diff --git a/alembic/versions/0010_add_taxonomy_info.py b/alembic/versions/0010_add_taxonomy_info.py index fc629f0..88bd44b 100644 --- a/alembic/versions/0010_add_taxonomy_info.py +++ b/alembic/versions/0010_add_taxonomy_info.py @@ -6,6 +6,7 @@ """ import sqlalchemy as sa + from alembic import op revision = "0010_add_taxonomy_info" @@ -30,9 +31,7 @@ def upgrade() -> None: sa.Column("augustus_dataset_name", sa.Text(), nullable=True), sa.Column("genetic_code_id", sa.Integer(), nullable=True), sa.PrimaryKeyConstraint("taxon_id"), - sa.ForeignKeyConstraint( - ["taxon_id"], ["organism.taxon_id"], ondelete="CASCADE" - ), + sa.ForeignKeyConstraint(["taxon_id"], ["organism.taxon_id"], ondelete="CASCADE"), ) # 2. Migrate existing augustus_dataset_name values from organism into taxonomy_info diff --git a/alembic/versions/0011_assembly_first_and_stages.py b/alembic/versions/0011_assembly_first_and_stages.py new file mode 100644 index 0000000..d3fee53 --- /dev/null +++ b/alembic/versions/0011_assembly_first_and_stages.py @@ -0,0 +1,153 @@ +"""Assembly-first manifest flow: nullable fields, status, and stage reporting tables. + +Revision ID: 0011_assembly_first_and_stages +Revises: 0010_add_taxonomy_info +Create Date: 2026-05-05 +""" + +import sqlalchemy as sa +from sqlalchemy.dialects.postgresql import JSONB, UUID + +from alembic import op + +revision = "0011_assembly_first_and_stages" +down_revision = "0010_add_taxonomy_info" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ── 1. Make late-known assembly fields nullable ────────────────────────── + op.alter_column("assembly", "assembly_name", existing_type=sa.Text(), nullable=True) + op.alter_column("assembly", "coverage", existing_type=sa.Float(), nullable=True) + op.alter_column("assembly", "program", existing_type=sa.Text(), nullable=True) + + # ── 2. Add lifecycle status to assembly ─────────────────────────────────── + op.add_column( + "assembly", + sa.Column( + "status", + sa.Text(), + nullable=False, + server_default="requested", + ), + ) + op.create_check_constraint( + "ck_assembly_status", + "assembly", + "status IN ('requested', 'running', 'curating', 'completed', 'failed', 'cancelled')", + ) + + # ── 3. assembly_stage catalog ───────────────────────────────────────────── + op.create_table( + "assembly_stage", + sa.Column("name", sa.Text(), primary_key=True), + sa.Column("category", sa.Text(), nullable=False), + sa.Column("is_active", sa.Boolean(), nullable=False, server_default=sa.true()), + sa.CheckConstraint("category IN ('pipeline', 'manual')", name="ck_assembly_stage_category"), + ) + + op.execute( + """ + INSERT INTO assembly_stage (name, category) VALUES + ('genomeassembly', 'pipeline'), + ('ascc', 'pipeline'), + ('treeval', 'pipeline'), + ('curation-pretext', 'pipeline'), + ('manual-curation', 'manual') + """ + ) + + # ── 4. assembly_stage_run ───────────────────────────────────────────────── + op.create_table( + "assembly_stage_run", + sa.Column( + "id", + UUID(as_uuid=True), + primary_key=True, + server_default=sa.text("uuid_generate_v4()"), + ), + sa.Column( + "assembly_id", + UUID(as_uuid=True), + sa.ForeignKey("assembly.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column( + "stage_name", + sa.Text(), + sa.ForeignKey("assembly_stage.name"), + nullable=False, + ), + sa.Column("status", sa.Text(), nullable=False), + sa.Column("external_run_id", sa.Text(), nullable=True), + sa.Column("attempt", sa.Integer(), nullable=False, server_default="1"), + sa.Column("stats", JSONB(), nullable=False, server_default="{}"), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.CheckConstraint( + "status IN ('running', 'succeeded', 'failed', 'cancelled')", + name="ck_assembly_stage_run_status", + ), + sa.UniqueConstraint( + "assembly_id", "stage_name", "attempt", name="uq_stage_run_assembly_stage_attempt" + ), + ) + op.create_index("ix_assembly_stage_run_assembly_id", "assembly_stage_run", ["assembly_id"]) + + # ── 5. assembly_stage_run_file ──────────────────────────────────────────── + op.create_table( + "assembly_stage_run_file", + sa.Column( + "id", + UUID(as_uuid=True), + primary_key=True, + server_default=sa.text("uuid_generate_v4()"), + ), + sa.Column( + "assembly_stage_run_id", + UUID(as_uuid=True), + sa.ForeignKey("assembly_stage_run.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column("storage_type", sa.Text(), nullable=False), + sa.Column("storage_uri", sa.Text(), nullable=False), + sa.Column("storage_details", JSONB(), nullable=False, server_default="{}"), + sa.Column("sha256sum", sa.Text(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + ) + op.create_index( + "ix_assembly_stage_run_file_run_id", + "assembly_stage_run_file", + ["assembly_stage_run_id"], + ) + + +def downgrade() -> None: + op.drop_table("assembly_stage_run_file") + op.drop_table("assembly_stage_run") + op.drop_table("assembly_stage") + + op.drop_constraint("ck_assembly_status", "assembly", type_="check") + op.drop_column("assembly", "status") + + op.alter_column("assembly", "program", existing_type=sa.Text(), nullable=False) + op.alter_column("assembly", "coverage", existing_type=sa.Float(), nullable=False) + op.alter_column("assembly", "assembly_name", existing_type=sa.Text(), nullable=False) diff --git a/alembic/versions/0012_assembly_manifest_columns.py b/alembic/versions/0012_assembly_manifest_columns.py new file mode 100644 index 0000000..89b1e7e --- /dev/null +++ b/alembic/versions/0012_assembly_manifest_columns.py @@ -0,0 +1,60 @@ +"""Add specimen sample IDs and manifest JSON to assembly. + +Revision ID: 0012_assembly_manifest_columns +Revises: 0011_assembly_first_and_stages +Create Date: 2026-05-08 + +Adds three nullable columns to the assembly table: +- long_read_specimen_sample_id: the specimen sample used for long reads (PacBio / ONT) +- hic_specimen_sample_id: the specimen sample used for Hi-C reads (optional) +- manifest_json: persisted JSON manifest generated at intent time + +All columns are nullable so existing rows are unaffected. +""" + +import sqlalchemy as sa +from sqlalchemy.dialects.postgresql import JSONB, UUID + +from alembic import op + +revision = "0012_assembly_manifest_columns" +down_revision = "0011_assembly_first_and_stages" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column( + "assembly", + sa.Column( + "long_read_specimen_sample_id", + UUID(as_uuid=True), + sa.ForeignKey("sample.id"), + nullable=True, + ), + ) + op.add_column( + "assembly", + sa.Column( + "hic_specimen_sample_id", + UUID(as_uuid=True), + sa.ForeignKey("sample.id"), + nullable=True, + ), + ) + op.add_column( + "assembly", + sa.Column("manifest_json", JSONB(), nullable=True), + ) + op.create_index( + "idx_assembly_intent_version_key", + "assembly", + ["taxon_id", "long_read_specimen_sample_id"], + ) + + +def downgrade() -> None: + op.drop_index("idx_assembly_intent_version_key", table_name="assembly") + op.drop_column("assembly", "manifest_json") + op.drop_column("assembly", "hic_specimen_sample_id") + op.drop_column("assembly", "long_read_specimen_sample_id") diff --git a/app/api/v1/api.py b/app/api/v1/api.py index 60377c7..3d46215 100644 --- a/app/api/v1/api.py +++ b/app/api/v1/api.py @@ -47,9 +47,7 @@ api_router.include_router(reads.router, prefix="/reads", tags=["reads"]) api_router.include_router(qc_reads.router, prefix="/qc-reads", tags=["qc-reads"]) api_router.include_router(genome_notes.router, prefix="/genome-notes", tags=["genome-notes"]) -api_router.include_router( - taxonomy_info.router, prefix="/taxonomy-info", tags=["taxonomy-info"] -) +api_router.include_router(taxonomy_info.router, prefix="/taxonomy-info", tags=["taxonomy-info"]) # XML export endpoints api_router.include_router(xml_export.router, prefix="/xml-export", tags=["xml-export"]) diff --git a/app/api/v1/endpoints/assemblies.py b/app/api/v1/endpoints/assemblies.py index 89b5b58..48faf02 100644 --- a/app/api/v1/endpoints/assemblies.py +++ b/app/api/v1/endpoints/assemblies.py @@ -2,16 +2,23 @@ from uuid import UUID from fastapi import APIRouter, Body, Depends, HTTPException, Query -from fastapi.responses import Response +from fastapi.responses import JSONResponse from sqlalchemy.orm import Session from app.core.dependencies import get_current_active_user, get_db from app.core.errors import AppError from app.core.pagination import Pagination, apply_pagination, pagination_params from app.core.policy import policy -from app.models.assembly import Assembly, AssemblyFile, AssemblyRun, AssemblySubmission +from app.models.assembly import ( + Assembly, + AssemblyFile, + AssemblyRun, + AssemblyStageRun, + AssemblySubmission, +) from app.models.experiment import Experiment from app.models.organism import Organism +from app.models.qc_read import QcRead from app.models.read import Read from app.models.sample import Sample from app.models.user import User @@ -25,7 +32,10 @@ AssemblyFileUpdate, AssemblyIntent, AssemblyIntentCancel, - AssemblyIntentResponse, + AssemblySpecimenSampleDiscoveryResponse, + AssemblyStageRunCreate, + AssemblyStageRunOut, + AssemblyStageRunUpdate, AssemblySubmissionCreate, AssemblySubmissionUpdate, AssemblyUpdate, @@ -37,45 +47,150 @@ AssemblySubmission as AssemblySubmissionSchema, ) from app.schemas.common import SubmissionStatus -from app.services.assembly_helper import determine_assembly_data_types, generate_assembly_manifest +from app.services.assembly_helper import ( + determine_assembly_data_types, + generate_assembly_manifest_json, + get_available_assembly_data_types, +) from app.services.assembly_service import ( assembly_file_service, assembly_service, + assembly_stage_run_service, assembly_submission_service, ) from app.services.organism_service import organism_service router = APIRouter() + # TODO remove tax_id refs and rely solely on taxon_id in organism table for all relationships and queries. def _organism_taxon_id(organism: Any) -> int: return organism.taxon_id if hasattr(organism, "taxon_id") else organism.tax_id -""" -@router.get("/", response_model=List[AssemblySchema]) -def read_assemblies( +def _build_sample_metadata_by_id( + db: Session, experiments: List[Experiment] +) -> Dict[str, Dict[str, Any]]: + """Build sample metadata mapping used in per-read manifest entries.""" + sample_ids = {exp.sample_id for exp in experiments if getattr(exp, "sample_id", None)} + if not sample_ids: + return {} + + sample_rows = db.query(Sample).filter(Sample.id.in_(list(sample_ids))).all() + return { + str(sample.id): { + "bpa_sample_id": getattr(sample, "bpa_sample_id", None), + "specimen_id": getattr(sample, "specimen_id", None), + } + for sample in sample_rows + } + + +def _build_specimen_metadata(*samples: Sample) -> Dict[str, Dict[str, Any]]: + return { + str(sample.id): { + "bpa_sample_id": getattr(sample, "bpa_sample_id", None), + "specimen_id": getattr(sample, "specimen_id", None), + } + for sample in samples + if sample is not None + } + + +def _validate_specimen_sample( + db: Session, sample_id: UUID, taxon_id: int, field_name: str +) -> Sample: + """Validate a specimen sample: must exist, kind='specimen', and belong to taxon_id.""" + sample = db.query(Sample).filter(Sample.id == sample_id).first() + if not sample: + raise AppError( + status_code=422, + code="specimen_sample_not_found", + message=f"{field_name} not found", + details={"field": field_name, "sample_id": str(sample_id)}, + ) + if sample.kind != "specimen": + raise AppError( + status_code=422, + code="specimen_sample_invalid_kind", + message=f"{field_name} must be a specimen sample (kind='specimen'), got kind='{sample.kind}'", + details={"field": field_name, "sample_id": str(sample_id), "kind": sample.kind}, + ) + if sample.taxon_id != taxon_id: + raise AppError( + status_code=422, + code="specimen_sample_taxon_mismatch", + message=f"{field_name} does not belong to taxon_id {taxon_id}", + details={ + "field": field_name, + "sample_id": str(sample_id), + "sample_taxon_id": sample.taxon_id, + "expected_taxon_id": taxon_id, + }, + ) + return sample + + +def _get_lineage_sample_ids_for_specimen(db: Session, specimen_sample: Sample) -> List[UUID]: + """Return specimen sample id plus any derived samples linked from it.""" + derived_samples = ( + db.query(Sample) + .filter( + Sample.derived_from_sample_id == specimen_sample.id, + Sample.taxon_id == specimen_sample.taxon_id, + Sample.kind == "derived", + ) + .all() + ) + return [specimen_sample.id] + [sample.id for sample in derived_samples] + + +@router.get("/specimen-samples/{taxon_id}", response_model=AssemblySpecimenSampleDiscoveryResponse) +def get_specimen_samples_for_assembly( + *, db: Session = Depends(get_db), - skip: int = 0, - limit: int = 100, - organism_id: Optional[UUID] = Query(None, description="Filter by organism ID"), - sample_id: Optional[UUID] = Query(None, description="Filter by sample ID"), - experiment_id: Optional[UUID] = Query(None, description="Filter by experiment ID"), + taxon_id: int, current_user: User = Depends(get_current_active_user), ) -> Any: - Retrieve assemblies. - # All users can read assemblies - query = db.query(Assembly) - if organism_id: - query = query.filter(Assembly.organism_id == organism_id) - if sample_id: - query = query.filter(Assembly.sample_id == sample_id) - if experiment_id: - query = query.filter(Assembly.experiment_id == experiment_id) + """Return specimen samples and available assembly data types for a taxon.""" + organism = db.query(Organism).filter(Organism.taxon_id == taxon_id).first() + if not organism: + raise HTTPException(status_code=404, detail=f"Organism with taxon_id {taxon_id} not found") + + organism_taxon_id = _organism_taxon_id(organism) + specimen_samples = ( + db.query(Sample) + .filter(Sample.taxon_id == organism_taxon_id, Sample.kind == "specimen") + .all() + ) - assemblies = query.offset(skip).limit(limit).all() - return assemblies -""" + specimen_sample_options = [] + for specimen_sample in specimen_samples: + lineage_sample_ids = _get_lineage_sample_ids_for_specimen(db, specimen_sample) + experiments = ( + db.query(Experiment).filter(Experiment.sample_id.in_(lineage_sample_ids)).all() + ) + experiment_ids = [experiment.id for experiment in experiments] + qc_reads = ( + db.query(QcRead).filter(QcRead.experiment_id.in_(experiment_ids)).all() + if experiment_ids + else [] + ) + + specimen_sample_options.append( + { + "sample_id": specimen_sample.id, + "specimen_id": specimen_sample.specimen_id, + "sex": specimen_sample.sex, + "available_data_types": get_available_assembly_data_types(experiments), + "qc_reads": qc_reads, + } + ) + + return { + "taxon_id": organism_taxon_id, + "specimen_samples": specimen_sample_options, + } @router.get("/pipeline-inputs") @@ -91,23 +206,19 @@ def get_pipeline_inputs( Returns a list of objects with scientific_name and files mapping for each organism. Files mapping contains read file names as keys and their bioplatforms_urls as values. """ - print(f"Organism taxon_id: {taxon_id}") if taxon_id is None: raise HTTPException(status_code=422, detail="taxon_id query parameter is required") if db is None: raise HTTPException(status_code=422, detail="database session is required") organism = organism_service.get_by_taxon_id(db, taxon_id) if not organism: - print(f"Organism with taxon_id '{taxon_id}' not found") raise HTTPException( status_code=404, detail=f"Organism with taxon_id '{taxon_id}' not found" ) - # Get all samples for this organism organism_taxon_id = _organism_taxon_id(organism) samples = db.query(Sample).filter(Sample.taxon_id == organism_taxon_id).all() if not samples: - print(f"No samples found for organism with taxon_id '{taxon_id}'") return [ { "scientific_name": organism.scientific_name, @@ -116,30 +227,19 @@ def get_pipeline_inputs( } ] - # Get all experiments and reads for these samples result = [] files_dict = {} - # Collect all reads for this organism through the sample->experiment->read relationship for sample in samples: - print(f"Sample {sample.id} found for organism with taxon_id '{taxon_id}'") - # Get experiments for this sample experiments = db.query(Experiment).filter(Experiment.sample_id == sample.id).all() - for experiment in experiments: - print(f"Experiment {experiment.id} found for sample {sample.id}") - # Get reads for this experiment reads = db.query(Read).filter(Read.experiment_id == experiment.id).all() - if reads is None: continue - for read in reads: - print(f"Read {read.id} found for experiment {experiment.id}") if read.file_name and read.bioplatforms_url: files_dict[read.file_name] = read.bioplatforms_url - # Create the result object result.append( { "scientific_name": organism.scientific_name, @@ -147,7 +247,6 @@ def get_pipeline_inputs( "files": files_dict, } ) - return result @@ -163,182 +262,69 @@ def get_pipeline_inputs_by_taxon_id( Returns a nested structure with taxon_id as the top level key, and scientific_name and files mapping for that organism. - Files mapping contains read file names as keys and their bioplatforms_urls as values. """ - print(f"Tax ID: {taxon_id}") - - # Check if taxon_id was provided if taxon_id is None: raise HTTPException(status_code=422, detail="taxon_id query parameter is required") - # Get all organisms with this taxon_id organism = organism_service.get_by_taxon_id(db, int(taxon_id)) if not organism: - print(f"No organisms found with tax ID '{taxon_id}'") return {taxon_id: {}} - # Initialize result structure result = {taxon_id: {"scientific_name": organism.scientific_name, "files": {}}} organism_taxon_id = _organism_taxon_id(organism) samples = db.query(Sample).filter(Sample.taxon_id == organism_taxon_id).all() if not samples: - print(f"No samples found for organism with taxon_id {taxon_id}") return result for sample in samples: - print(f"Sample {sample.id} found for organism {taxon_id}") experiments = db.query(Experiment).filter(Experiment.sample_id == sample.id).all() - for experiment in experiments: - print(f"Experiment {experiment.id} found for sample {sample.id}") reads = db.query(Read).filter(Read.experiment_id == experiment.id).all() - for read in reads: - print(f"Read {read.id} found for experiment {experiment.id}") if read.file_name and read.bioplatforms_url: result[taxon_id]["files"][read.file_name] = read.bioplatforms_url return result -def _get_manifest_inputs_by_taxon_id(db: Session, taxon_id: int): - organism = db.query(Organism).filter(Organism.taxon_id == taxon_id).first() - if not organism: - raise HTTPException(status_code=404, detail=f"Organism with taxon_id {taxon_id} not found") - - selected_sample_id = _get_optimal_sample_id_for_taxon_id( - db, taxon_id, organism_taxon_id=_organism_taxon_id(organism) - ) - if not selected_sample_id: - raise HTTPException( - status_code=404, - detail=f"No specimen sample found for taxon_id {taxon_id}", - ) - sample = db.query(Sample).filter(Sample.id == selected_sample_id).first() - if not sample: - raise HTTPException( - status_code=404, - detail=f"Selected specimen sample not found for taxon_id {taxon_id}", - ) - - sample_ids = [ - row[0] - for row in db.query(Sample.id).filter(Sample.taxon_id == _organism_taxon_id(organism)).all() - ] - if not sample_ids: - raise HTTPException( - status_code=404, - detail=f"No samples found for taxon_id {taxon_id}", - ) - - experiments = db.query(Experiment).filter(Experiment.sample_id.in_(sample_ids)).all() - if not experiments: - raise HTTPException( - status_code=404, - detail=f"No experiments found for taxon_id {taxon_id}", - ) - - experiment_ids = [exp.id for exp in experiments] - reads = db.query(Read).filter(Read.experiment_id.in_(experiment_ids)).all() - if not reads: - raise HTTPException( - status_code=404, - detail=f"No reads found for taxon_id {taxon_id}", - ) - - return organism, sample, reads, experiments - - -def _get_optimal_sample_id_for_taxon_id( - db: Session, taxon_id: int, organism_taxon_id: Optional[int] = None -) -> UUID | None: - # TODO: Replace with long-read aware sample selection. - if organism_taxon_id is None: - organism = db.query(Organism).filter(Organism.taxon_id == taxon_id).first() - if not organism: - return None - organism_taxon_id = organism.taxon_id - - sample = ( - db.query(Sample) - .filter(Sample.taxon_id == organism_taxon_id, Sample.kind == "specimen") - .order_by(Sample.created_at.asc()) - .first() - ) - return sample.id if sample else None - - -# Backward-compatible aliases retained for tests while the API contract migrates. -_get_manifest_inputs_by_tax_id = _get_manifest_inputs_by_taxon_id -_get_optimal_sample_id_for_tax_id = _get_optimal_sample_id_for_taxon_id - - -def _build_sample_metadata_by_id( - db: Session, experiments: List[Experiment] -) -> Dict[str, Dict[str, Any]]: - """Build sample metadata mapping used in per-read manifest entries.""" - sample_ids = {exp.sample_id for exp in experiments if getattr(exp, "sample_id", None)} - if not sample_ids: - return {} - - sample_rows = db.query(Sample).filter(Sample.id.in_(list(sample_ids))).all() - return { - str(sample.id): { - "bpa_sample_id": getattr(sample, "bpa_sample_id", None), - "specimen_id": getattr(sample, "specimen_id", None), - } - for sample in sample_rows - } - - @router.get("/manifest/{taxon_id}") def get_assembly_manifest( *, db: Session = Depends(get_db), taxon_id: int, - version: Optional[int] = Query(None, description="Reserved manifest version to retrieve"), + version: Optional[int] = Query(None, description="Assembly version to retrieve"), current_user: User = Depends(get_current_active_user), ) -> Any: """ - Retrieve the latest reserved assembly manifest YAML for an organism by taxon_id. + Retrieve the stored manifest JSON for the latest requested assembly for a taxon. - Returns YAML manifest with: - - scientific_name and taxon_id from organism - - reads grouped by platform (PACBIO_SMRT, Hi-C) - - PACBIO_SMRT: Only files ending in .ccs.bam or hifi_reads.bam - - Hi-C: Includes read_number and lane_number - - Returns: - YAML string with manifest data + Returns the manifest that was generated and stored when the intent was created. """ - organism, selected_sample, reads, experiments = _get_manifest_inputs_by_taxon_id(db, taxon_id) + organism = db.query(Organism).filter(Organism.taxon_id == taxon_id).first() + if not organism: + raise HTTPException(status_code=404, detail=f"Organism with taxon_id {taxon_id} not found") - run_query = ( - db.query(AssemblyRun) + assembly_query = ( + db.query(Assembly) .filter( - AssemblyRun.taxon_id == _organism_taxon_id(organism), - AssemblyRun.sample_id == selected_sample.id, + Assembly.taxon_id == _organism_taxon_id(organism), + Assembly.status == "requested", ) - .order_by(AssemblyRun.created_at.desc()) + .order_by(Assembly.created_at.desc()) ) if version is not None: - run_query = run_query.filter(AssemblyRun.version == version) - assembly_run = run_query.first() - if not assembly_run: - raise HTTPException(status_code=404, detail="No reserved assembly manifest found") - - sample_metadata_by_id = _build_sample_metadata_by_id(db, experiments) - yaml_content = generate_assembly_manifest( - organism, - reads, - experiments, - assembly_run.tol_id, - assembly_run.version, - sample_metadata_by_id, - ) + assembly_query = assembly_query.filter(Assembly.version == version) + assembly = assembly_query.first() + if not assembly: + raise HTTPException(status_code=404, detail="No requested assembly manifest found") - # Return as YAML response - return Response(content=yaml_content, media_type="application/x-yaml") + if assembly.manifest_json is None: + raise HTTPException( + status_code=404, + detail="No manifest stored for this assembly. Re-submit an intent to generate one.", + ) + + return JSONResponse(content=assembly.manifest_json) @router.post("/intent/{taxon_id}") @@ -347,66 +333,175 @@ def create_assembly_intent( *, db: Session = Depends(get_db), taxon_id: int, - intent_in: Optional[AssemblyIntent] = Body(default=None), + intent_in: AssemblyIntent = Body(...), current_user: User = Depends(get_current_active_user), -) -> Response: +) -> Any: """ - Reserve the next assembly version and return a manifest as YAML. + Create an assembly record using explicit specimen sample IDs and return its manifest as JSON. + + The caller must supply: + - long_read_specimen_sample_id: the specimen sample used for PacBio / ONT long reads + - hic_specimen_sample_id (optional): the specimen sample used for Hi-C reads - Returns: - YAML response with manifest data including assembly_run_id, version, status, and manifest fields + Both samples must be kind='specimen' and belong to the given taxon_id. + + Returns JSON with assembly_id, version, status, and the generated manifest. """ - organism, selected_sample, reads, experiments = _get_manifest_inputs_by_taxon_id(db, taxon_id) + # 1. Resolve organism + organism = db.query(Organism).filter(Organism.taxon_id == taxon_id).first() + if not organism: + raise HTTPException(status_code=404, detail=f"Organism with taxon_id {taxon_id} not found") + org_taxon_id = _organism_taxon_id(organism) + + # 2. Validate long_read_specimen_sample_id + long_read_sample = _validate_specimen_sample( + db, intent_in.long_read_specimen_sample_id, org_taxon_id, "long_read_specimen_sample_id" + ) + + # 3. Validate hic_specimen_sample_id if provided + hic_sample = None + if intent_in.hic_specimen_sample_id: + hic_sample = _validate_specimen_sample( + db, intent_in.hic_specimen_sample_id, org_taxon_id, "hic_specimen_sample_id" + ) + + # 4. Fetch long-read experiments (PacBio or ONT only) and their reads + long_read_lineage_sample_ids = _get_lineage_sample_ids_for_specimen(db, long_read_sample) + long_read_sample_id_map = { + str(sample_id): str(long_read_sample.id) for sample_id in long_read_lineage_sample_ids + } + long_read_experiments = ( + db.query(Experiment) + .filter( + Experiment.sample_id.in_(long_read_lineage_sample_ids), + Experiment.platform.in_(["PACBIO_SMRT", "OXFORD_NANOPORE"]), + ) + .all() + ) + if not long_read_experiments: + raise AppError( + status_code=422, + code="no_long_read_experiments", + message="No long-read experiments (PacBio or ONT) found for long_read_specimen_sample_id", + details={"long_read_specimen_sample_id": str(long_read_sample.id)}, + ) + + long_read_exp_ids = [e.id for e in long_read_experiments] + long_reads = db.query(Read).filter(Read.experiment_id.in_(long_read_exp_ids)).all() + + # 5. Fetch Hi-C experiments and reads if hic_sample is provided + hic_experiments: List[Experiment] = [] + hic_reads: List[Read] = [] + hic_sample_id_map: Dict[str, str] = {} + if hic_sample: + hic_lineage_sample_ids = _get_lineage_sample_ids_for_specimen(db, hic_sample) + hic_sample_id_map = { + str(sample_id): str(hic_sample.id) for sample_id in hic_lineage_sample_ids + } + hic_experiments = ( + db.query(Experiment) + .filter( + Experiment.sample_id.in_(hic_lineage_sample_ids), + Experiment.platform == "ILLUMINA", + ) + .all() + ) + hic_experiments = [ + e for e in hic_experiments if (e.library_strategy or "").upper() == "HI-C" + ] + if not hic_experiments: + raise AppError( + status_code=422, + code="no_hic_experiments", + message="No Hi-C experiments (ILLUMINA + Hi-C) found for hic_specimen_sample_id", + details={"hic_specimen_sample_id": str(hic_sample.id)}, + ) + hic_exp_ids = [e.id for e in hic_experiments] + hic_reads = db.query(Read).filter(Read.experiment_id.in_(hic_exp_ids)).all() + + # 6. Determine data_types from the relevant experiments + all_experiments = long_read_experiments + hic_experiments try: - data_types = determine_assembly_data_types(experiments) + data_types = determine_assembly_data_types(all_experiments) except ValueError as exc: raise AppError( status_code=400, code="assembly_intent_invalid_data_types", message=str(exc), - details={ - "taxon_id": taxon_id, - "sample_id": str(selected_sample.id), - }, + details={"taxon_id": taxon_id}, ) from exc - next_version = assembly_service.get_next_version( + # 7. Create assembly (version is assigned inside the service) + assembly = assembly_service.create_from_intent( db, - taxon_id=_organism_taxon_id(organism), - sample_id=selected_sample.id, + taxon_id=org_taxon_id, + long_read_specimen_sample_id=long_read_sample.id, + hic_specimen_sample_id=hic_sample.id if hic_sample else None, data_types=data_types, + tol_id=intent_in.tol_id, + project_id=None, + manifest_json=None, # filled in step 9 ) - run = AssemblyRun( - taxon_id=_organism_taxon_id(organism), - sample_id=selected_sample.id, - data_types=data_types, - version=next_version, - tol_id=intent_in.tol_id if intent_in else None, - status="reserved", - ) - db.add(run) - db.commit() - db.refresh(run) + try: + # 8. Build sample metadata and generate JSON manifest + all_reads = long_reads + hic_reads + sample_metadata_by_id = _build_specimen_metadata(long_read_sample, hic_sample) + sequencing_sample_to_specimen_sample_id = { + **long_read_sample_id_map, + **hic_sample_id_map, + } + manifest_data = generate_assembly_manifest_json( + organism=organism, + reads=all_reads, + experiments=all_experiments, + tol_id=assembly.tol_id, + version=assembly.version, + long_read_sample_id=long_read_sample.id, + hic_sample_id=hic_sample.id if hic_sample else None, + sample_metadata_by_id=sample_metadata_by_id, + sequencing_sample_to_specimen_sample_id=sequencing_sample_to_specimen_sample_id, + ) - sample_metadata_by_id = _build_sample_metadata_by_id(db, experiments) - manifest_yaml = generate_assembly_manifest( - organism, reads, experiments, run.tol_id, run.version, sample_metadata_by_id + # 9. Validate that the manifest contains eligible reads then persist it + long_read_keys = {"PACBIO_SMRT", "OXFORD_NANOPORE"} + if not any(k in manifest_data["reads"] for k in long_read_keys): + raise AppError( + status_code=422, + code="no_eligible_long_reads", + message=( + "No eligible long reads found for long_read_specimen_sample_id. " + "PacBio reads must end in .ccs.bam or hifi_reads.bam." + ), + details={"long_read_specimen_sample_id": str(long_read_sample.id)}, + ) + + if hic_sample and "Hi-C" not in manifest_data["reads"]: + raise AppError( + status_code=422, + code="no_eligible_hic_reads", + message="No eligible Hi-C reads found for hic_specimen_sample_id", + details={"hic_specimen_sample_id": str(hic_sample.id)}, + ) + + assembly.manifest_json = manifest_data + db.add(assembly) + db.commit() + db.refresh(assembly) + except Exception: + db.delete(assembly) + db.commit() + raise + + return JSONResponse( + content={ + "assembly_id": str(assembly.id), + "version": assembly.version, + "status": assembly.status, + "manifest": manifest_data, + } ) - # Build complete YAML response including metadata and manifest - import yaml - - response_data = { - "assembly_run_id": str(run.id), - "version": run.version, - "status": run.status, - "manifest": yaml.safe_load(manifest_yaml), - } - yaml_response = yaml.dump(response_data, default_flow_style=False, sort_keys=False) - - return Response(content=yaml_response, media_type="application/x-yaml") - @router.post("/intent/{taxon_id}/cancel") @policy("assemblies:write") @@ -417,54 +512,49 @@ def cancel_assembly_intent( cancel_in: AssemblyIntentCancel, current_user: User = Depends(get_current_active_user), ) -> Any: - """Cancel a reserved assembly intent by ID.""" + """Cancel a requested assembly by ID.""" organism = db.query(Organism).filter(Organism.taxon_id == taxon_id).first() if not organism: raise HTTPException(status_code=404, detail=f"Organism with taxon_id {taxon_id} not found") - selected_sample_id = _get_optimal_sample_id_for_taxon_id( - db, taxon_id, organism_taxon_id=_organism_taxon_id(organism) - ) - if not selected_sample_id: - raise HTTPException( - status_code=404, - detail=f"No specimen sample found for taxon_id {taxon_id}", - ) - - run = ( - db.query(AssemblyRun) + assembly = ( + db.query(Assembly) .filter( - AssemblyRun.id == cancel_in.assembly_run_id, - AssemblyRun.taxon_id == _organism_taxon_id(organism), - AssemblyRun.sample_id == selected_sample_id, - AssemblyRun.status == "reserved", + Assembly.id == cancel_in.assembly_id, + Assembly.taxon_id == _organism_taxon_id(organism), + Assembly.status == "requested", ) .first() ) - if not run: - raise HTTPException(status_code=404, detail="No reserved assembly intent found to cancel") - if cancel_in.version is not None and cancel_in.version != run.version: + if not assembly: + raise HTTPException(status_code=404, detail="No requested assembly found to cancel") + if cancel_in.version is not None and cancel_in.version != assembly.version: raise AppError( status_code=409, code="assembly_intent_version_mismatch", - message="Requested version does not match the reserved assembly intent", + message="Requested version does not match the assembly", details={ - "assembly_run_id": str(run.id), + "assembly_id": str(assembly.id), "requested_version": cancel_in.version, - "actual_version": run.version, + "actual_version": assembly.version, }, ) - run.status = "cancelled" - db.add(run) + assembly.status = "cancelled" + db.add(assembly) db.commit() - db.refresh(run) + db.refresh(assembly) return { - "id": str(run.id), - "taxon_id": run.taxon_id, - "sample_id": str(run.sample_id), - "version": run.version, - "status": run.status, + "id": str(assembly.id), + "taxon_id": assembly.taxon_id, + "long_read_specimen_sample_id": str(assembly.long_read_specimen_sample_id) + if assembly.long_read_specimen_sample_id + else None, + "hic_specimen_sample_id": str(assembly.hic_specimen_sample_id) + if assembly.hic_specimen_sample_id + else None, + "version": assembly.version, + "status": assembly.status, } @@ -475,8 +565,14 @@ def get_optimal_sample_id( taxon_id: int, current_user: User = Depends(get_current_active_user), ) -> Any: - sample_id = _get_optimal_sample_id_for_taxon_id(db, taxon_id) - return {"sample_id": str(sample_id) if sample_id else None} + """Return the first specimen sample for an organism (informational helper).""" + sample = ( + db.query(Sample) + .filter(Sample.taxon_id == taxon_id, Sample.kind == "specimen") + .order_by(Sample.created_at.asc()) + .first() + ) + return {"sample_id": str(sample.id) if sample else None} @router.post("/from-experiments/{taxon_id}", response_model=AssemblySchema) @@ -491,17 +587,12 @@ def create_assembly_from_experiments( """ Create assembly based on all experiments for an organism (by taxon_id). - Automatically determines data_types by analyzing experiment platforms: - - PACBIO_SMRT: platform == "PACBIO_SMRT" - - OXFORD_NANOPORE: platform == "OXFORD_NANOPORE" - - Hi-C: platform == "ILLUMINA" AND library_strategy == "Hi-C" - - The taxon_id is automatically determined from taxon_id. + Automatically determines data_types by analyzing experiment platforms. The data_types field is auto-detected but can be overridden. """ try: assembly, platform_info = assembly_service.create_from_experiments( - db, taxon_id, assembly_in + db, taxon_id=taxon_id, assembly_in=assembly_in ) return assembly except ValueError as e: @@ -520,6 +611,27 @@ def create_assembly( return assembly +@router.get("/{assembly_id}/manifest") +def get_manifest_by_assembly_id( + *, + db: Session = Depends(get_db), + assembly_id: UUID, + current_user: User = Depends(get_current_active_user), +) -> Any: + """Retrieve the stored manifest JSON for a specific assembly.""" + assembly = db.query(Assembly).filter(Assembly.id == assembly_id).first() + if not assembly: + raise HTTPException(status_code=404, detail="Assembly not found") + + if assembly.manifest_json is None: + raise HTTPException( + status_code=404, + detail="No manifest stored for this assembly. Re-submit an intent to generate one.", + ) + + return JSONResponse(content=assembly.manifest_json) + + @router.get("/{assembly_id}", response_model=AssemblySchema) def read_assembly( *, @@ -527,8 +639,6 @@ def read_assembly( assembly_id: UUID, current_user: User = Depends(get_current_active_user), ) -> Any: - # Get assembly by ID. - # All users can read assembly details assembly = db.query(Assembly).filter(Assembly.id == assembly_id).first() if not assembly: raise HTTPException(status_code=404, detail="Assembly not found") @@ -566,12 +676,9 @@ def delete_assembly( assembly_id: UUID, current_user: User = Depends(get_current_active_user), ) -> Any: - # Delete an assembly. - # Only superusers can delete assemblies assembly = db.query(Assembly).filter(Assembly.id == assembly_id).first() if not assembly: raise HTTPException(status_code=404, detail="Assembly not found") - db.delete(assembly) db.commit() return assembly @@ -594,7 +701,6 @@ def read_assembly_submissions( if status: query = query.filter(AssemblySubmission.status == status.value) submissions = apply_pagination(query, pagination).all() - return submissions @@ -607,12 +713,9 @@ def create_assembly_submission( current_user: User = Depends(get_current_active_user), ) -> Any: """Create new assembly submission.""" - - # Verify assembly exists assembly = assembly_service.get(db, id=submission_in.assembly_id) if not assembly: raise HTTPException(status_code=404, detail="Assembly not found") - submission = assembly_submission_service.create(db, obj_in=submission_in) return submission @@ -627,11 +730,9 @@ def update_assembly_submission( current_user: User = Depends(get_current_active_user), ) -> Any: """Update an assembly submission.""" - submission = assembly_submission_service.get(db, id=submission_id) if not submission: raise HTTPException(status_code=404, detail="Assembly submission not found") - submission = assembly_submission_service.update(db, db_obj=submission, obj_in=submission_in) return submission @@ -660,7 +761,6 @@ def read_assembly_files( ) else: files = assembly_file_service.get_by_assembly_id(db, assembly_id=assembly_id) - return files @@ -674,16 +774,11 @@ def create_assembly_file( current_user: User = Depends(get_current_active_user), ) -> Any: """Add a file to an assembly.""" - - # Verify assembly exists assembly = assembly_service.get(db, id=assembly_id) if not assembly: raise HTTPException(status_code=404, detail="Assembly not found") - - # Ensure assembly_id matches if file_in.assembly_id != assembly_id: raise HTTPException(status_code=400, detail="Assembly ID mismatch") - file = assembly_file_service.create(db, obj_in=file_in) return file @@ -698,11 +793,9 @@ def update_assembly_file( current_user: User = Depends(get_current_active_user), ) -> Any: """Update an assembly file.""" - file = assembly_file_service.get(db, id=file_id) if not file: raise HTTPException(status_code=404, detail="Assembly file not found") - file = assembly_file_service.update(db, db_obj=file, obj_in=file_in) return file @@ -719,6 +812,68 @@ def delete_assembly_file( file = assembly_file_service.get(db, id=file_id) if not file: raise HTTPException(status_code=404, detail="Assembly file not found") - assembly_file_service.remove(db, id=file_id) return {"message": "File deleted successfully"} + + +# ========================================== +# Assembly Stage Run endpoints +# ========================================== + + +@router.get("/{assembly_id}/stage-runs", response_model=List[AssemblyStageRunOut]) +def list_stage_runs( + *, + db: Session = Depends(get_db), + assembly_id: UUID, + current_user: User = Depends(get_current_active_user), +) -> Any: + """List all stage runs for an assembly, newest first.""" + assembly = assembly_service.get(db, id=assembly_id) + if not assembly: + raise HTTPException(status_code=404, detail="Assembly not found") + return assembly_stage_run_service.get_by_assembly_id(db, assembly_id=assembly_id) + + +@router.post("/{assembly_id}/stage-runs", response_model=AssemblyStageRunOut) +@policy("assemblies:write") +def create_stage_run( + *, + db: Session = Depends(get_db), + assembly_id: UUID, + run_in: AssemblyStageRunCreate, + current_user: User = Depends(get_current_active_user), +) -> Any: + """Report a stage run result for an assembly.""" + assembly = assembly_service.get(db, id=assembly_id) + if not assembly: + raise HTTPException(status_code=404, detail="Assembly not found") + return assembly_stage_run_service.create_with_files( + db, + assembly_id=assembly_id, + run_in=run_in, + ) + + +@router.patch("/{assembly_id}/stage-runs/{stage_run_id}", response_model=AssemblyStageRunOut) +@policy("assemblies:write") +def update_stage_run( + *, + db: Session = Depends(get_db), + assembly_id: UUID, + stage_run_id: UUID, + update_in: AssemblyStageRunUpdate, + current_user: User = Depends(get_current_active_user), +) -> Any: + """Update status, stats, or files for an existing stage run.""" + stage_run = ( + db.query(AssemblyStageRun) + .filter( + AssemblyStageRun.id == stage_run_id, + AssemblyStageRun.assembly_id == assembly_id, + ) + .first() + ) + if not stage_run: + raise HTTPException(status_code=404, detail="Stage run not found") + return assembly_stage_run_service.update_with_files(db, db_obj=stage_run, update_in=update_in) diff --git a/app/api/v1/endpoints/samples.py b/app/api/v1/endpoints/samples.py index b4230f1..fc9c185 100644 --- a/app/api/v1/endpoints/samples.py +++ b/app/api/v1/endpoints/samples.py @@ -6,6 +6,7 @@ from uuid import UUID from fastapi import APIRouter, Depends, HTTPException, Query, status +from sqlalchemy import or_ from sqlalchemy.orm import Session from sqlalchemy.orm.attributes import flag_modified @@ -15,6 +16,7 @@ from app.models.experiment import Experiment from app.models.organism import Organism from app.models.project import Project +from app.models.read import Read from app.models.sample import Sample, SampleSubmission from app.models.user import User from app.schemas.bulk_import import BulkImportResponse, BulkSampleImport @@ -25,6 +27,7 @@ SampleSubmissionCreate, SampleSubmissionUpdate, SampleUpdate, + SpecimenSampleHierarchyResponse, ) from app.schemas.sample import SampleSubmission as SampleSubmissionSchema from app.utils.mapping import to_float @@ -69,7 +72,69 @@ def read_samples( return samples -@router.get("/by-specimen/{taxon_id}/{specimen_id}", response_model=SampleSchema) +@router.get( + "/by-specimen/{taxon_id}/{specimen_id:path}/related", + response_model=SpecimenSampleHierarchyResponse, +) +def get_samples_experiments_and_reads_for_specimen( + *, + db: Session = Depends(get_db), + taxon_id: int, + specimen_id: str, + current_user: User = Depends(get_current_active_user), +) -> Any: + """Return the specimen sample, related samples, and nested experiments/reads.""" + organism = db.query(Organism).filter(Organism.taxon_id == taxon_id).first() + if not organism: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Organism with taxon_id {taxon_id} not found", + ) + + specimen_sample = ( + db.query(Sample) + .filter( + Sample.taxon_id == organism.taxon_id, + Sample.specimen_id == specimen_id, + Sample.kind == SampleKind.SPECIMEN, + ) + .first() + ) + if not specimen_sample: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Specimen sample not found for taxon_id {taxon_id} and specimen_id '{specimen_id}'", + ) + + related_samples = ( + db.query(Sample) + .filter( + Sample.taxon_id == organism.taxon_id, + or_( + Sample.id == specimen_sample.id, Sample.derived_from_sample_id == specimen_sample.id + ), + ) + .all() + ) + + related_payload = [] + for sample in related_samples: + experiments = db.query(Experiment).filter(Experiment.sample_id == sample.id).all() + experiment_payload = [] + for experiment in experiments: + reads = db.query(Read).filter(Read.experiment_id == experiment.id).all() + experiment_payload.append({"experiment": experiment, "reads": reads}) + + related_payload.append({"sample": sample, "experiments": experiment_payload}) + + return { + "taxon_id": organism.taxon_id, + "specimen_id": specimen_id, + "samples": related_payload, + } + + +@router.get("/by-specimen/{taxon_id}/{specimen_id:path}", response_model=SampleSchema) def get_specimen_by_taxid_and_specimen_id( *, db: Session = Depends(get_db), @@ -83,9 +148,6 @@ def get_specimen_by_taxid_and_specimen_id( This finds the unique specimen sample for a given organism (by taxon_id) and specimen_id combination. """ - # All users can read samples - - # First, find the organism by taxon_id organism = db.query(Organism).filter(Organism.taxon_id == taxon_id).first() if not organism: raise HTTPException( @@ -93,7 +155,6 @@ def get_specimen_by_taxid_and_specimen_id( detail=f"Organism with taxon_id {taxon_id} not found", ) - # Then find the specimen sample sample = ( db.query(Sample) .filter( diff --git a/app/models/__init__.py b/app/models/__init__.py index 558d4ad..5b16a88 100644 --- a/app/models/__init__.py +++ b/app/models/__init__.py @@ -5,10 +5,10 @@ from app.models.experiment import Experiment, ExperimentSubmission from app.models.genome_note import GenomeNote from app.models.organism import Organism -from app.models.taxonomy_info import TaxonomyInfo from app.models.project import Project from app.models.qc_read import QcRead, QcReadFile, QcReadSubmission from app.models.read import Read from app.models.sample import Sample, SampleSubmission +from app.models.taxonomy_info import TaxonomyInfo from app.models.token import RefreshToken from app.models.user import User diff --git a/app/models/assembly.py b/app/models/assembly.py index c7cdd2b..df14a95 100644 --- a/app/models/assembly.py +++ b/app/models/assembly.py @@ -2,6 +2,7 @@ from sqlalchemy import ( BigInteger, + Boolean, Column, DateTime, Float, @@ -9,6 +10,7 @@ Integer, PrimaryKeyConstraint, Text, + UniqueConstraint, func, ) from sqlalchemy import Enum as SQLAlchemyEnum @@ -33,7 +35,7 @@ class Assembly(Base): project_id = Column(UUID(as_uuid=True), ForeignKey("project.id"), nullable=True) # Assembly metadata fields - assembly_name = Column(Text, nullable=False) + assembly_name = Column(Text, nullable=True) assembly_type = Column(Text, nullable=False, default="clone or isolate") tol_id = Column(Text, nullable=True) data_types = Column( @@ -48,9 +50,10 @@ class Assembly(Base): ), nullable=False, ) - coverage = Column(Float, nullable=False) - program = Column(Text, nullable=False) + coverage = Column(Float, nullable=True) + program = Column(Text, nullable=True) mingaplength = Column(Float, nullable=True) + status = Column(Text, nullable=False, default="requested") moleculetype = Column( SQLAlchemyEnum("genomic DNA", "genomic RNA", name="molecule_type"), nullable=False, @@ -67,9 +70,17 @@ class Assembly(Base): onupdate=func.now(), ) + long_read_specimen_sample_id = Column( + UUID(as_uuid=True), ForeignKey("sample.id"), nullable=True + ) + hic_specimen_sample_id = Column(UUID(as_uuid=True), ForeignKey("sample.id"), nullable=True) + manifest_json = Column(JSONB, nullable=True) + # Relationships organism = relationship("Organism", backref="assemblies") - sample = relationship("Sample", backref="assemblies") + sample = relationship("Sample", foreign_keys=[sample_id], backref="assemblies") + long_read_specimen_sample = relationship("Sample", foreign_keys=[long_read_specimen_sample_id]) + hic_specimen_sample = relationship("Sample", foreign_keys=[hic_specimen_sample_id]) project = relationship("Project", backref="assemblies") @@ -240,3 +251,71 @@ class AssemblyRead(Base): "Assembly", backref=backref("assembly_reads", cascade="all, delete-orphan") ) read = relationship("Read", backref=backref("reads_assembly", cascade="all, delete-orphan")) + + +class AssemblyStage(Base): + """Catalog of known assembly pipeline/manual stages.""" + + __tablename__ = "assembly_stage" + + name = Column(Text, primary_key=True) + category = Column(Text, nullable=False) + is_active = Column(Boolean, nullable=False, default=True) + + +class AssemblyStageRun(Base): + """A single reported run of an assembly stage (pipeline or manual).""" + + __tablename__ = "assembly_stage_run" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + assembly_id = Column( + UUID(as_uuid=True), ForeignKey("assembly.id", ondelete="CASCADE"), nullable=False + ) + stage_name = Column(Text, ForeignKey("assembly_stage.name"), nullable=False) + status = Column(Text, nullable=False) + external_run_id = Column(Text, nullable=True) + attempt = Column(Integer, nullable=False, default=1) + stats = Column(JSONB, nullable=False, default=dict) + started_at = Column(DateTime(timezone=True), nullable=True) + completed_at = Column(DateTime(timezone=True), nullable=True) + + created_at = Column(DateTime(timezone=True), nullable=False, server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), + nullable=False, + server_default=func.now(), + onupdate=func.now(), + ) + + __table_args__ = ( + UniqueConstraint( + "assembly_id", "stage_name", "attempt", name="uq_stage_run_assembly_stage_attempt" + ), + ) + + assembly = relationship("Assembly", backref=backref("stage_runs", cascade="all, delete-orphan")) + stage = relationship("AssemblyStage", backref="runs") + + +class AssemblyStageRunFile(Base): + """Files attached to an assembly stage run.""" + + __tablename__ = "assembly_stage_run_file" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + assembly_stage_run_id = Column( + UUID(as_uuid=True), + ForeignKey("assembly_stage_run.id", ondelete="CASCADE"), + nullable=False, + ) + storage_type = Column(Text, nullable=False) + storage_uri = Column(Text, nullable=False) + storage_details = Column(JSONB, nullable=False, default=dict) + sha256sum = Column(Text, nullable=False) + + created_at = Column(DateTime(timezone=True), nullable=False, server_default=func.now()) + + stage_run = relationship( + "AssemblyStageRun", backref=backref("files", cascade="all, delete-orphan") + ) diff --git a/app/schemas/assembly.py b/app/schemas/assembly.py index 9ad3ad0..663f9d0 100644 --- a/app/schemas/assembly.py +++ b/app/schemas/assembly.py @@ -1,11 +1,12 @@ from datetime import datetime from enum import Enum -from typing import Dict, Optional +from typing import Any, Dict, List, Optional from uuid import UUID from pydantic import BaseModel, ConfigDict, Field from app.schemas.common import SubmissionStatus +from app.schemas.qc_read import QcReadOut class AssemblyDataTypes(str, Enum): @@ -19,6 +20,15 @@ class AssemblyDataTypes(str, Enum): PACBIO_SMRT_OXFORD_NANOPORE_HIC = "PACBIO_SMRT_OXFORD_NANOPORE_HIC" +class AssemblySpecimenSampleDataType(str, Enum): + """Atomic data types exposed by specimen-sample discovery.""" + + PACBIO_SMRT = "PACBIO_SMRT" + OXFORD_NANOPORE = "OXFORD_NANOPORE" + HIC = "Hi-C" + RNASEQ = "RNA-Seq" + + class AssemblyFileType(str, Enum): """Enum for assembly file types.""" @@ -35,16 +45,17 @@ class AssemblyBase(BaseModel): taxon_id: int sample_id: UUID project_id: Optional[UUID] = None - assembly_name: str + assembly_name: Optional[str] = None assembly_type: str = "clone or isolate" - tol_id: str + tol_id: Optional[str] = None data_types: AssemblyDataTypes - coverage: float - program: str + coverage: Optional[float] = None + program: Optional[str] = None mingaplength: Optional[float] = None moleculetype: str = "genomic DNA" description: Optional[str] = None version: int = 1 + status: str = "requested" # Schema for creating a new assembly @@ -75,21 +86,40 @@ class AssemblyIntent(BaseModel): """Schema for reserving an assembly version and generating a manifest.""" tol_id: Optional[str] = None + long_read_specimen_sample_id: UUID + hic_specimen_sample_id: Optional[UUID] = None class AssemblyIntentResponse(BaseModel): """Response envelope for assembly intent creation.""" - assembly_run_id: UUID + assembly_id: UUID version: int status: str - manifest_yaml: str + manifest_json: Dict[str, Any] + + +class AssemblySpecimenSampleOption(BaseModel): + """Assembly discovery metadata for a specimen sample.""" + + sample_id: UUID + specimen_id: Optional[str] = None + sex: Optional[str] = None + available_data_types: List[AssemblySpecimenSampleDataType] = Field(default_factory=list) + qc_reads: List[QcReadOut] = Field(default_factory=list) + + +class AssemblySpecimenSampleDiscoveryResponse(BaseModel): + """Response schema for assembly specimen-sample discovery.""" + + taxon_id: int + specimen_samples: List[AssemblySpecimenSampleOption] class AssemblyIntentCancel(BaseModel): """Schema for cancelling an existing assembly intent.""" - assembly_run_id: UUID + assembly_id: UUID version: Optional[int] = None @@ -109,6 +139,7 @@ class AssemblyUpdate(BaseModel): moleculetype: Optional[str] = None version_number: Optional[int] = None description: Optional[str] = None + status: Optional[str] = None # Schema for assembly in DB @@ -116,6 +147,9 @@ class AssemblyInDBBase(AssemblyBase): """Base schema for Assembly in DB, includes id and timestamps.""" id: UUID + long_read_specimen_sample_id: Optional[UUID] = None + hic_specimen_sample_id: Optional[UUID] = None + manifest_json: Optional[Dict[str, Any]] = None created_at: datetime updated_at: datetime @@ -252,3 +286,74 @@ class AssemblyFile(AssemblyFileInDBBase): """Schema for returning assembly file information.""" pass + + +# ========================================== +# AssemblyStageRun schemas +# ========================================== + + +class AssemblyStageRunFileCreate(BaseModel): + """File payload for a stage run.""" + + storage_type: str + storage_uri: str + storage_details: Dict[str, Any] = {} + sha256sum: str + + +class AssemblyStageRunCreate(BaseModel): + """Schema for reporting a stage run result.""" + + stage_name: str + status: str + external_run_id: Optional[str] = None + attempt: int = 1 + stats: Dict[str, Any] = {} + started_at: Optional[datetime] = None + completed_at: Optional[datetime] = None + files: List[AssemblyStageRunFileCreate] = [] + + +class AssemblyStageRunUpdate(BaseModel): + """Schema for updating an existing stage run. If files is provided, replaces all existing files.""" + + status: Optional[str] = None + external_run_id: Optional[str] = None + stats: Optional[Dict[str, Any]] = None + started_at: Optional[datetime] = None + completed_at: Optional[datetime] = None + files: Optional[List[AssemblyStageRunFileCreate]] = None + + +class AssemblyStageRunFileOut(BaseModel): + """Stage run file response schema.""" + + id: UUID + assembly_stage_run_id: UUID + storage_type: str + storage_uri: str + storage_details: Dict[str, Any] + sha256sum: str + created_at: datetime + + model_config = ConfigDict(from_attributes=True) + + +class AssemblyStageRunOut(BaseModel): + """Stage run response schema.""" + + id: UUID + assembly_id: UUID + stage_name: str + status: str + external_run_id: Optional[str] + attempt: int + stats: Dict[str, Any] + started_at: Optional[datetime] + completed_at: Optional[datetime] + created_at: datetime + updated_at: datetime + files: List[AssemblyStageRunFileOut] = [] + + model_config = ConfigDict(from_attributes=True) diff --git a/app/schemas/experiment.py b/app/schemas/experiment.py index 1fb4e7d..c147583 100644 --- a/app/schemas/experiment.py +++ b/app/schemas/experiment.py @@ -103,6 +103,37 @@ class Experiment(ExperimentInDBBase): pass +class ExperimentDetail(ExperimentInDBBase): + """Detailed experiment schema used by nested aggregate endpoints.""" + + project_id: UUID + bioplatforms_base_url: Optional[str] = None + design_description: Optional[str] = None + bpa_library_id: Optional[str] = None + library_strategy: Optional[str] = None + library_source: Optional[str] = None + insert_size: Optional[str] = None + library_construction_protocol: Optional[str] = None + library_selection: Optional[str] = None + library_layout: Optional[str] = None + instrument_model: Optional[str] = None + platform: Optional[str] = None + material_extracted_by: Optional[str] = None + library_prepared_by: Optional[str] = None + sequencing_kit: Optional[str] = None + flowcell_type: Optional[str] = None + base_caller_model: Optional[str] = None + data_owner: Optional[str] = None + project_collaborators: Optional[str] = None + extraction_method: Optional[str] = None + nucleic_acid_treatment: Optional[str] = None + extraction_protocol_doi: Optional[str] = None + nucleic_acid_conc: Optional[str] = None + nucleic_acid_volume: Optional[str] = None + gal: Optional[str] = None + raw_data_release_date: Optional[str] = None + + # Base ExperimentSubmission schema class ExperimentSubmissionBase(BaseModel): """Base ExperimentSubmission schema with common attributes.""" diff --git a/app/schemas/read.py b/app/schemas/read.py index 27d3b19..417ea80 100644 --- a/app/schemas/read.py +++ b/app/schemas/read.py @@ -69,6 +69,19 @@ class Read(ReadInDBBase): pass +class ReadDetail(ReadInDBBase): + """Detailed read schema used by nested aggregate endpoints.""" + + bpa_dataset_id: Optional[str] = None + file_name: Optional[str] = None + file_checksum: Optional[str] = None + file_format: Optional[str] = None + optional_file: bool + bioplatforms_url: Optional[str] = None + read_number: Optional[str] = None + lane_number: Optional[str] = None + + # Base ReadSubmission schema class ReadSubmissionBase(BaseModel): """Base ReadSubmission schema with common attributes.""" diff --git a/app/schemas/sample.py b/app/schemas/sample.py index fb014c9..af41b40 100644 --- a/app/schemas/sample.py +++ b/app/schemas/sample.py @@ -1,12 +1,13 @@ from datetime import datetime -from enum import Enum -from typing import Dict, Optional +from typing import Dict, List, Optional from uuid import UUID from pydantic import BaseModel, ConfigDict, Field # Enum for submission status from app.schemas.common import SampleKind, SubmissionStatus +from app.schemas.experiment import ExperimentDetail as ExperimentDetailSchema +from app.schemas.read import ReadDetail as ReadDetailSchema # Base Sample schema (aligns with schema.sql columns except id/timestamps/bpa_json) @@ -91,6 +92,28 @@ class Sample(SampleInDBBase): pass +class SampleExperimentReads(BaseModel): + """Nested experiment and read data for a sample.""" + + experiment: ExperimentDetailSchema + reads: List[ReadDetailSchema] = Field(default_factory=list) + + +class SpecimenSampleHierarchyItem(BaseModel): + """A related sample and its nested experiments and reads.""" + + sample: Sample + experiments: List[SampleExperimentReads] = Field(default_factory=list) + + +class SpecimenSampleHierarchyResponse(BaseModel): + """Response shape for all sample/experiment/read data linked to a specimen.""" + + taxon_id: int + specimen_id: str + samples: List[SpecimenSampleHierarchyItem] = Field(default_factory=list) + + # Base SampleSubmission schema class SampleSubmissionBase(BaseModel): """Base SampleSubmission schema with common attributes.""" diff --git a/app/services/assembly_helper.py b/app/services/assembly_helper.py index 2d444d0..ce76943 100644 --- a/app/services/assembly_helper.py +++ b/app/services/assembly_helper.py @@ -1,9 +1,8 @@ """Helper functions for assembly operations.""" import logging -from typing import Any, Dict, List, Set - -import yaml +from typing import Any, Dict, List, Optional, Set, Tuple +from uuid import UUID from app.models.experiment import Experiment from app.models.organism import Organism @@ -13,44 +12,66 @@ logger = logging.getLogger(__name__) -def determine_assembly_data_types(experiments: List[Experiment]) -> AssemblyDataTypes: - """Determine assembly data_types based on experiments. - - Rules: - - PACBIO_SMRT exists if: platform == "PACBIO_SMRT" - - OXFORD_NANOPORE exists if: platform == "OXFORD_NANOPORE" - - Hi-C exists if: platform == "ILLUMINA" AND library_strategy in ("Hi-C", "WGS") - - Args: - experiments: List of Experiment objects - - Returns: - AssemblyDataTypes enum value based on detected platforms - - Raises: - ValueError: If no valid sequencing platforms are detected - """ +def _detect_assembly_data_type_flags( + experiments: List[Experiment], +) -> Tuple[bool, bool, bool, bool]: + """Return booleans for PacBio, ONT, and Hi-C using assembly classification rules.""" has_pacbio = False has_nanopore = False has_hic = False + has_rnaseq = False for exp in experiments: platform = exp.platform.upper() if exp.platform else "" library_strategy = exp.library_strategy.upper() if exp.library_strategy else "" - # Check for PacBio if platform == "PACBIO_SMRT" and library_strategy in ("WGS", "WGA"): has_pacbio = True - - # Check for Oxford Nanopore if platform == "OXFORD_NANOPORE" and library_strategy in ("WGS", "WGA"): has_nanopore = True - - # Check for Hi-C (Illumina + Hi-C or WGS library strategy) - if platform == "ILLUMINA" and library_strategy in ("HI-C", "WGS"): + if platform == "ILLUMINA" and library_strategy == "HI-C": has_hic = True + if platform == "ILLUMINA" and library_strategy == "RNA-SEQ": + has_rnaseq = True + + return has_pacbio, has_nanopore, has_hic, has_rnaseq + + +def get_available_assembly_data_types(experiments: List[Experiment]) -> List[str]: + """Return atomic data types available for a specimen sample. + + This is used by discovery flows, so a sample with only Hi-C data should still + report ["Hi-C"] even though that combination is not sufficient for creating an + assembly intent on its own. + """ + has_pacbio, has_nanopore, has_hic, has_rnaseq = _detect_assembly_data_type_flags(experiments) + available_data_types: List[str] = [] + + if has_pacbio: + available_data_types.append("PACBIO_SMRT") + if has_nanopore: + available_data_types.append("OXFORD_NANOPORE") + if has_hic: + available_data_types.append("Hi-C") + if has_rnaseq: + available_data_types.append("RNA-Seq") + + return available_data_types + + +def determine_assembly_data_types(experiments: List[Experiment]) -> AssemblyDataTypes: + """Determine assembly data_types based on experiments. + + Rules: + - PACBIO_SMRT exists if: platform == "PACBIO_SMRT" + - OXFORD_NANOPORE exists if: platform == "OXFORD_NANOPORE" + - Hi-C exists if: platform == "ILLUMINA" AND library_strategy == "Hi-C" + + Raises: + ValueError: If no valid sequencing platforms are detected + """ + has_pacbio, has_nanopore, has_hic, has_rnaseq = _detect_assembly_data_type_flags(experiments) - # Determine the appropriate enum value based on combinations if has_pacbio and has_nanopore and has_hic: return AssemblyDataTypes.PACBIO_SMRT_OXFORD_NANOPORE_HIC elif has_pacbio and has_nanopore: @@ -64,7 +85,6 @@ def determine_assembly_data_types(experiments: List[Experiment]) -> AssemblyData elif has_nanopore: return AssemblyDataTypes.OXFORD_NANOPORE else: - # TODO decide if we relax this requirement and still return the manifest noting the available data types are not supported raise ValueError( "No valid data types detected in experiments. " "Expected PACBIO_SMRT (with or without Hi-C), OXFORD_NANOPORE (with or without Hi-C), or ILLUMINA with Hi-C." @@ -72,14 +92,7 @@ def determine_assembly_data_types(experiments: List[Experiment]) -> AssemblyData def get_detected_platforms(experiments: List[Experiment]) -> dict: - """Get a summary of detected platforms for debugging/logging. - - Args: - experiments: List of Experiment objects - - Returns: - Dictionary with platform detection details - """ + """Get a summary of detected platforms for debugging/logging.""" platforms: Set[str] = set() library_strategies: Set[str] = set() @@ -108,44 +121,53 @@ def _normalize_read_number(read_number: str | None) -> str | None: return None -def generate_assembly_manifest( +def generate_assembly_manifest_json( organism: Organism, reads: List[Read], experiments: List[Experiment], tol_id: str | None, version: int, + long_read_sample_id: UUID, + hic_sample_id: Optional[UUID] = None, sample_metadata_by_id: Dict[str, Dict[str, Any]] | None = None, -) -> str: - """Generate assembly manifest YAML from organism and reads data. + sequencing_sample_to_specimen_sample_id: Dict[str, str] | None = None, +) -> Dict[str, Any]: + """Generate an assembly manifest as a JSON-serialisable dict. - Groups reads by bpa_package_id (from Experiment), then by platform type. + Reads are routed to sections based on which specimen sample they belong to: + - Reads from long_read_sample_id experiments → PACBIO_SMRT or OXFORD_NANOPORE section + - Reads from hic_sample_id experiments → Hi-C section (omitted when hic_sample_id is None) - Rules: - - PACBIO_SMRT: Only include files ending in .ccs.bam or hifi_reads.bam - - Hi-C: Split reads into r1/r2 groups by read_number + PacBio filtering: only files ending in .ccs.bam or hifi_reads.bam are included. + ONT: all reads are included. + Hi-C: reads are split into r1/r2 by read_number and include lane_number. Args: organism: Organism object - reads: List of Read objects - experiments: List of Experiment objects (to determine platform) + reads: Combined list of Read objects from both specimen samples + experiments: Combined list of Experiment objects from both specimen samples tol_id: ToL ID for the assembly (optional) version: Assembly version number + long_read_sample_id: sample.id of the long-read specimen sample + hic_sample_id: sample.id of the Hi-C specimen sample (optional) sample_metadata_by_id: Sample metadata keyed by sample.id as string (optional) Returns: - YAML string formatted as assembly manifest + Dict representing the manifest (JSON-serialisable) """ logger.info( - f"Generating manifest for organism: {organism.scientific_name} (taxon_id: {organism.taxon_id})" + "Generating manifest for organism: %s (taxon_id: %s)", + organism.scientific_name, + organism.taxon_id, ) - logger.info(f"Total experiments: {len(experiments)}, Total reads: {len(reads)}") + logger.info("Total experiments: %d, Total reads: %d", len(experiments), len(reads)) + + long_read_sample_str = str(long_read_sample_id) + hic_sample_str = str(hic_sample_id) if hic_sample_id else None # Build experiment info map: experiment.id → metadata - exp_info_map = {} + exp_info_map: Dict[Any, Dict[str, Any]] = {} for exp in experiments: - logger.info( - f"Experiment {exp.id}: platform={exp.platform}, library_strategy={exp.library_strategy}" - ) exp_info_map[exp.id] = { "platform": exp.platform.upper() if exp.platform else "", "library_strategy": exp.library_strategy.upper() if exp.library_strategy else "", @@ -154,99 +176,142 @@ def generate_assembly_manifest( "sample_id": str(exp.sample_id) if getattr(exp, "sample_id", None) else None, } - # Group reads by bpa_package_id per platform pacbio_by_package: Dict[str, Any] = {} + ont_by_package: Dict[str, Any] = {} hic_by_package: Dict[str, Any] = {} for read in reads: if not read.experiment_id: - logger.debug(f"Read {read.id} has no experiment_id, skipping") + logger.debug("Read %s has no experiment_id, skipping", read.id) continue exp_info = exp_info_map.get(read.experiment_id) if not exp_info: - logger.debug(f"Read {read.id} has no matching experiment, skipping") + logger.debug("Read %s has no matching experiment, skipping", read.id) continue platform = exp_info["platform"] library_strategy = exp_info["library_strategy"] bpa_package_id = exp_info["bpa_package_id"] sample_id = exp_info["sample_id"] - sample_meta = (sample_metadata_by_id or {}).get(sample_id, {}) if sample_id else {} - - logger.debug( - f"Read {read.id} (file: {read.file_name}): platform={platform}, library_strategy={library_strategy}" + resolved_sample_id = ( + sequencing_sample_to_specimen_sample_id.get(sample_id, sample_id) + if sample_id and sequencing_sample_to_specimen_sample_id + else sample_id + ) + sample_meta = ( + (sample_metadata_by_id or {}).get(resolved_sample_id, {}) if resolved_sample_id else {} ) - # PacBio SMRT reads (only .ccs.bam or hifi_reads.bam) - if platform == "PACBIO_SMRT" and read.file_name: - if read.file_name.endswith(".ccs.bam") or read.file_name.endswith("hifi_reads.bam"): - logger.info(f"Adding PacBio read: {read.file_name}") - if bpa_package_id not in pacbio_by_package: - entry: Dict[str, Any] = { - "sample_id": sample_id, + # Route by specimen sample, then by platform + is_long_read_sample = resolved_sample_id == long_read_sample_str + is_hic_sample = hic_sample_str is not None and resolved_sample_id == hic_sample_str + + if is_long_read_sample: + if platform == "PACBIO_SMRT" and library_strategy in ("WGS", "WGA") and read.file_name: + if read.file_name.endswith(".ccs.bam") or read.file_name.endswith("hifi_reads.bam"): + logger.info("Adding PacBio read: %s", read.file_name) + if bpa_package_id not in pacbio_by_package: + entry: Dict[str, Any] = { + "sample_id": resolved_sample_id, + "bpa_sample_id": sample_meta.get("bpa_sample_id"), + "specimen_id": sample_meta.get("specimen_id"), + "resources": [], + } + if exp_info["bioplatforms_base_url"]: + entry["bioplatforms_base_url"] = exp_info["bioplatforms_base_url"] + pacbio_by_package[bpa_package_id] = entry + pacbio_by_package[bpa_package_id]["resources"].append( + {"md5sum": read.file_checksum, "url": read.bioplatforms_url} + ) + else: + logger.debug( + "Skipping PacBio read %s — not .ccs.bam or hifi_reads.bam", + read.file_name, + ) + elif platform == "OXFORD_NANOPORE" and library_strategy in ("WGS", "WGA"): + logger.info("Adding ONT read: %s", read.file_name) + if bpa_package_id not in ont_by_package: + entry = { + "sample_id": resolved_sample_id, "bpa_sample_id": sample_meta.get("bpa_sample_id"), "specimen_id": sample_meta.get("specimen_id"), "resources": [], } if exp_info["bioplatforms_base_url"]: entry["bioplatforms_base_url"] = exp_info["bioplatforms_base_url"] - pacbio_by_package[bpa_package_id] = entry - pacbio_by_package[bpa_package_id]["resources"].append( + ont_by_package[bpa_package_id] = entry + ont_by_package[bpa_package_id]["resources"].append( {"md5sum": read.file_checksum, "url": read.bioplatforms_url} ) + else: logger.debug( - f"Skipping PacBio read {read.file_name} - doesn't match .ccs.bam or hifi_reads.bam" + "Long-read sample read %s skipped: platform=%s not a long-read platform", + read.file_name, + platform, ) - # Hi-C reads (Illumina + Hi-C or WGS library strategy) - elif platform == "ILLUMINA" and library_strategy in ("HI-C", "WGS"): - logger.info(f"Adding Hi-C read: {read.file_name} (library_strategy={library_strategy})") - if bpa_package_id not in hic_by_package: - hic_by_package[bpa_package_id] = { - "sample_id": sample_id, - "bpa_sample_id": sample_meta.get("bpa_sample_id"), - "specimen_id": sample_meta.get("specimen_id"), - "resources": {"r1": [], "r2": []}, - } - rkey = _normalize_read_number(read.read_number) - if rkey in ("r1", "r2"): - hic_by_package[bpa_package_id]["resources"][rkey].append( - { - "url": read.bioplatforms_url, - "md5sum": read.file_checksum, - "lane_number": read.lane_number, - } + if is_hic_sample: + if platform == "ILLUMINA" and library_strategy == "HI-C": + logger.info( + "Adding Hi-C read: %s (library_strategy=%s)", read.file_name, library_strategy ) + if bpa_package_id not in hic_by_package: + hic_by_package[bpa_package_id] = { + "sample_id": resolved_sample_id, + "bpa_sample_id": sample_meta.get("bpa_sample_id"), + "specimen_id": sample_meta.get("specimen_id"), + "resources": {"r1": [], "r2": []}, + } + rkey = _normalize_read_number(read.read_number) + if rkey in ("r1", "r2"): + hic_by_package[bpa_package_id]["resources"][rkey].append( + { + "url": read.bioplatforms_url, + "md5sum": read.file_checksum, + "lane_number": read.lane_number, + } + ) + else: + logger.debug( + "Hi-C read %s has unrecognised read_number=%s, skipping", + read.file_name, + read.read_number, + ) else: logger.debug( - f"Hi-C read {read.file_name} has unrecognized read_number={read.read_number}, skipping" + "Hi-C sample read %s skipped: platform=%s library_strategy=%s not Hi-C", + read.file_name, + platform, + library_strategy, ) - else: + elif not is_long_read_sample: logger.debug( - f"Read {read.file_name} doesn't match criteria: platform={platform}, library_strategy={library_strategy}" + "Read %s sample_id=%s resolved_sample_id=%s does not match either specimen sample, skipping", + read.id, + sample_id, + resolved_sample_id, ) - # Build manifest structure - manifest = { - "scientific_name": organism.scientific_name, - "taxon_id": organism.taxon_id, - "tolid": tol_id, - "version": version, - "reads": {}, - } - + reads_section: Dict[str, Any] = {} if pacbio_by_package: - manifest["reads"]["PACBIO_SMRT"] = pacbio_by_package - logger.info(f"Added {len(pacbio_by_package)} PacBio packages to manifest") - + reads_section["PACBIO_SMRT"] = pacbio_by_package + logger.info("Added %d PacBio packages to manifest", len(pacbio_by_package)) + if ont_by_package: + reads_section["OXFORD_NANOPORE"] = ont_by_package + logger.info("Added %d ONT packages to manifest", len(ont_by_package)) if hic_by_package: - manifest["reads"]["Hi-C"] = hic_by_package - logger.info(f"Added {len(hic_by_package)} Hi-C packages to manifest") + reads_section["Hi-C"] = hic_by_package + logger.info("Added %d Hi-C packages to manifest", len(hic_by_package)) - if not pacbio_by_package and not hic_by_package: + if not reads_section: logger.warning("No reads matched the filtering criteria!") - # Convert to YAML - return yaml.dump(manifest, default_flow_style=False, sort_keys=False) + return { + "scientific_name": organism.scientific_name, + "taxon_id": organism.taxon_id, + "tolid": tol_id, + "version": version, + "reads": reads_section, + } diff --git a/app/services/assembly_service.py b/app/services/assembly_service.py index 060a8eb..a72ee17 100644 --- a/app/services/assembly_service.py +++ b/app/services/assembly_service.py @@ -9,6 +9,8 @@ AssemblyFile, AssemblyRead, AssemblyRun, + AssemblyStageRun, + AssemblyStageRunFile, AssemblySubmission, ) from app.models.experiment import Experiment @@ -18,6 +20,8 @@ AssemblyCreate, AssemblyFileCreate, AssemblyFileUpdate, + AssemblyStageRunCreate, + AssemblyStageRunUpdate, AssemblySubmissionCreate, AssemblySubmissionUpdate, AssemblyUpdate, @@ -155,6 +159,10 @@ def get_next_version( sample_id: UUID, data_types: str, ) -> int: + # Assembly is the primary source of truth for version numbers. + # AssemblyRun is also checked during the transition period to avoid conflicts + # with legacy reservations. Remove the AssemblyRun check once legacy records + # are resolved and the table is dropped. max_assembly = ( db.query(func.max(Assembly.version)) .filter( @@ -175,6 +183,69 @@ def get_next_version( ) return max(max_assembly or 0, max_run or 0) + 1 + def get_next_version_for_intent( + self, + db: Session, + *, + taxon_id: int, + long_read_specimen_sample_id: UUID, + ) -> int: + """Return the next version scoped by (taxon_id, long_read_specimen_sample_id). + + This is the versioning strategy for the intent flow. Versions are shared + across all assemblies for the same taxon + long-read specimen, regardless + of data_types or hic_specimen_sample_id. + """ + max_version = ( + db.query(func.max(Assembly.version)) + .filter( + Assembly.taxon_id == taxon_id, + Assembly.long_read_specimen_sample_id == long_read_specimen_sample_id, + ) + .scalar() + ) + return (max_version or 0) + 1 + + def create_from_intent( + self, + db: Session, + *, + taxon_id: int, + long_read_specimen_sample_id: UUID, + hic_specimen_sample_id: Optional[UUID], + data_types: str, + tol_id: Optional[str], + project_id: Optional[UUID], + manifest_json: Optional[dict] = None, + ) -> Assembly: + """Create an Assembly at manifest-request time with status='requested'. + + Versioning is scoped by (taxon_id, long_read_specimen_sample_id) only — + data_types and hic_specimen_sample_id are excluded from the version key. + sample_id is set to long_read_specimen_sample_id for backward compatibility. + """ + version = self.get_next_version_for_intent( + db, + taxon_id=taxon_id, + long_read_specimen_sample_id=long_read_specimen_sample_id, + ) + assembly = Assembly( + taxon_id=taxon_id, + sample_id=long_read_specimen_sample_id, + long_read_specimen_sample_id=long_read_specimen_sample_id, + hic_specimen_sample_id=hic_specimen_sample_id, + data_types=data_types, + version=version, + tol_id=tol_id, + project_id=project_id, + status="requested", + manifest_json=manifest_json, + ) + db.add(assembly) + db.commit() + db.refresh(assembly) + return assembly + class AssemblySubmissionService( BaseService[AssemblySubmission, AssemblySubmissionCreate, AssemblySubmissionUpdate] @@ -233,7 +304,93 @@ def get_by_assembly_id(self, db: Session, assembly_id: UUID) -> List[AssemblyRea return db.query(AssemblyRead).filter(AssemblyRead.assembly_id == assembly_id).all() +class AssemblyStageRunService( + BaseService[AssemblyStageRun, AssemblyStageRunCreate, AssemblyStageRunUpdate] +): + """Service for AssemblyStageRun operations.""" + + def get_by_assembly_id(self, db: Session, assembly_id: UUID) -> List[AssemblyStageRun]: + return ( + db.query(AssemblyStageRun) + .filter(AssemblyStageRun.assembly_id == assembly_id) + .order_by(AssemblyStageRun.created_at.desc()) + .all() + ) + + def create_with_files( + self, + db: Session, + *, + assembly_id: UUID, + run_in: AssemblyStageRunCreate, + ) -> AssemblyStageRun: + run = AssemblyStageRun( + assembly_id=assembly_id, + stage_name=run_in.stage_name, + status=run_in.status, + external_run_id=run_in.external_run_id, + attempt=run_in.attempt, + stats=run_in.stats, + started_at=run_in.started_at, + completed_at=run_in.completed_at, + ) + db.add(run) + db.flush() + for f in run_in.files: + db.add( + AssemblyStageRunFile( + assembly_stage_run_id=run.id, + storage_type=f.storage_type, + storage_uri=f.storage_uri, + storage_details=f.storage_details, + sha256sum=f.sha256sum, + ) + ) + db.commit() + db.refresh(run) + return run + + def update_with_files( + self, + db: Session, + *, + db_obj: AssemblyStageRun, + update_in: AssemblyStageRunUpdate, + ) -> AssemblyStageRun: + if update_in.status is not None: + db_obj.status = update_in.status + if update_in.external_run_id is not None: + db_obj.external_run_id = update_in.external_run_id + if update_in.stats is not None: + db_obj.stats = update_in.stats + if update_in.started_at is not None: + db_obj.started_at = update_in.started_at + if update_in.completed_at is not None: + db_obj.completed_at = update_in.completed_at + + if update_in.files is not None: + # Replace all files + db.query(AssemblyStageRunFile).filter( + AssemblyStageRunFile.assembly_stage_run_id == db_obj.id + ).delete() + for f in update_in.files: + db.add( + AssemblyStageRunFile( + assembly_stage_run_id=db_obj.id, + storage_type=f.storage_type, + storage_uri=f.storage_uri, + storage_details=f.storage_details, + sha256sum=f.sha256sum, + ) + ) + + db.commit() + db.refresh(db_obj) + return db_obj + + assembly_service = AssemblyService(Assembly) assembly_submission_service = AssemblySubmissionService(AssemblySubmission) assembly_file_service = AssemblyFileService(AssemblyFile) assembly_read_service = AssemblyReadService(AssemblyRead) +assembly_stage_run_service = AssemblyStageRunService(AssemblyStageRun) diff --git a/app/services/taxonomy_info_service.py b/app/services/taxonomy_info_service.py index bc1813a..8990109 100644 --- a/app/services/taxonomy_info_service.py +++ b/app/services/taxonomy_info_service.py @@ -53,9 +53,7 @@ def delete(self, db: Session, *, taxon_id: int) -> Optional[TaxonomyInfo]: db.commit() return ti - def bulk_import( - self, db: Session, *, data: Dict[str, Dict[str, Any]] - ) -> BulkImportResponse: + def bulk_import(self, db: Session, *, data: Dict[str, Dict[str, Any]]) -> BulkImportResponse: created_count = 0 skipped_count = 0 errors: List[str] = [] @@ -87,9 +85,7 @@ def bulk_import( continue # Reject duplicates - existing = ( - db.query(TaxonomyInfo).filter(TaxonomyInfo.taxon_id == taxon_id).first() - ) + existing = db.query(TaxonomyInfo).filter(TaxonomyInfo.taxon_id == taxon_id).first() if existing: errors.append(f"{key}: taxonomy_info for taxon_id {taxon_id} already exists") skipped_count += 1 diff --git a/schema.sql b/schema.sql index 2dc818e..5fd90eb 100644 --- a/schema.sql +++ b/schema.sql @@ -84,13 +84,30 @@ CREATE TABLE organism ( ncbi_order TEXT, ncbi_family TEXT, busco_dataset_name TEXT, - augustus_dataset_name TEXT, bpa_json JSONB, taxonomy_lineage_json JSONB, created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); +-- ========================================== +-- Taxonomy info table (1:1 extension of organism) +-- ========================================== + +CREATE TABLE taxonomy_info ( + taxon_id INT PRIMARY KEY REFERENCES organism(taxon_id) ON DELETE CASCADE, + busco_odb10_dataset_name TEXT, + busco_odb12_dataset_name TEXT, + find_plastid BOOLEAN, + hic_motif TEXT, + mitochondrial_genetic_code_id INTEGER, + mitohifi_reference_species TEXT, + oatk_hmm_name TEXT, + defined_class TEXT, + augustus_dataset_name TEXT, + genetic_code_id INTEGER +); + -- ========================================== -- Accession registry table -- ========================================== @@ -484,21 +501,37 @@ CREATE TABLE assembly ( project_id UUID REFERENCES project(id), -- Assembly metadata - assembly_name TEXT NOT NULL, + assembly_name TEXT, assembly_type TEXT NOT NULL DEFAULT 'clone or isolate', tol_id TEXT, data_types assembly_data_types NOT NULL, - coverage FLOAT NOT NULL, - program TEXT NOT NULL, + coverage FLOAT, + program TEXT, mingaplength FLOAT, moleculetype molecule_type NOT NULL DEFAULT 'genomic DNA', description TEXT, - -- Auto-incremented version per (data_types, taxon_id, sample_id) + -- Intent-flow specimen samples + -- long_read_specimen_sample_id: the specimen sample supplying PacBio / ONT reads + -- hic_specimen_sample_id: the specimen sample supplying Hi-C reads (optional) + long_read_specimen_sample_id UUID REFERENCES sample(id), + hic_specimen_sample_id UUID REFERENCES sample(id), + + -- Persisted manifest generated at intent time + manifest_json JSONB, + + -- Version is scoped by (taxon_id, long_read_specimen_sample_id) for intent-flow assemblies version INTEGER NOT NULL DEFAULT 1, + -- Assembly lifecycle status + status TEXT NOT NULL DEFAULT 'requested', + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + CONSTRAINT ck_assembly_status CHECK ( + status IN ('requested', 'running', 'curating', 'completed', 'failed', 'cancelled') + ) ); CREATE TABLE assembly_run ( @@ -532,9 +565,12 @@ CREATE TABLE assembly_file ( CREATE INDEX idx_assembly_file_assembly_id ON assembly_file(assembly_id); CREATE INDEX idx_assembly_file_type ON assembly_file(assembly_id, file_type); --- Index for version lookups +-- Index for legacy version lookups (generic create flow) CREATE INDEX idx_assembly_version_key ON assembly(data_types, taxon_id, sample_id, version); +-- Index for intent-flow version lookups (scoped by taxon + long-read specimen sample) +CREATE INDEX idx_assembly_intent_version_key ON assembly(taxon_id, long_read_specimen_sample_id); + -- Assembly submission table (simplified - no broker integration) CREATE TABLE assembly_submission ( @@ -572,6 +608,57 @@ CREATE TABLE assembly_read ( PRIMARY KEY (assembly_id, read_id) ); +-- ========================================== +-- Assembly stage catalog and reporting tables +-- ========================================== + +CREATE TABLE assembly_stage ( + name TEXT PRIMARY KEY, + category TEXT NOT NULL, + is_active BOOLEAN NOT NULL DEFAULT TRUE, + CONSTRAINT ck_assembly_stage_category CHECK (category IN ('pipeline', 'manual')) +); + +-- Seed rows +INSERT INTO assembly_stage (name, category) VALUES + ('genomeassembly', 'pipeline'), + ('ascc', 'pipeline'), + ('treeval', 'pipeline'), + ('curation-pretext', 'pipeline'), + ('manual-curation', 'manual'); + +CREATE TABLE assembly_stage_run ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + assembly_id UUID NOT NULL REFERENCES assembly(id) ON DELETE CASCADE, + stage_name TEXT NOT NULL REFERENCES assembly_stage(name), + status TEXT NOT NULL, + external_run_id TEXT, + attempt INTEGER NOT NULL DEFAULT 1, + stats JSONB NOT NULL DEFAULT '{}', + started_at TIMESTAMPTZ, + completed_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT ck_assembly_stage_run_status CHECK ( + status IN ('running', 'succeeded', 'failed', 'cancelled') + ), + CONSTRAINT uq_stage_run_assembly_stage_attempt UNIQUE (assembly_id, stage_name, attempt) +); + +CREATE INDEX ix_assembly_stage_run_assembly_id ON assembly_stage_run(assembly_id); + +CREATE TABLE assembly_stage_run_file ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + assembly_stage_run_id UUID NOT NULL REFERENCES assembly_stage_run(id) ON DELETE CASCADE, + storage_type TEXT NOT NULL, + storage_uri TEXT NOT NULL, + storage_details JSONB NOT NULL DEFAULT '{}', + sha256sum TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX ix_assembly_stage_run_file_run_id ON assembly_stage_run_file(assembly_stage_run_id); + -- ========================================== -- Genome note tables -- ========================================== diff --git a/tests/unit/endpoints/test_endpoints_assemblies.py b/tests/unit/endpoints/test_endpoints_assemblies.py index 2f8b8b6..5c1bdee 100644 --- a/tests/unit/endpoints/test_endpoints_assemblies.py +++ b/tests/unit/endpoints/test_endpoints_assemblies.py @@ -100,6 +100,7 @@ def test_create_assembly_from_experiments_success(monkeypatch): program="hifiasm", moleculetype="genomic DNA", version=1, + status="requested", created_at=datetime.now(timezone.utc), updated_at=datetime.now(timezone.utc), ) @@ -181,44 +182,24 @@ def mock_create_raises(*args, **kwargs): def test_get_assembly_manifest_success(monkeypatch): - """Test successful manifest generation.""" + """GET /manifest/{taxon_id} returns stored manifest JSON for latest requested assembly.""" client = TestClient(app) - # Mock database queries - organism = SimpleNamespace( - scientific_name="Test Species", - taxon_id=172942, - ) - sample = SimpleNamespace( - id="550e8400-e29b-41d4-a716-446655440000", - taxon_id=172942, - kind="specimen", - bpa_sample_id="102.100.100/9000", - specimen_id="SPEC-001", - ) - experiment = SimpleNamespace( - id="exp-1", - sample_id="550e8400-e29b-41d4-a716-446655440000", - platform="PACBIO_SMRT", - library_strategy="WGS", - bpa_package_id="pkg-exp-1", - bioplatforms_base_url=None, - ) - read = SimpleNamespace( - id="read-1", - experiment_id="exp-1", - file_name="sample.ccs.bam", - file_checksum="abc123", - bioplatforms_url="https://example.com/1", - read_number=None, - lane_number=None, - ) - assembly_run = SimpleNamespace( + organism = SimpleNamespace(scientific_name="Test Species", taxon_id=172942) + manifest_json = { + "scientific_name": "Test Species", + "taxon_id": 172942, + "tolid": "tol-123", + "version": 1, + "reads": {"PACBIO_SMRT": {"pkg-exp-1": {"resources": []}}}, + } + requested_assembly = SimpleNamespace( id="run-1", taxon_id=172942, - sample_id="550e8400-e29b-41d4-a716-446655440000", tol_id="tol-123", version=1, + status="requested", + manifest_json=manifest_json, ) class MockQuery: @@ -243,22 +224,10 @@ def __init__(self): def query(self, model): self.call_count += 1 - if self.call_count == 1: # organism query + if self.call_count == 1: return MockQuery(organism) - elif self.call_count == 2: # selected specimen sample query - return MockQuery(sample) - elif self.call_count == 3: # selected sample by ID query - return MockQuery(sample) - elif self.call_count == 4: # all sample IDs query - return MockQuery([(sample.id,)]) - elif self.call_count == 5: # experiments query - return MockQuery([experiment]) - elif self.call_count == 6: # reads query - return MockQuery([read]) - elif self.call_count == 7: # latest assembly_run query - return MockQuery(assembly_run) - elif self.call_count == 8: # sample metadata query for per-read manifest fields - return MockQuery([sample]) + if self.call_count == 2: + return MockQuery(requested_assembly) return MockQuery([]) app.dependency_overrides[assemblies.get_current_active_user] = lambda: SimpleNamespace( @@ -269,13 +238,8 @@ def query(self, model): resp = client.get("/api/v1/assemblies/manifest/172942") assert resp.status_code == 200 - assert resp.headers["content-type"] == "application/x-yaml" - assert b"scientific_name: Test Species" in resp.content - assert b"taxon_id: 172942" in resp.content - assert b"sample_id: 550e8400-e29b-41d4-a716-446655440000" in resp.content - assert b"bpa_sample_id: 102.100.100/9000" in resp.content - assert b"specimen_id: SPEC-001" in resp.content - assert b"PACBIO_SMRT:" in resp.content + assert resp.headers["content-type"].startswith("application/json") + assert resp.json() == manifest_json def test_get_assembly_manifest_organism_not_found(): @@ -309,33 +273,57 @@ def first(self): def test_create_assembly_intent_invalid_data_types_returns_app_error(monkeypatch): client = TestClient(app) - organism = SimpleNamespace( - scientific_name="Test Species", - taxon_id=172942, + organism = SimpleNamespace(scientific_name="Test Species", taxon_id=172942) + long_read_sample_id = uuid4() + long_read_sample = SimpleNamespace(id=long_read_sample_id, taxon_id=172942, kind="specimen") + invalid_long_read_exp = SimpleNamespace( + id="e1", + sample_id=long_read_sample_id, + platform="PACBIO_SMRT", + library_strategy="RNA", ) - reads = [SimpleNamespace(id="r1", experiment_id="e1")] - experiments = [SimpleNamespace(id="e1", platform="UNKNOWN", library_strategy="UNKNOWN")] - monkeypatch.setattr( - assemblies, - "_get_manifest_inputs_by_taxon_id", - lambda db, taxon_id: ( - organism, - SimpleNamespace(id="550e8400-e29b-41d4-a716-446655440000"), - reads, - experiments, - ), - ) + class _Q: + def __init__(self, value): + self.value = value + + def filter(self, *_a, **_k): + return self + + def all(self): + return self.value if isinstance(self.value, list) else [] + + def first(self): + return self.value if not isinstance(self.value, list) else None + + class _DB: + def __init__(self): + self.calls = 0 + + def query(self, _model): + self.calls += 1 + if self.calls == 1: + return _Q(organism) + if self.calls == 2: + return _Q(long_read_sample) + if self.calls == 3: + return _Q([]) + if self.calls == 4: + return _Q([invalid_long_read_exp]) + if self.calls == 5: + return _Q([]) + return _Q([]) app.dependency_overrides[assemblies.get_current_active_user] = lambda: SimpleNamespace( is_active=True, roles=["curator"], is_superuser=False ) - app.dependency_overrides[assemblies.get_db] = _override_db(_FakeSession()) + app.dependency_overrides[assemblies.get_db] = _override_db(_DB()) resp = client.post( "/api/v1/assemblies/intent/172942", json={ "tol_id": "tol-123", + "long_read_specimen_sample_id": str(long_read_sample_id), }, ) @@ -345,14 +333,300 @@ def test_create_assembly_intent_invalid_data_types_returns_app_error(monkeypatch assert "No valid data types detected in experiments" in body["error"]["message"] -def test_create_assembly_intent_allows_empty_body(monkeypatch): +def test_create_assembly_intent_requires_specimen_sample_id(): + client = TestClient(app) + + app.dependency_overrides[assemblies.get_current_active_user] = lambda: SimpleNamespace( + is_active=True, roles=["curator"], is_superuser=False + ) + app.dependency_overrides[assemblies.get_db] = _override_db(_FakeSession()) + + resp = client.post( + "/api/v1/assemblies/intent/172942", + json={"tol_id": "tol-123"}, + ) + + assert resp.status_code == 422 + + +def test_get_specimen_samples_for_assembly_returns_discovery_options(): client = TestClient(app) - organism = SimpleNamespace( - scientific_name="Test Species", + organism = SimpleNamespace(scientific_name="Test Species", taxon_id=172942) + specimen_a_id = uuid4() + derived_a_id = uuid4() + specimen_b_id = uuid4() + specimen_c_id = uuid4() + specimen_d_id = uuid4() + specimen_a_pacbio_exp_id = uuid4() + specimen_a_hic_exp_id = uuid4() + specimen_b_ont_exp_id = uuid4() + specimen_d_rna_exp_id = uuid4() + + specimen_a = SimpleNamespace( + id=specimen_a_id, taxon_id=172942, + kind="specimen", + specimen_id="SPEC-001", + sex="female", + ) + derived_a = SimpleNamespace( + id=derived_a_id, + taxon_id=172942, + kind="derived", + derived_from_sample_id=specimen_a_id, + ) + specimen_b = SimpleNamespace( + id=specimen_b_id, + taxon_id=172942, + kind="specimen", + specimen_id=None, + sex="male", + ) + specimen_c = SimpleNamespace( + id=specimen_c_id, + taxon_id=172942, + kind="specimen", + specimen_id="SPEC-003", + sex="unknown", + ) + specimen_d = SimpleNamespace( + id=specimen_d_id, + taxon_id=172942, + kind="specimen", + specimen_id="SPEC-004", + sex="female", + ) + + specimen_a_experiments = [ + SimpleNamespace( + id=specimen_a_pacbio_exp_id, + sample_id=derived_a_id, + platform="PACBIO_SMRT", + library_strategy="WGS", + ), + SimpleNamespace( + id=specimen_a_hic_exp_id, + sample_id=specimen_a_id, + platform="ILLUMINA", + library_strategy="Hi-C", + ), + ] + specimen_a_qc_reads = [ + SimpleNamespace( + id=uuid4(), + experiment_id=specimen_a_pacbio_exp_id, + base_count=1000, + read_count=100, + qc_bases_removed=10, + qc_reads_removed=2, + mean_gc_content=0.45, + n50_length=5000, + created_at="2026-01-01T00:00:00Z", + updated_at="2026-01-01T00:00:00Z", + files=[], + submission_records=[], + ) + ] + specimen_b_experiments = [ + SimpleNamespace( + id=specimen_b_ont_exp_id, + sample_id=specimen_b_id, + platform="OXFORD_NANOPORE", + library_strategy="WGA", + ) + ] + specimen_d_experiments = [ + SimpleNamespace( + id=specimen_d_rna_exp_id, + sample_id=specimen_d_id, + platform="ILLUMINA", + library_strategy="RNA-Seq", + ) + ] + + class _Q: + def __init__(self, value): + self.value = value + + def filter(self, *_a, **_k): + return self + + def all(self): + return self.value if isinstance(self.value, list) else [] + + def first(self): + return self.value if not isinstance(self.value, list) else None + + class _DB: + def __init__(self): + self.calls = 0 + + def query(self, _model): + self.calls += 1 + if self.calls == 1: + return _Q(organism) + if self.calls == 2: + return _Q([specimen_a, specimen_b, specimen_c, specimen_d]) + if self.calls == 3: + return _Q([derived_a]) + if self.calls == 4: + return _Q(specimen_a_experiments) + if self.calls == 5: + return _Q(specimen_a_qc_reads) + if self.calls == 6: + return _Q([]) + if self.calls == 7: + return _Q(specimen_b_experiments) + if self.calls == 8: + return _Q([]) + if self.calls == 9: + return _Q([]) + if self.calls == 10: + return _Q([]) + if self.calls == 11: + return _Q([]) + if self.calls == 12: + return _Q(specimen_d_experiments) + if self.calls == 13: + return _Q([]) + return _Q([]) + + app.dependency_overrides[assemblies.get_current_active_user] = lambda: SimpleNamespace( + is_active=True, roles=["admin"], is_superuser=False + ) + app.dependency_overrides[assemblies.get_db] = _override_db(_DB()) + + resp = client.get("/api/v1/assemblies/specimen-samples/172942") + + assert resp.status_code == 200 + body = resp.json() + assert body["taxon_id"] == 172942 + assert body["specimen_samples"] == [ + { + "sample_id": str(specimen_a_id), + "specimen_id": "SPEC-001", + "sex": "female", + "available_data_types": ["PACBIO_SMRT", "Hi-C"], + "qc_reads": [ + { + "id": str(specimen_a_qc_reads[0].id), + "experiment_id": str(specimen_a_pacbio_exp_id), + "base_count": 1000, + "read_count": 100, + "qc_bases_removed": 10, + "qc_reads_removed": 2, + "mean_gc_content": 0.45, + "n50_length": 5000, + "created_at": "2026-01-01T00:00:00Z", + "updated_at": "2026-01-01T00:00:00Z", + "files": [], + "submission_records": [], + } + ], + }, + { + "sample_id": str(specimen_b_id), + "specimen_id": None, + "sex": "male", + "available_data_types": ["OXFORD_NANOPORE"], + "qc_reads": [], + }, + { + "sample_id": str(specimen_c_id), + "specimen_id": "SPEC-003", + "sex": "unknown", + "available_data_types": [], + "qc_reads": [], + }, + { + "sample_id": str(specimen_d_id), + "specimen_id": "SPEC-004", + "sex": "female", + "available_data_types": ["RNA-Seq"], + "qc_reads": [], + }, + ] + + +def test_get_specimen_samples_for_assembly_returns_404_for_unknown_taxon(): + client = TestClient(app) + + class _Q: + def filter(self, *_a, **_k): + return self + + def first(self): + return None + + class _DB: + def query(self, _model): + return _Q() + + app.dependency_overrides[assemblies.get_current_active_user] = lambda: SimpleNamespace( + is_active=True, roles=["admin"], is_superuser=False + ) + app.dependency_overrides[assemblies.get_db] = _override_db(_DB()) + + resp = client.get("/api/v1/assemblies/specimen-samples/999999") + + assert resp.status_code == 404 + assert "not found" in resp.json()["error"]["message"] + + +def test_create_assembly_intent_rejects_non_specimen_sample(): + client = TestClient(app) + + organism = SimpleNamespace(scientific_name="Test Species", taxon_id=172942) + derived_sample_id = uuid4() + derived_sample = SimpleNamespace(id=derived_sample_id, taxon_id=172942, kind="derived") + + class _Q: + def __init__(self, value): + self.value = value + + def filter(self, *_a, **_k): + return self + + def first(self): + return self.value + + class _DB: + def __init__(self): + self.calls = 0 + + def query(self, _model): + self.calls += 1 + if self.calls == 1: + return _Q(organism) + return _Q(derived_sample) + + app.dependency_overrides[assemblies.get_current_active_user] = lambda: SimpleNamespace( + is_active=True, roles=["curator"], is_superuser=False + ) + app.dependency_overrides[assemblies.get_db] = _override_db(_DB()) + + resp = client.post( + "/api/v1/assemblies/intent/172942", + json={"long_read_specimen_sample_id": str(derived_sample_id)}, + ) + + assert resp.status_code == 422 + assert resp.json()["error"]["code"] == "specimen_sample_invalid_kind" + + +def test_create_assembly_intent_success_returns_json_manifest(monkeypatch): + client = TestClient(app) + + organism = SimpleNamespace(scientific_name="Test Species", taxon_id=172942) + long_read_sample_id = uuid4() + selected_sample = SimpleNamespace( + id=long_read_sample_id, + taxon_id=172942, + kind="specimen", + bpa_sample_id="102.100.100/9000", + specimen_id="SPEC-001", ) - selected_sample = SimpleNamespace(id="550e8400-e29b-41d4-a716-446655440000") run_id = uuid4() reads = [ SimpleNamespace( @@ -368,7 +642,7 @@ def test_create_assembly_intent_allows_empty_body(monkeypatch): experiments = [ SimpleNamespace( id="e1", - sample_id=selected_sample.id, + sample_id=long_read_sample_id, platform="PACBIO_SMRT", library_strategy="WGS", bpa_package_id="pkg-e1", @@ -376,42 +650,61 @@ def test_create_assembly_intent_allows_empty_body(monkeypatch): ) ] + mock_assembly = SimpleNamespace( + id=run_id, + version=1, + status="requested", + tol_id=None, + manifest_json=None, + ) monkeypatch.setattr( - assemblies, - "_get_manifest_inputs_by_taxon_id", - lambda db, taxon_id: (organism, selected_sample, reads, experiments), + assemblies.assembly_service, + "create_from_intent", + lambda db, **kwargs: mock_assembly, ) - monkeypatch.setattr(assemblies.assembly_service, "get_next_version", lambda *_, **__: 1) class _FakeIntentDB: def __init__(self): - self.last_added = None + self.calls = 0 def query(self, _model): + self.calls += 1 + class _Q: + def __init__(self, value): + self.value = value + def filter(self, *_a, **_k): return self def all(self): - return [ - SimpleNamespace( - id=selected_sample.id, - bpa_sample_id="102.100.100/9000", - specimen_id="SPEC-001", - ) - ] + return self.value if isinstance(self.value, list) else [] - return _Q() + def first(self): + return self.value if not isinstance(self.value, list) else None + + if self.calls == 1: + return _Q(organism) + if self.calls == 2: + return _Q(selected_sample) + if self.calls == 3: + return _Q([]) + if self.calls == 4: + return _Q(experiments) + if self.calls == 5: + return _Q(reads) + return _Q([]) def add(self, _obj): - self.last_added = _obj return None def commit(self): return None - def refresh(self, obj): - obj.id = run_id + def refresh(self, _obj): + return None + + def delete(self, _obj): return None app.dependency_overrides[assemblies.get_current_active_user] = lambda: SimpleNamespace( @@ -419,36 +712,149 @@ def refresh(self, obj): ) app.dependency_overrides[assemblies.get_db] = _override_db(_FakeIntentDB()) - resp = client.post("/api/v1/assemblies/intent/172942") + resp = client.post( + "/api/v1/assemblies/intent/172942", + json={"long_read_specimen_sample_id": str(long_read_sample_id)}, + ) assert resp.status_code == 200 - import yaml as _yaml - - body = _yaml.safe_load(resp.content) - assert body["assembly_run_id"] == str(run_id) + body = resp.json() + assert body["assembly_id"] == str(run_id) assert body["version"] == 1 - assert body["status"] == "reserved" + assert body["status"] == "requested" assert "manifest" in body +def test_create_assembly_intent_resolves_reads_via_derived_samples(monkeypatch): + client = TestClient(app) + + organism = SimpleNamespace(scientific_name="Test Species", taxon_id=172942) + long_read_sample_id = uuid4() + derived_sample_id = uuid4() + specimen_sample = SimpleNamespace( + id=long_read_sample_id, + taxon_id=172942, + kind="specimen", + bpa_sample_id="102.100.100/9000", + specimen_id="SPEC-001", + ) + derived_sample = SimpleNamespace( + id=derived_sample_id, + taxon_id=172942, + kind="derived", + derived_from_sample_id=long_read_sample_id, + ) + run_id = uuid4() + reads = [ + SimpleNamespace( + id="r1", + experiment_id="e1", + file_name="sample.ccs.bam", + file_checksum="abc123", + bioplatforms_url="https://example.com/1", + read_number=None, + lane_number=None, + ) + ] + experiments = [ + SimpleNamespace( + id="e1", + sample_id=derived_sample_id, + platform="PACBIO_SMRT", + library_strategy="WGS", + bpa_package_id="pkg-e1", + bioplatforms_base_url=None, + ) + ] + + mock_assembly = SimpleNamespace( + id=run_id, + version=1, + status="requested", + tol_id=None, + manifest_json=None, + ) + monkeypatch.setattr( + assemblies.assembly_service, + "create_from_intent", + lambda db, **kwargs: mock_assembly, + ) + + class _FakeIntentDB: + def __init__(self): + self.calls = 0 + + def query(self, _model): + self.calls += 1 + + class _Q: + def __init__(self, value): + self.value = value + + def filter(self, *_a, **_k): + return self + + def all(self): + return self.value if isinstance(self.value, list) else [] + + def first(self): + return self.value if not isinstance(self.value, list) else None + + if self.calls == 1: + return _Q(organism) + if self.calls == 2: + return _Q(specimen_sample) + if self.calls == 3: + return _Q([derived_sample]) + if self.calls == 4: + return _Q(experiments) + if self.calls == 5: + return _Q(reads) + return _Q([]) + + def add(self, _obj): + return None + + def commit(self): + return None + + def refresh(self, _obj): + return None + + def delete(self, _obj): + return None + + app.dependency_overrides[assemblies.get_current_active_user] = lambda: SimpleNamespace( + is_active=True, roles=["curator"], is_superuser=False + ) + app.dependency_overrides[assemblies.get_db] = _override_db(_FakeIntentDB()) + + resp = client.post( + "/api/v1/assemblies/intent/172942", + json={"long_read_specimen_sample_id": str(long_read_sample_id)}, + ) + + assert resp.status_code == 200 + body = resp.json() + assert body["manifest"]["reads"]["PACBIO_SMRT"]["pkg-e1"]["sample_id"] == str( + long_read_sample_id + ) + assert body["manifest"]["reads"]["PACBIO_SMRT"]["pkg-e1"]["specimen_id"] == "SPEC-001" + + def test_cancel_assembly_intent_success(monkeypatch): client = TestClient(app) organism = SimpleNamespace(taxon_id=172942) - selected_sample_id = "550e8400-e29b-41d4-a716-446655440000" + long_read_sample_id = uuid4() run_id = uuid4() run = SimpleNamespace( id=run_id, taxon_id=172942, - sample_id=selected_sample_id, + long_read_specimen_sample_id=long_read_sample_id, + hic_specimen_sample_id=None, version=2, - status="reserved", - ) - - monkeypatch.setattr( - assemblies, - "_get_optimal_sample_id_for_taxon_id", - lambda db, taxon_id, organism_taxon_id=None: selected_sample_id, + status="requested", ) class _Q: @@ -490,7 +896,7 @@ def refresh(self, _obj): resp = client.post( "/api/v1/assemblies/intent/172942/cancel", - json={"assembly_run_id": str(run_id)}, + json={"assembly_id": str(run_id)}, ) assert resp.status_code == 200 @@ -503,13 +909,6 @@ def test_cancel_assembly_intent_not_found(monkeypatch): client = TestClient(app) organism = SimpleNamespace(taxon_id=172942) - selected_sample_id = "550e8400-e29b-41d4-a716-446655440000" - - monkeypatch.setattr( - assemblies, - "_get_optimal_sample_id_for_taxon_id", - lambda db, taxon_id, organism_taxon_id=None: selected_sample_id, - ) class _Q: def __init__(self, value): @@ -541,8 +940,319 @@ def query(self, _model): resp = client.post( "/api/v1/assemblies/intent/172942/cancel", - json={"assembly_run_id": str(uuid4())}, + json={"assembly_id": str(uuid4())}, + ) + + assert resp.status_code == 404 + assert "No requested assembly found to cancel" in resp.json()["error"]["message"] + + +# ── New manifest-by-assembly-id tests ────────────────────────────────────── + + +def test_get_manifest_by_assembly_id_success(monkeypatch): + """GET /{assembly_id}/manifest returns stored manifest JSON for a known assembly.""" + from uuid import uuid4 as _uuid4 + + client = TestClient(app) + assembly_id = _uuid4() + manifest_json = { + "scientific_name": "Test Species", + "taxon_id": 172942, + "tolid": "tol-999", + "version": 3, + "reads": {"PACBIO_SMRT": {"pkg-exp-1": {"resources": []}}}, + } + + assembly = SimpleNamespace( + id=assembly_id, + taxon_id=172942, + tol_id="tol-999", + version=3, + status="requested", + manifest_json=manifest_json, + ) + + class MockQuery: + def __init__(self, rv): + self.rv = rv + + def filter(self, *a, **k): + return self + + def all(self): + return self.rv if isinstance(self.rv, list) else [] + + def first(self): + return self.rv if not isinstance(self.rv, list) else None + + class MockDB: + def query(self, _m): + return MockQuery(assembly) + + app.dependency_overrides[assemblies.get_current_active_user] = lambda: SimpleNamespace( + is_active=True, roles=["admin"], is_superuser=False + ) + app.dependency_overrides[assemblies.get_db] = lambda: MockDB() + + resp = client.get(f"/api/v1/assemblies/{assembly_id}/manifest") + + assert resp.status_code == 200 + assert resp.headers["content-type"].startswith("application/json") + assert resp.json() == manifest_json + + +def test_get_manifest_by_assembly_id_not_found(): + """GET /{assembly_id}/manifest returns 404 when assembly does not exist.""" + from uuid import uuid4 as _uuid4 + + client = TestClient(app) + + class MockDB: + def query(self, _m): + return self + + def filter(self, *a, **k): + return self + + def first(self): + return None + + app.dependency_overrides[assemblies.get_current_active_user] = lambda: SimpleNamespace( + is_active=True, roles=["admin"], is_superuser=False + ) + app.dependency_overrides[assemblies.get_db] = lambda: MockDB() + + resp = client.get(f"/api/v1/assemblies/{_uuid4()}/manifest") + assert resp.status_code == 404 + assert "Assembly not found" in resp.json()["error"]["message"] + + +# ── Stage-run endpoint tests ──────────────────────────────────────────────── + + +def _make_stage_run(assembly_id=None): + """Return a SimpleNamespace that satisfies AssemblyStageRunOut.""" + from datetime import datetime, timezone + + return SimpleNamespace( + id=uuid4(), + assembly_id=assembly_id or uuid4(), + stage_name="genomeassembly", + status="succeeded", + external_run_id="ext-123", + attempt=1, + stats={"n50": 10000}, + started_at=datetime(2026, 1, 1, tzinfo=timezone.utc), + completed_at=datetime(2026, 1, 2, tzinfo=timezone.utc), + created_at=datetime(2026, 1, 1, tzinfo=timezone.utc), + updated_at=datetime(2026, 1, 2, tzinfo=timezone.utc), + files=[], + ) + + +def test_list_stage_runs(monkeypatch): + """GET /{assembly_id}/stage-runs returns list of stage runs.""" + client = TestClient(app) + assembly_id = uuid4() + stage_run = _make_stage_run(assembly_id=assembly_id) + + monkeypatch.setattr( + assemblies.assembly_service, "get", lambda db, id: SimpleNamespace(id=assembly_id) + ) + monkeypatch.setattr( + assemblies.assembly_stage_run_service, + "get_by_assembly_id", + lambda db, assembly_id: [stage_run], + ) + + app.dependency_overrides[assemblies.get_current_active_user] = lambda: SimpleNamespace( + is_active=True, roles=["admin"], is_superuser=False + ) + app.dependency_overrides[assemblies.get_db] = _override_db(_FakeSession()) + + resp = client.get(f"/api/v1/assemblies/{assembly_id}/stage-runs") + assert resp.status_code == 200 + body = resp.json() + assert isinstance(body, list) + assert len(body) == 1 + assert body[0]["stage_name"] == "genomeassembly" + assert body[0]["status"] == "succeeded" + + +def test_list_stage_runs_assembly_not_found(monkeypatch): + """GET /{assembly_id}/stage-runs returns 404 when assembly missing.""" + client = TestClient(app) + + monkeypatch.setattr(assemblies.assembly_service, "get", lambda db, id: None) + + app.dependency_overrides[assemblies.get_current_active_user] = lambda: SimpleNamespace( + is_active=True, roles=["admin"], is_superuser=False ) + app.dependency_overrides[assemblies.get_db] = _override_db(_FakeSession()) + resp = client.get(f"/api/v1/assemblies/{uuid4()}/stage-runs") assert resp.status_code == 404 - assert "No reserved assembly intent found" in resp.json()["error"]["message"] + + +def test_create_stage_run_success(monkeypatch): + """POST /{assembly_id}/stage-runs creates a stage run with files.""" + client = TestClient(app) + assembly_id = uuid4() + stage_run = _make_stage_run(assembly_id=assembly_id) + stage_run.files = [ + SimpleNamespace( + id=uuid4(), + assembly_stage_run_id=stage_run.id, + storage_type="s3", + storage_uri="s3://bucket/key", + storage_details={"region": "ap-southeast-2"}, + sha256sum="deadbeef", + created_at=stage_run.created_at, + ) + ] + + monkeypatch.setattr( + assemblies.assembly_service, "get", lambda db, id: SimpleNamespace(id=assembly_id) + ) + monkeypatch.setattr( + assemblies.assembly_stage_run_service, + "create_with_files", + lambda db, **kwargs: stage_run, + ) + + app.dependency_overrides[assemblies.get_current_active_user] = lambda: SimpleNamespace( + is_active=True, roles=["curator"], is_superuser=False + ) + app.dependency_overrides[assemblies.get_db] = _override_db(_FakeSession()) + + resp = client.post( + f"/api/v1/assemblies/{assembly_id}/stage-runs", + json={ + "stage_name": "genomeassembly", + "status": "succeeded", + "attempt": 1, + "stats": {"n50": 10000}, + "files": [ + { + "storage_type": "s3", + "storage_uri": "s3://bucket/key", + "storage_details": {"region": "ap-southeast-2"}, + "sha256sum": "deadbeef", + } + ], + }, + ) + + assert resp.status_code == 200 + body = resp.json() + assert body["stage_name"] == "genomeassembly" + assert body["status"] == "succeeded" + assert len(body["files"]) == 1 + assert body["files"][0]["storage_type"] == "s3" + assert body["files"][0]["sha256sum"] == "deadbeef" + + +def test_update_stage_run_status(monkeypatch): + """PATCH /{assembly_id}/stage-runs/{stage_run_id} updates status.""" + client = TestClient(app) + assembly_id = uuid4() + stage_run = _make_stage_run(assembly_id=assembly_id) + updated_run = _make_stage_run(assembly_id=assembly_id) + updated_run.id = stage_run.id + updated_run.status = "failed" + + class _Q: + def filter(self, *_a, **_k): + return self + + def first(self): + return stage_run + + monkeypatch.setattr( + assemblies.assembly_stage_run_service, + "update_with_files", + lambda db, db_obj, update_in: updated_run, + ) + + app.dependency_overrides[assemblies.get_current_active_user] = lambda: SimpleNamespace( + is_active=True, roles=["curator"], id=uuid4(), is_superuser=False + ) + + class _DB: + def query(self, _m): + return _Q() + + app.dependency_overrides[assemblies.get_db] = lambda: _DB() + + resp = client.patch( + f"/api/v1/assemblies/{assembly_id}/stage-runs/{stage_run.id}", + json={"status": "failed"}, + ) + + assert resp.status_code == 200 + assert resp.json()["status"] == "failed" + + +def test_update_stage_run_replaces_files(monkeypatch): + """PATCH replaces all files when files list is provided.""" + from datetime import datetime, timezone + + client = TestClient(app) + assembly_id = uuid4() + stage_run = _make_stage_run(assembly_id=assembly_id) + new_file = SimpleNamespace( + id=uuid4(), + assembly_stage_run_id=stage_run.id, + storage_type="gcs", + storage_uri="gs://bucket/new-key", + storage_details={}, + sha256sum="cafebabe", + created_at=datetime(2026, 1, 3, tzinfo=timezone.utc), + ) + updated_run = _make_stage_run(assembly_id=assembly_id) + updated_run.id = stage_run.id + updated_run.files = [new_file] + + class _Q: + def filter(self, *_a, **_k): + return self + + def first(self): + return stage_run + + monkeypatch.setattr( + assemblies.assembly_stage_run_service, + "update_with_files", + lambda db, db_obj, update_in: updated_run, + ) + + app.dependency_overrides[assemblies.get_current_active_user] = lambda: SimpleNamespace( + is_active=True, roles=["curator"], id=uuid4(), is_superuser=False + ) + + class _DB: + def query(self, _m): + return _Q() + + app.dependency_overrides[assemblies.get_db] = lambda: _DB() + + resp = client.patch( + f"/api/v1/assemblies/{assembly_id}/stage-runs/{stage_run.id}", + json={ + "files": [ + { + "storage_type": "gcs", + "storage_uri": "gs://bucket/new-key", + "storage_details": {}, + "sha256sum": "cafebabe", + } + ] + }, + ) + + assert resp.status_code == 200 + body = resp.json() + assert len(body["files"]) == 1 + assert body["files"][0]["sha256sum"] == "cafebabe" + assert body["files"][0]["storage_type"] == "gcs" diff --git a/tests/unit/endpoints/test_endpoints_samples.py b/tests/unit/endpoints/test_endpoints_samples.py index 2805fb6..c5db9cf 100644 --- a/tests/unit/endpoints/test_endpoints_samples.py +++ b/tests/unit/endpoints/test_endpoints_samples.py @@ -1,5 +1,6 @@ import uuid from types import SimpleNamespace +from urllib.parse import quote from fastapi.testclient import TestClient @@ -34,3 +35,474 @@ def test_sample_not_found(): resp = client.get(f"/api/v1/samples/{uuid.uuid4()}") assert resp.status_code == 404 + + +def test_get_samples_experiments_and_reads_for_specimen(): + client = TestClient(app) + + taxon_id = 172942 + specimen_id = "SPEC-001" + specimen_sample_id = uuid.uuid4() + derived_sample_id = uuid.uuid4() + experiment_1_id = uuid.uuid4() + experiment_2_id = uuid.uuid4() + read_1_id = uuid.uuid4() + read_2_id = uuid.uuid4() + + organism = SimpleNamespace(taxon_id=taxon_id, scientific_name="Test species") + specimen_sample = SimpleNamespace( + id=specimen_sample_id, + taxon_id=taxon_id, + specimen_id=specimen_id, + kind="specimen", + bpa_sample_id=None, + specimen_id_description=None, + identified_by=None, + specimen_custodian=None, + sample_custodian=None, + lifestage="adult", + sex="female", + organism_part="whole organism", + region_and_locality="region", + state_or_region=None, + country_or_sea="Australia", + indigenous_location=None, + latitude=None, + longitude=None, + elevation=None, + depth=None, + habitat="forest", + collection_method=None, + collection_date=None, + collected_by="collector", + collecting_institution="institution", + collection_permit=None, + data_context=None, + bioplatforms_project_id=None, + title=None, + sample_same_as=None, + sample_derived_from=None, + specimen_voucher=None, + tolid=None, + preservation_method=None, + preservation_temperature=None, + project_name=None, + biosample_accession=None, + derived_from_sample_id=None, + extensions=None, + created_at="2026-01-01T00:00:00Z", + updated_at="2026-01-01T00:00:00Z", + ) + derived_sample = SimpleNamespace( + id=derived_sample_id, + taxon_id=taxon_id, + specimen_id=specimen_id, + kind="derived", + bpa_sample_id="BPA-001", + specimen_id_description=None, + identified_by=None, + specimen_custodian=None, + sample_custodian=None, + lifestage="adult", + sex="female", + organism_part="tissue", + region_and_locality="region", + state_or_region=None, + country_or_sea="Australia", + indigenous_location=None, + latitude=None, + longitude=None, + elevation=None, + depth=None, + habitat="forest", + collection_method=None, + collection_date=None, + collected_by="collector", + collecting_institution="institution", + collection_permit=None, + data_context=None, + bioplatforms_project_id=None, + title=None, + sample_same_as=None, + sample_derived_from=None, + specimen_voucher=None, + tolid=None, + preservation_method=None, + preservation_temperature=None, + project_name=None, + biosample_accession=None, + derived_from_sample_id=specimen_sample_id, + extensions=None, + created_at="2026-01-01T00:00:00Z", + updated_at="2026-01-01T00:00:00Z", + ) + experiment_1 = SimpleNamespace( + id=experiment_1_id, + sample_id=specimen_sample_id, + project_id=uuid.uuid4(), + bpa_package_id="PKG-001", + bioplatforms_base_url="https://example.com/pkg-001", + design_description="long-read library", + bpa_library_id="LIB-001", + library_strategy="WGS", + library_source="GENOMIC", + insert_size=None, + library_construction_protocol="protocol", + library_selection="size selected", + library_layout="SINGLE", + instrument_model="Sequel II", + platform="PACBIO_SMRT", + material_extracted_by="lab-a", + library_prepared_by="lab-b", + sequencing_kit="kit-a", + flowcell_type="flowcell-a", + base_caller_model=None, + data_owner="owner-a", + project_collaborators="collab-a", + extraction_method="method-a", + nucleic_acid_treatment=None, + extraction_protocol_doi=None, + nucleic_acid_conc="10", + nucleic_acid_volume="5", + gal=None, + raw_data_release_date="2026-01-10", + created_at="2026-01-01T00:00:00Z", + updated_at="2026-01-01T00:00:00Z", + ) + experiment_2 = SimpleNamespace( + id=experiment_2_id, + sample_id=derived_sample_id, + project_id=uuid.uuid4(), + bpa_package_id="PKG-002", + bioplatforms_base_url="https://example.com/pkg-002", + design_description="derived library", + bpa_library_id="LIB-002", + library_strategy="Hi-C", + library_source="GENOMIC", + insert_size=None, + library_construction_protocol="protocol-2", + library_selection="selection-2", + library_layout="PAIRED", + instrument_model="NovaSeq", + platform="ILLUMINA", + material_extracted_by="lab-c", + library_prepared_by="lab-d", + sequencing_kit="kit-b", + flowcell_type="flowcell-b", + base_caller_model=None, + data_owner="owner-b", + project_collaborators="collab-b", + extraction_method="method-b", + nucleic_acid_treatment=None, + extraction_protocol_doi=None, + nucleic_acid_conc="20", + nucleic_acid_volume="10", + gal=None, + raw_data_release_date="2026-02-10", + created_at="2026-01-01T00:00:00Z", + updated_at="2026-01-01T00:00:00Z", + ) + read_1 = SimpleNamespace( + id=read_1_id, + experiment_id=experiment_1_id, + bpa_resource_id="RES-001", + bpa_dataset_id="DATASET-001", + file_name="reads1.ccs.bam", + file_checksum="md5-1", + file_format="bam", + optional_file=False, + bioplatforms_url="https://example.com/reads1.ccs.bam", + read_number=None, + lane_number=None, + created_at="2026-01-01T00:00:00Z", + updated_at="2026-01-01T00:00:00Z", + ) + read_2 = SimpleNamespace( + id=read_2_id, + experiment_id=experiment_2_id, + bpa_resource_id="RES-002", + bpa_dataset_id="DATASET-002", + file_name="reads2_R1.fastq.gz", + file_checksum="md5-2", + file_format="fastq.gz", + optional_file=True, + bioplatforms_url="https://example.com/reads2_R1.fastq.gz", + read_number="1", + lane_number="L001", + created_at="2026-01-01T00:00:00Z", + updated_at="2026-01-01T00:00:00Z", + ) + + class _Q: + def __init__(self, value): + self.value = value + + def filter(self, *_a, **_k): + return self + + def first(self): + return self.value if not isinstance(self.value, list) else None + + def all(self): + return self.value if isinstance(self.value, list) else [] + + class _DB: + def __init__(self): + self.calls = 0 + + def query(self, _model): + self.calls += 1 + if self.calls == 1: + return _Q(organism) + if self.calls == 2: + return _Q(specimen_sample) + if self.calls == 3: + return _Q([specimen_sample, derived_sample]) + if self.calls == 4: + return _Q([experiment_1]) + if self.calls == 5: + return _Q([read_1]) + if self.calls == 6: + return _Q([experiment_2]) + if self.calls == 7: + return _Q([read_2]) + return _Q([]) + + app.dependency_overrides[samples.get_current_active_user] = lambda: SimpleNamespace( + is_active=True, roles=["admin"], is_superuser=False + ) + app.dependency_overrides[samples.get_db] = _override_db(_DB()) + + resp = client.get(f"/api/v1/samples/by-specimen/{taxon_id}/{specimen_id}/related") + + assert resp.status_code == 200 + body = resp.json() + assert body["taxon_id"] == taxon_id + assert body["specimen_id"] == specimen_id + assert len(body["samples"]) == 2 + assert body["samples"][0]["sample"]["id"] == str(specimen_sample_id) + assert body["samples"][0]["experiments"][0]["experiment"]["id"] == str(experiment_1_id) + assert body["samples"][0]["experiments"][0]["experiment"]["platform"] == "PACBIO_SMRT" + assert body["samples"][0]["experiments"][0]["reads"][0]["id"] == str(read_1_id) + assert body["samples"][0]["experiments"][0]["reads"][0]["file_name"] == "reads1.ccs.bam" + assert body["samples"][1]["sample"]["id"] == str(derived_sample_id) + assert body["samples"][1]["experiments"][0]["experiment"]["id"] == str(experiment_2_id) + assert body["samples"][1]["experiments"][0]["experiment"]["library_layout"] == "PAIRED" + assert body["samples"][1]["experiments"][0]["reads"][0]["id"] == str(read_2_id) + assert body["samples"][1]["experiments"][0]["reads"][0]["lane_number"] == "L001" + + +def test_get_samples_experiments_and_reads_for_specimen_not_found(): + client = TestClient(app) + + taxon_id = 172942 + specimen_id = "SPEC-404" + organism = SimpleNamespace(taxon_id=taxon_id, scientific_name="Test species") + + class _Q: + def __init__(self, value): + self.value = value + + def filter(self, *_a, **_k): + return self + + def first(self): + return self.value + + class _DB: + def __init__(self): + self.calls = 0 + + def query(self, _model): + self.calls += 1 + if self.calls == 1: + return _Q(organism) + return _Q(None) + + app.dependency_overrides[samples.get_current_active_user] = lambda: SimpleNamespace( + is_active=True, roles=["admin"], is_superuser=False + ) + app.dependency_overrides[samples.get_db] = _override_db(_DB()) + + resp = client.get(f"/api/v1/samples/by-specimen/{taxon_id}/{specimen_id}/related") + assert resp.status_code == 404 + + +def test_get_specimen_by_taxid_and_specimen_id_accepts_encoded_path_value(): + client = TestClient(app) + + taxon_id = 172942 + specimen_id = "SPEC / 001" + encoded_specimen_id = quote(specimen_id, safe="") + specimen_sample_id = uuid.uuid4() + + organism = SimpleNamespace(taxon_id=taxon_id, scientific_name="Test species") + specimen_sample = SimpleNamespace( + id=specimen_sample_id, + taxon_id=taxon_id, + specimen_id=specimen_id, + kind="specimen", + bpa_sample_id=None, + specimen_id_description=None, + identified_by=None, + specimen_custodian=None, + sample_custodian=None, + lifestage="adult", + sex="female", + organism_part="whole organism", + region_and_locality="region", + state_or_region=None, + country_or_sea="Australia", + indigenous_location=None, + latitude=None, + longitude=None, + elevation=None, + depth=None, + habitat="forest", + collection_method=None, + collection_date=None, + collected_by="collector", + collecting_institution="institution", + collection_permit=None, + data_context=None, + bioplatforms_project_id=None, + title=None, + sample_same_as=None, + sample_derived_from=None, + specimen_voucher=None, + tolid=None, + preservation_method=None, + preservation_temperature=None, + project_name=None, + biosample_accession=None, + derived_from_sample_id=None, + extensions=None, + created_at="2026-01-01T00:00:00Z", + updated_at="2026-01-01T00:00:00Z", + ) + + class _Q: + def __init__(self, value): + self.value = value + + def filter(self, *_a, **_k): + return self + + def first(self): + return self.value + + class _DB: + def __init__(self): + self.calls = 0 + + def query(self, _model): + self.calls += 1 + if self.calls == 1: + return _Q(organism) + return _Q(specimen_sample) + + app.dependency_overrides[samples.get_current_active_user] = lambda: SimpleNamespace( + is_active=True, roles=["admin"], is_superuser=False + ) + app.dependency_overrides[samples.get_db] = _override_db(_DB()) + + resp = client.get(f"/api/v1/samples/by-specimen/{taxon_id}/{encoded_specimen_id}") + + assert resp.status_code == 200 + body = resp.json() + assert body["specimen_id"] == specimen_id + + +def test_get_samples_experiments_and_reads_for_specimen_accepts_encoded_path_value(): + client = TestClient(app) + + taxon_id = 172942 + specimen_id = "SPEC / 001" + encoded_specimen_id = quote(specimen_id, safe="") + specimen_sample_id = uuid.uuid4() + + organism = SimpleNamespace(taxon_id=taxon_id, scientific_name="Test species") + specimen_sample = SimpleNamespace( + id=specimen_sample_id, + taxon_id=taxon_id, + specimen_id=specimen_id, + kind="specimen", + bpa_sample_id=None, + specimen_id_description=None, + identified_by=None, + specimen_custodian=None, + sample_custodian=None, + lifestage="adult", + sex="female", + organism_part="whole organism", + region_and_locality="region", + state_or_region=None, + country_or_sea="Australia", + indigenous_location=None, + latitude=None, + longitude=None, + elevation=None, + depth=None, + habitat="forest", + collection_method=None, + collection_date=None, + collected_by="collector", + collecting_institution="institution", + collection_permit=None, + data_context=None, + bioplatforms_project_id=None, + title=None, + sample_same_as=None, + sample_derived_from=None, + specimen_voucher=None, + tolid=None, + preservation_method=None, + preservation_temperature=None, + project_name=None, + biosample_accession=None, + derived_from_sample_id=None, + extensions=None, + created_at="2026-01-01T00:00:00Z", + updated_at="2026-01-01T00:00:00Z", + ) + + class _Q: + def __init__(self, value): + self.value = value + + def filter(self, *_a, **_k): + return self + + def first(self): + return self.value if not isinstance(self.value, list) else None + + def all(self): + return self.value if isinstance(self.value, list) else [] + + class _DB: + def __init__(self): + self.calls = 0 + + def query(self, _model): + self.calls += 1 + if self.calls == 1: + return _Q(organism) + if self.calls == 2: + return _Q(specimen_sample) + if self.calls == 3: + return _Q([specimen_sample]) + if self.calls == 4: + return _Q([]) + return _Q([]) + + app.dependency_overrides[samples.get_current_active_user] = lambda: SimpleNamespace( + is_active=True, roles=["admin"], is_superuser=False + ) + app.dependency_overrides[samples.get_db] = _override_db(_DB()) + + resp = client.get(f"/api/v1/samples/by-specimen/{taxon_id}/{encoded_specimen_id}/related") + + assert resp.status_code == 200 + body = resp.json() + assert body["specimen_id"] == specimen_id + assert body["samples"][0]["sample"]["specimen_id"] == specimen_id diff --git a/tests/unit/endpoints/test_endpoints_taxonomy_info.py b/tests/unit/endpoints/test_endpoints_taxonomy_info.py index 7971f48..afef115 100644 --- a/tests/unit/endpoints/test_endpoints_taxonomy_info.py +++ b/tests/unit/endpoints/test_endpoints_taxonomy_info.py @@ -152,7 +152,9 @@ def test_create_taxonomy_info_success(monkeypatch): "taxonomy_info_service", SimpleNamespace(create=lambda db, ti_in: ti), ) - resp = client.post("/api/v1/taxonomy-info/", json={"taxon_id": 5077, "augustus_dataset_name": "anidulans"}) + resp = client.post( + "/api/v1/taxonomy-info/", json={"taxon_id": 5077, "augustus_dataset_name": "anidulans"} + ) assert resp.status_code == 201 assert resp.json()["taxon_id"] == 5077 @@ -352,9 +354,7 @@ def test_bulk_import_rejects_mismatched_inner_taxon_id(monkeypatch): "taxonomy_info_service", SimpleNamespace(bulk_import=lambda db, data: result), ) - resp = client.post( - "/api/v1/taxonomy-info/bulk-import", json={"5077": {"taxon_id": 9999}} - ) + resp = client.post("/api/v1/taxonomy-info/bulk-import", json={"5077": {"taxon_id": 9999}}) assert resp.status_code == 200 body = resp.json() assert body["skipped_count"] == 1 diff --git a/tests/unit/services/test_assembly_helper.py b/tests/unit/services/test_assembly_helper.py index 1e89408..4ef56bc 100644 --- a/tests/unit/services/test_assembly_helper.py +++ b/tests/unit/services/test_assembly_helper.py @@ -1,9 +1,9 @@ """Tests for assembly helper functions.""" from unittest.mock import Mock +from uuid import UUID import pytest -import yaml from app.models.experiment import Experiment from app.models.organism import Organism @@ -11,139 +11,142 @@ from app.schemas.assembly import AssemblyDataTypes from app.services.assembly_helper import ( determine_assembly_data_types, - generate_assembly_manifest, + generate_assembly_manifest_json, + get_available_assembly_data_types, get_detected_platforms, ) +LONG_READ_SAMPLE_ID = UUID("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa") +HIC_SAMPLE_ID = UUID("bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb") +LONG_READ_SAMPLE_STR = str(LONG_READ_SAMPLE_ID) +HIC_SAMPLE_STR = str(HIC_SAMPLE_ID) + class TestDetermineAssemblyDataTypes: """Tests for determine_assembly_data_types function.""" def test_pacbio_only(self): - """Test detection of PacBio SMRT only.""" - experiments = [ - Mock(platform="PACBIO_SMRT", library_strategy="WGS"), - ] - result = determine_assembly_data_types(experiments) - assert result == AssemblyDataTypes.PACBIO_SMRT + experiments = [Mock(platform="PACBIO_SMRT", library_strategy="WGS")] + assert determine_assembly_data_types(experiments) == AssemblyDataTypes.PACBIO_SMRT def test_oxford_nanopore_only(self): - """Test detection of Oxford Nanopore only.""" - experiments = [ - Mock(platform="OXFORD_NANOPORE", library_strategy="WGS"), - ] - result = determine_assembly_data_types(experiments) - assert result == AssemblyDataTypes.OXFORD_NANOPORE + experiments = [Mock(platform="OXFORD_NANOPORE", library_strategy="WGS")] + assert determine_assembly_data_types(experiments) == AssemblyDataTypes.OXFORD_NANOPORE - def test_illumina_hic(self): - """Test that Illumina-only raises error (no long-read platform).""" - experiments = [ - Mock(platform="ILLUMINA", library_strategy="Hi-C"), - ] + def test_illumina_hic_alone_raises_error(self): + experiments = [Mock(platform="ILLUMINA", library_strategy="Hi-C")] with pytest.raises(ValueError, match="No valid data types detected in experiments"): determine_assembly_data_types(experiments) - def test_illumina_wgs_treated_as_hic(self): - """Test that ILLUMINA + WGS is treated as Hi-C.""" + def test_illumina_wgs_not_treated_as_hic(self): experiments = [ Mock(platform="PACBIO_SMRT", library_strategy="WGS"), Mock(platform="ILLUMINA", library_strategy="WGS"), ] - result = determine_assembly_data_types(experiments) - assert result == AssemblyDataTypes.PACBIO_SMRT_HIC + assert determine_assembly_data_types(experiments) == AssemblyDataTypes.PACBIO_SMRT def test_pacbio_and_hic(self): - """Test detection of PacBio + Hi-C combination.""" experiments = [ Mock(platform="PACBIO_SMRT", library_strategy="WGS"), Mock(platform="ILLUMINA", library_strategy="Hi-C"), ] - result = determine_assembly_data_types(experiments) - assert result == AssemblyDataTypes.PACBIO_SMRT_HIC + assert determine_assembly_data_types(experiments) == AssemblyDataTypes.PACBIO_SMRT_HIC def test_nanopore_and_hic(self): - """Test detection of Oxford Nanopore + Hi-C combination.""" experiments = [ Mock(platform="OXFORD_NANOPORE", library_strategy="WGS"), - Mock(platform="ILLUMINA", library_strategy="WGS"), + Mock(platform="ILLUMINA", library_strategy="Hi-C"), ] - result = determine_assembly_data_types(experiments) - assert result == AssemblyDataTypes.OXFORD_NANOPORE_HIC + assert determine_assembly_data_types(experiments) == AssemblyDataTypes.OXFORD_NANOPORE_HIC def test_pacbio_and_nanopore(self): - """Test detection of PacBio + Oxford Nanopore combination.""" experiments = [ Mock(platform="PACBIO_SMRT", library_strategy="WGS"), Mock(platform="OXFORD_NANOPORE", library_strategy="WGS"), ] - result = determine_assembly_data_types(experiments) - assert result == AssemblyDataTypes.PACBIO_SMRT_OXFORD_NANOPORE + assert ( + determine_assembly_data_types(experiments) + == AssemblyDataTypes.PACBIO_SMRT_OXFORD_NANOPORE + ) def test_all_three_platforms(self): - """Test detection of all three platform types.""" experiments = [ Mock(platform="PACBIO_SMRT", library_strategy="WGS"), Mock(platform="OXFORD_NANOPORE", library_strategy="WGS"), Mock(platform="ILLUMINA", library_strategy="Hi-C"), ] - result = determine_assembly_data_types(experiments) - assert result == AssemblyDataTypes.PACBIO_SMRT_OXFORD_NANOPORE_HIC + assert ( + determine_assembly_data_types(experiments) + == AssemblyDataTypes.PACBIO_SMRT_OXFORD_NANOPORE_HIC + ) def test_case_insensitive_platform(self): - """Test that platform detection is case-insensitive.""" - experiments = [ - Mock(platform="pacbio_smrt", library_strategy="WGS"), - ] - result = determine_assembly_data_types(experiments) - assert result == AssemblyDataTypes.PACBIO_SMRT + experiments = [Mock(platform="pacbio_smrt", library_strategy="WGS")] + assert determine_assembly_data_types(experiments) == AssemblyDataTypes.PACBIO_SMRT def test_case_insensitive_library_strategy(self): - """Test that library strategy detection is case-insensitive.""" experiments = [ Mock(platform="OXFORD_NANOPORE", library_strategy="WGS"), - Mock(platform="ILLUMINA", library_strategy="wgs"), + Mock(platform="ILLUMINA", library_strategy="hi-c"), ] - result = determine_assembly_data_types(experiments) - assert result == AssemblyDataTypes.OXFORD_NANOPORE_HIC + assert determine_assembly_data_types(experiments) == AssemblyDataTypes.OXFORD_NANOPORE_HIC def test_no_valid_platforms_raises_error(self): - """Test that no valid platforms raises ValueError.""" - experiments = [ - Mock(platform="UNKNOWN", library_strategy="WGS"), - ] + experiments = [Mock(platform="UNKNOWN", library_strategy="WGS")] with pytest.raises(ValueError, match="No valid data types detected in experiments"): determine_assembly_data_types(experiments) def test_empty_experiments_raises_error(self): - """Test that empty experiments list raises ValueError.""" with pytest.raises(ValueError, match="No valid data types detected in experiments"): determine_assembly_data_types([]) def test_none_platform_ignored(self): - """Test that experiments with None platform are ignored.""" experiments = [ Mock(platform=None, library_strategy="WGS"), Mock(platform="PACBIO_SMRT", library_strategy="WGS"), ] - result = determine_assembly_data_types(experiments) - assert result == AssemblyDataTypes.PACBIO_SMRT + assert determine_assembly_data_types(experiments) == AssemblyDataTypes.PACBIO_SMRT + + +class TestGetAvailableAssemblyDataTypes: + """Tests for discovery-oriented assembly data type detection.""" + + def test_hic_only_is_reported_for_discovery(self): + experiments = [Mock(platform="ILLUMINA", library_strategy="Hi-C")] + assert get_available_assembly_data_types(experiments) == ["Hi-C"] + + def test_returns_atomic_data_types_in_stable_order(self): + experiments = [ + Mock(platform="ILLUMINA", library_strategy="Hi-C"), + Mock(platform="OXFORD_NANOPORE", library_strategy="WGS"), + Mock(platform="PACBIO_SMRT", library_strategy="WGA"), + ] + assert get_available_assembly_data_types(experiments) == [ + "PACBIO_SMRT", + "OXFORD_NANOPORE", + "Hi-C", + ] + + def test_ignores_non_assembly_illumina_wgs(self): + experiments = [Mock(platform="ILLUMINA", library_strategy="WGS")] + assert get_available_assembly_data_types(experiments) == [] + + def test_rnaseq_is_reported_for_discovery(self): + experiments = [Mock(platform="ILLUMINA", library_strategy="RNA-Seq")] + assert get_available_assembly_data_types(experiments) == ["RNA-Seq"] class TestGetDetectedPlatforms: """Tests for get_detected_platforms function.""" def test_single_platform(self): - """Test detection of single platform.""" - experiments = [ - Mock(platform="PACBIO_SMRT", library_strategy="WGS"), - ] + experiments = [Mock(platform="PACBIO_SMRT", library_strategy="WGS")] result = get_detected_platforms(experiments) assert result["platforms"] == ["PACBIO_SMRT"] assert result["library_strategies"] == ["WGS"] assert result["experiment_count"] == 1 def test_multiple_platforms(self): - """Test detection of multiple platforms.""" experiments = [ Mock(platform="PACBIO_SMRT", library_strategy="WGS"), Mock(platform="ILLUMINA", library_strategy="Hi-C"), @@ -154,15 +157,22 @@ def test_multiple_platforms(self): assert result["experiment_count"] == 2 def test_empty_experiments(self): - """Test with empty experiments list.""" result = get_detected_platforms([]) assert result["platforms"] == [] assert result["library_strategies"] == [] assert result["experiment_count"] == 0 +# ────────────────────────────────────────────────────────────────────────────── +# Helpers for building mock experiments / reads +# ────────────────────────────────────────────────────────────────────────────── + + def _make_pacbio_experiment( - exp_id="exp1", bpa_package_id="pkg-001", sample_id="sample-uuid-1", bioplatforms_base_url=None + exp_id="exp1", + bpa_package_id="pkg-001", + sample_id=LONG_READ_SAMPLE_STR, + bioplatforms_base_url=None, ): return Mock( id=exp_id, @@ -174,8 +184,27 @@ def _make_pacbio_experiment( ) +def _make_ont_experiment( + exp_id="exp-ont", + bpa_package_id="pkg-ont", + sample_id=LONG_READ_SAMPLE_STR, + bioplatforms_base_url=None, +): + return Mock( + id=exp_id, + platform="OXFORD_NANOPORE", + library_strategy="WGS", + bpa_package_id=bpa_package_id, + bioplatforms_base_url=bioplatforms_base_url, + sample_id=sample_id, + ) + + def _make_hic_experiment( - exp_id="exp2", bpa_package_id="pkg-002", sample_id="sample-uuid-2", bioplatforms_base_url=None + exp_id="exp2", + bpa_package_id="pkg-002", + sample_id=HIC_SAMPLE_STR, + bioplatforms_base_url=None, ): return Mock( id=exp_id, @@ -187,11 +216,13 @@ def _make_hic_experiment( ) -class TestGenerateAssemblyManifest: - """Tests for generate_assembly_manifest function.""" +class TestGenerateAssemblyManifestJson: + """Tests for generate_assembly_manifest_json function.""" + + # ── PacBio ──────────────────────────────────────────────────────────────── def test_pacbio_reads_filtered_by_extension(self): - """Test that only .ccs.bam and hifi_reads.bam files are included for PacBio.""" + """Only .ccs.bam and hifi_reads.bam files are included for PacBio.""" organism = Mock(scientific_name="Test Species", taxon_id=12345) experiments = [_make_pacbio_experiment()] reads = [ @@ -224,10 +255,11 @@ def test_pacbio_reads_filtered_by_extension(self): ), ] - result = generate_assembly_manifest(organism, reads, experiments, "tol1", 1) - data = yaml.safe_load(result) + result = generate_assembly_manifest_json( + organism, reads, experiments, "tol1", 1, LONG_READ_SAMPLE_ID + ) - pacbio = data["reads"]["PACBIO_SMRT"] + pacbio = result["reads"]["PACBIO_SMRT"] assert "pkg-001" in pacbio resources = pacbio["pkg-001"]["resources"] urls = [r["url"] for r in resources] @@ -235,10 +267,67 @@ def test_pacbio_reads_filtered_by_extension(self): assert "https://example.com/2" in urls assert "https://example.com/3" not in urls assert all("md5sum" in r for r in resources) - assert all("file_name" not in r for r in resources) + + def test_pacbio_read_without_file_name_skipped(self): + """PacBio reads with no file_name are silently skipped.""" + organism = Mock(scientific_name="Test Species", taxon_id=12345) + experiments = [_make_pacbio_experiment()] + reads = [ + Mock( + id="r1", + experiment_id="exp1", + file_name=None, + file_checksum="abc123", + bioplatforms_url="https://example.com/1", + read_number=None, + lane_number=None, + ), + ] + + result = generate_assembly_manifest_json( + organism, reads, experiments, "tol1", 1, LONG_READ_SAMPLE_ID + ) + assert result["reads"] == {} + + # ── Oxford Nanopore ─────────────────────────────────────────────────────── + + def test_ont_reads_included_without_extension_filter(self): + """All ONT reads are included regardless of file extension.""" + organism = Mock(scientific_name="Test Species", taxon_id=12345) + experiments = [_make_ont_experiment()] + reads = [ + Mock( + id="r1", + experiment_id="exp-ont", + file_name="sample.fastq.gz", + file_checksum="abc123", + bioplatforms_url="https://example.com/ont1", + read_number=None, + lane_number=None, + ), + Mock( + id="r2", + experiment_id="exp-ont", + file_name="sample.pod5", + file_checksum="def456", + bioplatforms_url="https://example.com/ont2", + read_number=None, + lane_number=None, + ), + ] + + result = generate_assembly_manifest_json( + organism, reads, experiments, "tol1", 1, LONG_READ_SAMPLE_ID + ) + + assert "OXFORD_NANOPORE" in result["reads"] + resources = result["reads"]["OXFORD_NANOPORE"]["pkg-ont"]["resources"] + assert len(resources) == 2 + + # ── Hi-C ───────────────────────────────────────────────────────────────── def test_hic_reads_include_metadata(self): - """Test that Hi-C reads include lane_number and are split by r1/r2.""" + """Hi-C reads include lane_number and are split by r1/r2.""" organism = Mock(scientific_name="Test Species", taxon_id=12345) experiments = [_make_hic_experiment()] reads = [ @@ -253,10 +342,11 @@ def test_hic_reads_include_metadata(self): ), ] - result = generate_assembly_manifest(organism, reads, experiments, "tol1", 1) - data = yaml.safe_load(result) + result = generate_assembly_manifest_json( + organism, reads, experiments, "tol1", 1, LONG_READ_SAMPLE_ID, HIC_SAMPLE_ID + ) - hic = data["reads"]["Hi-C"] + hic = result["reads"]["Hi-C"] assert "pkg-002" in hic r1_resources = hic["pkg-002"]["resources"]["r1"] assert len(r1_resources) == 1 @@ -264,7 +354,7 @@ def test_hic_reads_include_metadata(self): assert r1_resources[0]["md5sum"] == "abc123" def test_hic_reads_split_into_r1_r2(self): - """Test that Hi-C reads with read_number 1 and 2 are split into r1 and r2.""" + """Hi-C reads with read_number 1 and 2 are split into r1 and r2.""" organism = Mock(scientific_name="Test Species", taxon_id=12345) experiments = [_make_hic_experiment()] reads = [ @@ -288,17 +378,18 @@ def test_hic_reads_split_into_r1_r2(self): ), ] - result = generate_assembly_manifest(organism, reads, experiments, "tol1", 1) - data = yaml.safe_load(result) + result = generate_assembly_manifest_json( + organism, reads, experiments, "tol1", 1, LONG_READ_SAMPLE_ID, HIC_SAMPLE_ID + ) - resources = data["reads"]["Hi-C"]["pkg-002"]["resources"] + resources = result["reads"]["Hi-C"]["pkg-002"]["resources"] assert len(resources["r1"]) == 1 assert len(resources["r2"]) == 1 assert resources["r1"][0]["url"] == "https://example.com/r1" assert resources["r2"][0]["url"] == "https://example.com/r2" - def test_wgs_treated_as_hic(self): - """Test that ILLUMINA + WGS is treated as Hi-C.""" + def test_wgs_not_treated_as_hic(self): + """ILLUMINA + WGS does not populate the Hi-C section.""" organism = Mock(scientific_name="Test Species", taxon_id=12345) experiments = [ Mock( @@ -307,7 +398,7 @@ def test_wgs_treated_as_hic(self): library_strategy="WGS", bpa_package_id="pkg-wgs", bioplatforms_base_url=None, - sample_id="s1", + sample_id=HIC_SAMPLE_STR, ) ] reads = [ @@ -322,113 +413,265 @@ def test_wgs_treated_as_hic(self): ), ] - result = generate_assembly_manifest(organism, reads, experiments, "tol1", 1) - data = yaml.safe_load(result) + result = generate_assembly_manifest_json( + organism, reads, experiments, "tol1", 1, LONG_READ_SAMPLE_ID, HIC_SAMPLE_ID + ) - assert "Hi-C" in data["reads"] - assert "pkg-wgs" in data["reads"]["Hi-C"] + assert "Hi-C" not in result["reads"] - def test_empty_reads_dict(self): - """Test that empty reads result in empty reads dict.""" + def test_hic_section_omitted_when_no_hic_sample_id(self): + """Hi-C section is absent when hic_sample_id is not supplied.""" organism = Mock(scientific_name="Test Species", taxon_id=12345) experiments = [ + _make_pacbio_experiment(), + _make_hic_experiment(), # hic experiment present but hic_sample_id not passed + ] + reads = [ Mock( - id="exp1", - platform="UNKNOWN", + id="r1", + experiment_id="exp1", + file_name="sample.ccs.bam", + file_checksum="abc123", + bioplatforms_url="https://example.com/1", + read_number=None, + lane_number=None, + ), + Mock( + id="r2", + experiment_id="exp2", + file_name="hic_R1.fastq.gz", + file_checksum="def456", + bioplatforms_url="https://example.com/2", + read_number="1", + lane_number="001", + ), + ] + + # hic_sample_id is None → Hi-C section must be absent + result = generate_assembly_manifest_json( + organism, reads, experiments, "tol1", 1, LONG_READ_SAMPLE_ID, hic_sample_id=None + ) + + assert "Hi-C" not in result["reads"] + assert "PACBIO_SMRT" in result["reads"] + + def test_reads_routed_by_specimen_sample_not_platform(self): + """Reads from long_read_sample_id are never placed in Hi-C even if ILLUMINA.""" + organism = Mock(scientific_name="Test Species", taxon_id=12345) + # An ILLUMINA experiment that belongs to the long_read_specimen_sample_id + experiments = [ + Mock( + id="exp-ill", + platform="ILLUMINA", library_strategy="WGS", - bpa_package_id="pkg-x", + bpa_package_id="pkg-ill", bioplatforms_base_url=None, - sample_id="s1", + sample_id=LONG_READ_SAMPLE_STR, # belongs to long-read sample ) ] - reads = [] + reads = [ + Mock( + id="r1", + experiment_id="exp-ill", + file_name="sample_R1.fastq.gz", + file_checksum="abc123", + bioplatforms_url="https://example.com/1", + read_number="1", + lane_number="001", + ), + ] - result = generate_assembly_manifest(organism, reads, experiments, "tol1", 1) + # Illumina experiment on long-read sample → should NOT appear in Hi-C section + result = generate_assembly_manifest_json( + organism, reads, experiments, "tol1", 1, LONG_READ_SAMPLE_ID, HIC_SAMPLE_ID + ) - assert "reads: {}" in result + assert result["reads"] == {} - def test_organism_metadata_included(self): - """Test that organism metadata is included in manifest.""" - organism = Mock(scientific_name="Saiphos equalis", taxon_id=172942) - experiments = [] - reads = [] + # ── Mixed long-read + Hi-C ──────────────────────────────────────────────── - result = generate_assembly_manifest(organism, reads, experiments, "tol123", 2) + def test_multiple_platform_types(self): + """Manifest with PacBio (long-read sample) and Hi-C (hic sample) packages.""" + organism = Mock(scientific_name="Test Species", taxon_id=12345) + experiments = [ + _make_pacbio_experiment(exp_id="exp1", bpa_package_id="pkg-pacbio"), + _make_hic_experiment(exp_id="exp2", bpa_package_id="pkg-hic"), + ] + reads = [ + Mock( + id="r1", + experiment_id="exp1", + file_name="sample.ccs.bam", + file_checksum="abc123", + bioplatforms_url="https://example.com/1", + read_number=None, + lane_number=None, + ), + Mock( + id="r2", + experiment_id="exp2", + file_name="hic_R1.fastq.gz", + file_checksum="def456", + bioplatforms_url="https://example.com/2", + read_number="1", + lane_number="001", + ), + ] - assert "scientific_name: Saiphos equalis" in result - assert "taxon_id: 172942" in result - assert "tolid: tol123" in result - assert "version: 2" in result + result = generate_assembly_manifest_json( + organism, reads, experiments, "tol1", 1, LONG_READ_SAMPLE_ID, HIC_SAMPLE_ID + ) - def test_sample_metadata_included_at_package_level(self): - """Test that sample metadata appears at the bpa_package_id level.""" - organism = Mock(scientific_name="Saiphos equalis", taxon_id=172942) + assert "PACBIO_SMRT" in result["reads"] + assert "Hi-C" in result["reads"] + assert "pkg-pacbio" in result["reads"]["PACBIO_SMRT"] + assert "pkg-hic" in result["reads"]["Hi-C"] + + def test_same_specimen_sample_can_supply_long_reads_and_hic(self): + """When the same specimen sample ID is used for both roles, both sections are populated.""" + organism = Mock(scientific_name="Test Species", taxon_id=12345) experiments = [ + _make_pacbio_experiment( + exp_id="exp-pb", + bpa_package_id="pkg-pacbio", + sample_id=LONG_READ_SAMPLE_STR, + ), + _make_hic_experiment( + exp_id="exp-hic", + bpa_package_id="pkg-hic", + sample_id=LONG_READ_SAMPLE_STR, + ), + ] + reads = [ Mock( - id="exp1", - sample_id="550e8400-e29b-41d4-a716-446655440000", - platform="PACBIO_SMRT", - library_strategy="WGS", - bpa_package_id="pkg-001", - bioplatforms_base_url=None, - ) + id="r1", + experiment_id="exp-pb", + file_name="sample.ccs.bam", + file_checksum="abc123", + bioplatforms_url="https://example.com/pb", + read_number=None, + lane_number=None, + ), + Mock( + id="r2", + experiment_id="exp-hic", + file_name="hic_R1.fastq.gz", + file_checksum="def456", + bioplatforms_url="https://example.com/hic", + read_number="1", + lane_number="001", + ), + ] + + result = generate_assembly_manifest_json( + organism, reads, experiments, "tol1", 1, LONG_READ_SAMPLE_ID, LONG_READ_SAMPLE_ID + ) + + assert "PACBIO_SMRT" in result["reads"] + assert "Hi-C" in result["reads"] + + def test_derived_sample_reads_are_attributed_to_parent_specimen(self): + """Reads from derived sequencing samples are routed to the selected parent specimen.""" + organism = Mock(scientific_name="Test Species", taxon_id=12345) + derived_sample_id = "cccccccc-cccc-cccc-cccc-cccccccccccc" + experiments = [ + _make_pacbio_experiment( + exp_id="exp-derived", + bpa_package_id="pkg-derived", + sample_id=derived_sample_id, + ), ] reads = [ Mock( id="r1", - experiment_id="exp1", + experiment_id="exp-derived", file_name="sample.ccs.bam", file_checksum="abc123", - bioplatforms_url="https://example.com/1", + bioplatforms_url="https://example.com/pb", read_number=None, lane_number=None, - ) + ), ] sample_metadata_by_id = { - "550e8400-e29b-41d4-a716-446655440000": { + LONG_READ_SAMPLE_STR: { "bpa_sample_id": "102.100.100/9000", "specimen_id": "SPEC-001", } } - result = generate_assembly_manifest( - organism, reads, experiments, "tol123", 2, sample_metadata_by_id + result = generate_assembly_manifest_json( + organism, + reads, + experiments, + "tol1", + 1, + LONG_READ_SAMPLE_ID, + sample_metadata_by_id=sample_metadata_by_id, + sequencing_sample_to_specimen_sample_id={ + derived_sample_id: LONG_READ_SAMPLE_STR, + }, ) - data = yaml.safe_load(result) - pkg = data["reads"]["PACBIO_SMRT"]["pkg-001"] - assert pkg["sample_id"] == "550e8400-e29b-41d4-a716-446655440000" - assert pkg["bpa_sample_id"] == "102.100.100/9000" + pkg = result["reads"]["PACBIO_SMRT"]["pkg-derived"] + assert pkg["sample_id"] == LONG_READ_SAMPLE_STR assert pkg["specimen_id"] == "SPEC-001" - def test_bioplatforms_base_url_included_when_set(self): - """Test that bioplatforms_base_url appears in the manifest when set on the experiment.""" + def test_pacbio_and_ont_in_same_long_read_sample(self): + """Both PacBio and ONT reads from the same long-read specimen appear in separate sections.""" organism = Mock(scientific_name="Test Species", taxon_id=12345) experiments = [ - _make_pacbio_experiment(bioplatforms_base_url="https://base.example.com/pkg-001") + _make_pacbio_experiment(exp_id="exp-pb", bpa_package_id="pkg-pb"), + _make_ont_experiment(exp_id="exp-ont", bpa_package_id="pkg-ont"), ] reads = [ Mock( id="r1", - experiment_id="exp1", + experiment_id="exp-pb", file_name="sample.ccs.bam", - file_checksum="abc123", - bioplatforms_url="https://example.com/1", + file_checksum="abc", + bioplatforms_url="https://example.com/pb", + read_number=None, + lane_number=None, + ), + Mock( + id="r2", + experiment_id="exp-ont", + file_name="sample.fastq.gz", + file_checksum="def", + bioplatforms_url="https://example.com/ont", read_number=None, lane_number=None, ), ] - result = generate_assembly_manifest(organism, reads, experiments, "tol1", 1) - data = yaml.safe_load(result) + result = generate_assembly_manifest_json( + organism, reads, experiments, "tol1", 1, LONG_READ_SAMPLE_ID + ) - pkg = data["reads"]["PACBIO_SMRT"]["pkg-001"] - assert pkg["bioplatforms_base_url"] == "https://base.example.com/pkg-001" + assert "PACBIO_SMRT" in result["reads"] + assert "OXFORD_NANOPORE" in result["reads"] - def test_bioplatforms_base_url_omitted_when_none(self): - """Test that bioplatforms_base_url is absent from the manifest when not set.""" + # ── Metadata ────────────────────────────────────────────────────────────── + + def test_empty_reads_dict(self): + """Empty reads result in an empty reads section.""" organism = Mock(scientific_name="Test Species", taxon_id=12345) - experiments = [_make_pacbio_experiment(bioplatforms_base_url=None)] + result = generate_assembly_manifest_json(organism, [], [], "tol1", 1, LONG_READ_SAMPLE_ID) + assert result["reads"] == {} + + def test_organism_metadata_included(self): + """Organism metadata is present in the manifest.""" + organism = Mock(scientific_name="Saiphos equalis", taxon_id=172942) + result = generate_assembly_manifest_json(organism, [], [], "tol123", 2, LONG_READ_SAMPLE_ID) + assert result["scientific_name"] == "Saiphos equalis" + assert result["taxon_id"] == 172942 + assert result["tolid"] == "tol123" + assert result["version"] == 2 + + def test_sample_metadata_included_at_package_level(self): + """Sample metadata (bpa_sample_id, specimen_id) appears at the package level.""" + organism = Mock(scientific_name="Saiphos equalis", taxon_id=172942) + experiments = [_make_pacbio_experiment()] reads = [ Mock( id="r1", @@ -438,23 +681,39 @@ def test_bioplatforms_base_url_omitted_when_none(self): bioplatforms_url="https://example.com/1", read_number=None, lane_number=None, - ), + ) ] + sample_metadata_by_id = { + LONG_READ_SAMPLE_STR: { + "bpa_sample_id": "102.100.100/9000", + "specimen_id": "SPEC-001", + } + } - result = generate_assembly_manifest(organism, reads, experiments, "tol1", 1) - data = yaml.safe_load(result) + result = generate_assembly_manifest_json( + organism, + reads, + experiments, + "tol123", + 2, + LONG_READ_SAMPLE_ID, + sample_metadata_by_id=sample_metadata_by_id, + ) - pkg = data["reads"]["PACBIO_SMRT"]["pkg-001"] - assert "bioplatforms_base_url" not in pkg + pkg = result["reads"]["PACBIO_SMRT"]["pkg-001"] + assert pkg["sample_id"] == LONG_READ_SAMPLE_STR + assert pkg["bpa_sample_id"] == "102.100.100/9000" + assert pkg["specimen_id"] == "SPEC-001" - def test_reads_without_experiment_id_skipped(self): - """Test that reads without experiment_id are skipped.""" + def test_bioplatforms_base_url_included_when_set(self): organism = Mock(scientific_name="Test Species", taxon_id=12345) - experiments = [_make_pacbio_experiment()] + experiments = [ + _make_pacbio_experiment(bioplatforms_base_url="https://base.example.com/pkg-001") + ] reads = [ Mock( id="r1", - experiment_id=None, + experiment_id="exp1", file_name="sample.ccs.bam", file_checksum="abc123", bioplatforms_url="https://example.com/1", @@ -463,17 +722,15 @@ def test_reads_without_experiment_id_skipped(self): ), ] - result = generate_assembly_manifest(organism, reads, experiments, "tol1", 1) - - assert "reads: {}" in result + result = generate_assembly_manifest_json( + organism, reads, experiments, "tol1", 1, LONG_READ_SAMPLE_ID + ) + pkg = result["reads"]["PACBIO_SMRT"]["pkg-001"] + assert pkg["bioplatforms_base_url"] == "https://base.example.com/pkg-001" - def test_multiple_platform_types(self): - """Test manifest with both PacBio and Hi-C packages.""" + def test_bioplatforms_base_url_omitted_when_none(self): organism = Mock(scientific_name="Test Species", taxon_id=12345) - experiments = [ - _make_pacbio_experiment(exp_id="exp1", bpa_package_id="pkg-pacbio"), - _make_hic_experiment(exp_id="exp2", bpa_package_id="pkg-hic"), - ] + experiments = [_make_pacbio_experiment(bioplatforms_base_url=None)] reads = [ Mock( id="r1", @@ -484,27 +741,36 @@ def test_multiple_platform_types(self): read_number=None, lane_number=None, ), + ] + + result = generate_assembly_manifest_json( + organism, reads, experiments, "tol1", 1, LONG_READ_SAMPLE_ID + ) + pkg = result["reads"]["PACBIO_SMRT"]["pkg-001"] + assert "bioplatforms_base_url" not in pkg + + def test_reads_without_experiment_id_skipped(self): + organism = Mock(scientific_name="Test Species", taxon_id=12345) + experiments = [_make_pacbio_experiment()] + reads = [ Mock( - id="r2", - experiment_id="exp2", - file_name="hic_R1.fastq.gz", - file_checksum="def456", - bioplatforms_url="https://example.com/2", - read_number="1", - lane_number="001", + id="r1", + experiment_id=None, + file_name="sample.ccs.bam", + file_checksum="abc123", + bioplatforms_url="https://example.com/1", + read_number=None, + lane_number=None, ), ] - result = generate_assembly_manifest(organism, reads, experiments, "tol1", 1) - data = yaml.safe_load(result) - - assert "PACBIO_SMRT" in data["reads"] - assert "Hi-C" in data["reads"] - assert "pkg-pacbio" in data["reads"]["PACBIO_SMRT"] - assert "pkg-hic" in data["reads"]["Hi-C"] + result = generate_assembly_manifest_json( + organism, reads, experiments, "tol1", 1, LONG_READ_SAMPLE_ID + ) + assert result["reads"] == {} def test_multiple_reads_same_package(self): - """Test that multiple reads under the same experiment are grouped together.""" + """Multiple reads under the same experiment are grouped together.""" organism = Mock(scientific_name="Test Species", taxon_id=12345) experiments = [_make_pacbio_experiment()] reads = [ @@ -528,8 +794,15 @@ def test_multiple_reads_same_package(self): ), ] - result = generate_assembly_manifest(organism, reads, experiments, "tol1", 1) - data = yaml.safe_load(result) - - resources = data["reads"]["PACBIO_SMRT"]["pkg-001"]["resources"] + result = generate_assembly_manifest_json( + organism, reads, experiments, "tol1", 1, LONG_READ_SAMPLE_ID + ) + resources = result["reads"]["PACBIO_SMRT"]["pkg-001"]["resources"] assert len(resources) == 2 + + def test_manifest_is_dict_not_string(self): + """generate_assembly_manifest_json returns a dict, not a YAML/JSON string.""" + organism = Mock(scientific_name="Test Species", taxon_id=12345) + result = generate_assembly_manifest_json(organism, [], [], "tol1", 1, LONG_READ_SAMPLE_ID) + assert isinstance(result, dict) + assert "reads" in result diff --git a/tests/unit/services/test_assembly_service.py b/tests/unit/services/test_assembly_service.py index e2fc1bc..7d55de0 100644 --- a/tests/unit/services/test_assembly_service.py +++ b/tests/unit/services/test_assembly_service.py @@ -13,25 +13,25 @@ from app.schemas.assembly import AssemblyCreate, AssemblyCreateFromExperiments, AssemblyDataTypes from app.services.assembly_service import AssemblyService +LONG_READ_SAMPLE_ID = uuid.uuid4() +HIC_SAMPLE_ID = uuid.uuid4() + @pytest.fixture def mock_db(): - """Create a mock database session.""" return MagicMock(spec=Session) @pytest.fixture def assembly_service(): - """Create an AssemblyService instance.""" return AssemblyService(Assembly) @pytest.fixture def sample_assembly_create(): - """Create a sample AssemblyCreate schema.""" return AssemblyCreate( taxon_id=172942, - sample_id=uuid.uuid4(), + sample_id=LONG_READ_SAMPLE_ID, project_id=uuid.uuid4(), assembly_name="Test Assembly", assembly_type="clone or isolate", @@ -44,57 +44,39 @@ def sample_assembly_create(): class TestCreateAssembly: - """Tests for create method with auto-increment versioning.""" + """Tests for create method with auto-increment versioning (generic flow).""" def test_create_first_version(self, mock_db, assembly_service, sample_assembly_create): - """Test creating first version (version 1).""" - # Mock query to return None (no existing versions) mock_query = Mock() mock_query.filter.return_value.scalar.return_value = None mock_db.query.return_value = mock_query - - # Mock the assembly object that will be created - created_assembly = Assembly( - id=uuid.uuid4(), - taxon_id=sample_assembly_create.taxon_id, - sample_id=sample_assembly_create.sample_id, - data_types=sample_assembly_create.data_types, - version=1, - ) mock_db.add = Mock() mock_db.commit = Mock() mock_db.refresh = Mock() with patch.object(Assembly, "__init__", return_value=None): - result = assembly_service.create(mock_db, obj_in=sample_assembly_create) + assembly_service.create(mock_db, obj_in=sample_assembly_create) - # Verify version was set to 1 mock_db.add.assert_called_once() mock_db.commit.assert_called_once() def test_create_increments_version(self, mock_db, assembly_service, sample_assembly_create): - """Test that version increments from existing max version.""" - # Mock query to return existing max version of 3 mock_query = Mock() mock_query.filter.return_value.scalar.return_value = 3 mock_db.query.return_value = mock_query - mock_db.add = Mock() mock_db.commit = Mock() mock_db.refresh = Mock() with patch.object(Assembly, "__init__", return_value=None): - result = assembly_service.create(mock_db, obj_in=sample_assembly_create) + assembly_service.create(mock_db, obj_in=sample_assembly_create) - # Verify version was incremented to 4 mock_db.add.assert_called_once() mock_db.commit.assert_called_once() def test_create_version_per_combination(self, mock_db, assembly_service): - """Test that version is per (data_types, taxon_id, sample_id) combination.""" sample_id = uuid.uuid4() - # Create two assemblies with different data_types assembly1 = AssemblyCreate( taxon_id=172942, sample_id=sample_id, @@ -106,7 +88,6 @@ def test_create_version_per_combination(self, mock_db, assembly_service): program="hifiasm", moleculetype="genomic DNA", ) - assembly2 = AssemblyCreate( taxon_id=172942, sample_id=sample_id, @@ -119,7 +100,6 @@ def test_create_version_per_combination(self, mock_db, assembly_service): moleculetype="genomic DNA", ) - # Mock query to return None for both (different combinations) mock_query = Mock() mock_query.filter.return_value.scalar.return_value = None mock_db.query.return_value = mock_query @@ -131,7 +111,6 @@ def test_create_version_per_combination(self, mock_db, assembly_service): assembly_service.create(mock_db, obj_in=assembly1) assembly_service.create(mock_db, obj_in=assembly2) - # Both should get version 1 since they have different data_types assert mock_db.add.call_count == 2 @@ -139,12 +118,8 @@ class TestCreateFromExperiments: """Tests for create_from_experiments method.""" def test_create_from_experiments_success(self, mock_db, assembly_service): - """Test successful assembly creation from experiments.""" taxon_id = 172942 - organism = Organism( - taxon_id=taxon_id, - scientific_name="Test Species", - ) + organism = Organism(taxon_id=taxon_id, scientific_name="Test Species") sample = Sample(id=uuid.uuid4(), taxon_id=taxon_id) experiments = [ Experiment( @@ -155,11 +130,10 @@ def test_create_from_experiments_success(self, mock_db, assembly_service): ), ] - # Mock database queries mock_db.query.return_value.filter.return_value.first.return_value = organism mock_db.query.return_value.filter.return_value.all.side_effect = [ - [sample], # samples query - experiments, # experiments query + [sample], + experiments, ] assembly_in = AssemblyCreateFromExperiments( @@ -174,23 +148,17 @@ def test_create_from_experiments_success(self, mock_db, assembly_service): with patch.object(assembly_service, "create") as mock_create: mock_create.return_value = Mock( - id=uuid.uuid4(), - data_types=AssemblyDataTypes.PACBIO_SMRT, - version=1, + id=uuid.uuid4(), data_types=AssemblyDataTypes.PACBIO_SMRT, version=1 ) - assembly, platform_info = assembly_service.create_from_experiments( mock_db, taxon_id=taxon_id, assembly_in=assembly_in ) - # Verify platform info was returned assert "platforms" in platform_info assert "library_strategies" in platform_info assert "experiment_count" in platform_info def test_create_from_experiments_organism_not_found(self, mock_db, assembly_service): - """Test error when organism not found.""" - taxon_id = 999999 mock_db.query.return_value.filter.return_value.first.return_value = None assembly_in = AssemblyCreateFromExperiments( @@ -205,18 +173,13 @@ def test_create_from_experiments_organism_not_found(self, mock_db, assembly_serv with pytest.raises(ValueError, match="Organism with taxon_id 999999 not found"): assembly_service.create_from_experiments( - mock_db, taxon_id=taxon_id, assembly_in=assembly_in + mock_db, taxon_id=999999, assembly_in=assembly_in ) def test_create_from_experiments_no_samples(self, mock_db, assembly_service): - """Test error when no samples found for organism.""" taxon_id = 172942 - organism = Organism( - taxon_id=taxon_id, - scientific_name="Test Species", - ) + organism = Organism(taxon_id=taxon_id, scientific_name="Test Species") - # Mock organism found but no samples mock_db.query.return_value.filter.return_value.first.return_value = organism mock_db.query.return_value.filter.return_value.all.return_value = [] @@ -236,19 +199,14 @@ def test_create_from_experiments_no_samples(self, mock_db, assembly_service): ) def test_create_from_experiments_no_experiments(self, mock_db, assembly_service): - """Test error when no experiments found.""" taxon_id = 172942 - organism = Organism( - taxon_id=taxon_id, - scientific_name="Test Species", - ) + organism = Organism(taxon_id=taxon_id, scientific_name="Test Species") sample = Sample(id=uuid.uuid4(), taxon_id=taxon_id) - # Mock organism and samples found but no experiments mock_db.query.return_value.filter.return_value.first.return_value = organism mock_db.query.return_value.filter.return_value.all.side_effect = [ - [sample], # samples query - [], # experiments query - empty + [sample], + [], ] assembly_in = AssemblyCreateFromExperiments( @@ -266,63 +224,183 @@ def test_create_from_experiments_no_experiments(self, mock_db, assembly_service) mock_db, taxon_id=taxon_id, assembly_in=assembly_in ) - def test_create_from_experiments_overrides_data_types(self, mock_db, assembly_service): - """Test that data_types is overridden based on experiments.""" - taxon_id = 172942 - organism = Organism( - taxon_id=taxon_id, - scientific_name="Test Species", + +class TestGetNextVersionForIntent: + """Tests for get_next_version_for_intent — versioning by (taxon_id, long_read_specimen_sample_id).""" + + def test_first_version_when_no_prior_assemblies(self, mock_db, assembly_service): + mock_query = Mock() + mock_query.filter.return_value.scalar.return_value = None + mock_db.query.return_value = mock_query + + version = assembly_service.get_next_version_for_intent( + mock_db, taxon_id=172942, long_read_specimen_sample_id=LONG_READ_SAMPLE_ID ) - sample = Sample(id=uuid.uuid4(), taxon_id=taxon_id) - experiments = [ - Experiment( - id=uuid.uuid4(), - sample_id=sample.id, - platform="OXFORD_NANOPORE", - library_strategy="WGS", - ), - Experiment( - id=uuid.uuid4(), - sample_id=sample.id, - platform="ILLUMINA", - library_strategy="Hi-C", - ), - ] + assert version == 1 - mock_db.query.return_value.filter.return_value.first.return_value = organism - mock_db.query.return_value.filter.return_value.all.side_effect = [ - [sample], - experiments, - ] + def test_increments_from_existing_max(self, mock_db, assembly_service): + mock_query = Mock() + mock_query.filter.return_value.scalar.return_value = 4 + mock_db.query.return_value = mock_query - # User provides PACBIO_SMRT which should be used instead of auto-detection - assembly_in = AssemblyCreateFromExperiments( - sample_id=sample.id, - assembly_name="Test Assembly", - assembly_type="clone or isolate", - tol_id="tol-008", - data_types=AssemblyDataTypes.PACBIO_SMRT, # Explicitly provided, should be used - coverage=50.0, - program="hifiasm", - moleculetype="genomic DNA", + version = assembly_service.get_next_version_for_intent( + mock_db, taxon_id=172942, long_read_specimen_sample_id=LONG_READ_SAMPLE_ID ) + assert version == 5 - with patch.object(assembly_service, "create") as mock_create: - # Verify that create is called with overridden data_types - mock_create.return_value = Mock( - id=uuid.uuid4(), - data_types=AssemblyDataTypes.OXFORD_NANOPORE_HIC, - version=1, + def test_different_hic_sample_does_not_split_version_sequence(self, mock_db, assembly_service): + """Two intents with the same long_read_specimen_sample_id but different hic_specimen_sample_id + must share the same version counter (hic_specimen_sample_id is NOT part of the key).""" + call_count = [0] + max_versions = [2] # One prior assembly exists for this (taxon_id, long_read) pair + + mock_query = Mock() + mock_query.filter.return_value.scalar.return_value = max_versions[0] + mock_db.query.return_value = mock_query + + v1 = assembly_service.get_next_version_for_intent( + mock_db, taxon_id=172942, long_read_specimen_sample_id=LONG_READ_SAMPLE_ID + ) + # Both calls use the same (taxon_id, long_read_specimen_sample_id) key, so both + # return 3 from the same mock (in real DB the second would see version=3 → return 4). + # This test verifies the QUERY only filters on taxon_id + long_read_specimen_sample_id. + assert v1 == 3 + + +class TestCreateFromIntent: + """Tests for create_from_intent — new signature with specimen sample IDs and manifest persistence.""" + + def test_creates_assembly_with_requested_status(self, mock_db, assembly_service): + mock_query = Mock() + mock_query.filter.return_value.scalar.return_value = None + mock_db.query.return_value = mock_query + mock_db.add = Mock() + mock_db.commit = Mock() + mock_db.refresh = Mock() + + with patch.object(Assembly, "__init__", return_value=None): + assembly_service.create_from_intent( + mock_db, + taxon_id=172942, + long_read_specimen_sample_id=LONG_READ_SAMPLE_ID, + hic_specimen_sample_id=None, + data_types="PACBIO_SMRT", + tol_id="tol-999", + project_id=None, + manifest_json=None, ) - assembly, platform_info = assembly_service.create_from_experiments( - mock_db, taxon_id=taxon_id, assembly_in=assembly_in + mock_db.add.assert_called_once() + added_obj = mock_db.add.call_args[0][0] + assert isinstance(added_obj, Assembly) + mock_db.commit.assert_called_once() + + def test_version_is_next_based_on_long_read_sample(self, mock_db, assembly_service): + """Version is derived from (taxon_id, long_read_specimen_sample_id) only.""" + mock_query = Mock() + mock_query.filter.return_value.scalar.return_value = 2 # existing max = 2 → next = 3 + mock_db.query.return_value = mock_query + mock_db.add = Mock() + mock_db.commit = Mock() + mock_db.refresh = Mock() + + created_versions = [] + + def capture_init(self, **kwargs): + created_versions.append(kwargs.get("version")) + + with patch.object(Assembly, "__init__", capture_init): + assembly_service.create_from_intent( + mock_db, + taxon_id=172942, + long_read_specimen_sample_id=LONG_READ_SAMPLE_ID, + hic_specimen_sample_id=None, + data_types="PACBIO_SMRT", + tol_id=None, + project_id=None, + ) + + assert created_versions == [3] + + def test_manifest_json_persisted(self, mock_db, assembly_service): + """manifest_json is stored on the Assembly object when provided.""" + mock_query = Mock() + mock_query.filter.return_value.scalar.return_value = None + mock_db.query.return_value = mock_query + mock_db.add = Mock() + mock_db.commit = Mock() + mock_db.refresh = Mock() + + sample_manifest = {"scientific_name": "Test", "reads": {"PACBIO_SMRT": {}}} + stored_manifests = [] + + def capture_init(self, **kwargs): + stored_manifests.append(kwargs.get("manifest_json")) + + with patch.object(Assembly, "__init__", capture_init): + assembly_service.create_from_intent( + mock_db, + taxon_id=172942, + long_read_specimen_sample_id=LONG_READ_SAMPLE_ID, + hic_specimen_sample_id=None, + data_types="PACBIO_SMRT", + tol_id=None, + project_id=None, + manifest_json=sample_manifest, + ) + + assert stored_manifests == [sample_manifest] + + def test_hic_specimen_sample_id_stored(self, mock_db, assembly_service): + """hic_specimen_sample_id is stored on the Assembly.""" + mock_query = Mock() + mock_query.filter.return_value.scalar.return_value = None + mock_db.query.return_value = mock_query + mock_db.add = Mock() + mock_db.commit = Mock() + mock_db.refresh = Mock() + + hic_ids = [] + + def capture_init(self, **kwargs): + hic_ids.append(kwargs.get("hic_specimen_sample_id")) + + with patch.object(Assembly, "__init__", capture_init): + assembly_service.create_from_intent( + mock_db, + taxon_id=172942, + long_read_specimen_sample_id=LONG_READ_SAMPLE_ID, + hic_specimen_sample_id=HIC_SAMPLE_ID, + data_types="PACBIO_SMRT_HIC", + tol_id=None, + project_id=None, ) - # Verify create was called - mock_create.assert_called_once() - call_args = mock_create.call_args - created_assembly_in = call_args.kwargs["obj_in"] + assert hic_ids == [HIC_SAMPLE_ID] + + def test_sample_id_set_to_long_read_specimen_sample_id(self, mock_db, assembly_service): + """sample_id is set to long_read_specimen_sample_id for backward compatibility.""" + mock_query = Mock() + mock_query.filter.return_value.scalar.return_value = None + mock_db.query.return_value = mock_query + mock_db.add = Mock() + mock_db.commit = Mock() + mock_db.refresh = Mock() + + sample_ids = [] + + def capture_init(self, **kwargs): + sample_ids.append(kwargs.get("sample_id")) + + with patch.object(Assembly, "__init__", capture_init): + assembly_service.create_from_intent( + mock_db, + taxon_id=172942, + long_read_specimen_sample_id=LONG_READ_SAMPLE_ID, + hic_specimen_sample_id=None, + data_types="PACBIO_SMRT", + tol_id=None, + project_id=None, + ) - # The data_types should be determined from experiments (ILLUMINA+WGS = Hi-C) - assert created_assembly_in.taxon_id == 172942 + assert sample_ids == [LONG_READ_SAMPLE_ID]