feat: taxonomy info change and assembly manifest update

alembic/versions/0002_update_taxon_cols.py

@@ -0,0 +1,52 @@
+"""Remove outdated columns from taxonomy_info table.
+
+Revision ID: 0002_update_taxon_cols
+Revises: 0001_initial_schema
+Create Date: 2026-05-11
+
+Removing columns "defined_class" (redundant with "ncbi_class"), "mito_ref" (redundant with "mitohifi_reference_species"), and
+"busco_dataset_name" (replaced by "busco_odb10_dataset_name" and "busco_odb12_dataset_name"). Also adding "hic_specimen_sample_ids"
+column to assembly table, to replace "hic_specimen_sample_id".
+"""
+
+import sqlalchemy as sa
+from sqlalchemy.dialects.postgresql import JSONB
+
+from alembic import op
+
+revision = "0002_update_taxon_cols"
+down_revision = "0001_initial_schema"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.drop_column("taxonomy_info", "defined_class")
+    op.execute(
+        sa.text(
+            """
+            UPDATE taxonomy_info
+            SET mitohifi_reference_species = COALESCE(mitohifi_reference_species, mito_ref)
+            WHERE mito_ref IS NOT NULL
+            """
+        )
+    )
+    op.drop_column("taxonomy_info", "mito_ref")
+    op.drop_column("taxonomy_info", "busco_dataset_name")
+    op.add_column("assembly", sa.Column("hic_specimen_sample_ids", JSONB, nullable=True))
+
+
+def downgrade() -> None:
+    op.add_column(
+        "taxonomy_info",
+        sa.Column("defined_class", sa.Text(), nullable=True),
+    )
+    op.add_column(
+        "taxonomy_info",
+        sa.Column("mito_ref", sa.Text(), nullable=True),
+    )
+    op.add_column(
+        "taxonomy_info",
+        sa.Column("busco_dataset_name", sa.Text(), nullable=True),
+    )
+    op.drop_column("assembly", "hic_specimen_sample_ids")

app/api/v1/endpoints/assemblies.py

@@ -323,7 +323,13 @@ def get_assembly_manifest(
             detail="No manifest stored for this assembly. Re-submit an intent to generate one.",
         )
 
-    return JSONResponse(content=assembly.manifest_json)
+    return JSONResponse(
+        content={
+            "assembly_id": str(assembly.id),
+            "version": assembly.version,
+            "manifest": assembly.manifest_json,
+        }
+    )
 
 
 @router.post("/intent/{taxon_id}")
@@ -360,12 +366,15 @@ def create_assembly_intent(
         db, intent_in.long_read_specimen_sample_id, org_taxon_id, "long_read_specimen_sample_id"
     )
 
-    # 3. Validate hic_specimen_sample_id if provided
-    hic_sample = None
-    if intent_in.hic_specimen_sample_id:
-        hic_sample = _validate_specimen_sample(
-            db, intent_in.hic_specimen_sample_id, org_taxon_id, "hic_specimen_sample_id"
-        )
+    # 3. Validate hic_specimen_sample_ids if provided
+    hic_samples: List[Sample] = []
+    if intent_in.hic_specimen_sample_ids:
+        for idx, hic_sid in enumerate(intent_in.hic_specimen_sample_ids):
+            hic_samples.append(
+                _validate_specimen_sample(
+                    db, hic_sid, org_taxon_id, f"hic_specimen_sample_ids[{idx}]"
+                )
+            )
 
     # 4. Fetch long-read experiments (PacBio or ONT only) and their reads
     long_read_lineage_sample_ids = _get_lineage_sample_ids_for_specimen(db, long_read_sample)
@@ -391,35 +400,34 @@ def create_assembly_intent(
     long_read_exp_ids = [e.id for e in long_read_experiments]
     long_reads = db.query(Read).filter(Read.experiment_id.in_(long_read_exp_ids)).all()
 
-    # 5. Fetch Hi-C experiments and reads if hic_sample is provided
+    # 5. Fetch Hi-C experiments and reads for each hic_sample
     hic_experiments: List[Experiment] = []
     hic_reads: List[Read] = []
     hic_sample_id_map: Dict[str, str] = {}
-    if hic_sample:
+    for hic_sample in hic_samples:
         hic_lineage_sample_ids = _get_lineage_sample_ids_for_specimen(db, hic_sample)
-        hic_sample_id_map = {
-            str(sample_id): str(hic_sample.id) for sample_id in hic_lineage_sample_ids
-        }
-        hic_experiments = (
+        hic_sample_id_map.update(
+            {str(sample_id): str(hic_sample.id) for sample_id in hic_lineage_sample_ids}
+        )
+        hic_exps = (
             db.query(Experiment)
             .filter(
                 Experiment.sample_id.in_(hic_lineage_sample_ids),
                 Experiment.platform == "ILLUMINA",
             )
             .all()
         )
-        hic_experiments = [
-            e for e in hic_experiments if (e.library_strategy or "").upper() == "HI-C"
-        ]
-        if not hic_experiments:
+        hic_exps = [e for e in hic_exps if (e.library_strategy or "").upper() == "HI-C"]
+        if not hic_exps:
             raise AppError(
                 status_code=422,
                 code="no_hic_experiments",
-                message="No Hi-C experiments (ILLUMINA + Hi-C) found for hic_specimen_sample_id",
+                message="No Hi-C experiments (ILLUMINA + Hi-C) found for hic_specimen_sample_ids",
                 details={"hic_specimen_sample_id": str(hic_sample.id)},
             )
-        hic_exp_ids = [e.id for e in hic_experiments]
-        hic_reads = db.query(Read).filter(Read.experiment_id.in_(hic_exp_ids)).all()
+        hic_exp_ids = [e.id for e in hic_exps]
+        hic_experiments.extend(hic_exps)
+        hic_reads.extend(db.query(Read).filter(Read.experiment_id.in_(hic_exp_ids)).all())
 
     # 6. Determine data_types from the relevant experiments
     # TODO review and remove this step now that we pass in the specimen_id
@@ -439,7 +447,7 @@ def create_assembly_intent(
         db,
         taxon_id=org_taxon_id,
         long_read_specimen_sample_id=long_read_sample.id,
-        hic_specimen_sample_id=hic_sample.id if hic_sample else None,
+        hic_specimen_sample_ids=[s.id for s in hic_samples] if hic_samples else None,
         # TODO review if we need data_types when creating experiment
         data_types=data_types,
         tol_id=intent_in.tol_id,
@@ -450,7 +458,7 @@ def create_assembly_intent(
     try:
         # 8. Build sample metadata and generate JSON manifest
         all_reads = long_reads + hic_reads
-        sample_metadata_by_id = _build_specimen_metadata(long_read_sample, hic_sample)
+        sample_metadata_by_id = _build_specimen_metadata(long_read_sample, *hic_samples)
         sequencing_sample_to_specimen_sample_id = {
             **long_read_sample_id_map,
             **hic_sample_id_map,
@@ -462,16 +470,19 @@ def create_assembly_intent(
             experiments=all_experiments,
             # TODO tolid from sample (reported by broker) not input from caller
             tol_id=assembly.tol_id,
+            assembly_id=str(assembly.id),
             version=assembly.version,
             long_read_sample_id=long_read_sample.id,
-            hic_sample_id=hic_sample.id if hic_sample else None,
+            hic_sample_ids=[s.id for s in hic_samples] if hic_samples else None,
             sample_metadata_by_id=sample_metadata_by_id,
             sequencing_sample_to_specimen_sample_id=sequencing_sample_to_specimen_sample_id,
         )
 
         # 9. Validate that the manifest contains eligible reads then persist it
+        # TODO decide whether to keep this validation.
+        """
         long_read_keys = {"PACBIO_SMRT", "OXFORD_NANOPORE"}
-        if not any(k in manifest_data["reads"] for k in long_read_keys):
+        if not any(k in manifest_data["read_files"] for k in long_read_keys):
             raise AppError(
                 status_code=422,
                 code="no_eligible_long_reads",
@@ -482,14 +493,14 @@ def create_assembly_intent(
                 details={"long_read_specimen_sample_id": str(long_read_sample.id)},
             )
 
-        if hic_sample and "Hi-C" not in manifest_data["reads"]:
+        if hic_samples and "Hi-C" not in manifest_data["read_files"]:
             raise AppError(
                 status_code=422,
                 code="no_eligible_hic_reads",
-                message="No eligible Hi-C reads found for hic_specimen_sample_id",
-                details={"hic_specimen_sample_id": str(hic_sample.id)},
+                message="No eligible Hi-C reads found for hic_specimen_sample_ids",
+                details={"hic_specimen_sample_ids": [str(s.id) for s in hic_samples]},
             )
-
+        """
         assembly.manifest_json = manifest_data
         db.add(assembly)
         db.commit()
@@ -503,7 +514,6 @@ def create_assembly_intent(
         content={
             "assembly_id": str(assembly.id),
             "version": assembly.version,
-            "status": assembly.status,
             "manifest": manifest_data,
         }
     )
@@ -556,9 +566,7 @@ def cancel_assembly_intent(
         "long_read_specimen_sample_id": str(assembly.long_read_specimen_sample_id)
         if assembly.long_read_specimen_sample_id
         else None,
-        "hic_specimen_sample_id": str(assembly.hic_specimen_sample_id)
-        if assembly.hic_specimen_sample_id
-        else None,
+        "hic_specimen_sample_ids": assembly.hic_specimen_sample_ids or [],
         "version": assembly.version,
         "status": assembly.status,
     }
@@ -635,7 +643,13 @@ def get_manifest_by_assembly_id(
             detail="No manifest stored for this assembly. Re-submit an intent to generate one.",
         )
 
-    return JSONResponse(content=assembly.manifest_json)
+    return JSONResponse(
+        content={
+            "assembly_id": str(assembly.id),
+            "version": assembly.version,
+            "manifest": assembly.manifest_json,
+        }
+    )
 
 
 @router.get("/{assembly_id}", response_model=AssemblySchema)

app/api/v1/endpoints/taxonomy_info.py

@@ -8,7 +8,11 @@
 from app.core.policy import policy
 from app.models.taxonomy_info import TaxonomyInfo
 from app.models.user import User
-from app.schemas.bulk_import import BulkImportResponse, BulkTaxonomyInfoImport
+from app.schemas.bulk_import import (
+    BulkImportResponse,
+    BulkNcbiRefreshRequest,
+    BulkTaxonomyInfoImport,
+)
 from app.schemas.taxonomy_info import (
     TaxonomyInfo as TaxonomyInfoSchema,
 )
@@ -50,6 +54,40 @@ def bulk_import_taxonomy_info(
     return taxonomy_info_service.bulk_import(db, data=data.root)
 
 
+@router.post("/bulk-upsert", response_model=BulkImportResponse)
+@policy("taxonomy_info:bulk_upsert")
+def bulk_upsert_taxonomy_info(
+    *,
+    db: Session = Depends(get_db),
+    data: BulkTaxonomyInfoImport,
+    current_user: User = Depends(get_current_active_user),
+) -> Any:
+    """
+    Bulk upsert taxonomy info from a dictionary keyed by taxon_id.
+
+    Inserts new rows and updates existing ones. Re-fetches NCBI data for all rows.
+    The taxon_id key must reference an existing organism.
+    """
+    return taxonomy_info_service.bulk_upsert(db, data=data.root)
+
+
+@router.post("/bulk-ncbi-refresh", response_model=BulkImportResponse)
+@policy("taxonomy_info:bulk_ncbi_refresh")
+def bulk_ncbi_refresh_taxonomy_info(
+    *,
+    db: Session = Depends(get_db),
+    data: BulkNcbiRefreshRequest,
+    current_user: User = Depends(get_current_active_user),
+) -> Any:
+    """
+    Re-sync NCBI taxonomy fields for existing taxonomy_info rows.
+
+    Only updates ncbi_* columns — upstream fields are left unchanged.
+    Rows that do not yet have a taxonomy_info record are skipped.
+    """
+    return taxonomy_info_service.bulk_ncbi_refresh(db, taxon_ids=data.taxon_ids)
+
+
 @router.get("/{taxon_id}", response_model=TaxonomyInfoSchema)
 def get_taxonomy_info(
     *,

app/core/policy.py

@@ -66,6 +66,8 @@
     "taxonomy_info:update": ["curator", "admin"],
     "taxonomy_info:delete": ["admin", "superuser"],
     "taxonomy_info:bulk_import": ["curator", "admin"],
+    "taxonomy_info:bulk_upsert": ["curator", "admin"],
+    "taxonomy_info:bulk_ncbi_refresh": ["curator", "admin"],
 }
 
 

app/models/assembly.py

@@ -74,6 +74,7 @@ class Assembly(Base):
         UUID(as_uuid=True), ForeignKey("sample.id"), nullable=True
     )
     hic_specimen_sample_id = Column(UUID(as_uuid=True), ForeignKey("sample.id"), nullable=True)
+    hic_specimen_sample_ids = Column(JSONB, nullable=True)
     manifest_json = Column(JSONB, nullable=True)
 
     # Relationships

app/models/taxonomy_info.py

@@ -23,16 +23,13 @@ class TaxonomyInfo(Base):
     ncbi_tax_string = Column(Text, nullable=True)
     ncbi_full_lineage = Column(Text, nullable=True)
     ncbi_last_synced_at = Column(DateTime(timezone=True), nullable=True)
-    mito_ref = Column(Text, nullable=True)
-    busco_dataset_name = Column(Text, nullable=True)
     busco_odb10_dataset_name = Column(Text, nullable=True)
     busco_odb12_dataset_name = Column(Text, nullable=True)
     find_plastid = Column(Boolean, nullable=True)
     hic_motif = Column(Text, nullable=True)
     mitochondrial_genetic_code_id = Column(Integer, nullable=True)
     mitohifi_reference_species = Column(Text, nullable=True)
     oatk_hmm_name = Column(Text, nullable=True)
-    defined_class = Column(Text, nullable=True)
     augustus_dataset_name = Column(Text, nullable=True)
     genetic_code_id = Column(Integer, nullable=True)
 

app/schemas/assembly.py

@@ -86,7 +86,7 @@ class AssemblyIntent(BaseModel):
 
     tol_id: Optional[str] = None
     long_read_specimen_sample_id: UUID
-    hic_specimen_sample_id: Optional[UUID] = None
+    hic_specimen_sample_ids: Optional[List[UUID]] = None
 
 
 class AssemblyIntentResponse(BaseModel):

app/schemas/bulk_import.py

@@ -42,10 +42,17 @@ class BulkTaxonomyInfoImport(RootModel[Dict[int, TaxonomyInfoUpdate]]):
     """Bulk taxonomy info import keyed by taxon_id."""
 
 
+class BulkNcbiRefreshRequest(BaseModel):
+    """Request body for bulk NCBI taxonomy refresh."""
+
+    taxon_ids: List[int]
+
+
 class BulkImportResponse(BaseModel):
     """Schema for bulk import response."""
 
     created_count: int
+    updated_count: Optional[int] = 0
     skipped_count: Optional[int] = 0
     message: str
     errors: Optional[List[str]] = None  # List of all errors with context

app/schemas/taxonomy_info.py

@@ -17,30 +17,24 @@ class TaxonomyInfoBase(BaseModel):
     ncbi_tax_string: Optional[str] = None
     ncbi_full_lineage: Optional[str] = None
     ncbi_last_synced_at: Optional[datetime] = None
-    mito_ref: Optional[str] = None
-    busco_dataset_name: Optional[str] = None
     busco_odb10_dataset_name: Optional[str] = None
     busco_odb12_dataset_name: Optional[str] = None
     find_plastid: Optional[bool] = None
     hic_motif: Optional[str] = None
     mitochondrial_genetic_code_id: Optional[int] = None
     mitohifi_reference_species: Optional[str] = None
     oatk_hmm_name: Optional[str] = None
-    defined_class: Optional[str] = None
     augustus_dataset_name: Optional[str] = None
     genetic_code_id: Optional[int] = None
 
 
 class TaxonomyInfoWriteBase(BaseModel):
-    busco_dataset_name: Optional[str] = None
     busco_odb10_dataset_name: Optional[str] = None
     busco_odb12_dataset_name: Optional[str] = None
     find_plastid: Optional[bool] = None
     hic_motif: Optional[str] = None
     mitochondrial_genetic_code_id: Optional[int] = None
-    mitohifi_reference_species: Optional[str] = None
     oatk_hmm_name: Optional[str] = None
-    defined_class: Optional[str] = None
     augustus_dataset_name: Optional[str] = None
     genetic_code_id: Optional[int] = None