From 12a83356e30fbc1a8bfd559a3d0a94be6c7c0850 Mon Sep 17 00:00:00 2001 From: Emily Marshall Date: Wed, 13 May 2026 14:17:34 +1000 Subject: [PATCH 01/14] feat: update taxonomy_info and organism schema, adding taxonomy details from ncbi --- .../0014_add_ncbi_taxonomy_details.py | 97 +++++++++++++++++++ app/api/v1/endpoints/assemblies.py | 2 +- app/services/assembly_helper.py | 24 ++--- schema.sql | 43 +++++--- 4 files changed, 138 insertions(+), 28 deletions(-) create mode 100644 alembic/versions/0014_add_ncbi_taxonomy_details.py diff --git a/alembic/versions/0014_add_ncbi_taxonomy_details.py b/alembic/versions/0014_add_ncbi_taxonomy_details.py new file mode 100644 index 0000000..ea71af2 --- /dev/null +++ b/alembic/versions/0014_add_ncbi_taxonomy_details.py @@ -0,0 +1,97 @@ +"""Remove submission_xml from assembly_submission table. + +Revision ID: 0014_add_ncbi_taxonomy_details +Revises: 0013_remove_assembly_submission_xml +Create Date: 2026-05-13 + +We are adding in NCBI taxonomy details for organisms, sourced through external API lookup to NCBI's Datasets API v2. +""" + +import sqlalchemy as sa + +from alembic import op + +revision = "0014_add_ncbi_taxonomy_details" +down_revision = "0013_remove_assembly_submission_xml" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.drop_column("organism", "common_name_source") + op.alter_column("organism", "genus", new_column_name="bpa_genus") + op.alter_column("organism", "species", new_column_name="bpa_species") + op.alter_column("organism", "common_name", new_column_name="bpa_common_name") + op.alter_column("organism", "infraspecific_epithet", new_column_name="bpa_infraspecific_epithet") + op.alter_column("organism", "culture_or_strain_id", new_column_name="bpa_culture_or_strain_id") + op.alter_column("organism", "authority", new_column_name="bpa_authority") + op.alter_column("organism", "scientific_name", new_column_name="bpa_scientific_name") + + op.drop_column("organism", "tax_string") + op.drop_column("organism", "ncbi_order") + op.drop_column("organism", "ncbi_family") + op.drop_column("organism", "busco_dataset_name") + op.drop_column("organism", "taxonomy_lineage_json") + + # Add new columns for NCBI taxonomy details + op.add_column("taxonomy_info", sa.Column("ncbi_taxon_id", sa.Integer(), nullable=True)) + op.add_column("taxonomy_info", sa.Column("ncbi_rank", sa.String(), nullable=True)) + op.add_column("taxonomy_info", sa.Column("ncbi_scientific_name", sa.String(), nullable=True)) + op.add_column("taxonomy_info", sa.Column("ncbi_authority", sa.String(), nullable=True)) + op.add_column("taxonomy_info", sa.Column("ncbi_common_name", sa.String(), nullable=True)) + op.add_column("taxonomy_info", sa.Column("ncbi_class", sa.String(), nullable=True)) + op.add_column("taxonomy_info", sa.Column("ncbi_order", sa.String(), nullable=True)) + op.add_column("taxonomy_info", sa.Column("ncbi_family", sa.String(), nullable=True)) + op.add_column("taxonomy_info", sa.Column("ncbi_lineage", sa.JSON(), nullable=True)) + op.add_column("taxonomy_info", sa.Column("ncbi_tax_string", sa.String(), nullable=True)) + op.add_column("taxonomy_info", sa.Column("ncbi_full_lineage", sa.String(), nullable=True)) + op.add_column("taxonomy_info", sa.Column("mito_ref", sa.String(), nullable=True)) + op.add_column("taxonomy_info", sa.Column("busco_dataset_name", sa.String(), nullable=True)) + + +def downgrade() -> None: + # Drop columns added by the upgrade, in reverse order + op.drop_column("taxonomy_info", "busco_dataset_name") + op.drop_column("taxonomy_info", "mito_ref") + op.drop_column("taxonomy_info", "ncbi_full_lineage") + op.drop_column("taxonomy_info", "ncbi_tax_string") + op.drop_column("taxonomy_info", "ncbi_lineage") + op.drop_column("taxonomy_info", "ncbi_family") + op.drop_column("taxonomy_info", "ncbi_order") + op.drop_column("taxonomy_info", "ncbi_class") + op.drop_column("taxonomy_info", "ncbi_common_name") + op.drop_column("taxonomy_info", "ncbi_authority") + op.drop_column("taxonomy_info", "ncbi_scientific_name") + op.drop_column("taxonomy_info", "ncbi_rank") + op.drop_column("taxonomy_info", "ncbi_taxon_id") + + # Restore columns that were dropped by the upgrade + op.add_column( + "organism", + sa.Column("taxonomy_lineage_json", postgresql.JSONB(astext_type=sa.Text()), nullable=True), + ) + op.add_column("organism", sa.Column("busco_dataset_name", sa.Text(), nullable=True)) + op.add_column("organism", sa.Column("ncbi_family", sa.Text(), nullable=True)) + op.add_column("organism", sa.Column("ncbi_order", sa.Text(), nullable=True)) + op.add_column("organism", sa.Column("tax_string", sa.Text(), nullable=True)) + + # Rename BPA columns back to their original names + op.alter_column("organism", "bpa_scientific_name", new_column_name="scientific_name") + op.alter_column("organism", "bpa_authority", new_column_name="authority") + op.alter_column( + "organism", + "bpa_culture_or_strain_id", + new_column_name="culture_or_strain_id", + ) + op.alter_column( + "organism", + "bpa_infraspecific_epithet", + new_column_name="infraspecific_epithet", + ) + op.alter_column("organism", "bpa_common_name", new_column_name="common_name") + op.alter_column("organism", "bpa_species", new_column_name="species") + op.alter_column("organism", "bpa_genus", new_column_name="genus") + + # Restore dropped BPA source column + op.add_column("organism", sa.Column("common_name_source", sa.Text(), nullable=True)) + diff --git a/app/api/v1/endpoints/assemblies.py b/app/api/v1/endpoints/assemblies.py index 8e0a41d..76802f7 100644 --- a/app/api/v1/endpoints/assemblies.py +++ b/app/api/v1/endpoints/assemblies.py @@ -457,7 +457,7 @@ def create_assembly_intent( } manifest_data = generate_assembly_manifest_json( organism=organism, - taxonomy_information=getattr(organism, "taxonomy_info", None), + taxonomy_info=getattr(organism, "taxonomy_info", None), reads=all_reads, experiments=all_experiments, # TODO tolid from sample (reported by broker) not input from caller diff --git a/app/services/assembly_helper.py b/app/services/assembly_helper.py index d218055..1d64fcb 100644 --- a/app/services/assembly_helper.py +++ b/app/services/assembly_helper.py @@ -124,7 +124,7 @@ def _normalize_read_number(read_number: str | None) -> str | None: def generate_assembly_manifest_json( organism: Organism, - taxonomy_information: Optional[TaxonomyInfo], + taxonomy_info: Optional[TaxonomyInfo], reads: List[Read], experiments: List[Experiment], tol_id: str | None, @@ -146,7 +146,7 @@ def generate_assembly_manifest_json( Args: organism: Organism object - taxonomy_information: TaxonomyInfo object related to the organism when available + taxonomy_info: TaxonomyInfo object related to the organism when available reads: Combined list of Read objects from both specimen samples experiments: Combined list of Experiment objects from both specimen samples tol_id: ToL ID for the assembly (optional) @@ -316,20 +316,20 @@ def generate_assembly_manifest_json( "taxon_id": organism.taxon_id, "tolid": tol_id, "version": version, - "busco_odb10_dataset_name": getattr(taxonomy_information, "busco_odb10_dataset_name", None), - "busco_odb12_dataset_name": getattr(taxonomy_information, "busco_odb12_dataset_name", None), - "find_plastid": getattr(taxonomy_information, "find_plastid", None), - "hic_motif": getattr(taxonomy_information, "hic_motif", None), + "busco_odb10_dataset_name": getattr(taxonomy_info, "busco_odb10_dataset_name", None), + "busco_odb12_dataset_name": getattr(taxonomy_info, "busco_odb12_dataset_name", None), + "find_plastid": getattr(taxonomy_info, "find_plastid", None), + "hic_motif": getattr(taxonomy_info, "hic_motif", None), "mitochondrial_genetic_code_id": getattr( - taxonomy_information, "mitochondrial_genetic_code_id", None + taxonomy_info, "mitochondrial_genetic_code_id", None ), "mitohifi_reference_species": getattr( - taxonomy_information, "mitohifi_reference_species", None + taxonomy_info, "mitohifi_reference_species", None ), - "oatk_hmm_name": getattr(taxonomy_information, "oatk_hmm_name", None), - "augustus_dataset_name": getattr(taxonomy_information, "augustus_dataset_name", None), - "genetic_code_id": getattr(taxonomy_information, "genetic_code_id", None), - "defined_class": getattr(taxonomy_information, "defined_class", None), + "oatk_hmm_name": getattr(taxonomy_info, "oatk_hmm_name", None), + "augustus_dataset_name": getattr(taxonomy_info, "augustus_dataset_name", None), + "genetic_code_id": getattr(taxonomy_info, "genetic_code_id", None), + "defined_class": getattr(taxonomy_info, "defined_class", None), "reads": reads_section, } return manifest diff --git a/schema.sql b/schema.sql index 1d5890e..66beb39 100644 --- a/schema.sql +++ b/schema.sql @@ -66,26 +66,23 @@ CREATE TABLE refresh_token ( CREATE TABLE organism ( taxon_id int PRIMARY KEY, -- We need to check that the scientific name = the tax id level, because we have a bunch of tax_ids that are the same for organisms which should have a more granular taxid level - scientific_name TEXT, - genus TEXT, - species TEXT, - common_name TEXT, - common_name_source TEXT, + bpa_scientific_name TEXT, + bpa_genus TEXT, + bpa_species TEXT, + bpa_common_name TEXT, -- TODO check common name is coming through from mapper, and set common_name_source -- family TEXT, -- order_or_group TEXT, -- class TEXT, -- phylum TEXT, - infraspecific_epithet TEXT, - culture_or_strain_id TEXT, - authority TEXT, - atol_scientific_name TEXT, - tax_string TEXT, - ncbi_order TEXT, - ncbi_family TEXT, - busco_dataset_name TEXT, + bpa_infraspecific_epithet TEXT, + bpa_culture_or_strain_id TEXT, + bpa_authority TEXT, + scientific_name TEXT, + + -- # TODO delete bpa_json bpa_json JSONB, - taxonomy_lineage_json JSONB, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); @@ -96,6 +93,23 @@ CREATE TABLE organism ( CREATE TABLE taxonomy_info ( taxon_id INT PRIMARY KEY REFERENCES organism(taxon_id) ON DELETE CASCADE, + ncbi_taxon_id INTEGER, + + ncbi_rank TEXT, + ncbi_scientific_name TEXT, + ncbi_authority TEXT, + ncbi_scientific_name TEXT, + ncbi_authority TEXT, + ncbi_common_name TEXT, + ncbi_class TEXT, + ncbi_order TEXT, + ncbi_family TEXT, + ncbi_lineage JSONB, + ncbi_tax_string TEXT, + ncbi_full_lineage TEXT, + mito_ref TEXT + + busco_dataset_name TEXT, busco_odb10_dataset_name TEXT, busco_odb12_dataset_name TEXT, find_plastid BOOLEAN, @@ -103,7 +117,6 @@ CREATE TABLE taxonomy_info ( mitochondrial_genetic_code_id INTEGER, mitohifi_reference_species TEXT, oatk_hmm_name TEXT, - defined_class TEXT, augustus_dataset_name TEXT, genetic_code_id INTEGER ); From b50566fb098b41eede5dd82b56b5334119e023a2 Mon Sep 17 00:00:00 2001 From: Emily Marshall Date: Wed, 13 May 2026 14:29:04 +1000 Subject: [PATCH 02/14] feat: update taxonomy and organism fields in schemas and models --- app/models/organism.py | 19 ++++------- app/models/taxonomy_info.py | 14 ++++++++ app/schemas/organism.py | 63 +++++++++++------------------------- app/schemas/taxonomy_info.py | 15 ++++++++- 4 files changed, 53 insertions(+), 58 deletions(-) diff --git a/app/models/organism.py b/app/models/organism.py index 6eda0f9..011cd93 100644 --- a/app/models/organism.py +++ b/app/models/organism.py @@ -15,21 +15,16 @@ class Organism(Base): __tablename__ = "organism" taxon_id = Column(Integer, primary_key=True) + bpa_scientific_name = Column(Text, nullable=True) + bpa_genus = Column(Text, nullable=True) + bpa_species = Column(Text, nullable=True) + bpa_common_name = Column(Text, nullable=True) + bpa_infraspecific_epithet = Column(Text, nullable=True) + bpa_culture_or_strain_id = Column(Text, nullable=True) + bpa_authority = Column(Text, nullable=True) scientific_name = Column(Text, nullable=True) - common_name = Column(Text, nullable=True) - common_name_source = Column(Text, nullable=True) - genus = Column(Text, nullable=True) - species = Column(Text, nullable=True) - infraspecific_epithet = Column(Text, nullable=True) - culture_or_strain_id = Column(Text, nullable=True) - authority = Column(Text, nullable=True) atol_scientific_name = Column(Text, nullable=True) - tax_string = Column(Text, nullable=True) - ncbi_order = Column(Text, nullable=True) - ncbi_family = Column(Text, nullable=True) - busco_dataset_name = Column(Text, nullable=True) bpa_json = Column(JSONB, nullable=True) - taxonomy_lineage_json = Column(JSONB, nullable=True) created_at = Column(DateTime(timezone=True), nullable=False, server_default=func.now()) updated_at = Column( DateTime(timezone=True), diff --git a/app/models/taxonomy_info.py b/app/models/taxonomy_info.py index 0b35d82..609419d 100644 --- a/app/models/taxonomy_info.py +++ b/app/models/taxonomy_info.py @@ -1,4 +1,5 @@ from sqlalchemy import Boolean, Column, ForeignKey, Integer, Text +from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.orm import relationship from app.db.session import Base @@ -10,6 +11,19 @@ class TaxonomyInfo(Base): taxon_id = Column( Integer, ForeignKey("organism.taxon_id", ondelete="CASCADE"), primary_key=True ) + ncbi_taxon_id = Column(Integer, nullable=True) + ncbi_rank = Column(Text, nullable=True) + ncbi_scientific_name = Column(Text, nullable=True) + ncbi_authority = Column(Text, nullable=True) + ncbi_common_name = Column(Text, nullable=True) + ncbi_class = Column(Text, nullable=True) + ncbi_order = Column(Text, nullable=True) + ncbi_family = Column(Text, nullable=True) + ncbi_lineage = Column(JSONB, nullable=True) + ncbi_tax_string = Column(Text, nullable=True) + ncbi_full_lineage = Column(Text, nullable=True) + mito_ref = Column(Text, nullable=True) + busco_dataset_name = Column(Text, nullable=True) busco_odb10_dataset_name = Column(Text, nullable=True) busco_odb12_dataset_name = Column(Text, nullable=True) find_plastid = Column(Boolean, nullable=True) diff --git a/app/schemas/organism.py b/app/schemas/organism.py index b1b90b2..2ae1ad3 100644 --- a/app/schemas/organism.py +++ b/app/schemas/organism.py @@ -1,43 +1,27 @@ from __future__ import annotations from datetime import datetime -from typing import TYPE_CHECKING, Dict, Optional +from typing import Dict, Optional -from pydantic import BaseModel, ConfigDict, model_validator +from pydantic import BaseModel, ConfigDict -from app.schemas.common import SubmissionStatus # noqa: F401 – kept for consumers - -if TYPE_CHECKING: - from app.schemas.taxonomy_info import TaxonomyInfo as TaxonomyInfoSchema +from app.schemas.taxonomy_info import TaxonomyInfo as TaxonomyInfoSchema class OrganismBase(BaseModel): """Base Organism schema with common attributes.""" taxon_id: int + bpa_scientific_name: Optional[str] = None + bpa_genus: Optional[str] = None + bpa_species: Optional[str] = None + bpa_common_name: Optional[str] = None + bpa_infraspecific_epithet: Optional[str] = None + bpa_culture_or_strain_id: Optional[str] = None + bpa_authority: Optional[str] = None scientific_name: Optional[str] = None - common_name: Optional[str] = None - common_name_source: Optional[str] = None - genus: Optional[str] = None - species: Optional[str] = None - infraspecific_epithet: Optional[str] = None - culture_or_strain_id: Optional[str] = None - authority: Optional[str] = None atol_scientific_name: Optional[str] = None - tax_string: Optional[str] = None - ncbi_order: Optional[str] = None - ncbi_family: Optional[str] = None - busco_dataset_name: Optional[str] = None bpa_json: Optional[Dict] = None - taxonomy_lineage_json: Optional[Dict] = None - - @model_validator(mode="before") - @classmethod - def _coerce_legacy_keys(cls, data): - if isinstance(data, dict) and "taxon_id" not in data and "tax_id" in data: - data = dict(data) - data["taxon_id"] = data.pop("tax_id") - return data class OrganismCreate(OrganismBase): @@ -49,21 +33,16 @@ class OrganismCreate(OrganismBase): class OrganismUpdate(BaseModel): """Schema for updating an existing organism.""" + bpa_scientific_name: Optional[str] = None + bpa_genus: Optional[str] = None + bpa_species: Optional[str] = None + bpa_common_name: Optional[str] = None + bpa_infraspecific_epithet: Optional[str] = None + bpa_culture_or_strain_id: Optional[str] = None + bpa_authority: Optional[str] = None scientific_name: Optional[str] = None - common_name: Optional[str] = None - common_name_source: Optional[str] = None - genus: Optional[str] = None - species: Optional[str] = None - infraspecific_epithet: Optional[str] = None - culture_or_strain_id: Optional[str] = None - authority: Optional[str] = None atol_scientific_name: Optional[str] = None - tax_string: Optional[str] = None - ncbi_order: Optional[str] = None - ncbi_family: Optional[str] = None - busco_dataset_name: Optional[str] = None bpa_json: Optional[Dict] = None - taxonomy_lineage_json: Optional[Dict] = None class OrganismInDBBase(OrganismBase): @@ -72,16 +51,10 @@ class OrganismInDBBase(OrganismBase): created_at: datetime updated_at: datetime - model_config = ConfigDict(from_attributes=True, populate_by_name=True) + model_config = ConfigDict(from_attributes=True) class Organism(OrganismInDBBase): """Schema for returning organism information.""" taxonomy_info: Optional[TaxonomyInfoSchema] = None - - -# Resolve forward references now that all schemas are defined. -from app.schemas.taxonomy_info import TaxonomyInfo as TaxonomyInfoSchema # noqa: E402 - -Organism.model_rebuild() diff --git a/app/schemas/taxonomy_info.py b/app/schemas/taxonomy_info.py index f93743e..bdc67e2 100644 --- a/app/schemas/taxonomy_info.py +++ b/app/schemas/taxonomy_info.py @@ -1,9 +1,22 @@ -from typing import Optional +from typing import Any, Optional from pydantic import BaseModel, ConfigDict class TaxonomyInfoBase(BaseModel): + ncbi_taxon_id: Optional[int] = None + ncbi_rank: Optional[str] = None + ncbi_scientific_name: Optional[str] = None + ncbi_authority: Optional[str] = None + ncbi_common_name: Optional[str] = None + ncbi_class: Optional[str] = None + ncbi_order: Optional[str] = None + ncbi_family: Optional[str] = None + ncbi_lineage: Optional[list[dict[str, Any]]] = None + ncbi_tax_string: Optional[str] = None + ncbi_full_lineage: Optional[str] = None + mito_ref: Optional[str] = None + busco_dataset_name: Optional[str] = None busco_odb10_dataset_name: Optional[str] = None busco_odb12_dataset_name: Optional[str] = None find_plastid: Optional[bool] = None From 20274c90675f2d4395939c5a971bed2f93b767ae Mon Sep 17 00:00:00 2001 From: Emily Marshall Date: Thu, 14 May 2026 10:57:18 +1000 Subject: [PATCH 03/14] feat: add dependency for requests library --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index e9653b6..ef351be 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,7 @@ dependencies = [ "alembic>=1.12.0", "python-multipart>=0.0.6", "email-validator>=2.0.0", + "requests>=2.31.0", ] [project.optional-dependencies] From 0b59f73a3ea89bfdc3c10f0d9f797cb6de1c4cd2 Mon Sep 17 00:00:00 2001 From: Emily Marshall Date: Thu, 14 May 2026 10:57:58 +1000 Subject: [PATCH 04/14] feat: add taxonomy lookup code --- app/models/organism.py | 2 - app/schemas/organism.py | 4 +- app/services/ncbi_taxonomy_service.py | 451 ++++++++++++++++++++++++++ app/services/organism_service.py | 51 ++- app/services/taxonomy_info_service.py | 17 +- 5 files changed, 488 insertions(+), 37 deletions(-) create mode 100644 app/services/ncbi_taxonomy_service.py diff --git a/app/models/organism.py b/app/models/organism.py index 011cd93..4bbdc88 100644 --- a/app/models/organism.py +++ b/app/models/organism.py @@ -22,8 +22,6 @@ class Organism(Base): bpa_infraspecific_epithet = Column(Text, nullable=True) bpa_culture_or_strain_id = Column(Text, nullable=True) bpa_authority = Column(Text, nullable=True) - scientific_name = Column(Text, nullable=True) - atol_scientific_name = Column(Text, nullable=True) bpa_json = Column(JSONB, nullable=True) created_at = Column(DateTime(timezone=True), nullable=False, server_default=func.now()) updated_at = Column( diff --git a/app/schemas/organism.py b/app/schemas/organism.py index 2ae1ad3..689be83 100644 --- a/app/schemas/organism.py +++ b/app/schemas/organism.py @@ -19,8 +19,6 @@ class OrganismBase(BaseModel): bpa_infraspecific_epithet: Optional[str] = None bpa_culture_or_strain_id: Optional[str] = None bpa_authority: Optional[str] = None - scientific_name: Optional[str] = None - atol_scientific_name: Optional[str] = None bpa_json: Optional[Dict] = None @@ -40,7 +38,7 @@ class OrganismUpdate(BaseModel): bpa_infraspecific_epithet: Optional[str] = None bpa_culture_or_strain_id: Optional[str] = None bpa_authority: Optional[str] = None - scientific_name: Optional[str] = None + bpa_scientific_name: Optional[str] = None atol_scientific_name: Optional[str] = None bpa_json: Optional[Dict] = None diff --git a/app/services/ncbi_taxonomy_service.py b/app/services/ncbi_taxonomy_service.py new file mode 100644 index 0000000..b4666c8 --- /dev/null +++ b/app/services/ncbi_taxonomy_service.py @@ -0,0 +1,451 @@ +import logging +import time + +import requests + +logger = logging.getLogger(__name__) + + +class NcbiTaxonomyService: + related_mitos = {} + no_mitos = [] + + def configure_logging() -> None: + logging.basicConfig( + level=logging.DEBUG, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + ) + + def normalize_tax_id(value: int | str | None) -> int | str | None: + if value is None: + return None + if isinstance(value, int): + return value + stripped = str(value).strip() + if stripped.isdigit(): + return int(stripped) + return stripped + + def chunked(items: list[int | str], size: int) -> list[list[int | str]]: + return [items[i : i + size] for i in range(0, len(items), size)] + + def build_taxonomy_url(tax_ids: list[int | str], endpoint: str) -> str: + taxons = ",".join(str(tax_id) for tax_id in tax_ids) + if endpoint == "taxonomy": + return ( + "https://api.ncbi.nlm.nih.gov/datasets/v2/taxonomy/" + f"taxon/{taxons}/dataset_report" + ) + elif endpoint == "organelle": + return ( + "https://api.ncbi.nlm.nih.gov/datasets/v2/organelle/" + f"taxon/{taxons}/dataset_report" + ) + + def fetch_reports( + tax_ids: list[int | str], + endpoint: str, + *, + max_retries: int = 3, + backoff_seconds: int = 1, + timeout_seconds: int = 20, + sleep_fn=time.sleep, + ) -> list[dict]: + """Fetch taxonomy report dicts for a batch of tax_ids. + + Raises RuntimeError if request fails after retries. + """ + url = build_taxonomy_url(tax_ids, endpoint) + + for attempt in range(1, max_retries + 1): + try: + response = requests.get(url, timeout=timeout_seconds) + response.raise_for_status() + payload = response.json() + reports = payload.get("reports", []) + return reports + except (requests.RequestException, ValueError, RuntimeError) as exc: + logger.warning( + "Bulk fetch failed for tax_ids %s (attempt %s/%s): %s", + tax_ids, + attempt, + max_retries, + exc, + ) + if attempt == max_retries: + raise RuntimeError( + "Unable to fetch taxonomy for tax_ids " + f"{tax_ids} after {max_retries} attempts" + ) from exc + sleep_fn(backoff_seconds * (2 ** (attempt - 1))) + + raise RuntimeError("Unexpected retry loop exit") + + def extract_taxonomy_fields(report: dict) -> dict: + """Extract required taxonomy fields from an NCBI report object.""" + taxonomy = report.get("taxonomy") or {} + classification = taxonomy.get("classification") or {} + current_rank = taxonomy.get("rank") + current_name = (taxonomy.get("current_scientific_name") or {}).get("name") + lineage = build_lineage( + classification, current_rank=current_rank, current_name=current_name + ) + + return { + "taxon_id": taxonomy.get("tax_id"), + "ncbi_rank": current_rank, + "ncbi_scientific_name": current_name, + "ncbi_authority": (taxonomy.get("current_scientific_name") or {}).get("authority"), + "ncbi_common_name": taxonomy.get("curator_common_name"), + "ncbi_class": (classification.get("class") or {}).get("name"), + "ncbi_order": (classification.get("order") or {}).get("name"), + "ncbi_family": (classification.get("family") or {}).get("name"), + "ncbi_lineage": lineage, + "ncbi_tax_string": lineage_to_string(lineage), + "ncbi_full_lineage": process_parents(taxonomy.get("parents")), + "mito_ref": organelle_ref_lookup( + taxonomy.get("parents"), taxonomy.get("tax_id"), organelle_type="Mitochondrion" + ), + } + + def has_useful_taxonomy_data(extracted: dict) -> bool: + """Return True if at least one expected extracted value is present.""" + return any(value is not None for value in extracted.values()) + + def build_lineage( + classification: dict, + *, + current_rank: str | None, + current_name: str | None, + ) -> list[dict]: + """Build ordered lineage as rank/name objects. + + We keep API-provided classification order, which is largest -> smallest in NCBI payloads. + """ + lineage: list[dict] = [] + + if isinstance(classification, dict): + domain_name = classification.get("domain", {}).get("name") + if domain_name: + lineage.append({"rank": "domain", "name": domain_name}) + for rank, node in classification.items(): + if not isinstance(node, dict): + continue + name = node.get("name") + if not name: + continue + if rank == "domain": + continue + lineage.append({"rank": str(rank), "name": name}) + + if current_name: + current_rank_label = str(current_rank).lower() if current_rank else "current" + if not lineage or lineage[-1]["name"] != current_name: + lineage.append({"rank": current_rank_label, "name": current_name}) + + return lineage + + def lineage_to_string(lineage: list[dict]) -> str | None: + names = [item.get("name") for item in lineage if item.get("name")] + return "; ".join(names) if names else None + + def get_report_query_taxid(report: dict) -> int | str | None: + query = report.get("query") or [] + if not query: + return None + return normalize_tax_id(query[0]) + + def prepare_unique_taxa( + input_taxa: list[dict], + ) -> tuple[list[int | str], dict[int | str, str | None], list[dict]]: + """Return (unique_taxids, raw_name_by_taxid, pre_collected_unmapped).""" + unique_taxids: list[int | str] = [] + raw_name_by_taxid: dict[int | str, str | None] = {} + pre_collected_unmapped: list[dict] = [] + seen_taxids: set[int | str] = set() + + # First pass: normalize IDs, record missing IDs, and preserve input order for unique IDs. + for item in input_taxa: + tax_id = normalize_tax_id(item.get("tax_id")) + raw_name = item.get("scientific_name") + + if tax_id is None: + logger.error("Skipping record with missing tax_id (scientific_name=%s)", raw_name) + pre_collected_unmapped.append( + {"taxon_id": None, "raw_scientific_name": raw_name, "error": "missing_tax_id"} + ) + continue + + if tax_id in seen_taxids: + logger.debug("Skipping duplicate tax_id %s", tax_id) + continue + + seen_taxids.add(tax_id) + unique_taxids.append(tax_id) + raw_name_by_taxid[tax_id] = raw_name + + return unique_taxids, raw_name_by_taxid, pre_collected_unmapped + + def process_batch( + taxon_batch: list[int | str], + raw_name_by_taxid: dict[int | str, str | None], + ) -> tuple[list[dict], list[dict]]: + mapped_batch: list[dict] = [] + unmapped_batch: list[dict] = [] + + # Batch fetch: if this fails, mark all IDs in the batch as unmapped. + logger.info("Processing batch of %s tax_ids", len(taxon_batch)) + logger.debug("Batch tax_ids: %s", taxon_batch) + try: + reports = fetch_reports(taxon_batch, endpoint="taxonomy") + except RuntimeError as exc: + logger.error("Batch fetch failed for tax_ids %s: %s", taxon_batch, exc) + for tax_id in taxon_batch: + unmapped_batch.append( + { + "taxon_id": tax_id, + "raw_scientific_name": raw_name_by_taxid.get(tax_id), + "error": str(exc), + } + ) + return mapped_batch, unmapped_batch + + received_query_taxids: set[int | str] = set() + for report in reports: + query_taxid = get_report_query_taxid(report) + if query_taxid is not None: + received_query_taxids.add(query_taxid) + + extracted = extract_taxonomy_fields(report) # query taxid + if not has_useful_taxonomy_data(extracted): + logger.error("Unexpected response structure for tax_id %s", query_taxid) + unmapped_batch.append( + { + "taxon_id": query_taxid, + "raw_scientific_name": raw_name_by_taxid.get(query_taxid), + "error": "unexpected response structure", + } + ) + continue + + if extracted.get("taxon_id") is not None: + extracted["supplied_taxon_id"] = query_taxid + extracted["supplied_name"] = raw_name_by_taxid.get(query_taxid) + logger.debug("Mapped tax_id %s", extracted.get("taxon_id")) + mapped_batch.append(extracted) + continue + + error_reason = (((report.get("errors") or [{}])[0]).get("reason")) or "missing taxonomy" + logger.warning("Tax_id %s returned error payload: %s", query_taxid, error_reason) + unmapped_batch.append( + { + "taxon_id": query_taxid, + "raw_scientific_name": raw_name_by_taxid.get(query_taxid), + "error": error_reason, + } + ) + + # If API did not return a report for a requested ID, mark it explicitly. + missing_taxids = [tax_id for tax_id in taxon_batch if tax_id not in received_query_taxids] + for tax_id in missing_taxids: + logger.warning("No report returned for requested tax_id %s", tax_id) + unmapped_batch.append( + { + "taxon_id": tax_id, + "raw_scientific_name": raw_name_by_taxid.get(tax_id), + "error": "no report returned", + } + ) + + return mapped_batch, unmapped_batch + + def process_taxa( + input_taxa: list[dict], *, batch_size: int = 20 + ) -> tuple[list[dict], list[dict]]: + """Return (mapped_taxa, unmapped_taxa).""" + mapped: list[dict] = [] + unmapped: list[dict] = [] + + logger.info("Starting taxonomy processing for %s input records", len(input_taxa)) + unique_taxids, raw_name_by_taxid, pre_collected_unmapped = prepare_unique_taxa(input_taxa) + unmapped.extend(pre_collected_unmapped) + logger.info( + "Prepared %s unique tax_ids (%s records pre-marked unmapped)", + len(unique_taxids), + len(pre_collected_unmapped), + ) + + # Second pass: fetch and process in bulk batches. + for taxon_batch in chunked(unique_taxids, batch_size): + mapped_batch, unmapped_batch = process_batch(taxon_batch, raw_name_by_taxid) + mapped.extend(mapped_batch) + unmapped.extend(unmapped_batch) + + logger.info("Finished processing: %s mapped, %s unmapped", len(mapped), len(unmapped)) + return mapped, unmapped + + def prepare_unique_parents( + input_parents: list[int], + ) -> list[int | str]: + """Return (unique_parent_taxids).""" + unique_taxids: list[int | str] = [] + seen_taxids: set[int | str] = set() + + # First pass: preserve input order for unique IDs. + for taxid in input_parents: + if taxid is None: + logger.error("Skipping record with missing tax_id %s", taxid) + continue + + if taxid in seen_taxids: + logger.debug("Skipping duplicate tax_id %s", taxid) + continue + + seen_taxids.add(taxid) + unique_taxids.append(taxid) + + return unique_taxids + + def process_parent_batch( + taxon_batch: list[int | str], + ) -> tuple[list[dict], list[dict]]: + mapped_batch: dict[int:str] + unmapped_batch: list[dict] = [] + + mapped_batch = {} + + # Batch fetch: if this fails, mark all IDs in the batch as unmapped. + logger.info("Processing batch of %s tax_ids", len(taxon_batch)) + # logger.debug("Batch tax_ids: %s", taxon_batch) + try: + reports = fetch_reports(taxon_batch, endpoint="taxonomy") + except RuntimeError as exc: + logger.error("Batch fetch failed for tax_ids %s: %s", taxon_batch, exc) + for tax_id in taxon_batch: + unmapped_batch.append( + { + "taxon_id": tax_id, + "error": str(exc), + } + ) + return mapped_batch, unmapped_batch + + received_query_taxids: set[int | str] = set() + for report in reports: + query_taxid = get_report_query_taxid(report) + if query_taxid is not None: + received_query_taxids.add(query_taxid) + + sci_name = extract_sci_name(report) + if not sci_name: + logger.error("Unexpected response structure for tax_id %s", query_taxid) + unmapped_batch.append( + { + "taxon_id": query_taxid, + "error": "unexpected response structure", + } + ) + continue + + if sci_name is not None: + # logger.debug("Mapped taxon %s", sci_name) + mapped_batch.update(sci_name) + continue + + error_reason = (((report.get("errors") or [{}])[0]).get("reason")) or "missing taxonomy" + logger.warning("Tax_id %s returned error payload: %s", query_taxid, error_reason) + unmapped_batch.append( + { + "taxon_id": query_taxid, + "error": error_reason, + } + ) + # If API did not return a report for a requested ID, mark it explicitly. + missing_taxids = [tax_id for tax_id in taxon_batch if tax_id not in received_query_taxids] + for tax_id in missing_taxids: + logger.warning("No report returned for requested tax_id %s", tax_id) + unmapped_batch.append( + { + "taxon_id": tax_id, + "error": "no report returned", + } + ) + + return mapped_batch, unmapped_batch + + def extract_sci_name(report: dict) -> dict: + taxonomy = report.get("taxonomy") or {} + tax_id = taxonomy.get("tax_id") + sci_name = (taxonomy.get("current_scientific_name") or {}).get("name") + return {tax_id: sci_name} + + def process_parents(parent_taxids: list[int]) -> str: + ordered_parents = [] + + unique_parents = prepare_unique_parents(parent_taxids) + no_of_parents = len(unique_parents) + + for taxon_batch in chunked(unique_parents, no_of_parents): + mapped_parents, unmapped_parents = process_parent_batch(taxon_batch) + + for parent in unique_parents: + if parent == 1 or parent == 131567: + continue # do not append taxon "root" or "cellular organisms" + ordered_parent = mapped_parents.get(parent) + ordered_parents.append(ordered_parent) + + string_lineage = "; ".join(ordered_parents) + + logger.debug(f"Mapped parents = {mapped_parents}") + logger.debug(f"Unmapped parents = {unmapped_parents}") + return string_lineage + + def get_taxon_w_org(reports: list[dict], organelle_type: str, taxid: int) -> str | None: + """ + Checks API ouptut to determine if an organelle sequence is available for a species within that taxonomic rank. + If so, returns the species name. + """ + taxid_w_mito = {} + for report in reports: + if report.get("description") == organelle_type: + organism_info = report.get("organism", []) + scientific_name = organism_info.get("organism_name") + if organelle_type == "Mitochondrion": + taxid_w_mito[taxid] = scientific_name + related_mitos.update(taxid_w_mito) + return scientific_name + return None + + def organelle_ref_lookup( + taxid_lineage: list[int], species_taxid: int, organelle_type: str + ) -> str | None: + """ + Takes the taxonomic lineage and iterates in reverse order to find the lowest rank at which an organelle assembly is available. + """ + rev_order_lineage = taxid_lineage[::-1] + full_rev_lineage = rev_order_lineage.insert(0, species_taxid) + # is_plant = 33090 in taxid_lineage + + for taxid in rev_order_lineage: + if organelle_type == "Mitochondrion" and taxid in related_mitos: + logger.debug(f"{taxid} found in the saved list") + return related_mitos.get(taxid) + if organelle_type == "Mitochondrion" and taxid in no_mitos: + logger.debug(f"{taxid} found in the unsaved list") + continue + try: + reports = fetch_reports([taxid], endpoint="organelle") + except RuntimeError as exc: + logger.error("Fetch failed for tax_id %s: %s", taxid, exc) + if len(reports) == 0: + logger.debug(f"No {organelle_type} found for {taxid}") + no_mitos.append(taxid) + else: + taxon_check = get_taxon_w_org(reports, organelle_type, taxid) + if taxon_check is not None: + return taxon_check + + return None + + +ncbi_taxonomy_service = NcbiTaxonomyService() diff --git a/app/services/organism_service.py b/app/services/organism_service.py index 9160f84..99372e4 100644 --- a/app/services/organism_service.py +++ b/app/services/organism_service.py @@ -32,13 +32,13 @@ def get_multi_with_filters( *, skip: int = 0, limit: int = 100, - scientific_name: Optional[str] = None, + bpa_scientific_name: Optional[str] = None, taxon_id: Optional[int] = None, ) -> List[Organism]: """Get organisms with filters.""" query = db.query(Organism) - if scientific_name: - query = query.filter(Organism.scientific_name.ilike(f"%{scientific_name}%")) + if bpa_scientific_name: + query = query.filter(Organism.bpa_scientific_name.ilike(f"%{scientific_name}%")) if taxon_id is not None: query = query.filter(Organism.taxon_id == taxon_id) return query.offset(skip).limit(limit).all() @@ -120,7 +120,7 @@ def get_organism_prepared_payload( response = OrganismSubmissionJsonResponse( taxon_id=organism.taxon_id, - scientific_name=organism.scientific_name, + bpa_scientific_name=organism.bpa_scientific_name, samples=[], experiments=[], reads=[], @@ -166,23 +166,16 @@ def get_organism_prepared_payload( def create_organism(self, db: Session, *, organism_in: OrganismCreate) -> Organism: """Create a new organism and draft projects + submissions.""" - organism_label = organism_in.scientific_name or str(organism_in.taxon_id) + organism_label = organism_in.bpa_scientific_name or str(organism_in.taxon_id) organism = Organism( taxon_id=organism_in.taxon_id, - scientific_name=organism_in.scientific_name, - common_name=organism_in.common_name, - common_name_source=organism_in.common_name_source, - genus=organism_in.genus, - species=organism_in.species, - infraspecific_epithet=organism_in.infraspecific_epithet, - culture_or_strain_id=organism_in.culture_or_strain_id, - authority=organism_in.authority, - atol_scientific_name=organism_in.atol_scientific_name, - tax_string=organism_in.tax_string, - ncbi_order=organism_in.ncbi_order, - ncbi_family=organism_in.ncbi_family, - busco_dataset_name=organism_in.busco_dataset_name, - taxonomy_lineage_json=organism_in.taxonomy_lineage_json, + bpa_scientific_name=organism_in.bpa_scientific_name, + bpa_common_name=organism_in.bpa_common_name, + bpa_genus=organism_in.bpa_genus, + bpa_species=organism_in.bpa_species, + bpa_infraspecific_epithet=organism_in.bpa_infraspecific_epithet, + bpa_culture_or_strain_id=organism_in.bpa_culture_or_strain_id, + bpa_authority=organism_in.bpa_authority, bpa_json=organism_in.model_dump(exclude_unset=True), ) db.add(organism) @@ -328,23 +321,19 @@ def bulk_import_organisms( skipped_count += 1 continue - scientific_name = organism_data.get("scientific_name") + scientific_name = organism_data.get("bpa_scientific_name") organism_label = scientific_name or str(taxon_id) # Create organism and projects organism = Organism( taxon_id=taxon_id, - scientific_name=scientific_name, - genus=organism_data.get("genus"), - species=organism_data.get("species"), - infraspecific_epithet=organism_data.get("infraspecific_epithet"), - culture_or_strain_id=organism_data.get("culture_or_strain_id"), - authority=organism_data.get("authority"), - atol_scientific_name=organism_data.get("atol_scientific_name"), - tax_string=organism_data.get("tax_string"), - ncbi_order=organism_data.get("ncbi_order"), - ncbi_family=organism_data.get("ncbi_family"), - busco_dataset_name=organism_data.get("busco_dataset_name"), + bpa_common_name=organism_data.get("bpa_common_name"), + bpa_genus=organism_data.get("bpa_genus"), + bpa_species=organism_data.get("bpa_species"), + bpa_infraspecific_epithet=organism_data.get("bpa_infraspecific_epithet"), + bpa_culture_or_strain_id=organism_data.get("bpa_culture_or_strain_id"), + bpa_authority=organism_data.get("bpa_authority"), + bpa_scientific_name=organism_data.get("bpa_scientific_name"), bpa_json=organism_data, ) root_project = Project( diff --git a/app/services/taxonomy_info_service.py b/app/services/taxonomy_info_service.py index 8990109..a15aa69 100644 --- a/app/services/taxonomy_info_service.py +++ b/app/services/taxonomy_info_service.py @@ -6,6 +6,7 @@ from app.models.taxonomy_info import TaxonomyInfo from app.schemas.bulk_import import BulkImportResponse from app.schemas.taxonomy_info import TaxonomyInfoCreate, TaxonomyInfoUpdate +from app.services.ncbi_taxonomy_service import ncbi_taxonomy_service class TaxonomyInfoService: @@ -21,12 +22,19 @@ def create(self, db: Session, *, ti_in: TaxonomyInfoCreate) -> TaxonomyInfo: organism = db.query(Organism).filter(Organism.taxon_id == ti_in.taxon_id).first() if not organism: raise ValueError(f"Organism with taxon_id {ti_in.taxon_id} does not exist") - existing = db.query(TaxonomyInfo).filter(TaxonomyInfo.taxon_id == ti_in.taxon_id).first() if existing: raise ValueError(f"TaxonomyInfo for taxon_id {ti_in.taxon_id} already exists") ti = TaxonomyInfo(**ti_in.model_dump()) + ncbi_taxon_service = NCBITaxonomyService() + mapped, unmapped = ncbi_taxonomy_service.process_taxa(ti_in.taxon_id) + if not mapped: + raise ValueError(f"No mapped lineage found for taxon_id {ti_in.taxon_id}") + for field, value in mapped[0].items(): + setattr(ti, field, value) + + # TODO: Map the lineage to the taxonomy info db.add(ti) db.commit() db.refresh(ti) @@ -53,6 +61,13 @@ def delete(self, db: Session, *, taxon_id: int) -> Optional[TaxonomyInfo]: db.commit() return ti + """def upsert_info_from_ncbi(self, db: Session, *, taxon_id: int) -> Optional[TaxonomyInfo]: + organism = db.query(Organism).filter(Organism.taxon_id == taxon_id).first() + if not organism: + raise ValueError(f"Organism with taxon_id {taxon_id} does not exist") + + if """ + def bulk_import(self, db: Session, *, data: Dict[str, Dict[str, Any]]) -> BulkImportResponse: created_count = 0 skipped_count = 0 From d4cc0bd2d6892b8724a6032c04b67007658527a1 Mon Sep 17 00:00:00 2001 From: Emily Marshall Date: Mon, 18 May 2026 00:32:59 +1000 Subject: [PATCH 05/14] feat: fix ncbi lookup logic, add ncbi lookup to bulk-insert ops, add tests --- app/api/v1/endpoints/taxonomy_info.py | 8 +- app/schemas/bulk_import.py | 8 +- app/schemas/taxonomy_info.py | 20 +- app/services/ncbi_taxonomy_service.py | 785 +++++++++--------- app/services/organism_service.py | 12 +- app/services/taxonomy_info_service.py | 170 ++-- .../endpoints/test_endpoints_taxonomy_info.py | 37 +- .../services/test_taxonomy_info_service.py | 202 +++++ 8 files changed, 766 insertions(+), 476 deletions(-) create mode 100644 tests/unit/services/test_taxonomy_info_service.py diff --git a/app/api/v1/endpoints/taxonomy_info.py b/app/api/v1/endpoints/taxonomy_info.py index 68fe4b2..8639694 100644 --- a/app/api/v1/endpoints/taxonomy_info.py +++ b/app/api/v1/endpoints/taxonomy_info.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List +from typing import Any, List from fastapi import APIRouter, Depends, HTTPException, status from sqlalchemy.orm import Session @@ -8,7 +8,7 @@ from app.core.policy import policy from app.models.taxonomy_info import TaxonomyInfo from app.models.user import User -from app.schemas.bulk_import import BulkImportResponse +from app.schemas.bulk_import import BulkImportResponse, BulkTaxonomyInfoImport from app.schemas.taxonomy_info import ( TaxonomyInfo as TaxonomyInfoSchema, ) @@ -38,7 +38,7 @@ def list_taxonomy_info( def bulk_import_taxonomy_info( *, db: Session = Depends(get_db), - data: Dict[str, Dict[str, Any]], + data: BulkTaxonomyInfoImport, current_user: User = Depends(get_current_active_user), ) -> Any: """ @@ -47,7 +47,7 @@ def bulk_import_taxonomy_info( Insert-only — existing rows are skipped and reported as errors. The taxon_id key must reference an existing organism. """ - return taxonomy_info_service.bulk_import(db, data=data) + return taxonomy_info_service.bulk_import(db, data=data.root) @router.get("/{taxon_id}", response_model=TaxonomyInfoSchema) diff --git a/app/schemas/bulk_import.py b/app/schemas/bulk_import.py index 20afeab..7e99051 100644 --- a/app/schemas/bulk_import.py +++ b/app/schemas/bulk_import.py @@ -1,6 +1,8 @@ from typing import Any, Dict, List, Optional -from pydantic import BaseModel +from pydantic import BaseModel, RootModel + +from app.schemas.taxonomy_info import TaxonomyInfoUpdate class BulkOrganismImport(BaseModel): @@ -36,6 +38,10 @@ class BulkExperimentImport(BaseModel): experiments: Dict[str, Dict[str, Any]] # Dictionary of experiment data keyed by package_id +class BulkTaxonomyInfoImport(RootModel[Dict[int, TaxonomyInfoUpdate]]): + """Bulk taxonomy info import keyed by taxon_id.""" + + class BulkImportResponse(BaseModel): """Schema for bulk import response.""" diff --git a/app/schemas/taxonomy_info.py b/app/schemas/taxonomy_info.py index bdc67e2..460aec5 100644 --- a/app/schemas/taxonomy_info.py +++ b/app/schemas/taxonomy_info.py @@ -29,11 +29,27 @@ class TaxonomyInfoBase(BaseModel): genetic_code_id: Optional[int] = None -class TaxonomyInfoCreate(TaxonomyInfoBase): +class TaxonomyInfoWriteBase(BaseModel): + busco_dataset_name: Optional[str] = None + busco_odb10_dataset_name: Optional[str] = None + busco_odb12_dataset_name: Optional[str] = None + find_plastid: Optional[bool] = None + hic_motif: Optional[str] = None + mitochondrial_genetic_code_id: Optional[int] = None + mitohifi_reference_species: Optional[str] = None + oatk_hmm_name: Optional[str] = None + defined_class: Optional[str] = None + augustus_dataset_name: Optional[str] = None + genetic_code_id: Optional[int] = None + + model_config = ConfigDict(extra="forbid") + + +class TaxonomyInfoCreate(TaxonomyInfoWriteBase): taxon_id: int -class TaxonomyInfoUpdate(TaxonomyInfoBase): +class TaxonomyInfoUpdate(TaxonomyInfoWriteBase): pass diff --git a/app/services/ncbi_taxonomy_service.py b/app/services/ncbi_taxonomy_service.py index b4666c8..b9865ce 100644 --- a/app/services/ncbi_taxonomy_service.py +++ b/app/services/ncbi_taxonomy_service.py @@ -1,451 +1,428 @@ import logging import time +from typing import Any, Optional import requests logger = logging.getLogger(__name__) +# Adapted from: +# - /Users/emilylm/Repositories/biocommons/tutorial-fetch-ncbi-data/src/taxonomy_fetcher.py +# - /Users/emilylm/Repositories/biocommons/tutorial-fetch-ncbi-data/src/parent_lineage_enrichment.py +related_mitos: dict[int | str, str] = {} +no_mitos: set[int | str] = set() -class NcbiTaxonomyService: - related_mitos = {} - no_mitos = [] - def configure_logging() -> None: - logging.basicConfig( - level=logging.DEBUG, - format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", - ) - - def normalize_tax_id(value: int | str | None) -> int | str | None: - if value is None: - return None - if isinstance(value, int): - return value - stripped = str(value).strip() - if stripped.isdigit(): - return int(stripped) - return stripped - - def chunked(items: list[int | str], size: int) -> list[list[int | str]]: - return [items[i : i + size] for i in range(0, len(items), size)] - - def build_taxonomy_url(tax_ids: list[int | str], endpoint: str) -> str: - taxons = ",".join(str(tax_id) for tax_id in tax_ids) - if endpoint == "taxonomy": - return ( - "https://api.ncbi.nlm.nih.gov/datasets/v2/taxonomy/" - f"taxon/{taxons}/dataset_report" +def normalize_tax_id(value: int | str | None) -> int | str | None: + if value is None: + return None + if isinstance(value, int): + return value + stripped = str(value).strip() + if stripped.isdigit(): + return int(stripped) + return stripped + + +def chunked(items: list[int | str], size: int) -> list[list[int | str]]: + if size <= 0: + raise ValueError("size must be > 0") + return [items[i : i + size] for i in range(0, len(items), size)] + + +def build_taxonomy_url(tax_ids: list[int | str], endpoint: str) -> str: + taxons = ",".join(str(tax_id) for tax_id in tax_ids) + if endpoint == "taxonomy": + return f"https://api.ncbi.nlm.nih.gov/datasets/v2/taxonomy/taxon/{taxons}/dataset_report" + if endpoint == "organelle": + return f"https://api.ncbi.nlm.nih.gov/datasets/v2/organelle/taxon/{taxons}/dataset_report" + raise ValueError(f"Unsupported endpoint: {endpoint}") + + +def fetch_reports( + tax_ids: list[int | str], + endpoint: str, + *, + max_retries: int = 3, + backoff_seconds: int = 1, + timeout_seconds: int = 20, + sleep_fn=time.sleep, +) -> list[dict[str, Any]]: + """Fetch taxonomy report dicts for a batch of tax_ids.""" + url = build_taxonomy_url(tax_ids, endpoint) + logger.info("Fetching NCBI %s reports for tax_ids=%s", endpoint, tax_ids) + + for attempt in range(1, max_retries + 1): + try: + response = requests.get(url, timeout=timeout_seconds) + response.raise_for_status() + payload = response.json() + reports = payload.get("reports", []) + logger.info( + "Fetched %s NCBI %s reports for tax_ids=%s", + len(reports), + endpoint, + tax_ids, ) - elif endpoint == "organelle": - return ( - "https://api.ncbi.nlm.nih.gov/datasets/v2/organelle/" - f"taxon/{taxons}/dataset_report" + return reports + except (requests.RequestException, ValueError, RuntimeError) as exc: + logger.warning( + "NCBI %s fetch failed for tax_ids=%s (attempt %s/%s): %s", + endpoint, + tax_ids, + attempt, + max_retries, + exc, ) + if attempt == max_retries: + raise RuntimeError( + f"Unable to fetch {endpoint} reports for tax_ids {tax_ids} after {max_retries} attempts" + ) from exc + sleep_fn(backoff_seconds * (2 ** (attempt - 1))) + + raise RuntimeError("Unexpected retry loop exit") + + +def build_lineage( + classification: dict[str, Any], + *, + current_rank: str | None, + current_name: str | None, +) -> list[dict[str, str]]: + lineage: list[dict[str, str]] = [] + + if isinstance(classification, dict): + domain_name = (classification.get("domain") or {}).get("name") + if domain_name: + lineage.append({"rank": "domain", "name": domain_name}) + for rank, node in classification.items(): + if not isinstance(node, dict): + continue + if rank == "domain": + continue + name = node.get("name") + if not name: + continue + lineage.append({"rank": str(rank), "name": name}) - def fetch_reports( - tax_ids: list[int | str], - endpoint: str, - *, - max_retries: int = 3, - backoff_seconds: int = 1, - timeout_seconds: int = 20, - sleep_fn=time.sleep, - ) -> list[dict]: - """Fetch taxonomy report dicts for a batch of tax_ids. - - Raises RuntimeError if request fails after retries. - """ - url = build_taxonomy_url(tax_ids, endpoint) - - for attempt in range(1, max_retries + 1): - try: - response = requests.get(url, timeout=timeout_seconds) - response.raise_for_status() - payload = response.json() - reports = payload.get("reports", []) - return reports - except (requests.RequestException, ValueError, RuntimeError) as exc: - logger.warning( - "Bulk fetch failed for tax_ids %s (attempt %s/%s): %s", - tax_ids, - attempt, - max_retries, - exc, - ) - if attempt == max_retries: - raise RuntimeError( - "Unable to fetch taxonomy for tax_ids " - f"{tax_ids} after {max_retries} attempts" - ) from exc - sleep_fn(backoff_seconds * (2 ** (attempt - 1))) - - raise RuntimeError("Unexpected retry loop exit") - - def extract_taxonomy_fields(report: dict) -> dict: - """Extract required taxonomy fields from an NCBI report object.""" - taxonomy = report.get("taxonomy") or {} - classification = taxonomy.get("classification") or {} - current_rank = taxonomy.get("rank") - current_name = (taxonomy.get("current_scientific_name") or {}).get("name") - lineage = build_lineage( - classification, current_rank=current_rank, current_name=current_name - ) + if current_name: + current_rank_label = str(current_rank).lower() if current_rank else "current" + if not lineage or lineage[-1]["name"] != current_name: + lineage.append({"rank": current_rank_label, "name": current_name}) - return { - "taxon_id": taxonomy.get("tax_id"), - "ncbi_rank": current_rank, - "ncbi_scientific_name": current_name, - "ncbi_authority": (taxonomy.get("current_scientific_name") or {}).get("authority"), - "ncbi_common_name": taxonomy.get("curator_common_name"), - "ncbi_class": (classification.get("class") or {}).get("name"), - "ncbi_order": (classification.get("order") or {}).get("name"), - "ncbi_family": (classification.get("family") or {}).get("name"), - "ncbi_lineage": lineage, - "ncbi_tax_string": lineage_to_string(lineage), - "ncbi_full_lineage": process_parents(taxonomy.get("parents")), - "mito_ref": organelle_ref_lookup( - taxonomy.get("parents"), taxonomy.get("tax_id"), organelle_type="Mitochondrion" - ), - } - - def has_useful_taxonomy_data(extracted: dict) -> bool: - """Return True if at least one expected extracted value is present.""" - return any(value is not None for value in extracted.values()) - - def build_lineage( - classification: dict, - *, - current_rank: str | None, - current_name: str | None, - ) -> list[dict]: - """Build ordered lineage as rank/name objects. - - We keep API-provided classification order, which is largest -> smallest in NCBI payloads. - """ - lineage: list[dict] = [] - - if isinstance(classification, dict): - domain_name = classification.get("domain", {}).get("name") - if domain_name: - lineage.append({"rank": "domain", "name": domain_name}) - for rank, node in classification.items(): - if not isinstance(node, dict): - continue - name = node.get("name") - if not name: - continue - if rank == "domain": - continue - lineage.append({"rank": str(rank), "name": name}) - - if current_name: - current_rank_label = str(current_rank).lower() if current_rank else "current" - if not lineage or lineage[-1]["name"] != current_name: - lineage.append({"rank": current_rank_label, "name": current_name}) - - return lineage - - def lineage_to_string(lineage: list[dict]) -> str | None: - names = [item.get("name") for item in lineage if item.get("name")] - return "; ".join(names) if names else None - - def get_report_query_taxid(report: dict) -> int | str | None: - query = report.get("query") or [] - if not query: - return None - return normalize_tax_id(query[0]) - - def prepare_unique_taxa( - input_taxa: list[dict], - ) -> tuple[list[int | str], dict[int | str, str | None], list[dict]]: - """Return (unique_taxids, raw_name_by_taxid, pre_collected_unmapped).""" - unique_taxids: list[int | str] = [] - raw_name_by_taxid: dict[int | str, str | None] = {} - pre_collected_unmapped: list[dict] = [] - seen_taxids: set[int | str] = set() - - # First pass: normalize IDs, record missing IDs, and preserve input order for unique IDs. - for item in input_taxa: - tax_id = normalize_tax_id(item.get("tax_id")) - raw_name = item.get("scientific_name") - - if tax_id is None: - logger.error("Skipping record with missing tax_id (scientific_name=%s)", raw_name) - pre_collected_unmapped.append( - {"taxon_id": None, "raw_scientific_name": raw_name, "error": "missing_tax_id"} - ) - continue + return lineage - if tax_id in seen_taxids: - logger.debug("Skipping duplicate tax_id %s", tax_id) - continue - seen_taxids.add(tax_id) - unique_taxids.append(tax_id) - raw_name_by_taxid[tax_id] = raw_name +def lineage_to_string(lineage: list[dict[str, str]]) -> str | None: + names = [item.get("name") for item in lineage if item.get("name")] + return "; ".join(names) if names else None - return unique_taxids, raw_name_by_taxid, pre_collected_unmapped - def process_batch( - taxon_batch: list[int | str], - raw_name_by_taxid: dict[int | str, str | None], - ) -> tuple[list[dict], list[dict]]: - mapped_batch: list[dict] = [] - unmapped_batch: list[dict] = [] +def get_report_query_taxid(report: dict[str, Any]) -> int | str | None: + query = report.get("query") or [] + if not query: + return None + return normalize_tax_id(query[0]) - # Batch fetch: if this fails, mark all IDs in the batch as unmapped. - logger.info("Processing batch of %s tax_ids", len(taxon_batch)) - logger.debug("Batch tax_ids: %s", taxon_batch) - try: - reports = fetch_reports(taxon_batch, endpoint="taxonomy") - except RuntimeError as exc: - logger.error("Batch fetch failed for tax_ids %s: %s", taxon_batch, exc) - for tax_id in taxon_batch: - unmapped_batch.append( - { - "taxon_id": tax_id, - "raw_scientific_name": raw_name_by_taxid.get(tax_id), - "error": str(exc), - } - ) - return mapped_batch, unmapped_batch - - received_query_taxids: set[int | str] = set() - for report in reports: - query_taxid = get_report_query_taxid(report) - if query_taxid is not None: - received_query_taxids.add(query_taxid) - - extracted = extract_taxonomy_fields(report) # query taxid - if not has_useful_taxonomy_data(extracted): - logger.error("Unexpected response structure for tax_id %s", query_taxid) - unmapped_batch.append( - { - "taxon_id": query_taxid, - "raw_scientific_name": raw_name_by_taxid.get(query_taxid), - "error": "unexpected response structure", - } - ) - continue - if extracted.get("taxon_id") is not None: - extracted["supplied_taxon_id"] = query_taxid - extracted["supplied_name"] = raw_name_by_taxid.get(query_taxid) - logger.debug("Mapped tax_id %s", extracted.get("taxon_id")) - mapped_batch.append(extracted) - continue +def prepare_unique_taxa( + input_taxa: list[dict[str, Any]], +) -> tuple[list[int | str], dict[int | str, str | None], list[dict[str, Any]]]: + unique_taxids: list[int | str] = [] + raw_name_by_taxid: dict[int | str, str | None] = {} + pre_collected_unmapped: list[dict[str, Any]] = [] + seen_taxids: set[int | str] = set() - error_reason = (((report.get("errors") or [{}])[0]).get("reason")) or "missing taxonomy" - logger.warning("Tax_id %s returned error payload: %s", query_taxid, error_reason) - unmapped_batch.append( - { - "taxon_id": query_taxid, - "raw_scientific_name": raw_name_by_taxid.get(query_taxid), - "error": error_reason, - } - ) + for item in input_taxa: + tax_id = normalize_tax_id(item.get("tax_id")) + raw_name = item.get("scientific_name") - # If API did not return a report for a requested ID, mark it explicitly. - missing_taxids = [tax_id for tax_id in taxon_batch if tax_id not in received_query_taxids] - for tax_id in missing_taxids: - logger.warning("No report returned for requested tax_id %s", tax_id) - unmapped_batch.append( - { - "taxon_id": tax_id, - "raw_scientific_name": raw_name_by_taxid.get(tax_id), - "error": "no report returned", - } + if tax_id is None: + logger.error("Skipping record with missing tax_id (scientific_name=%s)", raw_name) + pre_collected_unmapped.append( + {"taxon_id": None, "raw_scientific_name": raw_name, "error": "missing_tax_id"} ) + continue - return mapped_batch, unmapped_batch + if tax_id in seen_taxids: + logger.debug("Skipping duplicate tax_id %s", tax_id) + continue - def process_taxa( - input_taxa: list[dict], *, batch_size: int = 20 - ) -> tuple[list[dict], list[dict]]: - """Return (mapped_taxa, unmapped_taxa).""" - mapped: list[dict] = [] - unmapped: list[dict] = [] - - logger.info("Starting taxonomy processing for %s input records", len(input_taxa)) - unique_taxids, raw_name_by_taxid, pre_collected_unmapped = prepare_unique_taxa(input_taxa) - unmapped.extend(pre_collected_unmapped) - logger.info( - "Prepared %s unique tax_ids (%s records pre-marked unmapped)", - len(unique_taxids), - len(pre_collected_unmapped), - ) + seen_taxids.add(tax_id) + unique_taxids.append(tax_id) + raw_name_by_taxid[tax_id] = raw_name - # Second pass: fetch and process in bulk batches. - for taxon_batch in chunked(unique_taxids, batch_size): - mapped_batch, unmapped_batch = process_batch(taxon_batch, raw_name_by_taxid) - mapped.extend(mapped_batch) - unmapped.extend(unmapped_batch) - - logger.info("Finished processing: %s mapped, %s unmapped", len(mapped), len(unmapped)) - return mapped, unmapped - - def prepare_unique_parents( - input_parents: list[int], - ) -> list[int | str]: - """Return (unique_parent_taxids).""" - unique_taxids: list[int | str] = [] - seen_taxids: set[int | str] = set() - - # First pass: preserve input order for unique IDs. - for taxid in input_parents: - if taxid is None: - logger.error("Skipping record with missing tax_id %s", taxid) - continue + return unique_taxids, raw_name_by_taxid, pre_collected_unmapped - if taxid in seen_taxids: - logger.debug("Skipping duplicate tax_id %s", taxid) - continue - seen_taxids.add(taxid) - unique_taxids.append(taxid) +def has_useful_taxonomy_data(extracted: dict[str, Any]) -> bool: + return any(value is not None for value in extracted.values()) - return unique_taxids - def process_parent_batch( - taxon_batch: list[int | str], - ) -> tuple[list[dict], list[dict]]: - mapped_batch: dict[int:str] - unmapped_batch: list[dict] = [] +def prepare_unique_parents(input_parents: list[int] | None) -> list[int | str]: + unique_taxids: list[int | str] = [] + seen_taxids: set[int | str] = set() - mapped_batch = {} + for taxid in input_parents or []: + if taxid is None or taxid in seen_taxids: + continue + seen_taxids.add(taxid) + unique_taxids.append(taxid) - # Batch fetch: if this fails, mark all IDs in the batch as unmapped. - logger.info("Processing batch of %s tax_ids", len(taxon_batch)) - # logger.debug("Batch tax_ids: %s", taxon_batch) - try: - reports = fetch_reports(taxon_batch, endpoint="taxonomy") - except RuntimeError as exc: - logger.error("Batch fetch failed for tax_ids %s: %s", taxon_batch, exc) - for tax_id in taxon_batch: - unmapped_batch.append( - { - "taxon_id": tax_id, - "error": str(exc), - } - ) - return mapped_batch, unmapped_batch - - received_query_taxids: set[int | str] = set() - for report in reports: - query_taxid = get_report_query_taxid(report) - if query_taxid is not None: - received_query_taxids.add(query_taxid) - - sci_name = extract_sci_name(report) - if not sci_name: - logger.error("Unexpected response structure for tax_id %s", query_taxid) - unmapped_batch.append( - { - "taxon_id": query_taxid, - "error": "unexpected response structure", - } - ) - continue + return unique_taxids - if sci_name is not None: - # logger.debug("Mapped taxon %s", sci_name) - mapped_batch.update(sci_name) - continue - error_reason = (((report.get("errors") or [{}])[0]).get("reason")) or "missing taxonomy" - logger.warning("Tax_id %s returned error payload: %s", query_taxid, error_reason) - unmapped_batch.append( - { - "taxon_id": query_taxid, - "error": error_reason, - } - ) - # If API did not return a report for a requested ID, mark it explicitly. - missing_taxids = [tax_id for tax_id in taxon_batch if tax_id not in received_query_taxids] - for tax_id in missing_taxids: - logger.warning("No report returned for requested tax_id %s", tax_id) - unmapped_batch.append( - { - "taxon_id": tax_id, - "error": "no report returned", - } - ) +def extract_sci_name(report: dict[str, Any]) -> dict[int, str]: + taxonomy = report.get("taxonomy") or {} + tax_id = taxonomy.get("tax_id") + sci_name = (taxonomy.get("current_scientific_name") or {}).get("name") + return {tax_id: sci_name} + + +def process_parent_batch( + taxon_batch: list[int | str], +) -> tuple[dict[int, str], list[dict[str, Any]]]: + mapped_batch: dict[int, str] = {} + unmapped_batch: list[dict[str, Any]] = [] + logger.info("Processing NCBI parent batch of %s tax_ids", len(taxon_batch)) + try: + reports = fetch_reports(taxon_batch, endpoint="taxonomy") + except RuntimeError as exc: + logger.error("NCBI parent batch fetch failed for tax_ids=%s: %s", taxon_batch, exc) + for tax_id in taxon_batch: + unmapped_batch.append({"taxon_id": tax_id, "error": str(exc)}) return mapped_batch, unmapped_batch - def extract_sci_name(report: dict) -> dict: - taxonomy = report.get("taxonomy") or {} - tax_id = taxonomy.get("tax_id") - sci_name = (taxonomy.get("current_scientific_name") or {}).get("name") - return {tax_id: sci_name} + received_query_taxids: set[int | str] = set() + for report in reports: + query_taxid = get_report_query_taxid(report) + if query_taxid is not None: + received_query_taxids.add(query_taxid) - def process_parents(parent_taxids: list[int]) -> str: - ordered_parents = [] + sci_name = extract_sci_name(report) + if not sci_name: + unmapped_batch.append( + {"taxon_id": query_taxid, "error": "unexpected response structure"} + ) + continue + mapped_batch.update(sci_name) - unique_parents = prepare_unique_parents(parent_taxids) - no_of_parents = len(unique_parents) + missing_taxids = [tax_id for tax_id in taxon_batch if tax_id not in received_query_taxids] + for tax_id in missing_taxids: + unmapped_batch.append({"taxon_id": tax_id, "error": "no report returned"}) - for taxon_batch in chunked(unique_parents, no_of_parents): - mapped_parents, unmapped_parents = process_parent_batch(taxon_batch) + return mapped_batch, unmapped_batch - for parent in unique_parents: - if parent == 1 or parent == 131567: - continue # do not append taxon "root" or "cellular organisms" - ordered_parent = mapped_parents.get(parent) - ordered_parents.append(ordered_parent) - string_lineage = "; ".join(ordered_parents) - - logger.debug(f"Mapped parents = {mapped_parents}") - logger.debug(f"Unmapped parents = {unmapped_parents}") - return string_lineage - - def get_taxon_w_org(reports: list[dict], organelle_type: str, taxid: int) -> str | None: - """ - Checks API ouptut to determine if an organelle sequence is available for a species within that taxonomic rank. - If so, returns the species name. - """ - taxid_w_mito = {} - for report in reports: - if report.get("description") == organelle_type: - organism_info = report.get("organism", []) - scientific_name = organism_info.get("organism_name") - if organelle_type == "Mitochondrion": - taxid_w_mito[taxid] = scientific_name - related_mitos.update(taxid_w_mito) - return scientific_name +def process_parents(parent_taxids: list[int] | None) -> str | None: + unique_parents = prepare_unique_parents(parent_taxids) + if not unique_parents: return None - def organelle_ref_lookup( - taxid_lineage: list[int], species_taxid: int, organelle_type: str - ) -> str | None: - """ - Takes the taxonomic lineage and iterates in reverse order to find the lowest rank at which an organelle assembly is available. - """ - rev_order_lineage = taxid_lineage[::-1] - full_rev_lineage = rev_order_lineage.insert(0, species_taxid) - # is_plant = 33090 in taxid_lineage - - for taxid in rev_order_lineage: - if organelle_type == "Mitochondrion" and taxid in related_mitos: - logger.debug(f"{taxid} found in the saved list") - return related_mitos.get(taxid) - if organelle_type == "Mitochondrion" and taxid in no_mitos: - logger.debug(f"{taxid} found in the unsaved list") - continue - try: - reports = fetch_reports([taxid], endpoint="organelle") - except RuntimeError as exc: - logger.error("Fetch failed for tax_id %s: %s", taxid, exc) - if len(reports) == 0: - logger.debug(f"No {organelle_type} found for {taxid}") - no_mitos.append(taxid) - else: - taxon_check = get_taxon_w_org(reports, organelle_type, taxid) - if taxon_check is not None: - return taxon_check + mapped_parents, unmapped_parents = process_parent_batch(unique_parents) + ordered_parents: list[str] = [] + for parent in unique_parents: + if parent in {1, 131567}: + continue + ordered_parent = mapped_parents.get(parent) + if ordered_parent: + ordered_parents.append(ordered_parent) + logger.debug("Mapped parents=%s unmapped=%s", mapped_parents, unmapped_parents) + return "; ".join(ordered_parents) if ordered_parents else None + + +def get_taxon_w_org( + reports: list[dict[str, Any]], + organelle_type: str, + taxid: int | str, +) -> str | None: + for report in reports: + if report.get("description") != organelle_type: + continue + organism_info = report.get("organism") or {} + scientific_name = organism_info.get("organism_name") + if organelle_type == "Mitochondrion" and scientific_name: + related_mitos[taxid] = scientific_name + return scientific_name + return None + + +def organelle_ref_lookup( + taxid_lineage: list[int] | None, + species_taxid: int | None, + organelle_type: str, +) -> str | None: + if species_taxid is None: return None + rev_order_lineage = list((taxid_lineage or [])[::-1]) + rev_order_lineage.insert(0, species_taxid) + + for taxid in rev_order_lineage: + if organelle_type == "Mitochondrion" and taxid in related_mitos: + return related_mitos.get(taxid) + if organelle_type == "Mitochondrion" and taxid in no_mitos: + continue + + try: + reports = fetch_reports([taxid], endpoint="organelle") + except RuntimeError as exc: + logger.error("Organelle fetch failed for tax_id=%s: %s", taxid, exc) + continue + + if not reports: + no_mitos.add(taxid) + continue + + taxon_check = get_taxon_w_org(reports, organelle_type, taxid) + if taxon_check is not None: + return taxon_check + + return None + + +def extract_taxonomy_fields(report: dict[str, Any]) -> dict[str, Any]: + taxonomy = report.get("taxonomy") or {} + classification = taxonomy.get("classification") or {} + current_rank = taxonomy.get("rank") + current_name = (taxonomy.get("current_scientific_name") or {}).get("name") + lineage = build_lineage(classification, current_rank=current_rank, current_name=current_name) + + return { + "taxon_id": taxonomy.get("tax_id"), + "ncbi_taxon_id": taxonomy.get("tax_id"), + "ncbi_rank": current_rank, + "ncbi_scientific_name": current_name, + "ncbi_authority": (taxonomy.get("current_scientific_name") or {}).get("authority"), + "ncbi_common_name": taxonomy.get("curator_common_name"), + "ncbi_class": (classification.get("class") or {}).get("name"), + "ncbi_order": (classification.get("order") or {}).get("name"), + "ncbi_family": (classification.get("family") or {}).get("name"), + "ncbi_lineage": lineage, + "ncbi_tax_string": lineage_to_string(lineage), + "ncbi_full_lineage": process_parents(taxonomy.get("parents")), + "mito_ref": organelle_ref_lookup( + taxonomy.get("parents"), taxonomy.get("tax_id"), organelle_type="Mitochondrion" + ), + } + + +def process_batch( + taxon_batch: list[int | str], + raw_name_by_taxid: dict[int | str, str | None], +) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + mapped_batch: list[dict[str, Any]] = [] + unmapped_batch: list[dict[str, Any]] = [] + + logger.info("Processing NCBI taxonomy batch of %s tax_ids", len(taxon_batch)) + try: + reports = fetch_reports(taxon_batch, endpoint="taxonomy") + except RuntimeError as exc: + logger.error("NCBI taxonomy batch fetch failed for tax_ids=%s: %s", taxon_batch, exc) + for tax_id in taxon_batch: + unmapped_batch.append( + { + "taxon_id": tax_id, + "raw_scientific_name": raw_name_by_taxid.get(tax_id), + "error": str(exc), + } + ) + return mapped_batch, unmapped_batch + + received_query_taxids: set[int | str] = set() + for report in reports: + query_taxid = get_report_query_taxid(report) + if query_taxid is not None: + received_query_taxids.add(query_taxid) + + extracted = extract_taxonomy_fields(report) + if not has_useful_taxonomy_data(extracted): + unmapped_batch.append( + { + "taxon_id": query_taxid, + "raw_scientific_name": raw_name_by_taxid.get(query_taxid), + "error": "unexpected response structure", + } + ) + continue + + if extracted.get("taxon_id") is not None: + extracted["supplied_taxon_id"] = query_taxid + extracted["supplied_name"] = raw_name_by_taxid.get(query_taxid) + mapped_batch.append(extracted) + logger.info("Mapped NCBI taxonomy for requested taxon_id=%s", query_taxid) + continue + + error_reason = (((report.get("errors") or [{}])[0]).get("reason")) or "missing taxonomy" + unmapped_batch.append( + { + "taxon_id": query_taxid, + "raw_scientific_name": raw_name_by_taxid.get(query_taxid), + "error": error_reason, + } + ) + + missing_taxids = [tax_id for tax_id in taxon_batch if tax_id not in received_query_taxids] + for tax_id in missing_taxids: + unmapped_batch.append( + { + "taxon_id": tax_id, + "raw_scientific_name": raw_name_by_taxid.get(tax_id), + "error": "no report returned", + } + ) -ncbi_taxonomy_service = NcbiTaxonomyService() + return mapped_batch, unmapped_batch + + +def process_taxa( + input_taxa: list[dict[str, Any]], + *, + batch_size: int = 20, +) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + mapped: list[dict[str, Any]] = [] + unmapped: list[dict[str, Any]] = [] + + logger.info("Starting NCBI taxonomy processing for %s input records", len(input_taxa)) + unique_taxids, raw_name_by_taxid, pre_collected_unmapped = prepare_unique_taxa(input_taxa) + unmapped.extend(pre_collected_unmapped) + logger.info( + "Prepared %s unique tax_ids for NCBI processing (%s pre-marked unmapped)", + len(unique_taxids), + len(pre_collected_unmapped), + ) + + for taxon_batch in chunked(unique_taxids, batch_size): + mapped_batch, unmapped_batch = process_batch(taxon_batch, raw_name_by_taxid) + mapped.extend(mapped_batch) + unmapped.extend(unmapped_batch) + + logger.info("Finished NCBI processing: %s mapped, %s unmapped", len(mapped), len(unmapped)) + return mapped, unmapped + + +def fetch_taxonomy_for_taxon_id( + taxon_id: int, + scientific_name: Optional[str] = None, +) -> tuple[Optional[dict[str, Any]], list[dict[str, Any]]]: + input_taxa = [{"tax_id": taxon_id, "scientific_name": scientific_name}] + mapped, unmapped = process_taxa(input_taxa, batch_size=1) + return (mapped[0] if mapped else None), unmapped + + +def fetch_taxonomy_for_taxon_ids( + taxa: dict[int, Optional[str]], + *, + batch_size: int = 20, +) -> tuple[dict[int, dict[str, Any]], list[dict[str, Any]]]: + input_taxa = [ + {"tax_id": taxon_id, "scientific_name": scientific_name} + for taxon_id, scientific_name in taxa.items() + ] + mapped, unmapped = process_taxa(input_taxa, batch_size=batch_size) + mapped_by_taxon_id = { + int(item["taxon_id"]): item for item in mapped if item.get("taxon_id") is not None + } + return mapped_by_taxon_id, unmapped diff --git a/app/services/organism_service.py b/app/services/organism_service.py index 99372e4..98edd9d 100644 --- a/app/services/organism_service.py +++ b/app/services/organism_service.py @@ -1,3 +1,4 @@ +import logging from typing import Any, Dict, List, Optional from uuid import UUID @@ -14,6 +15,8 @@ from app.schemas.organism import OrganismCreate, OrganismUpdate from app.services.base_service import BaseService +logger = logging.getLogger(__name__) + class OrganismService(BaseService[Organism, OrganismCreate, OrganismUpdate]): """Service for Organism operations.""" @@ -167,6 +170,7 @@ def get_organism_prepared_payload( def create_organism(self, db: Session, *, organism_in: OrganismCreate) -> Organism: """Create a new organism and draft projects + submissions.""" organism_label = organism_in.bpa_scientific_name or str(organism_in.taxon_id) + logger.info("Creating organism taxon_id=%s label=%s", organism_in.taxon_id, organism_label) organism = Organism( taxon_id=organism_in.taxon_id, bpa_scientific_name=organism_in.bpa_scientific_name, @@ -250,6 +254,7 @@ def create_organism(self, db: Session, *, organism_in: OrganismCreate) -> Organi db.commit() db.refresh(organism) + logger.info("Created organism taxon_id=%s", organism.taxon_id) return organism def update_organism( @@ -321,7 +326,9 @@ def bulk_import_organisms( skipped_count += 1 continue - scientific_name = organism_data.get("bpa_scientific_name") + scientific_name = organism_data.get("bpa_scientific_name") or organism_data.get( + "scientific_name" + ) organism_label = scientific_name or str(taxon_id) # Create organism and projects @@ -333,7 +340,7 @@ def bulk_import_organisms( bpa_infraspecific_epithet=organism_data.get("bpa_infraspecific_epithet"), bpa_culture_or_strain_id=organism_data.get("bpa_culture_or_strain_id"), bpa_authority=organism_data.get("bpa_authority"), - bpa_scientific_name=organism_data.get("bpa_scientific_name"), + bpa_scientific_name=scientific_name, bpa_json=organism_data, ) root_project = Project( @@ -400,6 +407,7 @@ def bulk_import_organisms( project_id=genomic_data_project.id, prepared_payload=genomic_payload ) ) + db.commit() created_count += 1 except Exception as e: diff --git a/app/services/taxonomy_info_service.py b/app/services/taxonomy_info_service.py index a15aa69..80fbdc2 100644 --- a/app/services/taxonomy_info_service.py +++ b/app/services/taxonomy_info_service.py @@ -1,3 +1,4 @@ +import logging from typing import Any, Dict, List, Optional from sqlalchemy.orm import Session @@ -6,12 +7,35 @@ from app.models.taxonomy_info import TaxonomyInfo from app.schemas.bulk_import import BulkImportResponse from app.schemas.taxonomy_info import TaxonomyInfoCreate, TaxonomyInfoUpdate -from app.services.ncbi_taxonomy_service import ncbi_taxonomy_service +from app.services.ncbi_taxonomy_service import ( + fetch_taxonomy_for_taxon_id, + fetch_taxonomy_for_taxon_ids, +) + +logger = logging.getLogger(__name__) class TaxonomyInfoService: """Service for TaxonomyInfo CRUD and bulk import operations.""" + @staticmethod + def _apply_payload_values(ti: TaxonomyInfo, values: Dict[str, Any]) -> None: + for field, value in values.items(): + if field == "taxon_id": + continue + setattr(ti, field, value) + + @staticmethod + def _apply_ncbi_values(ti: TaxonomyInfo, mapped: Dict[str, Any]) -> List[str]: + mapped_columns = {column.name for column in TaxonomyInfo.__table__.columns} + applied_fields: list[str] = [] + for field, value in mapped.items(): + if field not in mapped_columns or field == "taxon_id": + continue + setattr(ti, field, value) + applied_fields.append(field) + return applied_fields + def get(self, db: Session, taxon_id: int) -> Optional[TaxonomyInfo]: return db.query(TaxonomyInfo).filter(TaxonomyInfo.taxon_id == taxon_id).first() @@ -26,20 +50,65 @@ def create(self, db: Session, *, ti_in: TaxonomyInfoCreate) -> TaxonomyInfo: if existing: raise ValueError(f"TaxonomyInfo for taxon_id {ti_in.taxon_id} already exists") - ti = TaxonomyInfo(**ti_in.model_dump()) - ncbi_taxon_service = NCBITaxonomyService() - mapped, unmapped = ncbi_taxonomy_service.process_taxa(ti_in.taxon_id) - if not mapped: - raise ValueError(f"No mapped lineage found for taxon_id {ti_in.taxon_id}") - for field, value in mapped[0].items(): - setattr(ti, field, value) + ti = self.populate_from_ncbi_lookup( + db, + taxon_id=ti_in.taxon_id, + scientific_name=getattr(organism, "bpa_scientific_name", None), + commit=False, + ) + if ti is None: + ti = TaxonomyInfo(taxon_id=ti_in.taxon_id) + db.add(ti) - # TODO: Map the lineage to the taxonomy info - db.add(ti) + self._apply_payload_values(ti, ti_in.model_dump(exclude_unset=True)) db.commit() db.refresh(ti) return ti + def populate_from_ncbi_lookup( + self, + db: Session, + *, + taxon_id: int, + scientific_name: Optional[str] = None, + commit: bool = False, + ) -> Optional[TaxonomyInfo]: + organism = db.query(Organism).filter(Organism.taxon_id == taxon_id).first() + if not organism: + raise ValueError(f"Organism with taxon_id {taxon_id} does not exist") + + logger.info("Starting NCBI taxonomy enrichment for organism taxon_id=%s", taxon_id) + mapped, unmapped = fetch_taxonomy_for_taxon_id(taxon_id, scientific_name=scientific_name) + if not mapped: + logger.warning( + "NCBI enrichment returned no mapped taxonomy for taxon_id=%s; unmapped=%s", + taxon_id, + unmapped, + ) + return None + + ti = db.query(TaxonomyInfo).filter(TaxonomyInfo.taxon_id == taxon_id).first() + created = False + if not ti: + ti = TaxonomyInfo(taxon_id=taxon_id) + db.add(ti) + created = True + + applied_fields = self._apply_ncbi_values(ti, mapped) + db.flush() + logger.info( + "NCBI taxonomy enrichment %s taxonomy_info for taxon_id=%s; applied_fields=%s", + "created" if created else "updated", + taxon_id, + applied_fields, + ) + + if commit: + db.commit() + db.refresh(ti) + + return ti + def update( self, db: Session, *, taxon_id: int, ti_in: TaxonomyInfoUpdate ) -> Optional[TaxonomyInfo]: @@ -68,62 +137,69 @@ def delete(self, db: Session, *, taxon_id: int) -> Optional[TaxonomyInfo]: if """ - def bulk_import(self, db: Session, *, data: Dict[str, Dict[str, Any]]) -> BulkImportResponse: + def bulk_import( + self, db: Session, *, data: Dict[int, TaxonomyInfoUpdate] + ) -> BulkImportResponse: created_count = 0 skipped_count = 0 errors: List[str] = [] + candidates: List[tuple[int, TaxonomyInfoUpdate, Organism]] = [] - for key, row in data.items(): + for taxon_id, row in data.items(): try: - # Resolve taxon_id from top-level key - try: - taxon_id = int(key) - except (ValueError, TypeError): - errors.append(f"{key}: top-level key is not a valid integer taxon_id") - skipped_count += 1 - continue - - # If inner object also supplies taxon_id it must match the key - inner_taxon_id = row.get("taxon_id") - if inner_taxon_id is not None and int(inner_taxon_id) != taxon_id: - errors.append( - f"{key}: inner taxon_id ({inner_taxon_id}) does not match top-level key" - ) - skipped_count += 1 - continue - - # Validate organism exists organism = db.query(Organism).filter(Organism.taxon_id == taxon_id).first() if not organism: - errors.append(f"{key}: organism with taxon_id {taxon_id} does not exist") + errors.append(f"{taxon_id}: organism with taxon_id {taxon_id} does not exist") skipped_count += 1 continue - # Reject duplicates existing = db.query(TaxonomyInfo).filter(TaxonomyInfo.taxon_id == taxon_id).first() if existing: - errors.append(f"{key}: taxonomy_info for taxon_id {taxon_id} already exists") + errors.append( + f"{taxon_id}: taxonomy_info for taxon_id {taxon_id} already exists" + ) skipped_count += 1 continue + except Exception as e: + errors.append(f"{taxon_id}: {str(e)}") + db.rollback() + skipped_count += 1 + continue + + candidates.append((taxon_id, row, organism)) + + scientific_names_by_taxon_id = { + taxon_id: getattr(organism, "bpa_scientific_name", None) + for taxon_id, _, organism in candidates + } + ncbi_by_taxon_id, unmapped = fetch_taxonomy_for_taxon_ids(scientific_names_by_taxon_id) + if unmapped: + logger.warning("NCBI bulk enrichment returned unmapped taxon_ids: %s", unmapped) - ti = TaxonomyInfo( - taxon_id=taxon_id, - busco_odb10_dataset_name=row.get("busco_odb10_dataset_name"), - busco_odb12_dataset_name=row.get("busco_odb12_dataset_name"), - find_plastid=row.get("find_plastid"), - hic_motif=row.get("hic_motif"), - mitochondrial_genetic_code_id=row.get("mitochondrial_genetic_code_id"), - mitohifi_reference_species=row.get("mitohifi_reference_species"), - oatk_hmm_name=row.get("oatk_hmm_name"), - defined_class=row.get("defined_class"), - augustus_dataset_name=row.get("augustus_dataset_name"), - genetic_code_id=row.get("genetic_code_id"), - ) + for taxon_id, row, _organism in candidates: + try: + ti = TaxonomyInfo(taxon_id=taxon_id) db.add(ti) + + mapped = ncbi_by_taxon_id.get(taxon_id) + if mapped: + applied_fields = self._apply_ncbi_values(ti, mapped) + logger.info( + "NCBI taxonomy enrichment created taxonomy_info for taxon_id=%s during bulk import; applied_fields=%s", + taxon_id, + applied_fields, + ) + else: + logger.warning( + "NCBI enrichment returned no mapped taxonomy during bulk import for taxon_id=%s", + taxon_id, + ) + + self._apply_payload_values(ti, row.model_dump(exclude_unset=True)) db.commit() created_count += 1 except Exception as e: - errors.append(f"{key}: {str(e)}") + errors.append(f"{taxon_id}: {str(e)}") db.rollback() skipped_count += 1 diff --git a/tests/unit/endpoints/test_endpoints_taxonomy_info.py b/tests/unit/endpoints/test_endpoints_taxonomy_info.py index afef115..50fcb27 100644 --- a/tests/unit/endpoints/test_endpoints_taxonomy_info.py +++ b/tests/unit/endpoints/test_endpoints_taxonomy_info.py @@ -195,6 +195,17 @@ def _raise(db, ti_in): assert "already exists" in resp.json()["error"]["message"] +def test_create_taxonomy_info_rejects_ncbi_fields(): + client = TestClient(app) + app.dependency_overrides[ti_module.get_current_active_user] = _override_user + app.dependency_overrides[ti_module.get_db] = _override_db(_FakeSession()) + resp = client.post( + "/api/v1/taxonomy-info/", + json={"taxon_id": 5077, "ncbi_rank": "species"}, + ) + assert resp.status_code == 422 + + # --------------------------------------------------------------------------- # Update # --------------------------------------------------------------------------- @@ -339,26 +350,20 @@ def test_bulk_import_skips_duplicates(monkeypatch): assert any("already exists" in e for e in body["errors"]) -def test_bulk_import_rejects_mismatched_inner_taxon_id(monkeypatch): +def test_bulk_import_rejects_inner_taxon_id_field(monkeypatch): client = TestClient(app) app.dependency_overrides[ti_module.get_current_active_user] = _override_user app.dependency_overrides[ti_module.get_db] = _override_db(_FakeSession()) - result = BulkImportResponse( - created_count=0, - skipped_count=1, - message="TaxonomyInfo import complete. Created: 0, Skipped: 1", - errors=["5077: inner taxon_id (9999) does not match top-level key"], - ) - monkeypatch.setattr( - ti_module, - "taxonomy_info_service", - SimpleNamespace(bulk_import=lambda db, data: result), - ) resp = client.post("/api/v1/taxonomy-info/bulk-import", json={"5077": {"taxon_id": 9999}}) - assert resp.status_code == 200 - body = resp.json() - assert body["skipped_count"] == 1 - assert any("does not match" in e for e in body["errors"]) + assert resp.status_code == 422 + + +def test_bulk_import_rejects_ncbi_fields(): + client = TestClient(app) + app.dependency_overrides[ti_module.get_current_active_user] = _override_user + app.dependency_overrides[ti_module.get_db] = _override_db(_FakeSession()) + resp = client.post("/api/v1/taxonomy-info/bulk-import", json={"5077": {"ncbi_rank": "species"}}) + assert resp.status_code == 422 # --------------------------------------------------------------------------- diff --git a/tests/unit/services/test_taxonomy_info_service.py b/tests/unit/services/test_taxonomy_info_service.py new file mode 100644 index 0000000..d863111 --- /dev/null +++ b/tests/unit/services/test_taxonomy_info_service.py @@ -0,0 +1,202 @@ +import pytest + +from app.models.organism import Organism +from app.models.taxonomy_info import TaxonomyInfo +from app.schemas.bulk_import import BulkTaxonomyInfoImport +from app.schemas.taxonomy_info import TaxonomyInfoCreate +from app.services import taxonomy_info_service as ti_service_module + + +class _Query: + def __init__(self, session, model): + self.session = session + self.model = model + self._taxon_id = None + + def filter(self, *criteria, **_kwargs): + for criterion in criteria: + right = getattr(criterion, "right", None) + value = getattr(right, "value", None) + if value is not None: + self._taxon_id = value + return self + + def first(self): + store = self.session.data.get(self.model, {}) + if self._taxon_id is None: + return next(iter(store.values()), None) + return store.get(self._taxon_id) + + +class _Session: + def __init__(self, data=None): + self.data = data or {} + self.commit_count = 0 + self.flush_count = 0 + self.refresh_count = 0 + + def query(self, model): + return _Query(self, model) + + def add(self, obj): + if isinstance(obj, (Organism, TaxonomyInfo)): + self.data.setdefault(type(obj), {}) + self.data[type(obj)][obj.taxon_id] = obj + + def commit(self): + self.commit_count += 1 + + def flush(self): + self.flush_count += 1 + + def refresh(self, _obj): + self.refresh_count += 1 + + def rollback(self): + pass + + +def test_populate_from_ncbi_lookup_creates_taxonomy_info(monkeypatch): + organism = Organism(taxon_id=5077, bpa_scientific_name="Penicillium") + db = _Session({Organism: {5077: organism}, TaxonomyInfo: {}}) + + monkeypatch.setattr( + ti_service_module, + "fetch_taxonomy_for_taxon_id", + lambda taxon_id, scientific_name=None: ( + { + "taxon_id": taxon_id, + "ncbi_taxon_id": taxon_id, + "ncbi_rank": "species", + "ncbi_scientific_name": "Penicillium test", + "ncbi_order": "Eurotiales", + "mito_ref": "Penicillium chrysogenum", + }, + [], + ), + ) + + ti = ti_service_module.taxonomy_info_service.populate_from_ncbi_lookup( + db, + taxon_id=5077, + scientific_name="Penicillium", + commit=False, + ) + + assert ti is not None + assert ti.taxon_id == 5077 + assert ti.ncbi_taxon_id == 5077 + assert ti.ncbi_rank == "species" + assert ti.ncbi_order == "Eurotiales" + assert ti.mito_ref == "Penicillium chrysogenum" + assert db.data[TaxonomyInfo][5077] is ti + assert db.flush_count == 1 + assert db.commit_count == 0 + + +def test_create_taxonomy_info_fetches_ncbi_and_applies_payload(monkeypatch): + organism = Organism(taxon_id=5303, bpa_scientific_name="Agaricus") + db = _Session({Organism: {5303: organism}, TaxonomyInfo: {}}) + + monkeypatch.setattr( + ti_service_module, + "fetch_taxonomy_for_taxon_id", + lambda taxon_id, scientific_name=None: ( + { + "taxon_id": taxon_id, + "ncbi_taxon_id": taxon_id, + "ncbi_rank": "species", + "ncbi_scientific_name": "Agaricus test", + }, + [], + ), + ) + + ti = ti_service_module.taxonomy_info_service.create( + db, + ti_in=TaxonomyInfoCreate( + taxon_id=5303, + genetic_code_id=11, + augustus_dataset_name="agaricus_aug", + ), + ) + + assert ti.taxon_id == 5303 + assert ti.ncbi_taxon_id == 5303 + assert ti.ncbi_rank == "species" + assert ti.genetic_code_id == 11 + assert ti.augustus_dataset_name == "agaricus_aug" + assert db.commit_count == 1 + assert db.refresh_count == 1 + + +def test_taxonomy_info_create_rejects_ncbi_fields(): + with pytest.raises(Exception): + TaxonomyInfoCreate( + taxon_id=5304, + ncbi_rank="genus", + genetic_code_id=3, + ) + + +def test_bulk_import_batches_ncbi_lookup_and_creates_taxonomy_info(monkeypatch): + organisms = { + 9612: Organism(taxon_id=9612, bpa_scientific_name="Canis lupus"), + 9685: Organism(taxon_id=9685, bpa_scientific_name="Felis catus"), + } + db = _Session({Organism: organisms, TaxonomyInfo: {}}) + calls = [] + + monkeypatch.setattr( + ti_service_module, + "fetch_taxonomy_for_taxon_ids", + lambda taxa, batch_size=20: ( + calls.append((taxa, batch_size)) + or { + 9612: { + "taxon_id": 9612, + "ncbi_taxon_id": 9612, + "ncbi_rank": "species", + "mito_ref": "Canis lupus familiaris", + }, + 9685: { + "taxon_id": 9685, + "ncbi_taxon_id": 9685, + "ncbi_rank": "species", + "mito_ref": "Felis silvestris catus", + }, + }, + [], + ), + ) + + result = ti_service_module.taxonomy_info_service.bulk_import( + db, + data=BulkTaxonomyInfoImport.model_validate( + { + "9612": {"genetic_code_id": 2}, + "9685": {"genetic_code_id": 1}, + } + ).root, + ) + + assert calls == [({9612: "Canis lupus", 9685: "Felis catus"}, 20)] + assert result.created_count == 2 + assert result.skipped_count == 0 + saved_dog = db.data[TaxonomyInfo][9612] + saved_cat = db.data[TaxonomyInfo][9685] + assert saved_dog.ncbi_taxon_id == 9612 + assert saved_dog.ncbi_rank == "species" + assert saved_dog.mito_ref == "Canis lupus familiaris" + assert saved_dog.genetic_code_id == 2 + assert saved_cat.ncbi_taxon_id == 9685 + assert saved_cat.ncbi_rank == "species" + assert saved_cat.mito_ref == "Felis silvestris catus" + assert saved_cat.genetic_code_id == 1 + + +def test_bulk_import_schema_rejects_ncbi_fields(): + with pytest.raises(Exception): + BulkTaxonomyInfoImport.model_validate( + {"9612": {"ncbi_rank": "species", "genetic_code_id": 2}} + ) From 78e3ca462f6b5ee62fed6a1ba2f40611fb235613 Mon Sep 17 00:00:00 2001 From: Emily Marshall Date: Mon, 18 May 2026 01:16:12 +1000 Subject: [PATCH 06/14] feat: remove unnecessary function, remove repeated db lookup --- app/services/ncbi_taxonomy_service.py | 9 --------- app/services/taxonomy_info_service.py | 13 +++++++------ 2 files changed, 7 insertions(+), 15 deletions(-) diff --git a/app/services/ncbi_taxonomy_service.py b/app/services/ncbi_taxonomy_service.py index b9865ce..1b1f484 100644 --- a/app/services/ncbi_taxonomy_service.py +++ b/app/services/ncbi_taxonomy_service.py @@ -403,15 +403,6 @@ def process_taxa( return mapped, unmapped -def fetch_taxonomy_for_taxon_id( - taxon_id: int, - scientific_name: Optional[str] = None, -) -> tuple[Optional[dict[str, Any]], list[dict[str, Any]]]: - input_taxa = [{"tax_id": taxon_id, "scientific_name": scientific_name}] - mapped, unmapped = process_taxa(input_taxa, batch_size=1) - return (mapped[0] if mapped else None), unmapped - - def fetch_taxonomy_for_taxon_ids( taxa: dict[int, Optional[str]], *, diff --git a/app/services/taxonomy_info_service.py b/app/services/taxonomy_info_service.py index 80fbdc2..1477d87 100644 --- a/app/services/taxonomy_info_service.py +++ b/app/services/taxonomy_info_service.py @@ -7,10 +7,7 @@ from app.models.taxonomy_info import TaxonomyInfo from app.schemas.bulk_import import BulkImportResponse from app.schemas.taxonomy_info import TaxonomyInfoCreate, TaxonomyInfoUpdate -from app.services.ncbi_taxonomy_service import ( - fetch_taxonomy_for_taxon_id, - fetch_taxonomy_for_taxon_ids, -) +from app.services.ncbi_taxonomy_service import fetch_taxonomy_for_taxon_ids logger = logging.getLogger(__name__) @@ -54,6 +51,7 @@ def create(self, db: Session, *, ti_in: TaxonomyInfoCreate) -> TaxonomyInfo: db, taxon_id=ti_in.taxon_id, scientific_name=getattr(organism, "bpa_scientific_name", None), + organism=organism, commit=False, ) if ti is None: @@ -71,14 +69,17 @@ def populate_from_ncbi_lookup( *, taxon_id: int, scientific_name: Optional[str] = None, + organism: Optional[Organism] = None, commit: bool = False, ) -> Optional[TaxonomyInfo]: - organism = db.query(Organism).filter(Organism.taxon_id == taxon_id).first() + if organism is None: + organism = db.query(Organism).filter(Organism.taxon_id == taxon_id).first() if not organism: raise ValueError(f"Organism with taxon_id {taxon_id} does not exist") logger.info("Starting NCBI taxonomy enrichment for organism taxon_id=%s", taxon_id) - mapped, unmapped = fetch_taxonomy_for_taxon_id(taxon_id, scientific_name=scientific_name) + mapped_dict, unmapped = fetch_taxonomy_for_taxon_ids({taxon_id: scientific_name}) + mapped = mapped_dict.get(taxon_id) if not mapped: logger.warning( "NCBI enrichment returned no mapped taxonomy for taxon_id=%s; unmapped=%s", From 9b56541c527ef19d1fb25dc24b7e8865651e5cf0 Mon Sep 17 00:00:00 2001 From: Emily Marshall Date: Mon, 18 May 2026 01:22:37 +1000 Subject: [PATCH 07/14] test: update tests --- .../services/test_taxonomy_info_service.py | 42 ++++++++++++------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/tests/unit/services/test_taxonomy_info_service.py b/tests/unit/services/test_taxonomy_info_service.py index d863111..8f9af35 100644 --- a/tests/unit/services/test_taxonomy_info_service.py +++ b/tests/unit/services/test_taxonomy_info_service.py @@ -59,18 +59,22 @@ def rollback(self): def test_populate_from_ncbi_lookup_creates_taxonomy_info(monkeypatch): organism = Organism(taxon_id=5077, bpa_scientific_name="Penicillium") db = _Session({Organism: {5077: organism}, TaxonomyInfo: {}}) + calls = [] monkeypatch.setattr( ti_service_module, - "fetch_taxonomy_for_taxon_id", - lambda taxon_id, scientific_name=None: ( - { - "taxon_id": taxon_id, - "ncbi_taxon_id": taxon_id, - "ncbi_rank": "species", - "ncbi_scientific_name": "Penicillium test", - "ncbi_order": "Eurotiales", - "mito_ref": "Penicillium chrysogenum", + "fetch_taxonomy_for_taxon_ids", + lambda taxa, batch_size=20: ( + calls.append((taxa, batch_size)) + or { + 5077: { + "taxon_id": 5077, + "ncbi_taxon_id": 5077, + "ncbi_rank": "species", + "ncbi_scientific_name": "Penicillium test", + "ncbi_order": "Eurotiales", + "mito_ref": "Penicillium chrysogenum", + } }, [], ), @@ -84,6 +88,7 @@ def test_populate_from_ncbi_lookup_creates_taxonomy_info(monkeypatch): ) assert ti is not None + assert calls == [({5077: "Penicillium"}, 20)] assert ti.taxon_id == 5077 assert ti.ncbi_taxon_id == 5077 assert ti.ncbi_rank == "species" @@ -97,16 +102,20 @@ def test_populate_from_ncbi_lookup_creates_taxonomy_info(monkeypatch): def test_create_taxonomy_info_fetches_ncbi_and_applies_payload(monkeypatch): organism = Organism(taxon_id=5303, bpa_scientific_name="Agaricus") db = _Session({Organism: {5303: organism}, TaxonomyInfo: {}}) + calls = [] monkeypatch.setattr( ti_service_module, - "fetch_taxonomy_for_taxon_id", - lambda taxon_id, scientific_name=None: ( - { - "taxon_id": taxon_id, - "ncbi_taxon_id": taxon_id, - "ncbi_rank": "species", - "ncbi_scientific_name": "Agaricus test", + "fetch_taxonomy_for_taxon_ids", + lambda taxa, batch_size=20: ( + calls.append((taxa, batch_size)) + or { + 5303: { + "taxon_id": 5303, + "ncbi_taxon_id": 5303, + "ncbi_rank": "species", + "ncbi_scientific_name": "Agaricus test", + } }, [], ), @@ -122,6 +131,7 @@ def test_create_taxonomy_info_fetches_ncbi_and_applies_payload(monkeypatch): ) assert ti.taxon_id == 5303 + assert calls == [({5303: "Agaricus"}, 20)] assert ti.ncbi_taxon_id == 5303 assert ti.ncbi_rank == "species" assert ti.genetic_code_id == 11 From 0c19c04edac4e7a80ef7e026ce612523e6e1c876 Mon Sep 17 00:00:00 2001 From: Emily Marshall Date: Mon, 18 May 2026 11:08:21 +1000 Subject: [PATCH 08/14] feat: update ncbi information for taxonomy info objects that already exist --- app/services/taxonomy_info_service.py | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/app/services/taxonomy_info_service.py b/app/services/taxonomy_info_service.py index 1477d87..f934b7e 100644 --- a/app/services/taxonomy_info_service.py +++ b/app/services/taxonomy_info_service.py @@ -43,9 +43,6 @@ def create(self, db: Session, *, ti_in: TaxonomyInfoCreate) -> TaxonomyInfo: organism = db.query(Organism).filter(Organism.taxon_id == ti_in.taxon_id).first() if not organism: raise ValueError(f"Organism with taxon_id {ti_in.taxon_id} does not exist") - existing = db.query(TaxonomyInfo).filter(TaxonomyInfo.taxon_id == ti_in.taxon_id).first() - if existing: - raise ValueError(f"TaxonomyInfo for taxon_id {ti_in.taxon_id} already exists") ti = self.populate_from_ncbi_lookup( db, @@ -154,13 +151,6 @@ def bulk_import( skipped_count += 1 continue - existing = db.query(TaxonomyInfo).filter(TaxonomyInfo.taxon_id == taxon_id).first() - if existing: - errors.append( - f"{taxon_id}: taxonomy_info for taxon_id {taxon_id} already exists" - ) - skipped_count += 1 - continue except Exception as e: errors.append(f"{taxon_id}: {str(e)}") db.rollback() @@ -179,14 +169,19 @@ def bulk_import( for taxon_id, row, _organism in candidates: try: - ti = TaxonomyInfo(taxon_id=taxon_id) - db.add(ti) + ti = db.query(TaxonomyInfo).filter(TaxonomyInfo.taxon_id == taxon_id).first() + created = False + if not ti: + ti = TaxonomyInfo(taxon_id=taxon_id) + db.add(ti) + created = True mapped = ncbi_by_taxon_id.get(taxon_id) if mapped: applied_fields = self._apply_ncbi_values(ti, mapped) logger.info( - "NCBI taxonomy enrichment created taxonomy_info for taxon_id=%s during bulk import; applied_fields=%s", + "NCBI taxonomy enrichment %s taxonomy_info for taxon_id=%s; applied_fields=%s", + "created" if created else "updated", taxon_id, applied_fields, ) From 77a58a67da9f3fe4005dc6cac7c8be358fa94a1d Mon Sep 17 00:00:00 2001 From: Emily Marshall Date: Mon, 18 May 2026 16:01:23 +1000 Subject: [PATCH 09/14] fix: shorten migration version label, linting --- .../0014_add_ncbi_taxonomy_details.py | 11 +- app/services/assembly_helper.py | 4 +- schema.sql | 22 +-- uv.lock | 131 ++++++++++++++++++ 4 files changed, 149 insertions(+), 19 deletions(-) diff --git a/alembic/versions/0014_add_ncbi_taxonomy_details.py b/alembic/versions/0014_add_ncbi_taxonomy_details.py index ea71af2..e7b0c95 100644 --- a/alembic/versions/0014_add_ncbi_taxonomy_details.py +++ b/alembic/versions/0014_add_ncbi_taxonomy_details.py @@ -1,7 +1,7 @@ """Remove submission_xml from assembly_submission table. Revision ID: 0014_add_ncbi_taxonomy_details -Revises: 0013_remove_assembly_submission_xml +Revises: 0013_remove_assembly_sub_col Create Date: 2026-05-13 We are adding in NCBI taxonomy details for organisms, sourced through external API lookup to NCBI's Datasets API v2. @@ -12,7 +12,7 @@ from alembic import op revision = "0014_add_ncbi_taxonomy_details" -down_revision = "0013_remove_assembly_submission_xml" +down_revision = "0013_remove_assembly_sub_col" branch_labels = None depends_on = None @@ -22,7 +22,9 @@ def upgrade() -> None: op.alter_column("organism", "genus", new_column_name="bpa_genus") op.alter_column("organism", "species", new_column_name="bpa_species") op.alter_column("organism", "common_name", new_column_name="bpa_common_name") - op.alter_column("organism", "infraspecific_epithet", new_column_name="bpa_infraspecific_epithet") + op.alter_column( + "organism", "infraspecific_epithet", new_column_name="bpa_infraspecific_epithet" + ) op.alter_column("organism", "culture_or_strain_id", new_column_name="bpa_culture_or_strain_id") op.alter_column("organism", "authority", new_column_name="bpa_authority") op.alter_column("organism", "scientific_name", new_column_name="bpa_scientific_name") @@ -32,7 +34,7 @@ def upgrade() -> None: op.drop_column("organism", "ncbi_family") op.drop_column("organism", "busco_dataset_name") op.drop_column("organism", "taxonomy_lineage_json") - + # Add new columns for NCBI taxonomy details op.add_column("taxonomy_info", sa.Column("ncbi_taxon_id", sa.Integer(), nullable=True)) op.add_column("taxonomy_info", sa.Column("ncbi_rank", sa.String(), nullable=True)) @@ -94,4 +96,3 @@ def downgrade() -> None: # Restore dropped BPA source column op.add_column("organism", sa.Column("common_name_source", sa.Text(), nullable=True)) - diff --git a/app/services/assembly_helper.py b/app/services/assembly_helper.py index 1d64fcb..9bc5698 100644 --- a/app/services/assembly_helper.py +++ b/app/services/assembly_helper.py @@ -323,9 +323,7 @@ def generate_assembly_manifest_json( "mitochondrial_genetic_code_id": getattr( taxonomy_info, "mitochondrial_genetic_code_id", None ), - "mitohifi_reference_species": getattr( - taxonomy_info, "mitohifi_reference_species", None - ), + "mitohifi_reference_species": getattr(taxonomy_info, "mitohifi_reference_species", None), "oatk_hmm_name": getattr(taxonomy_info, "oatk_hmm_name", None), "augustus_dataset_name": getattr(taxonomy_info, "augustus_dataset_name", None), "genetic_code_id": getattr(taxonomy_info, "genetic_code_id", None), diff --git a/schema.sql b/schema.sql index 66beb39..3f1c06d 100644 --- a/schema.sql +++ b/schema.sql @@ -82,7 +82,7 @@ CREATE TABLE organism ( -- # TODO delete bpa_json bpa_json JSONB, - + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); @@ -94,19 +94,19 @@ CREATE TABLE organism ( CREATE TABLE taxonomy_info ( taxon_id INT PRIMARY KEY REFERENCES organism(taxon_id) ON DELETE CASCADE, ncbi_taxon_id INTEGER, - + ncbi_rank TEXT, ncbi_scientific_name TEXT, ncbi_authority TEXT, - ncbi_scientific_name TEXT, - ncbi_authority TEXT, - ncbi_common_name TEXT, - ncbi_class TEXT, - ncbi_order TEXT, - ncbi_family TEXT, - ncbi_lineage JSONB, - ncbi_tax_string TEXT, - ncbi_full_lineage TEXT, + ncbi_scientific_name TEXT, + ncbi_authority TEXT, + ncbi_common_name TEXT, + ncbi_class TEXT, + ncbi_order TEXT, + ncbi_family TEXT, + ncbi_lineage JSONB, + ncbi_tax_string TEXT, + ncbi_full_lineage TEXT, mito_ref TEXT busco_dataset_name TEXT, diff --git a/uv.lock b/uv.lock index 208426e..ffdc9e4 100644 --- a/uv.lock +++ b/uv.lock @@ -63,6 +63,7 @@ dependencies = [ { name = "pydantic-settings" }, { name = "python-jose", extra = ["cryptography"] }, { name = "python-multipart" }, + { name = "requests" }, { name = "sqlalchemy" }, { name = "uvicorn", extra = ["standard"] }, ] @@ -101,6 +102,7 @@ requires-dist = [ { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.23.0" }, { name = "python-jose", extras = ["cryptography"], specifier = ">=3.3.0" }, { name = "python-multipart", specifier = ">=0.0.6" }, + { name = "requests", specifier = ">=2.31.0" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.4.0" }, { name = "sqlalchemy", specifier = ">=2.0.22" }, { name = "uvicorn", extras = ["standard"], specifier = ">=0.23.2" }, @@ -296,6 +298,111 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/db/3c/33bac158f8ab7f89b2e59426d5fe2e4f63f7ed25df84c036890172b412b5/cfgv-3.5.0-py2.py3-none-any.whl", hash = "sha256:a8dc6b26ad22ff227d2634a65cb388215ce6cc96bbcc5cfde7641ae87e8dacc0", size = 7445, upload-time = "2025-11-19T20:55:50.744Z" }, ] +[[package]] +name = "charset-normalizer" +version = "3.4.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e7/a1/67fe25fac3c7642725500a3f6cfe5821ad557c3abb11c9d20d12c7008d3e/charset_normalizer-3.4.7.tar.gz", hash = "sha256:ae89db9e5f98a11a4bf50407d4363e7b09b31e55bc117b4f7d80aab97ba009e5", size = 144271, upload-time = "2026-04-02T09:28:39.342Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/26/08/0f303cb0b529e456bb116f2d50565a482694fbb94340bf56d44677e7ed03/charset_normalizer-3.4.7-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cdd68a1fb318e290a2077696b7eb7a21a49163c455979c639bf5a5dcdc46617d", size = 315182, upload-time = "2026-04-02T09:25:40.673Z" }, + { url = "https://files.pythonhosted.org/packages/24/47/b192933e94b546f1b1fe4df9cc1f84fcdbf2359f8d1081d46dd029b50207/charset_normalizer-3.4.7-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e17b8d5d6a8c47c85e68ca8379def1303fd360c3e22093a807cd34a71cd082b8", size = 209329, upload-time = "2026-04-02T09:25:42.354Z" }, + { url = "https://files.pythonhosted.org/packages/c2/b4/01fa81c5ca6141024d89a8fc15968002b71da7f825dd14113207113fabbd/charset_normalizer-3.4.7-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:511ef87c8aec0783e08ac18565a16d435372bc1ac25a91e6ac7f5ef2b0bff790", size = 231230, upload-time = "2026-04-02T09:25:44.281Z" }, + { url = "https://files.pythonhosted.org/packages/20/f7/7b991776844dfa058017e600e6e55ff01984a063290ca5622c0b63162f68/charset_normalizer-3.4.7-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:007d05ec7321d12a40227aae9e2bc6dca73f3cb21058999a1df9e193555a9dcc", size = 225890, upload-time = "2026-04-02T09:25:45.475Z" }, + { url = "https://files.pythonhosted.org/packages/20/e7/bed0024a0f4ab0c8a9c64d4445f39b30c99bd1acd228291959e3de664247/charset_normalizer-3.4.7-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cf29836da5119f3c8a8a70667b0ef5fdca3bb12f80fd06487cfa575b3909b393", size = 216930, upload-time = "2026-04-02T09:25:46.58Z" }, + { url = "https://files.pythonhosted.org/packages/e2/ab/b18f0ab31cdd7b3ddb8bb76c4a414aeb8160c9810fdf1bc62f269a539d87/charset_normalizer-3.4.7-cp310-cp310-manylinux_2_31_armv7l.whl", hash = "sha256:12d8baf840cc7889b37c7c770f478adea7adce3dcb3944d02ec87508e2dcf153", size = 202109, upload-time = "2026-04-02T09:25:48.031Z" }, + { url = "https://files.pythonhosted.org/packages/82/e5/7e9440768a06dfb3075936490cb82dbf0ee20a133bf0dd8551fa096914ec/charset_normalizer-3.4.7-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d560742f3c0d62afaccf9f41fe485ed69bd7661a241f86a3ef0f0fb8b1a397af", size = 214684, upload-time = "2026-04-02T09:25:49.245Z" }, + { url = "https://files.pythonhosted.org/packages/71/94/8c61d8da9f062fdf457c80acfa25060ec22bf1d34bbeaca4350f13bcfd07/charset_normalizer-3.4.7-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b14b2d9dac08e28bb8046a1a0434b1750eb221c8f5b87a68f4fa11a6f97b5e34", size = 212785, upload-time = "2026-04-02T09:25:50.671Z" }, + { url = "https://files.pythonhosted.org/packages/66/cd/6e9889c648e72c0ab2e5967528bb83508f354d706637bc7097190c874e13/charset_normalizer-3.4.7-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:bc17a677b21b3502a21f66a8cc64f5bfad4df8a0b8434d661666f8ce90ac3af1", size = 203055, upload-time = "2026-04-02T09:25:51.802Z" }, + { url = "https://files.pythonhosted.org/packages/92/2e/7a951d6a08aefb7eb8e1b54cdfb580b1365afdd9dd484dc4bee9e5d8f258/charset_normalizer-3.4.7-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:750e02e074872a3fad7f233b47734166440af3cdea0add3e95163110816d6752", size = 232502, upload-time = "2026-04-02T09:25:53.388Z" }, + { url = "https://files.pythonhosted.org/packages/58/d5/abcf2d83bf8e0a1286df55cd0dc1d49af0da4282aa77e986df343e7de124/charset_normalizer-3.4.7-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:4e5163c14bffd570ef2affbfdd77bba66383890797df43dc8b4cc7d6f500bf53", size = 214295, upload-time = "2026-04-02T09:25:54.765Z" }, + { url = "https://files.pythonhosted.org/packages/47/3a/7d4cd7ed54be99973a0dc176032cba5cb1f258082c31fa6df35cff46acfc/charset_normalizer-3.4.7-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:6ed74185b2db44f41ef35fd1617c5888e59792da9bbc9190d6c7300617182616", size = 227145, upload-time = "2026-04-02T09:25:55.904Z" }, + { url = "https://files.pythonhosted.org/packages/1d/98/3a45bf8247889cf28262ebd3d0872edff11565b2a1e3064ccb132db3fbb0/charset_normalizer-3.4.7-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:94e1885b270625a9a828c9793b4d52a64445299baa1fea5a173bf1d3dd9a1a5a", size = 218884, upload-time = "2026-04-02T09:25:57.074Z" }, + { url = "https://files.pythonhosted.org/packages/ad/80/2e8b7f8915ed5c9ef13aa828d82738e33888c485b65ebf744d615040c7ea/charset_normalizer-3.4.7-cp310-cp310-win32.whl", hash = "sha256:6785f414ae0f3c733c437e0f3929197934f526d19dfaa75e18fdb4f94c6fb374", size = 148343, upload-time = "2026-04-02T09:25:58.199Z" }, + { url = "https://files.pythonhosted.org/packages/35/1b/3b8c8c77184af465ee9ad88b5aea46ea6b2e1f7b9dc9502891e37af21e30/charset_normalizer-3.4.7-cp310-cp310-win_amd64.whl", hash = "sha256:6696b7688f54f5af4462118f0bfa7c1621eeb87154f77fa04b9295ce7a8f2943", size = 159174, upload-time = "2026-04-02T09:25:59.322Z" }, + { url = "https://files.pythonhosted.org/packages/be/c1/feb40dca40dbb21e0a908801782d9288c64fc8d8e562c2098e9994c8c21b/charset_normalizer-3.4.7-cp310-cp310-win_arm64.whl", hash = "sha256:66671f93accb62ed07da56613636f3641f1a12c13046ce91ffc923721f23c008", size = 147805, upload-time = "2026-04-02T09:26:00.756Z" }, + { url = "https://files.pythonhosted.org/packages/c2/d7/b5b7020a0565c2e9fa8c09f4b5fa6232feb326b8c20081ccded47ea368fd/charset_normalizer-3.4.7-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7641bb8895e77f921102f72833904dcd9901df5d6d72a2ab8f31d04b7e51e4e7", size = 309705, upload-time = "2026-04-02T09:26:02.191Z" }, + { url = "https://files.pythonhosted.org/packages/5a/53/58c29116c340e5456724ecd2fff4196d236b98f3da97b404bc5e51ac3493/charset_normalizer-3.4.7-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:202389074300232baeb53ae2569a60901f7efadd4245cf3a3bf0617d60b439d7", size = 206419, upload-time = "2026-04-02T09:26:03.583Z" }, + { url = "https://files.pythonhosted.org/packages/b2/02/e8146dc6591a37a00e5144c63f29fb7c97a734ea8a111190783c0e60ab63/charset_normalizer-3.4.7-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:30b8d1d8c52a48c2c5690e152c169b673487a2a58de1ec7393196753063fcd5e", size = 227901, upload-time = "2026-04-02T09:26:04.738Z" }, + { url = "https://files.pythonhosted.org/packages/fb/73/77486c4cd58f1267bf17db420e930c9afa1b3be3fe8c8b8ebbebc9624359/charset_normalizer-3.4.7-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:532bc9bf33a68613fd7d65e4b1c71a6a38d7d42604ecf239c77392e9b4e8998c", size = 222742, upload-time = "2026-04-02T09:26:06.36Z" }, + { url = "https://files.pythonhosted.org/packages/a1/fa/f74eb381a7d94ded44739e9d94de18dc5edc9c17fb8c11f0a6890696c0a9/charset_normalizer-3.4.7-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2fe249cb4651fd12605b7288b24751d8bfd46d35f12a20b1ba33dea122e690df", size = 214061, upload-time = "2026-04-02T09:26:08.347Z" }, + { url = "https://files.pythonhosted.org/packages/dc/92/42bd3cefcf7687253fb86694b45f37b733c97f59af3724f356fa92b8c344/charset_normalizer-3.4.7-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:65bcd23054beab4d166035cabbc868a09c1a49d1efe458fe8e4361215df40265", size = 199239, upload-time = "2026-04-02T09:26:09.823Z" }, + { url = "https://files.pythonhosted.org/packages/4c/3d/069e7184e2aa3b3cddc700e3dd267413dc259854adc3380421c805c6a17d/charset_normalizer-3.4.7-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:08e721811161356f97b4059a9ba7bafb23ea5ee2255402c42881c214e173c6b4", size = 210173, upload-time = "2026-04-02T09:26:10.953Z" }, + { url = "https://files.pythonhosted.org/packages/62/51/9d56feb5f2e7074c46f93e0ebdbe61f0848ee246e2f0d89f8e20b89ebb8f/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e060d01aec0a910bdccb8be71faf34e7799ce36950f8294c8bf612cba65a2c9e", size = 209841, upload-time = "2026-04-02T09:26:12.142Z" }, + { url = "https://files.pythonhosted.org/packages/d2/59/893d8f99cc4c837dda1fe2f1139079703deb9f321aabcb032355de13b6c7/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:38c0109396c4cfc574d502df99742a45c72c08eff0a36158b6f04000043dbf38", size = 200304, upload-time = "2026-04-02T09:26:13.711Z" }, + { url = "https://files.pythonhosted.org/packages/7d/1d/ee6f3be3464247578d1ed5c46de545ccc3d3ff933695395c402c21fa6b77/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:1c2a768fdd44ee4a9339a9b0b130049139b8ce3c01d2ce09f67f5a68048d477c", size = 229455, upload-time = "2026-04-02T09:26:14.941Z" }, + { url = "https://files.pythonhosted.org/packages/54/bb/8fb0a946296ea96a488928bdce8ef99023998c48e4713af533e9bb98ef07/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:1a87ca9d5df6fe460483d9a5bbf2b18f620cbed41b432e2bddb686228282d10b", size = 210036, upload-time = "2026-04-02T09:26:16.478Z" }, + { url = "https://files.pythonhosted.org/packages/9a/bc/015b2387f913749f82afd4fcba07846d05b6d784dd16123cb66860e0237d/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d635aab80466bc95771bb78d5370e74d36d1fe31467b6b29b8b57b2a3cd7d22c", size = 224739, upload-time = "2026-04-02T09:26:17.751Z" }, + { url = "https://files.pythonhosted.org/packages/17/ab/63133691f56baae417493cba6b7c641571a2130eb7bceba6773367ab9ec5/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ae196f021b5e7c78e918242d217db021ed2a6ace2bc6ae94c0fc596221c7f58d", size = 216277, upload-time = "2026-04-02T09:26:18.981Z" }, + { url = "https://files.pythonhosted.org/packages/06/6d/3be70e827977f20db77c12a97e6a9f973631a45b8d186c084527e53e77a4/charset_normalizer-3.4.7-cp311-cp311-win32.whl", hash = "sha256:adb2597b428735679446b46c8badf467b4ca5f5056aae4d51a19f9570301b1ad", size = 147819, upload-time = "2026-04-02T09:26:20.295Z" }, + { url = "https://files.pythonhosted.org/packages/20/d9/5f67790f06b735d7c7637171bbfd89882ad67201891b7275e51116ed8207/charset_normalizer-3.4.7-cp311-cp311-win_amd64.whl", hash = "sha256:8e385e4267ab76874ae30db04c627faaaf0b509e1ccc11a95b3fc3e83f855c00", size = 159281, upload-time = "2026-04-02T09:26:21.74Z" }, + { url = "https://files.pythonhosted.org/packages/ca/83/6413f36c5a34afead88ce6f66684d943d91f233d76dd083798f9602b75ae/charset_normalizer-3.4.7-cp311-cp311-win_arm64.whl", hash = "sha256:d4a48e5b3c2a489fae013b7589308a40146ee081f6f509e047e0e096084ceca1", size = 147843, upload-time = "2026-04-02T09:26:22.901Z" }, + { url = "https://files.pythonhosted.org/packages/0c/eb/4fc8d0a7110eb5fc9cc161723a34a8a6c200ce3b4fbf681bc86feee22308/charset_normalizer-3.4.7-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:eca9705049ad3c7345d574e3510665cb2cf844c2f2dcfe675332677f081cbd46", size = 311328, upload-time = "2026-04-02T09:26:24.331Z" }, + { url = "https://files.pythonhosted.org/packages/f8/e3/0fadc706008ac9d7b9b5be6dc767c05f9d3e5df51744ce4cc9605de7b9f4/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6178f72c5508bfc5fd446a5905e698c6212932f25bcdd4b47a757a50605a90e2", size = 208061, upload-time = "2026-04-02T09:26:25.568Z" }, + { url = "https://files.pythonhosted.org/packages/42/f0/3dd1045c47f4a4604df85ec18ad093912ae1344ac706993aff91d38773a2/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e1421b502d83040e6d7fb2fb18dff63957f720da3d77b2fbd3187ceb63755d7b", size = 229031, upload-time = "2026-04-02T09:26:26.865Z" }, + { url = "https://files.pythonhosted.org/packages/dc/67/675a46eb016118a2fbde5a277a5d15f4f69d5f3f5f338e5ee2f8948fcf43/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:edac0f1ab77644605be2cbba52e6b7f630731fc42b34cb0f634be1a6eface56a", size = 225239, upload-time = "2026-04-02T09:26:28.044Z" }, + { url = "https://files.pythonhosted.org/packages/4b/f8/d0118a2f5f23b02cd166fa385c60f9b0d4f9194f574e2b31cef350ad7223/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5649fd1c7bade02f320a462fdefd0b4bd3ce036065836d4f42e0de958038e116", size = 216589, upload-time = "2026-04-02T09:26:29.239Z" }, + { url = "https://files.pythonhosted.org/packages/b1/f1/6d2b0b261b6c4ceef0fcb0d17a01cc5bc53586c2d4796fa04b5c540bc13d/charset_normalizer-3.4.7-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:203104ed3e428044fd943bc4bf45fa73c0730391f9621e37fe39ecf477b128cb", size = 202733, upload-time = "2026-04-02T09:26:30.5Z" }, + { url = "https://files.pythonhosted.org/packages/6f/c0/7b1f943f7e87cc3db9626ba17807d042c38645f0a1d4415c7a14afb5591f/charset_normalizer-3.4.7-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:298930cec56029e05497a76988377cbd7457ba864beeea92ad7e844fe74cd1f1", size = 212652, upload-time = "2026-04-02T09:26:31.709Z" }, + { url = "https://files.pythonhosted.org/packages/38/dd/5a9ab159fe45c6e72079398f277b7d2b523e7f716acc489726115a910097/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:708838739abf24b2ceb208d0e22403dd018faeef86ddac04319a62ae884c4f15", size = 211229, upload-time = "2026-04-02T09:26:33.282Z" }, + { url = "https://files.pythonhosted.org/packages/d5/ff/531a1cad5ca855d1c1a8b69cb71abfd6d85c0291580146fda7c82857caa1/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0f7eb884681e3938906ed0434f20c63046eacd0111c4ba96f27b76084cd679f5", size = 203552, upload-time = "2026-04-02T09:26:34.845Z" }, + { url = "https://files.pythonhosted.org/packages/c1/4c/a5fb52d528a8ca41f7598cb619409ece30a169fbdf9cdce592e53b46c3a6/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:4dc1e73c36828f982bfe79fadf5919923f8a6f4df2860804db9a98c48824ce8d", size = 230806, upload-time = "2026-04-02T09:26:36.152Z" }, + { url = "https://files.pythonhosted.org/packages/59/7a/071feed8124111a32b316b33ae4de83d36923039ef8cf48120266844285b/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:aed52fea0513bac0ccde438c188c8a471c4e0f457c2dd20cdbf6ea7a450046c7", size = 212316, upload-time = "2026-04-02T09:26:37.672Z" }, + { url = "https://files.pythonhosted.org/packages/fd/35/f7dba3994312d7ba508e041eaac39a36b120f32d4c8662b8814dab876431/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:fea24543955a6a729c45a73fe90e08c743f0b3334bbf3201e6c4bc1b0c7fa464", size = 227274, upload-time = "2026-04-02T09:26:38.93Z" }, + { url = "https://files.pythonhosted.org/packages/8a/2d/a572df5c9204ab7688ec1edc895a73ebded3b023bb07364710b05dd1c9be/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:bb6d88045545b26da47aa879dd4a89a71d1dce0f0e549b1abcb31dfe4a8eac49", size = 218468, upload-time = "2026-04-02T09:26:40.17Z" }, + { url = "https://files.pythonhosted.org/packages/86/eb/890922a8b03a568ca2f336c36585a4713c55d4d67bf0f0c78924be6315ca/charset_normalizer-3.4.7-cp312-cp312-win32.whl", hash = "sha256:2257141f39fe65a3fdf38aeccae4b953e5f3b3324f4ff0daf9f15b8518666a2c", size = 148460, upload-time = "2026-04-02T09:26:41.416Z" }, + { url = "https://files.pythonhosted.org/packages/35/d9/0e7dffa06c5ab081f75b1b786f0aefc88365825dfcd0ac544bdb7b2b6853/charset_normalizer-3.4.7-cp312-cp312-win_amd64.whl", hash = "sha256:5ed6ab538499c8644b8a3e18debabcd7ce684f3fa91cf867521a7a0279cab2d6", size = 159330, upload-time = "2026-04-02T09:26:42.554Z" }, + { url = "https://files.pythonhosted.org/packages/9e/5d/481bcc2a7c88ea6b0878c299547843b2521ccbc40980cb406267088bc701/charset_normalizer-3.4.7-cp312-cp312-win_arm64.whl", hash = "sha256:56be790f86bfb2c98fb742ce566dfb4816e5a83384616ab59c49e0604d49c51d", size = 147828, upload-time = "2026-04-02T09:26:44.075Z" }, + { url = "https://files.pythonhosted.org/packages/c1/3b/66777e39d3ae1ddc77ee606be4ec6d8cbd4c801f65e5a1b6f2b11b8346dd/charset_normalizer-3.4.7-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f496c9c3cc02230093d8330875c4c3cdfc3b73612a5fd921c65d39cbcef08063", size = 309627, upload-time = "2026-04-02T09:26:45.198Z" }, + { url = "https://files.pythonhosted.org/packages/2e/4e/b7f84e617b4854ade48a1b7915c8ccfadeba444d2a18c291f696e37f0d3b/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ea948db76d31190bf08bd371623927ee1339d5f2a0b4b1b4a4439a65298703c", size = 207008, upload-time = "2026-04-02T09:26:46.824Z" }, + { url = "https://files.pythonhosted.org/packages/c4/bb/ec73c0257c9e11b268f018f068f5d00aa0ef8c8b09f7753ebd5f2880e248/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a277ab8928b9f299723bc1a2dabb1265911b1a76341f90a510368ca44ad9ab66", size = 228303, upload-time = "2026-04-02T09:26:48.397Z" }, + { url = "https://files.pythonhosted.org/packages/85/fb/32d1f5033484494619f701e719429c69b766bfc4dbc61aa9e9c8c166528b/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3bec022aec2c514d9cf199522a802bd007cd588ab17ab2525f20f9c34d067c18", size = 224282, upload-time = "2026-04-02T09:26:49.684Z" }, + { url = "https://files.pythonhosted.org/packages/fa/07/330e3a0dda4c404d6da83b327270906e9654a24f6c546dc886a0eb0ffb23/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e044c39e41b92c845bc815e5ae4230804e8e7bc29e399b0437d64222d92809dd", size = 215595, upload-time = "2026-04-02T09:26:50.915Z" }, + { url = "https://files.pythonhosted.org/packages/e3/7c/fc890655786e423f02556e0216d4b8c6bcb6bdfa890160dc66bf52dee468/charset_normalizer-3.4.7-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:f495a1652cf3fbab2eb0639776dad966c2fb874d79d87ca07f9d5f059b8bd215", size = 201986, upload-time = "2026-04-02T09:26:52.197Z" }, + { url = "https://files.pythonhosted.org/packages/d8/97/bfb18b3db2aed3b90cf54dc292ad79fdd5ad65c4eae454099475cbeadd0d/charset_normalizer-3.4.7-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e712b419df8ba5e42b226c510472b37bd57b38e897d3eca5e8cfd410a29fa859", size = 211711, upload-time = "2026-04-02T09:26:53.49Z" }, + { url = "https://files.pythonhosted.org/packages/6f/a5/a581c13798546a7fd557c82614a5c65a13df2157e9ad6373166d2a3e645d/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7804338df6fcc08105c7745f1502ba68d900f45fd770d5bdd5288ddccb8a42d8", size = 210036, upload-time = "2026-04-02T09:26:54.975Z" }, + { url = "https://files.pythonhosted.org/packages/8c/bf/b3ab5bcb478e4193d517644b0fb2bf5497fbceeaa7a1bc0f4d5b50953861/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:481551899c856c704d58119b5025793fa6730adda3571971af568f66d2424bb5", size = 202998, upload-time = "2026-04-02T09:26:56.303Z" }, + { url = "https://files.pythonhosted.org/packages/e7/4e/23efd79b65d314fa320ec6017b4b5834d5c12a58ba4610aa353af2e2f577/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:f59099f9b66f0d7145115e6f80dd8b1d847176df89b234a5a6b3f00437aa0832", size = 230056, upload-time = "2026-04-02T09:26:57.554Z" }, + { url = "https://files.pythonhosted.org/packages/b9/9f/1e1941bc3f0e01df116e68dc37a55c4d249df5e6fa77f008841aef68264f/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:f59ad4c0e8f6bba240a9bb85504faa1ab438237199d4cce5f622761507b8f6a6", size = 211537, upload-time = "2026-04-02T09:26:58.843Z" }, + { url = "https://files.pythonhosted.org/packages/80/0f/088cbb3020d44428964a6c97fe1edfb1b9550396bf6d278330281e8b709c/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:3dedcc22d73ec993f42055eff4fcfed9318d1eeb9a6606c55892a26964964e48", size = 226176, upload-time = "2026-04-02T09:27:00.437Z" }, + { url = "https://files.pythonhosted.org/packages/6a/9f/130394f9bbe06f4f63e22641d32fc9b202b7e251c9aef4db044324dac493/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:64f02c6841d7d83f832cd97ccf8eb8a906d06eb95d5276069175c696b024b60a", size = 217723, upload-time = "2026-04-02T09:27:02.021Z" }, + { url = "https://files.pythonhosted.org/packages/73/55/c469897448a06e49f8fa03f6caae97074fde823f432a98f979cc42b90e69/charset_normalizer-3.4.7-cp313-cp313-win32.whl", hash = "sha256:4042d5c8f957e15221d423ba781e85d553722fc4113f523f2feb7b188cc34c5e", size = 148085, upload-time = "2026-04-02T09:27:03.192Z" }, + { url = "https://files.pythonhosted.org/packages/5d/78/1b74c5bbb3f99b77a1715c91b3e0b5bdb6fe302d95ace4f5b1bec37b0167/charset_normalizer-3.4.7-cp313-cp313-win_amd64.whl", hash = "sha256:3946fa46a0cf3e4c8cb1cc52f56bb536310d34f25f01ca9b6c16afa767dab110", size = 158819, upload-time = "2026-04-02T09:27:04.454Z" }, + { url = "https://files.pythonhosted.org/packages/68/86/46bd42279d323deb8687c4a5a811fd548cb7d1de10cf6535d099877a9a9f/charset_normalizer-3.4.7-cp313-cp313-win_arm64.whl", hash = "sha256:80d04837f55fc81da168b98de4f4b797ef007fc8a79ab71c6ec9bc4dd662b15b", size = 147915, upload-time = "2026-04-02T09:27:05.971Z" }, + { url = "https://files.pythonhosted.org/packages/97/c8/c67cb8c70e19ef1960b97b22ed2a1567711de46c4ddf19799923adc836c2/charset_normalizer-3.4.7-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:c36c333c39be2dbca264d7803333c896ab8fa7d4d6f0ab7edb7dfd7aea6e98c0", size = 309234, upload-time = "2026-04-02T09:27:07.194Z" }, + { url = "https://files.pythonhosted.org/packages/99/85/c091fdee33f20de70d6c8b522743b6f831a2f1cd3ff86de4c6a827c48a76/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c2aed2e5e41f24ea8ef1590b8e848a79b56f3a5564a65ceec43c9d692dc7d8a", size = 208042, upload-time = "2026-04-02T09:27:08.749Z" }, + { url = "https://files.pythonhosted.org/packages/87/1c/ab2ce611b984d2fd5d86a5a8a19c1ae26acac6bad967da4967562c75114d/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:54523e136b8948060c0fa0bc7b1b50c32c186f2fceee897a495406bb6e311d2b", size = 228706, upload-time = "2026-04-02T09:27:09.951Z" }, + { url = "https://files.pythonhosted.org/packages/a8/29/2b1d2cb00bf085f59d29eb773ce58ec2d325430f8c216804a0a5cd83cbca/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:715479b9a2802ecac752a3b0efa2b0b60285cf962ee38414211abdfccc233b41", size = 224727, upload-time = "2026-04-02T09:27:11.175Z" }, + { url = "https://files.pythonhosted.org/packages/47/5c/032c2d5a07fe4d4855fea851209cca2b6f03ebeb6d4e3afdb3358386a684/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bd6c2a1c7573c64738d716488d2cdd3c00e340e4835707d8fdb8dc1a66ef164e", size = 215882, upload-time = "2026-04-02T09:27:12.446Z" }, + { url = "https://files.pythonhosted.org/packages/2c/c2/356065d5a8b78ed04499cae5f339f091946a6a74f91e03476c33f0ab7100/charset_normalizer-3.4.7-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:c45e9440fb78f8ddabcf714b68f936737a121355bf59f3907f4e17721b9d1aae", size = 200860, upload-time = "2026-04-02T09:27:13.721Z" }, + { url = "https://files.pythonhosted.org/packages/0c/cd/a32a84217ced5039f53b29f460962abb2d4420def55afabe45b1c3c7483d/charset_normalizer-3.4.7-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3534e7dcbdcf757da6b85a0bbf5b6868786d5982dd959b065e65481644817a18", size = 211564, upload-time = "2026-04-02T09:27:15.272Z" }, + { url = "https://files.pythonhosted.org/packages/44/86/58e6f13ce26cc3b8f4a36b94a0f22ae2f00a72534520f4ae6857c4b81f89/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e8ac484bf18ce6975760921bb6148041faa8fef0547200386ea0b52b5d27bf7b", size = 211276, upload-time = "2026-04-02T09:27:16.834Z" }, + { url = "https://files.pythonhosted.org/packages/8f/fe/d17c32dc72e17e155e06883efa84514ca375f8a528ba2546bee73fc4df81/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a5fe03b42827c13cdccd08e6c0247b6a6d4b5e3cdc53fd1749f5896adcdc2356", size = 201238, upload-time = "2026-04-02T09:27:18.229Z" }, + { url = "https://files.pythonhosted.org/packages/6a/29/f33daa50b06525a237451cdb6c69da366c381a3dadcd833fa5676bc468b3/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:2d6eb928e13016cea4f1f21d1e10c1cebd5a421bc57ddf5b1142ae3f86824fab", size = 230189, upload-time = "2026-04-02T09:27:19.445Z" }, + { url = "https://files.pythonhosted.org/packages/b6/6e/52c84015394a6a0bdcd435210a7e944c5f94ea1055f5cc5d56c5fe368e7b/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:e74327fb75de8986940def6e8dee4f127cc9752bee7355bb323cc5b2659b6d46", size = 211352, upload-time = "2026-04-02T09:27:20.79Z" }, + { url = "https://files.pythonhosted.org/packages/8c/d7/4353be581b373033fb9198bf1da3cf8f09c1082561e8e922aa7b39bf9fe8/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:d6038d37043bced98a66e68d3aa2b6a35505dc01328cd65217cefe82f25def44", size = 227024, upload-time = "2026-04-02T09:27:22.063Z" }, + { url = "https://files.pythonhosted.org/packages/30/45/99d18aa925bd1740098ccd3060e238e21115fffbfdcb8f3ece837d0ace6c/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:7579e913a5339fb8fa133f6bbcfd8e6749696206cf05acdbdca71a1b436d8e72", size = 217869, upload-time = "2026-04-02T09:27:23.486Z" }, + { url = "https://files.pythonhosted.org/packages/5c/05/5ee478aa53f4bb7996482153d4bfe1b89e0f087f0ab6b294fcf92d595873/charset_normalizer-3.4.7-cp314-cp314-win32.whl", hash = "sha256:5b77459df20e08151cd6f8b9ef8ef1f961ef73d85c21a555c7eed5b79410ec10", size = 148541, upload-time = "2026-04-02T09:27:25.146Z" }, + { url = "https://files.pythonhosted.org/packages/48/77/72dcb0921b2ce86420b2d79d454c7022bf5be40202a2a07906b9f2a35c97/charset_normalizer-3.4.7-cp314-cp314-win_amd64.whl", hash = "sha256:92a0a01ead5e668468e952e4238cccd7c537364eb7d851ab144ab6627dbbe12f", size = 159634, upload-time = "2026-04-02T09:27:26.642Z" }, + { url = "https://files.pythonhosted.org/packages/c6/a3/c2369911cd72f02386e4e340770f6e158c7980267da16af8f668217abaa0/charset_normalizer-3.4.7-cp314-cp314-win_arm64.whl", hash = "sha256:67f6279d125ca0046a7fd386d01b311c6363844deac3e5b069b514ba3e63c246", size = 148384, upload-time = "2026-04-02T09:27:28.271Z" }, + { url = "https://files.pythonhosted.org/packages/94/09/7e8a7f73d24dba1f0035fbbf014d2c36828fc1bf9c88f84093e57d315935/charset_normalizer-3.4.7-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:effc3f449787117233702311a1b7d8f59cba9ced946ba727bdc329ec69028e24", size = 330133, upload-time = "2026-04-02T09:27:29.474Z" }, + { url = "https://files.pythonhosted.org/packages/8d/da/96975ddb11f8e977f706f45cddd8540fd8242f71ecdb5d18a80723dcf62c/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fbccdc05410c9ee21bbf16a35f4c1d16123dcdeb8a1d38f33654fa21d0234f79", size = 216257, upload-time = "2026-04-02T09:27:30.793Z" }, + { url = "https://files.pythonhosted.org/packages/e5/e8/1d63bf8ef2d388e95c64b2098f45f84758f6d102a087552da1485912637b/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:733784b6d6def852c814bce5f318d25da2ee65dd4839a0718641c696e09a2960", size = 234851, upload-time = "2026-04-02T09:27:32.44Z" }, + { url = "https://files.pythonhosted.org/packages/9b/40/e5ff04233e70da2681fa43969ad6f66ca5611d7e669be0246c4c7aaf6dc8/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a89c23ef8d2c6b27fd200a42aa4ac72786e7c60d40efdc76e6011260b6e949c4", size = 233393, upload-time = "2026-04-02T09:27:34.03Z" }, + { url = "https://files.pythonhosted.org/packages/be/c1/06c6c49d5a5450f76899992f1ee40b41d076aee9279b49cf9974d2f313d5/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6c114670c45346afedc0d947faf3c7f701051d2518b943679c8ff88befe14f8e", size = 223251, upload-time = "2026-04-02T09:27:35.369Z" }, + { url = "https://files.pythonhosted.org/packages/2b/9f/f2ff16fb050946169e3e1f82134d107e5d4ae72647ec8a1b1446c148480f/charset_normalizer-3.4.7-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:a180c5e59792af262bf263b21a3c49353f25945d8d9f70628e73de370d55e1e1", size = 206609, upload-time = "2026-04-02T09:27:36.661Z" }, + { url = "https://files.pythonhosted.org/packages/69/d5/a527c0cd8d64d2eab7459784fb4169a0ac76e5a6fc5237337982fd61347e/charset_normalizer-3.4.7-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3c9a494bc5ec77d43cea229c4f6db1e4d8fe7e1bbffa8b6f0f0032430ff8ab44", size = 220014, upload-time = "2026-04-02T09:27:38.019Z" }, + { url = "https://files.pythonhosted.org/packages/7e/80/8a7b8104a3e203074dc9aa2c613d4b726c0e136bad1cc734594b02867972/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8d828b6667a32a728a1ad1d93957cdf37489c57b97ae6c4de2860fa749b8fc1e", size = 218979, upload-time = "2026-04-02T09:27:39.37Z" }, + { url = "https://files.pythonhosted.org/packages/02/9a/b759b503d507f375b2b5c153e4d2ee0a75aa215b7f2489cf314f4541f2c0/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cf1493cd8607bec4d8a7b9b004e699fcf8f9103a9284cc94962cb73d20f9d4a3", size = 209238, upload-time = "2026-04-02T09:27:40.722Z" }, + { url = "https://files.pythonhosted.org/packages/c2/4e/0f3f5d47b86bdb79256e7290b26ac847a2832d9a4033f7eb2cd4bcf4bb5b/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0c96c3b819b5c3e9e165495db84d41914d6894d55181d2d108cc1a69bfc9cce0", size = 236110, upload-time = "2026-04-02T09:27:42.33Z" }, + { url = "https://files.pythonhosted.org/packages/96/23/bce28734eb3ed2c91dcf93abeb8a5cf393a7b2749725030bb630e554fdd8/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:752a45dc4a6934060b3b0dab47e04edc3326575f82be64bc4fc293914566503e", size = 219824, upload-time = "2026-04-02T09:27:43.924Z" }, + { url = "https://files.pythonhosted.org/packages/2c/6f/6e897c6984cc4d41af319b077f2f600fc8214eb2fe2d6bcb79141b882400/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:8778f0c7a52e56f75d12dae53ae320fae900a8b9b4164b981b9c5ce059cd1fcb", size = 233103, upload-time = "2026-04-02T09:27:45.348Z" }, + { url = "https://files.pythonhosted.org/packages/76/22/ef7bd0fe480a0ae9b656189ec00744b60933f68b4f42a7bb06589f6f576a/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ce3412fbe1e31eb81ea42f4169ed94861c56e643189e1e75f0041f3fe7020abe", size = 225194, upload-time = "2026-04-02T09:27:46.706Z" }, + { url = "https://files.pythonhosted.org/packages/c5/a7/0e0ab3e0b5bc1219bd80a6a0d4d72ca74d9250cb2382b7c699c147e06017/charset_normalizer-3.4.7-cp314-cp314t-win32.whl", hash = "sha256:c03a41a8784091e67a39648f70c5f97b5b6a37f216896d44d2cdcb82615339a0", size = 159827, upload-time = "2026-04-02T09:27:48.053Z" }, + { url = "https://files.pythonhosted.org/packages/7a/1d/29d32e0fb40864b1f878c7f5a0b343ae676c6e2b271a2d55cc3a152391da/charset_normalizer-3.4.7-cp314-cp314t-win_amd64.whl", hash = "sha256:03853ed82eeebbce3c2abfdbc98c96dc205f32a79627688ac9a27370ea61a49c", size = 174168, upload-time = "2026-04-02T09:27:49.795Z" }, + { url = "https://files.pythonhosted.org/packages/de/32/d92444ad05c7a6e41fb2036749777c163baf7a0301a040cb672d6b2b1ae9/charset_normalizer-3.4.7-cp314-cp314t-win_arm64.whl", hash = "sha256:c35abb8bfff0185efac5878da64c45dafd2b37fb0383add1be155a763c1f083d", size = 153018, upload-time = "2026-04-02T09:27:51.116Z" }, + { url = "https://files.pythonhosted.org/packages/db/8f/61959034484a4a7c527811f4721e75d02d653a35afb0b6054474d8185d4c/charset_normalizer-3.4.7-py3-none-any.whl", hash = "sha256:3dce51d0f5e7951f8bb4900c257dad282f49190fdbebecd4ba99bcc41fef404d", size = 61958, upload-time = "2026-04-02T09:28:37.794Z" }, +] + [[package]] name = "click" version = "8.3.1" @@ -1280,6 +1387,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, ] +[[package]] +name = "requests" +version = "2.34.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, +] + [[package]] name = "rsa" version = "4.9.1" @@ -1455,6 +1577,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, ] +[[package]] +name = "urllib3" +version = "2.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, +] + [[package]] name = "uvicorn" version = "0.38.0" From b80147059d009c4214f91ebd71c75bb0c3893449 Mon Sep 17 00:00:00 2001 From: Emily Marshall Date: Mon, 18 May 2026 21:14:31 +1000 Subject: [PATCH 10/14] feat: re-introduce o.scientific_name, populate from ti.ncbi_sci.._name or else o.bpa_sci.._name --- .../0014_add_ncbi_taxonomy_details.py | 18 +++++++ app/models/organism.py | 1 + app/models/taxonomy_info.py | 3 +- app/schemas/organism.py | 38 +++++++++------ app/schemas/taxonomy_info.py | 2 + app/services/organism_service.py | 32 ++++++++++++- app/services/taxonomy_info_service.py | 48 +++++++++++++++---- schema.sql | 8 ++-- .../endpoints/test_endpoints_organisms.py | 26 ++++------ .../services/test_taxonomy_info_service.py | 30 ++++++++++++ 10 files changed, 159 insertions(+), 47 deletions(-) diff --git a/alembic/versions/0014_add_ncbi_taxonomy_details.py b/alembic/versions/0014_add_ncbi_taxonomy_details.py index e7b0c95..13922e5 100644 --- a/alembic/versions/0014_add_ncbi_taxonomy_details.py +++ b/alembic/versions/0014_add_ncbi_taxonomy_details.py @@ -8,6 +8,7 @@ """ import sqlalchemy as sa +from sqlalchemy.dialects import postgresql from alembic import op @@ -28,6 +29,7 @@ def upgrade() -> None: op.alter_column("organism", "culture_or_strain_id", new_column_name="bpa_culture_or_strain_id") op.alter_column("organism", "authority", new_column_name="bpa_authority") op.alter_column("organism", "scientific_name", new_column_name="bpa_scientific_name") + op.add_column("organism", sa.Column("scientific_name", sa.Text(), nullable=True)) op.drop_column("organism", "tax_string") op.drop_column("organism", "ncbi_order") @@ -47,8 +49,22 @@ def upgrade() -> None: op.add_column("taxonomy_info", sa.Column("ncbi_lineage", sa.JSON(), nullable=True)) op.add_column("taxonomy_info", sa.Column("ncbi_tax_string", sa.String(), nullable=True)) op.add_column("taxonomy_info", sa.Column("ncbi_full_lineage", sa.String(), nullable=True)) + op.add_column( + "taxonomy_info", + sa.Column("ncbi_last_synced_at", sa.DateTime(timezone=True), nullable=True), + ) op.add_column("taxonomy_info", sa.Column("mito_ref", sa.String(), nullable=True)) op.add_column("taxonomy_info", sa.Column("busco_dataset_name", sa.String(), nullable=True)) + op.execute("UPDATE organism SET scientific_name = bpa_scientific_name") + op.execute( + """ + UPDATE organism + SET scientific_name = taxonomy_info.ncbi_scientific_name + FROM taxonomy_info + WHERE taxonomy_info.taxon_id = organism.taxon_id + AND taxonomy_info.ncbi_scientific_name IS NOT NULL + """ + ) def downgrade() -> None: @@ -58,6 +74,7 @@ def downgrade() -> None: op.drop_column("taxonomy_info", "ncbi_full_lineage") op.drop_column("taxonomy_info", "ncbi_tax_string") op.drop_column("taxonomy_info", "ncbi_lineage") + op.drop_column("taxonomy_info", "ncbi_last_synced_at") op.drop_column("taxonomy_info", "ncbi_family") op.drop_column("taxonomy_info", "ncbi_order") op.drop_column("taxonomy_info", "ncbi_class") @@ -76,6 +93,7 @@ def downgrade() -> None: op.add_column("organism", sa.Column("ncbi_family", sa.Text(), nullable=True)) op.add_column("organism", sa.Column("ncbi_order", sa.Text(), nullable=True)) op.add_column("organism", sa.Column("tax_string", sa.Text(), nullable=True)) + op.drop_column("organism", "scientific_name") # Rename BPA columns back to their original names op.alter_column("organism", "bpa_scientific_name", new_column_name="scientific_name") diff --git a/app/models/organism.py b/app/models/organism.py index 4bbdc88..95549f9 100644 --- a/app/models/organism.py +++ b/app/models/organism.py @@ -15,6 +15,7 @@ class Organism(Base): __tablename__ = "organism" taxon_id = Column(Integer, primary_key=True) + scientific_name = Column(Text, nullable=True) bpa_scientific_name = Column(Text, nullable=True) bpa_genus = Column(Text, nullable=True) bpa_species = Column(Text, nullable=True) diff --git a/app/models/taxonomy_info.py b/app/models/taxonomy_info.py index 609419d..0263908 100644 --- a/app/models/taxonomy_info.py +++ b/app/models/taxonomy_info.py @@ -1,4 +1,4 @@ -from sqlalchemy import Boolean, Column, ForeignKey, Integer, Text +from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, Text from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.orm import relationship @@ -22,6 +22,7 @@ class TaxonomyInfo(Base): ncbi_lineage = Column(JSONB, nullable=True) ncbi_tax_string = Column(Text, nullable=True) ncbi_full_lineage = Column(Text, nullable=True) + ncbi_last_synced_at = Column(DateTime(timezone=True), nullable=True) mito_ref = Column(Text, nullable=True) busco_dataset_name = Column(Text, nullable=True) busco_odb10_dataset_name = Column(Text, nullable=True) diff --git a/app/schemas/organism.py b/app/schemas/organism.py index 689be83..a09a7d3 100644 --- a/app/schemas/organism.py +++ b/app/schemas/organism.py @@ -3,15 +3,12 @@ from datetime import datetime from typing import Dict, Optional -from pydantic import BaseModel, ConfigDict +from pydantic import BaseModel, ConfigDict, model_validator from app.schemas.taxonomy_info import TaxonomyInfo as TaxonomyInfoSchema -class OrganismBase(BaseModel): - """Base Organism schema with common attributes.""" - - taxon_id: int +class OrganismWriteBase(BaseModel): bpa_scientific_name: Optional[str] = None bpa_genus: Optional[str] = None bpa_species: Optional[str] = None @@ -21,16 +18,21 @@ class OrganismBase(BaseModel): bpa_authority: Optional[str] = None bpa_json: Optional[Dict] = None - -class OrganismCreate(OrganismBase): - """Schema for creating a new organism.""" - - pass + # TODO this is for backwards compatability - we can change this once we update the data mapper field names + @model_validator(mode="before") + @classmethod + def _coerce_scientific_name(cls, data): + if isinstance(data, dict) and "bpa_scientific_name" not in data and "scientific_name" in data: + data = dict(data) + data["bpa_scientific_name"] = data["scientific_name"] + return data -class OrganismUpdate(BaseModel): - """Schema for updating an existing organism.""" +class OrganismBase(BaseModel): + """Base Organism schema with common attributes.""" + taxon_id: int + scientific_name: Optional[str] = None bpa_scientific_name: Optional[str] = None bpa_genus: Optional[str] = None bpa_species: Optional[str] = None @@ -38,11 +40,19 @@ class OrganismUpdate(BaseModel): bpa_infraspecific_epithet: Optional[str] = None bpa_culture_or_strain_id: Optional[str] = None bpa_authority: Optional[str] = None - bpa_scientific_name: Optional[str] = None - atol_scientific_name: Optional[str] = None bpa_json: Optional[Dict] = None +class OrganismCreate(OrganismWriteBase): + """Schema for creating a new organism.""" + + taxon_id: int + + +class OrganismUpdate(OrganismWriteBase): + """Schema for updating an existing organism.""" + + class OrganismInDBBase(OrganismBase): """Base schema for Organism in DB, includes id and timestamps.""" diff --git a/app/schemas/taxonomy_info.py b/app/schemas/taxonomy_info.py index 460aec5..43468eb 100644 --- a/app/schemas/taxonomy_info.py +++ b/app/schemas/taxonomy_info.py @@ -1,3 +1,4 @@ +from datetime import datetime from typing import Any, Optional from pydantic import BaseModel, ConfigDict @@ -15,6 +16,7 @@ class TaxonomyInfoBase(BaseModel): ncbi_lineage: Optional[list[dict[str, Any]]] = None ncbi_tax_string: Optional[str] = None ncbi_full_lineage: Optional[str] = None + ncbi_last_synced_at: Optional[datetime] = None mito_ref: Optional[str] = None busco_dataset_name: Optional[str] = None busco_odb10_dataset_name: Optional[str] = None diff --git a/app/services/organism_service.py b/app/services/organism_service.py index 98edd9d..b97639a 100644 --- a/app/services/organism_service.py +++ b/app/services/organism_service.py @@ -18,6 +18,24 @@ logger = logging.getLogger(__name__) +def resolve_scientific_name( + bpa_scientific_name: Optional[str], + ncbi_scientific_name: Optional[str] = None, +) -> Optional[str]: + return ncbi_scientific_name or bpa_scientific_name + + +def sync_organism_scientific_name( + organism: Organism, + *, + ncbi_scientific_name: Optional[str] = None, +) -> None: + organism.scientific_name = resolve_scientific_name( + organism.bpa_scientific_name, + ncbi_scientific_name, + ) + + class OrganismService(BaseService[Organism, OrganismCreate, OrganismUpdate]): """Service for Organism operations.""" @@ -41,7 +59,7 @@ def get_multi_with_filters( """Get organisms with filters.""" query = db.query(Organism) if bpa_scientific_name: - query = query.filter(Organism.bpa_scientific_name.ilike(f"%{scientific_name}%")) + query = query.filter(Organism.bpa_scientific_name.ilike(f"%{bpa_scientific_name}%")) if taxon_id is not None: query = query.filter(Organism.taxon_id == taxon_id) return query.offset(skip).limit(limit).all() @@ -123,7 +141,7 @@ def get_organism_prepared_payload( response = OrganismSubmissionJsonResponse( taxon_id=organism.taxon_id, - bpa_scientific_name=organism.bpa_scientific_name, + scientific_name=organism.scientific_name, samples=[], experiments=[], reads=[], @@ -173,6 +191,7 @@ def create_organism(self, db: Session, *, organism_in: OrganismCreate) -> Organi logger.info("Creating organism taxon_id=%s label=%s", organism_in.taxon_id, organism_label) organism = Organism( taxon_id=organism_in.taxon_id, + scientific_name=organism_in.bpa_scientific_name, bpa_scientific_name=organism_in.bpa_scientific_name, bpa_common_name=organism_in.bpa_common_name, bpa_genus=organism_in.bpa_genus, @@ -286,6 +305,14 @@ def update_organism( new_bpa_json[field] = value organism.bpa_json = new_bpa_json + sync_organism_scientific_name( + organism, + ncbi_scientific_name=getattr( + getattr(organism, "taxonomy_info", None), + "ncbi_scientific_name", + None, + ), + ) db.add(organism) db.commit() db.refresh(organism) @@ -334,6 +361,7 @@ def bulk_import_organisms( # Create organism and projects organism = Organism( taxon_id=taxon_id, + scientific_name=scientific_name, bpa_common_name=organism_data.get("bpa_common_name"), bpa_genus=organism_data.get("bpa_genus"), bpa_species=organism_data.get("bpa_species"), diff --git a/app/services/taxonomy_info_service.py b/app/services/taxonomy_info_service.py index f934b7e..a74cd34 100644 --- a/app/services/taxonomy_info_service.py +++ b/app/services/taxonomy_info_service.py @@ -1,4 +1,5 @@ import logging +from datetime import datetime, timezone from typing import Any, Dict, List, Optional from sqlalchemy.orm import Session @@ -8,6 +9,7 @@ from app.schemas.bulk_import import BulkImportResponse from app.schemas.taxonomy_info import TaxonomyInfoCreate, TaxonomyInfoUpdate from app.services.ncbi_taxonomy_service import fetch_taxonomy_for_taxon_ids +from app.services.organism_service import sync_organism_scientific_name logger = logging.getLogger(__name__) @@ -43,6 +45,9 @@ def create(self, db: Session, *, ti_in: TaxonomyInfoCreate) -> TaxonomyInfo: organism = db.query(Organism).filter(Organism.taxon_id == ti_in.taxon_id).first() if not organism: raise ValueError(f"Organism with taxon_id {ti_in.taxon_id} does not exist") + existing = db.query(TaxonomyInfo).filter(TaxonomyInfo.taxon_id == ti_in.taxon_id).first() + if existing: + raise ValueError(f"TaxonomyInfo for taxon_id {ti_in.taxon_id} already exists") ti = self.populate_from_ncbi_lookup( db, @@ -56,6 +61,10 @@ def create(self, db: Session, *, ti_in: TaxonomyInfoCreate) -> TaxonomyInfo: db.add(ti) self._apply_payload_values(ti, ti_in.model_dump(exclude_unset=True)) + sync_organism_scientific_name( + organism, + ncbi_scientific_name=getattr(ti, "ncbi_scientific_name", None), + ) db.commit() db.refresh(ti) return ti @@ -93,6 +102,11 @@ def populate_from_ncbi_lookup( created = True applied_fields = self._apply_ncbi_values(ti, mapped) + ti.ncbi_last_synced_at = datetime.now(timezone.utc) + sync_organism_scientific_name( + organism, + ncbi_scientific_name=ti.ncbi_scientific_name, + ) db.flush() logger.info( "NCBI taxonomy enrichment %s taxonomy_info for taxon_id=%s; applied_fields=%s", @@ -113,8 +127,15 @@ def update( ti = db.query(TaxonomyInfo).filter(TaxonomyInfo.taxon_id == taxon_id).first() if not ti: return None + organism = db.query(Organism).filter(Organism.taxon_id == taxon_id).first() for field, value in ti_in.model_dump(exclude_unset=True).items(): setattr(ti, field, value) + if organism: + sync_organism_scientific_name( + organism, + ncbi_scientific_name=ti.ncbi_scientific_name, + ) + db.add(organism) db.add(ti) db.commit() db.refresh(ti) @@ -124,6 +145,10 @@ def delete(self, db: Session, *, taxon_id: int) -> Optional[TaxonomyInfo]: ti = db.query(TaxonomyInfo).filter(TaxonomyInfo.taxon_id == taxon_id).first() if not ti: return None + organism = db.query(Organism).filter(Organism.taxon_id == taxon_id).first() + if organism: + sync_organism_scientific_name(organism, ncbi_scientific_name=None) + db.add(organism) db.delete(ti) db.commit() return ti @@ -150,6 +175,11 @@ def bulk_import( errors.append(f"{taxon_id}: organism with taxon_id {taxon_id} does not exist") skipped_count += 1 continue + existing = db.query(TaxonomyInfo).filter(TaxonomyInfo.taxon_id == taxon_id).first() + if existing: + errors.append(f"{taxon_id}: taxonomy_info for taxon_id {taxon_id} already exists") + skipped_count += 1 + continue except Exception as e: errors.append(f"{taxon_id}: {str(e)}") @@ -167,21 +197,18 @@ def bulk_import( if unmapped: logger.warning("NCBI bulk enrichment returned unmapped taxon_ids: %s", unmapped) - for taxon_id, row, _organism in candidates: + for taxon_id, row, organism in candidates: try: - ti = db.query(TaxonomyInfo).filter(TaxonomyInfo.taxon_id == taxon_id).first() - created = False - if not ti: - ti = TaxonomyInfo(taxon_id=taxon_id) - db.add(ti) - created = True + ti = TaxonomyInfo(taxon_id=taxon_id) + db.add(ti) mapped = ncbi_by_taxon_id.get(taxon_id) if mapped: applied_fields = self._apply_ncbi_values(ti, mapped) + ti.ncbi_last_synced_at = datetime.now(timezone.utc) logger.info( "NCBI taxonomy enrichment %s taxonomy_info for taxon_id=%s; applied_fields=%s", - "created" if created else "updated", + "created", taxon_id, applied_fields, ) @@ -192,6 +219,11 @@ def bulk_import( ) self._apply_payload_values(ti, row.model_dump(exclude_unset=True)) + sync_organism_scientific_name( + organism, + ncbi_scientific_name=ti.ncbi_scientific_name, + ) + db.add(organism) db.commit() created_count += 1 except Exception as e: diff --git a/schema.sql b/schema.sql index 3f1c06d..ed60329 100644 --- a/schema.sql +++ b/schema.sql @@ -94,12 +94,9 @@ CREATE TABLE organism ( CREATE TABLE taxonomy_info ( taxon_id INT PRIMARY KEY REFERENCES organism(taxon_id) ON DELETE CASCADE, ncbi_taxon_id INTEGER, - ncbi_rank TEXT, ncbi_scientific_name TEXT, ncbi_authority TEXT, - ncbi_scientific_name TEXT, - ncbi_authority TEXT, ncbi_common_name TEXT, ncbi_class TEXT, ncbi_order TEXT, @@ -107,8 +104,8 @@ CREATE TABLE taxonomy_info ( ncbi_lineage JSONB, ncbi_tax_string TEXT, ncbi_full_lineage TEXT, - mito_ref TEXT - + ncbi_last_synced_at TIMESTAMPTZ, + mito_ref TEXT, busco_dataset_name TEXT, busco_odb10_dataset_name TEXT, busco_odb12_dataset_name TEXT, @@ -117,6 +114,7 @@ CREATE TABLE taxonomy_info ( mitochondrial_genetic_code_id INTEGER, mitohifi_reference_species TEXT, oatk_hmm_name TEXT, + defined_class TEXT, augustus_dataset_name TEXT, genetic_code_id INTEGER ); diff --git a/tests/unit/endpoints/test_endpoints_organisms.py b/tests/unit/endpoints/test_endpoints_organisms.py index 1613c24..03dd5a3 100644 --- a/tests/unit/endpoints/test_endpoints_organisms.py +++ b/tests/unit/endpoints/test_endpoints_organisms.py @@ -51,20 +51,14 @@ def test_organisms_list_and_not_found(monkeypatch): base_org = { "taxon_id": 1, "scientific_name": "Sci", - "common_name": "Com", - "common_name_source": None, - "genus": None, - "species": None, - "infraspecific_epithet": None, - "culture_or_strain_id": None, - "authority": None, - "atol_scientific_name": None, - "tax_string": None, - "ncbi_order": None, - "ncbi_family": None, - "busco_dataset_name": None, + "bpa_scientific_name": "Sci", + "bpa_common_name": "Com", + "bpa_genus": None, + "bpa_species": None, + "bpa_infraspecific_epithet": None, + "bpa_culture_or_strain_id": None, + "bpa_authority": None, "bpa_json": None, - "taxonomy_lineage_json": None, "taxonomy_info": None, "created_at": now, "updated_at": now, @@ -95,9 +89,8 @@ def test_create_organism(monkeypatch): fake_service = SimpleNamespace( create_organism=lambda db, organism_in: { **organism_in.model_dump(), - "common_name_source": None, + "scientific_name": organism_in.bpa_scientific_name, "bpa_json": None, - "taxonomy_lineage_json": None, "created_at": now, "updated_at": now, } @@ -125,9 +118,8 @@ def test_create_organism_without_scientific_name(monkeypatch): fake_service = SimpleNamespace( create_organism=lambda db, organism_in: { **organism_in.model_dump(), - "common_name_source": None, + "scientific_name": organism_in.bpa_scientific_name, "bpa_json": None, - "taxonomy_lineage_json": None, "created_at": now, "updated_at": now, } diff --git a/tests/unit/services/test_taxonomy_info_service.py b/tests/unit/services/test_taxonomy_info_service.py index 8f9af35..3439dc3 100644 --- a/tests/unit/services/test_taxonomy_info_service.py +++ b/tests/unit/services/test_taxonomy_info_service.py @@ -43,6 +43,10 @@ def add(self, obj): self.data.setdefault(type(obj), {}) self.data[type(obj)][obj.taxon_id] = obj + def delete(self, obj): + store = self.data.get(type(obj), {}) + store.pop(obj.taxon_id, None) + def commit(self): self.commit_count += 1 @@ -94,6 +98,8 @@ def test_populate_from_ncbi_lookup_creates_taxonomy_info(monkeypatch): assert ti.ncbi_rank == "species" assert ti.ncbi_order == "Eurotiales" assert ti.mito_ref == "Penicillium chrysogenum" + assert ti.ncbi_last_synced_at is not None + assert organism.scientific_name == "Penicillium test" assert db.data[TaxonomyInfo][5077] is ti assert db.flush_count == 1 assert db.commit_count == 0 @@ -134,8 +140,10 @@ def test_create_taxonomy_info_fetches_ncbi_and_applies_payload(monkeypatch): assert calls == [({5303: "Agaricus"}, 20)] assert ti.ncbi_taxon_id == 5303 assert ti.ncbi_rank == "species" + assert ti.ncbi_last_synced_at is not None assert ti.genetic_code_id == 11 assert ti.augustus_dataset_name == "agaricus_aug" + assert organism.scientific_name == "Agaricus test" assert db.commit_count == 1 assert db.refresh_count == 1 @@ -167,12 +175,14 @@ def test_bulk_import_batches_ncbi_lookup_and_creates_taxonomy_info(monkeypatch): "taxon_id": 9612, "ncbi_taxon_id": 9612, "ncbi_rank": "species", + "ncbi_scientific_name": "Canis lupus familiaris", "mito_ref": "Canis lupus familiaris", }, 9685: { "taxon_id": 9685, "ncbi_taxon_id": 9685, "ncbi_rank": "species", + "ncbi_scientific_name": "Felis silvestris catus", "mito_ref": "Felis silvestris catus", }, }, @@ -198,11 +208,31 @@ def test_bulk_import_batches_ncbi_lookup_and_creates_taxonomy_info(monkeypatch): assert saved_dog.ncbi_taxon_id == 9612 assert saved_dog.ncbi_rank == "species" assert saved_dog.mito_ref == "Canis lupus familiaris" + assert saved_dog.ncbi_last_synced_at is not None assert saved_dog.genetic_code_id == 2 assert saved_cat.ncbi_taxon_id == 9685 assert saved_cat.ncbi_rank == "species" assert saved_cat.mito_ref == "Felis silvestris catus" + assert saved_cat.ncbi_last_synced_at is not None assert saved_cat.genetic_code_id == 1 + assert organisms[9612].scientific_name == "Canis lupus familiaris" + assert organisms[9685].scientific_name == "Felis silvestris catus" + + +def test_delete_taxonomy_info_falls_back_to_bpa_scientific_name(): + organism = Organism( + taxon_id=5303, + scientific_name="Agaricus test", + bpa_scientific_name="Agaricus", + ) + ti = TaxonomyInfo(taxon_id=5303, ncbi_scientific_name="Agaricus test") + db = _Session({Organism: {5303: organism}, TaxonomyInfo: {5303: ti}}) + + deleted = ti_service_module.taxonomy_info_service.delete(db, taxon_id=5303) + + assert deleted is ti + assert organism.scientific_name == "Agaricus" + assert db.commit_count == 1 def test_bulk_import_schema_rejects_ncbi_fields(): From dbf9bb1806f74ebce59c577f0f6b1d05e4d143cc Mon Sep 17 00:00:00 2001 From: Emily Marshall Date: Mon, 18 May 2026 21:16:33 +1000 Subject: [PATCH 11/14] feat: add shim for legacy data structure, until data mapper is updated --- app/schemas/organism.py | 24 +++++++++++++++++-- app/services/organism_service.py | 17 ++++++++----- .../endpoints/test_endpoints_organisms.py | 7 ++++++ 3 files changed, 40 insertions(+), 8 deletions(-) diff --git a/app/schemas/organism.py b/app/schemas/organism.py index a09a7d3..04c1cc3 100644 --- a/app/schemas/organism.py +++ b/app/schemas/organism.py @@ -22,9 +22,29 @@ class OrganismWriteBase(BaseModel): @model_validator(mode="before") @classmethod def _coerce_scientific_name(cls, data): - if isinstance(data, dict) and "bpa_scientific_name" not in data and "scientific_name" in data: + # TODO: Remove this legacy field mapping once all callers send bpa_* organism fields. + if not isinstance(data, dict): + return data + + legacy_to_bpa = { + "scientific_name": "bpa_scientific_name", + "genus": "bpa_genus", + "species": "bpa_species", + "common_name": "bpa_common_name", + "infraspecific_epithet": "bpa_infraspecific_epithet", + "culture_or_strain_id": "bpa_culture_or_strain_id", + "authority": "bpa_authority", + } + + needs_copy = any( + bpa_field not in data and legacy_field in data + for legacy_field, bpa_field in legacy_to_bpa.items() + ) + if needs_copy: data = dict(data) - data["bpa_scientific_name"] = data["scientific_name"] + for legacy_field, bpa_field in legacy_to_bpa.items(): + if bpa_field not in data and legacy_field in data: + data[bpa_field] = data[legacy_field] return data diff --git a/app/services/organism_service.py b/app/services/organism_service.py index b97639a..dc24c3e 100644 --- a/app/services/organism_service.py +++ b/app/services/organism_service.py @@ -362,12 +362,17 @@ def bulk_import_organisms( organism = Organism( taxon_id=taxon_id, scientific_name=scientific_name, - bpa_common_name=organism_data.get("bpa_common_name"), - bpa_genus=organism_data.get("bpa_genus"), - bpa_species=organism_data.get("bpa_species"), - bpa_infraspecific_epithet=organism_data.get("bpa_infraspecific_epithet"), - bpa_culture_or_strain_id=organism_data.get("bpa_culture_or_strain_id"), - bpa_authority=organism_data.get("bpa_authority"), + # TODO: Remove these legacy fallbacks once bulk organism callers send bpa_* fields only. + bpa_common_name=organism_data.get("bpa_common_name") + or organism_data.get("common_name"), + bpa_genus=organism_data.get("bpa_genus") or organism_data.get("genus"), + bpa_species=organism_data.get("bpa_species") or organism_data.get("species"), + bpa_infraspecific_epithet=organism_data.get("bpa_infraspecific_epithet") + or organism_data.get("infraspecific_epithet"), + bpa_culture_or_strain_id=organism_data.get("bpa_culture_or_strain_id") + or organism_data.get("culture_or_strain_id"), + bpa_authority=organism_data.get("bpa_authority") + or organism_data.get("authority"), bpa_scientific_name=scientific_name, bpa_json=organism_data, ) diff --git a/tests/unit/endpoints/test_endpoints_organisms.py b/tests/unit/endpoints/test_endpoints_organisms.py index 03dd5a3..ed4cbd8 100644 --- a/tests/unit/endpoints/test_endpoints_organisms.py +++ b/tests/unit/endpoints/test_endpoints_organisms.py @@ -100,12 +100,19 @@ def test_create_organism(monkeypatch): "taxon_id": 1, "scientific_name": "Sci", "common_name": "Com", + "genus": "Canis", + "species": "lupus", } resp = client.post("/api/v1/organisms", json=payload) assert resp.status_code == 200 body = resp.json() assert body["taxon_id"] == 1 + assert body["scientific_name"] == "Sci" + assert body["bpa_scientific_name"] == "Sci" + assert body["bpa_common_name"] == "Com" + assert body["bpa_genus"] == "Canis" + assert body["bpa_species"] == "lupus" def test_create_organism_without_scientific_name(monkeypatch): From 711659ae3f58eb44cdfa1c8f59b84e18474eede0 Mon Sep 17 00:00:00 2001 From: Emily Marshall Date: Mon, 18 May 2026 21:26:37 +1000 Subject: [PATCH 12/14] docs: organism.scientific_name and taxonomy_info.last_synced_at behaviour --- docs/ncbi_taxonomy_sync.md | 127 +++++++++++++++++++++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 docs/ncbi_taxonomy_sync.md diff --git a/docs/ncbi_taxonomy_sync.md b/docs/ncbi_taxonomy_sync.md new file mode 100644 index 0000000..5e6fdf3 --- /dev/null +++ b/docs/ncbi_taxonomy_sync.md @@ -0,0 +1,127 @@ +## NCBI taxonomy sync behavior + +This document describes the current application-level behavior for organism scientific names and NCBI taxonomy enrichment. + +### Field roles + +`organism.bpa_scientific_name` +- Scientific name supplied by the caller / BPA source payload. + +`taxonomy_info.ncbi_scientific_name` +- Scientific name returned by the NCBI taxonomy lookup. + +`organism.scientific_name` +- Canonical app-facing scientific name. +- This is a derived field maintained by application logic. + +`taxonomy_info.ncbi_last_synced_at` +- Timestamp of the latest successful NCBI sync. +- Only updated when an NCBI lookup succeeds and returns mapped taxonomy data. + +### Canonical scientific name rule + +The canonical rule is: + +```text +organism.scientific_name = + taxonomy_info.ncbi_scientific_name if present + else organism.bpa_scientific_name +``` + +This rule is recomputed during relevant writes instead of using a one-time copy or "set if null" behavior. + +### Current write behavior + +#### Organism create + +When a new organism is created: +- `organism.bpa_scientific_name` is stored from the caller payload +- `organism.scientific_name` is initially set to `bpa_scientific_name` + +#### Organism update + +When an organism is updated: +- BPA/source fields are updated +- `organism.scientific_name` is recomputed +- if related `taxonomy_info.ncbi_scientific_name` exists, it still takes precedence + +#### TaxonomyInfo create + +When `taxonomy_info` is created: +- the service performs an NCBI taxonomy lookup +- mapped NCBI fields are stored on `taxonomy_info` +- `taxonomy_info.ncbi_last_synced_at` is set to the current timestamp +- `organism.scientific_name` is recomputed using the canonical rule + +#### TaxonomyInfo bulk import + +When `taxonomy_info` is bulk imported: +- taxon IDs are collected first +- NCBI lookup is performed in batches +- mapped NCBI fields are stored per row +- `taxonomy_info.ncbi_last_synced_at` is set for successfully mapped rows +- `organism.scientific_name` is recomputed per organism + +#### TaxonomyInfo delete + +When `taxonomy_info` is deleted: +- `organism.scientific_name` falls back to `organism.bpa_scientific_name` + +### Current endpoint semantics + +`POST /api/v1/organisms` +- Creates organism rows and related project records. +- Does not trigger NCBI taxonomy enrichment. + +`POST /api/v1/taxonomy-info` +- Creates a taxonomy info row and performs NCBI enrichment. + +`POST /api/v1/taxonomy-info/bulk-import` +- Bulk creates taxonomy info rows and performs batched NCBI enrichment. + +At the moment, the taxonomy-info create/bulk-import flows are treated as insert-oriented paths rather than explicit upsert endpoints. + +### Caller-owned vs NCBI-owned fields + +Callers should not provide `ncbi_*` fields in taxonomy-info request payloads. + +Those fields are lookup-owned and come from the NCBI enrichment step. This is enforced by the taxonomy-info write schemas. + +### Temporary organism field compatibility shim + +The API is in a transition period where some callers still send legacy organism field names without the `bpa_` prefix. + +Current temporary input mapping: +- `scientific_name -> bpa_scientific_name` +- `genus -> bpa_genus` +- `species -> bpa_species` +- `common_name -> bpa_common_name` +- `infraspecific_epithet -> bpa_infraspecific_epithet` +- `culture_or_strain_id -> bpa_culture_or_strain_id` +- `authority -> bpa_authority` + +This compatibility exists in two places: + +1. `app/schemas/organism.py` +- The Pydantic write schema maps legacy input fields to `bpa_*` fields before validation. + +2. `app/services/organism_service.py` +- The raw `bulk_import_organisms(...)` path applies the same legacy fallbacks because it bypasses the organism Pydantic schema. + +### Cleanup once callers are updated + +Once all callers send only `bpa_*` organism fields, remove: + +1. The legacy field coercion validator in `app/schemas/organism.py` +2. The legacy fallback reads in `app/services/organism_service.py` bulk import + +These sections are marked with `TODO` comments in the code. + +### Why this design was chosen + +This design keeps: +- provenance fields separate (`bpa_scientific_name`, `ncbi_scientific_name`) +- one canonical field for the rest of the app (`organism.scientific_name`) +- application logic simple for read paths that need a single scientific name + +It also avoids hidden state by always recomputing the canonical field from a deterministic precedence rule. From 24e8e931c1d7f6b9529984203f5d78032923dcc0 Mon Sep 17 00:00:00 2001 From: Emily Marshall Date: Mon, 18 May 2026 22:16:22 +1000 Subject: [PATCH 13/14] feat: archive old migrations --- .../0002_timestamptz_broker_idx.py | 76 +- .../0003_add_tol_id_to_assembly.py | 4 +- .../0004_add_assembly_run.py | 0 .../0005_org_sci_name_nullable.py | 0 ...consolidated_submission_schema_refactor.py | 14 +- .../0007_add_qc_read_tables.py | 0 .../0008_add_base_url_to_experiment.py | 0 .../0009_use_taxon_id_as_organism_pk.py | 0 .../0010_add_taxonomy_info.py | 0 .../0011_assembly_first_and_stages.py | 0 .../0012_assembly_manifest_columns.py | 0 .../0013_remove_assembly_submission_xml.py | 4 +- .../0014_add_ncbi_taxonomy_details.py | 0 alembic/base_schema/0001_initial_schema.sql | 806 ++++++++++++++++++ alembic/versions/0001_initial_schema.py | 4 +- docs/migration_workflow.md | 9 + 16 files changed, 899 insertions(+), 18 deletions(-) rename alembic/{versions => archive}/0002_timestamptz_broker_idx.py (72%) rename alembic/{versions => archive}/0003_add_tol_id_to_assembly.py (83%) rename alembic/{versions => archive}/0004_add_assembly_run.py (100%) rename alembic/{versions => archive}/0005_org_sci_name_nullable.py (100%) rename alembic/{versions => archive}/0006_consolidated_submission_schema_refactor.py (91%) rename alembic/{versions => archive}/0007_add_qc_read_tables.py (100%) rename alembic/{versions => archive}/0008_add_base_url_to_experiment.py (100%) rename alembic/{versions => archive}/0009_use_taxon_id_as_organism_pk.py (100%) rename alembic/{versions => archive}/0010_add_taxonomy_info.py (100%) rename alembic/{versions => archive}/0011_assembly_first_and_stages.py (100%) rename alembic/{versions => archive}/0012_assembly_manifest_columns.py (100%) rename alembic/{versions => archive}/0013_remove_assembly_submission_xml.py (86%) rename alembic/{versions => archive}/0014_add_ncbi_taxonomy_details.py (100%) create mode 100644 alembic/base_schema/0001_initial_schema.sql diff --git a/alembic/versions/0002_timestamptz_broker_idx.py b/alembic/archive/0002_timestamptz_broker_idx.py similarity index 72% rename from alembic/versions/0002_timestamptz_broker_idx.py rename to alembic/archive/0002_timestamptz_broker_idx.py index 049a865..b2133d5 100644 --- a/alembic/versions/0002_timestamptz_broker_idx.py +++ b/alembic/archive/0002_timestamptz_broker_idx.py @@ -9,6 +9,50 @@ depends_on = None +def _alter_column_to_timestamptz(table: str, col: str) -> None: + op.execute( + f""" + DO $$ BEGIN + IF EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_schema = 'public' + AND table_name = '{table}' + AND column_name = '{col}' + AND data_type = 'timestamp without time zone' + ) THEN + ALTER TABLE {table} + ALTER COLUMN {col} + TYPE TIMESTAMPTZ + USING {col} AT TIME ZONE 'UTC'; + END IF; + END $$; + """ + ) + + +def _alter_column_to_timestamp(table: str, col: str) -> None: + op.execute( + f""" + DO $$ BEGIN + IF EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_schema = 'public' + AND table_name = '{table}' + AND column_name = '{col}' + AND data_type = 'timestamp with time zone' + ) THEN + ALTER TABLE {table} + ALTER COLUMN {col} + TYPE TIMESTAMP + USING {col}::timestamp; + END IF; + END $$; + """ + ) + + def upgrade() -> None: # Timestamp columns -> TIMESTAMPTZ (treat existing values as UTC) tables = { @@ -42,6 +86,7 @@ def upgrade() -> None: ], "read": ["created_at", "updated_at"], "read_submission": ["created_at", "updated_at", "lock_acquired_at", "lock_expires_at"], + "qc_read_submission": ["created_at", "updated_at", "lock_acquired_at", "lock_expires_at"], "submission_attempt": ["lock_acquired_at", "lock_expires_at", "created_at", "updated_at"], "submission_event": ["at"], "assembly": ["created_at", "updated_at"], @@ -53,9 +98,7 @@ def upgrade() -> None: for table, columns in tables.items(): for col in columns: - op.execute( - f"ALTER TABLE {table} ALTER COLUMN {col} TYPE TIMESTAMPTZ USING {col} AT TIME ZONE 'UTC'" - ) + _alter_column_to_timestamptz(table, col) # Sample latitude/longitude checks (skip if already exists from schema.sql) op.execute( @@ -102,9 +145,25 @@ def upgrade() -> None: op.execute( "CREATE INDEX IF NOT EXISTS idx_experiment_submission_lock_expires_at ON experiment_submission (lock_expires_at)" ) - op.execute("CREATE INDEX IF NOT EXISTS idx_read_submission_status ON read_submission (status)") op.execute( - "CREATE INDEX IF NOT EXISTS idx_read_submission_lock_expires_at ON read_submission (lock_expires_at)" + """ + DO $$ BEGIN + IF to_regclass('public.read_submission') IS NOT NULL THEN + CREATE INDEX IF NOT EXISTS idx_read_submission_status ON read_submission (status); + CREATE INDEX IF NOT EXISTS idx_read_submission_lock_expires_at ON read_submission (lock_expires_at); + END IF; + END $$; + """ + ) + op.execute( + """ + DO $$ BEGIN + IF to_regclass('public.qc_read_submission') IS NOT NULL THEN + CREATE INDEX IF NOT EXISTS idx_qc_read_submission_status ON qc_read_submission (status); + CREATE INDEX IF NOT EXISTS idx_qc_read_submission_lock_expires_at ON qc_read_submission (lock_expires_at); + END IF; + END $$; + """ ) op.execute( "CREATE INDEX IF NOT EXISTS idx_submission_attempt_status ON submission_attempt (status)" @@ -118,6 +177,8 @@ def downgrade() -> None: # Drop indexes op.execute("DROP INDEX IF EXISTS idx_submission_attempt_lock_expires_at") op.execute("DROP INDEX IF EXISTS idx_submission_attempt_status") + op.execute("DROP INDEX IF EXISTS idx_qc_read_submission_lock_expires_at") + op.execute("DROP INDEX IF EXISTS idx_qc_read_submission_status") op.execute("DROP INDEX IF EXISTS idx_read_submission_lock_expires_at") op.execute("DROP INDEX IF EXISTS idx_read_submission_status") op.execute("DROP INDEX IF EXISTS idx_experiment_submission_lock_expires_at") @@ -163,6 +224,7 @@ def downgrade() -> None: ], "read": ["created_at", "updated_at"], "read_submission": ["created_at", "updated_at", "lock_acquired_at", "lock_expires_at"], + "qc_read_submission": ["created_at", "updated_at", "lock_acquired_at", "lock_expires_at"], "submission_attempt": ["lock_acquired_at", "lock_expires_at", "created_at", "updated_at"], "submission_event": ["at"], "assembly": ["created_at", "updated_at"], @@ -174,6 +236,4 @@ def downgrade() -> None: for table, columns in tables.items(): for col in columns: - op.execute( - f"ALTER TABLE {table} ALTER COLUMN {col} TYPE TIMESTAMP USING {col}::timestamp" - ) + _alter_column_to_timestamp(table, col) diff --git a/alembic/versions/0003_add_tol_id_to_assembly.py b/alembic/archive/0003_add_tol_id_to_assembly.py similarity index 83% rename from alembic/versions/0003_add_tol_id_to_assembly.py rename to alembic/archive/0003_add_tol_id_to_assembly.py index 0054b94..b58ace9 100644 --- a/alembic/versions/0003_add_tol_id_to_assembly.py +++ b/alembic/archive/0003_add_tol_id_to_assembly.py @@ -17,8 +17,8 @@ def upgrade(): - op.add_column("assembly", sa.Column("tol_id", sa.Text(), nullable=True)) + op.add_column("assembly", sa.Column("tol_id", sa.Text(), nullable=True), if_not_exists=True) def downgrade(): - op.drop_column("assembly", "tol_id") + op.drop_column("assembly", "tol_id", if_exists=True) diff --git a/alembic/versions/0004_add_assembly_run.py b/alembic/archive/0004_add_assembly_run.py similarity index 100% rename from alembic/versions/0004_add_assembly_run.py rename to alembic/archive/0004_add_assembly_run.py diff --git a/alembic/versions/0005_org_sci_name_nullable.py b/alembic/archive/0005_org_sci_name_nullable.py similarity index 100% rename from alembic/versions/0005_org_sci_name_nullable.py rename to alembic/archive/0005_org_sci_name_nullable.py diff --git a/alembic/versions/0006_consolidated_submission_schema_refactor.py b/alembic/archive/0006_consolidated_submission_schema_refactor.py similarity index 91% rename from alembic/versions/0006_consolidated_submission_schema_refactor.py rename to alembic/archive/0006_consolidated_submission_schema_refactor.py index 2648e36..f45838b 100644 --- a/alembic/versions/0006_consolidated_submission_schema_refactor.py +++ b/alembic/archive/0006_consolidated_submission_schema_refactor.py @@ -30,6 +30,7 @@ def upgrade(): op.add_column( "sample_submission", sa.Column("project_id", postgresql.UUID(as_uuid=True), nullable=False), + if_not_exists=True, ) op.create_foreign_key( "fk_sample_submission_project_id", @@ -43,12 +44,14 @@ def upgrade(): "sample_submission", ["project_id"], unique=False, + if_not_exists=True, ) # 2) Add experiment.project_id (NOT NULL, FK to project with CASCADE) op.add_column( "experiment", sa.Column("project_id", postgresql.UUID(as_uuid=True), nullable=False), + if_not_exists=True, ) op.create_foreign_key( "fk_experiment_project_id", @@ -63,6 +66,7 @@ def upgrade(): "experiment", ["project_id"], unique=False, + if_not_exists=True, ) # 3) Drop read_submission.project_id (derivable via experiment) @@ -126,11 +130,13 @@ def downgrade(): ) # 2) Drop experiment.project_id - op.drop_index("idx_experiment_project_id", table_name="experiment") + op.drop_index("idx_experiment_project_id", table_name="experiment", if_exists=True) op.drop_constraint("fk_experiment_project_id", "experiment", type_="foreignkey") - op.drop_column("experiment", "project_id") + op.drop_column("experiment", "project_id", if_exists=True) # 1) Drop sample_submission.project_id - op.drop_index("idx_sample_submission_project_id", table_name="sample_submission") + op.drop_index( + "idx_sample_submission_project_id", table_name="sample_submission", if_exists=True + ) op.drop_constraint("fk_sample_submission_project_id", "sample_submission", type_="foreignkey") - op.drop_column("sample_submission", "project_id") + op.drop_column("sample_submission", "project_id", if_exists=True) diff --git a/alembic/versions/0007_add_qc_read_tables.py b/alembic/archive/0007_add_qc_read_tables.py similarity index 100% rename from alembic/versions/0007_add_qc_read_tables.py rename to alembic/archive/0007_add_qc_read_tables.py diff --git a/alembic/versions/0008_add_base_url_to_experiment.py b/alembic/archive/0008_add_base_url_to_experiment.py similarity index 100% rename from alembic/versions/0008_add_base_url_to_experiment.py rename to alembic/archive/0008_add_base_url_to_experiment.py diff --git a/alembic/versions/0009_use_taxon_id_as_organism_pk.py b/alembic/archive/0009_use_taxon_id_as_organism_pk.py similarity index 100% rename from alembic/versions/0009_use_taxon_id_as_organism_pk.py rename to alembic/archive/0009_use_taxon_id_as_organism_pk.py diff --git a/alembic/versions/0010_add_taxonomy_info.py b/alembic/archive/0010_add_taxonomy_info.py similarity index 100% rename from alembic/versions/0010_add_taxonomy_info.py rename to alembic/archive/0010_add_taxonomy_info.py diff --git a/alembic/versions/0011_assembly_first_and_stages.py b/alembic/archive/0011_assembly_first_and_stages.py similarity index 100% rename from alembic/versions/0011_assembly_first_and_stages.py rename to alembic/archive/0011_assembly_first_and_stages.py diff --git a/alembic/versions/0012_assembly_manifest_columns.py b/alembic/archive/0012_assembly_manifest_columns.py similarity index 100% rename from alembic/versions/0012_assembly_manifest_columns.py rename to alembic/archive/0012_assembly_manifest_columns.py diff --git a/alembic/versions/0013_remove_assembly_submission_xml.py b/alembic/archive/0013_remove_assembly_submission_xml.py similarity index 86% rename from alembic/versions/0013_remove_assembly_submission_xml.py rename to alembic/archive/0013_remove_assembly_submission_xml.py index eced902..25eac0c 100644 --- a/alembic/versions/0013_remove_assembly_submission_xml.py +++ b/alembic/archive/0013_remove_assembly_submission_xml.py @@ -1,6 +1,6 @@ """Remove submission_xml from assembly_submission table. -Revision ID: 0013_remove_assembly_submission_xml +Revision ID: 0013_remove_assembly_sub_col Revises: 0012_assembly_manifest_columns Create Date: 2026-05-11 @@ -12,7 +12,7 @@ from alembic import op -revision = "0013_remove_assembly_submission_xml" +revision = "0013_remove_assembly_sub_col" down_revision = "0012_assembly_manifest_columns" branch_labels = None depends_on = None diff --git a/alembic/versions/0014_add_ncbi_taxonomy_details.py b/alembic/archive/0014_add_ncbi_taxonomy_details.py similarity index 100% rename from alembic/versions/0014_add_ncbi_taxonomy_details.py rename to alembic/archive/0014_add_ncbi_taxonomy_details.py diff --git a/alembic/base_schema/0001_initial_schema.sql b/alembic/base_schema/0001_initial_schema.sql new file mode 100644 index 0000000..ed60329 --- /dev/null +++ b/alembic/base_schema/0001_initial_schema.sql @@ -0,0 +1,806 @@ +-- PostgreSQL schema for biological metadata tracking system +-- Based on ER diagram and requirements +-- NOTE: This file is kept in sync with the database after each migration +-- Regenerate with: docker compose exec db pg_dump -U -d --schema-only --no-owner --no-acl + +-- Enable UUID extension +CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; + +-- Create ENUM types +CREATE TYPE submission_status AS ENUM ('draft', 'ready', 'submitting', 'rejected', 'accepted', 'replaced'); +CREATE TYPE authority_type AS ENUM ('ENA', 'NCBI', 'DDBJ'); +CREATE TYPE molecule_type AS ENUM ('genomic DNA', 'genomic RNA'); +CREATE TYPE assembly_data_types AS ENUM ( + 'PACBIO_SMRT', + 'PACBIO_SMRT_HIC', + 'OXFORD_NANOPORE', + 'OXFORD_NANOPORE_HIC', + 'PACBIO_SMRT_OXFORD_NANOPORE', + 'PACBIO_SMRT_OXFORD_NANOPORE_HIC' +); +CREATE TYPE assembly_file_type AS ENUM ( + 'FASTA', + 'QC_REPORT', + 'STATISTICS', + 'OTHER' +); +CREATE TYPE entity_type AS ENUM ('organism', 'sample', 'experiment', 'read', 'assembly', 'project', 'qc_read'); +CREATE TYPE project_type AS ENUM ('root', 'genomic_data', 'assembly'); +CREATE TYPE sample_kind AS ENUM ('specimen', 'derived'); +-- ========================================== +-- Users and Authentication +-- ========================================== + +CREATE TABLE users ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + username TEXT NOT NULL UNIQUE, + email TEXT UNIQUE, + hashed_password TEXT NOT NULL, + full_name TEXT, + roles TEXT[] NOT NULL DEFAULT '{}', + is_active BOOLEAN NOT NULL DEFAULT TRUE, + is_superuser BOOLEAN NOT NULL DEFAULT FALSE, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +-- ========================================== +-- Authentication refresh tokens table +-- ========================================== + +CREATE TABLE refresh_token ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + token_hash TEXT NOT NULL, + user_id UUID NOT NULL REFERENCES users(id), + expires_at TIMESTAMPTZ NOT NULL, + revoked BOOLEAN NOT NULL DEFAULT FALSE, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +-- ========================================== +-- Organism tables +-- ========================================== + +-- Main organism table +CREATE TABLE organism ( + taxon_id int PRIMARY KEY, + -- We need to check that the scientific name = the tax id level, because we have a bunch of tax_ids that are the same for organisms which should have a more granular taxid level + bpa_scientific_name TEXT, + bpa_genus TEXT, + bpa_species TEXT, + bpa_common_name TEXT, + -- TODO check common name is coming through from mapper, and set common_name_source + -- family TEXT, + -- order_or_group TEXT, + -- class TEXT, + -- phylum TEXT, + bpa_infraspecific_epithet TEXT, + bpa_culture_or_strain_id TEXT, + bpa_authority TEXT, + scientific_name TEXT, + + -- # TODO delete bpa_json + bpa_json JSONB, + + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +-- ========================================== +-- Taxonomy info table (1:1 extension of organism) +-- ========================================== + +CREATE TABLE taxonomy_info ( + taxon_id INT PRIMARY KEY REFERENCES organism(taxon_id) ON DELETE CASCADE, + ncbi_taxon_id INTEGER, + ncbi_rank TEXT, + ncbi_scientific_name TEXT, + ncbi_authority TEXT, + ncbi_common_name TEXT, + ncbi_class TEXT, + ncbi_order TEXT, + ncbi_family TEXT, + ncbi_lineage JSONB, + ncbi_tax_string TEXT, + ncbi_full_lineage TEXT, + ncbi_last_synced_at TIMESTAMPTZ, + mito_ref TEXT, + busco_dataset_name TEXT, + busco_odb10_dataset_name TEXT, + busco_odb12_dataset_name TEXT, + find_plastid BOOLEAN, + hic_motif TEXT, + mitochondrial_genetic_code_id INTEGER, + mitohifi_reference_species TEXT, + oatk_hmm_name TEXT, + defined_class TEXT, + augustus_dataset_name TEXT, + genetic_code_id INTEGER +); + +-- ========================================== +-- Accession registry table +-- ========================================== + +CREATE TABLE accession_registry ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + authority authority_type NOT NULL, + accession TEXT NOT NULL UNIQUE, + secondary_accession TEXT, + entity_type entity_type NOT NULL, + entity_id UUID NOT NULL, + accepted_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE (authority, entity_type, entity_id), + UNIQUE (authority, accession) +); + +CREATE UNIQUE INDEX uq_registry_full + ON accession_registry (accession, authority, entity_type, entity_id); + +-- ========================================== +-- project tables +-- ========================================== + +-- Main project table +CREATE TABLE project ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + taxon_id INT NOT NULL REFERENCES organism(taxon_id) ON DELETE CASCADE, + project_type project_type NOT NULL, + project_accession TEXT UNIQUE, + study_type TEXT NOT NULL, + alias TEXT NOT NULL, + title TEXT NOT NULL, + description TEXT NOT NULL, + centre_name TEXT, + study_attributes JSONB, + -- TODO remove submitted_at, should only be for submission table + submitted_at TIMESTAMPTZ, + status submission_status NOT NULL DEFAULT 'draft', + authority authority_type NOT NULL DEFAULT 'ENA', + -- TODO confirm if we want study attributes, and enforece schema for json (or include as seperate table) + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE UNIQUE INDEX uq_one_project_type_per_organism + ON project (taxon_id, project_type); + +-- Project submission table +CREATE TABLE IF NOT EXISTS project_submission ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + project_id UUID NOT NULL REFERENCES project(id) ON DELETE CASCADE, + authority authority_type NOT NULL DEFAULT 'ENA', + status submission_status NOT NULL DEFAULT 'draft', + + prepared_payload JSONB, + response_payload JSONB, + + accession TEXT, + + -- constant to help the composite FK + entity_type_const entity_type NOT NULL DEFAULT 'project' CHECK (entity_type_const = 'project'), + + submitted_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + -- attempt linkage + attempt_id UUID, + finalised_attempt_id UUID, + + -- broker lease/claim fields (attempt-scoped) + lock_acquired_at TIMESTAMPTZ, + lock_expires_at TIMESTAMPTZ, + + CONSTRAINT fk_self_project_accession + FOREIGN KEY (accession, authority, entity_type_const, project_id) + REFERENCES accession_registry (accession, authority, entity_type, entity_id) + DEFERRABLE INITIALLY DEFERRED +); + +-- Only one accepted submission per project+authority with accession +CREATE UNIQUE INDEX IF NOT EXISTS uq_project_one_accepted + ON project_submission (project_id, authority) + WHERE status = 'accepted' AND accession IS NOT NULL; + +-- Broker claim indexes +CREATE INDEX IF NOT EXISTS idx_project_submission_attempt ON project_submission (attempt_id); +CREATE INDEX IF NOT EXISTS idx_project_submission_finalised_attempt ON project_submission (finalised_attempt_id); +CREATE INDEX IF NOT EXISTS idx_project_submission_status ON project_submission (status); +CREATE INDEX IF NOT EXISTS idx_project_submission_lock_expires_at ON project_submission (lock_expires_at); + +-- ========================================== +-- Sample tables +-- ========================================== + +-- Main sample table +CREATE TABLE sample ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + taxon_id INT NOT NULL REFERENCES organism(taxon_id) ON DELETE CASCADE, + bpa_sample_id TEXT, + specimen_id TEXT, + specimen_id_description TEXT, + identified_by TEXT, + specimen_custodian TEXT, + sample_custodian TEXT, + lifestage TEXT NOT NULL, + sex TEXT NOT NULL, + organism_part TEXT NOT NULL, + region_and_locality TEXT NOT NULL, + state_or_region TEXT, + country_or_sea TEXT NOT NULL, + indigenous_location TEXT, + latitude FLOAT, + longitude FLOAT, + elevation FLOAT, + depth FLOAT, + habitat TEXT NOT NULL, + collection_method TEXT, + collection_date TEXT, + collected_by TEXT NOT NULL, + collecting_institution TEXT NOT NULL, + collection_permit TEXT, + data_context TEXT, + bioplatforms_project_id TEXT, + title TEXT, + sample_same_as TEXT, + sample_derived_from TEXT, + specimen_voucher TEXT, + tolid TEXT, + preservation_method TEXT, + preservation_temperature TEXT, + project_name TEXT, + biosample_accession TEXT, + + derived_from_sample_id UUID REFERENCES sample(id) ON DELETE CASCADE, + kind sample_kind NOT NULL, + + CONSTRAINT derived_from_sample_id_matches_kind CHECK ( + (kind = 'specimen' AND derived_from_sample_id IS NULL) + OR + (kind = 'derived' AND derived_from_sample_id IS NOT NULL) + ), + CONSTRAINT chk_sample_latitude CHECK (latitude IS NULL OR latitude BETWEEN -90 AND 90), + CONSTRAINT chk_sample_longitude CHECK (longitude IS NULL OR longitude BETWEEN -180 AND 180), + extensions JSONB, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +-- Sample submission table +CREATE TABLE sample_submission ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + sample_id UUID NOT NULL REFERENCES sample(id) ON DELETE CASCADE, + authority authority_type NOT NULL DEFAULT 'ENA', + prepared_payload JSONB NOT NULL, + response_payload JSONB, + accession TEXT, + biosample_accession TEXT, + -- TODO undecided whether to keep biosample_accession here or rely on the accession_registry table + status submission_status NOT NULL DEFAULT 'draft', + submitted_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + -- constant to help the composite FK + entity_type_const entity_type NOT NULL DEFAULT 'sample' CHECK (entity_type_const = 'sample'), + + -- prerequisite FK (for normalized accession lookups) + -- TODO remove: samples are not project-scoped; project context (if any) is derived via experiments + project_id UUID NOT NULL REFERENCES project(id), + + -- attempt linkage + attempt_id UUID, + finalised_attempt_id UUID, + + -- broker lease/claim fields (attempt-scoped) + lock_acquired_at TIMESTAMPTZ, + lock_expires_at TIMESTAMPTZ, + + CONSTRAINT fk_self_accession + FOREIGN KEY (accession, authority, entity_type_const, sample_id) + REFERENCES accession_registry (accession, authority, entity_type, entity_id) + DEFERRABLE INITIALLY DEFERRED +); + +CREATE UNIQUE INDEX uq_sample_one_accepted + ON sample_submission (sample_id, authority) + WHERE status = 'accepted' AND accession IS NOT NULL; + -- TODO uniqueness constraint above? + -- TODO consider if we want to keep track of former submissions that have been replaced/modified + +-- Broker claim indexes +CREATE INDEX IF NOT EXISTS idx_sample_submission_attempt ON sample_submission (attempt_id); +CREATE INDEX IF NOT EXISTS idx_sample_submission_finalised_attempt ON sample_submission (finalised_attempt_id); +CREATE INDEX IF NOT EXISTS idx_sample_submission_status ON sample_submission (status); +CREATE INDEX IF NOT EXISTS idx_sample_submission_lock_expires_at ON sample_submission (lock_expires_at); +CREATE INDEX IF NOT EXISTS idx_sample_submission_project_id ON sample_submission (project_id); + +-- Support parent/child lookups for derived samples +CREATE INDEX IF NOT EXISTS idx_sample_derived_from_sample_id ON sample(derived_from_sample_id); + +-- Enforce uniqueness: one specimen sample per (taxon_id, specimen_id) +CREATE UNIQUE INDEX IF NOT EXISTS uq_specimen_per_organism_specimen_id + ON sample (taxon_id, specimen_id) + WHERE kind = 'specimen' AND specimen_id IS NOT NULL; + +-- Enforce uniqueness: bpa_sample_id must be unique for derived samples +CREATE UNIQUE INDEX IF NOT EXISTS uq_derived_bpa_sample_id + ON sample (bpa_sample_id) + WHERE kind = 'derived' AND bpa_sample_id IS NOT NULL; + +-- Index for efficient lookup by taxon_id + specimen_id +CREATE INDEX IF NOT EXISTS idx_sample_organism_specimen_lookup + ON sample (taxon_id, specimen_id) + WHERE specimen_id IS NOT NULL; + +-- UNIQUE (sample_id, authority) WHERE status = 'accepted' AND accession IS NOT NULL + +-- ========================================== +-- Experiment tables +-- ========================================== + +-- Main experiment table +CREATE TABLE experiment ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + sample_id UUID NOT NULL REFERENCES sample(id) ON DELETE CASCADE, + project_id UUID NOT NULL REFERENCES project(id) ON DELETE CASCADE, + bpa_package_id TEXT UNIQUE NOT NULL, + bioplatforms_base_url TEXT, + design_description TEXT, + bpa_library_id TEXT, + library_strategy TEXT, + library_source TEXT, + insert_size TEXT, + library_construction_protocol TEXT, + library_selection TEXT, + library_layout TEXT, + instrument_model TEXT, + platform TEXT, + material_extracted_by TEXT, + library_prepared_by TEXT, + sequencing_kit TEXT, + flowcell_type TEXT, + base_caller_model TEXT, + data_owner TEXT, + project_collaborators TEXT, + extraction_method TEXT, + nucleic_acid_treatment TEXT, + extraction_protocol_doi TEXT, + nucleic_acid_conc TEXT, + nucleic_acid_volume TEXT, + gal TEXT, + raw_data_release_date TEXT, + + -- bpa_dataset_id TEXT UNIQUE NOT NULL, + extensions JSONB, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +-- Experiment submission table +CREATE TABLE experiment_submission ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + experiment_id UUID NOT NULL REFERENCES experiment(id) ON DELETE CASCADE, + authority authority_type NOT NULL DEFAULT 'ENA', + status submission_status NOT NULL DEFAULT 'draft', + + project_accession TEXT, + sample_accession TEXT, + prepared_payload JSONB, + response_payload JSONB, + + accession TEXT, + + -- constant to help the composite FK + entity_type_const entity_type NOT NULL DEFAULT 'experiment' CHECK (entity_type_const = 'experiment'), + + submitted_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + attempt_id UUID, + finalised_attempt_id UUID, + + -- broker lease/claim fields + lock_acquired_at TIMESTAMPTZ, + lock_expires_at TIMESTAMPTZ, + + -- When accession is present, it must exist in the registry AND map to this same experiment: + CONSTRAINT fk_self_accession + FOREIGN KEY (accession, authority, entity_type_const, experiment_id) + REFERENCES accession_registry (accession, authority, entity_type, entity_id) + DEFERRABLE INITIALLY DEFERRED +); + +-- Broker lease/claim index +CREATE INDEX IF NOT EXISTS idx_experiment_submission_attempt ON experiment_submission (attempt_id); +CREATE INDEX IF NOT EXISTS idx_experiment_submission_finalised_attempt ON experiment_submission (finalised_attempt_id); +CREATE INDEX IF NOT EXISTS idx_experiment_submission_status ON experiment_submission (status); +CREATE INDEX IF NOT EXISTS idx_experiment_submission_lock_expires_at ON experiment_submission (lock_expires_at); + +-- TODO consider if we want to keep track of former submissions that have been replaced/modified +CREATE UNIQUE INDEX uq_exp_one_accepted + ON experiment_submission (experiment_id, authority) + WHERE status = 'accepted' AND accession IS NOT NULL; + +/* + -- When status = accepted, both upstream accessions must be present + CONSTRAINT chk_ready_to_send + CHECK ( + status IN ('draft','ready','submitted','rejected') + OR (project_accession IS NOT NULL AND sample_accession IS NOT NULL AND accession IS NOT NULL) + ) +*/ + +-- ========================================== +-- Read tables +-- ========================================== + +CREATE TABLE read ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + experiment_id UUID NOT NULL REFERENCES experiment(id) ON DELETE CASCADE, + bpa_resource_id TEXT UNIQUE, + bpa_dataset_id TEXT, + file_name TEXT NOT NULL, + file_checksum TEXT, + file_format TEXT NOT NULL, + optional_file BOOLEAN NOT NULL DEFAULT TRUE, + bioplatforms_url TEXT, + read_number TEXT, + lane_number TEXT, + run_read_count TEXT, + run_base_count TEXT, + extensions JSONB, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +-- ========================================== +-- Broker Attempt table +-- ========================================== + +CREATE TABLE submission_attempt ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + taxon_id INT REFERENCES organism(taxon_id), + campaign_label TEXT, + -- TODO make ENUM + status TEXT NOT NULL DEFAULT 'processing', + lock_acquired_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + lock_expires_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_submission_attempt_status ON submission_attempt (status); +CREATE INDEX IF NOT EXISTS idx_submission_attempt_lock_expires_at ON submission_attempt (lock_expires_at); + +-- ========================================== +-- Submission events (append-only audit trail) +-- ========================================== +-- Note: Each submission creates multiple rows (claimed -> released/accepted/rejected) +-- This provides full audit history but means ~2x rows per submission. +-- TODO: If table size becomes an issue, consider switching to single-row-per-submission +-- with UPDATE instead of INSERT for state transitions. + +CREATE TABLE submission_event ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + attempt_id UUID NOT NULL REFERENCES submission_attempt(id) ON DELETE CASCADE, + entity_type entity_type NOT NULL, + submission_id UUID NOT NULL, + action TEXT NOT NULL CHECK (action IN ('claimed','accepted','rejected','released','expired','progress')), + accession TEXT, + details JSONB, + at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_submission_event_attempt ON submission_event (attempt_id); +CREATE INDEX IF NOT EXISTS idx_submission_event_entity ON submission_event (entity_type, submission_id); + +-- ========================================== +-- Assembly tables +-- ========================================== + +-- Main assembly table +CREATE TABLE assembly ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + taxon_id INT REFERENCES organism(taxon_id) NOT NULL, + sample_id UUID REFERENCES sample(id) NOT NULL, + project_id UUID REFERENCES project(id), + + -- Assembly metadata + assembly_name TEXT, + assembly_type TEXT NOT NULL DEFAULT 'clone or isolate', + tol_id TEXT, + data_types assembly_data_types NOT NULL, + coverage FLOAT, + program TEXT, + mingaplength FLOAT, + moleculetype molecule_type NOT NULL DEFAULT 'genomic DNA', + description TEXT, + + -- Intent-flow specimen samples + -- long_read_specimen_sample_id: the specimen sample supplying PacBio / ONT reads + -- hic_specimen_sample_id: the specimen sample supplying Hi-C reads (optional) + long_read_specimen_sample_id UUID REFERENCES sample(id), + hic_specimen_sample_id UUID REFERENCES sample(id), + + -- Persisted manifest generated at intent time + manifest_json JSONB, + + -- Version is scoped by (taxon_id, long_read_specimen_sample_id) for intent-flow assemblies + version INTEGER NOT NULL DEFAULT 1, + + -- Assembly lifecycle status + status TEXT NOT NULL DEFAULT 'requested', + + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + CONSTRAINT ck_assembly_status CHECK ( + status IN ('requested', 'running', 'curating', 'completed', 'failed', 'cancelled') + ) +); + +CREATE TABLE assembly_run ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + taxon_id INT REFERENCES organism(taxon_id) NOT NULL, + sample_id UUID REFERENCES sample(id) NOT NULL, + data_types assembly_data_types NOT NULL, + version INTEGER NOT NULL, + tol_id TEXT, + status TEXT NOT NULL DEFAULT 'reserved', + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE TABLE assembly_file ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + assembly_id UUID REFERENCES assembly(id) ON DELETE CASCADE NOT NULL, + file_type assembly_file_type NOT NULL, + file_name TEXT NOT NULL, + file_location TEXT NOT NULL, + file_size BIGINT, + file_checksum TEXT, + file_checksum_method TEXT DEFAULT 'MD5', + file_format TEXT, + description TEXT, + metadata JSONB, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX idx_assembly_file_assembly_id ON assembly_file(assembly_id); +CREATE INDEX idx_assembly_file_type ON assembly_file(assembly_id, file_type); + +-- Index for legacy version lookups (generic create flow) +CREATE INDEX idx_assembly_version_key ON assembly(data_types, taxon_id, sample_id, version); + +-- Index for intent-flow version lookups (scoped by taxon + long-read specimen sample) +CREATE INDEX idx_assembly_intent_version_key ON assembly(taxon_id, long_read_specimen_sample_id); + + +-- Assembly submission table (simplified - no broker integration) +CREATE TABLE assembly_submission ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + assembly_id UUID NOT NULL REFERENCES assembly(id) ON DELETE CASCADE, + authority authority_type NOT NULL DEFAULT 'ENA', + status submission_status NOT NULL DEFAULT 'draft', + + -- ENA accessions + accession TEXT, + sample_accession TEXT, + project_accession TEXT, + + -- Submission payloads + manifest_json JSONB, + response_payload JSONB, + + -- Metadata + submitted_at TIMESTAMPTZ, + submitted_by UUID REFERENCES users(id), + + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +-- Only one accepted submission per assembly+authority +CREATE UNIQUE INDEX uq_assembly_one_accepted + ON assembly_submission (assembly_id, authority) + WHERE status = 'accepted' AND accession IS NOT NULL; + +CREATE TABLE assembly_read ( + assembly_id UUID NOT NULL REFERENCES assembly(id), + read_id UUID NOT NULL REFERENCES read(id), + PRIMARY KEY (assembly_id, read_id) +); + +-- ========================================== +-- Assembly stage catalog and reporting tables +-- ========================================== + +CREATE TABLE assembly_stage ( + name TEXT PRIMARY KEY, + category TEXT NOT NULL, + is_active BOOLEAN NOT NULL DEFAULT TRUE, + CONSTRAINT ck_assembly_stage_category CHECK (category IN ('pipeline', 'manual')) +); + +-- Seed rows +INSERT INTO assembly_stage (name, category) VALUES + ('genomeassembly', 'pipeline'), + ('ascc', 'pipeline'), + ('treeval', 'pipeline'), + ('curation-pretext', 'pipeline'), + ('manual-curation', 'manual'); + +CREATE TABLE assembly_stage_run ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + assembly_id UUID NOT NULL REFERENCES assembly(id) ON DELETE CASCADE, + stage_name TEXT NOT NULL REFERENCES assembly_stage(name), + status TEXT NOT NULL, + external_run_id TEXT, + attempt INTEGER NOT NULL DEFAULT 1, + stats JSONB NOT NULL DEFAULT '{}', + started_at TIMESTAMPTZ, + completed_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT ck_assembly_stage_run_status CHECK ( + status IN ('running', 'succeeded', 'failed', 'cancelled') + ), + CONSTRAINT uq_stage_run_assembly_stage_attempt UNIQUE (assembly_id, stage_name, attempt) +); + +CREATE INDEX ix_assembly_stage_run_assembly_id ON assembly_stage_run(assembly_id); + +CREATE TABLE assembly_stage_run_file ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + assembly_stage_run_id UUID NOT NULL REFERENCES assembly_stage_run(id) ON DELETE CASCADE, + storage_type TEXT NOT NULL, + storage_uri TEXT NOT NULL, + storage_details JSONB NOT NULL DEFAULT '{}', + sha256sum TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX ix_assembly_stage_run_file_run_id ON assembly_stage_run_file(assembly_stage_run_id); + +-- ========================================== +-- Genome note tables +-- ========================================== + +-- Main genome_note table +-- Tracks versioned genome notes linked to assemblies +-- Each organism can have multiple draft versions but only one published version +CREATE TABLE genome_note ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + taxon_id INT NOT NULL REFERENCES organism(taxon_id) ON DELETE CASCADE, + assembly_id UUID NOT NULL REFERENCES assembly(id) ON DELETE CASCADE, + + -- Versioning: auto-increment per organism + version INTEGER NOT NULL, + + -- Content and metadata + title TEXT NOT NULL, + note_url TEXT NOT NULL, -- URL hosting the genome note + + -- Publication status + is_published BOOLEAN NOT NULL DEFAULT FALSE, + published_at TIMESTAMPTZ, + + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + -- Ensure version uniqueness per organism + UNIQUE (taxon_id, version) +); + +-- Ensure only one published note per organism +CREATE UNIQUE INDEX uq_genome_note_one_published_per_organism + ON genome_note (taxon_id) + WHERE is_published = TRUE; + +-- Indexes for efficient lookups +CREATE INDEX idx_genome_note_taxon_id ON genome_note(taxon_id); +CREATE INDEX idx_genome_note_assembly_id ON genome_note(assembly_id); +CREATE INDEX idx_genome_note_published ON genome_note(taxon_id, is_published) + WHERE is_published = TRUE; + +-- ========================================== +-- QC Read tables +-- ========================================== + +CREATE TABLE qc_read ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + experiment_id UUID NOT NULL REFERENCES experiment(id) ON DELETE CASCADE, + base_count BIGINT NOT NULL, + read_count BIGINT NOT NULL, + qc_bases_removed BIGINT NOT NULL, + qc_reads_removed BIGINT NOT NULL, + mean_gc_content FLOAT NOT NULL, + n50_length BIGINT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE TABLE qc_read_file ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + qc_read_id UUID NOT NULL REFERENCES qc_read(id) ON DELETE CASCADE, + file_type TEXT NOT NULL CHECK (file_type IN ('cram', 'fastq_r1', 'fastq_r2')), + storage_backend TEXT NOT NULL, + storage_profile TEXT NOT NULL, + bucket_name TEXT NOT NULL, + path_to_file TEXT NOT NULL, + md5_checksum TEXT NOT NULL CHECK (md5_checksum ~ '^[a-f0-9]{32}$'), + sha256_checksum TEXT NOT NULL CHECK (sha256_checksum ~ '^[a-f0-9]{64}$'), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE TABLE qc_read_submission ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + qc_read_id UUID NOT NULL REFERENCES qc_read(id) ON DELETE CASCADE, + experiment_id UUID REFERENCES experiment(id) ON DELETE CASCADE, + authority authority_type NOT NULL DEFAULT 'ENA', + status submission_status NOT NULL DEFAULT 'draft', + + prepared_payload JSONB NOT NULL, + response_payload JSONB, + + accession TEXT, + + -- constant to help the composite FK + entity_type_const entity_type NOT NULL DEFAULT 'qc_read' CHECK (entity_type_const = 'qc_read'), + + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + -- attempt linkage + attempt_id UUID, + finalised_attempt_id UUID, + + -- broker lease/claim fields + lock_acquired_at TIMESTAMPTZ, + lock_expires_at TIMESTAMPTZ, + + CONSTRAINT fk_qc_read_submission_accession + FOREIGN KEY (accession, authority, entity_type_const, qc_read_id) + REFERENCES accession_registry (accession, authority, entity_type, entity_id) + DEFERRABLE INITIALLY DEFERRED +); + +-- QC read indexes +CREATE INDEX idx_qc_read_experiment_id ON qc_read(experiment_id); +CREATE INDEX idx_qc_read_file_qc_read_id ON qc_read_file(qc_read_id); +CREATE INDEX idx_qc_read_submission_attempt ON qc_read_submission (attempt_id); +CREATE INDEX idx_qc_read_submission_experiment_id ON qc_read_submission (experiment_id); +CREATE INDEX idx_qc_read_submission_finalised_attempt ON qc_read_submission (finalised_attempt_id); +CREATE INDEX idx_qc_read_submission_lock_expires_at ON qc_read_submission (lock_expires_at); +CREATE INDEX idx_qc_read_submission_status ON qc_read_submission (status); + +CREATE UNIQUE INDEX uq_qc_read_one_accepted + ON qc_read_submission (qc_read_id, authority) + WHERE status = 'accepted' AND accession IS NOT NULL; + +-- ========================================== +-- BPA initiative table +-- ========================================== + +CREATE TABLE bpa_initiative ( + project_code TEXT PRIMARY KEY, + title TEXT NOT NULL, + url TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +-- Create indexes for common query patterns +CREATE INDEX idx_sample_taxon_id ON sample(taxon_id); +CREATE INDEX idx_experiment_sample_id ON experiment(sample_id); +CREATE INDEX idx_experiment_project_id ON experiment(project_id); +CREATE INDEX idx_assembly_sample_id ON assembly(sample_id); +CREATE INDEX idx_assembly_taxon_id ON assembly(taxon_id); diff --git a/alembic/versions/0001_initial_schema.py b/alembic/versions/0001_initial_schema.py index 44b70fb..5adfc30 100644 --- a/alembic/versions/0001_initial_schema.py +++ b/alembic/versions/0001_initial_schema.py @@ -1,4 +1,4 @@ -"""Initial schema load from schema.sql.""" +"""Initial schema load from frozen bootstrap SQL.""" from pathlib import Path @@ -12,7 +12,7 @@ def upgrade() -> None: - schema_path = Path(__file__).resolve().parents[2] / "schema.sql" + schema_path = Path(__file__).resolve().parents[1] / "base_schema" / "0001_initial_schema.sql" op.execute(schema_path.read_text()) diff --git a/docs/migration_workflow.md b/docs/migration_workflow.md index 1dabd09..465b950 100644 --- a/docs/migration_workflow.md +++ b/docs/migration_workflow.md @@ -1,4 +1,13 @@ +## Schema files + +- `alembic/base_schema/0001_initial_schema.sql` is the frozen bootstrap schema used only by `0001_initial_schema.py`. +- Do not edit that file after it has been established, unless you are intentionally rewriting migration history. +- `schema.sql` is the mutable current schema snapshot for the present database shape. +- New migrations should update the database schema and then `schema.sql` can be refreshed to match the current state. + +--- + ```bash # Check current migration version docker compose exec api alembic current From 9fb105d6ba39f451d17eeb1dc8274087ea7a8726 Mon Sep 17 00:00:00 2001 From: Emily Marshall Date: Mon, 18 May 2026 23:02:31 +1000 Subject: [PATCH 14/14] style: linting --- app/services/taxonomy_info_service.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/services/taxonomy_info_service.py b/app/services/taxonomy_info_service.py index a74cd34..f40acd9 100644 --- a/app/services/taxonomy_info_service.py +++ b/app/services/taxonomy_info_service.py @@ -177,7 +177,9 @@ def bulk_import( continue existing = db.query(TaxonomyInfo).filter(TaxonomyInfo.taxon_id == taxon_id).first() if existing: - errors.append(f"{taxon_id}: taxonomy_info for taxon_id {taxon_id} already exists") + errors.append( + f"{taxon_id}: taxonomy_info for taxon_id {taxon_id} already exists" + ) skipped_count += 1 continue