feat: add ncbi taxonomy information to database

alembic/versions/0002_timestamptz_broker_idx.py → alembic/archive/0002_timestamptz_broker_idx.py

@@ -9,6 +9,50 @@
 depends_on = None
 
 
+def _alter_column_to_timestamptz(table: str, col: str) -> None:
+    op.execute(
+        f"""
+        DO $$ BEGIN
+            IF EXISTS (
+                SELECT 1
+                FROM information_schema.columns
+                WHERE table_schema = 'public'
+                  AND table_name = '{table}'
+                  AND column_name = '{col}'
+                  AND data_type = 'timestamp without time zone'
+            ) THEN
+                ALTER TABLE {table}
+                ALTER COLUMN {col}
+                TYPE TIMESTAMPTZ
+                USING {col} AT TIME ZONE 'UTC';
+            END IF;
+        END $$;
+        """
+    )
+
+
+def _alter_column_to_timestamp(table: str, col: str) -> None:
+    op.execute(
+        f"""
+        DO $$ BEGIN
+            IF EXISTS (
+                SELECT 1
+                FROM information_schema.columns
+                WHERE table_schema = 'public'
+                  AND table_name = '{table}'
+                  AND column_name = '{col}'
+                  AND data_type = 'timestamp with time zone'
+            ) THEN
+                ALTER TABLE {table}
+                ALTER COLUMN {col}
+                TYPE TIMESTAMP
+                USING {col}::timestamp;
+            END IF;
+        END $$;
+        """
+    )
+
+
 def upgrade() -> None:
     # Timestamp columns -> TIMESTAMPTZ (treat existing values as UTC)
     tables = {
@@ -42,6 +86,7 @@ def upgrade() -> None:
         ],
         "read": ["created_at", "updated_at"],
         "read_submission": ["created_at", "updated_at", "lock_acquired_at", "lock_expires_at"],
+        "qc_read_submission": ["created_at", "updated_at", "lock_acquired_at", "lock_expires_at"],
         "submission_attempt": ["lock_acquired_at", "lock_expires_at", "created_at", "updated_at"],
         "submission_event": ["at"],
         "assembly": ["created_at", "updated_at"],
@@ -53,9 +98,7 @@ def upgrade() -> None:
 
     for table, columns in tables.items():
         for col in columns:
-            op.execute(
-                f"ALTER TABLE {table} ALTER COLUMN {col} TYPE TIMESTAMPTZ USING {col} AT TIME ZONE 'UTC'"
-            )
+            _alter_column_to_timestamptz(table, col)
 
     # Sample latitude/longitude checks (skip if already exists from schema.sql)
     op.execute(
@@ -102,9 +145,25 @@ def upgrade() -> None:
     op.execute(
         "CREATE INDEX IF NOT EXISTS idx_experiment_submission_lock_expires_at ON experiment_submission (lock_expires_at)"
     )
-    op.execute("CREATE INDEX IF NOT EXISTS idx_read_submission_status ON read_submission (status)")
     op.execute(
-        "CREATE INDEX IF NOT EXISTS idx_read_submission_lock_expires_at ON read_submission (lock_expires_at)"
+        """
+        DO $$ BEGIN
+            IF to_regclass('public.read_submission') IS NOT NULL THEN
+                CREATE INDEX IF NOT EXISTS idx_read_submission_status ON read_submission (status);
+                CREATE INDEX IF NOT EXISTS idx_read_submission_lock_expires_at ON read_submission (lock_expires_at);
+            END IF;
+        END $$;
+        """
+    )
+    op.execute(
+        """
+        DO $$ BEGIN
+            IF to_regclass('public.qc_read_submission') IS NOT NULL THEN
+                CREATE INDEX IF NOT EXISTS idx_qc_read_submission_status ON qc_read_submission (status);
+                CREATE INDEX IF NOT EXISTS idx_qc_read_submission_lock_expires_at ON qc_read_submission (lock_expires_at);
+            END IF;
+        END $$;
+        """
     )
     op.execute(
         "CREATE INDEX IF NOT EXISTS idx_submission_attempt_status ON submission_attempt (status)"
@@ -118,6 +177,8 @@ def downgrade() -> None:
     # Drop indexes
     op.execute("DROP INDEX IF EXISTS idx_submission_attempt_lock_expires_at")
     op.execute("DROP INDEX IF EXISTS idx_submission_attempt_status")
+    op.execute("DROP INDEX IF EXISTS idx_qc_read_submission_lock_expires_at")
+    op.execute("DROP INDEX IF EXISTS idx_qc_read_submission_status")
     op.execute("DROP INDEX IF EXISTS idx_read_submission_lock_expires_at")
     op.execute("DROP INDEX IF EXISTS idx_read_submission_status")
     op.execute("DROP INDEX IF EXISTS idx_experiment_submission_lock_expires_at")
@@ -163,6 +224,7 @@ def downgrade() -> None:
         ],
         "read": ["created_at", "updated_at"],
         "read_submission": ["created_at", "updated_at", "lock_acquired_at", "lock_expires_at"],
+        "qc_read_submission": ["created_at", "updated_at", "lock_acquired_at", "lock_expires_at"],
         "submission_attempt": ["lock_acquired_at", "lock_expires_at", "created_at", "updated_at"],
         "submission_event": ["at"],
         "assembly": ["created_at", "updated_at"],
@@ -174,6 +236,4 @@ def downgrade() -> None:
 
     for table, columns in tables.items():
         for col in columns:
-            op.execute(
-                f"ALTER TABLE {table} ALTER COLUMN {col} TYPE TIMESTAMP USING {col}::timestamp"
-            )
+            _alter_column_to_timestamp(table, col)

alembic/versions/0003_add_tol_id_to_assembly.py → alembic/archive/0003_add_tol_id_to_assembly.py

@@ -17,8 +17,8 @@
 
 
 def upgrade():
-    op.add_column("assembly", sa.Column("tol_id", sa.Text(), nullable=True))
+    op.add_column("assembly", sa.Column("tol_id", sa.Text(), nullable=True), if_not_exists=True)
 
 
 def downgrade():
-    op.drop_column("assembly", "tol_id")
+    op.drop_column("assembly", "tol_id", if_exists=True)

alembic/versions/0006_consolidated_submission_schema_refactor.py → alembic/archive/0006_consolidated_submission_schema_refactor.py

@@ -30,6 +30,7 @@ def upgrade():
     op.add_column(
         "sample_submission",
         sa.Column("project_id", postgresql.UUID(as_uuid=True), nullable=False),
+        if_not_exists=True,
     )
     op.create_foreign_key(
         "fk_sample_submission_project_id",
@@ -43,12 +44,14 @@ def upgrade():
         "sample_submission",
         ["project_id"],
         unique=False,
+        if_not_exists=True,
     )
 
     # 2) Add experiment.project_id (NOT NULL, FK to project with CASCADE)
     op.add_column(
         "experiment",
         sa.Column("project_id", postgresql.UUID(as_uuid=True), nullable=False),
+        if_not_exists=True,
     )
     op.create_foreign_key(
         "fk_experiment_project_id",
@@ -63,6 +66,7 @@ def upgrade():
         "experiment",
         ["project_id"],
         unique=False,
+        if_not_exists=True,
     )
 
     # 3) Drop read_submission.project_id (derivable via experiment)
@@ -126,11 +130,13 @@ def downgrade():
     )
 
     # 2) Drop experiment.project_id
-    op.drop_index("idx_experiment_project_id", table_name="experiment")
+    op.drop_index("idx_experiment_project_id", table_name="experiment", if_exists=True)
     op.drop_constraint("fk_experiment_project_id", "experiment", type_="foreignkey")
-    op.drop_column("experiment", "project_id")
+    op.drop_column("experiment", "project_id", if_exists=True)
 
     # 1) Drop sample_submission.project_id
-    op.drop_index("idx_sample_submission_project_id", table_name="sample_submission")
+    op.drop_index(
+        "idx_sample_submission_project_id", table_name="sample_submission", if_exists=True
+    )
     op.drop_constraint("fk_sample_submission_project_id", "sample_submission", type_="foreignkey")
-    op.drop_column("sample_submission", "project_id")
+    op.drop_column("sample_submission", "project_id", if_exists=True)

alembic/archive/0014_add_ncbi_taxonomy_details.py

@@ -0,0 +1,116 @@
+"""Remove submission_xml from assembly_submission table.
+
+Revision ID: 0014_add_ncbi_taxonomy_details
+Revises: 0013_remove_assembly_sub_col
+Create Date: 2026-05-13
+
+We are adding in NCBI taxonomy details for organisms, sourced through external API lookup to NCBI's Datasets API v2.
+"""
+
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+from alembic import op
+
+revision = "0014_add_ncbi_taxonomy_details"
+down_revision = "0013_remove_assembly_sub_col"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.drop_column("organism", "common_name_source")
+    op.alter_column("organism", "genus", new_column_name="bpa_genus")
+    op.alter_column("organism", "species", new_column_name="bpa_species")
+    op.alter_column("organism", "common_name", new_column_name="bpa_common_name")
+    op.alter_column(
+        "organism", "infraspecific_epithet", new_column_name="bpa_infraspecific_epithet"
+    )
+    op.alter_column("organism", "culture_or_strain_id", new_column_name="bpa_culture_or_strain_id")
+    op.alter_column("organism", "authority", new_column_name="bpa_authority")
+    op.alter_column("organism", "scientific_name", new_column_name="bpa_scientific_name")
+    op.add_column("organism", sa.Column("scientific_name", sa.Text(), nullable=True))
+
+    op.drop_column("organism", "tax_string")
+    op.drop_column("organism", "ncbi_order")
+    op.drop_column("organism", "ncbi_family")
+    op.drop_column("organism", "busco_dataset_name")
+    op.drop_column("organism", "taxonomy_lineage_json")
+
+    # Add new columns for NCBI taxonomy details
+    op.add_column("taxonomy_info", sa.Column("ncbi_taxon_id", sa.Integer(), nullable=True))
+    op.add_column("taxonomy_info", sa.Column("ncbi_rank", sa.String(), nullable=True))
+    op.add_column("taxonomy_info", sa.Column("ncbi_scientific_name", sa.String(), nullable=True))
+    op.add_column("taxonomy_info", sa.Column("ncbi_authority", sa.String(), nullable=True))
+    op.add_column("taxonomy_info", sa.Column("ncbi_common_name", sa.String(), nullable=True))
+    op.add_column("taxonomy_info", sa.Column("ncbi_class", sa.String(), nullable=True))
+    op.add_column("taxonomy_info", sa.Column("ncbi_order", sa.String(), nullable=True))
+    op.add_column("taxonomy_info", sa.Column("ncbi_family", sa.String(), nullable=True))
+    op.add_column("taxonomy_info", sa.Column("ncbi_lineage", sa.JSON(), nullable=True))
+    op.add_column("taxonomy_info", sa.Column("ncbi_tax_string", sa.String(), nullable=True))
+    op.add_column("taxonomy_info", sa.Column("ncbi_full_lineage", sa.String(), nullable=True))
+    op.add_column(
+        "taxonomy_info",
+        sa.Column("ncbi_last_synced_at", sa.DateTime(timezone=True), nullable=True),
+    )
+    op.add_column("taxonomy_info", sa.Column("mito_ref", sa.String(), nullable=True))
+    op.add_column("taxonomy_info", sa.Column("busco_dataset_name", sa.String(), nullable=True))
+    op.execute("UPDATE organism SET scientific_name = bpa_scientific_name")
+    op.execute(
+        """
+        UPDATE organism
+        SET scientific_name = taxonomy_info.ncbi_scientific_name
+        FROM taxonomy_info
+        WHERE taxonomy_info.taxon_id = organism.taxon_id
+          AND taxonomy_info.ncbi_scientific_name IS NOT NULL
+        """
+    )
+
+
+def downgrade() -> None:
+    # Drop columns added by the upgrade, in reverse order
+    op.drop_column("taxonomy_info", "busco_dataset_name")
+    op.drop_column("taxonomy_info", "mito_ref")
+    op.drop_column("taxonomy_info", "ncbi_full_lineage")
+    op.drop_column("taxonomy_info", "ncbi_tax_string")
+    op.drop_column("taxonomy_info", "ncbi_lineage")
+    op.drop_column("taxonomy_info", "ncbi_last_synced_at")
+    op.drop_column("taxonomy_info", "ncbi_family")
+    op.drop_column("taxonomy_info", "ncbi_order")
+    op.drop_column("taxonomy_info", "ncbi_class")
+    op.drop_column("taxonomy_info", "ncbi_common_name")
+    op.drop_column("taxonomy_info", "ncbi_authority")
+    op.drop_column("taxonomy_info", "ncbi_scientific_name")
+    op.drop_column("taxonomy_info", "ncbi_rank")
+    op.drop_column("taxonomy_info", "ncbi_taxon_id")
+
+    # Restore columns that were dropped by the upgrade
+    op.add_column(
+        "organism",
+        sa.Column("taxonomy_lineage_json", postgresql.JSONB(astext_type=sa.Text()), nullable=True),
+    )
+    op.add_column("organism", sa.Column("busco_dataset_name", sa.Text(), nullable=True))
+    op.add_column("organism", sa.Column("ncbi_family", sa.Text(), nullable=True))
+    op.add_column("organism", sa.Column("ncbi_order", sa.Text(), nullable=True))
+    op.add_column("organism", sa.Column("tax_string", sa.Text(), nullable=True))
+    op.drop_column("organism", "scientific_name")
+
+    # Rename BPA columns back to their original names
+    op.alter_column("organism", "bpa_scientific_name", new_column_name="scientific_name")
+    op.alter_column("organism", "bpa_authority", new_column_name="authority")
+    op.alter_column(
+        "organism",
+        "bpa_culture_or_strain_id",
+        new_column_name="culture_or_strain_id",
+    )
+    op.alter_column(
+        "organism",
+        "bpa_infraspecific_epithet",
+        new_column_name="infraspecific_epithet",
+    )
+    op.alter_column("organism", "bpa_common_name", new_column_name="common_name")
+    op.alter_column("organism", "bpa_species", new_column_name="species")
+    op.alter_column("organism", "bpa_genus", new_column_name="genus")
+
+    # Restore dropped BPA source column
+    op.add_column("organism", sa.Column("common_name_source", sa.Text(), nullable=True))