diff --git a/alembic/versions/0002_timestamptz_and_broker_indexes.py b/alembic/versions/0002_timestamptz_broker_idx.py similarity index 100% rename from alembic/versions/0002_timestamptz_and_broker_indexes.py rename to alembic/versions/0002_timestamptz_broker_idx.py diff --git a/alembic/versions/0003_add_tol_id_to_assembly.py b/alembic/versions/0003_add_tol_id_to_assembly.py new file mode 100644 index 0000000..0054b94 --- /dev/null +++ b/alembic/versions/0003_add_tol_id_to_assembly.py @@ -0,0 +1,24 @@ +"""Add tol_id to assembly. + +Revision ID: 0003_add_tol_id_to_assembly +Revises: 0002_timestamptz_broker_idx +Create Date: 2026-02-24 00:00:00.000000 +""" + +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "0003_add_tol_id_to_assembly" +down_revision = "0002_timestamptz_broker_idx" +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column("assembly", sa.Column("tol_id", sa.Text(), nullable=True)) + + +def downgrade(): + op.drop_column("assembly", "tol_id") diff --git a/alembic/versions/0004_add_assembly_run.py b/alembic/versions/0004_add_assembly_run.py new file mode 100644 index 0000000..82b5998 --- /dev/null +++ b/alembic/versions/0004_add_assembly_run.py @@ -0,0 +1,36 @@ +"""Add assembly_run table. + +Revision ID: 0004_add_assembly_run +Revises: 0003_add_tol_id_to_assembly +Create Date: 2026-02-24 00:00:00.000000 +""" + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "0004_add_assembly_run" +down_revision = "0003_add_tol_id_to_assembly" +branch_labels = None +depends_on = None + + +def upgrade(): + op.execute( + """ + CREATE TABLE IF NOT EXISTS assembly_run ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + organism_key TEXT REFERENCES organism(grouping_key) NOT NULL, + sample_id UUID REFERENCES sample(id) NOT NULL, + data_types assembly_data_types NOT NULL, + version INTEGER NOT NULL, + tol_id TEXT, + status TEXT NOT NULL DEFAULT 'reserved', + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() + ); + """ + ) + + +def downgrade(): + op.execute("DROP TABLE IF EXISTS assembly_run;") diff --git a/app/api/v1/endpoints/assemblies.py b/app/api/v1/endpoints/assemblies.py index a2ba3e7..d43d31f 100644 --- a/app/api/v1/endpoints/assemblies.py +++ b/app/api/v1/endpoints/assemblies.py @@ -5,9 +5,10 @@ from sqlalchemy.orm import Session from app.core.dependencies import get_current_active_user, get_db +from app.core.errors import AppError from app.core.pagination import Pagination, apply_pagination, pagination_params from app.core.policy import policy -from app.models.assembly import Assembly, AssemblyFile, AssemblySubmission +from app.models.assembly import Assembly, AssemblyFile, AssemblyRun, AssemblySubmission from app.models.experiment import Experiment from app.models.organism import Organism from app.models.read import Read @@ -21,6 +22,7 @@ AssemblyCreateFromExperiments, AssemblyFileCreate, AssemblyFileUpdate, + AssemblyIntent, AssemblySubmissionCreate, AssemblySubmissionUpdate, AssemblyUpdate, @@ -32,7 +34,7 @@ AssemblySubmission as AssemblySubmissionSchema, ) from app.schemas.common import SubmissionStatus -from app.services.assembly_helper import generate_assembly_manifest +from app.services.assembly_helper import determine_assembly_data_types, generate_assembly_manifest from app.services.assembly_service import ( assembly_file_service, assembly_service, @@ -205,15 +207,52 @@ def get_pipeline_inputs_by_tax_id( return result +def _get_manifest_inputs_by_tax_id(db: Session, tax_id: int, sample_id: UUID): + organism = db.query(Organism).filter(Organism.tax_id == tax_id).first() + if not organism: + raise HTTPException(status_code=404, detail=f"Organism with tax_id {tax_id} not found") + + sample = db.query(Sample).filter(Sample.id == sample_id).first() + # if not sample or sample.organism_key != organism.grouping_key: + if not sample: + raise HTTPException(status_code=404, detail="Sample not found for this organism") + + experiments = db.query(Experiment).filter(Experiment.sample_id == sample_id).all() + if not experiments: + raise HTTPException( + status_code=404, + detail=f"No experiments found for organism {organism.grouping_key} and sample {sample_id} (tax_id: {tax_id})", + ) + + experiment_ids = [exp.id for exp in experiments] + reads = db.query(Read).filter(Read.experiment_id.in_(experiment_ids)).all() + if not reads: + raise HTTPException( + status_code=404, + detail=f"No reads found for organism {organism.grouping_key} (tax_id: {tax_id})", + ) + + return organism, reads, experiments + + +def _get_optimal_sample_id_for_tax_id(db: Session, tax_id: int) -> UUID | None: + # TODO: Implement specimen/long-read selection logic. + # Placeholder for now; return None to indicate no automatic selection. + _ = db, tax_id + return None + + @router.get("/manifest/{tax_id}") def get_assembly_manifest( *, db: Session = Depends(get_db), tax_id: int, + sample_id: UUID = Query(..., description="Sample ID for the manifest"), + version: Optional[int] = Query(None, description="Reserved manifest version to retrieve"), current_user: User = Depends(get_current_active_user), ) -> Any: """ - Generate assembly manifest YAML for an organism by tax_id. + Retrieve the latest reserved assembly manifest YAML for an organism by tax_id. Returns YAML manifest with: - scientific_name and taxon_id from organism @@ -226,46 +265,92 @@ def get_assembly_manifest( """ from fastapi.responses import Response - # Get organism by tax_id - organism = db.query(Organism).filter(Organism.tax_id == tax_id).first() - if not organism: - raise HTTPException(status_code=404, detail=f"Organism with tax_id {tax_id} not found") + organism, reads, experiments = _get_manifest_inputs_by_tax_id(db, tax_id, sample_id) - # Get all samples for this organism - samples = db.query(Sample).filter(Sample.organism_key == organism.grouping_key).all() - if not samples: - raise HTTPException( - status_code=404, - detail=f"No samples found for organism {organism.grouping_key} (tax_id: {tax_id})", + run_query = ( + db.query(AssemblyRun) + .filter( + AssemblyRun.organism_key == organism.grouping_key, + AssemblyRun.sample_id == sample_id, ) + .order_by(AssemblyRun.created_at.desc()) + ) + if version is not None: + run_query = run_query.filter(AssemblyRun.version == version) + assembly_run = run_query.first() + if not assembly_run: + raise HTTPException(status_code=404, detail="No reserved assembly manifest found") + + yaml_content = generate_assembly_manifest( + organism, reads, experiments, assembly_run.tol_id, assembly_run.version + ) - # Get all experiments for these samples - sample_ids = [sample.id for sample in samples] - experiments = db.query(Experiment).filter(Experiment.sample_id.in_(sample_ids)).all() + # Return as YAML response + return Response(content=yaml_content, media_type="application/x-yaml") - if not experiments: - raise HTTPException( - status_code=404, - detail=f"No experiments found for organism {organism.grouping_key} (tax_id: {tax_id})", - ) - # Get all reads for these experiments - experiment_ids = [exp.id for exp in experiments] - reads = db.query(Read).filter(Read.experiment_id.in_(experiment_ids)).all() +@router.post("/intent/{tax_id}") +@policy("assemblies:write") +def create_assembly_intent( + *, + db: Session = Depends(get_db), + tax_id: int, + intent_in: AssemblyIntent, + current_user: User = Depends(get_current_active_user), +) -> Any: + """ + Reserve the next assembly version and return a manifest. + """ + from fastapi.responses import Response - if not reads: - raise HTTPException( - status_code=404, - detail=f"No reads found for organism {organism.grouping_key} (tax_id: {tax_id})", - ) + organism, reads, experiments = _get_manifest_inputs_by_tax_id(db, tax_id, intent_in.sample_id) + try: + data_types = determine_assembly_data_types(experiments) + except ValueError as exc: + raise AppError( + status_code=400, + code="assembly_intent_invalid_data_types", + message=str(exc), + details={ + "tax_id": tax_id, + "sample_id": str(intent_in.sample_id), + }, + ) from exc + + next_version = assembly_service.get_next_version( + db, + organism_key=organism.grouping_key, + sample_id=intent_in.sample_id, + data_types=data_types, + ) - # Generate YAML manifest - yaml_content = generate_assembly_manifest(organism, reads, experiments) + run = AssemblyRun( + organism_key=organism.grouping_key, + sample_id=intent_in.sample_id, + data_types=data_types, + version=next_version, + tol_id=intent_in.tol_id, + status="reserved", + ) + db.add(run) + db.commit() + db.refresh(run) - # Return as YAML response + yaml_content = generate_assembly_manifest(organism, reads, experiments, run.tol_id, run.version) return Response(content=yaml_content, media_type="application/x-yaml") +@router.get("/optimal-sample/{tax_id}") +def get_optimal_sample_id( + *, + db: Session = Depends(get_db), + tax_id: int, + current_user: User = Depends(get_current_active_user), +) -> Any: + sample_id = _get_optimal_sample_id_for_tax_id(db, tax_id) + return {"sample_id": str(sample_id) if sample_id else None} + + @router.post("/from-experiments/{tax_id}", response_model=AssemblySchema) @policy("assemblies:write") def create_assembly_from_experiments( diff --git a/app/models/assembly.py b/app/models/assembly.py index c1b8a3f..397150a 100644 --- a/app/models/assembly.py +++ b/app/models/assembly.py @@ -35,6 +35,7 @@ class Assembly(Base): # Assembly metadata fields assembly_name = Column(Text, nullable=False) assembly_type = Column(Text, nullable=False, default="clone or isolate") + tol_id = Column(Text, nullable=True) data_types = Column( SQLAlchemyEnum( "PACBIO_SMRT", @@ -72,6 +73,46 @@ class Assembly(Base): project = relationship("Project", backref="assemblies") +class AssemblyRun(Base): + """ + AssemblyRun model for reserving versions and tracking assembly intents. + + This model corresponds to the 'assembly_run' table in the database. + """ + + __tablename__ = "assembly_run" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organism_key = Column("organism_key", ForeignKey("organism.grouping_key"), nullable=False) + sample_id = Column(UUID(as_uuid=True), ForeignKey("sample.id"), nullable=False) + data_types = Column( + SQLAlchemyEnum( + "PACBIO_SMRT", + "PACBIO_SMRT_HIC", + "OXFORD_NANOPORE", + "OXFORD_NANOPORE_HIC", + "PACBIO_SMRT_OXFORD_NANOPORE", + "PACBIO_SMRT_OXFORD_NANOPORE_HIC", + name="assembly_data_types", + ), + nullable=False, + ) + version = Column(Integer, nullable=False) + tol_id = Column(Text, nullable=True) + status = Column(Text, nullable=False, default="reserved") + + created_at = Column(DateTime(timezone=True), nullable=False, server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), + nullable=False, + server_default=func.now(), + onupdate=func.now(), + ) + + organism = relationship("Organism", backref="assembly_runs") + sample = relationship("Sample", backref="assembly_runs") + + class AssemblySubmission(Base): """ AssemblySubmission model for storing assembly submission data to ENA. diff --git a/app/schemas/assembly.py b/app/schemas/assembly.py index 13f4f92..fcd2908 100644 --- a/app/schemas/assembly.py +++ b/app/schemas/assembly.py @@ -37,6 +37,7 @@ class AssemblyBase(BaseModel): project_id: Optional[UUID] = None assembly_name: str assembly_type: str = "clone or isolate" + tol_id: str data_types: AssemblyDataTypes coverage: float program: str @@ -61,6 +62,7 @@ class AssemblyCreateFromExperiments(BaseModel): project_id: Optional[UUID] = None assembly_name: str assembly_type: str = "clone or isolate" + tol_id: str data_types: Optional[AssemblyDataTypes] = None # Auto-detected, can be overridden coverage: float program: str @@ -69,6 +71,13 @@ class AssemblyCreateFromExperiments(BaseModel): description: Optional[str] = None +class AssemblyIntent(BaseModel): + """Schema for reserving an assembly version and generating a manifest.""" + + sample_id: UUID + tol_id: Optional[str] = None + + # Schema for updating an existing assembly class AssemblyUpdate(BaseModel): """Schema for updating an existing assembly.""" @@ -78,6 +87,7 @@ class AssemblyUpdate(BaseModel): project_id: Optional[UUID] = None assembly_name: Optional[str] = None assembly_type: Optional[str] = None + tol_id: Optional[str] = None coverage: Optional[float] = None program: Optional[str] = None mingaplength: Optional[float] = None diff --git a/app/services/assembly_helper.py b/app/services/assembly_helper.py index 1b312ea..82f6649 100644 --- a/app/services/assembly_helper.py +++ b/app/services/assembly_helper.py @@ -39,11 +39,11 @@ def determine_assembly_data_types(experiments: List[Experiment]) -> AssemblyData library_strategy = exp.library_strategy.upper() if exp.library_strategy else "" # Check for PacBio - if platform == "PACBIO_SMRT": + if platform == "PACBIO_SMRT" and library_strategy in ("WGS", "WGA"): has_pacbio = True # Check for Oxford Nanopore - if platform == "OXFORD_NANOPORE": + if platform == "OXFORD_NANOPORE" and library_strategy in ("WGS", "WGA"): has_nanopore = True # Check for Hi-C (Illumina + Hi-C or WGS library strategy) @@ -96,7 +96,11 @@ def get_detected_platforms(experiments: List[Experiment]) -> dict: def generate_assembly_manifest( - organism: Organism, reads: List[Read], experiments: List[Experiment] + organism: Organism, + reads: List[Read], + experiments: List[Experiment], + tol_id: str | None, + version: int, ) -> str: """Generate assembly manifest YAML from organism and reads data. @@ -110,6 +114,8 @@ def generate_assembly_manifest( organism: Organism object reads: List of Read objects experiments: List of Experiment objects (to determine platform) + tol_id: ToL ID for the assembly (optional) + version: Assembly version number Returns: YAML string formatted as assembly manifest @@ -185,6 +191,8 @@ def generate_assembly_manifest( manifest = { "scientific_name": organism.scientific_name, "taxon_id": organism.tax_id, + "tolid": tol_id, + "version": version, "reads": {}, } diff --git a/app/services/assembly_service.py b/app/services/assembly_service.py index 2b3d3a2..8a8e772 100644 --- a/app/services/assembly_service.py +++ b/app/services/assembly_service.py @@ -4,7 +4,13 @@ from sqlalchemy import func from sqlalchemy.orm import Session -from app.models.assembly import Assembly, AssemblyFile, AssemblyRead, AssemblySubmission +from app.models.assembly import ( + Assembly, + AssemblyFile, + AssemblyRead, + AssemblyRun, + AssemblySubmission, +) from app.models.experiment import Experiment from app.models.organism import Organism from app.models.sample import Sample @@ -145,6 +151,34 @@ def create_from_experiments( return assembly, platform_info + def get_next_version( + self, + db: Session, + *, + organism_key: str, + sample_id: UUID, + data_types: str, + ) -> int: + max_assembly = ( + db.query(func.max(Assembly.version)) + .filter( + Assembly.data_types == data_types, + Assembly.organism_key == organism_key, + Assembly.sample_id == sample_id, + ) + .scalar() + ) + max_run = ( + db.query(func.max(AssemblyRun.version)) + .filter( + AssemblyRun.data_types == data_types, + AssemblyRun.organism_key == organism_key, + AssemblyRun.sample_id == sample_id, + ) + .scalar() + ) + return max(max_assembly or 0, max_run or 0) + 1 + class AssemblySubmissionService( BaseService[AssemblySubmission, AssemblySubmissionCreate, AssemblySubmissionUpdate] diff --git a/docs/migration_workflow.md b/docs/migration_workflow.md new file mode 100644 index 0000000..1dabd09 --- /dev/null +++ b/docs/migration_workflow.md @@ -0,0 +1,453 @@ + +```bash +# Check current migration version +docker compose exec api alembic current + +# Rollback one migration +docker compose exec api alembic downgrade -1 + +# Rollback to specific version +docker compose exec api alembic downgrade + +# Restore from backup (last resort) +docker compose exec db psql -U postgres -d atol_db < backup.sql +``` + +--- + +## Traditional Server Deployment (Non-Docker Production) + +For production environments using VMs, bare metal servers, or Kubernetes (without Docker Compose). + +### Setup Requirements + +1. **Python environment** + ```bash + # Install Python 3.12+ and create virtual environment + python3 -m venv venv + source venv/bin/activate + pip install -r requirements.txt # or use uv + ``` + +2. **Environment variables** + ```bash + # Set database connection in .env or environment + export DATABASE_URI="postgresql://user:password@db-host:5432/atol_db" + ``` + +3. **Alembic configuration** + - Ensure `alembic.ini` points to correct database + - Or use environment variable: `ALEMBIC_CONFIG` if needed + +### Pre-deployment Checklist + +Same as Docker-based deployment, plus: + +1. **Verify database connectivity** + ```bash + # Test connection from application server + psql -h db-host -U postgres -d atol_db -c "SELECT version();" + ``` + +2. **Check migration files are deployed** + ```bash + ls -la alembic/versions/ + # Ensure new migration files are present + ``` + +3. **Verify Alembic can connect** + ```bash + alembic current + # Should show current migration version + ``` + +### Deployment Steps + +#### Option 1: Maintenance Window + +1. **Backup database** + ```bash + # From database server or application server with access + pg_dump -h db-host -U postgres -d atol_db -F c -f backup_$(date +%Y%m%d_%H%M%S).dump + + # Or SQL format + pg_dump -h db-host -U postgres -d atol_db > backup_$(date +%Y%m%d_%H%M%S).sql + ``` + +2. **Stop application** + ```bash + # Systemd + sudo systemctl stop atol-api + + # Or supervisor + sudo supervisorctl stop atol-api + + # Or Kubernetes + kubectl scale deployment atol-api --replicas=0 + ``` + +3. **Deploy new code** + ```bash + # Pull latest code + git pull origin main + + # Install dependencies if changed + pip install -r requirements.txt + ``` + +4. **Run migration** + ```bash + # Activate virtual environment + source venv/bin/activate + + # Run migration + alembic upgrade head + + # Verify + alembic current + ``` + +5. **Start application** + ```bash + # Systemd + sudo systemctl start atol-api + + # Or supervisor + sudo supervisorctl start atol-api + + # Or Kubernetes + kubectl scale deployment atol-api --replicas=3 + ``` + +6. **Monitor** + ```bash + # Check logs + sudo journalctl -u atol-api -f + + # Or supervisor + sudo tail -f /var/log/atol-api/error.log + + # Or Kubernetes + kubectl logs -f deployment/atol-api + ``` + +#### Option 2: Zero-Downtime (Rolling Deployment) + +1. **Ensure migration is backward-compatible** + - Add columns as nullable + - Don't drop columns yet + - Application code works with both old and new schema + +2. **Apply migration (while app is running)** + ```bash + # SSH to application server or use CI/CD + source venv/bin/activate + alembic upgrade head + ``` + +3. **Deploy new application code (rolling)** + ```bash + # Kubernetes rolling update + kubectl set image deployment/atol-api api=atol-api:v2.0 + kubectl rollout status deployment/atol-api + + # Or manual rolling restart with load balancer + # Update server 1, wait, update server 2, etc. + ``` + +4. **Verify deployment** + ```bash + # Check health endpoint + curl https://api.example.com/health + + # Monitor logs across all instances + kubectl logs -l app=atol-api --tail=100 + ``` + +### Rollback Procedure (Non-Docker) + +```bash +# Check current version +alembic current + +# Rollback one migration +alembic downgrade -1 + +# Rollback to specific version +alembic downgrade + +# Restore from backup (last resort) +pg_restore -h db-host -U postgres -d atol_db -c backup.dump +# Or for SQL format: +psql -h db-host -U postgres -d atol_db < backup.sql + +# Restart application +sudo systemctl restart atol-api +``` + +### CI/CD Integration + +#### GitHub Actions Example + +```yaml +# .github/workflows/deploy.yml +name: Deploy to Production + +on: + push: + branches: [main] + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.12' + + - name: Install dependencies + run: | + pip install -r requirements.txt + + - name: Run migrations + env: + DATABASE_URI: ${{ secrets.DATABASE_URI }} + run: | + alembic upgrade head + + - name: Deploy application + run: | + # Your deployment script + ./deploy.sh +``` + +#### GitLab CI Example + +```yaml +# .gitlab-ci.yml +stages: + - migrate + - deploy + +migrate: + stage: migrate + script: + - pip install -r requirements.txt + - alembic upgrade head + only: + - main + +deploy: + stage: deploy + script: + - ./deploy.sh + only: + - main +``` + +### Kubernetes-Specific Considerations + +1. **Use init containers for migrations** + ```yaml + apiVersion: apps/v1 + kind: Deployment + metadata: + name: atol-api + spec: + template: + spec: + initContainers: + - name: migrate + image: atol-api:latest + command: ["alembic", "upgrade", "head"] + env: + - name: DATABASE_URI + valueFrom: + secretKeyRef: + name: db-secret + key: uri + containers: + - name: api + image: atol-api:latest + ``` + +2. **Use Jobs for one-off migrations** + ```yaml + apiVersion: batch/v1 + kind: Job + metadata: + name: db-migration-v2 + spec: + template: + spec: + containers: + - name: migrate + image: atol-api:latest + command: ["alembic", "upgrade", "head"] + restartPolicy: Never + ``` + +### Production Best Practices (Non-Docker) + +1. **Database connection pooling** + - Use PgBouncer or similar for connection pooling + - Migrations should use direct connection, not pooler + +2. **Migration locks** + - Alembic uses database locks to prevent concurrent migrations + - Ensure only one migration process runs at a time + +3. **Monitoring** + ```bash + # Check migration status + alembic current + + # View migration history + alembic history --verbose + + # Check database version + psql -h db-host -U postgres -d atol_db -c "SELECT * FROM alembic_version;" + ``` + +4. **Automated backups before migrations** + ```bash + #!/bin/bash + # pre-migration-backup.sh + + BACKUP_DIR="/backups/db" + TIMESTAMP=$(date +%Y%m%d_%H%M%S) + + # Create backup + pg_dump -h db-host -U postgres -d atol_db -F c -f "$BACKUP_DIR/pre_migration_$TIMESTAMP.dump" + + # Run migration + alembic upgrade head + + # Keep last 7 days of backups + find $BACKUP_DIR -name "pre_migration_*.dump" -mtime +7 -delete + ``` + +5. **Health checks** + - Ensure health endpoint checks database connectivity + - Load balancer should remove unhealthy instances during migration + +## Migration File Best Practices + +### Structure + +```python +"""Brief description of what this migration does. + +Revision ID: xxxx +Revises: yyyy +Create Date: 2026-02-09 + +Detailed explanation of changes: +- What tables are affected +- What columns are added/removed +- Any data transformations +""" + +def upgrade() -> None: + # 1. Add new columns as nullable first (for zero-downtime) + # 2. Migrate data if needed + # 3. Make columns NOT NULL + # 4. Add constraints and indexes + # 5. Drop old columns last + pass + +def downgrade() -> None: + # Reverse all changes in opposite order + pass +``` + +### Tips + +- **Idempotent operations**: Use `IF EXISTS` / `IF NOT EXISTS` where possible +- **Data migration**: Include SQL for migrating existing data +- **Indexes**: Create indexes CONCURRENTLY in production to avoid locks +- **Foreign keys**: Add them after data is populated +- **Transactions**: Alembic runs migrations in transactions by default +- **Testing**: Always test both upgrade and downgrade + +## Common Patterns + +### Adding a Required Column + +```python +def upgrade() -> None: + # Step 1: Add as nullable + op.add_column('table_name', sa.Column('new_col', sa.Text(), nullable=True)) + + # Step 2: Populate with default/migrated data + op.execute("UPDATE table_name SET new_col = 'default_value'") + + # Step 3: Make NOT NULL + op.alter_column('table_name', 'new_col', nullable=False) +``` + +### Renaming a Column + +```python +def upgrade() -> None: + op.alter_column('table_name', 'old_name', new_column_name='new_name') +``` + +### Dropping a Table with Foreign Keys + +```python +def upgrade() -> None: + # Drop dependent tables/constraints first + op.drop_constraint('fk_name', 'dependent_table', type_='foreignkey') + op.drop_table('table_name') +``` + +## Troubleshooting + +### Migration not visible in container +- Ensure `alembic` directory is mounted in `docker-compose.yml` +- Restart containers: `docker compose restart api` + +### Migration already applied +- Check: `docker compose exec api alembic current` +- View history: `docker compose exec api alembic history` + +### Database connection errors +- Ensure database is running: `docker compose ps` +- Check connection string in `.env` +- Run from within container: `docker compose exec api alembic ...` + +### Circular import errors +- Restart containers to clear Python cache +- Check for naming conflicts with Python packages + +## Monitoring + +After applying migrations in production: + +```bash +# Check migration status +docker compose exec api alembic current + +# View migration history +docker compose exec api alembic history + +# Check database size +docker compose exec db psql -U postgres -d atol_db -c " + SELECT pg_size_pretty(pg_database_size('atol_db'));" + +# Monitor application logs +docker compose logs -f api + +# Check for errors +docker compose logs api | grep ERROR +``` + +## References + +- [Alembic Documentation](https://alembic.sqlalchemy.org/) +- [SQLAlchemy Documentation](https://docs.sqlalchemy.org/) +- Project migrations: `alembic/versions/` diff --git a/schema.sql b/schema.sql index 075d106..c3b3648 100644 --- a/schema.sql +++ b/schema.sql @@ -548,6 +548,7 @@ CREATE TABLE assembly ( -- Assembly metadata assembly_name TEXT NOT NULL, assembly_type TEXT NOT NULL DEFAULT 'clone or isolate', + tol_id TEXT, data_types assembly_data_types NOT NULL, coverage FLOAT NOT NULL, program TEXT NOT NULL, @@ -562,6 +563,18 @@ CREATE TABLE assembly ( updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); +CREATE TABLE assembly_run ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + organism_key TEXT REFERENCES organism(grouping_key) NOT NULL, + sample_id UUID REFERENCES sample(id) NOT NULL, + data_types assembly_data_types NOT NULL, + version INTEGER NOT NULL, + tol_id TEXT, + status TEXT NOT NULL DEFAULT 'reserved', + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + CREATE TABLE assembly_file ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), assembly_id UUID REFERENCES assembly(id) ON DELETE CASCADE NOT NULL, diff --git a/tests/unit/endpoints/test_endpoints_assemblies.py b/tests/unit/endpoints/test_endpoints_assemblies.py index 1a0f055..e69a86f 100644 --- a/tests/unit/endpoints/test_endpoints_assemblies.py +++ b/tests/unit/endpoints/test_endpoints_assemblies.py @@ -95,6 +95,7 @@ def test_create_assembly_from_experiments_success(monkeypatch): sample_id=uuid4(), assembly_name="Test Assembly", assembly_type="clone or isolate", + tol_id="tol-123", data_types="PACBIO_SMRT", coverage=50.0, program="hifiasm", @@ -126,6 +127,7 @@ def test_create_assembly_from_experiments_success(monkeypatch): "sample_id": "550e8400-e29b-41d4-a716-446655440000", "assembly_name": "Test Assembly", "assembly_type": "clone or isolate", + "tol_id": "tol-123", "coverage": 50.0, "program": "hifiasm", "moleculetype": "genomic DNA", @@ -165,6 +167,7 @@ def mock_create_raises(*args, **kwargs): "sample_id": "550e8400-e29b-41d4-a716-446655440000", "assembly_name": "Test Assembly", "assembly_type": "clone or isolate", + "tol_id": "tol-123", "coverage": 50.0, "program": "hifiasm", "moleculetype": "genomic DNA", @@ -188,10 +191,12 @@ def test_get_assembly_manifest_success(monkeypatch): scientific_name="Test Species", tax_id=172942, ) - sample = SimpleNamespace(id="sample-1", organism_key="test_organism") + sample = SimpleNamespace( + id="550e8400-e29b-41d4-a716-446655440000", organism_key="test_organism" + ) experiment = SimpleNamespace( id="exp-1", - sample_id="sample-1", + sample_id="550e8400-e29b-41d4-a716-446655440000", platform="PACBIO_SMRT", library_strategy="WGS", ) @@ -204,6 +209,13 @@ def test_get_assembly_manifest_success(monkeypatch): read_number=None, lane_number=None, ) + assembly_run = SimpleNamespace( + id="run-1", + organism_key="test_organism", + sample_id="sample-1", + tol_id="tol-123", + version=1, + ) class MockQuery: def __init__(self, return_value): @@ -212,6 +224,9 @@ def __init__(self, return_value): def filter(self, *args, **kwargs): return self + def order_by(self, *args, **kwargs): + return self + def first(self): return self.return_value if not isinstance(self.return_value, list) else None @@ -226,12 +241,14 @@ def query(self, model): self.call_count += 1 if self.call_count == 1: # organism query return MockQuery(organism) - elif self.call_count == 2: # samples query - return MockQuery([sample]) + elif self.call_count == 2: # sample query + return MockQuery(sample) elif self.call_count == 3: # experiments query return MockQuery([experiment]) elif self.call_count == 4: # reads query return MockQuery([read]) + elif self.call_count == 5: # latest assembly_run query + return MockQuery(assembly_run) return MockQuery([]) app.dependency_overrides[assemblies.get_current_active_user] = lambda: SimpleNamespace( @@ -239,7 +256,9 @@ def query(self, model): ) app.dependency_overrides[assemblies.get_db] = lambda: MockDB() - resp = client.get("/api/v1/assemblies/manifest/172942") + resp = client.get( + "/api/v1/assemblies/manifest/172942?sample_id=550e8400-e29b-41d4-a716-446655440000" + ) assert resp.status_code == 200 assert resp.headers["content-type"] == "application/x-yaml" @@ -267,10 +286,44 @@ def first(self): ) app.dependency_overrides[assemblies.get_db] = lambda: MockDB() - resp = client.get("/api/v1/assemblies/manifest/999999") + resp = client.get( + "/api/v1/assemblies/manifest/999999?sample_id=550e8400-e29b-41d4-a716-446655440000" + ) assert resp.status_code == 404 response_data = resp.json() # Error format may be either {"detail": ...} or {"error": {"message": ...}} error_msg = response_data.get("detail") or response_data.get("error", {}).get("message", "") assert "not found" in error_msg + + +def test_create_assembly_intent_invalid_data_types_returns_app_error(monkeypatch): + client = TestClient(app) + + organism = SimpleNamespace(grouping_key="test_organism") + reads = [SimpleNamespace(id="r1", experiment_id="e1")] + experiments = [SimpleNamespace(id="e1", platform="UNKNOWN", library_strategy="UNKNOWN")] + + monkeypatch.setattr( + assemblies, + "_get_manifest_inputs_by_tax_id", + lambda db, tax_id, sample_id: (organism, reads, experiments), + ) + + app.dependency_overrides[assemblies.get_current_active_user] = lambda: SimpleNamespace( + is_active=True, roles=["curator"], is_superuser=False + ) + app.dependency_overrides[assemblies.get_db] = _override_db(_FakeSession()) + + resp = client.post( + "/api/v1/assemblies/intent/172942", + json={ + "sample_id": "550e8400-e29b-41d4-a716-446655440000", + "tol_id": "tol-123", + }, + ) + + assert resp.status_code == 400 + body = resp.json() + assert body["error"]["code"] == "assembly_intent_invalid_data_types" + assert "No valid sequencing platforms detected" in body["error"]["message"] diff --git a/tests/unit/services/test_assembly_helper.py b/tests/unit/services/test_assembly_helper.py index eb30bdb..b323cb5 100644 --- a/tests/unit/services/test_assembly_helper.py +++ b/tests/unit/services/test_assembly_helper.py @@ -197,7 +197,7 @@ def test_pacbio_reads_filtered_by_extension(self): ), ] - result = generate_assembly_manifest(organism, reads, experiments) + result = generate_assembly_manifest(organism, reads, experiments, "tol1", 1) assert "PACBIO_SMRT:" in result assert "sample.ccs.bam" in result @@ -220,7 +220,7 @@ def test_hic_reads_include_metadata(self): ), ] - result = generate_assembly_manifest(organism, reads, experiments) + result = generate_assembly_manifest(organism, reads, experiments, "tol1", 1) assert "Hi-C:" in result assert "hic_R1.fastq.gz" in result @@ -243,7 +243,7 @@ def test_wgs_treated_as_hic(self): ), ] - result = generate_assembly_manifest(organism, reads, experiments) + result = generate_assembly_manifest(organism, reads, experiments, "tol1", 1) assert "Hi-C:" in result assert "sample_R1.fastq.gz" in result @@ -254,7 +254,7 @@ def test_empty_reads_dict(self): experiments = [Mock(id="exp1", platform="UNKNOWN", library_strategy="WGS")] reads = [] - result = generate_assembly_manifest(organism, reads, experiments) + result = generate_assembly_manifest(organism, reads, experiments, "tol1", 1) assert "reads: {}" in result @@ -264,10 +264,12 @@ def test_organism_metadata_included(self): experiments = [] reads = [] - result = generate_assembly_manifest(organism, reads, experiments) + result = generate_assembly_manifest(organism, reads, experiments, "tol123", 2) assert "scientific_name: Saiphos equalis" in result assert "taxon_id: 172942" in result + assert "tolid: tol123" in result + assert "version: 2" in result def test_reads_without_experiment_id_skipped(self): """Test that reads without experiment_id are skipped.""" @@ -285,7 +287,7 @@ def test_reads_without_experiment_id_skipped(self): ), ] - result = generate_assembly_manifest(organism, reads, experiments) + result = generate_assembly_manifest(organism, reads, experiments, "tol1", 1) assert "sample.ccs.bam" not in result assert "reads: {}" in result @@ -318,7 +320,7 @@ def test_multiple_platform_types(self): ), ] - result = generate_assembly_manifest(organism, reads, experiments) + result = generate_assembly_manifest(organism, reads, experiments, "tol1", 1) assert "PACBIO_SMRT:" in result assert "Hi-C:" in result diff --git a/tests/unit/services/test_assembly_service.py b/tests/unit/services/test_assembly_service.py index c39b4b3..4d8ca25 100644 --- a/tests/unit/services/test_assembly_service.py +++ b/tests/unit/services/test_assembly_service.py @@ -35,6 +35,7 @@ def sample_assembly_create(): project_id=uuid.uuid4(), assembly_name="Test Assembly", assembly_type="clone or isolate", + tol_id="tol-001", data_types=AssemblyDataTypes.PACBIO_SMRT, coverage=50.0, program="hifiasm", @@ -99,6 +100,7 @@ def test_create_version_per_combination(self, mock_db, assembly_service): sample_id=sample_id, assembly_name="Assembly 1", assembly_type="clone or isolate", + tol_id="tol-002", data_types=AssemblyDataTypes.PACBIO_SMRT, coverage=50.0, program="hifiasm", @@ -110,6 +112,7 @@ def test_create_version_per_combination(self, mock_db, assembly_service): sample_id=sample_id, assembly_name="Assembly 2", assembly_type="clone or isolate", + tol_id="tol-003", data_types=AssemblyDataTypes.PACBIO_SMRT_HIC, coverage=50.0, program="hifiasm", @@ -164,6 +167,7 @@ def test_create_from_experiments_success(self, mock_db, assembly_service): sample_id=sample.id, assembly_name="Test Assembly", assembly_type="clone or isolate", + tol_id="tol-004", coverage=50.0, program="hifiasm", moleculetype="genomic DNA", @@ -194,6 +198,7 @@ def test_create_from_experiments_organism_not_found(self, mock_db, assembly_serv sample_id=uuid.uuid4(), assembly_name="Test Assembly", assembly_type="clone or isolate", + tol_id="tol-005", coverage=50.0, program="hifiasm", moleculetype="genomic DNA", @@ -221,6 +226,7 @@ def test_create_from_experiments_no_samples(self, mock_db, assembly_service): sample_id=uuid.uuid4(), assembly_name="Test Assembly", assembly_type="clone or isolate", + tol_id="tol-006", coverage=50.0, program="hifiasm", moleculetype="genomic DNA", @@ -252,6 +258,7 @@ def test_create_from_experiments_no_experiments(self, mock_db, assembly_service) sample_id=sample.id, assembly_name="Test Assembly", assembly_type="clone or isolate", + tol_id="tol-007", coverage=50.0, program="hifiasm", moleculetype="genomic DNA", @@ -297,6 +304,7 @@ def test_create_from_experiments_overrides_data_types(self, mock_db, assembly_se sample_id=sample.id, assembly_name="Test Assembly", assembly_type="clone or isolate", + tol_id="tol-008", data_types=AssemblyDataTypes.PACBIO_SMRT, # Explicitly provided, should be used coverage=50.0, program="hifiasm",