diff --git a/.gitignore b/.gitignore index 0cffff1..543133b 100644 --- a/.gitignore +++ b/.gitignore @@ -15,5 +15,4 @@ postman/ .coverage htmlcov/ coverage.xml -.claude/ .windsurf/ diff --git a/BROKER_PREREQUISITES_ENHANCEMENT.md b/BROKER_PREREQUISITES_ENHANCEMENT.md deleted file mode 100644 index 1dc7afc..0000000 --- a/BROKER_PREREQUISITES_ENHANCEMENT.md +++ /dev/null @@ -1,115 +0,0 @@ -# Broker Prerequisites Enhancement - -## Problem -The broker API was returning null values for `prerequisites`, `validation_hints`, and `file_metadata` fields because: -1. Missing accession data in submission payloads -2. No way to distinguish between "not required" vs "required but not yet submitted" - -## Solution -Enhanced the broker contract to include **existing** accessions (resolved from `accession_registry`), allowing clients to decide whether dependency state is complete enough to submit. - -### Key Architecture Understanding - -**Crucial clarification**: Accessions are stored in different places: -- **Existing accessions**: Stored in database row fields (`project_accession`, `sample_accession`, etc.) and `accession_registry` table -- **prepared_payload**: Contains data for ENA submission, NOT existing accessions -- **Required accessions**: Specified in payload using `expected_*_accession` fields for dependencies not yet submitted - -### Changes Made - -#### 1. BrokerPrerequisites Schema -```python -class BrokerPrerequisites(BaseModel): - # Existing accessions (from database row fields) - project_accession: Optional[str] = None - sample_accession: Optional[str] = None - experiment_accession: Optional[str] = None - run_accession: Optional[str] = None - study_accession: Optional[str] = None - analysis_accession: Optional[str] = None -``` - -#### 2. Enhanced Prerequisite Extraction Logic -- **Existing accessions**: Looked up from `accession_registry` via FK relationships on submission tables - -#### 3. Updated Test Cases -- Tests now verify both existing and required accessions -- Demonstrates mixed states where some dependencies exist and others don't - -### Usage Examples - -#### Sample Submission with Existing Project Accession -**Database state**: `accession_registry` contains a project accession for the sample's `project_id` -```json -{ - "alias": "sample-1", - "requires_project_accession": true -} -``` -Result: -```json -{ - "prerequisites": { - "project_accession": "PRJ123456" // ✅ Resolved from accession_registry - } -} -``` - -#### Experiment with Missing Study Accession -**Database state**: `accession_registry` does not contain a project accession for the experiment's `project_id` -```json -{ - "alias": "experiment-1", - "requires_study_accession": true, - "expected_study_accession": "PRJ123456" // Required but not submitted -} -``` -Result: -```json -{ - "prerequisites": { - "sample_accession": "SAMEA123456", // ✅ Resolved from accession_registry - "study_accession": null // ❌ Missing from registry - } -} -``` - -#### Run with Missing Experiment Accession -**Database state**: `accession_registry` does not contain an experiment accession for the run's `experiment_id` -```json -{ - "alias": "run-1", - "file_name": "reads.fastq.gz", - "file_format": "fastq", - "expected_experiment_accession": "ERX123456" // Required but not submitted -} -``` -Result: -```json -{ - "prerequisites": { - "experiment_accession": null // ❌ Missing from registry - }, - "files": [ - {"filename": "reads.fastq.gz", "filetype": "fastq"} - ] -} -``` - -### Data Flow Summary - -1. **prepared_payload**: Contains ENA submission data (metadata, files, etc.) -2. **Database row fields**: Store actual accessions once submitted (`*_accession` fields) -3. **Broker response**: Shows resolved accessions (or `null`), and clients decide completeness - -### Client Benefits -1. **Clear dependency visibility**: See what exists vs what is missing -2. **Single source of truth**: Accessions resolved from `accession_registry` - -### Payload Fields for Required Accessions -- `expected_*_accession`: Placeholder for the accession that will be assigned -- `requires_*_accession`: Boolean flag indicating the dependency is required -- `requires_project_accession`: For samples -- `requires_study_accession`: For experiments (maps to `required_project_accession` in broker prerequisites) - -This enhancement maintains backward compatibility while providing much richer dependency information to broker clients, with the correct understanding of where different types of data are stored. diff --git a/README-old.md.md b/README-old.md.md new file mode 100644 index 0000000..630fb66 --- /dev/null +++ b/README-old.md.md @@ -0,0 +1,383 @@ +# Canopy: A Metadata Tracking System for the Australian Tree of Life (AToL) data + +Canopy is a FastAPI backend used to track and manage genomic data for the Australian Tree of Life (AToL) project. + +> **Handover note:** This README is the entry point and is kept complete and current. For a deeper, evidence-based handover pack (system architecture, lifecycle rules, broker flows, troubleshooting, and open questions), see [`docs/handover/README.md`](docs/handover/README.md). + +## Overview + +Canopy manages metadata across core biological entities (Organism, Sample, Experiment, Read, Assembly, Project, BPA Initiative, Genome Note, QC Read), and models each submission lifecycle with a two-table pattern per entity: +- Main: current state +- Submission: staged for external submission (e.g., ENA) + +A dedicated broker workflow enables integration with external submission pipelines. The broker can claim work (with short leases) and report outcomes/accessions back to Canopy. Helper endpoints provide attempt summaries and item listings to power a simple dashboard or external UI. + +## Features + +- Authentication & tokens: JWT access, refresh, and logout endpoints +- Role-based access control: user, curator, broker, genome_launcher, admin, superuser +- RESTful APIs for core entities (organisms, samples, experiments, reads, assemblies, projects, genome notes, BPA initiatives, QC reads) +- Submission workflow endpoints (sample-submissions, experiment-submissions, qc-read-submissions) +- Broker endpoints to support external submission pipelines: + - Claim drafts and obtain a lease: `/api/v1/broker/organisms/{taxon_id}/claim` + - ENA broker contract endpoints: `/api/v1/broker/claims/ready`, `/api/v1/broker/claims/entity`, `/api/v1/broker/validation`, `/api/v1/broker/reports/{attempt_id}` + - Renew lease, finalise, and report results: `/api/v1/broker/attempts/{attempt_id}/...` + - Lightweight ToLID persistence/reporting endpoints: `/api/v1/broker/tolids/...` + - Attempt listing and summaries for dashboard views +- Bulk import endpoints for organisms, samples, experiments, and taxonomy info +- Assembly intent flow: generates a manifest from specimen/lineage data for pipeline consumption (see [`docs/assembly_reporting_api.md`](docs/assembly_reporting_api.md)) +- PostgreSQL database with UUID primary keys and JSONB fields; schema migrated via Alembic, with `schema.sql` kept as a current schema snapshot +- Docker Compose for local development, plus an alternative local (non-Docker) run mode + +## Tech Stack + +- Python 3.12, FastAPI, Uvicorn +- SQLAlchemy 2.x ORM +- Pydantic v2 + pydantic-settings +- PostgreSQL +- JWT auth via python-jose[cryptography]; password hashing via passlib[bcrypt] +- Alembic for schema migrations (`alembic/versions/`) +- Docker & Docker Compose +- Deployed via GitHub Actions to AWS (image pushed to ECR, deploy triggered via Lambda) — see [`.github/workflows/build-and-deploy-dev.yml`](.github/workflows/build-and-deploy-dev.yml) + +## Project Structure + +``` +app/ +├── main.py +├── api/ +│ └── v1/ +│ ├── api.py +│ └── endpoints/ +│ ├── auth.py +│ ├── users.py +│ ├── organisms.py +│ ├── samples.py +│ ├── sample_submissions.py +│ ├── experiments.py +│ ├── experiment_submissions.py +│ ├── reads.py +│ ├── qc_reads.py +│ ├── assemblies.py +│ ├── projects.py +│ ├── genome_notes.py* +│ └── broker.py +├── core/ +│ ├── dependencies.py +│ ├── security.py +│ └── settings.py +├── db/ +│ └── session.py +├── models/ +│ ├── user.py, token.py +│ ├── organism.py, sample.py, experiment.py, read.py, qc_read.py +│ ├── assembly.py, project.py, genome_note.py* +│ └── (SQLAlchemy models) +└── schemas/ + ├── user.py + ├── organism.py, sample.py, experiment.py, read.py, qc_read.py + ├── assembly.py, project.py, genome_note.py* + └── (Pydantic schemas) + +# Top-level +pyproject.toml, uv.lock, docker-compose.yml, Dockerfile, schema.sql, scripts/, docs/ +``` + +## Getting Started + +### Prerequisites + +- Docker and Docker Compose +- [uv](https://docs.astral.sh/uv/) for local (non-Docker) development +- Postgres available for Alembic migrations + +### Running the Application (Docker) + +1) Create your environment file from the template: + +```bash +cp .env.example .env +``` + +2) Edit `.env` and set the required values. At minimum: +- `POSTGRES_USER=postgres` +- `POSTGRES_PASSWORD=` (default example uses `postgres`) +- `POSTGRES_DB=atol_db` +- `POSTGRES_PORT=5432` # container port (host is mapped to 5433) +- `POSTGRES_SERVER=db` # do not change; this is the Docker service name +- `JWT_SECRET_KEY=` +- `JWT_ALGORITHM=HS256` + +Generate a secure secret (macOS/Linux): + +```bash +openssl rand -hex 32 +``` + +3) Build and start the stack: + +```bash +docker compose up -d --build # Compose v2 +# or +docker-compose up -d --build # older Compose +``` + +4) Tail logs (optional) and wait for the API to be ready: + +```bash +docker compose logs -f api +``` + +5) Open the API docs: + +- Swagger UI: http://localhost:8000/api/v1/docs +- ReDoc: http://localhost:8000/api/v1/redoc + +6) Stop the stack: + +```bash +docker compose down +``` + +To reset the database (this will delete your data): + +```bash +docker compose down -v +``` + +Note on code changes (Docker): `scripts/entrypoint.sh` only enables uvicorn `--reload` when `ENVIRONMENT=dev`. If reload isn't enabled, restart the API container to pick up code changes: + +```bash +docker compose restart api +``` + +#### Local database access (via Docker) +Connect to the Postgres database hosted on the `db` container using `psql`. + +- Quick one-liner (uses credentials created by `.env` during container init): + + ```bash + docker compose exec db psql -U postgres -d atol_db + ``` + +- Or open a shell in the container first, then run psql: + + ```bash + docker compose exec -it db bash + psql -U postgres -d atol_db + ``` + +Once in psql, you can run SQL and handy meta-commands: + +```sql +-- List databases +\l +-- Connect to a database (if needed) +\c atol_db +-- List schemas +\dn +-- List tables (optionally by schema) +\dt +\dt public.* +-- Describe a table +\d+ public.users +-- Run a query +SELECT NOW(); +-- Quit psql +\q +``` + +### Create your first user + +On a fresh database you will likely want to create an initial user (e.g. an admin/broker) so you can log in and use secured endpoints. Use the helper script from the project root on your host machine: + +```bash +python scripts/create_user.py \ + --host localhost \ + --port 5433 \ + --dbname atol_db \ + --user postgres \ + --password \ + --username admin \ + --email admin@example.org \ + --user-password \ + --role user +``` + +You will need to manually update the role for your first user in the database. + +Then obtain a JWT to call secured endpoints: + +```bash +# Exchange username/password for tokens +curl -s -X POST http://localhost:8000/api/v1/auth/login \ + -H 'Content-Type: application/x-www-form-urlencoded' \ + -d 'username=admin&password=' +``` + +Export the returned `access_token` and use it in the `Authorization: Bearer ` header. + +### Broker integration quickstart + +The broker API provides claim/report endpoints to integrate with an external submission pipeline. + +- Claim items for an organism (example): + + ```bash + TOKEN= + curl -s -X POST "http://localhost:8000/api/v1/broker/organisms//claim?per_type_limit=100" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"lease_duration_minutes": 30}' + ``` + +- Report results for an attempt (example shape): + + ```bash + curl -s -X POST "http://localhost:8000/api/v1/broker/attempts//report" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"samples": [], "experiments": [], "reads": [], "projects": []}' + ``` + +For the lightweight ToLID persistence/reporting flow, see [docs/tolid_broker_api.md](docs/tolid_broker_api.md). +For a deeper, evidence-based walkthrough of the broker and submission lifecycle, see [docs/handover/broker_and_submission_flows.md](docs/handover/broker_and_submission_flows.md). + +## Bulk Import API + +The system provides API endpoints for bulk importing organisms, samples, experiments, and taxonomy info. These endpoints allow you to import data in the same format as the standalone import script but through authenticated API calls. + +### Bulk Import Endpoints + +- `/api/v1/organisms/bulk-import` - Bulk import organisms +- `/api/v1/samples/bulk-import` - Bulk import samples +- `/api/v1/experiments/bulk-import` - Bulk import experiments + +All bulk import endpoints: +- Require authentication with 'curator' or 'admin' role +- Accept JSON data in the same format as the standalone import script +- Return counts of created and skipped records + +See [docs/bulk_import_api.md](docs/bulk_import_api.md) for request formats and curl examples. + +## API Documentation + +Once the application is running, you can access the interactive API documentation: + +- Swagger UI: http://localhost:8000/api/v1/docs +- ReDoc: http://localhost:8000/api/v1/redoc + +## Role-Based Access Control + +The system implements role-based access control with the following roles: + +- **user**: Basic read access to data +- **curator**: Can create and update biological entities +- **broker**: Can claim and report biological entities +- **genome_launcher**: Can access data required by genome assembly & post-assembly pipelines and report results +- **admin**: Full access to all endpoints +- **superuser**: Special role with delete permissions + +## Development (run without Docker) + +Prefer Docker for a consistent setup. This section shows how to run the FastAPI app directly on your machine as an alternative. + +1) Sync the environment with uv (creates `.venv` in the project root): + +```bash +uv sync --dev +# Optional: source .venv/bin/activate +``` + +2) Configure environment + +- Copy `.env.example` to `.env` and fill the required values (particularly `JWT_SECRET_KEY`, `JWT_ALGORITHM`). +- Choose ONE of the database options below: + +Option A — reuse the Docker Postgres (recommended) + +```bash +# Start only the database service +docker compose up -d db + +# In your .env (or environment), point to the compose database on host 5433 +POSTGRES_SERVER=localhost +POSTGRES_PORT=5433 +POSTGRES_USER=postgres +POSTGRES_PASSWORD= +POSTGRES_DB=atol_db +``` + +Option B — use a local Postgres (no Docker) + +```bash +# Ensure Postgres is running locally (port 5432) +# Create the database (migrations will handle schema) +createdb -h localhost -U postgres atol_db || true + +# In your .env (or environment), point to your local instance +POSTGRES_SERVER=localhost +POSTGRES_PORT=5432 +POSTGRES_USER=postgres +POSTGRES_PASSWORD= +POSTGRES_DB=atol_db +``` + +3) Run the application + +```bash +uv run alembic upgrade head +uv run uvicorn app.main:app --reload --host 0.0.0.0 --port 8000 +``` + +4) Open the docs at http://localhost:8000/api/v1/docs + +### Linting + +- Install hooks: `uv run pre-commit install` +- Run all checks: `uv run pre-commit run --all-files` +- Install commit message hook: `uv run pre-commit install --hook-type commit-msg` (requires commitlint hook configured) + +### Database migrations (Alembic) + +- Apply migrations: `uv run alembic upgrade head` +- Create a new revision: `uv run alembic revision --autogenerate -m "description"` +- Docker Compose runs migrations automatically on container start via `scripts/entrypoint.sh`. +- `scripts/entrypoint.sh` supports `APP_MODE=serve` (default) and `APP_MODE=migrate` (run migrations then exit). +- For Alembic conventions, migration patterns, and troubleshooting, see [docs/migration_workflow.md](docs/migration_workflow.md). + +### Environment configuration + +Configuration is provided via environment variables loaded from `.env` (see `.env.example`). Key settings: + +- Database + - `POSTGRES_SERVER` (Docker service name is `db`) + - `POSTGRES_USER` + - `POSTGRES_PASSWORD` + - `POSTGRES_DB` + - `POSTGRES_PORT` (container port, default 5432; host is `5433` via compose) +- Auth + - `JWT_SECRET_KEY` (required) + - `JWT_ALGORITHM` (e.g. `HS256`) + - `JWT_ACCESS_TOKEN_EXPIRE_MINUTES` + - `JWT_REFRESH_TOKEN_EXPIRE_DAYS` +- CORS + - `BACKEND_CORS_ORIGINS` (JSON array of origins) + +For the full environment variable reference derived from code, see [docs/handover/config_reference.md](docs/handover/config_reference.md). + +## Deployment + +The app is built and deployed to AWS via GitHub Actions: +- [`.github/workflows/build-and-deploy-dev.yml`](.github/workflows/build-and-deploy-dev.yml) builds a Docker image, pushes it to ECR, and invokes an AWS Lambda function to deploy it (on every push to `main`). +- [`.github/workflows/build-publish.yml`](.github/workflows/build-publish.yml) builds and pushes release-tagged images to ECR. + +The underlying AWS runtime infrastructure (where the Lambda deploys to, networking, secrets) is managed by another team — see [docs/handover/setup_and_operations.md](docs/handover/setup_and_operations.md) for what's verifiable from this repo, and [docs/handover/open_questions.md](docs/handover/open_questions.md) for what isn't. + +## Handover Documentation + +For a deeper, evidence-based handover pack covering system architecture, lifecycle rules, broker/submission flows, environment config, troubleshooting, and known gaps, start at [docs/handover/README.md](docs/handover/README.md). + +## License + +This project is licensed under the GPL-3.0-or-later License. diff --git a/README.md b/README.md index f86f804..b8f3f25 100644 --- a/README.md +++ b/README.md @@ -1,381 +1,231 @@ -# Canopy: A Metadata Tracking System for the Australian Tree of Life (AToL) data +# Canopy -Canopy is a FastAPI backend used to track and manage genomic data for the Australian Tree of Life (AToL) project. +Canopy is a FastAPI backend for managing metadata and submission-tracking records for the Australian Tree of Life project. -## Overview +This `README` is the central repository document. It gives the verified high-level picture of the application and links to more detailed maintainer documentation where the detail would otherwise make this file unwieldy. -Canopy manages metadata across core biological entities (Organism, Sample, Experiment, Read, Assembly, Project, BPA Initiative, Genome Note), and models each submission lifecycle with a two-table pattern per entity: -- Main: current state -- Submission: staged for external submission (e.g., ENA) +## Verified Scope -A dedicated broker workflow enables integration with external submission pipelines. The broker can claim work (with short leases) and report outcomes/accessions back to Canopy. Helper endpoints provide attempt summaries and item listings to power a simple dashboard or external UI. +The current codebase stores and serves metadata for: -## Features +- organisms +- taxonomy enrichment records +- samples and specimen-to-derived sample lineage +- experiments +- reads +- QC-read results +- assemblies, assembly runs, and assembly stage runs +- projects +- BPA initiatives +- genome notes +- users and refresh tokens +- broker submission attempts, events, and ToLID request state -- Authentication & tokens: JWT access, refresh, and logout endpoints -- Role-based access control: user, curator, broker, genome_launcher, admin, superuser -- RESTful APIs for core entities (organisms, samples, experiments, reads, assemblies, projects, genome notes, BPA initiatives) -- Submission workflow endpoints (sample-submissions, experiment-submissions, read-submissions) -- Broker endpoints to support external submission pipelines: - - Claim drafts and obtain a lease: `/api/v1/broker/organisms/{taxon_id}/claim` - - ENA broker contract endpoints: `/api/v1/broker/claims/ready`, `/api/v1/broker/claims/entity`, `/api/v1/broker/validation`, `/api/v1/broker/reports/{attempt_id}` - - Renew lease, finalise, and report results: `/api/v1/broker/attempts/{attempt_id}/...` - - Lightweight ToLID persistence/reporting endpoints: `/api/v1/broker/tolids/...` - - Attempt listing and summaries for dashboard views -- Bulk import endpoints for organisms, samples, and experiments -- XML export endpoints for downstream systems -- PostgreSQL database with UUID primary keys and JSONB fields; schema bootstrapped via Docker using `schema.sql` -- Docker Compose for local development, plus an alternative local (non-Docker) run mode +Primary implementation paths: -## Tech Stack +- API app: `app/main.py` +- Versioned router: `app/api/v1/api.py` +- Runtime settings: `app/core/settings.py` +- Models: `app/models/` +- Schemas: `app/schemas/` +- Shared business logic: `app/services/` +- Migrations: `alembic/versions/` -- Python 3.10, FastAPI, Uvicorn -- SQLAlchemy 2.x ORM -- Pydantic v2 + pydantic-settings -- PostgreSQL 14 -- JWT auth via python-jose[cryptography]; password hashing via passlib[bcrypt] -- Alembic (available); schema initialization via `schema.sql` in Docker -- Docker & Docker Compose +## How The System Works -## Project Structure +### Core pattern -``` -app/ -├── main.py -├── api/ -│ └── v1/ -│ ├── api.py -│ └── endpoints/ -│ ├── auth.py -│ ├── users.py -│ ├── organisms.py -│ ├── samples.py -│ ├── sample_submissions.py -│ ├── experiments.py -│ ├── experiment_submissions.py -│ ├── reads.py -│ ├── read_submissions.py -│ ├── assemblies.py -│ ├── projects.py -│ ├── bpa_initiatives.py -│ ├── genome_notes.py -│ ├── broker.py -│ └── xml_export.py -├── core/ -│ ├── dependencies.py -│ ├── security.py -│ └── settings.py -├── db/ -│ └── session.py -├── models/ -│ ├── user.py, token.py -│ ├── organism.py, sample.py, experiment.py, read.py -│ ├── assembly.py, project.py, bpa_initiative.py, genome_note.py -│ ├── accession_registry.py, broker.py -│ └── (SQLAlchemy models) -└── schemas/ - ├── user.py - ├── organism.py, sample.py, experiment.py, read.py - ├── assembly.py, project.py, bpa_initiative.py, genome_note.py - └── (Pydantic schemas) - -# Top-level -pyproject.toml, uv.lock, docker-compose.yml, Dockerfile, schema.sql, scripts/, docs/, data/ -``` - -## Getting Started - - ### Prerequisites - - - Docker and Docker Compose - - [uv](https://docs.astral.sh/uv/) for local (non-Docker) development - - Postgres available for Alembic migrations - - ### Running the Application (Docker) - - 1) Create your environment file from the template: - - ```bash - cp .env.example .env - ``` - - 2) Edit `.env` and set the required values. At minimum: - - `POSTGRES_USER=postgres` - - `POSTGRES_PASSWORD=` (default example uses `postgres`) - - `POSTGRES_DB=atol_db` - - `POSTGRES_PORT=5432` # container port (host is mapped to 5433) - - `POSTGRES_SERVER=db` # do not change; this is the Docker service name - - `JWT_SECRET_KEY=` - - `JWT_ALGORITHM=HS256` - - Generate a secure secret (macOS/Linux): - - ```bash - openssl rand -hex 32 - ``` - - 3) Build and start the stack: +Several entities use a main-table plus submission-table pattern. - ```bash - docker compose up -d --build # Compose v2 - # or - docker-compose up -d --build # older Compose - ``` +- Main table: current application-facing record +- Submission table: staged payloads and submission lifecycle state for external workflows - 4) Tail logs (optional) and wait for the API to be ready: +This pattern is visible for: - ```bash - docker compose logs -f api - ``` +- projects +- samples +- experiments +- QC reads +- assemblies - 5) Open the API docs: +### Broker-facing workflow - - Swagger UI: http://localhost:8000/api/v1/docs - - ReDoc: http://localhost:8000/api/v1/redoc +The repo exposes broker endpoints under `/api/v1/broker/...` for claiming work, validating payload prerequisites, reporting outcomes, finalising failed attempts, and managing ToLID request state. - 6) Stop the stack: +The currently routed claim/report surface is: - ```bash - docker compose down - ``` +- `/broker/claims/ready` +- `/broker/claims/entity` +- `/broker/claims/batch` +- `/broker/validation` +- `/broker/reports/{attempt_id}` +- `/broker/attempts/{attempt_id}/finalise` - To reset the database (this will delete your data): +Detailed broker lifecycle notes are in [docs/handover/broker_and_submission_flows.md](docs/handover/broker_and_submission_flows.md). - ```bash - docker compose down -v - ``` +### Assembly workflow - Note on code changes (Docker): the API container runs uvicorn without `--reload`. If you edit code, restart the API container to pick up changes: +The current assembly flow is centered on: - ```bash - docker compose restart api - ``` +1. creating an assembly intent +2. generating and storing an assembly manifest +3. registering assembly pipeline runs by GitHub repo and commit +4. reporting QC-read outputs +5. reporting per-stage results - #### Local database access (via Docker) - Connect to the Postgres database hosted on the `db` container using `psql`. +Detailed assembly instructions are in [docs/assembly_reporting_api.md](docs/assembly_reporting_api.md). - - Quick one-liner (uses credentials created by `.env` during container init): +## Active Documentation - ```bash - docker compose exec db psql -U postgres -d atol_db - ``` +### Start here for maintainers - - Or open a shell in the container first, then run psql: +- [docs/handover/README.md](docs/handover/README.md) - ```bash - docker compose exec -it db bash - psql -U postgres -d atol_db - ``` +### Feature and operations docs verified against current code - Once in psql, you can run SQL and handy meta-commands: +- [docs/assembly_reporting_api.md](docs/assembly_reporting_api.md) +- [docs/auth_refresh_tokens.md](docs/auth_refresh_tokens.md) +- [docs/bulk_import_api.md](docs/bulk_import_api.md) +- [docs/migration_workflow.md](docs/migration_workflow.md) +- [docs/ncbi_taxonomy_sync.md](docs/ncbi_taxonomy_sync.md) +- [docs/tolid_broker_api.md](docs/tolid_broker_api.md) - ```sql - -- List databases - \l - -- Connect to a database (if needed) - \c atol_db - -- List schemas - \dn - -- List tables (optionally by schema) - \dt - \dt public.* - -- Describe a table - \d+ public.users - -- Run a query - SELECT NOW(); - -- Quit psql - \q - ``` +### Handover pack contents - ### Create your first user - On a fresh database you will likely want to create an initial user (e.g. an admin/broker) so you can log in and use secured endpoints. Use the helper script from the project root on your host machine: +- [docs/handover/system_overview.md](docs/handover/system_overview.md) +- [docs/handover/setup_and_operations.md](docs/handover/setup_and_operations.md) +- [docs/handover/broker_and_submission_flows.md](docs/handover/broker_and_submission_flows.md) +- [docs/handover/config_reference.md](docs/handover/config_reference.md) +- [docs/handover/troubleshooting.md](docs/handover/troubleshooting.md) +- [docs/handover/open_questions.md](docs/handover/open_questions.md) - ```bash - python scripts/create_user.py \ - --host localhost \ - --port 5433 \ - --dbname atol_db \ - --user postgres \ - --password \ - --username admin \ - --email admin@example.org \ - --user-password \ - --role user - ``` +## Local Development - You will need to manually update the role for your first user in the database. +### Requirements - Then obtain a JWT to call secured endpoints: +- Docker and Docker Compose for the container-based local path +- or `uv` plus a reachable PostgreSQL instance for the non-Docker local path - ```bash - # Exchange username/password for tokens - curl -s -X POST http://localhost:8000/api/v1/auth/login \ - -H 'Content-Type: application/x-www-form-urlencoded' \ - -d 'username=admin&password=' - ``` +### Environment and settings - Export the returned `access_token` and use it in the `Authorization: Bearer ` header. +Settings are defined in `app/core/settings.py`. - ### Broker integration quickstart - The broker API provides claim/report endpoints to integrate with an external submission pipeline. +At application import time, the code requires: - - Claim items for an organism (example): +- `JWT_SECRET_KEY` +- `JWT_ALGORITHM` +- `DATABASE_URI`, or enough `POSTGRES_*` values for `DATABASE_URI` to be derived - ```bash - TOKEN= - curl -s -X POST "http://localhost:8000/api/v1/broker/organisms//claim?per_type_limit=100" \ - -H "Authorization: Bearer $TOKEN" \ - -H "Content-Type: application/json" \ - -d '{"lease_duration_minutes": 30}' - ``` +At container entrypoint time, `scripts/entrypoint.sh` requires `DATABASE_URI` to exist in the shell environment before it will run migrations. - - Report results for an attempt (example shape): +The repository includes: - ```bash - curl -s -X POST "http://localhost:8000/api/v1/broker/attempts//report" \ - -H "Authorization: Bearer $TOKEN" \ - -H "Content-Type: application/json" \ - -d '{"samples": [], "experiments": [], "reads": [], "projects": []}' - ``` +- `.env.example` +- `docker-compose.yml` +- `Dockerfile` +- `scripts/entrypoint.sh` - For the flat ENA broker contract used by Canopy, see [docs/ena_broker_contract.md](docs/ena_broker_contract.md). - For the lightweight ToLID persistence/reporting flow, see [docs/tolid_broker_api.md](docs/tolid_broker_api.md). - For a deeper overview of attempt leasing and statuses, see the `broker` endpoints in `app/api/v1/endpoints/broker.py` and the interactive docs. +### Docker path - ## Bulk Import API +1. Copy the template: -The system provides API endpoints for bulk importing organisms, samples, and experiments. These endpoints allow you to import data in the same format as the standalone import script but through authenticated API calls. - -### Bulk Import Endpoints - -- `/api/v1/organisms/bulk-import` - Bulk import organisms -- `/api/v1/samples/bulk-import` - Bulk import samples -- `/api/v1/experiments/bulk-import` - Bulk import experiments - -All bulk import endpoints: -- Require authentication with 'curator' or 'admin' role -- Accept JSON data in the same format as the standalone import script -- Return counts of created and skipped records - -### Example usage - -See `docs/bulk_import_api.md` for request formats and curl examples for organisms, samples, and experiments. +```bash +cp .env.example .env +``` -4. Access the API documentation at http://localhost:8000/api/v1/docs +2. Set the required values in `.env`. -### Authentication +3. Start the stack: -The API uses JWT tokens for authentication. To authenticate: +```bash +docker compose up --build +``` -1. Create a user or use a default admin user -2. Get a token from `/auth/login` endpoint -3. Use the token in the Authorization header: `Bearer {token}` +4. Open the API docs: -## API Documentation +- `http://localhost:8000/api/v1/docs` +- `http://localhost:8000/api/v1/redoc` -Once the application is running, you can access the interactive API documentation: +The Docker Compose file maps: -- Swagger UI: http://localhost:8000/api/v1/docs -- ReDoc: http://localhost:8000/api/v1/redoc +- API: host `8000` -> container `8000` +- Postgres: host `5433` -> container `5432` -## Role-Based Access Control +### Non-Docker path -The system implements role-based access control with the following roles: +1. Install dependencies: -- **user**: Basic read access to data -- **curator**: Can create and update biological entities -- **broker**: Can claim and report biological entities -- **genome_launcher**: Can access data required by genome assembly & post-assembly pipelines and report results -- **admin**: Full access to all endpoints -- **superuser**: Special role with delete permissions +```bash +uv sync --dev --frozen +``` -## Development (run without Docker) +2. Export the required environment variables. - Prefer Docker for a consistent setup. This section shows how to run the FastAPI app directly on your machine as an alternative. +3. Apply migrations: - 1) Sync the environment with uv (creates `.venv` in the project root): +```bash +uv run alembic upgrade head +``` - ```bash - uv sync --dev - # Optional: source .venv/bin/activate - ``` +4. Start the API: - 2) Configure environment +```bash +uv run uvicorn app.main:app --host 0.0.0.0 --port 8000 --reload +``` - - Copy `.env.example` to `.env` and fill the required values (particularly `JWT_SECRET_KEY`, `JWT_ALGORITHM`). - - Choose ONE of the database options below: +## Database And Migrations - Option A — reuse the Docker Postgres (recommended) +- SQLAlchemy session setup is in `app/db/session.py`. +- Alembic config is in `alembic/env.py`. +- Active revisions are in `alembic/versions/`. +- `scripts/entrypoint.sh` runs `uv run alembic upgrade head` before starting the app. +- `schema.sql` is maintained as a schema snapshot, not the runtime bootstrap mechanism. - ```bash - # Start only the database service - docker compose up -d db +For the repo-local migration workflow, see [docs/migration_workflow.md](docs/migration_workflow.md). - # In your .env (or environment), point to the compose database on host 5433 - POSTGRES_SERVER=localhost - POSTGRES_PORT=5433 - POSTGRES_USER=postgres - POSTGRES_PASSWORD= - POSTGRES_DB=atol_db - ``` +## Authentication - Option B — use a local Postgres (no Docker) +The current authentication surface is: - ```bash - # Ensure Postgres is running locally (port 5432) - # Create the database (migrations will handle schema) - createdb -h localhost -U postgres atol_db || true +- `POST /api/v1/auth/login` +- `POST /api/v1/auth/refresh` +- `POST /api/v1/auth/logout` - # In your .env (or environment), point to your local instance - POSTGRES_SERVER=localhost - POSTGRES_PORT=5432 - POSTGRES_USER=postgres - POSTGRES_PASSWORD= - POSTGRES_DB=atol_db - ``` +Access tokens are JWTs. Refresh tokens are stored as hashed values in the `refresh_token` table. - 3) Run the application +Detailed auth behavior is in [docs/auth_refresh_tokens.md](docs/auth_refresh_tokens.md). - ```bash - uv run alembic upgrade head - uv run uvicorn app.main:app --reload --host 0.0.0.0 --port 8000 - ``` +## Useful Entry Points - 4) Open the docs at http://localhost:8000/api/v1/docs +### API docs and health - ### Linting +- OpenAPI JSON: `/api/v1/openapi.json` +- Swagger UI: `/api/v1/docs` +- ReDoc: `/api/v1/redoc` +- Health: `/health` +- Version: `/version` - - Install hooks: `uv run pre-commit install` - - Run all checks: `uv run pre-commit run --all-files` - - Install commit message hook: `uv run pre-commit install --hook-type commit-msg` (requires commitlint hook configured) +### Scripts - ### Database migrations (Alembic) +- `scripts/entrypoint.sh`: wait for DB, run migrations, start app +- `scripts/create_user.py`: create a user directly in the database +- `scripts/expire_leases.py`: expire broker leases directly from Python - - Apply migrations: `uv run alembic upgrade head` - - Create a new revision: `uv run alembic revision --autogenerate -m "description"` - - Docker Compose runs migrations automatically on container start via `scripts/entrypoint.sh`. - - `scripts/entrypoint.sh` supports `APP_MODE=serve` (default) and `APP_MODE=migrate` (run migrations then exit). +## Testing - ### Environment configuration +The repository contains unit tests under `tests/`. -Configuration is provided via environment variables loaded from `.env` (see `.env.example`). Key settings: +Run them with: -- Database - - `POSTGRES_SERVER` (Docker service name is `db`) - - `POSTGRES_USER` - - `POSTGRES_PASSWORD` - - `POSTGRES_DB` - - `POSTGRES_PORT` (container port, default 5432; host is `5433` via compose) -- Auth - - `JWT_SECRET_KEY` (required) - - `JWT_ALGORITHM` (e.g. `HS256`) - - `JWT_ACCESS_TOKEN_EXPIRE_MINUTES` (default 150) - - `JWT_REFRESH_TOKEN_EXPIRE_DAYS` (default 7) -- CORS - - `BACKEND_CORS_ORIGINS` (JSON array of origins) +```bash +pytest -q +``` -When using Docker, the database is initialized automatically on first run using `schema.sql` via the Postgres container's init hook. +## Known Limits Of This Repository -## License +The repository does not by itself tell us: -This project is licensed under the GPL-3.0-or-later License. +- which broker API surface is used in production +- who or what promotes submission rows from `draft` to `ready` +- the full production runtime topology behind the GitHub Actions deployment workflows +- any external runbooks or operational conventions outside version control diff --git a/app/api/v1/endpoints/broker.py b/app/api/v1/endpoints/broker.py index aeba69c..f2ffb61 100644 --- a/app/api/v1/endpoints/broker.py +++ b/app/api/v1/endpoints/broker.py @@ -1107,11 +1107,11 @@ def report_submission_outcomes( # ---------- Endpoints ---------- -# NOTE: More specific routes must come before parameterized routes in FastAPI -# Otherwise /claim would match /organisms/{taxon_id}/claim with taxon_id="claim" +# NOTE: More specific routes must come before parameterized routes in FastAPI. +# Legacy broker handlers are retained below for reference/unit coverage, but only +# the currently supported routes keep @router decorators. -@router.post("/claim", response_model=ClaimByEntityResponse) @policy("broker:claim") def claim_by_entity_ids( *, @@ -1548,7 +1548,6 @@ def claim_by_entity_ids( ) -@router.post("/organisms/{taxon_id}/claim", response_model=ClaimResponse) @policy("broker:claim") def claim_drafts_for_organism( *, @@ -1999,7 +1998,6 @@ def expire_leases( } -@router.post("/attempts/{attempt_id}/lease/renew") @policy("broker:claim") def renew_attempt_lease( *, @@ -2162,7 +2160,6 @@ def finalise_attempt( return {"attempt_id": str(attempt_id), "released": released, "status": attempt.status} -@router.post("/attempts/{attempt_id}/report", response_model=ReportResult) @policy("broker:claim") def report_results( *, @@ -3045,7 +3042,6 @@ def _get_attempt_items_with_relationships( } -@router.get("/attempts") @policy("broker:read") def list_attempts( *, @@ -3085,7 +3081,6 @@ def list_attempts( return {"items": results, "page": page, "page_size": page_size, "total": total} -@router.get("/attempts/{attempt_id}") @policy("broker:read") def get_attempt( *, @@ -3116,7 +3111,6 @@ def get_attempt( return result -@router.get("/attempts/{attempt_id}/items") @policy("broker:read") def get_attempt_items( *, @@ -3128,7 +3122,6 @@ def get_attempt_items( return {k: [e.model_dump() for e in v] for k, v in items.items()} -@router.get("/organisms/{taxon_id}/summary") @policy("broker:read") def organism_summary( *, diff --git a/app/core/settings.py b/app/core/settings.py index c878e50..9f7143f 100644 --- a/app/core/settings.py +++ b/app/core/settings.py @@ -25,7 +25,7 @@ class Settings(BaseSettings): # Security JWT_SECRET_KEY: Optional[str] = None JWT_ALGORITHM: Optional[str] = None - JWT_ACCESS_TOKEN_EXPIRE_MINUTES: int = 150 # 150 minutes TODO change to 15 min for prod + JWT_ACCESS_TOKEN_EXPIRE_MINUTES: int = 15 # 15 minutes JWT_REFRESH_TOKEN_EXPIRE_DAYS: int = 7 # 7 days DATABASE_URI: Optional[str] = None diff --git a/docs/auth_refresh_tokens.md b/docs/auth_refresh_tokens.md index ef6e9ee..58f3dd9 100644 --- a/docs/auth_refresh_tokens.md +++ b/docs/auth_refresh_tokens.md @@ -1,53 +1,66 @@ -# Authentication with Refresh Tokens +# Authentication And Refresh Tokens -This document explains how the authentication system works in the ATOL Canopy application with the new refresh token implementation. +This document describes the authentication behavior that is implemented in the current codebase. -## Overview +## Verified -The authentication system uses a stateful refresh token approach with JWT (JSON Web Tokens): +### Endpoints -1. **Access Tokens**: Short-lived JWTs (15 minutes) used for API authorization -2. **Refresh Tokens**: Longer-lived tokens (7 days) stored in the database and used to obtain new access tokens +| Method | Endpoint | Purpose | +| --- | --- | --- | +| `POST` | `/api/v1/auth/login` | Exchange username and password for an access token and refresh token | +| `POST` | `/api/v1/auth/refresh` | Rotate a valid refresh token and issue a new access token and refresh token | +| `POST` | `/api/v1/auth/logout` | Revoke all active refresh tokens for the authenticated user | -## Authentication Flow +### Access tokens -### 1. Login +- Access tokens are JWTs created by `create_access_token()` in `app/core/security.py`. +- The token payload contains: + - `sub`: the user ID as a string + - `exp`: the token expiry timestamp +- Protected endpoints resolve the current user by decoding the JWT in `app/core/dependencies.py`. -When a user logs in with valid credentials, the system: +### Refresh tokens -- Creates a short-lived access token (JWT) -- Generates a refresh token and stores it in the database -- Returns both tokens to the client +- Refresh tokens are random opaque strings generated by `generate_refresh_token()` in `app/core/security.py`. +- Only the SHA-256 hash of the refresh token is stored in the database table `refresh_token` (`app/models/token.py`). +- The refresh flow performs token rotation: + - the old refresh token row is marked `revoked = true` + - a new refresh token is generated + - a new `refresh_token` row is inserted -```http -POST /api/v1/auth/login -Content-Type: application/x-www-form-urlencoded +### Logout behavior -username=your_username&password=your_password -``` +- `POST /api/v1/auth/logout` revokes all non-revoked refresh tokens belonging to the current user. +- Logout does not invalidate already-issued JWT access tokens directly. The code only revokes refresh-token rows. -Response: +### Expiry configuration -```json -{ - "access_token": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9...", - "refresh_token": "random_secure_string", - "token_type": "bearer" -} -``` +- Access-token expiry is controlled by `JWT_ACCESS_TOKEN_EXPIRE_MINUTES`. +- Refresh-token expiry is controlled by `JWT_REFRESH_TOKEN_EXPIRE_DAYS`. +- Current code defaults: + - `JWT_ACCESS_TOKEN_EXPIRE_MINUTES = 150` + - `JWT_REFRESH_TOKEN_EXPIRE_DAYS = 7` -### 2. Using Access Tokens +### Failure modes verified in code -Include the access token in the `Authorization` header for protected API requests: +- Invalid username or password at login returns `401`. +- Inactive user at login returns `400`. +- Invalid, expired, revoked, deleted-user, or inactive-user refresh attempts return `401`. +- Invalid bearer token or missing `sub` claim returns `401`. + +## Example Requests + +### Login ```http -GET /api/v1/protected-endpoint -Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9... -``` +POST /api/v1/auth/login +Content-Type: application/x-www-form-urlencoded -### 3. Refreshing Tokens +username=your_username&password=your_password +``` -When the access token expires, use the refresh token to get a new pair of tokens: +### Refresh ```http POST /api/v1/auth/refresh @@ -58,53 +71,14 @@ Content-Type: application/json } ``` -Response: - -```json -{ - "access_token": "new_access_token", - "refresh_token": "new_refresh_token", - "token_type": "bearer" -} -``` - -**Note**: The system implements token rotation - each time you refresh, you get a new refresh token and the old one is invalidated. - -### 4. Logout - -To securely logout, revoke all refresh tokens: +### Logout ```http POST /api/v1/auth/logout -Authorization: Bearer your_access_token +Authorization: Bearer ``` -Response: - -```json -{ - "message": "Successfully logged out" -} -``` - -## Security Features - -1. **Token Rotation**: Each refresh operation invalidates the used refresh token and issues a new one -2. **Database Storage**: Refresh tokens are stored as secure hashes, not plaintext -3. **Automatic Expiration**: Both access and refresh tokens have expiration times -4. **Revocation**: Refresh tokens can be revoked by the user or the system -5. **Cascading Deletion**: When a user is deleted, all their refresh tokens are automatically removed - -## Implementation Details - -- Access tokens are stateless JWTs signed with a secret key -- Refresh tokens are securely generated random strings -- Refresh tokens are stored in the `refresh_tokens` table with a hash of the actual token -- Token validation includes checking expiration and revocation status - -## Best Practices for Client Applications +## Unknown From This Repo -1. Store the refresh token securely (e.g., in an HTTP-only cookie or secure storage) -2. Store the access token in memory (not localStorage) to prevent XSS attacks -3. Implement automatic token refresh when access tokens expire -4. Always call the logout endpoint when the user logs out +- There is no verified client-side guidance in this repo for where browser or desktop clients should store tokens. +- There is no server-side access-token denylist or forced JWT invalidation mechanism beyond normal expiry. diff --git a/docs/bulk_import_api.md b/docs/bulk_import_api.md index 7089624..2f27b98 100644 --- a/docs/bulk_import_api.md +++ b/docs/bulk_import_api.md @@ -1,159 +1,117 @@ -# Bulk Import API Documentation +# Bulk Import Workflows -This document describes how to use the bulk import API endpoints to import organisms, samples, and experiments into the database. +This document describes the bulk-import endpoints that are active in the current codebase. -## Authentication +## Verified -All bulk import endpoints require authentication and the user must have either the 'curator' or 'admin' role. +### Active bulk-import endpoints -## Endpoints Overview +| Method | Endpoint | Expected top-level payload shape | Notes | +| --- | --- | --- | --- | +| `POST` | `/api/v1/organisms/bulk-import` | raw dictionary keyed by `taxon_id` | No wrapper key such as `organisms` | +| `POST` | `/api/v1/samples/bulk-import` | raw dictionary keyed by `bpa_sample_id` | Mixed sample import path; defaults to specimen unless `kind` supplied | +| `POST` | `/api/v1/samples/bulk-import-specimens` | raw dictionary keyed by `taxon_id`, then `specimen_id` | Explicit specimen-only import | +| `POST` | `/api/v1/samples/bulk-import-derived` | raw dictionary keyed by sample key | Explicit derived-only import | +| `POST` | `/api/v1/experiments/bulk-import` | raw dictionary keyed by package ID | Can also create `read` rows from nested `runs` lists | +| `POST` | `/api/v1/taxonomy-info/bulk-import` | raw dictionary keyed by `taxon_id` | Insert-oriented, with NCBI enrichment | +| `POST` | `/api/v1/taxonomy-info/bulk-upsert` | raw dictionary keyed by `taxon_id` | Insert or update, with NCBI enrichment | +| `POST` | `/api/v1/taxonomy-info/bulk-ncbi-refresh` | object with `taxon_ids` list | Refresh existing taxonomy-info rows only | -| Endpoint | Method | Description | -|----------|--------|-------------| -| `/api/v1/organisms/bulk-import` | POST | Bulk import organisms | -| `/api/v1/samples/bulk-import` | POST | Bulk import samples | -| `/api/v1/experiments/bulk-import` | POST | Bulk import experiments | +### Authentication and roles -## Request Format +- Organism bulk import requires `organisms:bulk_import`. +- Sample bulk import endpoints require `samples:bulk_import`. +- Experiment bulk import requires `experiments:bulk_import`. +- Taxonomy bulk actions require the matching taxonomy policies defined in `app/core/policy.py`. -### Bulk Import Organisms +### Recommended import order -**Endpoint:** `/api/v1/organisms/bulk-import` +1. Organisms +2. Specimen samples +3. Derived samples +4. Experiments +5. Assembly intents or broker-driven submission workflows +6. Taxonomy-info import or refresh when NCBI enrichment is needed -**Request Body:** -```json -{ - "organisms": { - "123456": { - "taxon_id": 123456, - "scientific_name": "Organism scientific name", - "other_field": "value", - ... - }, - "789012": { - ... - } - } -} -``` +This ordering is verified from the code-level dependencies: -The request body should match the format of the JSON file in `data/unique_organisms.json`. Each organism is keyed by `taxon_id` and must contain at least `taxon_id` and `scientific_name`. +- samples require existing organisms +- experiments require existing samples and the organism’s `genomic_data` project +- taxonomy info requires existing organisms -**Response:** -```json -{ - "created_count": 10, - "skipped_count": 2, - "message": "Organism import complete. Created: 10, Skipped: 2" -} -``` +## Endpoint-specific behavior -### Bulk Import Samples +### `POST /api/v1/organisms/bulk-import` -**Endpoint:** `/api/v1/samples/bulk-import` +- Creates new `organism` rows. +- Also creates the paired `root` and `genomic_data` `project` rows plus draft `project_submission` rows for each organism. +- Skips rows when the organism already exists. +- Accepts both current `bpa_*` fields and legacy unprefixed fields for several organism attributes. + +Example payload: -**Request Body:** ```json { - "samples": { - "bpa_sample_id_1": { - "taxon_id": 123456, - "other_field": "value", - ... - }, - "bpa_sample_id_2": { - ... - } + "172942": { + "taxon_id": 172942, + "bpa_scientific_name": "Example species" } } ``` -The request body should match the format of the JSON file in `data/unique_samples.json`. Each sample is identified by its `bpa_sample_id` and can include `taxon_id` to link it to an organism. - -**Response:** -```json -{ - "created_count": 15, - "skipped_count": 3, - "message": "Sample import complete. Created samples: 15, Created submission records: 15, Skipped: 3" -} -``` +### `POST /api/v1/samples/bulk-import-specimens` -### Bulk Import Experiments +- Expects nested keys: outer `taxon_id`, inner `specimen_id`. +- Enforces one specimen sample per `(taxon_id, specimen_id)`. +- Forces `organism_part` to `"WHOLE ORGANISM"` in this explicit specimen import path. -**Endpoint:** `/api/v1/experiments/bulk-import` +Example payload: -**Request Body:** ```json { - "experiments": { - "bpa_package_id_1": { - "bpa_sample_id": "bpa_sample_id_1", - "other_field": "value", - ... - }, - "bpa_package_id_2": { - ... + "172942": { + "SPEC-001": { + "bpa_sample_id": "BPA-SPEC-001", + "lifestage": "adult" } } } ``` -The request body should match the format of the JSON file in `data/experiments.json`. Each experiment is identified by its `bpa_package_id` and must contain a `bpa_sample_id` to link it to a sample. +### `POST /api/v1/samples/bulk-import-derived` -**Response:** -```json -{ - "created_count": 20, - "skipped_count": 5, - "message": "Experiment import complete. Created experiments: 20, Created submission records: 20, Skipped: 5" -} -``` - -## Import Order - -When importing data, follow this order to ensure proper relationships: +- Requires `bpa_sample_id`. +- Requires `taxon_id`. +- Requires `specimen_id` so the code can find the parent specimen sample. +- Parent lookup is by `(taxon_id, specimen_id)` with `kind = specimen`. -1. Import organisms first -2. Import samples second (they may reference organisms) -3. Import experiments last (they reference samples) +### `POST /api/v1/samples/bulk-import` -## Error Handling +- Accepts a flatter legacy/importer-friendly path keyed by `bpa_sample_id`. +- Defaults missing required text fields such as `lifestage`, `sex`, and `habitat` to `"unknown"` in code. +- Defaults sample kind to `specimen` unless `kind` is supplied. -- If an entity already exists (by its unique key), it will be skipped -- If required fields are missing, the entity will be skipped -- If referenced entities don't exist (e.g., a sample references a non-existent organism), the entity will still be created but without the relationship +### `POST /api/v1/experiments/bulk-import` -## Example Usage with curl +- Requires `bpa_sample_id` in each experiment payload so the sample can be resolved. +- Requires `bpa_library_id`. +- Creates `experiment` and `experiment_submission` rows. +- If a `runs` list exists, the importer also creates `read` rows. +- If the experiment already exists, the importer still attempts to create any missing reads from the nested `runs` list. -### Import Organisms -```bash -curl -X POST "http://localhost:8000/api/v1/organisms/bulk-import" \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer YOUR_TOKEN" \ - -d @data/unique_organisms.json -``` +### Taxonomy bulk operations -### Import Samples -```bash -curl -X POST "http://localhost:8000/api/v1/samples/bulk-import" \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer YOUR_TOKEN" \ - -d @data/unique_samples.json -``` +- `bulk-import` skips already-fully-synced rows and only creates new rows when NCBI enrichment returns mapped data. +- `bulk-upsert` can insert or update rows and re-fetches NCBI data for all supplied taxon IDs. +- `bulk-ncbi-refresh` updates only existing `taxonomy_info` rows. -### Import Experiments -```bash -curl -X POST "http://localhost:8000/api/v1/experiments/bulk-import" \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer YOUR_TOKEN" \ - -d @data/experiments.json -``` +## Important constraints and sharp edges -## Converting Standalone Script Data to API Format +- The organism, sample, and experiment bulk endpoints do not share one uniform payload shape. +- Several bulk paths still contain compatibility logic for older field names or older caller behavior. +- Some bulk paths emit partial diagnostics through response `errors`; some older code paths also still use `print(...)` for local debugging. -If you have data in a different format, you may need to transform it to match the expected format for these endpoints. The format should match the JSON files used by the standalone import script: +## Unknown From This Repo -- `data/unique_organisms.json` -- `data/unique_samples.json` -- `data/experiments.json` +- There are no canonical example input files in the tracked repository proving the exact upstream export format for every current caller. +- The repo does not document who owns the promotion from imported `draft` submission rows to `ready`. diff --git a/docs/handover/README.md b/docs/handover/README.md new file mode 100644 index 0000000..c03c694 --- /dev/null +++ b/docs/handover/README.md @@ -0,0 +1,39 @@ +# Canopy Handover Pack + +This directory contains hander documentation for future maintainers of the Canopy codebase + +## What is covered + +The docs cover: + +- what the system currently does +- where major workflows live in the code +- how local setup and migrations work +- how broker and ToLID flows behave +- deployment to AWS (basic details only - this is largely managed by the BioCloud team) +- manual overrides when things don't go to plan +- where the known gaps and human-owned unknowns still are + +This pack is intentionally maintainer-oriented. It favors verified behavior and operator tasks over broad narrative description. + +## Recommended Reading Order + +1. [system_overview.md](/Users/emilylm/Repositories/atol-database-v2/docs/handover/system_overview.md) +2. [setup_and_operations.md](/Users/emilylm/Repositories/atol-database-v2/docs/handover/setup_and_operations.md) +3. [broker_and_submission_flows.md](/Users/emilylm/Repositories/atol-database-v2/docs/handover/broker_and_submission_flows.md) +4. [config_reference.md](/Users/emilylm/Repositories/atol-database-v2/docs/handover/config_reference.md) +5. [known_issues.md](/Users/emilylm/Repositories/atol-database-v2/docs/handover/known_issues.md) +6. [troubleshooting.md](/Users/emilylm/Repositories/atol-database-v2/docs/handover/troubleshooting.md) +7. [open_questions.md](/Users/emilylm/Repositories/atol-database-v2/docs/handover/open_questions.md) + +## Document Map + +| File | What it covers | When to use it | +| --- | --- | --- | +| [system_overview.md](/Users/emilylm/Repositories/atol-database-v2/docs/handover/system_overview.md) | Repo structure, core entities, main lifecycle patterns, and external interaction points | Use this first when orienting a new maintainer | +| [setup_and_operations.md](/Users/emilylm/Repositories/atol-database-v2/docs/handover/setup_and_operations.md) | Docker startup path, local non-Docker workflow, migrations, scripts, and recurring operator actions | Use this for local setup and day-to-day maintenance | +| [broker_and_submission_flows.md](/Users/emilylm/Repositories/atol-database-v2/docs/handover/broker_and_submission_flows.md) | Submission states, broker claim/report behavior, ToLID flow, and attempt leasing | Use this when debugging broker-facing workflows | +| [config_reference.md](/Users/emilylm/Repositories/atol-database-v2/docs/handover/config_reference.md) | Environment variables and workflow-level config surfaced by the repo | Use this when wiring environments or checking startup failures | +| [known_issues.md](/Users/emilylm/Repositories/atol-database-v2/docs/handover/known_issues.md) | Current code issues, operational gaps, and external dependencies | Use this when planning fixes, prioritising debt, or handing work to a new maintainer | +| [troubleshooting.md](/Users/emilylm/Repositories/atol-database-v2/docs/handover/troubleshooting.md) | Symptom-driven maintainer guide | Use this during incidents or confusing behavior | +| [open_questions.md](/Users/emilylm/Repositories/atol-database-v2/docs/handover/open_questions.md) | Gaps that require human knowledge outside the repo | Use this when deciding what tribal knowledge still needs to be captured | diff --git a/docs/handover/broker_and_submission_flows.md b/docs/handover/broker_and_submission_flows.md new file mode 100644 index 0000000..6e48bb8 --- /dev/null +++ b/docs/handover/broker_and_submission_flows.md @@ -0,0 +1,71 @@ +# Broker And Submission Flows + +## Verified + +### Active broker API surfaces + +The repo currently exposes these broker submission endpoints: + +- `POST /api/v1/broker/claims/ready` +- `POST /api/v1/broker/claims/entity` +- `POST /api/v1/broker/claims/batch` +- `POST /api/v1/broker/validation` +- `POST /api/v1/broker/reports/{attempt_id}` +- `POST /api/v1/broker/attempts/{attempt_id}/finalise` + +The claim, validation, and report endpoints above use a shared broker schema from `app/schemas/broker_contract.py`. +Claims return a single `entities` list where each item includes its own `type`, `id`, payload, prerequisites, and file metadata. + +### Legacy handler status + +- The module still contains older claim/report helper functions such as `claim_by_entity_ids()`, `claim_drafts_for_organism()`, `renew_attempt_lease()`, `report_results()`, `list_attempts()`, `get_attempt()`, `get_attempt_items()`, and `organism_summary()`. +- These are out-dated. I've removed the routes that were exposing these functions, but I've kept the functions for the moment (didn't have time to remove). Remove in future! + +### Submission states and attempt leasing + +- `SubmissionAttempt` rows track claim attempts and lease expiry (`app/models/broker.py`). +- The current broker claim endpoints can claim submissions in either `draft` or `ready` because `CLAIMABLE_SUBMISSION_STATES = ("draft", "ready")` in `app/api/v1/endpoints/broker.py`. +- Claimed rows move to `submitting`, receive `attempt_id`, `lock_acquired_at`, and `lock_expires_at`, and emit `submission_event` rows with `action="claimed"`. +- Finalising an attempt releases remaining `submitting` rows back to `draft` and marks the attempt `complete`. +- Expiring stale leases resets expired `submitting` rows back to `draft` and marks expired attempts as `expired`. + +### Broker report behavior + +- `POST /api/v1/broker/reports/{attempt_id}` updates the entities claimed under one broker attempt. +- `_map_report_state_to_submission_status()` maps: + - `completed`, `accepted`, `success` -> `accepted` + - `failed`, `rejected`, `error` -> `rejected` + - `submitting`, `processing` -> `submitting` +- Accepted reports write `accession` to the submission row and attempt to insert into `accession_registry`. +- Sample reports can also store `secondary_accession` into sample.`biosample_accession`. +- Rejected reports create a new draft submission row for the same entity so the entity becomes claimable again. +- A database integrity conflict during accession registration is translated into HTTP `409` with a message pointing maintainers to `accession_registry`. + +### Submission prerequisites and validation + +- The current broker validation and claim code resolves existing prerequisite accessions from `accession_registry`, not only from payload fields (`_extract_broker_prerequisites()` in `app/api/v1/endpoints/broker.py`). +- Validation rules currently enforced by `POST /api/v1/broker/validation` are: + - sample: payload must exist + - experiment: payload, `sample_accession`, and `project_accession` must exist + - run: payload, `experiment_accession`, and run file metadata must exist +- Project claims do not include prerequisites. + +### ToLID flow + +- The ToLID flow does not use broker lease semantics. +- `GET /api/v1/broker/tolids/by-specimen-accession/{specimen_id}` resolves a specimen sample from an accepted sample accession and returns either: + - stored `tolid_request` state + - virtual `not_requested` state if no row exists yet +- `POST /api/v1/broker/tolids/{sample_id}/report` lazily creates a `tolid_request` row when needed. +- Allowed reported statuses are `pending`, `assigned`, and `failed`; `not_requested` is rejected by schema validation (`app/schemas/tolid.py`). +- Reporting `assigned` also mirrors the ToLID value onto `sample.tolid`. + +Note: while Canopy stores ToLIDs, and keeps track of ToLID requests, the action of retrying tolids in "pending" state needs to be handled seperately. + +### Submission lifecycle rules outside broker.py + +- Organism creation creates draft `project_submission` rows immediately. +- Sample creation creates one draft `sample_submission`. +- Experiment creation creates one draft `experiment_submission`. +- Assembly QC reporting creates one draft `qc_read_submission`. +- Sample and experiment edits can create replacement draft submission rows as described in `system_overview.md`. diff --git a/docs/handover/config_reference.md b/docs/handover/config_reference.md new file mode 100644 index 0000000..f74cea4 --- /dev/null +++ b/docs/handover/config_reference.md @@ -0,0 +1,62 @@ +# Config Reference + +## Verified + +### Application and entrypoint settings + +| Name | Source | Required | Verified behavior | +| --- | --- | --- | --- | +| `PROJECT_NAME` | `app/core/settings.py` | no | FastAPI title; default `atol-canopy` | +| `API_V1_STR` | `app/core/settings.py` | no | API prefix and docs prefix; default `/api/v1` | +| `DEBUG` | `app/core/settings.py` | no | Present in settings model; no direct behavior found outside settings object | +| `ENVIRONMENT` | `app/core/settings.py`, `app/main.py`, `scripts/entrypoint.sh` | effectively yes for predictable behavior | `dev` enables debug logging and `uvicorn --reload`; non-`prod` with no explicit CORS origins defaults to `["*"]` | +| `APP_VERSION` | `app/core/settings.py`, `Dockerfile`, `app/main.py` | no | Returned by `GET /version`; default `dev`; Docker build can inject a release value | +| `PORT` | `scripts/entrypoint.sh`, `Dockerfile` | no | Runtime listen port for the entrypoint; default `8000` | +| `APP_MODE` | `scripts/entrypoint.sh` | no | `serve` starts API; `migrate` runs migrations then exits | + +### Database settings + +| Name | Source | Required | Verified behavior | +| --- | --- | --- | --- | +| `DATABASE_URI` | `app/core/settings.py`, `scripts/entrypoint.sh`, `alembic/env.py` | yes for entrypoint; yes or derivable for app import | Used by SQLAlchemy engine and Alembic; entrypoint exits if it is unset | +| `POSTGRES_SERVER` | `app/core/settings.py` | required if deriving `DATABASE_URI` | Used only to derive `DATABASE_URI` inside settings | +| `POSTGRES_USER` | `app/core/settings.py` | required if deriving `DATABASE_URI` | Used only to derive `DATABASE_URI` inside settings | +| `POSTGRES_PASSWORD` | `app/core/settings.py` | required if deriving `DATABASE_URI` | Used only to derive `DATABASE_URI` inside settings | +| `POSTGRES_DB` | `app/core/settings.py` | required if deriving `DATABASE_URI` | Used only to derive `DATABASE_URI` inside settings | +| `POSTGRES_PORT` | `app/core/settings.py` | required if deriving `DATABASE_URI` | Used only to derive `DATABASE_URI` inside settings | + +### Auth and token settings + +| Name | Source | Required | Verified behavior | +| --- | --- | --- | --- | +| `JWT_SECRET_KEY` | `app/core/settings.py`, `app/core/security.py`, `app/core/dependencies.py` | yes | Required at startup; used to sign and verify access tokens | +| `JWT_ALGORITHM` | `app/core/settings.py`, `app/core/security.py`, `app/core/dependencies.py` | yes | Required at startup; used for JWT encode/decode | +| `JWT_ACCESS_TOKEN_EXPIRE_MINUTES` | `app/core/settings.py`, `app/api/v1/endpoints/auth.py` | no | Access-token lifetime in minutes; default `150` | +| `JWT_REFRESH_TOKEN_EXPIRE_DAYS` | `app/core/settings.py`, `app/api/v1/endpoints/auth.py` | no | Refresh-token lifetime in days; default `7` | + +### CORS settings + +| Name | Source | Required | Verified behavior | +| --- | --- | --- | --- | +| `BACKEND_CORS_ORIGINS` | `app/core/settings.py`, `app/main.py` | no | If set, FastAPI installs CORS middleware with those origins; if unset and `ENVIRONMENT != "prod"`, settings default it to `["*"]`; `["*"]` is rejected when `ENVIRONMENT == "prod"` | + +### Repo automation secrets and workflow vars + +These are not application settings, but they are verified in GitHub workflow files: + +| Name | Source | Verified behavior | +| --- | --- | --- | +| `AWS_ROLE_ECR_PUSH` | `.github/workflows/build-and-deploy-dev.yml`, `build-publish.yml` | GitHub Actions assumes this AWS role before pushing images | +| `GITHUB_TOKEN` | `.github/workflows/release-please.yml` | Used by release-please | +| `AWS_REGION` | workflow env | Set to `ap-southeast-2` in the repo workflows | +| `IMAGE_REPO` | workflow env | ECR repository used by build jobs | +| `DEPLOY_FUNCTION_NAME` | workflow env | Lambda function invoked by the dev deployment workflow | + +## Inferences + +- `BACKEND_CORS_ORIGINS` is expected to be supplied in a Pydantic-compatible list form such as a JSON array string, because it is typed as `List[str]` in `Settings`. + +## Unknown From This Repo + +- The real production values for any of these settings. +- Whether there are additional deployment-time environment variables injected outside the repository. diff --git a/docs/handover/known_issues.md b/docs/handover/known_issues.md new file mode 100644 index 0000000..2e5993b --- /dev/null +++ b/docs/handover/known_issues.md @@ -0,0 +1,61 @@ +# Known Issues And Outstanding Work + +## TO DO: Canopy & Broker + +- Persist broker logs and broker error responses so they can be reviewed later. + +- Persist BPA import logs, mapper outputs, and bulk-import failures. Bulk import endpoints return error lists, but import history is not stored. (side note, some sample import paths still use `print(...)` rather than structured logging e.g. in [samples.py](/Users/emilylm/Repositories/atol-database-v2/app/api/v1/endpoints/samples.py)) + +- Decide whether `bpa_package_id` should stay in the request body for `POST /api/v1/assemblies/{assembly_id}/qc-reads/report`. It is currently a required body field in [qc_read.py](/Users/emilylm/Repositories/atol-database-v2/app/schemas/qc_read.py) and is resolved in [assemblies.py](/Users/emilylm/Repositories/atol-database-v2/app/api/v1/endpoints/assemblies.py). From PO feedback, `bpa_package_id` should be moved to a path variable. + +- Broker support for parent and child projects is missing. There is no project hierarchy model or broker payload support in [project.py](/Users/emilylm/Repositories/atol-database-v2/app/models/project.py) or [broker.py](/Users/emilylm/Repositories/atol-database-v2/app/api/v1/endpoints/broker.py). We were awaiting confirmation from ENA about how to register and confirm parent-child project relos. Need to confirm this and implement the heirarchy. + +- Assembly stage reporting is one-shot per `(assembly_run_id, stage_name)`. A second `POST` for the same stage returns a conflict. `PATCH` replaces the file list; it does not append. See [assembly_service.py](/Users/emilylm/Repositories/atol-database-v2/app/services/assembly_service.py). This is likely undesirable - may want to update so that we can add more result files and/or metadata for an assembly stage we have already reported results for. + +- `POST /api/v1/assemblies/{assembly_id}/qc-reads/report` always creates a new `QcRead`, new files, and a new draft `QcReadSubmission`. There is no deduplication, uniqueness check, or delete path inside that workflow. See [assemblies.py](/Users/emilylm/Repositories/atol-database-v2/app/api/v1/endpoints/assemblies.py). May need to change this behaviour if we want more control. + +- ToLID external-id selection should be reviewed. [tolid_service.py](/Users/emilylm/Repositories/atol-database-v2/app/services/tolid_service.py) currently uses `SampleSubmission.accession` and only falls back to `sample.biosample_accession`. We actually want to use the external / BioSample accession as the `external_id` for the ToLIDs we register -> so we need to change this (may require changes to the broker) + +- Project titles are not validated against any ENA minimum length in Canopy. Recent submission attempts (using the broker) have revealed that ENA requires titles of at least 20 characters. That check should be added in [project.py](/Users/emilylm/Repositories/atol-database-v2/app/schemas/project.py) or before broker submission. May need to pad `title` field when char length is too short. + +- The current `qc-reads/report` payload is inconsistent with the genome launcher. The current Canopy request shape is defined in [qc_read.py](/Users/emilylm/Repositories/atol-database-v2/app/schemas/qc_read.py). The genome launcher has different fields for qc_read files. Need to change fields in Canopy, or in the genome launcher - or a shim could be added. + +- Annotationa are not implemented. + +- Genome notes are largely not implemented. There exists separate CRUD and rudimentary publish/unpublish feature in [genome_notes.py](/Users/emilylm/Repositories/atol-database-v2/app/api/v1/endpoints/genome_notes.py) and [genome_note.py](/Users/emilylm/Repositories/atol-database-v2/app/models/genome_note.py), but this hasn't been tested or properly implemented. We need to implement endpoints that 1) return the metadata required for a genome note and 2) allow reporting/storing of a published genome note back into the db (might be the actual genome note data or just a DOI/link). + +- Ensure all API endpoints have explicit `@policy(...)` protection. Several authenticated endpoints still rely only on authentication and have no policy decorator. + +- Remove the hard-coded repository path from [create_user.py](/Users/emilylm/Repositories/atol-database-v2/scripts/create_user.py). All script sin `scripts/` folder can probably be removed and managed elsewhere. + +## TODO: Operations + +- Set up a scheduler to import data from BPA data portal. + +- Set up a scheduler for retrying or polling `pending` ToLID requests. = + +- Define log collection and log retention for deployed environments. Logs are available in AWS CloudWatch. + +- Set up scheduled database backup, dump, and restore procedures. + +- When prod environment created, dev db should be refreshed from prod db snapshots regularly.. Something to discuss with BioCloud team (will need to decide on & implement a DR protocol) + +## Data And Access Management + +- Backfill `experiment.bioplatforms_base_url` where it is missing in existing environments (missing in dev AWS environment currently - because it is read from BPA data which had already been imported). + +- Patch manually assigned ToLIDs for the benchmarking datasets if those records must exist in Canopy. (can use the `/tolid/report` endpoint) + +- New roles should be created for 1) each service user (genome launcher and broker) and 2) admin and/or curator users for AToL team members who need to interact with the database. Credentials should be kept confidential. Supported roles in code include `admin`, `curator`, `broker`, `genome_launcher`, and `superuser`. + +## Open Questions + +- How can ENA parent and child project relationships be verified for private / unreleased projects? + +- How long are deployed Canopy logs retained in AWS? + +- Does the dev environment still need `bioplatforms_base_url` backfill and manual ToLID backfill? + +## Resolved since docs creted + +- Duplicate `assembly_run` registration is now handled. Creating the same `(assembly_id, github_repo, git_commit)` combination returns `409` through [assemblies.py](/Users/emilylm/Repositories/atol-database-v2/app/api/v1/endpoints/assemblies.py) and [assembly_service.py](/Users/emilylm/Repositories/atol-database-v2/app/services/assembly_service.py). diff --git a/docs/handover/setup_and_operations.md b/docs/handover/setup_and_operations.md new file mode 100644 index 0000000..f055059 --- /dev/null +++ b/docs/handover/setup_and_operations.md @@ -0,0 +1,66 @@ +# Local Setup And Operations + +### Container startup path + +- `docker-compose.yml` defines two services: + - `api` on host port `8000` + - `db` on host port `5433`, container port `5432` +- The API container runs `scripts/entrypoint.sh` from `Dockerfile`. +- `scripts/entrypoint.sh` does the following: + 1. requires `DATABASE_URI` to be present in the environment + 2. waits for the database to answer `SELECT 1` + 3. runs `uv run alembic upgrade head` + 4. starts `uvicorn` + 5. enables `--reload` only when `ENVIRONMENT=dev` + +### Local Docker workflow (recommended for local development) + +1. Copy `.env.example` to `.env`. +2. Set at least the JWT variables and Postgres variables needed by the app. +3. Start the stack with `docker compose up --build`. +4. API docs are served from `/api/v1/docs`, `/api/v1/redoc`, and `/api/v1/openapi.json`. + +### Local non-Docker workflow + +1. Install dependencies with `uv sync --dev --frozen` (`Dockerfile`, `.github/workflows/lint.yml`). +2. Export the required environment variables described in `app/core/settings.py`. +3. Run migrations with `uv run alembic upgrade head`. +4. Start the API with `uv run uvicorn app.main:app --host 0.0.0.0 --port 8000 --reload`. + +### First-user creation + +- `scripts/create_user.py` creates a user directly in the database and hashes the password with the same helper used by the API. +- The script accepts either `--db-uri` or host/port/name/user/password fields. +- The script sets `roles=[role]`. +- The script sets `is_superuser=True` only when `--role superuser`. +e.g.: (TODO) + +### Recurring maintainer tasks TODO + +| Task | Verified implementation | +| --- | --- | +| Run tests | `pytest -q` passes in the current repo state | +| Apply schema changes | `uv run alembic upgrade head` | +| Expire stale broker leases from code | `scripts/expire_leases.py` | +| Expire stale broker leases via API | `POST /api/v1/admin/leases/expire` or `POST /api/v1/broker/leases/expire` | +| Check liveness | `GET /health` | +| Check app version | `GET /version` | +| Regenerate schema snapshot | Comment in `schema.sql` says to use `pg_dump --schema-only` after migrations | + +### CI/CD related tasks (added by BioCloud team - don't change without assistance) + +- `.github/workflows/build-and-deploy-dev.yml` builds a container image, pushes it to ECR, and invokes a Lambda deployment function. +- `.github/workflows/build-publish.yml` builds and pushes release-tagged images to ECR. +- These workflows prove that the repo is wired to AWS-hosted automation, but they do not describe the runtime environment behind the deployment target. + +### Some stuff to consider updating (TODO) + +- `scripts/entrypoint.sh` requires a literal `DATABASE_URI` shell variable even though `app/core/settings.py` can derive `DATABASE_URI` from `POSTGRES_*` values. This means `POSTGRES_*` values alone are enough for direct Python startup but not enough for the entrypoint script. +- `scripts/create_user.py` appends a hard-coded repository path to `sys.path`. That makes the script repo-location-sensitive. +- There are two lease-expiry implementations: + - `app/services/broker_service.py` + - `expire_stale_leases()` inside `app/api/v1/endpoints/broker.py` + +## Notes + +- `schema.sql` is a maintained schema snapshot used for documentation rather than the runtime bootstrap path. The actual initial database schema is in `0001_initial_schema.sql` which migrations are applied ontop of. When updating the database, we should always use and test Alembic migrations and *then* update `schema.sql` accordingly. diff --git a/docs/handover/system_overview.md b/docs/handover/system_overview.md new file mode 100644 index 0000000..05541ac --- /dev/null +++ b/docs/handover/system_overview.md @@ -0,0 +1,89 @@ +# System Overview + +### What the Canopy app does + +- The application is a FastAPI backend with a versioned API under `/api/v1` (`app/main.py`, `app/api/v1/api.py`). +- It stores 1) biological metadata imported from BPA, 2) submission-ready metadata objects for ENA and tracking of these submissions, 3) inputs and outputs for AToL's genome engine workflow (incl. qc, genome assembly and post-assembly pipelines). The key entities are: + - organisms and taxonomy information + - samples and sample lineage + - experiments and raw reads + - QC-read results + - assemblies, assembly runs, and assembly stage runs + - projects, genome notes*, and users + - broker submission objects, broker submission events, and ToLID request state +- Submission-facing entities use a main-table plus submission-table pattern for at least projects, samples, experiments, QC reads, and assemblies (`app/models/project.py`, `sample.py`, `experiment.py`, `qc_read.py`, `assembly.py`). + +### Codebase structure + +| Path | Verified role | +| --- | --- | +| `app/main.py` | FastAPI app creation, CORS setup, exception handlers, root, health, version | +| `app/api/v1/api.py` | Registers all active routers | +| `app/api/v1/endpoints/` | Route handlers and some light business rules *(should ideally move all business ogic to services)* | +| `app/models/` | SQLAlchemy models for tables and relationships | +| `app/schemas/` | Pydantic request/response models and enums | +| `app/services/` | Shared business logic for organism, experiment, taxonomy, assembly, broker-adjacent helpers, and ToLID state | +| `app/core/` | settings, auth dependencies, password/JWT helpers, pagination, policy wrapper, app error types | +| `app/config/ena-atol-map.json`** | ENA payload field mapping used by sample and experiment payload generation | +| `alembic/` | Schema migration entrypoint and revisions | +| `scripts/` | Container startup, lease-expiry helper, and user-creation script | +| `tests/` | Unit tests for routes, services, settings, security, and broker behavior | + +** This must be updated if either the AToL schema and/or ENA's ToL sample checklist [ERC000053](https://www.ebi.ac.uk/ena/browser/view/ERC000053) change. +### Entity and lifecycle patterns + +#### Organisms + +- Creating an organism also creates two `project` rows, one `root` and one `genomic_data`, plus matching `project_submission` draft rows (`app/services/organism_service.py`). +- `organism.scientific_name` is treated as the app-facing canonical name and is recomputed from `taxonomy_info.ncbi_scientific_name` when that exists; otherwise it falls back to `organism.bpa_scientific_name` (`app/services/organism_service.py`, `app/services/taxonomy_info_service.py`). + +#### Samples + +- Samples support `specimen` and `derived` kinds (`app/schemas/common.py`, `app/models/sample.py`). +- Derived samples must point to a specimen parent in the same `taxon_id`; specimen samples cannot have a parent (`app/api/v1/endpoints/samples.py`). +- Creating a sample also creates a draft `sample_submission` linked to the organism’s `genomic_data` project (`app/api/v1/endpoints/samples.py`). + +#### Experiments and reads + +- Creating an experiment requires an existing sample and the sample’s `genomic_data` project; it also creates a draft `experiment_submission` with `prepared_payload` built from `app/config/ena-atol-map.json` (`app/services/experiment_service.py`). +- Bulk experiment import also creates `read` rows from nested `runs` payloads (`app/services/experiment_service.py`). + +#### Taxonomy enrichment + +- Taxonomy-info create, bulk-import, bulk-upsert, and bulk-refresh all call NCBI lookup code in `app/services/ncbi_taxonomy_service.py` through `app/services/taxonomy_info_service.py`. +- Successful NCBI enrichment updates `taxonomy_info.ncbi_last_synced_at`, recomputes `organism.scientific_name`, and refreshes draft/ready project submission payloads for that organism (`app/services/taxonomy_info_service.py`, `app/services/organism_service.py`). +- Note: the bulk-insert endpoint sometimes returns before enriching all requested The endpoint is indopodent: so simply re-run the command until no more empty rows are returned. + +#### Assemblies + +- There are two assembly creation flows: + - generic `POST /api/v1/assemblies/` + - intent-driven `POST /api/v1/assemblies/intent/{taxon_id}` +- The intent flow validates specimen samples, resolves lineage-linked experiments and reads, generates a manifest JSON, stores it on the `assembly`, and returns `assembly_id`, `version`, and the manifest (`app/api/v1/endpoints/assemblies.py`, `app/services/assembly_helper.py`). +- "Assembly runs" (`assembly_run` table in db) represent a pipeline invocation for a specific assembly/manifest and are tracked/identified by `github_repo` plus `git_commit` (i.e. used to report results for different runs of the same assembly); "assembly stage runs" (`aasembly_stage_run` table in db) store one record per stage name per run (`app/models/assembly.py`, `app/services/assembly_service.py`). +- QC read reporting for assemblies creates `qc_read`, `qc_read_file`, `qc_read_assembly`, and a draft `qc_read_submission` (`app/api/v1/endpoints/assemblies.py`, `app/api/v1/endpoints/qc_reads.py`). +- Assembly stage reporting for assemblies accepts a flexible `data` object and a list of files. NOTE: at the moment, you can only report results for an assembly stage run via the POST /assemblies/{assembly_id}/runs/{run_id}/stage_runs/{stage_run_id} endpoint *once*, reporting more results will fail, and the respective PATCH endpoint will replace the existing files rather than appending them. (TODO: this should be fixed to allow appending files) + +#### Submission tables and status changes + +- Create paths for core objects (projects, samples, experiments, qc_reads) generally produce `draft` submission rows in the respective `*_submission` tables. +- Sample and experiment updates have status-aware behavior: + - `submitting`: update blocked + - `draft` or `ready`: existing submission row updated in place and reset to `draft` + - `accepted`: existing submission row marked `replaced`, then a new `draft` row is created preserving accession fields + - `rejected` or `replaced`: a new `draft` row is created preserving accession fields where present + (`app/api/v1/endpoints/samples.py`, `app/services/experiment_service.py`) + +### External interaction points visible in code + +- Broker-facing submission APIs live in `app/api/v1/endpoints/broker.py`. +- ToLID durable state is exposed through broker routes but handled by `app/services/tolid_service.py`. +- NCBI taxonomy lookup is the only outbound HTTP integration directly evidenced in application code (`app/services/ncbi_taxonomy_service.py`). + +## Notes + +- Canopy is a metadata store and broker-facing coordination service. It doesn't directly interact with ENA servers. Rather, it provides the data needed for submission, which is requested and submitted by a seperate ENA broker service and then reported back to Canopy (including, importantly, the accessions of submitted objects). + +## To check + +- Whether `POST /api/v1/assemblies/from-experiments/{taxon_id}` is still actively used now that the assembly intent flow exists. diff --git a/docs/handover/troubleshooting.md b/docs/handover/troubleshooting.md new file mode 100644 index 0000000..a26c63c --- /dev/null +++ b/docs/handover/troubleshooting.md @@ -0,0 +1,27 @@ +# Troubleshooting Guide + +## Verified + +| Symptom | Likely component | Files, endpoints, or scripts to inspect | Verified next checks | +| --- | --- | --- | --- | +| API container exits before serving requests | Startup config or DB connectivity | `scripts/entrypoint.sh`, `docker-compose.yml`, `app/core/settings.py` | Confirm `DATABASE_URI` is set in the shell environment seen by the entrypoint, not just derivable from `POSTGRES_*`; confirm DB answers `SELECT 1`; confirm Alembic can reach the same URI | +| App import fails with a settings error | Missing JWT or DB settings | `app/core/settings.py` | Check `JWT_SECRET_KEY`, `JWT_ALGORITHM`, and either `DATABASE_URI` or all `POSTGRES_*`; in `prod`, confirm `BACKEND_CORS_ORIGINS` is not `["*"]` | +| Login works but refresh fails with `401 Invalid or expired refresh token` | Refresh-token lookup or revocation | `app/api/v1/endpoints/auth.py`, `app/models/token.py` | Check whether the token hash exists in `refresh_token`, whether `expires_at` is still in the future, and whether `revoked` is already true | +| Sample or experiment update is rejected while broker work is in progress | Submission lock state | `app/api/v1/endpoints/samples.py`, `app/services/experiment_service.py`, broker attempt routes | Check the latest submission row status; both code paths block edits when the latest submission is `submitting` | +| Broker cannot claim work that maintainers expect to be available | Submission status or stale lease | `app/api/v1/endpoints/broker.py`, `scripts/expire_leases.py`, `POST /api/v1/admin/leases/expire`, `POST /api/v1/broker/leases/expire` | Check whether rows are in `draft` or `ready`; the routed broker claim contract accepts both; check `lock_expires_at`; expire stale leases if needed | +| Broker report returns `409` mentioning integrity constraints | Accession registry conflict | `app/api/v1/endpoints/broker.py`, `app/models/accession_registry.py` | Inspect `accession_registry` for an existing row with the same accession bound to a different entity; inspect the affected submission row’s accession FK fields | +| ToLID lookup by accession returns `404` | Sample accession resolution or sample kind | `app/services/tolid_service.py`, `app/models/tolid_request.py`, `GET /api/v1/broker/tolids/...` | Confirm there is an accepted `sample_submission.accession` or `sample.biosample_accession`; confirm the resolved sample has `kind = specimen` | +| ToLID report cannot create a row | Missing accepted sample accession | `app/services/tolid_service.py` | Check whether `_fallback_external_id()` can find a sample submission accession or `sample.biosample_accession`; without one, reporting returns `tolid_external_id_missing` | +| Assembly intent returns `422` for specimen samples | Sample lineage or experiment-platform validation | `app/api/v1/endpoints/assemblies.py`, `app/services/assembly_helper.py` | Confirm each supplied sample exists, is `kind='specimen'`, and matches the route `taxon_id`; confirm long-read experiments exist with platform `PACBIO_SMRT` or `OXFORD_NANOPORE`; confirm Hi-C experiments are `ILLUMINA` plus library strategy `HI-C` | +| Assembly QC read reporting returns `422` | Manifest mismatch or lineage mismatch | `POST /api/v1/assemblies/{assembly_id}/qc-reads/report`, `app/api/v1/endpoints/assemblies.py` | Confirm `bpa_package_id` resolves to an experiment; confirm the experiment sample belongs to the assembly’s allowed specimen lineage; confirm the package ID exists in `assembly.manifest_json["read_files"]`; confirm the supplied source MD5 values match `read.file_checksum` rows for that experiment | +| Genome note publish returns `409` | Existing published note for the same organism | `app/services/genome_note_service.py`, `app/api/v1/endpoints/genome_notes.py` | Check whether another `genome_note` row for the same `taxon_id` already has `is_published = true` | +| Taxonomy bulk import skips rows unexpectedly | Missing organism rows or unmapped NCBI enrichment | `app/services/taxonomy_info_service.py`, `app/services/ncbi_taxonomy_service.py` | Check whether each `taxon_id` already exists in `organism`; inspect `errors`, `ncbi_retryable_count`, and `ncbi_retryable_taxon_ids` in the bulk response | +| User-creation script fails in an unexpected checkout location | Hard-coded import path in script | `scripts/create_user.py` | Check the `sys.path.append("/Users/emilylm/Repositories/atol-database-v2")` line and adjust it if the repo is moved | + +## Verified Sharp Edges And Technical Debt + +- There are two different lease-expiry implementations with overlapping responsibilities. +- Legacy broker helper functions still exist in `app/api/v1/endpoints/broker.py` even though the older routes are no longer exposed. +- Submission status `ready` is part of the API contract, but this repo does not show who sets it. +- Several code comments and docstrings are stale. One example is the PacBio filtering note in `app/services/assembly_helper.py`, while tests in `tests/unit/services/test_assembly_helper.py` assert that all PacBio reads are currently included when they have `file_name`. +- The sample endpoint contains several `print(...)` debugging paths instead of structured logging. diff --git a/docs/migration_workflow.md b/docs/migration_workflow.md index 465b950..118b102 100644 --- a/docs/migration_workflow.md +++ b/docs/migration_workflow.md @@ -1,462 +1,81 @@ +# Migration Workflow -## Schema files +This document describes the migration workflow that is directly evidenced in the current repository. -- `alembic/base_schema/0001_initial_schema.sql` is the frozen bootstrap schema used only by `0001_initial_schema.py`. -- Do not edit that file after it has been established, unless you are intentionally rewriting migration history. -- `schema.sql` is the mutable current schema snapshot for the present database shape. -- New migrations should update the database schema and then `schema.sql` can be refreshed to match the current state. +## Verified ---- +### Migration framework -```bash -# Check current migration version -docker compose exec api alembic current - -# Rollback one migration -docker compose exec api alembic downgrade -1 - -# Rollback to specific version -docker compose exec api alembic downgrade - -# Restore from backup (last resort) -docker compose exec db psql -U postgres -d atol_db < backup.sql -``` - ---- - -## Traditional Server Deployment (Non-Docker Production) - -For production environments using VMs, bare metal servers, or Kubernetes (without Docker Compose). - -### Setup Requirements - -1. **Python environment** - ```bash - # Install Python 3.12+ and create virtual environment - python3 -m venv venv - source venv/bin/activate - pip install -r requirements.txt # or use uv - ``` - -2. **Environment variables** - ```bash - # Set database connection in .env or environment - export DATABASE_URI="postgresql://user:password@db-host:5432/atol_db" - ``` - -3. **Alembic configuration** - - Ensure `alembic.ini` points to correct database - - Or use environment variable: `ALEMBIC_CONFIG` if needed - -### Pre-deployment Checklist - -Same as Docker-based deployment, plus: - -1. **Verify database connectivity** - ```bash - # Test connection from application server - psql -h db-host -U postgres -d atol_db -c "SELECT version();" - ``` - -2. **Check migration files are deployed** - ```bash - ls -la alembic/versions/ - # Ensure new migration files are present - ``` - -3. **Verify Alembic can connect** - ```bash - alembic current - # Should show current migration version - ``` - -### Deployment Steps - -#### Option 1: Maintenance Window - -1. **Backup database** - ```bash - # From database server or application server with access - pg_dump -h db-host -U postgres -d atol_db -F c -f backup_$(date +%Y%m%d_%H%M%S).dump - - # Or SQL format - pg_dump -h db-host -U postgres -d atol_db > backup_$(date +%Y%m%d_%H%M%S).sql - ``` - -2. **Stop application** - ```bash - # Systemd - sudo systemctl stop atol-api - - # Or supervisor - sudo supervisorctl stop atol-api - - # Or Kubernetes - kubectl scale deployment atol-api --replicas=0 - ``` +- Alembic is configured in `alembic/env.py`. +- Active revisions are under `alembic/versions/`. +- Container startup runs `uv run alembic upgrade head` before serving the API (`scripts/entrypoint.sh`). -3. **Deploy new code** - ```bash - # Pull latest code - git pull origin main +### Current revision chain - # Install dependencies if changed - pip install -r requirements.txt - ``` +The current active revision sequence in `alembic/versions/` is: -4. **Run migration** - ```bash - # Activate virtual environment - source venv/bin/activate +1. `0001_initial_schema` +2. `0002_update_taxon_cols` +3. `0003_assembly_run_github` +4. `0004_qc_reads_assembly_refs` +5. `0005_add_tolid_requests` - # Run migration - alembic upgrade head +### Local developer workflow - # Verify - alembic current - ``` - -5. **Start application** - ```bash - # Systemd - sudo systemctl start atol-api - - # Or supervisor - sudo supervisorctl start atol-api - - # Or Kubernetes - kubectl scale deployment atol-api --replicas=3 - ``` - -6. **Monitor** - ```bash - # Check logs - sudo journalctl -u atol-api -f - - # Or supervisor - sudo tail -f /var/log/atol-api/error.log - - # Or Kubernetes - kubectl logs -f deployment/atol-api - ``` - -#### Option 2: Zero-Downtime (Rolling Deployment) - -1. **Ensure migration is backward-compatible** - - Add columns as nullable - - Don't drop columns yet - - Application code works with both old and new schema - -2. **Apply migration (while app is running)** - ```bash - # SSH to application server or use CI/CD - source venv/bin/activate - alembic upgrade head - ``` - -3. **Deploy new application code (rolling)** - ```bash - # Kubernetes rolling update - kubectl set image deployment/atol-api api=atol-api:v2.0 - kubectl rollout status deployment/atol-api - - # Or manual rolling restart with load balancer - # Update server 1, wait, update server 2, etc. - ``` - -4. **Verify deployment** - ```bash - # Check health endpoint - curl https://api.example.com/health - - # Monitor logs across all instances - kubectl logs -l app=atol-api --tail=100 - ``` - -### Rollback Procedure (Non-Docker) +1. Ensure the app can resolve `DATABASE_URI`. +2. Apply migrations: ```bash -# Check current version -alembic current - -# Rollback one migration -alembic downgrade -1 - -# Rollback to specific version -alembic downgrade - -# Restore from backup (last resort) -pg_restore -h db-host -U postgres -d atol_db -c backup.dump -# Or for SQL format: -psql -h db-host -U postgres -d atol_db < backup.sql - -# Restart application -sudo systemctl restart atol-api +uv run alembic upgrade head ``` -### CI/CD Integration - -#### GitHub Actions Example - -```yaml -# .github/workflows/deploy.yml -name: Deploy to Production - -on: - push: - branches: [main] - -jobs: - deploy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.12' +3. Run the test suite: - - name: Install dependencies - run: | - pip install -r requirements.txt - - - name: Run migrations - env: - DATABASE_URI: ${{ secrets.DATABASE_URI }} - run: | - alembic upgrade head - - - name: Deploy application - run: | - # Your deployment script - ./deploy.sh -``` - -#### GitLab CI Example - -```yaml -# .gitlab-ci.yml -stages: - - migrate - - deploy - -migrate: - stage: migrate - script: - - pip install -r requirements.txt - - alembic upgrade head - only: - - main - -deploy: - stage: deploy - script: - - ./deploy.sh - only: - - main +```bash +pytest -q ``` -### Kubernetes-Specific Considerations - -1. **Use init containers for migrations** - ```yaml - apiVersion: apps/v1 - kind: Deployment - metadata: - name: atol-api - spec: - template: - spec: - initContainers: - - name: migrate - image: atol-api:latest - command: ["alembic", "upgrade", "head"] - env: - - name: DATABASE_URI - valueFrom: - secretKeyRef: - name: db-secret - key: uri - containers: - - name: api - image: atol-api:latest - ``` - -2. **Use Jobs for one-off migrations** - ```yaml - apiVersion: batch/v1 - kind: Job - metadata: - name: db-migration-v2 - spec: - template: - spec: - containers: - - name: migrate - image: atol-api:latest - command: ["alembic", "upgrade", "head"] - restartPolicy: Never - ``` - -### Production Best Practices (Non-Docker) - -1. **Database connection pooling** - - Use PgBouncer or similar for connection pooling - - Migrations should use direct connection, not pooler - -2. **Migration locks** - - Alembic uses database locks to prevent concurrent migrations - - Ensure only one migration process runs at a time - -3. **Monitoring** - ```bash - # Check migration status - alembic current - - # View migration history - alembic history --verbose - - # Check database version - psql -h db-host -U postgres -d atol_db -c "SELECT * FROM alembic_version;" - ``` - -4. **Automated backups before migrations** - ```bash - #!/bin/bash - # pre-migration-backup.sh - - BACKUP_DIR="/backups/db" - TIMESTAMP=$(date +%Y%m%d_%H%M%S) - - # Create backup - pg_dump -h db-host -U postgres -d atol_db -F c -f "$BACKUP_DIR/pre_migration_$TIMESTAMP.dump" - - # Run migration - alembic upgrade head - - # Keep last 7 days of backups - find $BACKUP_DIR -name "pre_migration_*.dump" -mtime +7 -delete - ``` - -5. **Health checks** - - Ensure health endpoint checks database connectivity - - Load balancer should remove unhealthy instances during migration - -## Migration File Best Practices +### Container workflow -### Structure +- `docker-compose.yml` starts the API container. +- The API container entrypoint waits for the database and then runs: -```python -"""Brief description of what this migration does. - -Revision ID: xxxx -Revises: yyyy -Create Date: 2026-02-09 - -Detailed explanation of changes: -- What tables are affected -- What columns are added/removed -- Any data transformations -""" - -def upgrade() -> None: - # 1. Add new columns as nullable first (for zero-downtime) - # 2. Migrate data if needed - # 3. Make columns NOT NULL - # 4. Add constraints and indexes - # 5. Drop old columns last - pass - -def downgrade() -> None: - # Reverse all changes in opposite order - pass +```bash +uv run alembic upgrade head ``` -### Tips - -- **Idempotent operations**: Use `IF EXISTS` / `IF NOT EXISTS` where possible -- **Data migration**: Include SQL for migrating existing data -- **Indexes**: Create indexes CONCURRENTLY in production to avoid locks -- **Foreign keys**: Add them after data is populated -- **Transactions**: Alembic runs migrations in transactions by default -- **Testing**: Always test both upgrade and downgrade +- If `APP_MODE=migrate`, the entrypoint exits after migrations instead of starting the API. -## Common Patterns +### Schema snapshot maintenance -### Adding a Required Column +The comment at the top of `schema.sql` states that the file should be kept in sync with the database after migrations and regenerated using a schema-only `pg_dump`. -```python -def upgrade() -> None: - # Step 1: Add as nullable - op.add_column('table_name', sa.Column('new_col', sa.Text(), nullable=True)) - - # Step 2: Populate with default/migrated data - op.execute("UPDATE table_name SET new_col = 'default_value'") - - # Step 3: Make NOT NULL - op.alter_column('table_name', 'new_col', nullable=False) -``` - -### Renaming a Column - -```python -def upgrade() -> None: - op.alter_column('table_name', 'old_name', new_column_name='new_name') -``` +The verified instruction in `schema.sql` is: -### Dropping a Table with Foreign Keys - -```python -def upgrade() -> None: - # Drop dependent tables/constraints first - op.drop_constraint('fk_name', 'dependent_table', type_='foreignkey') - op.drop_table('table_name') +```bash +docker compose exec db pg_dump -U -d --schema-only --no-owner --no-acl ``` -## Troubleshooting - -### Migration not visible in container -- Ensure `alembic` directory is mounted in `docker-compose.yml` -- Restart containers: `docker compose restart api` +### What recent migrations changed -### Migration already applied -- Check: `docker compose exec api alembic current` -- View history: `docker compose exec api alembic history` +| Revision | Verified focus | +| --- | --- | +| `0002_update_taxon_cols` | Removes outdated taxonomy columns and adds `assembly.hic_specimen_sample_ids` | +| `0003_assembly_run_github` | Reworks assembly reporting around `assembly_run` and `assembly_stage_run` | +| `0004_qc_reads_assembly_refs` | Reworks QC-read reporting and adds the `qc_read_assembly` link table | +| `0005_add_tolid_requests` | Adds durable ToLID request state | -### Database connection errors -- Ensure database is running: `docker compose ps` -- Check connection string in `.env` -- Run from within container: `docker compose exec api alembic ...` +## Recommended maintainer practice -### Circular import errors -- Restart containers to clear Python cache -- Check for naming conflicts with Python packages +### For repo changes that include schema updates -## Monitoring - -After applying migrations in production: - -```bash -# Check migration status -docker compose exec api alembic current - -# View migration history -docker compose exec api alembic history - -# Check database size -docker compose exec db psql -U postgres -d atol_db -c " - SELECT pg_size_pretty(pg_database_size('atol_db'));" - -# Monitor application logs -docker compose logs -f api - -# Check for errors -docker compose logs api | grep ERROR -``` +1. Add or update the Alembic revision under `alembic/versions/`. +2. Apply the migration to a local database. +3. Update `schema.sql` so it matches the post-migration schema snapshot. +4. Run tests. +5. Update maintainer-facing docs if the schema change affects workflows or operations. -## References +## Unknown From This Repo -- [Alembic Documentation](https://alembic.sqlalchemy.org/) -- [SQLAlchemy Documentation](https://docs.sqlalchemy.org/) -- Project migrations: `alembic/versions/` +- There is no verified production rollback runbook in this repository. +- There is no verified repository-local command for creating new Alembic revisions documented here; the repo only proves how revisions are applied, not the team’s preferred authoring command. diff --git a/docs/xml_export_api.md b/docs/xml_export_api.md deleted file mode 100644 index d647894..0000000 --- a/docs/xml_export_api.md +++ /dev/null @@ -1,204 +0,0 @@ -# XML Export API Documentation - -This document provides information about the XML export functionality for ENA (European Nucleotide Archive) submissions in the AToL Database API. - -## Overview - -The XML Export API allows you to generate ENA-compliant XML files for sample, experiment, and run submissions. These XML files can be used for submitting data to ENA through their submission portal or API. - -## Endpoints - -### Sample XML Export - -#### Get XML for a Single Sample - -``` -GET /api/v1/xml-export/samples/{sample_id}/xml -``` - -Generates ENA-compliant XML for a specific sample identified by its UUID. - -**Parameters:** -- `sample_id`: UUID of the sample - -**Response:** -- Content-Type: `text/plain` -- Body: XML content for the sample - -#### Get XML for Multiple Samples - -``` -GET /api/v1/xml-export/samples/xml -``` - -Generates ENA-compliant XML for multiple samples based on provided filters. - -**Query Parameters:** -- `sample_ids`: (Optional) List of sample UUIDs to include -- `status`: (Optional) Filter by submission status - -**Response:** -- Content-Type: `text/plain` -- Body: XML content for all matching samples - -#### Get XML for Samples Associated with an Experiment - -``` -GET /api/v1/xml-export/experiments/package/{bpa_package_id}/samples/xml -``` - -Generates ENA-compliant XML for samples associated with a specific experiment package. - -**Parameters:** -- `bpa_package_id`: BPA package ID of the experiment - -**Response:** -- Content-Type: `text/plain` -- Body: XML content for associated samples - -### Experiment XML Export - -#### Get XML for a Single Experiment - -``` -GET /api/v1/xml-export/experiments/{experiment_id}/xml -``` - -Generates ENA-compliant XML for a specific experiment identified by its UUID. - -**Parameters:** -- `experiment_id`: UUID of the experiment - -**Response:** -- Content-Type: `text/plain` -- Body: XML content for the experiment - -#### Get XML for Multiple Experiments - -``` -GET /api/v1/xml-export/experiments/xml -``` - -Generates ENA-compliant XML for multiple experiments based on provided filters. - -**Query Parameters:** -- `experiment_ids`: (Optional) List of experiment UUIDs to include -- `status`: (Optional) Filter by submission status - -**Response:** -- Content-Type: `text/plain` -- Body: XML content for all matching experiments - -#### Get XML for an Experiment by Package ID - -``` -GET /api/v1/xml-export/experiments/package/{bpa_package_id}/xml -``` - -Generates ENA-compliant XML for a specific experiment identified by its BPA package ID. - -**Parameters:** -- `bpa_package_id`: BPA package ID of the experiment - -**Response:** -- Content-Type: `text/plain` -- Body: XML content for the experiment - -### Run XML Export - -#### Get XML for a Single Read/Run - -``` -GET /api/v1/xml-export/reads/{read_id}/xml -``` - -Generates ENA-compliant XML for a specific read/run identified by its UUID. - -**Parameters:** -- `read_id`: UUID of the read - -**Response:** -- Content-Type: `text/plain` -- Body: XML content for the read/run - -#### Get XML for Multiple Reads/Runs - -``` -GET /api/v1/xml-export/reads/xml -``` - -Generates ENA-compliant XML for multiple reads/runs based on provided filters. - -**Query Parameters:** -- `read_ids`: (Optional) List of read UUIDs to include -- `experiment_id`: (Optional) Filter by experiment ID -- `status`: (Optional) Filter by submission status - -**Response:** -- Content-Type: `text/plain` -- Body: XML content for all matching reads/runs - -#### Get XML for Reads/Runs Associated with an Experiment - -``` -GET /api/v1/xml-export/experiments/{experiment_id}/reads/xml -``` - -Generates ENA-compliant XML for reads/runs associated with a specific experiment. - -**Parameters:** -- `experiment_id`: UUID of the experiment - -**Response:** -- Content-Type: `text/plain` -- Body: XML content for associated reads/runs - -## XML Structure - -### Sample XML - -The generated sample XML follows the ENA sample schema and includes: - -- Sample identifiers -- Title -- Taxonomy information (taxon ID, scientific name, common name) -- Description -- Sample attributes -- ENA checklist reference - -### Experiment XML - -The generated experiment XML follows the ENA experiment schema and includes: - -- Experiment identifiers -- Title -- Study reference -- Design description and library information -- Platform and instrument model -- Experiment attributes - -### Run XML - -The generated run XML follows the ENA run schema and includes: - -- Run identifiers -- Experiment reference -- Platform and instrument model -- Data block with file information (filename, checksum, filetype, etc.) - -## Usage Example - -To retrieve XML for a specific experiment: - -``` -curl -X GET "http://localhost:8000/api/v1/xml-export/experiments/{experiment_id}/xml" -H "Authorization: Bearer {your_token}" -``` - -Replace `{experiment_id}` with the UUID of your experiment and `{your_token}` with your authentication token. - -## Notes - -- All endpoints require authentication -- The XML is generated from the `prepared_payload` field in the respective database tables -- If a record has no `prepared_payload` data, a 400 Bad Request error will be returned -- The XML is formatted according to ENA submission requirements diff --git a/tests/unit/endpoints/test_endpoints_broker_route_surface.py b/tests/unit/endpoints/test_endpoints_broker_route_surface.py new file mode 100644 index 0000000..e2d1725 --- /dev/null +++ b/tests/unit/endpoints/test_endpoints_broker_route_surface.py @@ -0,0 +1,43 @@ +from app.main import app + + +def test_broker_route_surface_exposes_supported_submission_routes(): + broker_routes = { + (method, route.path) + for route in app.routes + if hasattr(route, "methods") and route.path.startswith("/api/v1/broker") + for method in route.methods + } + + expected_routes = { + ("POST", "/api/v1/broker/claims/ready"), + ("POST", "/api/v1/broker/claims/entity"), + ("POST", "/api/v1/broker/claims/batch"), + ("POST", "/api/v1/broker/validation"), + ("POST", "/api/v1/broker/reports/{attempt_id}"), + ("POST", "/api/v1/broker/attempts/{attempt_id}/finalise"), + } + + assert expected_routes.issubset(broker_routes) + + +def test_broker_route_surface_does_not_expose_removed_legacy_routes(): + broker_routes = { + (method, route.path) + for route in app.routes + if hasattr(route, "methods") and route.path.startswith("/api/v1/broker") + for method in route.methods + } + + removed_routes = { + ("POST", "/api/v1/broker/claim"), + ("POST", "/api/v1/broker/organisms/{taxon_id}/claim"), + ("POST", "/api/v1/broker/attempts/{attempt_id}/lease/renew"), + ("POST", "/api/v1/broker/attempts/{attempt_id}/report"), + ("GET", "/api/v1/broker/attempts"), + ("GET", "/api/v1/broker/attempts/{attempt_id}"), + ("GET", "/api/v1/broker/attempts/{attempt_id}/items"), + ("GET", "/api/v1/broker/organisms/{taxon_id}/summary"), + } + + assert broker_routes.isdisjoint(removed_routes)