From 7934e7a78e29e69f3a596ba5cdf7acf9736d8519 Mon Sep 17 00:00:00 2001 From: Emily Marshall Date: Tue, 23 Jun 2026 22:48:10 +1000 Subject: [PATCH 1/9] feat: tighten access token expiration time --- app/core/settings.py | 2 +- .../broker_prerequisites_enhancement.md | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename BROKER_PREREQUISITES_ENHANCEMENT.md => docs/broker_prerequisites_enhancement.md (100%) diff --git a/app/core/settings.py b/app/core/settings.py index c878e50..9f7143f 100644 --- a/app/core/settings.py +++ b/app/core/settings.py @@ -25,7 +25,7 @@ class Settings(BaseSettings): # Security JWT_SECRET_KEY: Optional[str] = None JWT_ALGORITHM: Optional[str] = None - JWT_ACCESS_TOKEN_EXPIRE_MINUTES: int = 150 # 150 minutes TODO change to 15 min for prod + JWT_ACCESS_TOKEN_EXPIRE_MINUTES: int = 15 # 15 minutes JWT_REFRESH_TOKEN_EXPIRE_DAYS: int = 7 # 7 days DATABASE_URI: Optional[str] = None diff --git a/BROKER_PREREQUISITES_ENHANCEMENT.md b/docs/broker_prerequisites_enhancement.md similarity index 100% rename from BROKER_PREREQUISITES_ENHANCEMENT.md rename to docs/broker_prerequisites_enhancement.md From d4f9e98cffd28a11805c873dcb6bc06ca8bd3a90 Mon Sep 17 00:00:00 2001 From: Emily Marshall Date: Wed, 24 Jun 2026 00:07:06 +1000 Subject: [PATCH 2/9] docs: update handover docs --- README.md | 395 +-------------- docs/assembly_reporting_api.md | 2 + docs/auth_refresh_tokens.md | 128 ++--- docs/broker_prerequisites_enhancement.md | 2 + docs/bulk_import_api.md | 196 +++----- docs/handover/README.md | 34 ++ docs/handover/broker_and_submission_flows.md | 88 ++++ docs/handover/config_reference.md | 62 +++ docs/handover/doc_audit.md | 31 ++ docs/handover/open_questions.md | 30 ++ docs/handover/setup_and_operations.md | 74 +++ docs/handover/system_overview.md | 90 ++++ docs/handover/troubleshooting.md | 27 ++ docs/migration_workflow.md | 479 ++----------------- docs/ncbi_taxonomy_sync.md | 2 + docs/tolid_broker_api.md | 2 + docs/xml_export_api.md | 204 -------- 17 files changed, 644 insertions(+), 1202 deletions(-) create mode 100644 docs/handover/README.md create mode 100644 docs/handover/broker_and_submission_flows.md create mode 100644 docs/handover/config_reference.md create mode 100644 docs/handover/doc_audit.md create mode 100644 docs/handover/open_questions.md create mode 100644 docs/handover/setup_and_operations.md create mode 100644 docs/handover/system_overview.md create mode 100644 docs/handover/troubleshooting.md delete mode 100644 docs/xml_export_api.md diff --git a/README.md b/README.md index f86f804..52ffab1 100644 --- a/README.md +++ b/README.md @@ -1,381 +1,26 @@ -# Canopy: A Metadata Tracking System for the Australian Tree of Life (AToL) data +# Canopy Handover Entry Point -Canopy is a FastAPI backend used to track and manage genomic data for the Australian Tree of Life (AToL) project. +This repository contains a FastAPI application for tracking organism, sample, experiment, read, assembly, project, QC-read, genome-note, taxonomy, user, broker-attempt, and ToLID-request metadata for the Australian Tree of Life project. -## Overview +This `README` has been reduced to verified pointers. The detailed handover pack created from the current codebase is in `docs/handover/`. -Canopy manages metadata across core biological entities (Organism, Sample, Experiment, Read, Assembly, Project, BPA Initiative, Genome Note), and models each submission lifecycle with a two-table pattern per entity: -- Main: current state -- Submission: staged for external submission (e.g., ENA) +## Verified Starting Points -A dedicated broker workflow enables integration with external submission pipelines. The broker can claim work (with short leases) and report outcomes/accessions back to Canopy. Helper endpoints provide attempt summaries and item listings to power a simple dashboard or external UI. +- API app entrypoint: `app/main.py` +- Versioned API router: `app/api/v1/api.py` +- Runtime settings: `app/core/settings.py` +- Database session setup: `app/db/session.py` +- Docker startup path: `Dockerfile`, `docker-compose.yml`, `scripts/entrypoint.sh` +- Schema and migrations: `schema.sql`, `alembic/env.py`, `alembic/versions/` +- Test suite: `tests/` -## Features +## Handover Pack -- Authentication & tokens: JWT access, refresh, and logout endpoints -- Role-based access control: user, curator, broker, genome_launcher, admin, superuser -- RESTful APIs for core entities (organisms, samples, experiments, reads, assemblies, projects, genome notes, BPA initiatives) -- Submission workflow endpoints (sample-submissions, experiment-submissions, read-submissions) -- Broker endpoints to support external submission pipelines: - - Claim drafts and obtain a lease: `/api/v1/broker/organisms/{taxon_id}/claim` - - ENA broker contract endpoints: `/api/v1/broker/claims/ready`, `/api/v1/broker/claims/entity`, `/api/v1/broker/validation`, `/api/v1/broker/reports/{attempt_id}` - - Renew lease, finalise, and report results: `/api/v1/broker/attempts/{attempt_id}/...` - - Lightweight ToLID persistence/reporting endpoints: `/api/v1/broker/tolids/...` - - Attempt listing and summaries for dashboard views -- Bulk import endpoints for organisms, samples, and experiments -- XML export endpoints for downstream systems -- PostgreSQL database with UUID primary keys and JSONB fields; schema bootstrapped via Docker using `schema.sql` -- Docker Compose for local development, plus an alternative local (non-Docker) run mode - -## Tech Stack - -- Python 3.10, FastAPI, Uvicorn -- SQLAlchemy 2.x ORM -- Pydantic v2 + pydantic-settings -- PostgreSQL 14 -- JWT auth via python-jose[cryptography]; password hashing via passlib[bcrypt] -- Alembic (available); schema initialization via `schema.sql` in Docker -- Docker & Docker Compose - -## Project Structure - -``` -app/ -├── main.py -├── api/ -│ └── v1/ -│ ├── api.py -│ └── endpoints/ -│ ├── auth.py -│ ├── users.py -│ ├── organisms.py -│ ├── samples.py -│ ├── sample_submissions.py -│ ├── experiments.py -│ ├── experiment_submissions.py -│ ├── reads.py -│ ├── read_submissions.py -│ ├── assemblies.py -│ ├── projects.py -│ ├── bpa_initiatives.py -│ ├── genome_notes.py -│ ├── broker.py -│ └── xml_export.py -├── core/ -│ ├── dependencies.py -│ ├── security.py -│ └── settings.py -├── db/ -│ └── session.py -├── models/ -│ ├── user.py, token.py -│ ├── organism.py, sample.py, experiment.py, read.py -│ ├── assembly.py, project.py, bpa_initiative.py, genome_note.py -│ ├── accession_registry.py, broker.py -│ └── (SQLAlchemy models) -└── schemas/ - ├── user.py - ├── organism.py, sample.py, experiment.py, read.py - ├── assembly.py, project.py, bpa_initiative.py, genome_note.py - └── (Pydantic schemas) - -# Top-level -pyproject.toml, uv.lock, docker-compose.yml, Dockerfile, schema.sql, scripts/, docs/, data/ -``` - -## Getting Started - - ### Prerequisites - - - Docker and Docker Compose - - [uv](https://docs.astral.sh/uv/) for local (non-Docker) development - - Postgres available for Alembic migrations - - ### Running the Application (Docker) - - 1) Create your environment file from the template: - - ```bash - cp .env.example .env - ``` - - 2) Edit `.env` and set the required values. At minimum: - - `POSTGRES_USER=postgres` - - `POSTGRES_PASSWORD=` (default example uses `postgres`) - - `POSTGRES_DB=atol_db` - - `POSTGRES_PORT=5432` # container port (host is mapped to 5433) - - `POSTGRES_SERVER=db` # do not change; this is the Docker service name - - `JWT_SECRET_KEY=` - - `JWT_ALGORITHM=HS256` - - Generate a secure secret (macOS/Linux): - - ```bash - openssl rand -hex 32 - ``` - - 3) Build and start the stack: - - ```bash - docker compose up -d --build # Compose v2 - # or - docker-compose up -d --build # older Compose - ``` - - 4) Tail logs (optional) and wait for the API to be ready: - - ```bash - docker compose logs -f api - ``` - - 5) Open the API docs: - - - Swagger UI: http://localhost:8000/api/v1/docs - - ReDoc: http://localhost:8000/api/v1/redoc - - 6) Stop the stack: - - ```bash - docker compose down - ``` - - To reset the database (this will delete your data): - - ```bash - docker compose down -v - ``` - - Note on code changes (Docker): the API container runs uvicorn without `--reload`. If you edit code, restart the API container to pick up changes: - - ```bash - docker compose restart api - ``` - - #### Local database access (via Docker) - Connect to the Postgres database hosted on the `db` container using `psql`. - - - Quick one-liner (uses credentials created by `.env` during container init): - - ```bash - docker compose exec db psql -U postgres -d atol_db - ``` - - - Or open a shell in the container first, then run psql: - - ```bash - docker compose exec -it db bash - psql -U postgres -d atol_db - ``` - - Once in psql, you can run SQL and handy meta-commands: - - ```sql - -- List databases - \l - -- Connect to a database (if needed) - \c atol_db - -- List schemas - \dn - -- List tables (optionally by schema) - \dt - \dt public.* - -- Describe a table - \d+ public.users - -- Run a query - SELECT NOW(); - -- Quit psql - \q - ``` - - ### Create your first user - On a fresh database you will likely want to create an initial user (e.g. an admin/broker) so you can log in and use secured endpoints. Use the helper script from the project root on your host machine: - - ```bash - python scripts/create_user.py \ - --host localhost \ - --port 5433 \ - --dbname atol_db \ - --user postgres \ - --password \ - --username admin \ - --email admin@example.org \ - --user-password \ - --role user - ``` - - You will need to manually update the role for your first user in the database. - - Then obtain a JWT to call secured endpoints: - - ```bash - # Exchange username/password for tokens - curl -s -X POST http://localhost:8000/api/v1/auth/login \ - -H 'Content-Type: application/x-www-form-urlencoded' \ - -d 'username=admin&password=' - ``` - - Export the returned `access_token` and use it in the `Authorization: Bearer ` header. - - ### Broker integration quickstart - The broker API provides claim/report endpoints to integrate with an external submission pipeline. - - - Claim items for an organism (example): - - ```bash - TOKEN= - curl -s -X POST "http://localhost:8000/api/v1/broker/organisms//claim?per_type_limit=100" \ - -H "Authorization: Bearer $TOKEN" \ - -H "Content-Type: application/json" \ - -d '{"lease_duration_minutes": 30}' - ``` - - - Report results for an attempt (example shape): - - ```bash - curl -s -X POST "http://localhost:8000/api/v1/broker/attempts//report" \ - -H "Authorization: Bearer $TOKEN" \ - -H "Content-Type: application/json" \ - -d '{"samples": [], "experiments": [], "reads": [], "projects": []}' - ``` - - For the flat ENA broker contract used by Canopy, see [docs/ena_broker_contract.md](docs/ena_broker_contract.md). - For the lightweight ToLID persistence/reporting flow, see [docs/tolid_broker_api.md](docs/tolid_broker_api.md). - For a deeper overview of attempt leasing and statuses, see the `broker` endpoints in `app/api/v1/endpoints/broker.py` and the interactive docs. - - ## Bulk Import API - -The system provides API endpoints for bulk importing organisms, samples, and experiments. These endpoints allow you to import data in the same format as the standalone import script but through authenticated API calls. - -### Bulk Import Endpoints - -- `/api/v1/organisms/bulk-import` - Bulk import organisms -- `/api/v1/samples/bulk-import` - Bulk import samples -- `/api/v1/experiments/bulk-import` - Bulk import experiments - -All bulk import endpoints: -- Require authentication with 'curator' or 'admin' role -- Accept JSON data in the same format as the standalone import script -- Return counts of created and skipped records - -### Example usage - -See `docs/bulk_import_api.md` for request formats and curl examples for organisms, samples, and experiments. - -4. Access the API documentation at http://localhost:8000/api/v1/docs - -### Authentication - -The API uses JWT tokens for authentication. To authenticate: - -1. Create a user or use a default admin user -2. Get a token from `/auth/login` endpoint -3. Use the token in the Authorization header: `Bearer {token}` - -## API Documentation - -Once the application is running, you can access the interactive API documentation: - -- Swagger UI: http://localhost:8000/api/v1/docs -- ReDoc: http://localhost:8000/api/v1/redoc - -## Role-Based Access Control - -The system implements role-based access control with the following roles: - -- **user**: Basic read access to data -- **curator**: Can create and update biological entities -- **broker**: Can claim and report biological entities -- **genome_launcher**: Can access data required by genome assembly & post-assembly pipelines and report results -- **admin**: Full access to all endpoints -- **superuser**: Special role with delete permissions - -## Development (run without Docker) - - Prefer Docker for a consistent setup. This section shows how to run the FastAPI app directly on your machine as an alternative. - - 1) Sync the environment with uv (creates `.venv` in the project root): - - ```bash - uv sync --dev - # Optional: source .venv/bin/activate - ``` - - 2) Configure environment - - - Copy `.env.example` to `.env` and fill the required values (particularly `JWT_SECRET_KEY`, `JWT_ALGORITHM`). - - Choose ONE of the database options below: - - Option A — reuse the Docker Postgres (recommended) - - ```bash - # Start only the database service - docker compose up -d db - - # In your .env (or environment), point to the compose database on host 5433 - POSTGRES_SERVER=localhost - POSTGRES_PORT=5433 - POSTGRES_USER=postgres - POSTGRES_PASSWORD= - POSTGRES_DB=atol_db - ``` - - Option B — use a local Postgres (no Docker) - - ```bash - # Ensure Postgres is running locally (port 5432) - # Create the database (migrations will handle schema) - createdb -h localhost -U postgres atol_db || true - - # In your .env (or environment), point to your local instance - POSTGRES_SERVER=localhost - POSTGRES_PORT=5432 - POSTGRES_USER=postgres - POSTGRES_PASSWORD= - POSTGRES_DB=atol_db - ``` - - 3) Run the application - - ```bash - uv run alembic upgrade head - uv run uvicorn app.main:app --reload --host 0.0.0.0 --port 8000 - ``` - - 4) Open the docs at http://localhost:8000/api/v1/docs - - ### Linting - - - Install hooks: `uv run pre-commit install` - - Run all checks: `uv run pre-commit run --all-files` - - Install commit message hook: `uv run pre-commit install --hook-type commit-msg` (requires commitlint hook configured) - - ### Database migrations (Alembic) - - - Apply migrations: `uv run alembic upgrade head` - - Create a new revision: `uv run alembic revision --autogenerate -m "description"` - - Docker Compose runs migrations automatically on container start via `scripts/entrypoint.sh`. - - `scripts/entrypoint.sh` supports `APP_MODE=serve` (default) and `APP_MODE=migrate` (run migrations then exit). - - ### Environment configuration - -Configuration is provided via environment variables loaded from `.env` (see `.env.example`). Key settings: - -- Database - - `POSTGRES_SERVER` (Docker service name is `db`) - - `POSTGRES_USER` - - `POSTGRES_PASSWORD` - - `POSTGRES_DB` - - `POSTGRES_PORT` (container port, default 5432; host is `5433` via compose) -- Auth - - `JWT_SECRET_KEY` (required) - - `JWT_ALGORITHM` (e.g. `HS256`) - - `JWT_ACCESS_TOKEN_EXPIRE_MINUTES` (default 150) - - `JWT_REFRESH_TOKEN_EXPIRE_DAYS` (default 7) -- CORS - - `BACKEND_CORS_ORIGINS` (JSON array of origins) - -When using Docker, the database is initialized automatically on first run using `schema.sql` via the Postgres container's init hook. - -## License - -This project is licensed under the GPL-3.0-or-later License. +- Overview and structure: `docs/handover/README.md` +- Documentation audit: `docs/handover/doc_audit.md` +- System and lifecycle overview: `docs/handover/system_overview.md` +- Local setup and operations: `docs/handover/setup_and_operations.md` +- Broker and submission flows: `docs/handover/broker_and_submission_flows.md` +- Config and environment reference: `docs/handover/config_reference.md` +- Troubleshooting guide: `docs/handover/troubleshooting.md` +- Open questions and tribal knowledge gaps: `docs/handover/open_questions.md` diff --git a/docs/assembly_reporting_api.md b/docs/assembly_reporting_api.md index 40e0a51..4f91f9d 100644 --- a/docs/assembly_reporting_api.md +++ b/docs/assembly_reporting_api.md @@ -1,3 +1,5 @@ +> **Status:** Verified current against `app/api/v1/endpoints/assemblies.py`, `app/services/assembly_service.py`, `app/services/assembly_helper.py`, and `tests/unit/endpoints/test_endpoints_assemblies.py`. + # Assembly Reporting API This document describes how to use the assembly API to register a new assembly, then report pipeline results back to the database. diff --git a/docs/auth_refresh_tokens.md b/docs/auth_refresh_tokens.md index ef6e9ee..e98521c 100644 --- a/docs/auth_refresh_tokens.md +++ b/docs/auth_refresh_tokens.md @@ -1,53 +1,68 @@ -# Authentication with Refresh Tokens +> **Status:** Verified current against `app/api/v1/endpoints/auth.py`, `app/core/security.py`, `app/core/dependencies.py`, `app/models/token.py`, and `tests/unit/endpoints/test_endpoints_auth.py`. -This document explains how the authentication system works in the ATOL Canopy application with the new refresh token implementation. +# Authentication And Refresh Tokens -## Overview +This document describes the authentication behavior that is implemented in the current codebase. -The authentication system uses a stateful refresh token approach with JWT (JSON Web Tokens): +## Verified -1. **Access Tokens**: Short-lived JWTs (15 minutes) used for API authorization -2. **Refresh Tokens**: Longer-lived tokens (7 days) stored in the database and used to obtain new access tokens +### Endpoints -## Authentication Flow +| Method | Endpoint | Purpose | +| --- | --- | --- | +| `POST` | `/api/v1/auth/login` | Exchange username and password for an access token and refresh token | +| `POST` | `/api/v1/auth/refresh` | Rotate a valid refresh token and issue a new access token and refresh token | +| `POST` | `/api/v1/auth/logout` | Revoke all active refresh tokens for the authenticated user | -### 1. Login +### Access tokens -When a user logs in with valid credentials, the system: +- Access tokens are JWTs created by `create_access_token()` in `app/core/security.py`. +- The token payload contains: + - `sub`: the user ID as a string + - `exp`: the token expiry timestamp +- Protected endpoints resolve the current user by decoding the JWT in `app/core/dependencies.py`. -- Creates a short-lived access token (JWT) -- Generates a refresh token and stores it in the database -- Returns both tokens to the client +### Refresh tokens -```http -POST /api/v1/auth/login -Content-Type: application/x-www-form-urlencoded +- Refresh tokens are random opaque strings generated by `generate_refresh_token()` in `app/core/security.py`. +- Only the SHA-256 hash of the refresh token is stored in the database table `refresh_token` (`app/models/token.py`). +- The refresh flow performs token rotation: + - the old refresh token row is marked `revoked = true` + - a new refresh token is generated + - a new `refresh_token` row is inserted -username=your_username&password=your_password -``` +### Logout behavior -Response: +- `POST /api/v1/auth/logout` revokes all non-revoked refresh tokens belonging to the current user. +- Logout does not invalidate already-issued JWT access tokens directly. The code only revokes refresh-token rows. -```json -{ - "access_token": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9...", - "refresh_token": "random_secure_string", - "token_type": "bearer" -} -``` +### Expiry configuration -### 2. Using Access Tokens +- Access-token expiry is controlled by `JWT_ACCESS_TOKEN_EXPIRE_MINUTES`. +- Refresh-token expiry is controlled by `JWT_REFRESH_TOKEN_EXPIRE_DAYS`. +- Current code defaults: + - `JWT_ACCESS_TOKEN_EXPIRE_MINUTES = 150` + - `JWT_REFRESH_TOKEN_EXPIRE_DAYS = 7` -Include the access token in the `Authorization` header for protected API requests: +### Failure modes verified in code + +- Invalid username or password at login returns `401`. +- Inactive user at login returns `400`. +- Invalid, expired, revoked, deleted-user, or inactive-user refresh attempts return `401`. +- Invalid bearer token or missing `sub` claim returns `401`. + +## Example Requests + +### Login ```http -GET /api/v1/protected-endpoint -Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9... -``` +POST /api/v1/auth/login +Content-Type: application/x-www-form-urlencoded -### 3. Refreshing Tokens +username=your_username&password=your_password +``` -When the access token expires, use the refresh token to get a new pair of tokens: +### Refresh ```http POST /api/v1/auth/refresh @@ -58,53 +73,14 @@ Content-Type: application/json } ``` -Response: - -```json -{ - "access_token": "new_access_token", - "refresh_token": "new_refresh_token", - "token_type": "bearer" -} -``` - -**Note**: The system implements token rotation - each time you refresh, you get a new refresh token and the old one is invalidated. - -### 4. Logout - -To securely logout, revoke all refresh tokens: +### Logout ```http POST /api/v1/auth/logout -Authorization: Bearer your_access_token -``` - -Response: - -```json -{ - "message": "Successfully logged out" -} +Authorization: Bearer ``` -## Security Features - -1. **Token Rotation**: Each refresh operation invalidates the used refresh token and issues a new one -2. **Database Storage**: Refresh tokens are stored as secure hashes, not plaintext -3. **Automatic Expiration**: Both access and refresh tokens have expiration times -4. **Revocation**: Refresh tokens can be revoked by the user or the system -5. **Cascading Deletion**: When a user is deleted, all their refresh tokens are automatically removed - -## Implementation Details - -- Access tokens are stateless JWTs signed with a secret key -- Refresh tokens are securely generated random strings -- Refresh tokens are stored in the `refresh_tokens` table with a hash of the actual token -- Token validation includes checking expiration and revocation status - -## Best Practices for Client Applications +## Unknown From This Repo -1. Store the refresh token securely (e.g., in an HTTP-only cookie or secure storage) -2. Store the access token in memory (not localStorage) to prevent XSS attacks -3. Implement automatic token refresh when access tokens expire -4. Always call the logout endpoint when the user logs out +- There is no verified client-side guidance in this repo for where browser or desktop clients should store tokens. +- There is no server-side access-token denylist or forced JWT invalidation mechanism beyond normal expiry. diff --git a/docs/broker_prerequisites_enhancement.md b/docs/broker_prerequisites_enhancement.md index 1dc7afc..6813c62 100644 --- a/docs/broker_prerequisites_enhancement.md +++ b/docs/broker_prerequisites_enhancement.md @@ -1,3 +1,5 @@ +> **Status:** Partially current. The accession-resolution behavior matches `app/api/v1/endpoints/broker.py`, but this note is change-history oriented and does not fully describe the current broker contract. + # Broker Prerequisites Enhancement ## Problem diff --git a/docs/bulk_import_api.md b/docs/bulk_import_api.md index 7089624..04af680 100644 --- a/docs/bulk_import_api.md +++ b/docs/bulk_import_api.md @@ -1,159 +1,119 @@ -# Bulk Import API Documentation +> **Status:** Verified current against `app/api/v1/endpoints/organisms.py`, `app/api/v1/endpoints/samples.py`, `app/api/v1/endpoints/experiments.py`, `app/services/organism_service.py`, `app/services/experiment_service.py`, and the bulk-import endpoint tests. -This document describes how to use the bulk import API endpoints to import organisms, samples, and experiments into the database. +# Bulk Import Workflows -## Authentication +This document describes the bulk-import endpoints that are active in the current codebase. -All bulk import endpoints require authentication and the user must have either the 'curator' or 'admin' role. +## Verified -## Endpoints Overview +### Active bulk-import endpoints -| Endpoint | Method | Description | -|----------|--------|-------------| -| `/api/v1/organisms/bulk-import` | POST | Bulk import organisms | -| `/api/v1/samples/bulk-import` | POST | Bulk import samples | -| `/api/v1/experiments/bulk-import` | POST | Bulk import experiments | +| Method | Endpoint | Expected top-level payload shape | Notes | +| --- | --- | --- | --- | +| `POST` | `/api/v1/organisms/bulk-import` | raw dictionary keyed by `taxon_id` | No wrapper key such as `organisms` | +| `POST` | `/api/v1/samples/bulk-import` | raw dictionary keyed by `bpa_sample_id` | Mixed sample import path; defaults to specimen unless `kind` supplied | +| `POST` | `/api/v1/samples/bulk-import-specimens` | raw dictionary keyed by `taxon_id`, then `specimen_id` | Explicit specimen-only import | +| `POST` | `/api/v1/samples/bulk-import-derived` | raw dictionary keyed by sample key | Explicit derived-only import | +| `POST` | `/api/v1/experiments/bulk-import` | raw dictionary keyed by package ID | Can also create `read` rows from nested `runs` lists | +| `POST` | `/api/v1/taxonomy-info/bulk-import` | raw dictionary keyed by `taxon_id` | Insert-oriented, with NCBI enrichment | +| `POST` | `/api/v1/taxonomy-info/bulk-upsert` | raw dictionary keyed by `taxon_id` | Insert or update, with NCBI enrichment | +| `POST` | `/api/v1/taxonomy-info/bulk-ncbi-refresh` | object with `taxon_ids` list | Refresh existing taxonomy-info rows only | -## Request Format +### Authentication and roles -### Bulk Import Organisms +- Organism bulk import requires `organisms:bulk_import`. +- Sample bulk import endpoints require `samples:bulk_import`. +- Experiment bulk import requires `experiments:bulk_import`. +- Taxonomy bulk actions require the matching taxonomy policies defined in `app/core/policy.py`. -**Endpoint:** `/api/v1/organisms/bulk-import` +### Recommended import order -**Request Body:** -```json -{ - "organisms": { - "123456": { - "taxon_id": 123456, - "scientific_name": "Organism scientific name", - "other_field": "value", - ... - }, - "789012": { - ... - } - } -} -``` +1. Organisms +2. Specimen samples +3. Derived samples +4. Experiments +5. Assembly intents or broker-driven submission workflows +6. Taxonomy-info import or refresh when NCBI enrichment is needed -The request body should match the format of the JSON file in `data/unique_organisms.json`. Each organism is keyed by `taxon_id` and must contain at least `taxon_id` and `scientific_name`. +This ordering is verified from the code-level dependencies: -**Response:** -```json -{ - "created_count": 10, - "skipped_count": 2, - "message": "Organism import complete. Created: 10, Skipped: 2" -} -``` +- samples require existing organisms +- experiments require existing samples and the organism’s `genomic_data` project +- taxonomy info requires existing organisms -### Bulk Import Samples +## Endpoint-specific behavior -**Endpoint:** `/api/v1/samples/bulk-import` +### `POST /api/v1/organisms/bulk-import` -**Request Body:** -```json -{ - "samples": { - "bpa_sample_id_1": { - "taxon_id": 123456, - "other_field": "value", - ... - }, - "bpa_sample_id_2": { - ... - } - } -} -``` +- Creates new `organism` rows. +- Also creates the paired `root` and `genomic_data` `project` rows plus draft `project_submission` rows for each organism. +- Skips rows when the organism already exists. +- Accepts both current `bpa_*` fields and legacy unprefixed fields for several organism attributes. -The request body should match the format of the JSON file in `data/unique_samples.json`. Each sample is identified by its `bpa_sample_id` and can include `taxon_id` to link it to an organism. +Example payload: -**Response:** ```json { - "created_count": 15, - "skipped_count": 3, - "message": "Sample import complete. Created samples: 15, Created submission records: 15, Skipped: 3" + "172942": { + "taxon_id": 172942, + "bpa_scientific_name": "Example species" + } } ``` -### Bulk Import Experiments +### `POST /api/v1/samples/bulk-import-specimens` + +- Expects nested keys: outer `taxon_id`, inner `specimen_id`. +- Enforces one specimen sample per `(taxon_id, specimen_id)`. +- Forces `organism_part` to `"WHOLE ORGANISM"` in this explicit specimen import path. -**Endpoint:** `/api/v1/experiments/bulk-import` +Example payload: -**Request Body:** ```json { - "experiments": { - "bpa_package_id_1": { - "bpa_sample_id": "bpa_sample_id_1", - "other_field": "value", - ... - }, - "bpa_package_id_2": { - ... + "172942": { + "SPEC-001": { + "bpa_sample_id": "BPA-SPEC-001", + "lifestage": "adult" } } } ``` -The request body should match the format of the JSON file in `data/experiments.json`. Each experiment is identified by its `bpa_package_id` and must contain a `bpa_sample_id` to link it to a sample. +### `POST /api/v1/samples/bulk-import-derived` -**Response:** -```json -{ - "created_count": 20, - "skipped_count": 5, - "message": "Experiment import complete. Created experiments: 20, Created submission records: 20, Skipped: 5" -} -``` - -## Import Order - -When importing data, follow this order to ensure proper relationships: +- Requires `bpa_sample_id`. +- Requires `taxon_id`. +- Requires `specimen_id` so the code can find the parent specimen sample. +- Parent lookup is by `(taxon_id, specimen_id)` with `kind = specimen`. -1. Import organisms first -2. Import samples second (they may reference organisms) -3. Import experiments last (they reference samples) +### `POST /api/v1/samples/bulk-import` -## Error Handling +- Accepts a flatter legacy/importer-friendly path keyed by `bpa_sample_id`. +- Defaults missing required text fields such as `lifestage`, `sex`, and `habitat` to `"unknown"` in code. +- Defaults sample kind to `specimen` unless `kind` is supplied. -- If an entity already exists (by its unique key), it will be skipped -- If required fields are missing, the entity will be skipped -- If referenced entities don't exist (e.g., a sample references a non-existent organism), the entity will still be created but without the relationship +### `POST /api/v1/experiments/bulk-import` -## Example Usage with curl +- Requires `bpa_sample_id` in each experiment payload so the sample can be resolved. +- Requires `bpa_library_id`. +- Creates `experiment` and `experiment_submission` rows. +- If a `runs` list exists, the importer also creates `read` rows. +- If the experiment already exists, the importer still attempts to create any missing reads from the nested `runs` list. -### Import Organisms -```bash -curl -X POST "http://localhost:8000/api/v1/organisms/bulk-import" \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer YOUR_TOKEN" \ - -d @data/unique_organisms.json -``` +### Taxonomy bulk operations -### Import Samples -```bash -curl -X POST "http://localhost:8000/api/v1/samples/bulk-import" \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer YOUR_TOKEN" \ - -d @data/unique_samples.json -``` +- `bulk-import` skips already-fully-synced rows and only creates new rows when NCBI enrichment returns mapped data. +- `bulk-upsert` can insert or update rows and re-fetches NCBI data for all supplied taxon IDs. +- `bulk-ncbi-refresh` updates only existing `taxonomy_info` rows. -### Import Experiments -```bash -curl -X POST "http://localhost:8000/api/v1/experiments/bulk-import" \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer YOUR_TOKEN" \ - -d @data/experiments.json -``` +## Important constraints and sharp edges -## Converting Standalone Script Data to API Format +- The organism, sample, and experiment bulk endpoints do not share one uniform payload shape. +- Several bulk paths still contain compatibility logic for older field names or older caller behavior. +- Some bulk paths emit partial diagnostics through response `errors`; some older code paths also still use `print(...)` for local debugging. -If you have data in a different format, you may need to transform it to match the expected format for these endpoints. The format should match the JSON files used by the standalone import script: +## Unknown From This Repo -- `data/unique_organisms.json` -- `data/unique_samples.json` -- `data/experiments.json` +- There are no canonical example input files in the tracked repository proving the exact upstream export format for every current caller. +- The repo does not document who owns the promotion from imported `draft` submission rows to `ready`. diff --git a/docs/handover/README.md b/docs/handover/README.md new file mode 100644 index 0000000..37f8d8b --- /dev/null +++ b/docs/handover/README.md @@ -0,0 +1,34 @@ +# Canopy Handover Pack + +## Verified + +This handover pack was written from the current repository state, with the code, tests, migrations, scripts, and selected current docs treated as the source of truth. + +Use these files in this order: + +1. `doc_audit.md` +2. `system_overview.md` +3. `setup_and_operations.md` +4. `broker_and_submission_flows.md` +5. `config_reference.md` +6. `troubleshooting.md` +7. `open_questions.md` + +## Proposed Documentation Structure + +| File | Purpose | +| --- | --- | +| `docs/handover/README.md` | Entry point for the handover pack | +| `docs/handover/doc_audit.md` | Audit of existing docs, README, Postman, and generated notes | +| `docs/handover/system_overview.md` | What the system does, repo structure, key entities, main lifecycle rules | +| `docs/handover/setup_and_operations.md` | Local setup, Docker flow, Alembic flow, scripts, recurring operator tasks | +| `docs/handover/broker_and_submission_flows.md` | Submission table lifecycle, broker claims/reports, ToLID flow, assembly reporting handoff points | +| `docs/handover/config_reference.md` | Environment variables and config behavior derived from code | +| `docs/handover/troubleshooting.md` | Maintainer symptom-driven troubleshooting guide | +| `docs/handover/open_questions.md` | Questions for the human owner and tribal knowledge gaps not captured in the repo | + +## Unknown From This Repo + +- Which broker endpoints are used in production versus retained only for backward compatibility. +- Which human or external process promotes submission rows from `draft` to `ready`. +- The production runtime topology behind the GitHub Actions deployment automation. diff --git a/docs/handover/broker_and_submission_flows.md b/docs/handover/broker_and_submission_flows.md new file mode 100644 index 0000000..a0b9d21 --- /dev/null +++ b/docs/handover/broker_and_submission_flows.md @@ -0,0 +1,88 @@ +# Broker And Submission Flows + +## Verified + +### Active broker API surfaces + +The repo currently exposes two broker surfaces in the same router: + +#### Flat contract endpoints + +- `POST /api/v1/broker/claims/ready` +- `POST /api/v1/broker/claims/entity` +- `POST /api/v1/broker/claims/batch` +- `POST /api/v1/broker/validation` +- `POST /api/v1/broker/reports/{attempt_id}` + +These use `Broker*` request/response schemas from `app/schemas/broker_contract.py`. + +#### Legacy claim/report endpoints that are still routed + +- `POST /api/v1/broker/claim` +- `POST /api/v1/broker/organisms/{taxon_id}/claim` +- `POST /api/v1/broker/attempts/{attempt_id}/lease/renew` +- `POST /api/v1/broker/attempts/{attempt_id}/finalise` +- `POST /api/v1/broker/attempts/{attempt_id}/report` +- `GET /api/v1/broker/attempts` +- `GET /api/v1/broker/attempts/{attempt_id}` +- `GET /api/v1/broker/attempts/{attempt_id}/items` +- `GET /api/v1/broker/organisms/{taxon_id}/summary` + +### Submission states and attempt leasing + +- `SubmissionAttempt` rows track claim attempts and lease expiry (`app/models/broker.py`). +- The flat contract can claim submissions in either `draft` or `ready` because `CLAIMABLE_SUBMISSION_STATES = ("draft", "ready")` in `app/api/v1/endpoints/broker.py`. +- The legacy claim endpoints only select `draft` rows in their query bodies. +- Claimed rows move to `submitting`, receive `attempt_id`, `lock_acquired_at`, and `lock_expires_at`, and emit `submission_event` rows with `action="claimed"`. +- Lease renewal extends the attempt lease and propagates the new expiry to still-`submitting` rows. +- Finalising an attempt releases remaining `submitting` rows back to `draft` and marks the attempt `complete`. +- Expiring stale leases resets expired `submitting` rows back to `draft` and marks expired attempts as `expired`. + +### Broker report behavior + +- `POST /api/v1/broker/reports/{attempt_id}` updates claim items using the flat contract. +- `_map_report_state_to_submission_status()` maps: + - `completed`, `accepted`, `success` -> `accepted` + - `failed`, `rejected`, `error` -> `rejected` + - `submitting`, `processing` -> `submitting` +- Accepted reports write `accession` to the submission row and attempt to insert into `accession_registry`. +- Sample reports can also store `secondary_accession` into `biosample_accession`. +- Rejected reports create a new draft submission row for the same entity so the entity becomes claimable again. +- A database integrity conflict during accession registration is translated into HTTP `409` with a message pointing maintainers to `accession_registry`. + +### Submission prerequisites and validation + +- The broker flat contract resolves existing prerequisite accessions from `accession_registry`, not only from payload fields (`_extract_broker_prerequisites()` in `app/api/v1/endpoints/broker.py`). +- Validation rules currently enforced by `POST /api/v1/broker/validation` are: + - sample: payload must exist + - experiment: payload, `sample_accession`, and `project_accession` must exist + - run: payload, `experiment_accession`, and run file metadata must exist +- Project claims do not include prerequisites. + +### ToLID flow + +- The ToLID flow does not use broker lease semantics. +- `GET /api/v1/broker/tolids/by-specimen-accession/{specimen_id}` resolves a specimen sample from an accepted sample accession and returns either: + - stored `tolid_request` state + - virtual `not_requested` state if no row exists yet +- `POST /api/v1/broker/tolids/{sample_id}/report` lazily creates a `tolid_request` row when needed. +- Allowed reported statuses are `pending`, `assigned`, and `failed`; `not_requested` is rejected by schema validation (`app/schemas/tolid.py`). +- Reporting `assigned` also mirrors the ToLID value onto `sample.tolid`. + +### Submission lifecycle rules outside broker.py + +- Organism creation creates draft `project_submission` rows immediately. +- Sample creation creates one draft `sample_submission`. +- Experiment creation creates one draft `experiment_submission`. +- Assembly QC reporting creates one draft `qc_read_submission`. +- Sample and experiment edits can create replacement draft submission rows as described in `system_overview.md`. + +## Inferences + +- The flat broker contract appears to be the newer interface and the legacy claim/report endpoints appear to be retained for compatibility, because both sets are active and the newer set uses dedicated broker-contract schemas. + +## Unknown From This Repo + +- Which broker surface is used by current production callers. +- Who or what changes submission rows from `draft` to `ready`. +- Whether `ready` means human-reviewed, machine-validated, or both. diff --git a/docs/handover/config_reference.md b/docs/handover/config_reference.md new file mode 100644 index 0000000..f74cea4 --- /dev/null +++ b/docs/handover/config_reference.md @@ -0,0 +1,62 @@ +# Config Reference + +## Verified + +### Application and entrypoint settings + +| Name | Source | Required | Verified behavior | +| --- | --- | --- | --- | +| `PROJECT_NAME` | `app/core/settings.py` | no | FastAPI title; default `atol-canopy` | +| `API_V1_STR` | `app/core/settings.py` | no | API prefix and docs prefix; default `/api/v1` | +| `DEBUG` | `app/core/settings.py` | no | Present in settings model; no direct behavior found outside settings object | +| `ENVIRONMENT` | `app/core/settings.py`, `app/main.py`, `scripts/entrypoint.sh` | effectively yes for predictable behavior | `dev` enables debug logging and `uvicorn --reload`; non-`prod` with no explicit CORS origins defaults to `["*"]` | +| `APP_VERSION` | `app/core/settings.py`, `Dockerfile`, `app/main.py` | no | Returned by `GET /version`; default `dev`; Docker build can inject a release value | +| `PORT` | `scripts/entrypoint.sh`, `Dockerfile` | no | Runtime listen port for the entrypoint; default `8000` | +| `APP_MODE` | `scripts/entrypoint.sh` | no | `serve` starts API; `migrate` runs migrations then exits | + +### Database settings + +| Name | Source | Required | Verified behavior | +| --- | --- | --- | --- | +| `DATABASE_URI` | `app/core/settings.py`, `scripts/entrypoint.sh`, `alembic/env.py` | yes for entrypoint; yes or derivable for app import | Used by SQLAlchemy engine and Alembic; entrypoint exits if it is unset | +| `POSTGRES_SERVER` | `app/core/settings.py` | required if deriving `DATABASE_URI` | Used only to derive `DATABASE_URI` inside settings | +| `POSTGRES_USER` | `app/core/settings.py` | required if deriving `DATABASE_URI` | Used only to derive `DATABASE_URI` inside settings | +| `POSTGRES_PASSWORD` | `app/core/settings.py` | required if deriving `DATABASE_URI` | Used only to derive `DATABASE_URI` inside settings | +| `POSTGRES_DB` | `app/core/settings.py` | required if deriving `DATABASE_URI` | Used only to derive `DATABASE_URI` inside settings | +| `POSTGRES_PORT` | `app/core/settings.py` | required if deriving `DATABASE_URI` | Used only to derive `DATABASE_URI` inside settings | + +### Auth and token settings + +| Name | Source | Required | Verified behavior | +| --- | --- | --- | --- | +| `JWT_SECRET_KEY` | `app/core/settings.py`, `app/core/security.py`, `app/core/dependencies.py` | yes | Required at startup; used to sign and verify access tokens | +| `JWT_ALGORITHM` | `app/core/settings.py`, `app/core/security.py`, `app/core/dependencies.py` | yes | Required at startup; used for JWT encode/decode | +| `JWT_ACCESS_TOKEN_EXPIRE_MINUTES` | `app/core/settings.py`, `app/api/v1/endpoints/auth.py` | no | Access-token lifetime in minutes; default `150` | +| `JWT_REFRESH_TOKEN_EXPIRE_DAYS` | `app/core/settings.py`, `app/api/v1/endpoints/auth.py` | no | Refresh-token lifetime in days; default `7` | + +### CORS settings + +| Name | Source | Required | Verified behavior | +| --- | --- | --- | --- | +| `BACKEND_CORS_ORIGINS` | `app/core/settings.py`, `app/main.py` | no | If set, FastAPI installs CORS middleware with those origins; if unset and `ENVIRONMENT != "prod"`, settings default it to `["*"]`; `["*"]` is rejected when `ENVIRONMENT == "prod"` | + +### Repo automation secrets and workflow vars + +These are not application settings, but they are verified in GitHub workflow files: + +| Name | Source | Verified behavior | +| --- | --- | --- | +| `AWS_ROLE_ECR_PUSH` | `.github/workflows/build-and-deploy-dev.yml`, `build-publish.yml` | GitHub Actions assumes this AWS role before pushing images | +| `GITHUB_TOKEN` | `.github/workflows/release-please.yml` | Used by release-please | +| `AWS_REGION` | workflow env | Set to `ap-southeast-2` in the repo workflows | +| `IMAGE_REPO` | workflow env | ECR repository used by build jobs | +| `DEPLOY_FUNCTION_NAME` | workflow env | Lambda function invoked by the dev deployment workflow | + +## Inferences + +- `BACKEND_CORS_ORIGINS` is expected to be supplied in a Pydantic-compatible list form such as a JSON array string, because it is typed as `List[str]` in `Settings`. + +## Unknown From This Repo + +- The real production values for any of these settings. +- Whether there are additional deployment-time environment variables injected outside the repository. diff --git a/docs/handover/doc_audit.md b/docs/handover/doc_audit.md new file mode 100644 index 0000000..582668b --- /dev/null +++ b/docs/handover/doc_audit.md @@ -0,0 +1,31 @@ +# Documentation Audit + +## Verified + +| Document or file | Status | Evidence | Action needed | +| --- | --- | --- | --- | +| `README.md` | stale | References `xml_export.py` and `read_submissions.py`, but those routes are not registered in `app/api/v1/api.py`; says Docker uses `schema.sql` bootstrap and no reload, while `scripts/entrypoint.sh` runs `alembic upgrade head` and starts `uvicorn --reload` when `ENVIRONMENT=dev` | Keep only as a verified pointer to the handover pack | +| `docs/assembly_reporting_api.md` | verified current | Matches active assembly intent, run, QC-report, and stage-run endpoints in `app/api/v1/endpoints/assemblies.py`; supported by `tests/unit/endpoints/test_endpoints_assemblies.py` and `tests/unit/services/test_assembly_helper.py` | Keep; continue treating as the current assembly-specific API doc | +| `docs/auth_refresh_tokens.md` | verified current | Rewritten from `app/api/v1/endpoints/auth.py`, `app/core/security.py`, `app/core/dependencies.py`, `app/models/token.py`, and auth endpoint tests | Keep | +| `docs/broker_prerequisites_enhancement.md` | partially current | Accession lookup behavior matches `_extract_broker_prerequisites()` in `app/api/v1/endpoints/broker.py`; document discusses `validation_hints` and `file_metadata`, but current response schema exposes `files`, and code does not populate `file_metadata` | Do not use as operator guidance without rewriting | +| `docs/bulk_import_api.md` | verified current | Rewritten from current organism, sample, experiment, and taxonomy bulk-import code paths | Keep | +| `docs/migration_workflow.md` | verified current | Rewritten to cover only Alembic, `schema.sql`, Docker entrypoint behavior, and active revision files evidenced in the repo | Keep | +| `docs/ncbi_taxonomy_sync.md` | verified current | Matches canonical scientific-name recomputation in `app/services/organism_service.py` and NCBI enrichment behavior in `app/services/taxonomy_info_service.py`; backed by `tests/unit/services/test_taxonomy_info_service.py` | Keep | +| `docs/tolid_broker_api.md` | verified current | Matches `app/services/tolid_service.py`, `app/schemas/tolid.py`, and ToLID routes in `app/api/v1/endpoints/broker.py`; backed by `tests/unit/endpoints/test_endpoints_broker_tolids.py` | Keep | +| `docs/xml_export_api.md` | stale | No `xml-export` router is included in `app/api/v1/api.py`, and there is no active `app/api/v1/endpoints/xml_export.py` file | Removed from the repo during handover cleanup | +| `postman/Canopy.postman_collection.json` | stale | Contains outdated paths and examples such as organism lookup by grouping key; includes embedded example credentials/tokens; current router surface is different in `app/api/v1/api.py` | Regenerate a minimal core collection from the current API surface | +| `data/docs/MIGRATION_MERGE_INSTRUCTIONS.md` | stale | Refers to migration chain ending at `0009...` and files that are no longer current; active Alembic head is `0005_add_tolid_requests.py` under the current chain | Treat as historical notes only | +| `data/docs/MIGRATION_NOTES.md` | stale | Refers to `0010_rename_base_url_to_bioplatforms_base_url.py`, which is not in the active migration chain | Treat as historical notes only | +| `data/docs/PR_QC_READ_SUBMISSION.md` | stale | Mentions `POST /qc-callbacks` and inactive `read_submissions.py`; current QC reporting route is `POST /api/v1/assemblies/{assembly_id}/qc-reads/report` | Treat as historical PR commentary only | +| `data/docs/SCHEMA_COMPARISON_REPORT.md` | stale | Describes schema mismatches relative to an older `schema3.sql` dump that is not present as current source of truth | Treat as historical notes only | +| `data/docs/SCHEMA_UPDATE_SUMMARY.md` | stale | Refers to removed `read_submission` decisions and older QC-read schema shape; current schema and migrations have moved on | Treat as historical notes only | +| `data/docs/SCHEMA_VERIFICATION.md` | stale | Claims a structural state from 2026-04-30 that predates current assembly and ToLID migrations | Treat as historical notes only | +| `docs/handover/*.md` | missing before this update | No maintainer-oriented handover pack existed in the repo | Created in this update | + +## Inferences + +- The `data/docs/` files appear to be temporary change notes rather than maintained operator documentation, because they describe one-off migration merges, PR summaries, and schema comparisons rather than current runtime behavior. + +## Unknown From This Repo + +- Whether the stale docs should be deleted, archived elsewhere, or retained for audit history. diff --git a/docs/handover/open_questions.md b/docs/handover/open_questions.md new file mode 100644 index 0000000..2014d0b --- /dev/null +++ b/docs/handover/open_questions.md @@ -0,0 +1,30 @@ +# Open Questions And Tribal Knowledge Gaps + +## Unresolved Questions For The Human Owner + +1. Which broker API surface is actually used in production today: the flat contract (`/broker/claims/*`, `/broker/reports/{attempt_id}`) or the older organism/attempt endpoints? +2. What process promotes submission rows from `draft` to `ready`? I could not verify any code path in this repo that sets `status = "ready"` for project, sample, experiment, or QC-read submissions. +3. Which payload shapes are treated as canonical for bulk organism, sample, and experiment imports outside the request validation implied by this repo? +4. Is `POST /api/v1/assemblies/from-experiments/{taxon_id}` still an active workflow, or has the assembly intent flow replaced it operationally? +5. Should stale historical documents under `data/docs/` remain in the repo, or should they be archived elsewhere to avoid confusing maintainers? +6. What are the expected roles and minimum permissions for day-to-day maintainer accounts versus broker accounts versus genome-launcher accounts? +7. Are there manual post-deploy checks, rollback steps, or data-backup requirements that happen outside the GitHub Actions workflows? + +## High-Risk Tribal Knowledge Not Present In The Repo + +### Verified gaps + +- Production deployment topology is not documented. The repo shows ECR image publishing and Lambda-triggered deployment automation, but not the runtime environment or rollback procedure. +- Broker client behavior is not documented. The repo exposes broker contracts, but not the external worker implementation, retry policy, or release cadence. +- ToLID remote-service details are not documented. The repo stores ToLID state and exposes report endpoints, but the external request payload, retry rules, and error taxonomy are absent. +- Data-ingest source contracts are not documented. The repo contains code that accepts bulk organism/sample/experiment dictionaries, but not the upstream ownership or canonical sample files. +- The governance around `ready` status is absent. This is operationally important because broker claim behavior differs between the older and newer broker endpoints. + +### Inferences + +- Maintainers probably rely on conventions outside this repo for deciding when staged submission payloads are safe to send to the broker, because the code models `ready` but does not create it. + +### Unknown From This Repo + +- Which stale routes, docs, or backward-compatibility shims can be safely removed without breaking external callers. +- Whether production operators depend on direct database edits for emergency recovery. diff --git a/docs/handover/setup_and_operations.md b/docs/handover/setup_and_operations.md new file mode 100644 index 0000000..adbe49f --- /dev/null +++ b/docs/handover/setup_and_operations.md @@ -0,0 +1,74 @@ +# Local Setup And Operations + +## Verified + +### Container startup path + +- `docker-compose.yml` defines two services: + - `api` on host port `8000` + - `db` on host port `5433`, container port `5432` +- The API container runs `scripts/entrypoint.sh` from `Dockerfile`. +- `scripts/entrypoint.sh` does the following: + 1. requires `DATABASE_URI` to be present in the environment + 2. waits for the database to answer `SELECT 1` + 3. runs `uv run alembic upgrade head` + 4. starts `uvicorn` + 5. enables `--reload` only when `ENVIRONMENT=dev` + +### Local Docker workflow + +1. Copy `.env.example` to `.env`. +2. Set at least the JWT variables and Postgres variables needed by the app. +3. Start the stack with `docker compose up --build`. +4. API docs are served from `/api/v1/docs`, `/api/v1/redoc`, and `/api/v1/openapi.json`. + +### Non-Docker workflow + +The repo contains enough evidence for this local path: + +1. Install dependencies with `uv sync --dev --frozen` (`Dockerfile`, `.github/workflows/lint.yml`). +2. Export the required environment variables described in `app/core/settings.py`. +3. Run migrations with `uv run alembic upgrade head`. +4. Start the API with `uv run uvicorn app.main:app --host 0.0.0.0 --port 8000 --reload`. + +### First-user creation + +- `scripts/create_user.py` creates a user directly in the database and hashes the password with the same helper used by the API. +- The script accepts either `--db-uri` or host/port/name/user/password fields. +- The script sets `roles=[role]`. +- The script sets `is_superuser=True` only when `--role superuser`. + +### Recurring maintainer tasks visible in the repo + +| Task | Verified implementation | +| --- | --- | +| Run tests | `pytest -q` passes in the current repo state | +| Apply schema changes | `uv run alembic upgrade head` | +| Expire stale broker leases from code | `scripts/expire_leases.py` | +| Expire stale broker leases via API | `POST /api/v1/admin/leases/expire` or `POST /api/v1/broker/leases/expire` | +| Check liveness | `GET /health` | +| Check app version | `GET /version` | +| Regenerate schema snapshot | Comment in `schema.sql` says to use `pg_dump --schema-only` after migrations | + +### Deployment automation evidenced in the repo + +- `.github/workflows/build-and-deploy-dev.yml` builds a container image, pushes it to ECR, and invokes a Lambda deployment function. +- `.github/workflows/build-publish.yml` builds and pushes release-tagged images to ECR. +- These workflows prove that the repo is wired to AWS-hosted automation, but they do not describe the runtime environment behind the deployment target. + +### Operational sharp edges visible in code + +- `scripts/entrypoint.sh` requires a literal `DATABASE_URI` shell variable even though `app/core/settings.py` can derive `DATABASE_URI` from `POSTGRES_*` values. This means `POSTGRES_*` values alone are enough for direct Python startup but not enough for the entrypoint script. +- `scripts/create_user.py` appends a hard-coded repository path to `sys.path`. That makes the script repo-location-sensitive. +- There are two lease-expiry implementations: + - `app/services/broker_service.py` + - `expire_stale_leases()` inside `app/api/v1/endpoints/broker.py` + +## Inferences + +- `schema.sql` is a maintained schema snapshot rather than the runtime bootstrap path, because container startup applies Alembic migrations and does not execute `schema.sql` directly. + +## Unknown From This Repo + +- The exact production release checklist after a successful GitHub Actions deployment. +- Whether there are environment-specific manual steps around database backups, smoke tests, or rollback outside the repository. diff --git a/docs/handover/system_overview.md b/docs/handover/system_overview.md new file mode 100644 index 0000000..2e738f5 --- /dev/null +++ b/docs/handover/system_overview.md @@ -0,0 +1,90 @@ +# System Overview + +## Verified + +### What the application does + +- The application is a FastAPI backend with a versioned API under `/api/v1` (`app/main.py`, `app/api/v1/api.py`). +- It stores biological metadata and submission-preparation metadata for these main areas: + - organisms and taxonomy enrichment + - samples and sample lineage + - experiments and raw reads + - QC-read results + - assemblies, assembly runs, and assembly stage runs + - projects, BPA initiatives, genome notes, and users + - broker submission attempts, events, and ToLID request state +- Submission-facing entities use a main-table plus submission-table pattern for at least projects, samples, experiments, QC reads, and assemblies (`app/models/project.py`, `sample.py`, `experiment.py`, `qc_read.py`, `assembly.py`). + +### Codebase structure + +| Path | Verified role | +| --- | --- | +| `app/main.py` | FastAPI app creation, CORS setup, exception handlers, root, health, version | +| `app/api/v1/api.py` | Registers all active routers | +| `app/api/v1/endpoints/` | Route handlers and some business rules | +| `app/models/` | SQLAlchemy models for tables and relationships | +| `app/schemas/` | Pydantic request/response models and enums | +| `app/services/` | Shared business logic for organism, experiment, taxonomy, assembly, broker-adjacent helpers, and ToLID state | +| `app/core/` | settings, auth dependencies, password/JWT helpers, pagination, policy wrapper, app error type | +| `app/config/ena-atol-map.json` | ENA payload field mapping used by sample and experiment payload generation | +| `alembic/` | Schema migration entrypoint and revisions | +| `scripts/` | Container startup, lease-expiry helper, and user-creation script | +| `tests/` | Unit tests for routes, services, settings, security, and broker behavior | + +### Entity and lifecycle patterns + +#### Organisms + +- Creating an organism also creates two `project` rows, one `root` and one `genomic_data`, plus matching `project_submission` draft rows (`app/services/organism_service.py`). +- `organism.scientific_name` is treated as the app-facing canonical name and is recomputed from `taxonomy_info.ncbi_scientific_name` when that exists; otherwise it falls back to `organism.bpa_scientific_name` (`app/services/organism_service.py`, `app/services/taxonomy_info_service.py`). + +#### Samples + +- Samples support `specimen` and `derived` kinds (`app/schemas/common.py`, `app/models/sample.py`). +- Derived samples must point to a specimen parent in the same `taxon_id`; specimen samples cannot have a parent (`app/api/v1/endpoints/samples.py`). +- Creating a sample also creates a draft `sample_submission` linked to the organism’s `genomic_data` project (`app/api/v1/endpoints/samples.py`). + +#### Experiments and reads + +- Creating an experiment requires an existing sample and the sample’s `genomic_data` project; it also creates a draft `experiment_submission` with `prepared_payload` built from `app/config/ena-atol-map.json` (`app/services/experiment_service.py`). +- Bulk experiment import also creates `read` rows from nested `runs` payloads (`app/services/experiment_service.py`). + +#### Taxonomy enrichment + +- Taxonomy-info create, bulk-import, bulk-upsert, and bulk-refresh all call NCBI lookup code in `app/services/ncbi_taxonomy_service.py` through `app/services/taxonomy_info_service.py`. +- Successful NCBI enrichment updates `taxonomy_info.ncbi_last_synced_at`, recomputes `organism.scientific_name`, and refreshes draft/ready project submission payloads for that organism (`app/services/taxonomy_info_service.py`, `app/services/organism_service.py`). + +#### Assemblies + +- There are two assembly creation flows: + - generic `POST /api/v1/assemblies/` + - intent-driven `POST /api/v1/assemblies/intent/{taxon_id}` +- The intent flow validates specimen samples, resolves lineage-linked experiments and reads, generates a manifest JSON, stores it on the `assembly`, and returns `assembly_id`, `version`, and the manifest (`app/api/v1/endpoints/assemblies.py`, `app/services/assembly_helper.py`). +- Assembly runs record pipeline invocations by `github_repo` plus `git_commit`; assembly stage runs store one record per stage name per run (`app/models/assembly.py`, `app/services/assembly_service.py`). +- QC read reporting for assemblies creates `qc_read`, `qc_read_file`, `qc_read_assembly`, and a draft `qc_read_submission` (`app/api/v1/endpoints/assemblies.py`, `app/api/v1/endpoints/qc_reads.py`). + +#### Submission tables and status changes + +- Create paths generally produce `draft` submission rows. +- Sample and experiment updates have status-aware behavior: + - `submitting`: update blocked + - `draft` or `ready`: existing submission row updated in place and reset to `draft` + - `accepted`: existing submission row marked `replaced`, then a new `draft` row is created preserving accession fields + - `rejected` or `replaced`: a new `draft` row is created preserving accession fields where present + (`app/api/v1/endpoints/samples.py`, `app/services/experiment_service.py`) + +### External interaction points visible in code + +- Broker-facing submission APIs live in `app/api/v1/endpoints/broker.py`. +- ToLID durable state is exposed through broker routes but handled by `app/services/tolid_service.py`. +- NCBI taxonomy lookup is the only outbound HTTP integration directly evidenced in application code (`app/services/ncbi_taxonomy_service.py`). + +## Inferences + +- Canopy appears to be a metadata store and broker-facing coordination service rather than the component that directly submits all records outward, because the repo contains claim/report APIs for an external broker and no general ENA submission client implementation outside payload preparation and broker result handling. + +## Unknown From This Repo + +- Which external system or human process decides that a project/sample/experiment/QC-read submission is `ready`. +- Which client systems are authoritative producers for organism, sample, and experiment bulk-import payloads. +- Whether `POST /api/v1/assemblies/from-experiments/{taxon_id}` is still actively used now that the assembly intent flow exists. diff --git a/docs/handover/troubleshooting.md b/docs/handover/troubleshooting.md new file mode 100644 index 0000000..9aa9698 --- /dev/null +++ b/docs/handover/troubleshooting.md @@ -0,0 +1,27 @@ +# Troubleshooting Guide + +## Verified + +| Symptom | Likely component | Files, endpoints, or scripts to inspect | Verified next checks | +| --- | --- | --- | --- | +| API container exits before serving requests | Startup config or DB connectivity | `scripts/entrypoint.sh`, `docker-compose.yml`, `app/core/settings.py` | Confirm `DATABASE_URI` is set in the shell environment seen by the entrypoint, not just derivable from `POSTGRES_*`; confirm DB answers `SELECT 1`; confirm Alembic can reach the same URI | +| App import fails with a settings error | Missing JWT or DB settings | `app/core/settings.py` | Check `JWT_SECRET_KEY`, `JWT_ALGORITHM`, and either `DATABASE_URI` or all `POSTGRES_*`; in `prod`, confirm `BACKEND_CORS_ORIGINS` is not `["*"]` | +| Login works but refresh fails with `401 Invalid or expired refresh token` | Refresh-token lookup or revocation | `app/api/v1/endpoints/auth.py`, `app/models/token.py` | Check whether the token hash exists in `refresh_token`, whether `expires_at` is still in the future, and whether `revoked` is already true | +| Sample or experiment update is rejected while broker work is in progress | Submission lock state | `app/api/v1/endpoints/samples.py`, `app/services/experiment_service.py`, broker attempt routes | Check the latest submission row status; both code paths block edits when the latest submission is `submitting` | +| Broker cannot claim work that maintainers expect to be available | Submission status, claim surface, or stale lease | `app/api/v1/endpoints/broker.py`, `scripts/expire_leases.py`, `POST /api/v1/admin/leases/expire`, `POST /api/v1/broker/leases/expire` | Check whether rows are still `draft` versus `ready`; flat broker claims accept `draft` and `ready`, legacy organism claim accepts only `draft`; check `lock_expires_at`; expire stale leases if needed | +| Broker report returns `409` mentioning integrity constraints | Accession registry conflict | `app/api/v1/endpoints/broker.py`, `app/models/accession_registry.py` | Inspect `accession_registry` for an existing row with the same accession bound to a different entity; inspect the affected submission row’s accession FK fields | +| ToLID lookup by accession returns `404` | Sample accession resolution or sample kind | `app/services/tolid_service.py`, `app/models/tolid_request.py`, `GET /api/v1/broker/tolids/...` | Confirm there is an accepted `sample_submission.accession` or `sample.biosample_accession`; confirm the resolved sample has `kind = specimen` | +| ToLID report cannot create a row | Missing accepted sample accession | `app/services/tolid_service.py` | Check whether `_fallback_external_id()` can find a sample submission accession or `sample.biosample_accession`; without one, reporting returns `tolid_external_id_missing` | +| Assembly intent returns `422` for specimen samples | Sample lineage or experiment-platform validation | `app/api/v1/endpoints/assemblies.py`, `app/services/assembly_helper.py` | Confirm each supplied sample exists, is `kind='specimen'`, and matches the route `taxon_id`; confirm long-read experiments exist with platform `PACBIO_SMRT` or `OXFORD_NANOPORE`; confirm Hi-C experiments are `ILLUMINA` plus library strategy `HI-C` | +| Assembly QC read reporting returns `422` | Manifest mismatch or lineage mismatch | `POST /api/v1/assemblies/{assembly_id}/qc-reads/report`, `app/api/v1/endpoints/assemblies.py` | Confirm `bpa_package_id` resolves to an experiment; confirm the experiment sample belongs to the assembly’s allowed specimen lineage; confirm the package ID exists in `assembly.manifest_json["read_files"]`; confirm the supplied source MD5 values match `read.file_checksum` rows for that experiment | +| Genome note publish returns `409` | Existing published note for the same organism | `app/services/genome_note_service.py`, `app/api/v1/endpoints/genome_notes.py` | Check whether another `genome_note` row for the same `taxon_id` already has `is_published = true` | +| Taxonomy bulk import skips rows unexpectedly | Missing organism rows or unmapped NCBI enrichment | `app/services/taxonomy_info_service.py`, `app/services/ncbi_taxonomy_service.py` | Check whether each `taxon_id` already exists in `organism`; inspect `errors`, `ncbi_retryable_count`, and `ncbi_retryable_taxon_ids` in the bulk response | +| User-creation script fails in an unexpected checkout location | Hard-coded import path in script | `scripts/create_user.py` | Check the `sys.path.append("/Users/emilylm/Repositories/atol-database-v2")` line and adjust it if the repo is moved | + +## Verified Sharp Edges And Technical Debt + +- There are two different lease-expiry implementations with overlapping responsibilities. +- Broker APIs are duplicated across legacy and newer contract routes. +- Submission status `ready` is part of the API contract, but this repo does not show who sets it. +- Several code comments and docstrings are stale. One example is the PacBio filtering note in `app/services/assembly_helper.py`, while tests in `tests/unit/services/test_assembly_helper.py` assert that all PacBio reads are currently included when they have `file_name`. +- The sample endpoint contains several `print(...)` debugging paths instead of structured logging. diff --git a/docs/migration_workflow.md b/docs/migration_workflow.md index 465b950..48b90d7 100644 --- a/docs/migration_workflow.md +++ b/docs/migration_workflow.md @@ -1,462 +1,83 @@ +> **Status:** Verified current only for repository-local migration and schema maintenance steps. This file does not claim any production deployment procedure beyond what is visible in the repo. -## Schema files +# Migration Workflow -- `alembic/base_schema/0001_initial_schema.sql` is the frozen bootstrap schema used only by `0001_initial_schema.py`. -- Do not edit that file after it has been established, unless you are intentionally rewriting migration history. -- `schema.sql` is the mutable current schema snapshot for the present database shape. -- New migrations should update the database schema and then `schema.sql` can be refreshed to match the current state. +This document describes the migration workflow that is directly evidenced in the current repository. ---- +## Verified -```bash -# Check current migration version -docker compose exec api alembic current - -# Rollback one migration -docker compose exec api alembic downgrade -1 - -# Rollback to specific version -docker compose exec api alembic downgrade - -# Restore from backup (last resort) -docker compose exec db psql -U postgres -d atol_db < backup.sql -``` - ---- - -## Traditional Server Deployment (Non-Docker Production) - -For production environments using VMs, bare metal servers, or Kubernetes (without Docker Compose). - -### Setup Requirements - -1. **Python environment** - ```bash - # Install Python 3.12+ and create virtual environment - python3 -m venv venv - source venv/bin/activate - pip install -r requirements.txt # or use uv - ``` - -2. **Environment variables** - ```bash - # Set database connection in .env or environment - export DATABASE_URI="postgresql://user:password@db-host:5432/atol_db" - ``` - -3. **Alembic configuration** - - Ensure `alembic.ini` points to correct database - - Or use environment variable: `ALEMBIC_CONFIG` if needed - -### Pre-deployment Checklist - -Same as Docker-based deployment, plus: - -1. **Verify database connectivity** - ```bash - # Test connection from application server - psql -h db-host -U postgres -d atol_db -c "SELECT version();" - ``` - -2. **Check migration files are deployed** - ```bash - ls -la alembic/versions/ - # Ensure new migration files are present - ``` - -3. **Verify Alembic can connect** - ```bash - alembic current - # Should show current migration version - ``` - -### Deployment Steps - -#### Option 1: Maintenance Window - -1. **Backup database** - ```bash - # From database server or application server with access - pg_dump -h db-host -U postgres -d atol_db -F c -f backup_$(date +%Y%m%d_%H%M%S).dump - - # Or SQL format - pg_dump -h db-host -U postgres -d atol_db > backup_$(date +%Y%m%d_%H%M%S).sql - ``` - -2. **Stop application** - ```bash - # Systemd - sudo systemctl stop atol-api - - # Or supervisor - sudo supervisorctl stop atol-api +### Migration framework - # Or Kubernetes - kubectl scale deployment atol-api --replicas=0 - ``` +- Alembic is configured in `alembic/env.py`. +- Active revisions are under `alembic/versions/`. +- Container startup runs `uv run alembic upgrade head` before serving the API (`scripts/entrypoint.sh`). -3. **Deploy new code** - ```bash - # Pull latest code - git pull origin main +### Current revision chain - # Install dependencies if changed - pip install -r requirements.txt - ``` +The current active revision sequence in `alembic/versions/` is: -4. **Run migration** - ```bash - # Activate virtual environment - source venv/bin/activate +1. `0001_initial_schema` +2. `0002_update_taxon_cols` +3. `0003_assembly_run_github` +4. `0004_qc_reads_assembly_refs` +5. `0005_add_tolid_requests` - # Run migration - alembic upgrade head +### Local developer workflow - # Verify - alembic current - ``` - -5. **Start application** - ```bash - # Systemd - sudo systemctl start atol-api - - # Or supervisor - sudo supervisorctl start atol-api - - # Or Kubernetes - kubectl scale deployment atol-api --replicas=3 - ``` - -6. **Monitor** - ```bash - # Check logs - sudo journalctl -u atol-api -f - - # Or supervisor - sudo tail -f /var/log/atol-api/error.log - - # Or Kubernetes - kubectl logs -f deployment/atol-api - ``` - -#### Option 2: Zero-Downtime (Rolling Deployment) - -1. **Ensure migration is backward-compatible** - - Add columns as nullable - - Don't drop columns yet - - Application code works with both old and new schema - -2. **Apply migration (while app is running)** - ```bash - # SSH to application server or use CI/CD - source venv/bin/activate - alembic upgrade head - ``` - -3. **Deploy new application code (rolling)** - ```bash - # Kubernetes rolling update - kubectl set image deployment/atol-api api=atol-api:v2.0 - kubectl rollout status deployment/atol-api - - # Or manual rolling restart with load balancer - # Update server 1, wait, update server 2, etc. - ``` - -4. **Verify deployment** - ```bash - # Check health endpoint - curl https://api.example.com/health - - # Monitor logs across all instances - kubectl logs -l app=atol-api --tail=100 - ``` - -### Rollback Procedure (Non-Docker) +1. Ensure the app can resolve `DATABASE_URI`. +2. Apply migrations: ```bash -# Check current version -alembic current - -# Rollback one migration -alembic downgrade -1 - -# Rollback to specific version -alembic downgrade - -# Restore from backup (last resort) -pg_restore -h db-host -U postgres -d atol_db -c backup.dump -# Or for SQL format: -psql -h db-host -U postgres -d atol_db < backup.sql - -# Restart application -sudo systemctl restart atol-api -``` - -### CI/CD Integration - -#### GitHub Actions Example - -```yaml -# .github/workflows/deploy.yml -name: Deploy to Production - -on: - push: - branches: [main] - -jobs: - deploy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.12' - - - name: Install dependencies - run: | - pip install -r requirements.txt - - - name: Run migrations - env: - DATABASE_URI: ${{ secrets.DATABASE_URI }} - run: | - alembic upgrade head - - - name: Deploy application - run: | - # Your deployment script - ./deploy.sh +uv run alembic upgrade head ``` -#### GitLab CI Example - -```yaml -# .gitlab-ci.yml -stages: - - migrate - - deploy +3. Run the test suite: -migrate: - stage: migrate - script: - - pip install -r requirements.txt - - alembic upgrade head - only: - - main - -deploy: - stage: deploy - script: - - ./deploy.sh - only: - - main +```bash +pytest -q ``` -### Kubernetes-Specific Considerations - -1. **Use init containers for migrations** - ```yaml - apiVersion: apps/v1 - kind: Deployment - metadata: - name: atol-api - spec: - template: - spec: - initContainers: - - name: migrate - image: atol-api:latest - command: ["alembic", "upgrade", "head"] - env: - - name: DATABASE_URI - valueFrom: - secretKeyRef: - name: db-secret - key: uri - containers: - - name: api - image: atol-api:latest - ``` - -2. **Use Jobs for one-off migrations** - ```yaml - apiVersion: batch/v1 - kind: Job - metadata: - name: db-migration-v2 - spec: - template: - spec: - containers: - - name: migrate - image: atol-api:latest - command: ["alembic", "upgrade", "head"] - restartPolicy: Never - ``` - -### Production Best Practices (Non-Docker) - -1. **Database connection pooling** - - Use PgBouncer or similar for connection pooling - - Migrations should use direct connection, not pooler - -2. **Migration locks** - - Alembic uses database locks to prevent concurrent migrations - - Ensure only one migration process runs at a time - -3. **Monitoring** - ```bash - # Check migration status - alembic current - - # View migration history - alembic history --verbose - - # Check database version - psql -h db-host -U postgres -d atol_db -c "SELECT * FROM alembic_version;" - ``` - -4. **Automated backups before migrations** - ```bash - #!/bin/bash - # pre-migration-backup.sh - - BACKUP_DIR="/backups/db" - TIMESTAMP=$(date +%Y%m%d_%H%M%S) - - # Create backup - pg_dump -h db-host -U postgres -d atol_db -F c -f "$BACKUP_DIR/pre_migration_$TIMESTAMP.dump" - - # Run migration - alembic upgrade head - - # Keep last 7 days of backups - find $BACKUP_DIR -name "pre_migration_*.dump" -mtime +7 -delete - ``` - -5. **Health checks** - - Ensure health endpoint checks database connectivity - - Load balancer should remove unhealthy instances during migration - -## Migration File Best Practices +### Container workflow -### Structure +- `docker-compose.yml` starts the API container. +- The API container entrypoint waits for the database and then runs: -```python -"""Brief description of what this migration does. - -Revision ID: xxxx -Revises: yyyy -Create Date: 2026-02-09 - -Detailed explanation of changes: -- What tables are affected -- What columns are added/removed -- Any data transformations -""" - -def upgrade() -> None: - # 1. Add new columns as nullable first (for zero-downtime) - # 2. Migrate data if needed - # 3. Make columns NOT NULL - # 4. Add constraints and indexes - # 5. Drop old columns last - pass - -def downgrade() -> None: - # Reverse all changes in opposite order - pass +```bash +uv run alembic upgrade head ``` -### Tips - -- **Idempotent operations**: Use `IF EXISTS` / `IF NOT EXISTS` where possible -- **Data migration**: Include SQL for migrating existing data -- **Indexes**: Create indexes CONCURRENTLY in production to avoid locks -- **Foreign keys**: Add them after data is populated -- **Transactions**: Alembic runs migrations in transactions by default -- **Testing**: Always test both upgrade and downgrade +- If `APP_MODE=migrate`, the entrypoint exits after migrations instead of starting the API. -## Common Patterns +### Schema snapshot maintenance -### Adding a Required Column +The comment at the top of `schema.sql` states that the file should be kept in sync with the database after migrations and regenerated using a schema-only `pg_dump`. -```python -def upgrade() -> None: - # Step 1: Add as nullable - op.add_column('table_name', sa.Column('new_col', sa.Text(), nullable=True)) - - # Step 2: Populate with default/migrated data - op.execute("UPDATE table_name SET new_col = 'default_value'") - - # Step 3: Make NOT NULL - op.alter_column('table_name', 'new_col', nullable=False) -``` - -### Renaming a Column - -```python -def upgrade() -> None: - op.alter_column('table_name', 'old_name', new_column_name='new_name') -``` +The verified instruction in `schema.sql` is: -### Dropping a Table with Foreign Keys - -```python -def upgrade() -> None: - # Drop dependent tables/constraints first - op.drop_constraint('fk_name', 'dependent_table', type_='foreignkey') - op.drop_table('table_name') +```bash +docker compose exec db pg_dump -U -d --schema-only --no-owner --no-acl ``` -## Troubleshooting - -### Migration not visible in container -- Ensure `alembic` directory is mounted in `docker-compose.yml` -- Restart containers: `docker compose restart api` +### What recent migrations changed -### Migration already applied -- Check: `docker compose exec api alembic current` -- View history: `docker compose exec api alembic history` +| Revision | Verified focus | +| --- | --- | +| `0002_update_taxon_cols` | Removes outdated taxonomy columns and adds `assembly.hic_specimen_sample_ids` | +| `0003_assembly_run_github` | Reworks assembly reporting around `assembly_run` and `assembly_stage_run` | +| `0004_qc_reads_assembly_refs` | Reworks QC-read reporting and adds the `qc_read_assembly` link table | +| `0005_add_tolid_requests` | Adds durable ToLID request state | -### Database connection errors -- Ensure database is running: `docker compose ps` -- Check connection string in `.env` -- Run from within container: `docker compose exec api alembic ...` +## Recommended maintainer practice -### Circular import errors -- Restart containers to clear Python cache -- Check for naming conflicts with Python packages +### For repo changes that include schema updates -## Monitoring - -After applying migrations in production: - -```bash -# Check migration status -docker compose exec api alembic current - -# View migration history -docker compose exec api alembic history - -# Check database size -docker compose exec db psql -U postgres -d atol_db -c " - SELECT pg_size_pretty(pg_database_size('atol_db'));" - -# Monitor application logs -docker compose logs -f api - -# Check for errors -docker compose logs api | grep ERROR -``` +1. Add or update the Alembic revision under `alembic/versions/`. +2. Apply the migration to a local database. +3. Update `schema.sql` so it matches the post-migration schema snapshot. +4. Run tests. +5. Update maintainer-facing docs if the schema change affects workflows or operations. -## References +## Unknown From This Repo -- [Alembic Documentation](https://alembic.sqlalchemy.org/) -- [SQLAlchemy Documentation](https://docs.sqlalchemy.org/) -- Project migrations: `alembic/versions/` +- There is no verified production rollback runbook in this repository. +- There is no verified repository-local command for creating new Alembic revisions documented here; the repo only proves how revisions are applied, not the team’s preferred authoring command. diff --git a/docs/ncbi_taxonomy_sync.md b/docs/ncbi_taxonomy_sync.md index 86c6573..da1f291 100644 --- a/docs/ncbi_taxonomy_sync.md +++ b/docs/ncbi_taxonomy_sync.md @@ -1,3 +1,5 @@ +> **Status:** Verified current against `app/services/organism_service.py`, `app/services/taxonomy_info_service.py`, `app/services/ncbi_taxonomy_service.py`, and `tests/unit/services/test_taxonomy_info_service.py`. + ## NCBI taxonomy sync behavior This document describes the current application-level behavior for organism scientific names and NCBI taxonomy enrichment. diff --git a/docs/tolid_broker_api.md b/docs/tolid_broker_api.md index a42c058..691258e 100644 --- a/docs/tolid_broker_api.md +++ b/docs/tolid_broker_api.md @@ -1,3 +1,5 @@ +> **Status:** Verified current against `app/api/v1/endpoints/broker.py`, `app/services/tolid_service.py`, `app/schemas/tolid.py`, and `tests/unit/endpoints/test_endpoints_broker_tolids.py`. + # ToLID Broker API This document describes the simplified ToLID flow between Canopy and the external broker worker. diff --git a/docs/xml_export_api.md b/docs/xml_export_api.md deleted file mode 100644 index d647894..0000000 --- a/docs/xml_export_api.md +++ /dev/null @@ -1,204 +0,0 @@ -# XML Export API Documentation - -This document provides information about the XML export functionality for ENA (European Nucleotide Archive) submissions in the AToL Database API. - -## Overview - -The XML Export API allows you to generate ENA-compliant XML files for sample, experiment, and run submissions. These XML files can be used for submitting data to ENA through their submission portal or API. - -## Endpoints - -### Sample XML Export - -#### Get XML for a Single Sample - -``` -GET /api/v1/xml-export/samples/{sample_id}/xml -``` - -Generates ENA-compliant XML for a specific sample identified by its UUID. - -**Parameters:** -- `sample_id`: UUID of the sample - -**Response:** -- Content-Type: `text/plain` -- Body: XML content for the sample - -#### Get XML for Multiple Samples - -``` -GET /api/v1/xml-export/samples/xml -``` - -Generates ENA-compliant XML for multiple samples based on provided filters. - -**Query Parameters:** -- `sample_ids`: (Optional) List of sample UUIDs to include -- `status`: (Optional) Filter by submission status - -**Response:** -- Content-Type: `text/plain` -- Body: XML content for all matching samples - -#### Get XML for Samples Associated with an Experiment - -``` -GET /api/v1/xml-export/experiments/package/{bpa_package_id}/samples/xml -``` - -Generates ENA-compliant XML for samples associated with a specific experiment package. - -**Parameters:** -- `bpa_package_id`: BPA package ID of the experiment - -**Response:** -- Content-Type: `text/plain` -- Body: XML content for associated samples - -### Experiment XML Export - -#### Get XML for a Single Experiment - -``` -GET /api/v1/xml-export/experiments/{experiment_id}/xml -``` - -Generates ENA-compliant XML for a specific experiment identified by its UUID. - -**Parameters:** -- `experiment_id`: UUID of the experiment - -**Response:** -- Content-Type: `text/plain` -- Body: XML content for the experiment - -#### Get XML for Multiple Experiments - -``` -GET /api/v1/xml-export/experiments/xml -``` - -Generates ENA-compliant XML for multiple experiments based on provided filters. - -**Query Parameters:** -- `experiment_ids`: (Optional) List of experiment UUIDs to include -- `status`: (Optional) Filter by submission status - -**Response:** -- Content-Type: `text/plain` -- Body: XML content for all matching experiments - -#### Get XML for an Experiment by Package ID - -``` -GET /api/v1/xml-export/experiments/package/{bpa_package_id}/xml -``` - -Generates ENA-compliant XML for a specific experiment identified by its BPA package ID. - -**Parameters:** -- `bpa_package_id`: BPA package ID of the experiment - -**Response:** -- Content-Type: `text/plain` -- Body: XML content for the experiment - -### Run XML Export - -#### Get XML for a Single Read/Run - -``` -GET /api/v1/xml-export/reads/{read_id}/xml -``` - -Generates ENA-compliant XML for a specific read/run identified by its UUID. - -**Parameters:** -- `read_id`: UUID of the read - -**Response:** -- Content-Type: `text/plain` -- Body: XML content for the read/run - -#### Get XML for Multiple Reads/Runs - -``` -GET /api/v1/xml-export/reads/xml -``` - -Generates ENA-compliant XML for multiple reads/runs based on provided filters. - -**Query Parameters:** -- `read_ids`: (Optional) List of read UUIDs to include -- `experiment_id`: (Optional) Filter by experiment ID -- `status`: (Optional) Filter by submission status - -**Response:** -- Content-Type: `text/plain` -- Body: XML content for all matching reads/runs - -#### Get XML for Reads/Runs Associated with an Experiment - -``` -GET /api/v1/xml-export/experiments/{experiment_id}/reads/xml -``` - -Generates ENA-compliant XML for reads/runs associated with a specific experiment. - -**Parameters:** -- `experiment_id`: UUID of the experiment - -**Response:** -- Content-Type: `text/plain` -- Body: XML content for associated reads/runs - -## XML Structure - -### Sample XML - -The generated sample XML follows the ENA sample schema and includes: - -- Sample identifiers -- Title -- Taxonomy information (taxon ID, scientific name, common name) -- Description -- Sample attributes -- ENA checklist reference - -### Experiment XML - -The generated experiment XML follows the ENA experiment schema and includes: - -- Experiment identifiers -- Title -- Study reference -- Design description and library information -- Platform and instrument model -- Experiment attributes - -### Run XML - -The generated run XML follows the ENA run schema and includes: - -- Run identifiers -- Experiment reference -- Platform and instrument model -- Data block with file information (filename, checksum, filetype, etc.) - -## Usage Example - -To retrieve XML for a specific experiment: - -``` -curl -X GET "http://localhost:8000/api/v1/xml-export/experiments/{experiment_id}/xml" -H "Authorization: Bearer {your_token}" -``` - -Replace `{experiment_id}` with the UUID of your experiment and `{your_token}` with your authentication token. - -## Notes - -- All endpoints require authentication -- The XML is generated from the `prepared_payload` field in the respective database tables -- If a record has no `prepared_payload` data, a 400 Bad Request error will be returned -- The XML is formatted according to ENA submission requirements From 46b51997140b7e8ba62cf3f2f131c8f4971f2165 Mon Sep 17 00:00:00 2001 From: Emily Marshall Date: Wed, 24 Jun 2026 00:09:53 +1000 Subject: [PATCH 3/9] docs: update handover docs --- README.md | 243 +++++++++++++++++++++-- docs/broker_prerequisites_enhancement.md | 117 ----------- docs/handover/README.md | 85 +++++--- docs/handover/doc_audit.md | 31 --- docs/handover/open_questions.md | 5 +- 5 files changed, 277 insertions(+), 204 deletions(-) delete mode 100644 docs/broker_prerequisites_enhancement.md delete mode 100644 docs/handover/doc_audit.md diff --git a/README.md b/README.md index 52ffab1..d25af98 100644 --- a/README.md +++ b/README.md @@ -1,26 +1,227 @@ -# Canopy Handover Entry Point +# Canopy -This repository contains a FastAPI application for tracking organism, sample, experiment, read, assembly, project, QC-read, genome-note, taxonomy, user, broker-attempt, and ToLID-request metadata for the Australian Tree of Life project. +Canopy is a FastAPI backend for managing metadata and submission-tracking records for the Australian Tree of Life project. -This `README` has been reduced to verified pointers. The detailed handover pack created from the current codebase is in `docs/handover/`. +This `README` is the central repository document. It gives the verified high-level picture of the application and links to more detailed maintainer documentation where the detail would otherwise make this file unwieldy. -## Verified Starting Points +## Verified Scope -- API app entrypoint: `app/main.py` -- Versioned API router: `app/api/v1/api.py` +The current codebase stores and serves metadata for: + +- organisms +- taxonomy enrichment records +- samples and specimen-to-derived sample lineage +- experiments +- reads +- QC-read results +- assemblies, assembly runs, and assembly stage runs +- projects +- BPA initiatives +- genome notes +- users and refresh tokens +- broker submission attempts, events, and ToLID request state + +Primary implementation paths: + +- API app: `app/main.py` +- Versioned router: `app/api/v1/api.py` - Runtime settings: `app/core/settings.py` -- Database session setup: `app/db/session.py` -- Docker startup path: `Dockerfile`, `docker-compose.yml`, `scripts/entrypoint.sh` -- Schema and migrations: `schema.sql`, `alembic/env.py`, `alembic/versions/` -- Test suite: `tests/` - -## Handover Pack - -- Overview and structure: `docs/handover/README.md` -- Documentation audit: `docs/handover/doc_audit.md` -- System and lifecycle overview: `docs/handover/system_overview.md` -- Local setup and operations: `docs/handover/setup_and_operations.md` -- Broker and submission flows: `docs/handover/broker_and_submission_flows.md` -- Config and environment reference: `docs/handover/config_reference.md` -- Troubleshooting guide: `docs/handover/troubleshooting.md` -- Open questions and tribal knowledge gaps: `docs/handover/open_questions.md` +- Models: `app/models/` +- Schemas: `app/schemas/` +- Shared business logic: `app/services/` +- Migrations: `alembic/versions/` + +## How The System Works + +### Core pattern + +Several entities use a main-table plus submission-table pattern. + +- Main table: current application-facing record +- Submission table: staged payloads and submission lifecycle state for external workflows + +This pattern is visible for: + +- projects +- samples +- experiments +- QC reads +- assemblies + +### Broker-facing workflow + +The repo exposes broker endpoints under `/api/v1/broker/...` for claiming work, validating payload prerequisites, reporting outcomes, inspecting attempts, and managing ToLID request state. + +There are currently two broker API surfaces in the same router: + +- a newer flat contract: `/broker/claims/*`, `/broker/validation`, `/broker/reports/{attempt_id}` +- older claim/report endpoints that are still routed: `/broker/claim`, `/broker/organisms/{taxon_id}/claim`, `/broker/attempts/...` + +The repository proves both are active. It does not prove which one is authoritative in production. + +### Assembly workflow + +The current assembly flow is centered on: + +1. creating an assembly intent +2. generating and storing an assembly manifest +3. registering assembly pipeline runs by GitHub repo and commit +4. reporting QC-read outputs +5. reporting per-stage results + +Detailed assembly instructions are in [docs/assembly_reporting_api.md](docs/assembly_reporting_api.md). + +## Active Documentation + +### Start here for maintainers + +- [docs/handover/README.md](docs/handover/README.md) + +### Feature and operations docs verified against current code + +- [docs/assembly_reporting_api.md](docs/assembly_reporting_api.md) +- [docs/auth_refresh_tokens.md](docs/auth_refresh_tokens.md) +- [docs/bulk_import_api.md](docs/bulk_import_api.md) +- [docs/migration_workflow.md](docs/migration_workflow.md) +- [docs/ncbi_taxonomy_sync.md](docs/ncbi_taxonomy_sync.md) +- [docs/tolid_broker_api.md](docs/tolid_broker_api.md) + +### Handover pack contents + +- [docs/handover/system_overview.md](docs/handover/system_overview.md) +- [docs/handover/setup_and_operations.md](docs/handover/setup_and_operations.md) +- [docs/handover/broker_and_submission_flows.md](docs/handover/broker_and_submission_flows.md) +- [docs/handover/config_reference.md](docs/handover/config_reference.md) +- [docs/handover/troubleshooting.md](docs/handover/troubleshooting.md) +- [docs/handover/open_questions.md](docs/handover/open_questions.md) + +## Local Development + +### Requirements + +- Docker and Docker Compose for the container-based local path +- or `uv` plus a reachable PostgreSQL instance for the non-Docker local path + +### Environment and settings + +Settings are defined in `app/core/settings.py`. + +At application import time, the code requires: + +- `JWT_SECRET_KEY` +- `JWT_ALGORITHM` +- `DATABASE_URI`, or enough `POSTGRES_*` values for `DATABASE_URI` to be derived + +At container entrypoint time, `scripts/entrypoint.sh` requires `DATABASE_URI` to exist in the shell environment before it will run migrations. + +The repository includes: + +- `.env.example` +- `docker-compose.yml` +- `Dockerfile` +- `scripts/entrypoint.sh` + +### Docker path + +1. Copy the template: + +```bash +cp .env.example .env +``` + +2. Set the required values in `.env`. + +3. Start the stack: + +```bash +docker compose up --build +``` + +4. Open the API docs: + +- `http://localhost:8000/api/v1/docs` +- `http://localhost:8000/api/v1/redoc` + +The Docker Compose file maps: + +- API: host `8000` -> container `8000` +- Postgres: host `5433` -> container `5432` + +### Non-Docker path + +1. Install dependencies: + +```bash +uv sync --dev --frozen +``` + +2. Export the required environment variables. + +3. Apply migrations: + +```bash +uv run alembic upgrade head +``` + +4. Start the API: + +```bash +uv run uvicorn app.main:app --host 0.0.0.0 --port 8000 --reload +``` + +## Database And Migrations + +- SQLAlchemy session setup is in `app/db/session.py`. +- Alembic config is in `alembic/env.py`. +- Active revisions are in `alembic/versions/`. +- `scripts/entrypoint.sh` runs `uv run alembic upgrade head` before starting the app. +- `schema.sql` is maintained as a schema snapshot, not the runtime bootstrap mechanism. + +For the repo-local migration workflow, see [docs/migration_workflow.md](docs/migration_workflow.md). + +## Authentication + +The current authentication surface is: + +- `POST /api/v1/auth/login` +- `POST /api/v1/auth/refresh` +- `POST /api/v1/auth/logout` + +Access tokens are JWTs. Refresh tokens are stored as hashed values in the `refresh_token` table. + +Detailed auth behavior is in [docs/auth_refresh_tokens.md](docs/auth_refresh_tokens.md). + +## Useful Entry Points + +### API docs and health + +- OpenAPI JSON: `/api/v1/openapi.json` +- Swagger UI: `/api/v1/docs` +- ReDoc: `/api/v1/redoc` +- Health: `/health` +- Version: `/version` + +### Scripts + +- `scripts/entrypoint.sh`: wait for DB, run migrations, start app +- `scripts/create_user.py`: create a user directly in the database +- `scripts/expire_leases.py`: expire broker leases directly from Python + +## Testing + +The repository contains unit tests under `tests/`. + +Run them with: + +```bash +pytest -q +``` + +## Known Limits Of This Repository + +The repository does not by itself tell us: + +- which broker API surface is used in production +- who or what promotes submission rows from `draft` to `ready` +- the full production runtime topology behind the GitHub Actions deployment workflows +- any external runbooks or operational conventions outside version control diff --git a/docs/broker_prerequisites_enhancement.md b/docs/broker_prerequisites_enhancement.md deleted file mode 100644 index 6813c62..0000000 --- a/docs/broker_prerequisites_enhancement.md +++ /dev/null @@ -1,117 +0,0 @@ -> **Status:** Partially current. The accession-resolution behavior matches `app/api/v1/endpoints/broker.py`, but this note is change-history oriented and does not fully describe the current broker contract. - -# Broker Prerequisites Enhancement - -## Problem -The broker API was returning null values for `prerequisites`, `validation_hints`, and `file_metadata` fields because: -1. Missing accession data in submission payloads -2. No way to distinguish between "not required" vs "required but not yet submitted" - -## Solution -Enhanced the broker contract to include **existing** accessions (resolved from `accession_registry`), allowing clients to decide whether dependency state is complete enough to submit. - -### Key Architecture Understanding - -**Crucial clarification**: Accessions are stored in different places: -- **Existing accessions**: Stored in database row fields (`project_accession`, `sample_accession`, etc.) and `accession_registry` table -- **prepared_payload**: Contains data for ENA submission, NOT existing accessions -- **Required accessions**: Specified in payload using `expected_*_accession` fields for dependencies not yet submitted - -### Changes Made - -#### 1. BrokerPrerequisites Schema -```python -class BrokerPrerequisites(BaseModel): - # Existing accessions (from database row fields) - project_accession: Optional[str] = None - sample_accession: Optional[str] = None - experiment_accession: Optional[str] = None - run_accession: Optional[str] = None - study_accession: Optional[str] = None - analysis_accession: Optional[str] = None -``` - -#### 2. Enhanced Prerequisite Extraction Logic -- **Existing accessions**: Looked up from `accession_registry` via FK relationships on submission tables - -#### 3. Updated Test Cases -- Tests now verify both existing and required accessions -- Demonstrates mixed states where some dependencies exist and others don't - -### Usage Examples - -#### Sample Submission with Existing Project Accession -**Database state**: `accession_registry` contains a project accession for the sample's `project_id` -```json -{ - "alias": "sample-1", - "requires_project_accession": true -} -``` -Result: -```json -{ - "prerequisites": { - "project_accession": "PRJ123456" // ✅ Resolved from accession_registry - } -} -``` - -#### Experiment with Missing Study Accession -**Database state**: `accession_registry` does not contain a project accession for the experiment's `project_id` -```json -{ - "alias": "experiment-1", - "requires_study_accession": true, - "expected_study_accession": "PRJ123456" // Required but not submitted -} -``` -Result: -```json -{ - "prerequisites": { - "sample_accession": "SAMEA123456", // ✅ Resolved from accession_registry - "study_accession": null // ❌ Missing from registry - } -} -``` - -#### Run with Missing Experiment Accession -**Database state**: `accession_registry` does not contain an experiment accession for the run's `experiment_id` -```json -{ - "alias": "run-1", - "file_name": "reads.fastq.gz", - "file_format": "fastq", - "expected_experiment_accession": "ERX123456" // Required but not submitted -} -``` -Result: -```json -{ - "prerequisites": { - "experiment_accession": null // ❌ Missing from registry - }, - "files": [ - {"filename": "reads.fastq.gz", "filetype": "fastq"} - ] -} -``` - -### Data Flow Summary - -1. **prepared_payload**: Contains ENA submission data (metadata, files, etc.) -2. **Database row fields**: Store actual accessions once submitted (`*_accession` fields) -3. **Broker response**: Shows resolved accessions (or `null`), and clients decide completeness - -### Client Benefits -1. **Clear dependency visibility**: See what exists vs what is missing -2. **Single source of truth**: Accessions resolved from `accession_registry` - -### Payload Fields for Required Accessions -- `expected_*_accession`: Placeholder for the accession that will be assigned -- `requires_*_accession`: Boolean flag indicating the dependency is required -- `requires_project_accession`: For samples -- `requires_study_accession`: For experiments (maps to `required_project_accession` in broker prerequisites) - -This enhancement maintains backward compatibility while providing much richer dependency information to broker clients, with the correct understanding of where different types of data are stored. diff --git a/docs/handover/README.md b/docs/handover/README.md index 37f8d8b..a515a59 100644 --- a/docs/handover/README.md +++ b/docs/handover/README.md @@ -1,34 +1,55 @@ # Canopy Handover Pack -## Verified - -This handover pack was written from the current repository state, with the code, tests, migrations, scripts, and selected current docs treated as the source of truth. - -Use these files in this order: - -1. `doc_audit.md` -2. `system_overview.md` -3. `setup_and_operations.md` -4. `broker_and_submission_flows.md` -5. `config_reference.md` -6. `troubleshooting.md` -7. `open_questions.md` - -## Proposed Documentation Structure - -| File | Purpose | -| --- | --- | -| `docs/handover/README.md` | Entry point for the handover pack | -| `docs/handover/doc_audit.md` | Audit of existing docs, README, Postman, and generated notes | -| `docs/handover/system_overview.md` | What the system does, repo structure, key entities, main lifecycle rules | -| `docs/handover/setup_and_operations.md` | Local setup, Docker flow, Alembic flow, scripts, recurring operator tasks | -| `docs/handover/broker_and_submission_flows.md` | Submission table lifecycle, broker claims/reports, ToLID flow, assembly reporting handoff points | -| `docs/handover/config_reference.md` | Environment variables and config behavior derived from code | -| `docs/handover/troubleshooting.md` | Maintainer symptom-driven troubleshooting guide | -| `docs/handover/open_questions.md` | Questions for the human owner and tribal knowledge gaps not captured in the repo | - -## Unknown From This Repo - -- Which broker endpoints are used in production versus retained only for backward compatibility. -- Which human or external process promotes submission rows from `draft` to `ready`. -- The production runtime topology behind the GitHub Actions deployment automation. +This directory contains hander documentation for future maintainers of the Canopy codebase + +## What is covered + +The docs cover: + +- what the system currently does +- where major workflows live in the code +- how local setup and migrations work +- how broker and ToLID flows behave +- deployment to AWS (basic details only - this is largely managed by the BioCloud team) +- manual overrides when things don't go to plan +- where the known gaps and human-owned unknowns still are + +This pack is intentionally maintainer-oriented. It favors verified behavior and operator tasks over broad narrative description. + +## Recommended Reading Order + +1. [system_overview.md](/Users/emilylm/Repositories/atol-database-v2/docs/handover/system_overview.md) +2. [setup_and_operations.md](/Users/emilylm/Repositories/atol-database-v2/docs/handover/setup_and_operations.md) +3. [broker_and_submission_flows.md](/Users/emilylm/Repositories/atol-database-v2/docs/handover/broker_and_submission_flows.md) +4. [config_reference.md](/Users/emilylm/Repositories/atol-database-v2/docs/handover/config_reference.md) +5. [troubleshooting.md](/Users/emilylm/Repositories/atol-database-v2/docs/handover/troubleshooting.md) +6. [open_questions.md](/Users/emilylm/Repositories/atol-database-v2/docs/handover/open_questions.md) + +## Document Map + +| File | What it covers | When to use it | +| --- | --- | --- | +| [system_overview.md](/Users/emilylm/Repositories/atol-database-v2/docs/handover/system_overview.md) | Repo structure, core entities, main lifecycle patterns, and external interaction points | Use this first when orienting a new maintainer | +| [setup_and_operations.md](/Users/emilylm/Repositories/atol-database-v2/docs/handover/setup_and_operations.md) | Docker startup path, local non-Docker workflow, migrations, scripts, and recurring operator actions | Use this for local setup and day-to-day maintenance | +| [broker_and_submission_flows.md](/Users/emilylm/Repositories/atol-database-v2/docs/handover/broker_and_submission_flows.md) | Submission states, broker claim/report behavior, ToLID flow, and attempt leasing | Use this when debugging broker-facing workflows | +| [config_reference.md](/Users/emilylm/Repositories/atol-database-v2/docs/handover/config_reference.md) | Environment variables and workflow-level config surfaced by the repo | Use this when wiring environments or checking startup failures | +| [troubleshooting.md](/Users/emilylm/Repositories/atol-database-v2/docs/handover/troubleshooting.md) | Symptom-driven maintainer guide | Use this during incidents or confusing behavior | +| [open_questions.md](/Users/emilylm/Repositories/atol-database-v2/docs/handover/open_questions.md) | Gaps that require human knowledge outside the repo | Use this when deciding what tribal knowledge still needs to be captured | + +## Relationship To The Top-Level README + +- [README.md](/Users/emilylm/Repositories/atol-database-v2/README.md) is the repository’s central high-level document. +- This handover pack is the deeper maintainer set. +- The two should stay consistent, but they serve different depths: + - `README.md`: complete high-level repo document + - `docs/handover/`: detailed maintainer and operator documentation + +## Highest-Risk Gaps Still Not Solved By Documentation Alone + +These are verified as unresolved from the repo itself: + +- which broker endpoint family is authoritative in production +- which human or external process moves submission rows from `draft` to `ready` +- what the production runtime topology is behind the GitHub Actions deployment workflows + +Those gaps are tracked in [open_questions.md](/Users/emilylm/Repositories/atol-database-v2/docs/handover/open_questions.md). diff --git a/docs/handover/doc_audit.md b/docs/handover/doc_audit.md deleted file mode 100644 index 582668b..0000000 --- a/docs/handover/doc_audit.md +++ /dev/null @@ -1,31 +0,0 @@ -# Documentation Audit - -## Verified - -| Document or file | Status | Evidence | Action needed | -| --- | --- | --- | --- | -| `README.md` | stale | References `xml_export.py` and `read_submissions.py`, but those routes are not registered in `app/api/v1/api.py`; says Docker uses `schema.sql` bootstrap and no reload, while `scripts/entrypoint.sh` runs `alembic upgrade head` and starts `uvicorn --reload` when `ENVIRONMENT=dev` | Keep only as a verified pointer to the handover pack | -| `docs/assembly_reporting_api.md` | verified current | Matches active assembly intent, run, QC-report, and stage-run endpoints in `app/api/v1/endpoints/assemblies.py`; supported by `tests/unit/endpoints/test_endpoints_assemblies.py` and `tests/unit/services/test_assembly_helper.py` | Keep; continue treating as the current assembly-specific API doc | -| `docs/auth_refresh_tokens.md` | verified current | Rewritten from `app/api/v1/endpoints/auth.py`, `app/core/security.py`, `app/core/dependencies.py`, `app/models/token.py`, and auth endpoint tests | Keep | -| `docs/broker_prerequisites_enhancement.md` | partially current | Accession lookup behavior matches `_extract_broker_prerequisites()` in `app/api/v1/endpoints/broker.py`; document discusses `validation_hints` and `file_metadata`, but current response schema exposes `files`, and code does not populate `file_metadata` | Do not use as operator guidance without rewriting | -| `docs/bulk_import_api.md` | verified current | Rewritten from current organism, sample, experiment, and taxonomy bulk-import code paths | Keep | -| `docs/migration_workflow.md` | verified current | Rewritten to cover only Alembic, `schema.sql`, Docker entrypoint behavior, and active revision files evidenced in the repo | Keep | -| `docs/ncbi_taxonomy_sync.md` | verified current | Matches canonical scientific-name recomputation in `app/services/organism_service.py` and NCBI enrichment behavior in `app/services/taxonomy_info_service.py`; backed by `tests/unit/services/test_taxonomy_info_service.py` | Keep | -| `docs/tolid_broker_api.md` | verified current | Matches `app/services/tolid_service.py`, `app/schemas/tolid.py`, and ToLID routes in `app/api/v1/endpoints/broker.py`; backed by `tests/unit/endpoints/test_endpoints_broker_tolids.py` | Keep | -| `docs/xml_export_api.md` | stale | No `xml-export` router is included in `app/api/v1/api.py`, and there is no active `app/api/v1/endpoints/xml_export.py` file | Removed from the repo during handover cleanup | -| `postman/Canopy.postman_collection.json` | stale | Contains outdated paths and examples such as organism lookup by grouping key; includes embedded example credentials/tokens; current router surface is different in `app/api/v1/api.py` | Regenerate a minimal core collection from the current API surface | -| `data/docs/MIGRATION_MERGE_INSTRUCTIONS.md` | stale | Refers to migration chain ending at `0009...` and files that are no longer current; active Alembic head is `0005_add_tolid_requests.py` under the current chain | Treat as historical notes only | -| `data/docs/MIGRATION_NOTES.md` | stale | Refers to `0010_rename_base_url_to_bioplatforms_base_url.py`, which is not in the active migration chain | Treat as historical notes only | -| `data/docs/PR_QC_READ_SUBMISSION.md` | stale | Mentions `POST /qc-callbacks` and inactive `read_submissions.py`; current QC reporting route is `POST /api/v1/assemblies/{assembly_id}/qc-reads/report` | Treat as historical PR commentary only | -| `data/docs/SCHEMA_COMPARISON_REPORT.md` | stale | Describes schema mismatches relative to an older `schema3.sql` dump that is not present as current source of truth | Treat as historical notes only | -| `data/docs/SCHEMA_UPDATE_SUMMARY.md` | stale | Refers to removed `read_submission` decisions and older QC-read schema shape; current schema and migrations have moved on | Treat as historical notes only | -| `data/docs/SCHEMA_VERIFICATION.md` | stale | Claims a structural state from 2026-04-30 that predates current assembly and ToLID migrations | Treat as historical notes only | -| `docs/handover/*.md` | missing before this update | No maintainer-oriented handover pack existed in the repo | Created in this update | - -## Inferences - -- The `data/docs/` files appear to be temporary change notes rather than maintained operator documentation, because they describe one-off migration merges, PR summaries, and schema comparisons rather than current runtime behavior. - -## Unknown From This Repo - -- Whether the stale docs should be deleted, archived elsewhere, or retained for audit history. diff --git a/docs/handover/open_questions.md b/docs/handover/open_questions.md index 2014d0b..2599c3e 100644 --- a/docs/handover/open_questions.md +++ b/docs/handover/open_questions.md @@ -6,9 +6,8 @@ 2. What process promotes submission rows from `draft` to `ready`? I could not verify any code path in this repo that sets `status = "ready"` for project, sample, experiment, or QC-read submissions. 3. Which payload shapes are treated as canonical for bulk organism, sample, and experiment imports outside the request validation implied by this repo? 4. Is `POST /api/v1/assemblies/from-experiments/{taxon_id}` still an active workflow, or has the assembly intent flow replaced it operationally? -5. Should stale historical documents under `data/docs/` remain in the repo, or should they be archived elsewhere to avoid confusing maintainers? -6. What are the expected roles and minimum permissions for day-to-day maintainer accounts versus broker accounts versus genome-launcher accounts? -7. Are there manual post-deploy checks, rollback steps, or data-backup requirements that happen outside the GitHub Actions workflows? +5. What are the expected roles and minimum permissions for day-to-day maintainer accounts versus broker accounts versus genome-launcher accounts? +6. Are there manual post-deploy checks, rollback steps, or data-backup requirements that happen outside the GitHub Actions workflows? ## High-Risk Tribal Knowledge Not Present In The Repo From 58cee85505034c96286ba8c9b38e6478d5db16bd Mon Sep 17 00:00:00 2001 From: Emily Marshall Date: Wed, 24 Jun 2026 13:42:27 +1000 Subject: [PATCH 4/9] docs: update handover docs --- README-old.md.md | 383 ++++++++++++++++++++++++++ docs/handover/README.md | 18 -- docs/handover/setup_and_operations.md | 24 +- docs/handover/system_overview.md | 33 ++- 4 files changed, 407 insertions(+), 51 deletions(-) create mode 100644 README-old.md.md diff --git a/README-old.md.md b/README-old.md.md new file mode 100644 index 0000000..630fb66 --- /dev/null +++ b/README-old.md.md @@ -0,0 +1,383 @@ +# Canopy: A Metadata Tracking System for the Australian Tree of Life (AToL) data + +Canopy is a FastAPI backend used to track and manage genomic data for the Australian Tree of Life (AToL) project. + +> **Handover note:** This README is the entry point and is kept complete and current. For a deeper, evidence-based handover pack (system architecture, lifecycle rules, broker flows, troubleshooting, and open questions), see [`docs/handover/README.md`](docs/handover/README.md). + +## Overview + +Canopy manages metadata across core biological entities (Organism, Sample, Experiment, Read, Assembly, Project, BPA Initiative, Genome Note, QC Read), and models each submission lifecycle with a two-table pattern per entity: +- Main: current state +- Submission: staged for external submission (e.g., ENA) + +A dedicated broker workflow enables integration with external submission pipelines. The broker can claim work (with short leases) and report outcomes/accessions back to Canopy. Helper endpoints provide attempt summaries and item listings to power a simple dashboard or external UI. + +## Features + +- Authentication & tokens: JWT access, refresh, and logout endpoints +- Role-based access control: user, curator, broker, genome_launcher, admin, superuser +- RESTful APIs for core entities (organisms, samples, experiments, reads, assemblies, projects, genome notes, BPA initiatives, QC reads) +- Submission workflow endpoints (sample-submissions, experiment-submissions, qc-read-submissions) +- Broker endpoints to support external submission pipelines: + - Claim drafts and obtain a lease: `/api/v1/broker/organisms/{taxon_id}/claim` + - ENA broker contract endpoints: `/api/v1/broker/claims/ready`, `/api/v1/broker/claims/entity`, `/api/v1/broker/validation`, `/api/v1/broker/reports/{attempt_id}` + - Renew lease, finalise, and report results: `/api/v1/broker/attempts/{attempt_id}/...` + - Lightweight ToLID persistence/reporting endpoints: `/api/v1/broker/tolids/...` + - Attempt listing and summaries for dashboard views +- Bulk import endpoints for organisms, samples, experiments, and taxonomy info +- Assembly intent flow: generates a manifest from specimen/lineage data for pipeline consumption (see [`docs/assembly_reporting_api.md`](docs/assembly_reporting_api.md)) +- PostgreSQL database with UUID primary keys and JSONB fields; schema migrated via Alembic, with `schema.sql` kept as a current schema snapshot +- Docker Compose for local development, plus an alternative local (non-Docker) run mode + +## Tech Stack + +- Python 3.12, FastAPI, Uvicorn +- SQLAlchemy 2.x ORM +- Pydantic v2 + pydantic-settings +- PostgreSQL +- JWT auth via python-jose[cryptography]; password hashing via passlib[bcrypt] +- Alembic for schema migrations (`alembic/versions/`) +- Docker & Docker Compose +- Deployed via GitHub Actions to AWS (image pushed to ECR, deploy triggered via Lambda) — see [`.github/workflows/build-and-deploy-dev.yml`](.github/workflows/build-and-deploy-dev.yml) + +## Project Structure + +``` +app/ +├── main.py +├── api/ +│ └── v1/ +│ ├── api.py +│ └── endpoints/ +│ ├── auth.py +│ ├── users.py +│ ├── organisms.py +│ ├── samples.py +│ ├── sample_submissions.py +│ ├── experiments.py +│ ├── experiment_submissions.py +│ ├── reads.py +│ ├── qc_reads.py +│ ├── assemblies.py +│ ├── projects.py +│ ├── genome_notes.py* +│ └── broker.py +├── core/ +│ ├── dependencies.py +│ ├── security.py +│ └── settings.py +├── db/ +│ └── session.py +├── models/ +│ ├── user.py, token.py +│ ├── organism.py, sample.py, experiment.py, read.py, qc_read.py +│ ├── assembly.py, project.py, genome_note.py* +│ └── (SQLAlchemy models) +└── schemas/ + ├── user.py + ├── organism.py, sample.py, experiment.py, read.py, qc_read.py + ├── assembly.py, project.py, genome_note.py* + └── (Pydantic schemas) + +# Top-level +pyproject.toml, uv.lock, docker-compose.yml, Dockerfile, schema.sql, scripts/, docs/ +``` + +## Getting Started + +### Prerequisites + +- Docker and Docker Compose +- [uv](https://docs.astral.sh/uv/) for local (non-Docker) development +- Postgres available for Alembic migrations + +### Running the Application (Docker) + +1) Create your environment file from the template: + +```bash +cp .env.example .env +``` + +2) Edit `.env` and set the required values. At minimum: +- `POSTGRES_USER=postgres` +- `POSTGRES_PASSWORD=` (default example uses `postgres`) +- `POSTGRES_DB=atol_db` +- `POSTGRES_PORT=5432` # container port (host is mapped to 5433) +- `POSTGRES_SERVER=db` # do not change; this is the Docker service name +- `JWT_SECRET_KEY=` +- `JWT_ALGORITHM=HS256` + +Generate a secure secret (macOS/Linux): + +```bash +openssl rand -hex 32 +``` + +3) Build and start the stack: + +```bash +docker compose up -d --build # Compose v2 +# or +docker-compose up -d --build # older Compose +``` + +4) Tail logs (optional) and wait for the API to be ready: + +```bash +docker compose logs -f api +``` + +5) Open the API docs: + +- Swagger UI: http://localhost:8000/api/v1/docs +- ReDoc: http://localhost:8000/api/v1/redoc + +6) Stop the stack: + +```bash +docker compose down +``` + +To reset the database (this will delete your data): + +```bash +docker compose down -v +``` + +Note on code changes (Docker): `scripts/entrypoint.sh` only enables uvicorn `--reload` when `ENVIRONMENT=dev`. If reload isn't enabled, restart the API container to pick up code changes: + +```bash +docker compose restart api +``` + +#### Local database access (via Docker) +Connect to the Postgres database hosted on the `db` container using `psql`. + +- Quick one-liner (uses credentials created by `.env` during container init): + + ```bash + docker compose exec db psql -U postgres -d atol_db + ``` + +- Or open a shell in the container first, then run psql: + + ```bash + docker compose exec -it db bash + psql -U postgres -d atol_db + ``` + +Once in psql, you can run SQL and handy meta-commands: + +```sql +-- List databases +\l +-- Connect to a database (if needed) +\c atol_db +-- List schemas +\dn +-- List tables (optionally by schema) +\dt +\dt public.* +-- Describe a table +\d+ public.users +-- Run a query +SELECT NOW(); +-- Quit psql +\q +``` + +### Create your first user + +On a fresh database you will likely want to create an initial user (e.g. an admin/broker) so you can log in and use secured endpoints. Use the helper script from the project root on your host machine: + +```bash +python scripts/create_user.py \ + --host localhost \ + --port 5433 \ + --dbname atol_db \ + --user postgres \ + --password \ + --username admin \ + --email admin@example.org \ + --user-password \ + --role user +``` + +You will need to manually update the role for your first user in the database. + +Then obtain a JWT to call secured endpoints: + +```bash +# Exchange username/password for tokens +curl -s -X POST http://localhost:8000/api/v1/auth/login \ + -H 'Content-Type: application/x-www-form-urlencoded' \ + -d 'username=admin&password=' +``` + +Export the returned `access_token` and use it in the `Authorization: Bearer ` header. + +### Broker integration quickstart + +The broker API provides claim/report endpoints to integrate with an external submission pipeline. + +- Claim items for an organism (example): + + ```bash + TOKEN= + curl -s -X POST "http://localhost:8000/api/v1/broker/organisms//claim?per_type_limit=100" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"lease_duration_minutes": 30}' + ``` + +- Report results for an attempt (example shape): + + ```bash + curl -s -X POST "http://localhost:8000/api/v1/broker/attempts//report" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"samples": [], "experiments": [], "reads": [], "projects": []}' + ``` + +For the lightweight ToLID persistence/reporting flow, see [docs/tolid_broker_api.md](docs/tolid_broker_api.md). +For a deeper, evidence-based walkthrough of the broker and submission lifecycle, see [docs/handover/broker_and_submission_flows.md](docs/handover/broker_and_submission_flows.md). + +## Bulk Import API + +The system provides API endpoints for bulk importing organisms, samples, experiments, and taxonomy info. These endpoints allow you to import data in the same format as the standalone import script but through authenticated API calls. + +### Bulk Import Endpoints + +- `/api/v1/organisms/bulk-import` - Bulk import organisms +- `/api/v1/samples/bulk-import` - Bulk import samples +- `/api/v1/experiments/bulk-import` - Bulk import experiments + +All bulk import endpoints: +- Require authentication with 'curator' or 'admin' role +- Accept JSON data in the same format as the standalone import script +- Return counts of created and skipped records + +See [docs/bulk_import_api.md](docs/bulk_import_api.md) for request formats and curl examples. + +## API Documentation + +Once the application is running, you can access the interactive API documentation: + +- Swagger UI: http://localhost:8000/api/v1/docs +- ReDoc: http://localhost:8000/api/v1/redoc + +## Role-Based Access Control + +The system implements role-based access control with the following roles: + +- **user**: Basic read access to data +- **curator**: Can create and update biological entities +- **broker**: Can claim and report biological entities +- **genome_launcher**: Can access data required by genome assembly & post-assembly pipelines and report results +- **admin**: Full access to all endpoints +- **superuser**: Special role with delete permissions + +## Development (run without Docker) + +Prefer Docker for a consistent setup. This section shows how to run the FastAPI app directly on your machine as an alternative. + +1) Sync the environment with uv (creates `.venv` in the project root): + +```bash +uv sync --dev +# Optional: source .venv/bin/activate +``` + +2) Configure environment + +- Copy `.env.example` to `.env` and fill the required values (particularly `JWT_SECRET_KEY`, `JWT_ALGORITHM`). +- Choose ONE of the database options below: + +Option A — reuse the Docker Postgres (recommended) + +```bash +# Start only the database service +docker compose up -d db + +# In your .env (or environment), point to the compose database on host 5433 +POSTGRES_SERVER=localhost +POSTGRES_PORT=5433 +POSTGRES_USER=postgres +POSTGRES_PASSWORD= +POSTGRES_DB=atol_db +``` + +Option B — use a local Postgres (no Docker) + +```bash +# Ensure Postgres is running locally (port 5432) +# Create the database (migrations will handle schema) +createdb -h localhost -U postgres atol_db || true + +# In your .env (or environment), point to your local instance +POSTGRES_SERVER=localhost +POSTGRES_PORT=5432 +POSTGRES_USER=postgres +POSTGRES_PASSWORD= +POSTGRES_DB=atol_db +``` + +3) Run the application + +```bash +uv run alembic upgrade head +uv run uvicorn app.main:app --reload --host 0.0.0.0 --port 8000 +``` + +4) Open the docs at http://localhost:8000/api/v1/docs + +### Linting + +- Install hooks: `uv run pre-commit install` +- Run all checks: `uv run pre-commit run --all-files` +- Install commit message hook: `uv run pre-commit install --hook-type commit-msg` (requires commitlint hook configured) + +### Database migrations (Alembic) + +- Apply migrations: `uv run alembic upgrade head` +- Create a new revision: `uv run alembic revision --autogenerate -m "description"` +- Docker Compose runs migrations automatically on container start via `scripts/entrypoint.sh`. +- `scripts/entrypoint.sh` supports `APP_MODE=serve` (default) and `APP_MODE=migrate` (run migrations then exit). +- For Alembic conventions, migration patterns, and troubleshooting, see [docs/migration_workflow.md](docs/migration_workflow.md). + +### Environment configuration + +Configuration is provided via environment variables loaded from `.env` (see `.env.example`). Key settings: + +- Database + - `POSTGRES_SERVER` (Docker service name is `db`) + - `POSTGRES_USER` + - `POSTGRES_PASSWORD` + - `POSTGRES_DB` + - `POSTGRES_PORT` (container port, default 5432; host is `5433` via compose) +- Auth + - `JWT_SECRET_KEY` (required) + - `JWT_ALGORITHM` (e.g. `HS256`) + - `JWT_ACCESS_TOKEN_EXPIRE_MINUTES` + - `JWT_REFRESH_TOKEN_EXPIRE_DAYS` +- CORS + - `BACKEND_CORS_ORIGINS` (JSON array of origins) + +For the full environment variable reference derived from code, see [docs/handover/config_reference.md](docs/handover/config_reference.md). + +## Deployment + +The app is built and deployed to AWS via GitHub Actions: +- [`.github/workflows/build-and-deploy-dev.yml`](.github/workflows/build-and-deploy-dev.yml) builds a Docker image, pushes it to ECR, and invokes an AWS Lambda function to deploy it (on every push to `main`). +- [`.github/workflows/build-publish.yml`](.github/workflows/build-publish.yml) builds and pushes release-tagged images to ECR. + +The underlying AWS runtime infrastructure (where the Lambda deploys to, networking, secrets) is managed by another team — see [docs/handover/setup_and_operations.md](docs/handover/setup_and_operations.md) for what's verifiable from this repo, and [docs/handover/open_questions.md](docs/handover/open_questions.md) for what isn't. + +## Handover Documentation + +For a deeper, evidence-based handover pack covering system architecture, lifecycle rules, broker/submission flows, environment config, troubleshooting, and known gaps, start at [docs/handover/README.md](docs/handover/README.md). + +## License + +This project is licensed under the GPL-3.0-or-later License. diff --git a/docs/handover/README.md b/docs/handover/README.md index a515a59..d47f8e3 100644 --- a/docs/handover/README.md +++ b/docs/handover/README.md @@ -35,21 +35,3 @@ This pack is intentionally maintainer-oriented. It favors verified behavior and | [config_reference.md](/Users/emilylm/Repositories/atol-database-v2/docs/handover/config_reference.md) | Environment variables and workflow-level config surfaced by the repo | Use this when wiring environments or checking startup failures | | [troubleshooting.md](/Users/emilylm/Repositories/atol-database-v2/docs/handover/troubleshooting.md) | Symptom-driven maintainer guide | Use this during incidents or confusing behavior | | [open_questions.md](/Users/emilylm/Repositories/atol-database-v2/docs/handover/open_questions.md) | Gaps that require human knowledge outside the repo | Use this when deciding what tribal knowledge still needs to be captured | - -## Relationship To The Top-Level README - -- [README.md](/Users/emilylm/Repositories/atol-database-v2/README.md) is the repository’s central high-level document. -- This handover pack is the deeper maintainer set. -- The two should stay consistent, but they serve different depths: - - `README.md`: complete high-level repo document - - `docs/handover/`: detailed maintainer and operator documentation - -## Highest-Risk Gaps Still Not Solved By Documentation Alone - -These are verified as unresolved from the repo itself: - -- which broker endpoint family is authoritative in production -- which human or external process moves submission rows from `draft` to `ready` -- what the production runtime topology is behind the GitHub Actions deployment workflows - -Those gaps are tracked in [open_questions.md](/Users/emilylm/Repositories/atol-database-v2/docs/handover/open_questions.md). diff --git a/docs/handover/setup_and_operations.md b/docs/handover/setup_and_operations.md index adbe49f..f055059 100644 --- a/docs/handover/setup_and_operations.md +++ b/docs/handover/setup_and_operations.md @@ -1,7 +1,5 @@ # Local Setup And Operations -## Verified - ### Container startup path - `docker-compose.yml` defines two services: @@ -15,16 +13,14 @@ 4. starts `uvicorn` 5. enables `--reload` only when `ENVIRONMENT=dev` -### Local Docker workflow +### Local Docker workflow (recommended for local development) 1. Copy `.env.example` to `.env`. 2. Set at least the JWT variables and Postgres variables needed by the app. 3. Start the stack with `docker compose up --build`. 4. API docs are served from `/api/v1/docs`, `/api/v1/redoc`, and `/api/v1/openapi.json`. -### Non-Docker workflow - -The repo contains enough evidence for this local path: +### Local non-Docker workflow 1. Install dependencies with `uv sync --dev --frozen` (`Dockerfile`, `.github/workflows/lint.yml`). 2. Export the required environment variables described in `app/core/settings.py`. @@ -37,8 +33,9 @@ The repo contains enough evidence for this local path: - The script accepts either `--db-uri` or host/port/name/user/password fields. - The script sets `roles=[role]`. - The script sets `is_superuser=True` only when `--role superuser`. +e.g.: (TODO) -### Recurring maintainer tasks visible in the repo +### Recurring maintainer tasks TODO | Task | Verified implementation | | --- | --- | @@ -50,13 +47,13 @@ The repo contains enough evidence for this local path: | Check app version | `GET /version` | | Regenerate schema snapshot | Comment in `schema.sql` says to use `pg_dump --schema-only` after migrations | -### Deployment automation evidenced in the repo +### CI/CD related tasks (added by BioCloud team - don't change without assistance) - `.github/workflows/build-and-deploy-dev.yml` builds a container image, pushes it to ECR, and invokes a Lambda deployment function. - `.github/workflows/build-publish.yml` builds and pushes release-tagged images to ECR. - These workflows prove that the repo is wired to AWS-hosted automation, but they do not describe the runtime environment behind the deployment target. -### Operational sharp edges visible in code +### Some stuff to consider updating (TODO) - `scripts/entrypoint.sh` requires a literal `DATABASE_URI` shell variable even though `app/core/settings.py` can derive `DATABASE_URI` from `POSTGRES_*` values. This means `POSTGRES_*` values alone are enough for direct Python startup but not enough for the entrypoint script. - `scripts/create_user.py` appends a hard-coded repository path to `sys.path`. That makes the script repo-location-sensitive. @@ -64,11 +61,6 @@ The repo contains enough evidence for this local path: - `app/services/broker_service.py` - `expire_stale_leases()` inside `app/api/v1/endpoints/broker.py` -## Inferences - -- `schema.sql` is a maintained schema snapshot rather than the runtime bootstrap path, because container startup applies Alembic migrations and does not execute `schema.sql` directly. - -## Unknown From This Repo +## Notes -- The exact production release checklist after a successful GitHub Actions deployment. -- Whether there are environment-specific manual steps around database backups, smoke tests, or rollback outside the repository. +- `schema.sql` is a maintained schema snapshot used for documentation rather than the runtime bootstrap path. The actual initial database schema is in `0001_initial_schema.sql` which migrations are applied ontop of. When updating the database, we should always use and test Alembic migrations and *then* update `schema.sql` accordingly. diff --git a/docs/handover/system_overview.md b/docs/handover/system_overview.md index 2e738f5..05541ac 100644 --- a/docs/handover/system_overview.md +++ b/docs/handover/system_overview.md @@ -1,18 +1,16 @@ # System Overview -## Verified - -### What the application does +### What the Canopy app does - The application is a FastAPI backend with a versioned API under `/api/v1` (`app/main.py`, `app/api/v1/api.py`). -- It stores biological metadata and submission-preparation metadata for these main areas: - - organisms and taxonomy enrichment +- It stores 1) biological metadata imported from BPA, 2) submission-ready metadata objects for ENA and tracking of these submissions, 3) inputs and outputs for AToL's genome engine workflow (incl. qc, genome assembly and post-assembly pipelines). The key entities are: + - organisms and taxonomy information - samples and sample lineage - experiments and raw reads - QC-read results - assemblies, assembly runs, and assembly stage runs - - projects, BPA initiatives, genome notes, and users - - broker submission attempts, events, and ToLID request state + - projects, genome notes*, and users + - broker submission objects, broker submission events, and ToLID request state - Submission-facing entities use a main-table plus submission-table pattern for at least projects, samples, experiments, QC reads, and assemblies (`app/models/project.py`, `sample.py`, `experiment.py`, `qc_read.py`, `assembly.py`). ### Codebase structure @@ -21,16 +19,17 @@ | --- | --- | | `app/main.py` | FastAPI app creation, CORS setup, exception handlers, root, health, version | | `app/api/v1/api.py` | Registers all active routers | -| `app/api/v1/endpoints/` | Route handlers and some business rules | +| `app/api/v1/endpoints/` | Route handlers and some light business rules *(should ideally move all business ogic to services)* | | `app/models/` | SQLAlchemy models for tables and relationships | | `app/schemas/` | Pydantic request/response models and enums | | `app/services/` | Shared business logic for organism, experiment, taxonomy, assembly, broker-adjacent helpers, and ToLID state | -| `app/core/` | settings, auth dependencies, password/JWT helpers, pagination, policy wrapper, app error type | -| `app/config/ena-atol-map.json` | ENA payload field mapping used by sample and experiment payload generation | +| `app/core/` | settings, auth dependencies, password/JWT helpers, pagination, policy wrapper, app error types | +| `app/config/ena-atol-map.json`** | ENA payload field mapping used by sample and experiment payload generation | | `alembic/` | Schema migration entrypoint and revisions | | `scripts/` | Container startup, lease-expiry helper, and user-creation script | | `tests/` | Unit tests for routes, services, settings, security, and broker behavior | +** This must be updated if either the AToL schema and/or ENA's ToL sample checklist [ERC000053](https://www.ebi.ac.uk/ena/browser/view/ERC000053) change. ### Entity and lifecycle patterns #### Organisms @@ -53,6 +52,7 @@ - Taxonomy-info create, bulk-import, bulk-upsert, and bulk-refresh all call NCBI lookup code in `app/services/ncbi_taxonomy_service.py` through `app/services/taxonomy_info_service.py`. - Successful NCBI enrichment updates `taxonomy_info.ncbi_last_synced_at`, recomputes `organism.scientific_name`, and refreshes draft/ready project submission payloads for that organism (`app/services/taxonomy_info_service.py`, `app/services/organism_service.py`). +- Note: the bulk-insert endpoint sometimes returns before enriching all requested The endpoint is indopodent: so simply re-run the command until no more empty rows are returned. #### Assemblies @@ -60,12 +60,13 @@ - generic `POST /api/v1/assemblies/` - intent-driven `POST /api/v1/assemblies/intent/{taxon_id}` - The intent flow validates specimen samples, resolves lineage-linked experiments and reads, generates a manifest JSON, stores it on the `assembly`, and returns `assembly_id`, `version`, and the manifest (`app/api/v1/endpoints/assemblies.py`, `app/services/assembly_helper.py`). -- Assembly runs record pipeline invocations by `github_repo` plus `git_commit`; assembly stage runs store one record per stage name per run (`app/models/assembly.py`, `app/services/assembly_service.py`). +- "Assembly runs" (`assembly_run` table in db) represent a pipeline invocation for a specific assembly/manifest and are tracked/identified by `github_repo` plus `git_commit` (i.e. used to report results for different runs of the same assembly); "assembly stage runs" (`aasembly_stage_run` table in db) store one record per stage name per run (`app/models/assembly.py`, `app/services/assembly_service.py`). - QC read reporting for assemblies creates `qc_read`, `qc_read_file`, `qc_read_assembly`, and a draft `qc_read_submission` (`app/api/v1/endpoints/assemblies.py`, `app/api/v1/endpoints/qc_reads.py`). +- Assembly stage reporting for assemblies accepts a flexible `data` object and a list of files. NOTE: at the moment, you can only report results for an assembly stage run via the POST /assemblies/{assembly_id}/runs/{run_id}/stage_runs/{stage_run_id} endpoint *once*, reporting more results will fail, and the respective PATCH endpoint will replace the existing files rather than appending them. (TODO: this should be fixed to allow appending files) #### Submission tables and status changes -- Create paths generally produce `draft` submission rows. +- Create paths for core objects (projects, samples, experiments, qc_reads) generally produce `draft` submission rows in the respective `*_submission` tables. - Sample and experiment updates have status-aware behavior: - `submitting`: update blocked - `draft` or `ready`: existing submission row updated in place and reset to `draft` @@ -79,12 +80,10 @@ - ToLID durable state is exposed through broker routes but handled by `app/services/tolid_service.py`. - NCBI taxonomy lookup is the only outbound HTTP integration directly evidenced in application code (`app/services/ncbi_taxonomy_service.py`). -## Inferences +## Notes -- Canopy appears to be a metadata store and broker-facing coordination service rather than the component that directly submits all records outward, because the repo contains claim/report APIs for an external broker and no general ENA submission client implementation outside payload preparation and broker result handling. +- Canopy is a metadata store and broker-facing coordination service. It doesn't directly interact with ENA servers. Rather, it provides the data needed for submission, which is requested and submitted by a seperate ENA broker service and then reported back to Canopy (including, importantly, the accessions of submitted objects). -## Unknown From This Repo +## To check -- Which external system or human process decides that a project/sample/experiment/QC-read submission is `ready`. -- Which client systems are authoritative producers for organism, sample, and experiment bulk-import payloads. - Whether `POST /api/v1/assemblies/from-experiments/{taxon_id}` is still actively used now that the assembly intent flow exists. From c3ee459dd56eaee23ae0c8e8c29aa6fc6fd984d6 Mon Sep 17 00:00:00 2001 From: Emily Marshall Date: Thu, 25 Jun 2026 13:30:21 +1000 Subject: [PATCH 5/9] chore: update .gitignore --- .gitignore | 2 -- 1 file changed, 2 deletions(-) diff --git a/.gitignore b/.gitignore index 0cffff1..50c10b5 100644 --- a/.gitignore +++ b/.gitignore @@ -15,5 +15,3 @@ postman/ .coverage htmlcov/ coverage.xml -.claude/ -.windsurf/ From 52bcb180e8a693cc6015593f01b977d6cfb9b62a Mon Sep 17 00:00:00 2001 From: Emily Marshall Date: Fri, 26 Jun 2026 03:23:36 +1000 Subject: [PATCH 6/9] docs: update docs --- .gitignore | 1 + README.md | 14 ++++-- app/api/v1/endpoints/broker.py | 13 ++--- docs/handover/README.md | 6 ++- docs/handover/broker_and_submission_flows.md | 43 +++++----------- docs/handover/known_issues.md | 53 ++++++++++++++++++++ docs/handover/open_questions.md | 15 +++--- docs/handover/troubleshooting.md | 4 +- 8 files changed, 92 insertions(+), 57 deletions(-) create mode 100644 docs/handover/known_issues.md diff --git a/.gitignore b/.gitignore index 50c10b5..543133b 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,4 @@ postman/ .coverage htmlcov/ coverage.xml +.windsurf/ diff --git a/README.md b/README.md index d25af98..b8f3f25 100644 --- a/README.md +++ b/README.md @@ -50,14 +50,18 @@ This pattern is visible for: ### Broker-facing workflow -The repo exposes broker endpoints under `/api/v1/broker/...` for claiming work, validating payload prerequisites, reporting outcomes, inspecting attempts, and managing ToLID request state. +The repo exposes broker endpoints under `/api/v1/broker/...` for claiming work, validating payload prerequisites, reporting outcomes, finalising failed attempts, and managing ToLID request state. -There are currently two broker API surfaces in the same router: +The currently routed claim/report surface is: -- a newer flat contract: `/broker/claims/*`, `/broker/validation`, `/broker/reports/{attempt_id}` -- older claim/report endpoints that are still routed: `/broker/claim`, `/broker/organisms/{taxon_id}/claim`, `/broker/attempts/...` +- `/broker/claims/ready` +- `/broker/claims/entity` +- `/broker/claims/batch` +- `/broker/validation` +- `/broker/reports/{attempt_id}` +- `/broker/attempts/{attempt_id}/finalise` -The repository proves both are active. It does not prove which one is authoritative in production. +Detailed broker lifecycle notes are in [docs/handover/broker_and_submission_flows.md](docs/handover/broker_and_submission_flows.md). ### Assembly workflow diff --git a/app/api/v1/endpoints/broker.py b/app/api/v1/endpoints/broker.py index aeba69c..f2ffb61 100644 --- a/app/api/v1/endpoints/broker.py +++ b/app/api/v1/endpoints/broker.py @@ -1107,11 +1107,11 @@ def report_submission_outcomes( # ---------- Endpoints ---------- -# NOTE: More specific routes must come before parameterized routes in FastAPI -# Otherwise /claim would match /organisms/{taxon_id}/claim with taxon_id="claim" +# NOTE: More specific routes must come before parameterized routes in FastAPI. +# Legacy broker handlers are retained below for reference/unit coverage, but only +# the currently supported routes keep @router decorators. -@router.post("/claim", response_model=ClaimByEntityResponse) @policy("broker:claim") def claim_by_entity_ids( *, @@ -1548,7 +1548,6 @@ def claim_by_entity_ids( ) -@router.post("/organisms/{taxon_id}/claim", response_model=ClaimResponse) @policy("broker:claim") def claim_drafts_for_organism( *, @@ -1999,7 +1998,6 @@ def expire_leases( } -@router.post("/attempts/{attempt_id}/lease/renew") @policy("broker:claim") def renew_attempt_lease( *, @@ -2162,7 +2160,6 @@ def finalise_attempt( return {"attempt_id": str(attempt_id), "released": released, "status": attempt.status} -@router.post("/attempts/{attempt_id}/report", response_model=ReportResult) @policy("broker:claim") def report_results( *, @@ -3045,7 +3042,6 @@ def _get_attempt_items_with_relationships( } -@router.get("/attempts") @policy("broker:read") def list_attempts( *, @@ -3085,7 +3081,6 @@ def list_attempts( return {"items": results, "page": page, "page_size": page_size, "total": total} -@router.get("/attempts/{attempt_id}") @policy("broker:read") def get_attempt( *, @@ -3116,7 +3111,6 @@ def get_attempt( return result -@router.get("/attempts/{attempt_id}/items") @policy("broker:read") def get_attempt_items( *, @@ -3128,7 +3122,6 @@ def get_attempt_items( return {k: [e.model_dump() for e in v] for k, v in items.items()} -@router.get("/organisms/{taxon_id}/summary") @policy("broker:read") def organism_summary( *, diff --git a/docs/handover/README.md b/docs/handover/README.md index d47f8e3..c03c694 100644 --- a/docs/handover/README.md +++ b/docs/handover/README.md @@ -22,8 +22,9 @@ This pack is intentionally maintainer-oriented. It favors verified behavior and 2. [setup_and_operations.md](/Users/emilylm/Repositories/atol-database-v2/docs/handover/setup_and_operations.md) 3. [broker_and_submission_flows.md](/Users/emilylm/Repositories/atol-database-v2/docs/handover/broker_and_submission_flows.md) 4. [config_reference.md](/Users/emilylm/Repositories/atol-database-v2/docs/handover/config_reference.md) -5. [troubleshooting.md](/Users/emilylm/Repositories/atol-database-v2/docs/handover/troubleshooting.md) -6. [open_questions.md](/Users/emilylm/Repositories/atol-database-v2/docs/handover/open_questions.md) +5. [known_issues.md](/Users/emilylm/Repositories/atol-database-v2/docs/handover/known_issues.md) +6. [troubleshooting.md](/Users/emilylm/Repositories/atol-database-v2/docs/handover/troubleshooting.md) +7. [open_questions.md](/Users/emilylm/Repositories/atol-database-v2/docs/handover/open_questions.md) ## Document Map @@ -33,5 +34,6 @@ This pack is intentionally maintainer-oriented. It favors verified behavior and | [setup_and_operations.md](/Users/emilylm/Repositories/atol-database-v2/docs/handover/setup_and_operations.md) | Docker startup path, local non-Docker workflow, migrations, scripts, and recurring operator actions | Use this for local setup and day-to-day maintenance | | [broker_and_submission_flows.md](/Users/emilylm/Repositories/atol-database-v2/docs/handover/broker_and_submission_flows.md) | Submission states, broker claim/report behavior, ToLID flow, and attempt leasing | Use this when debugging broker-facing workflows | | [config_reference.md](/Users/emilylm/Repositories/atol-database-v2/docs/handover/config_reference.md) | Environment variables and workflow-level config surfaced by the repo | Use this when wiring environments or checking startup failures | +| [known_issues.md](/Users/emilylm/Repositories/atol-database-v2/docs/handover/known_issues.md) | Current code issues, operational gaps, and external dependencies | Use this when planning fixes, prioritising debt, or handing work to a new maintainer | | [troubleshooting.md](/Users/emilylm/Repositories/atol-database-v2/docs/handover/troubleshooting.md) | Symptom-driven maintainer guide | Use this during incidents or confusing behavior | | [open_questions.md](/Users/emilylm/Repositories/atol-database-v2/docs/handover/open_questions.md) | Gaps that require human knowledge outside the repo | Use this when deciding what tribal knowledge still needs to be captured | diff --git a/docs/handover/broker_and_submission_flows.md b/docs/handover/broker_and_submission_flows.md index a0b9d21..6e48bb8 100644 --- a/docs/handover/broker_and_submission_flows.md +++ b/docs/handover/broker_and_submission_flows.md @@ -4,55 +4,46 @@ ### Active broker API surfaces -The repo currently exposes two broker surfaces in the same router: - -#### Flat contract endpoints +The repo currently exposes these broker submission endpoints: - `POST /api/v1/broker/claims/ready` - `POST /api/v1/broker/claims/entity` - `POST /api/v1/broker/claims/batch` - `POST /api/v1/broker/validation` - `POST /api/v1/broker/reports/{attempt_id}` +- `POST /api/v1/broker/attempts/{attempt_id}/finalise` -These use `Broker*` request/response schemas from `app/schemas/broker_contract.py`. +The claim, validation, and report endpoints above use a shared broker schema from `app/schemas/broker_contract.py`. +Claims return a single `entities` list where each item includes its own `type`, `id`, payload, prerequisites, and file metadata. -#### Legacy claim/report endpoints that are still routed +### Legacy handler status -- `POST /api/v1/broker/claim` -- `POST /api/v1/broker/organisms/{taxon_id}/claim` -- `POST /api/v1/broker/attempts/{attempt_id}/lease/renew` -- `POST /api/v1/broker/attempts/{attempt_id}/finalise` -- `POST /api/v1/broker/attempts/{attempt_id}/report` -- `GET /api/v1/broker/attempts` -- `GET /api/v1/broker/attempts/{attempt_id}` -- `GET /api/v1/broker/attempts/{attempt_id}/items` -- `GET /api/v1/broker/organisms/{taxon_id}/summary` +- The module still contains older claim/report helper functions such as `claim_by_entity_ids()`, `claim_drafts_for_organism()`, `renew_attempt_lease()`, `report_results()`, `list_attempts()`, `get_attempt()`, `get_attempt_items()`, and `organism_summary()`. +- These are out-dated. I've removed the routes that were exposing these functions, but I've kept the functions for the moment (didn't have time to remove). Remove in future! ### Submission states and attempt leasing - `SubmissionAttempt` rows track claim attempts and lease expiry (`app/models/broker.py`). -- The flat contract can claim submissions in either `draft` or `ready` because `CLAIMABLE_SUBMISSION_STATES = ("draft", "ready")` in `app/api/v1/endpoints/broker.py`. -- The legacy claim endpoints only select `draft` rows in their query bodies. +- The current broker claim endpoints can claim submissions in either `draft` or `ready` because `CLAIMABLE_SUBMISSION_STATES = ("draft", "ready")` in `app/api/v1/endpoints/broker.py`. - Claimed rows move to `submitting`, receive `attempt_id`, `lock_acquired_at`, and `lock_expires_at`, and emit `submission_event` rows with `action="claimed"`. -- Lease renewal extends the attempt lease and propagates the new expiry to still-`submitting` rows. - Finalising an attempt releases remaining `submitting` rows back to `draft` and marks the attempt `complete`. - Expiring stale leases resets expired `submitting` rows back to `draft` and marks expired attempts as `expired`. ### Broker report behavior -- `POST /api/v1/broker/reports/{attempt_id}` updates claim items using the flat contract. +- `POST /api/v1/broker/reports/{attempt_id}` updates the entities claimed under one broker attempt. - `_map_report_state_to_submission_status()` maps: - `completed`, `accepted`, `success` -> `accepted` - `failed`, `rejected`, `error` -> `rejected` - `submitting`, `processing` -> `submitting` - Accepted reports write `accession` to the submission row and attempt to insert into `accession_registry`. -- Sample reports can also store `secondary_accession` into `biosample_accession`. +- Sample reports can also store `secondary_accession` into sample.`biosample_accession`. - Rejected reports create a new draft submission row for the same entity so the entity becomes claimable again. - A database integrity conflict during accession registration is translated into HTTP `409` with a message pointing maintainers to `accession_registry`. ### Submission prerequisites and validation -- The broker flat contract resolves existing prerequisite accessions from `accession_registry`, not only from payload fields (`_extract_broker_prerequisites()` in `app/api/v1/endpoints/broker.py`). +- The current broker validation and claim code resolves existing prerequisite accessions from `accession_registry`, not only from payload fields (`_extract_broker_prerequisites()` in `app/api/v1/endpoints/broker.py`). - Validation rules currently enforced by `POST /api/v1/broker/validation` are: - sample: payload must exist - experiment: payload, `sample_accession`, and `project_accession` must exist @@ -69,6 +60,8 @@ These use `Broker*` request/response schemas from `app/schemas/broker_contract.p - Allowed reported statuses are `pending`, `assigned`, and `failed`; `not_requested` is rejected by schema validation (`app/schemas/tolid.py`). - Reporting `assigned` also mirrors the ToLID value onto `sample.tolid`. +Note: while Canopy stores ToLIDs, and keeps track of ToLID requests, the action of retrying tolids in "pending" state needs to be handled seperately. + ### Submission lifecycle rules outside broker.py - Organism creation creates draft `project_submission` rows immediately. @@ -76,13 +69,3 @@ These use `Broker*` request/response schemas from `app/schemas/broker_contract.p - Experiment creation creates one draft `experiment_submission`. - Assembly QC reporting creates one draft `qc_read_submission`. - Sample and experiment edits can create replacement draft submission rows as described in `system_overview.md`. - -## Inferences - -- The flat broker contract appears to be the newer interface and the legacy claim/report endpoints appear to be retained for compatibility, because both sets are active and the newer set uses dedicated broker-contract schemas. - -## Unknown From This Repo - -- Which broker surface is used by current production callers. -- Who or what changes submission rows from `draft` to `ready`. -- Whether `ready` means human-reviewed, machine-validated, or both. diff --git a/docs/handover/known_issues.md b/docs/handover/known_issues.md new file mode 100644 index 0000000..25d44a7 --- /dev/null +++ b/docs/handover/known_issues.md @@ -0,0 +1,53 @@ +# Known Issues And Outstanding Work + +This page lists current codebase issues, operational gaps, and external dependencies that future maintainers should understand before changing behavior or planning follow-up work. + +## Current Code Issues + +- Broker and bulk-import observability is weak. The code logs some broker activity and returns per-record errors from bulk import endpoints, but there is no in-repo mechanism to persist broker run logs or bulk-import outcome logs for later audit. Some sample import and update paths still use `print(...)` instead of structured logging, especially in [samples.py](/Users/emilylm/Repositories/atol-database-v2/app/api/v1/endpoints/samples.py). + +- `POST /api/v1/assemblies/{assembly_id}/qc-reads/report` requires `bpa_package_id` in the request body. If the intended contract is to identify the package in the path instead, the API shape will need to change in [assemblies.py](/Users/emilylm/Repositories/atol-database-v2/app/api/v1/endpoints/assemblies.py) and [qc_read.py](/Users/emilylm/Repositories/atol-database-v2/app/schemas/qc_read.py). + +- Assembly stage reporting is effectively one-shot per `(assembly_run_id, stage_name)`. A second `POST` for the same stage is rejected, and `PATCH` replaces the full file list rather than appending to it. The current behavior lives in [assembly_service.py](/Users/emilylm/Repositories/atol-database-v2/app/services/assembly_service.py). + +- QC-read reporting always creates a new `QcRead`, new files, and a new draft submission row. There is no deduplication, merge, or reconciliation path in `POST /api/v1/assemblies/{assembly_id}/qc-reads/report`, so repeated reporting can accumulate duplicate or overlapping QC-read records. See [assemblies.py](/Users/emilylm/Repositories/atol-database-v2/app/api/v1/endpoints/assemblies.py). + +- ToLID external-id selection is likely wrong for the intended workflow. The ToLID service currently prefers the primary sample submission accession and only falls back to `sample.biosample_accession`. If the desired identifier is the BioSample or external accession such as `SAMEA...`, this needs changing in [tolid_service.py](/Users/emilylm/Repositories/atol-database-v2/app/services/tolid_service.py). + +- Project parent/child relationships are not represented in the current schema or broker contract. There is no project hierarchy model, join table, or broker payload support for those relationships in [project.py](/Users/emilylm/Repositories/atol-database-v2/app/models/project.py) or [broker.py](/Users/emilylm/Repositories/atol-database-v2/app/api/v1/endpoints/broker.py). If ENA requires that structure, Canopy cannot currently enforce it. + +- API authorization policy coverage is incomplete. Many endpoints require authentication but do not have an explicit `@policy(...)` decorator. This is especially noticeable across read endpoints in [assemblies.py](/Users/emilylm/Repositories/atol-database-v2/app/api/v1/endpoints/assemblies.py), [qc_reads.py](/Users/emilylm/Repositories/atol-database-v2/app/api/v1/endpoints/qc_reads.py), [reads.py](/Users/emilylm/Repositories/atol-database-v2/app/api/v1/endpoints/reads.py), [projects.py](/Users/emilylm/Repositories/atol-database-v2/app/api/v1/endpoints/projects.py), [organisms.py](/Users/emilylm/Repositories/atol-database-v2/app/api/v1/endpoints/organisms.py), and others. + +- Annotations are not implemented in this codebase. Genome notes do exist as a lightweight CRUD and publish/unpublish feature, but there is no annotation model or annotation API surface. Relevant files are [genome_note.py](/Users/emilylm/Repositories/atol-database-v2/app/models/genome_note.py) and [genome_notes.py](/Users/emilylm/Repositories/atol-database-v2/app/api/v1/endpoints/genome_notes.py). + +- Project titles are not validated against any ENA-specific minimum length in Canopy. If ENA rejects short titles, that failure will happen downstream rather than being prevented locally. See [project.py](/Users/emilylm/Repositories/atol-database-v2/app/models/project.py) and [project.py](/Users/emilylm/Repositories/atol-database-v2/app/schemas/project.py). + +- `scripts/create_user.py` is not portable because it contains a hard-coded repository path in `sys.path`. That should be removed or made relative. See [create_user.py](/Users/emilylm/Repositories/atol-database-v2/scripts/create_user.py). + +## Operational Gaps + +- There is no in-repo scheduler or automation for recurring BPA imports. The code exposes bulk-import APIs, but nothing in this repo schedules or triggers them automatically. + +- There is no in-repo scheduler or automation for retrying `pending` ToLID requests. The code exposes `/broker/tolids/pending` and report endpoints, but retry timing is left to an external broker process. + +- Experiment `bioplatforms_base_url` backfill remains a data remediation task. The field exists and is used by assembly helper logic, but the repo cannot tell us which rows in a live database are missing it. + +- Manual ToLID backfill is possible through existing endpoints or sample updates, but the repo cannot tell us which real datasets still need patching. + +- Backup and snapshot procedures are not documented in the repository. Regular dumps and environment refreshes may be advisable, but the implementation details sit outside this codebase. + +## Open Questions And External Dependencies + +- Whether the `qc-reads/report` request shape matches genome launcher cannot be confirmed from this repo alone. + +- How ENA expects parent and child project relationships to be represented still needs external confirmation. + +- Current hosted log-retention settings cannot be determined from the application code. + +- The actual user and role setup in dev or prod databases cannot be inferred from the repo, only the supported role names in [policy.py](/Users/emilylm/Repositories/atol-database-v2/app/core/policy.py). + +- Whether dev should be regularly refreshed from prod snapshots is an environment-management decision, not something the codebase answers. + +## No Longer An Active Code Issue + +- Duplicate `assembly_run` registration is already handled. Creating the same `(assembly_id, github_repo, git_commit)` combination returns `409` through [assemblies.py](/Users/emilylm/Repositories/atol-database-v2/app/api/v1/endpoints/assemblies.py) and [assembly_service.py](/Users/emilylm/Repositories/atol-database-v2/app/services/assembly_service.py). If failures still occur here, they are a different case than duplicate registration. diff --git a/docs/handover/open_questions.md b/docs/handover/open_questions.md index 2599c3e..9007440 100644 --- a/docs/handover/open_questions.md +++ b/docs/handover/open_questions.md @@ -2,12 +2,11 @@ ## Unresolved Questions For The Human Owner -1. Which broker API surface is actually used in production today: the flat contract (`/broker/claims/*`, `/broker/reports/{attempt_id}`) or the older organism/attempt endpoints? -2. What process promotes submission rows from `draft` to `ready`? I could not verify any code path in this repo that sets `status = "ready"` for project, sample, experiment, or QC-read submissions. -3. Which payload shapes are treated as canonical for bulk organism, sample, and experiment imports outside the request validation implied by this repo? -4. Is `POST /api/v1/assemblies/from-experiments/{taxon_id}` still an active workflow, or has the assembly intent flow replaced it operationally? -5. What are the expected roles and minimum permissions for day-to-day maintainer accounts versus broker accounts versus genome-launcher accounts? -6. Are there manual post-deploy checks, rollback steps, or data-backup requirements that happen outside the GitHub Actions workflows? +1. What process promotes submission rows from `draft` to `ready`? I could not verify any code path in this repo that sets `status = "ready"` for project, sample, experiment, or QC-read submissions. +2. Which payload shapes are treated as canonical for bulk organism, sample, and experiment imports outside the request validation implied by this repo? +3. Is `POST /api/v1/assemblies/from-experiments/{taxon_id}` still an active workflow, or has the assembly intent flow replaced it operationally? +4. What are the expected roles and minimum permissions for day-to-day maintainer accounts versus broker accounts versus genome-launcher accounts? +5. Are there manual post-deploy checks, rollback steps, or data-backup requirements that happen outside the GitHub Actions workflows? ## High-Risk Tribal Knowledge Not Present In The Repo @@ -17,7 +16,7 @@ - Broker client behavior is not documented. The repo exposes broker contracts, but not the external worker implementation, retry policy, or release cadence. - ToLID remote-service details are not documented. The repo stores ToLID state and exposes report endpoints, but the external request payload, retry rules, and error taxonomy are absent. - Data-ingest source contracts are not documented. The repo contains code that accepts bulk organism/sample/experiment dictionaries, but not the upstream ownership or canonical sample files. -- The governance around `ready` status is absent. This is operationally important because broker claim behavior differs between the older and newer broker endpoints. +- The governance around `ready` status is absent. This is operationally important because the routed broker claim contract accepts both `draft` and `ready`, while retained legacy helper code only queries `draft`. ### Inferences @@ -25,5 +24,5 @@ ### Unknown From This Repo -- Which stale routes, docs, or backward-compatibility shims can be safely removed without breaking external callers. +- Whether the retained non-routed legacy broker helper functions can be deleted outright without losing useful maintenance tooling. - Whether production operators depend on direct database edits for emergency recovery. diff --git a/docs/handover/troubleshooting.md b/docs/handover/troubleshooting.md index 9aa9698..a26c63c 100644 --- a/docs/handover/troubleshooting.md +++ b/docs/handover/troubleshooting.md @@ -8,7 +8,7 @@ | App import fails with a settings error | Missing JWT or DB settings | `app/core/settings.py` | Check `JWT_SECRET_KEY`, `JWT_ALGORITHM`, and either `DATABASE_URI` or all `POSTGRES_*`; in `prod`, confirm `BACKEND_CORS_ORIGINS` is not `["*"]` | | Login works but refresh fails with `401 Invalid or expired refresh token` | Refresh-token lookup or revocation | `app/api/v1/endpoints/auth.py`, `app/models/token.py` | Check whether the token hash exists in `refresh_token`, whether `expires_at` is still in the future, and whether `revoked` is already true | | Sample or experiment update is rejected while broker work is in progress | Submission lock state | `app/api/v1/endpoints/samples.py`, `app/services/experiment_service.py`, broker attempt routes | Check the latest submission row status; both code paths block edits when the latest submission is `submitting` | -| Broker cannot claim work that maintainers expect to be available | Submission status, claim surface, or stale lease | `app/api/v1/endpoints/broker.py`, `scripts/expire_leases.py`, `POST /api/v1/admin/leases/expire`, `POST /api/v1/broker/leases/expire` | Check whether rows are still `draft` versus `ready`; flat broker claims accept `draft` and `ready`, legacy organism claim accepts only `draft`; check `lock_expires_at`; expire stale leases if needed | +| Broker cannot claim work that maintainers expect to be available | Submission status or stale lease | `app/api/v1/endpoints/broker.py`, `scripts/expire_leases.py`, `POST /api/v1/admin/leases/expire`, `POST /api/v1/broker/leases/expire` | Check whether rows are in `draft` or `ready`; the routed broker claim contract accepts both; check `lock_expires_at`; expire stale leases if needed | | Broker report returns `409` mentioning integrity constraints | Accession registry conflict | `app/api/v1/endpoints/broker.py`, `app/models/accession_registry.py` | Inspect `accession_registry` for an existing row with the same accession bound to a different entity; inspect the affected submission row’s accession FK fields | | ToLID lookup by accession returns `404` | Sample accession resolution or sample kind | `app/services/tolid_service.py`, `app/models/tolid_request.py`, `GET /api/v1/broker/tolids/...` | Confirm there is an accepted `sample_submission.accession` or `sample.biosample_accession`; confirm the resolved sample has `kind = specimen` | | ToLID report cannot create a row | Missing accepted sample accession | `app/services/tolid_service.py` | Check whether `_fallback_external_id()` can find a sample submission accession or `sample.biosample_accession`; without one, reporting returns `tolid_external_id_missing` | @@ -21,7 +21,7 @@ ## Verified Sharp Edges And Technical Debt - There are two different lease-expiry implementations with overlapping responsibilities. -- Broker APIs are duplicated across legacy and newer contract routes. +- Legacy broker helper functions still exist in `app/api/v1/endpoints/broker.py` even though the older routes are no longer exposed. - Submission status `ready` is part of the API contract, but this repo does not show who sets it. - Several code comments and docstrings are stale. One example is the PacBio filtering note in `app/services/assembly_helper.py`, while tests in `tests/unit/services/test_assembly_helper.py` assert that all PacBio reads are currently included when they have `file_name`. - The sample endpoint contains several `print(...)` debugging paths instead of structured logging. From 3a64f809959050cfb13bb20c47a8b94b4cfd18f5 Mon Sep 17 00:00:00 2001 From: Emily Marshall Date: Fri, 26 Jun 2026 03:23:45 +1000 Subject: [PATCH 7/9] test: update tests --- .../test_endpoints_broker_route_surface.py | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 tests/unit/endpoints/test_endpoints_broker_route_surface.py diff --git a/tests/unit/endpoints/test_endpoints_broker_route_surface.py b/tests/unit/endpoints/test_endpoints_broker_route_surface.py new file mode 100644 index 0000000..e2d1725 --- /dev/null +++ b/tests/unit/endpoints/test_endpoints_broker_route_surface.py @@ -0,0 +1,43 @@ +from app.main import app + + +def test_broker_route_surface_exposes_supported_submission_routes(): + broker_routes = { + (method, route.path) + for route in app.routes + if hasattr(route, "methods") and route.path.startswith("/api/v1/broker") + for method in route.methods + } + + expected_routes = { + ("POST", "/api/v1/broker/claims/ready"), + ("POST", "/api/v1/broker/claims/entity"), + ("POST", "/api/v1/broker/claims/batch"), + ("POST", "/api/v1/broker/validation"), + ("POST", "/api/v1/broker/reports/{attempt_id}"), + ("POST", "/api/v1/broker/attempts/{attempt_id}/finalise"), + } + + assert expected_routes.issubset(broker_routes) + + +def test_broker_route_surface_does_not_expose_removed_legacy_routes(): + broker_routes = { + (method, route.path) + for route in app.routes + if hasattr(route, "methods") and route.path.startswith("/api/v1/broker") + for method in route.methods + } + + removed_routes = { + ("POST", "/api/v1/broker/claim"), + ("POST", "/api/v1/broker/organisms/{taxon_id}/claim"), + ("POST", "/api/v1/broker/attempts/{attempt_id}/lease/renew"), + ("POST", "/api/v1/broker/attempts/{attempt_id}/report"), + ("GET", "/api/v1/broker/attempts"), + ("GET", "/api/v1/broker/attempts/{attempt_id}"), + ("GET", "/api/v1/broker/attempts/{attempt_id}/items"), + ("GET", "/api/v1/broker/organisms/{taxon_id}/summary"), + } + + assert broker_routes.isdisjoint(removed_routes) From cb6c67cb00a955bb5e8b69622444e249331151b2 Mon Sep 17 00:00:00 2001 From: Emily Marshall Date: Fri, 26 Jun 2026 03:33:54 +1000 Subject: [PATCH 8/9] docs: update docs --- docs/assembly_reporting_api.md | 2 -- docs/auth_refresh_tokens.md | 2 -- docs/bulk_import_api.md | 2 -- docs/handover/open_questions.md | 28 ---------------------------- docs/migration_workflow.md | 2 -- docs/ncbi_taxonomy_sync.md | 2 -- docs/tolid_broker_api.md | 2 -- 7 files changed, 40 deletions(-) delete mode 100644 docs/handover/open_questions.md diff --git a/docs/assembly_reporting_api.md b/docs/assembly_reporting_api.md index 4f91f9d..40e0a51 100644 --- a/docs/assembly_reporting_api.md +++ b/docs/assembly_reporting_api.md @@ -1,5 +1,3 @@ -> **Status:** Verified current against `app/api/v1/endpoints/assemblies.py`, `app/services/assembly_service.py`, `app/services/assembly_helper.py`, and `tests/unit/endpoints/test_endpoints_assemblies.py`. - # Assembly Reporting API This document describes how to use the assembly API to register a new assembly, then report pipeline results back to the database. diff --git a/docs/auth_refresh_tokens.md b/docs/auth_refresh_tokens.md index e98521c..58f3dd9 100644 --- a/docs/auth_refresh_tokens.md +++ b/docs/auth_refresh_tokens.md @@ -1,5 +1,3 @@ -> **Status:** Verified current against `app/api/v1/endpoints/auth.py`, `app/core/security.py`, `app/core/dependencies.py`, `app/models/token.py`, and `tests/unit/endpoints/test_endpoints_auth.py`. - # Authentication And Refresh Tokens This document describes the authentication behavior that is implemented in the current codebase. diff --git a/docs/bulk_import_api.md b/docs/bulk_import_api.md index 04af680..2f27b98 100644 --- a/docs/bulk_import_api.md +++ b/docs/bulk_import_api.md @@ -1,5 +1,3 @@ -> **Status:** Verified current against `app/api/v1/endpoints/organisms.py`, `app/api/v1/endpoints/samples.py`, `app/api/v1/endpoints/experiments.py`, `app/services/organism_service.py`, `app/services/experiment_service.py`, and the bulk-import endpoint tests. - # Bulk Import Workflows This document describes the bulk-import endpoints that are active in the current codebase. diff --git a/docs/handover/open_questions.md b/docs/handover/open_questions.md deleted file mode 100644 index 9007440..0000000 --- a/docs/handover/open_questions.md +++ /dev/null @@ -1,28 +0,0 @@ -# Open Questions And Tribal Knowledge Gaps - -## Unresolved Questions For The Human Owner - -1. What process promotes submission rows from `draft` to `ready`? I could not verify any code path in this repo that sets `status = "ready"` for project, sample, experiment, or QC-read submissions. -2. Which payload shapes are treated as canonical for bulk organism, sample, and experiment imports outside the request validation implied by this repo? -3. Is `POST /api/v1/assemblies/from-experiments/{taxon_id}` still an active workflow, or has the assembly intent flow replaced it operationally? -4. What are the expected roles and minimum permissions for day-to-day maintainer accounts versus broker accounts versus genome-launcher accounts? -5. Are there manual post-deploy checks, rollback steps, or data-backup requirements that happen outside the GitHub Actions workflows? - -## High-Risk Tribal Knowledge Not Present In The Repo - -### Verified gaps - -- Production deployment topology is not documented. The repo shows ECR image publishing and Lambda-triggered deployment automation, but not the runtime environment or rollback procedure. -- Broker client behavior is not documented. The repo exposes broker contracts, but not the external worker implementation, retry policy, or release cadence. -- ToLID remote-service details are not documented. The repo stores ToLID state and exposes report endpoints, but the external request payload, retry rules, and error taxonomy are absent. -- Data-ingest source contracts are not documented. The repo contains code that accepts bulk organism/sample/experiment dictionaries, but not the upstream ownership or canonical sample files. -- The governance around `ready` status is absent. This is operationally important because the routed broker claim contract accepts both `draft` and `ready`, while retained legacy helper code only queries `draft`. - -### Inferences - -- Maintainers probably rely on conventions outside this repo for deciding when staged submission payloads are safe to send to the broker, because the code models `ready` but does not create it. - -### Unknown From This Repo - -- Whether the retained non-routed legacy broker helper functions can be deleted outright without losing useful maintenance tooling. -- Whether production operators depend on direct database edits for emergency recovery. diff --git a/docs/migration_workflow.md b/docs/migration_workflow.md index 48b90d7..118b102 100644 --- a/docs/migration_workflow.md +++ b/docs/migration_workflow.md @@ -1,5 +1,3 @@ -> **Status:** Verified current only for repository-local migration and schema maintenance steps. This file does not claim any production deployment procedure beyond what is visible in the repo. - # Migration Workflow This document describes the migration workflow that is directly evidenced in the current repository. diff --git a/docs/ncbi_taxonomy_sync.md b/docs/ncbi_taxonomy_sync.md index da1f291..86c6573 100644 --- a/docs/ncbi_taxonomy_sync.md +++ b/docs/ncbi_taxonomy_sync.md @@ -1,5 +1,3 @@ -> **Status:** Verified current against `app/services/organism_service.py`, `app/services/taxonomy_info_service.py`, `app/services/ncbi_taxonomy_service.py`, and `tests/unit/services/test_taxonomy_info_service.py`. - ## NCBI taxonomy sync behavior This document describes the current application-level behavior for organism scientific names and NCBI taxonomy enrichment. diff --git a/docs/tolid_broker_api.md b/docs/tolid_broker_api.md index 691258e..a42c058 100644 --- a/docs/tolid_broker_api.md +++ b/docs/tolid_broker_api.md @@ -1,5 +1,3 @@ -> **Status:** Verified current against `app/api/v1/endpoints/broker.py`, `app/services/tolid_service.py`, `app/schemas/tolid.py`, and `tests/unit/endpoints/test_endpoints_broker_tolids.py`. - # ToLID Broker API This document describes the simplified ToLID flow between Canopy and the external broker worker. From 5e5e2b372c81f1a958a152225546c8869a75ce1d Mon Sep 17 00:00:00 2001 From: Emily Marshall Date: Fri, 26 Jun 2026 03:56:36 +1000 Subject: [PATCH 9/9] docs: update docs --- docs/handover/known_issues.md | 60 ++++++++++++++++++++--------------- 1 file changed, 34 insertions(+), 26 deletions(-) diff --git a/docs/handover/known_issues.md b/docs/handover/known_issues.md index 25d44a7..2e5993b 100644 --- a/docs/handover/known_issues.md +++ b/docs/handover/known_issues.md @@ -1,53 +1,61 @@ # Known Issues And Outstanding Work -This page lists current codebase issues, operational gaps, and external dependencies that future maintainers should understand before changing behavior or planning follow-up work. +## TO DO: Canopy & Broker -## Current Code Issues +- Persist broker logs and broker error responses so they can be reviewed later. -- Broker and bulk-import observability is weak. The code logs some broker activity and returns per-record errors from bulk import endpoints, but there is no in-repo mechanism to persist broker run logs or bulk-import outcome logs for later audit. Some sample import and update paths still use `print(...)` instead of structured logging, especially in [samples.py](/Users/emilylm/Repositories/atol-database-v2/app/api/v1/endpoints/samples.py). +- Persist BPA import logs, mapper outputs, and bulk-import failures. Bulk import endpoints return error lists, but import history is not stored. (side note, some sample import paths still use `print(...)` rather than structured logging e.g. in [samples.py](/Users/emilylm/Repositories/atol-database-v2/app/api/v1/endpoints/samples.py)) -- `POST /api/v1/assemblies/{assembly_id}/qc-reads/report` requires `bpa_package_id` in the request body. If the intended contract is to identify the package in the path instead, the API shape will need to change in [assemblies.py](/Users/emilylm/Repositories/atol-database-v2/app/api/v1/endpoints/assemblies.py) and [qc_read.py](/Users/emilylm/Repositories/atol-database-v2/app/schemas/qc_read.py). +- Decide whether `bpa_package_id` should stay in the request body for `POST /api/v1/assemblies/{assembly_id}/qc-reads/report`. It is currently a required body field in [qc_read.py](/Users/emilylm/Repositories/atol-database-v2/app/schemas/qc_read.py) and is resolved in [assemblies.py](/Users/emilylm/Repositories/atol-database-v2/app/api/v1/endpoints/assemblies.py). From PO feedback, `bpa_package_id` should be moved to a path variable. -- Assembly stage reporting is effectively one-shot per `(assembly_run_id, stage_name)`. A second `POST` for the same stage is rejected, and `PATCH` replaces the full file list rather than appending to it. The current behavior lives in [assembly_service.py](/Users/emilylm/Repositories/atol-database-v2/app/services/assembly_service.py). +- Broker support for parent and child projects is missing. There is no project hierarchy model or broker payload support in [project.py](/Users/emilylm/Repositories/atol-database-v2/app/models/project.py) or [broker.py](/Users/emilylm/Repositories/atol-database-v2/app/api/v1/endpoints/broker.py). We were awaiting confirmation from ENA about how to register and confirm parent-child project relos. Need to confirm this and implement the heirarchy. -- QC-read reporting always creates a new `QcRead`, new files, and a new draft submission row. There is no deduplication, merge, or reconciliation path in `POST /api/v1/assemblies/{assembly_id}/qc-reads/report`, so repeated reporting can accumulate duplicate or overlapping QC-read records. See [assemblies.py](/Users/emilylm/Repositories/atol-database-v2/app/api/v1/endpoints/assemblies.py). +- Assembly stage reporting is one-shot per `(assembly_run_id, stage_name)`. A second `POST` for the same stage returns a conflict. `PATCH` replaces the file list; it does not append. See [assembly_service.py](/Users/emilylm/Repositories/atol-database-v2/app/services/assembly_service.py). This is likely undesirable - may want to update so that we can add more result files and/or metadata for an assembly stage we have already reported results for. -- ToLID external-id selection is likely wrong for the intended workflow. The ToLID service currently prefers the primary sample submission accession and only falls back to `sample.biosample_accession`. If the desired identifier is the BioSample or external accession such as `SAMEA...`, this needs changing in [tolid_service.py](/Users/emilylm/Repositories/atol-database-v2/app/services/tolid_service.py). +- `POST /api/v1/assemblies/{assembly_id}/qc-reads/report` always creates a new `QcRead`, new files, and a new draft `QcReadSubmission`. There is no deduplication, uniqueness check, or delete path inside that workflow. See [assemblies.py](/Users/emilylm/Repositories/atol-database-v2/app/api/v1/endpoints/assemblies.py). May need to change this behaviour if we want more control. -- Project parent/child relationships are not represented in the current schema or broker contract. There is no project hierarchy model, join table, or broker payload support for those relationships in [project.py](/Users/emilylm/Repositories/atol-database-v2/app/models/project.py) or [broker.py](/Users/emilylm/Repositories/atol-database-v2/app/api/v1/endpoints/broker.py). If ENA requires that structure, Canopy cannot currently enforce it. +- ToLID external-id selection should be reviewed. [tolid_service.py](/Users/emilylm/Repositories/atol-database-v2/app/services/tolid_service.py) currently uses `SampleSubmission.accession` and only falls back to `sample.biosample_accession`. We actually want to use the external / BioSample accession as the `external_id` for the ToLIDs we register -> so we need to change this (may require changes to the broker) -- API authorization policy coverage is incomplete. Many endpoints require authentication but do not have an explicit `@policy(...)` decorator. This is especially noticeable across read endpoints in [assemblies.py](/Users/emilylm/Repositories/atol-database-v2/app/api/v1/endpoints/assemblies.py), [qc_reads.py](/Users/emilylm/Repositories/atol-database-v2/app/api/v1/endpoints/qc_reads.py), [reads.py](/Users/emilylm/Repositories/atol-database-v2/app/api/v1/endpoints/reads.py), [projects.py](/Users/emilylm/Repositories/atol-database-v2/app/api/v1/endpoints/projects.py), [organisms.py](/Users/emilylm/Repositories/atol-database-v2/app/api/v1/endpoints/organisms.py), and others. +- Project titles are not validated against any ENA minimum length in Canopy. Recent submission attempts (using the broker) have revealed that ENA requires titles of at least 20 characters. That check should be added in [project.py](/Users/emilylm/Repositories/atol-database-v2/app/schemas/project.py) or before broker submission. May need to pad `title` field when char length is too short. -- Annotations are not implemented in this codebase. Genome notes do exist as a lightweight CRUD and publish/unpublish feature, but there is no annotation model or annotation API surface. Relevant files are [genome_note.py](/Users/emilylm/Repositories/atol-database-v2/app/models/genome_note.py) and [genome_notes.py](/Users/emilylm/Repositories/atol-database-v2/app/api/v1/endpoints/genome_notes.py). +- The current `qc-reads/report` payload is inconsistent with the genome launcher. The current Canopy request shape is defined in [qc_read.py](/Users/emilylm/Repositories/atol-database-v2/app/schemas/qc_read.py). The genome launcher has different fields for qc_read files. Need to change fields in Canopy, or in the genome launcher - or a shim could be added. -- Project titles are not validated against any ENA-specific minimum length in Canopy. If ENA rejects short titles, that failure will happen downstream rather than being prevented locally. See [project.py](/Users/emilylm/Repositories/atol-database-v2/app/models/project.py) and [project.py](/Users/emilylm/Repositories/atol-database-v2/app/schemas/project.py). +- Annotationa are not implemented. -- `scripts/create_user.py` is not portable because it contains a hard-coded repository path in `sys.path`. That should be removed or made relative. See [create_user.py](/Users/emilylm/Repositories/atol-database-v2/scripts/create_user.py). +- Genome notes are largely not implemented. There exists separate CRUD and rudimentary publish/unpublish feature in [genome_notes.py](/Users/emilylm/Repositories/atol-database-v2/app/api/v1/endpoints/genome_notes.py) and [genome_note.py](/Users/emilylm/Repositories/atol-database-v2/app/models/genome_note.py), but this hasn't been tested or properly implemented. We need to implement endpoints that 1) return the metadata required for a genome note and 2) allow reporting/storing of a published genome note back into the db (might be the actual genome note data or just a DOI/link). -## Operational Gaps +- Ensure all API endpoints have explicit `@policy(...)` protection. Several authenticated endpoints still rely only on authentication and have no policy decorator. -- There is no in-repo scheduler or automation for recurring BPA imports. The code exposes bulk-import APIs, but nothing in this repo schedules or triggers them automatically. +- Remove the hard-coded repository path from [create_user.py](/Users/emilylm/Repositories/atol-database-v2/scripts/create_user.py). All script sin `scripts/` folder can probably be removed and managed elsewhere. -- There is no in-repo scheduler or automation for retrying `pending` ToLID requests. The code exposes `/broker/tolids/pending` and report endpoints, but retry timing is left to an external broker process. +## TODO: Operations -- Experiment `bioplatforms_base_url` backfill remains a data remediation task. The field exists and is used by assembly helper logic, but the repo cannot tell us which rows in a live database are missing it. +- Set up a scheduler to import data from BPA data portal. -- Manual ToLID backfill is possible through existing endpoints or sample updates, but the repo cannot tell us which real datasets still need patching. +- Set up a scheduler for retrying or polling `pending` ToLID requests. = -- Backup and snapshot procedures are not documented in the repository. Regular dumps and environment refreshes may be advisable, but the implementation details sit outside this codebase. +- Define log collection and log retention for deployed environments. Logs are available in AWS CloudWatch. -## Open Questions And External Dependencies +- Set up scheduled database backup, dump, and restore procedures. -- Whether the `qc-reads/report` request shape matches genome launcher cannot be confirmed from this repo alone. +- When prod environment created, dev db should be refreshed from prod db snapshots regularly.. Something to discuss with BioCloud team (will need to decide on & implement a DR protocol) -- How ENA expects parent and child project relationships to be represented still needs external confirmation. +## Data And Access Management -- Current hosted log-retention settings cannot be determined from the application code. +- Backfill `experiment.bioplatforms_base_url` where it is missing in existing environments (missing in dev AWS environment currently - because it is read from BPA data which had already been imported). -- The actual user and role setup in dev or prod databases cannot be inferred from the repo, only the supported role names in [policy.py](/Users/emilylm/Repositories/atol-database-v2/app/core/policy.py). +- Patch manually assigned ToLIDs for the benchmarking datasets if those records must exist in Canopy. (can use the `/tolid/report` endpoint) -- Whether dev should be regularly refreshed from prod snapshots is an environment-management decision, not something the codebase answers. +- New roles should be created for 1) each service user (genome launcher and broker) and 2) admin and/or curator users for AToL team members who need to interact with the database. Credentials should be kept confidential. Supported roles in code include `admin`, `curator`, `broker`, `genome_launcher`, and `superuser`. -## No Longer An Active Code Issue +## Open Questions -- Duplicate `assembly_run` registration is already handled. Creating the same `(assembly_id, github_repo, git_commit)` combination returns `409` through [assemblies.py](/Users/emilylm/Repositories/atol-database-v2/app/api/v1/endpoints/assemblies.py) and [assembly_service.py](/Users/emilylm/Repositories/atol-database-v2/app/services/assembly_service.py). If failures still occur here, they are a different case than duplicate registration. +- How can ENA parent and child project relationships be verified for private / unreleased projects? + +- How long are deployed Canopy logs retained in AWS? + +- Does the dev environment still need `bioplatforms_base_url` backfill and manual ToLID backfill? + +## Resolved since docs creted + +- Duplicate `assembly_run` registration is now handled. Creating the same `(assembly_id, github_repo, git_commit)` combination returns `409` through [assemblies.py](/Users/emilylm/Repositories/atol-database-v2/app/api/v1/endpoints/assemblies.py) and [assembly_service.py](/Users/emilylm/Repositories/atol-database-v2/app/services/assembly_service.py).