diff --git a/app/core/config/settings.py b/app/core/config/settings.py index 9b21dd0b6c..c8bdac6209 100644 --- a/app/core/config/settings.py +++ b/app/core/config/settings.py @@ -242,9 +242,12 @@ class Settings(BaseSettings): database_url: str = DEFAULT_DATABASE_URL # Pool timeout and recycle are fixed constants in ``app/db/session.py``; # the background-task engine always derives its pool sizing from the two - # settings below. - database_pool_size: int = Field(default=15, gt=0) - database_max_overflow: int = Field(default=10, ge=0) + # settings below. Defaults are sized so one replica's two pooled engines + # cap at (25 + 15) * 2 = 80 PostgreSQL connections, preserving >= 20 raw + # server slots on PostgreSQL's default max_connections=100 for reserved + # connections, the migration path's two-connection peak, and operations. + database_pool_size: int = Field(default=25, gt=0) + database_max_overflow: int = Field(default=15, ge=0) database_migrate_on_startup: bool = True database_sqlite_pre_migrate_backup_enabled: bool = True database_sqlite_pre_migrate_backup_max_files: int = Field(default=5, ge=1) diff --git a/docker-compose.yml b/docker-compose.yml index 3c1d518035..b9c730521c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -92,6 +92,13 @@ services: fi exec docker-entrypoint.sh "$@" - codex-lb-postgres-entrypoint-guard + # Docker's default /dev/shm is 64MB. PostgreSQL parallel workers exchange + # tuples through dynamic shared memory under /dev/shm, so hash joins that + # spill past 64MB abort with "could not resize shared memory segment ... + # No space left on device". 1GB gives parallel query realistic headroom. + # (The Helm chart needs no equivalent: the bundled Bitnami PostgreSQL + # sub-chart mounts a memory-backed /dev/shm by default via shmVolume.) + shm_size: 1gb environment: POSTGRES_USER: codex_lb POSTGRES_PASSWORD: codex_lb diff --git a/docs/reference/settings.md b/docs/reference/settings.md index 229934236e..cb350e1607 100644 --- a/docs/reference/settings.md +++ b/docs/reference/settings.md @@ -32,11 +32,11 @@ the host side of the compose `ports` mapping instead. | Environment variable | Type | Default | | --- | --- | --- | | `CODEX_LB_DATABASE_ALEMBIC_AUTO_REMAP_ENABLED` | `bool` | `True` | -| `CODEX_LB_DATABASE_MAX_OVERFLOW` | `int` | `10` | +| `CODEX_LB_DATABASE_MAX_OVERFLOW` | `int` | `15` | | `CODEX_LB_DATABASE_MIGRATE_ON_STARTUP` | `bool` | `True` | | `CODEX_LB_DATABASE_MIGRATION_LOCK_TIMEOUT_SECONDS` | `float` | `300.0` | | `CODEX_LB_DATABASE_MIGRATIONS_FAIL_FAST` | `bool` | `True` | -| `CODEX_LB_DATABASE_POOL_SIZE` | `int` | `15` | +| `CODEX_LB_DATABASE_POOL_SIZE` | `int` | `25` | | `CODEX_LB_DATABASE_SQLITE_PRE_MIGRATE_BACKUP_ENABLED` | `bool` | `True` | | `CODEX_LB_DATABASE_SQLITE_PRE_MIGRATE_BACKUP_MAX_FILES` | `int` | `5` | | `CODEX_LB_DATABASE_SQLITE_STARTUP_CHECK_MODE` | `'quick' \| 'full' \| 'off'` | `'quick'` | diff --git a/openspec/changes/expand-postgres-shm-and-pool-headroom/proposal.md b/openspec/changes/expand-postgres-shm-and-pool-headroom/proposal.md new file mode 100644 index 0000000000..efc23843ad --- /dev/null +++ b/openspec/changes/expand-postgres-shm-and-pool-headroom/proposal.md @@ -0,0 +1,48 @@ +# Expand PostgreSQL /dev/shm and default pool headroom + +## Why + +Two independent capacity ceilings surfaced on a production single-replica +PostgreSQL deployment: + +- The Compose `postgres` service runs with Docker's default 64MB `/dev/shm`. + PostgreSQL parallel workers (`work_mem=32MB`, + `max_parallel_workers_per_gather=2`) exchange spill files through dynamic + shared memory under `/dev/shm`, so parallel hash joins abort with + `could not resize shared memory segment ... No space left on device`, + which asyncpg surfaces as `DiskFullError` on the request path. +- The default SQLAlchemy pool (`database_pool_size=15`, + `database_max_overflow=10`, fixed 30s checkout timeout) exhausts under + slow-query pile-ups: once 25 request-path checkouts are held, every further + request waits 30 seconds and fails with + `QueuePool limit of size 15 overflow 10 reached, connection timed out`. + +## What Changes + +- The Compose `postgres` service sets `shm_size: 1gb`. +- Default `database_pool_size` rises 15 → 25 and `database_max_overflow` + 10 → 15, keeping the per-replica two-engine cap at + `(25 + 15) * 2 = 80` application connections — inside PostgreSQL's default + `max_connections=100` with at least 20 raw server slots reserved (same + reserve rule the Helm capacity guidance already mandates). +- Helm deployments are unaffected: the chart always injects its own + `CODEX_LB_DATABASE_POOL_SIZE` / `CODEX_LB_DATABASE_MAX_OVERFLOW` values, + and the bundled Bitnami PostgreSQL sub-chart already mounts a + memory-backed `/dev/shm` (`shmVolume.enabled=true` by default). + +## Capabilities + +### Modified Capabilities + +- `deployment-installation`: the Compose Postgres profile provisions a + `/dev/shm` large enough for parallel query. +- `database-backends`: default pool sizing preserves the raw-slot reserve on + PostgreSQL's default `max_connections`. + +## Impact + +- SQLite deployments: none (pool sizing applies to pooled backends only). +- Helm deployments: none (chart values override both settings). +- Compose/manual PostgreSQL deployments: applying `shm_size` requires the + postgres container to be recreated (seconds of downtime); per-replica + worst-case application connections rise from 50 to 80. diff --git a/openspec/changes/expand-postgres-shm-and-pool-headroom/specs/database-backends/spec.md b/openspec/changes/expand-postgres-shm-and-pool-headroom/specs/database-backends/spec.md new file mode 100644 index 0000000000..a5231ef1e9 --- /dev/null +++ b/openspec/changes/expand-postgres-shm-and-pool-headroom/specs/database-backends/spec.md @@ -0,0 +1,27 @@ +## ADDED Requirements + +### Requirement: Default pool sizing preserves raw-slot reserve on default max_connections + +The default values of `database_pool_size` and `database_max_overflow` MUST +keep one replica's aggregate application connection capacity — +`(database_pool_size + database_max_overflow) * 2 pooled engines * 1 +supported worker` — at or below 80, so a single replica on PostgreSQL's +default `max_connections=100` retains at least 20 raw server slots for +PostgreSQL-reserved connections, the migration path's two-connection peak, +administration, and transient non-application clients. + +#### Scenario: Default single replica fits default max_connections + +- **WHEN** one replica runs with the default `database_pool_size` and + `database_max_overflow` +- **THEN** both pooled engines together cap at no more than 80 PostgreSQL + connections +- **AND** at least 20 raw server slots remain on a default + `max_connections=100` server + +#### Scenario: Operators can still tune the pool + +- **WHEN** `CODEX_LB_DATABASE_POOL_SIZE` or `CODEX_LB_DATABASE_MAX_OVERFLOW` + is set in the environment +- **THEN** the configured values override the defaults for both pooled + engines diff --git a/openspec/changes/expand-postgres-shm-and-pool-headroom/specs/deployment-installation/spec.md b/openspec/changes/expand-postgres-shm-and-pool-headroom/specs/deployment-installation/spec.md new file mode 100644 index 0000000000..05e880b54a --- /dev/null +++ b/openspec/changes/expand-postgres-shm-and-pool-headroom/specs/deployment-installation/spec.md @@ -0,0 +1,22 @@ +## ADDED Requirements + +### Requirement: Compose Postgres service sizes /dev/shm for parallel query + +The Docker Compose `postgres` service MUST set an explicit `shm_size` of at +least 1GB. Docker's default 64MB `/dev/shm` causes PostgreSQL parallel +workers to fail with `could not resize shared memory segment ... No space +left on device` once a parallel hash join spills past the segment. + +#### Scenario: Compose postgres service pins shm_size + +- **WHEN** `docker-compose.yml` is inspected +- **THEN** the `postgres` service declares `shm_size` of at least 1GB + +#### Scenario: Parallel hash join spills past 64MB + +- **GIVEN** the Compose `postgres` service is running with the declared + `shm_size` +- **WHEN** a parallel hash join spills more than 64MB of build tuples into + dynamic shared memory +- **THEN** the query does not fail with `could not resize shared memory + segment` diff --git a/openspec/changes/expand-postgres-shm-and-pool-headroom/tasks.md b/openspec/changes/expand-postgres-shm-and-pool-headroom/tasks.md new file mode 100644 index 0000000000..558d9c804e --- /dev/null +++ b/openspec/changes/expand-postgres-shm-and-pool-headroom/tasks.md @@ -0,0 +1,17 @@ +## 1. Implementation + +- [x] 1.1 Add `shm_size: 1gb` to the Compose `postgres` service. +- [x] 1.2 Raise default `database_pool_size` to 25 and + `database_max_overflow` to 15, documenting the 80-connection / + 20-raw-slot budget at the setting definition. +- [x] 1.3 Regenerate `docs/reference/settings.md`. + +## 2. Regression coverage + +- [x] 2.1 Policy-test that the Compose `postgres` service pins `shm_size`. +- [x] 2.2 Update the settings default assertion to the new pool size. + +## 3. Validation + +- [x] 3.1 Run the compose, db-session, settings, and Helm artifact suites. +- [x] 3.2 Run strict OpenSpec validation for this change. diff --git a/tests/unit/test_db_session.py b/tests/unit/test_db_session.py index e75bf781db..300fc966e3 100644 --- a/tests/unit/test_db_session.py +++ b/tests/unit/test_db_session.py @@ -738,7 +738,7 @@ async def test_init_background_db_derives_postgres_pool_size_from_main_pool() -> if os.environ.get("CODEX_LB_TEST_DATABASE_URL"): assert isinstance(pool, NullPool) else: - assert cast(Any, pool).size() == 15 + assert cast(Any, pool).size() == 25 if session_module._background_engine is not None: await session_module._background_engine.dispose() diff --git a/tests/unit/test_docker_compose_postgres.py b/tests/unit/test_docker_compose_postgres.py index 72b9591cc1..1accaa0150 100644 --- a/tests/unit/test_docker_compose_postgres.py +++ b/tests/unit/test_docker_compose_postgres.py @@ -11,6 +11,15 @@ def _compose() -> dict[str, Any]: return yaml.safe_load((repo_root / "docker-compose.yml").read_text(encoding="utf-8")) +def test_postgres_compose_service_sizes_dev_shm_for_parallel_query() -> None: + postgres = _compose()["services"]["postgres"] + + # Docker's default 64MB /dev/shm makes PostgreSQL parallel hash joins fail + # with "could not resize shared memory segment ... No space left on + # device" once they spill past the segment. Keep an explicit >= 1GB size. + assert postgres["shm_size"] == "1gb" + + def test_postgres18_compose_upgrade_helper_is_digest_pinned() -> None: services = _compose()["services"] postgres = services["postgres"] diff --git a/tests/unit/test_settings_trace_and_removed.py b/tests/unit/test_settings_trace_and_removed.py index dd197e890d..fcf9c16a92 100644 --- a/tests/unit/test_settings_trace_and_removed.py +++ b/tests/unit/test_settings_trace_and_removed.py @@ -153,7 +153,7 @@ def test_phase_3_removed_settings_are_listed_and_ignored(monkeypatch): settings = Settings() assert not hasattr(settings, "database_pool_recycle_seconds") assert not hasattr(settings, "drain_primary_threshold_pct") - assert settings.database_pool_size == 15 + assert settings.database_pool_size == 25 assert settings.soft_drain_enabled is True found = warn_removed_settings( {