fix(db): add postgres shm_size and raise default pool headroom

app/core/config/settings.py

@@ -242,9 +242,12 @@ class Settings(BaseSettings):
     database_url: str = DEFAULT_DATABASE_URL
     # Pool timeout and recycle are fixed constants in ``app/db/session.py``;
     # the background-task engine always derives its pool sizing from the two
-    # settings below.
-    database_pool_size: int = Field(default=15, gt=0)
-    database_max_overflow: int = Field(default=10, ge=0)
+    # settings below. Defaults are sized so one replica's two pooled engines
+    # cap at (25 + 15) * 2 = 80 PostgreSQL connections, preserving >= 20 raw
+    # server slots on PostgreSQL's default max_connections=100 for reserved
+    # connections, the migration path's two-connection peak, and operations.
+    database_pool_size: int = Field(default=25, gt=0)
+    database_max_overflow: int = Field(default=15, ge=0)
     database_migrate_on_startup: bool = True
     database_sqlite_pre_migrate_backup_enabled: bool = True
     database_sqlite_pre_migrate_backup_max_files: int = Field(default=5, ge=1)

docker-compose.yml

@@ -92,6 +92,13 @@ services:
         fi
         exec docker-entrypoint.sh "$@"
       - codex-lb-postgres-entrypoint-guard
+    # Docker's default /dev/shm is 64MB. PostgreSQL parallel workers exchange
+    # tuples through dynamic shared memory under /dev/shm, so hash joins that
+    # spill past 64MB abort with "could not resize shared memory segment ...
+    # No space left on device". 1GB gives parallel query realistic headroom.
+    # (The Helm chart needs no equivalent: the bundled Bitnami PostgreSQL
+    # sub-chart mounts a memory-backed /dev/shm by default via shmVolume.)
+    shm_size: 1gb
     environment:
       POSTGRES_USER: codex_lb
       POSTGRES_PASSWORD: codex_lb

docs/reference/settings.md

@@ -32,11 +32,11 @@ the host side of the compose `ports` mapping instead.
 | Environment variable | Type | Default |
 | --- | --- | --- |
 | `CODEX_LB_DATABASE_ALEMBIC_AUTO_REMAP_ENABLED` | `bool` | `True` |
-| `CODEX_LB_DATABASE_MAX_OVERFLOW` | `int` | `10` |
+| `CODEX_LB_DATABASE_MAX_OVERFLOW` | `int` | `15` |
 | `CODEX_LB_DATABASE_MIGRATE_ON_STARTUP` | `bool` | `True` |
 | `CODEX_LB_DATABASE_MIGRATION_LOCK_TIMEOUT_SECONDS` | `float` | `300.0` |
 | `CODEX_LB_DATABASE_MIGRATIONS_FAIL_FAST` | `bool` | `True` |
-| `CODEX_LB_DATABASE_POOL_SIZE` | `int` | `15` |
+| `CODEX_LB_DATABASE_POOL_SIZE` | `int` | `25` |
 | `CODEX_LB_DATABASE_SQLITE_PRE_MIGRATE_BACKUP_ENABLED` | `bool` | `True` |
 | `CODEX_LB_DATABASE_SQLITE_PRE_MIGRATE_BACKUP_MAX_FILES` | `int` | `5` |
 | `CODEX_LB_DATABASE_SQLITE_STARTUP_CHECK_MODE` | `'quick' \| 'full' \| 'off'` | `'quick'` |

openspec/changes/expand-postgres-shm-and-pool-headroom/proposal.md

@@ -0,0 +1,48 @@
+# Expand PostgreSQL /dev/shm and default pool headroom
+
+## Why
+
+Two independent capacity ceilings surfaced on a production single-replica
+PostgreSQL deployment:
+
+- The Compose `postgres` service runs with Docker's default 64MB `/dev/shm`.
+  PostgreSQL parallel workers (`work_mem=32MB`,
+  `max_parallel_workers_per_gather=2`) exchange spill files through dynamic
+  shared memory under `/dev/shm`, so parallel hash joins abort with
+  `could not resize shared memory segment ... No space left on device`,
+  which asyncpg surfaces as `DiskFullError` on the request path.
+- The default SQLAlchemy pool (`database_pool_size=15`,
+  `database_max_overflow=10`, fixed 30s checkout timeout) exhausts under
+  slow-query pile-ups: once 25 request-path checkouts are held, every further
+  request waits 30 seconds and fails with
+  `QueuePool limit of size 15 overflow 10 reached, connection timed out`.
+
+## What Changes
+
+- The Compose `postgres` service sets `shm_size: 1gb`.
+- Default `database_pool_size` rises 15 → 25 and `database_max_overflow`
+  10 → 15, keeping the per-replica two-engine cap at
+  `(25 + 15) * 2 = 80` application connections — inside PostgreSQL's default
+  `max_connections=100` with at least 20 raw server slots reserved (same
+  reserve rule the Helm capacity guidance already mandates).
+- Helm deployments are unaffected: the chart always injects its own
+  `CODEX_LB_DATABASE_POOL_SIZE` / `CODEX_LB_DATABASE_MAX_OVERFLOW` values,
+  and the bundled Bitnami PostgreSQL sub-chart already mounts a
+  memory-backed `/dev/shm` (`shmVolume.enabled=true` by default).
+
+## Capabilities
+
+### Modified Capabilities
+
+- `deployment-installation`: the Compose Postgres profile provisions a
+  `/dev/shm` large enough for parallel query.
+- `database-backends`: default pool sizing preserves the raw-slot reserve on
+  PostgreSQL's default `max_connections`.
+
+## Impact
+
+- SQLite deployments: none (pool sizing applies to pooled backends only).
+- Helm deployments: none (chart values override both settings).
+- Compose/manual PostgreSQL deployments: applying `shm_size` requires the
+  postgres container to be recreated (seconds of downtime); per-replica
+  worst-case application connections rise from 50 to 80.

openspec/changes/expand-postgres-shm-and-pool-headroom/specs/database-backends/spec.md

@@ -0,0 +1,27 @@
+## ADDED Requirements
+
+### Requirement: Default pool sizing preserves raw-slot reserve on default max_connections
+
+The default values of `database_pool_size` and `database_max_overflow` MUST
+keep one replica's aggregate application connection capacity —
+`(database_pool_size + database_max_overflow) * 2 pooled engines * 1
+supported worker` — at or below 80, so a single replica on PostgreSQL's
+default `max_connections=100` retains at least 20 raw server slots for
+PostgreSQL-reserved connections, the migration path's two-connection peak,
+administration, and transient non-application clients.
+
+#### Scenario: Default single replica fits default max_connections
+
+- **WHEN** one replica runs with the default `database_pool_size` and
+  `database_max_overflow`
+- **THEN** both pooled engines together cap at no more than 80 PostgreSQL
+  connections
+- **AND** at least 20 raw server slots remain on a default
+  `max_connections=100` server
+
+#### Scenario: Operators can still tune the pool
+
+- **WHEN** `CODEX_LB_DATABASE_POOL_SIZE` or `CODEX_LB_DATABASE_MAX_OVERFLOW`
+  is set in the environment
+- **THEN** the configured values override the defaults for both pooled
+  engines

openspec/changes/expand-postgres-shm-and-pool-headroom/specs/deployment-installation/spec.md

@@ -0,0 +1,22 @@
+## ADDED Requirements
+
+### Requirement: Compose Postgres service sizes /dev/shm for parallel query
+
+The Docker Compose `postgres` service MUST set an explicit `shm_size` of at
+least 1GB. Docker's default 64MB `/dev/shm` causes PostgreSQL parallel
+workers to fail with `could not resize shared memory segment ... No space
+left on device` once a parallel hash join spills past the segment.
+
+#### Scenario: Compose postgres service pins shm_size
+
+- **WHEN** `docker-compose.yml` is inspected
+- **THEN** the `postgres` service declares `shm_size` of at least 1GB
+
+#### Scenario: Parallel hash join spills past 64MB
+
+- **GIVEN** the Compose `postgres` service is running with the declared
+  `shm_size`
+- **WHEN** a parallel hash join spills more than 64MB of build tuples into
+  dynamic shared memory
+- **THEN** the query does not fail with `could not resize shared memory
+  segment`

openspec/changes/expand-postgres-shm-and-pool-headroom/tasks.md

@@ -0,0 +1,17 @@
+## 1. Implementation
+
+- [x] 1.1 Add `shm_size: 1gb` to the Compose `postgres` service.
+- [x] 1.2 Raise default `database_pool_size` to 25 and
+  `database_max_overflow` to 15, documenting the 80-connection /
+  20-raw-slot budget at the setting definition.
+- [x] 1.3 Regenerate `docs/reference/settings.md`.
+
+## 2. Regression coverage
+
+- [x] 2.1 Policy-test that the Compose `postgres` service pins `shm_size`.
+- [x] 2.2 Update the settings default assertion to the new pool size.
+
+## 3. Validation
+
+- [x] 3.1 Run the compose, db-session, settings, and Helm artifact suites.
+- [x] 3.2 Run strict OpenSpec validation for this change.

tests/unit/test_db_session.py

@@ -738,7 +738,7 @@ async def test_init_background_db_derives_postgres_pool_size_from_main_pool() ->
     if os.environ.get("CODEX_LB_TEST_DATABASE_URL"):
         assert isinstance(pool, NullPool)
     else:
-        assert cast(Any, pool).size() == 15
+        assert cast(Any, pool).size() == 25
 
     if session_module._background_engine is not None:
         await session_module._background_engine.dispose()

tests/unit/test_docker_compose_postgres.py

@@ -11,6 +11,15 @@ def _compose() -> dict[str, Any]:
     return yaml.safe_load((repo_root / "docker-compose.yml").read_text(encoding="utf-8"))
 
 
+def test_postgres_compose_service_sizes_dev_shm_for_parallel_query() -> None:
+    postgres = _compose()["services"]["postgres"]
+
+    # Docker's default 64MB /dev/shm makes PostgreSQL parallel hash joins fail
+    # with "could not resize shared memory segment ... No space left on
+    # device" once they spill past the segment. Keep an explicit >= 1GB size.
+    assert postgres["shm_size"] == "1gb"
+
+
 def test_postgres18_compose_upgrade_helper_is_digest_pinned() -> None:
     services = _compose()["services"]
     postgres = services["postgres"]

tests/unit/test_settings_trace_and_removed.py

@@ -153,7 +153,7 @@ def test_phase_3_removed_settings_are_listed_and_ignored(monkeypatch):
     settings = Settings()
     assert not hasattr(settings, "database_pool_recycle_seconds")
     assert not hasattr(settings, "drain_primary_threshold_pct")
-    assert settings.database_pool_size == 15
+    assert settings.database_pool_size == 25
     assert settings.soft_drain_enabled is True
     found = warn_removed_settings(
         {