deploys: safe ordering and a health check; take the migration runner back out

.github/workflows/cypress-tests.yml

@@ -85,9 +85,6 @@ jobs:
         # Build remaining services (caching OK for these)
         docker compose -f docker-compose.test.yml --env-file test.env build
 
-    - name: Apply and check test database migrations
-      run: bash ci/test-migrations.sh
-
     - name: Start services
       run: |
         # Start all services in detached mode
@@ -100,6 +97,24 @@ jobs:
         # Show running containers
         docker compose -f docker-compose.test.yml ps
 
+    - name: Run Database Migrations
+      env:
+        DATABASE_URL: postgres://postgres:PdwPNS2mDN73Vfbc@localhost:5432/polis-test
+        POSTGRES_DB: polis-test
+        POSTGRES_HOST: postgres:5432
+        POSTGRES_PASSWORD: PdwPNS2mDN73Vfbc
+        POSTGRES_PORT: 5432
+        POSTGRES_USER: postgres
+      run: |
+        echo "Installing postgres-client..."
+        sudo apt-get update && sudo apt-get install -y postgresql-client
+
+        echo "Making migration script executable..."
+        chmod +x server/bin/run-migrations.sh
+
+        echo "Running migrations..."
+        ./server/bin/run-migrations.sh
+
     - name: Check service health
       run: |
         # Check if key services are responding

.github/workflows/jest-server-test.yml

@@ -97,9 +97,6 @@ jobs:
         docker compose -f docker-compose.test.yml --env-file test.env build \
           postgres file-server ses-local oidc-simulator dynamodb
 
-    - name: Apply and check test database migrations
-      run: bash ci/test-migrations.sh
-
     - name: Start services
       run: |
         # Start only required services in detached mode (exclude server + math-python)
@@ -147,13 +144,6 @@ jobs:
         cd server
         npm run contract:check
 
-    - name: Check API migration readiness
-      working-directory: server
-      env:
-        DATABASE_URL: postgres://postgres:PdwPNS2mDN73Vfbc@localhost:5432/polis-test
-        DATABASE_SSL: "false"
-      run: node src/db/migrations.cjs
-
     - name: Run server integration tests
       run: |
         cd server

.github/workflows/python-ci.yml

@@ -11,12 +11,12 @@ on:
       - 'delphi/**/*.py'
       - 'delphi/requirements*.txt'
       - 'delphi/Dockerfile'
+      - 'scripts/*install.sh'
+      - 'scripts/application_stop.sh'
+      - 'scripts/validate_service.sh'
+      - 'scripts/test-deploy-hooks.sh'
+      - 'docker-compose*.yml'
       - '.github/workflows/python-ci.yml'
-      - 'ci/test-migrations.sh'
-      - 'docker-compose.test.yml'
-      - 'server/Dockerfile-db'
-      - 'server/postgres/**'
-      - 'queue-rs/polis-migrate/**'
       # server/src so a new server-side wildcard runs the projection-gate sweep
       - 'server/src/**'
       # Representative payload tests execute the probe planner and independent gate.
@@ -75,9 +75,6 @@ jobs:
         # Build all services in the test file (including delphi)
         docker compose -f docker-compose.test.yml --env-file .env build
 
-    - name: Apply and check test database migrations
-      run: bash ci/test-migrations.sh
-
     - name: 4. Start all services
       run: |
         # Start all services (including delphi) in detached mode
@@ -98,7 +95,7 @@ jobs:
 
     # The opt-in Postgres integration tests (require_polis_postgres) run here
     # against the compose `postgres` service, whose image bakes the polis
-    # migrations through polis-migrate during initialization and the explicit CI step,
+    # migrations (server/postgres/migrations/*.sql via docker-entrypoint-initdb.d),
     # so the votes / votes_latest_unique schema + on_vote_insert_update_unique_table
     # rule are already applied. The pytest step exports POLIS_TEST_POSTGRES_URL;
     # the cold-start generator under test is already baked into the delphi image
@@ -113,13 +110,8 @@ jobs:
         echo "Copying script to be tested into container..."
         docker compose -f docker-compose.test.yml cp delphi/polismath/run_math_pipeline.py delphi:/app/run_math_pipeline.py
         docker compose -f docker-compose.test.yml cp delphi/umap_narrative delphi:/app/umap_narrative
-        echo "Copying the compose files into container..."
-        # tests/test_compose_math_env.py asserts that every stack's delphi service
-        # receives the MATH_ENV its math service writes under. /app/tests has no
-        # checkout above it, so put the two files it parses beside it; without
-        # them the test skips instead of guarding the report pipeline's math_env.
-        docker compose -f docker-compose.test.yml cp docker-compose.yml delphi:/app/docker-compose.yml
-        docker compose -f docker-compose.test.yml cp docker-compose.test.yml delphi:/app/docker-compose.test.yml
+        # The same container-layout preparation and hook tests run on mm5.
+        bash scripts/test-deploy-hooks.sh
 
         # Wire a checkout-shaped root so delphi/tests/scripts collects AND RUNS the
         # pure-Python inventory sweep + the DB-free gate unit tests (the conftest
@@ -135,9 +127,6 @@ jobs:
         # its first item (observed: only server/src copied).
         # Recordings tests also load the packer and its digest helper from ci/.
         # battery_coverage.py is already included in the declared delphi/scripts.
-        # test_compose_math_env.py reads the deploy hook's per-role compose lines.
-        # test_before_install_hook.py runs scripts/before_install.sh against a fake docker.
-        # test_compose_math_env.py also reads the stop hook's per-role stop lines.
         # Representative payload tests import the box planner/gate and their
         # helpers from this same checkout root; keep their relative paths intact.
         # Light-shadow triage tests also import the compare/triage modules.
@@ -149,8 +138,7 @@ jobs:
           ci/private_cert/image_admission.py ci/probe_box/receipt.py \
           ci/probe_box/contracts.py ci/probe_box/light_shadow.py ci/probe_box/light_shadow_queries.py \
           ci/private_cert/images/light_shadow_compare.py ci/private_cert/images/light_shadow_triage.py \
-          ci/private_cert/images/recipe.py \
-          scripts/after_install.sh scripts/before_install.sh scripts/application_stop.sh; do
+          ci/private_cert/images/recipe.py; do
           docker compose -f docker-compose.test.yml exec -T delphi mkdir -p "/app/projgate/$(dirname "$rel")"
           docker compose -f docker-compose.test.yml cp "$rel" "delphi:/app/projgate/$rel" \
             || { echo "failed to copy scan input: $rel"; exit 1; }

.github/workflows/queue-rs-ci.yml

@@ -9,11 +9,6 @@ on:
   pull_request:
     paths:
       - 'queue-rs/**'
-      - 'server/index.ts'
-      - 'server/src/db/migrations.cjs'
-      - 'server/Dockerfile-db'
-      - 'server/postgres/init-migrations.sh'
-      - 'scripts/after_install.sh'
       - 'file-server/nginx/**'
       - 'file-server/nginx.Dockerfile'
       - 'server/postgres/migrations/**'
@@ -22,11 +17,6 @@ on:
     branches: [edge, stable]
     paths:
       - 'queue-rs/**'
-      - 'server/index.ts'
-      - 'server/src/db/migrations.cjs'
-      - 'server/Dockerfile-db'
-      - 'server/postgres/init-migrations.sh'
-      - 'scripts/after_install.sh'
       - 'file-server/nginx/**'
       - 'file-server/nginx.Dockerfile'
       - 'server/postgres/migrations/**'
@@ -120,40 +110,3 @@ jobs:
       - name: nginx routing switch (off, absent/up/502/dead, bad names and settings, method and body gate, truncation limit)
         working-directory: .
         run: queue-rs/polis-api/conformance/nginx-routing.sh
-
-  polis-migrate:
-    name: migration runner and startup refusal
-    runs-on: ubuntu-24.04
-    timeout-minutes: 30
-    env:
-      COMPOSE_PROJECT_NAME: polis-migrate-test-ci-${{ github.run_id }}-${{ github.run_attempt }}
-      POLIS_RECOVERY_PG_PORT: '55850'
-      RECOVERY_PG_PORT: '55850'
-    steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
-        with:
-          persist-credentials: false
-      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020
-        with:
-          node-version: '22.23.3'
-      - name: Build and lint the runner
-        working-directory: queue-rs
-        run: |
-          cargo fmt -p polis-migrate --check
-          cargo clippy --locked -p polis-migrate --all-targets -- -D warnings
-          cargo test --locked -p polis-migrate
-          cargo build --locked -p polis-migrate
-      - name: Compile the API entrypoint
-        working-directory: server
-        run: |
-          npm ci --ignore-scripts --no-audit --no-fund
-          npm run build
-      - name: Isolated real PostgreSQL proofs
-        run: |
-          docker compose -f queue-rs/polis-migrate/tests/compose.yml up -d --wait
-          python3 queue-rs/polis-migrate/tests/prove.py
-      - name: Prove the fresh database image and restart
-        run: bash queue-rs/polis-migrate/tests/fresh-image.sh
-      - name: Remove only the owned test project
-        if: always()
-        run: docker compose -f queue-rs/polis-migrate/tests/compose.yml down -v

CHANGELOG.md

@@ -31,20 +31,3 @@ Changes which have been merged to `edge` but are not yet versioned on `stable` c
 * ...
 
 
-
-## Migration runner
-
-Deployments now apply pending numbered migrations through `polis-migrate` before
-service replacement. The API refuses startup with pending or mismatched history.
-Existing databases require one-time catalog-checked adoption. See
-[upgrading](docs/upgrading.md) for the first CodeDeploy hook transition and the
-release-wide coordinator hold. PostgreSQL 17+ is required.
-
-The explicit release manifest admits the supported legacy schema then applies
-M19/M23/M24; M20/M21/M25/M26 stay outside the forward path. Deprecated M4/M5/M7
-are observation-only, never automatic destructive steps. Exact legacy type
-alternatives, a catalog-only first-deploy report, per-deployment upgrade notes
-and a source-to-release map are documented in
-[migration upgrade notes](docs/migration-upgrade-notes.md). Historical SQL
-checksums remain unchanged; no semantic version is invented for unversioned
-historical releases.

Makefile

@@ -177,7 +177,8 @@ refresh-devdb: ## Force dev DB mode (migrations), drop postgres_data volume, and
 # P-022 §C — poller recovery matrix (R01-R12) on a REAL Postgres
 # ---------------------------------------------------------------------------- #
 # Reuses docker-compose.test.yml's postgres service (built from
-# server/Dockerfile-db, which runs polis-migrate during fresh initialization) with docker-compose.recovery.yml overriding the
+# server/Dockerfile-db, which bakes server/postgres/migrations/*.sql into
+# docker-entrypoint-initdb.d) with docker-compose.recovery.yml overriding the
 # host port and making the data directory a tmpfs, so every `up` re-runs initdb
 # with the real migrations and nothing survives teardown.
 #

appspec.yml

@@ -23,4 +23,8 @@ hooks:
   ApplicationStop:
     - location: scripts/application_stop.sh
       timeout: 300
-      runas: root
+      runas: root
+  ValidateService:
+    - location: scripts/validate_service.sh
+      timeout: 1200
+      runas: root

bin/run-migrations.clj

@@ -1,5 +1,143 @@
 #!/usr/bin/env bb
-;; The 2021 ledger idea continues in polis-migrate. Do not replay all SQL here.
-(require '[babashka.process :as process])
-(let [result @(process/process ["bash" "server/bin/run-migrations.sh"] {:inherit true})]
-  (System/exit (:exit result)))
+
+(require '[babashka.pods :as pods]
+         '[babashka.deps :as deps]
+         '[clojure.pprint :as pp]
+         '[clojure.tools.cli :as cli]
+         '[clojure.java.io :as io]
+         '[clojure.string :as string])
+
+(pods/load-pod 'org.babashka/postgresql "0.0.1")
+(deps/add-deps '{:deps {honeysql/honeysql {:mvn/version "1.0.444"}}})
+
+(require '[pod.babashka.postgresql :as pg]
+         '[honeysql.core :as hsql]
+         '[honeysql.helpers :as hsqlh])
+
+
+
+(def db-url
+  (System/getenv "DATABASE_URL"))
+
+(defn heroku-url-spec [db-url]
+  (let [[_ user password host port db] (re-matches #"postgres://(?:(.+):(.*)@)?([^:]+)(?::(\d+))?/(.+)" db-url)]
+    {:dbtype "postgresql"
+     :host host
+     :dbname db
+     :port (or port 80)
+     :user user
+     :password password}))
+
+(defn execute-sql! [args]
+  (println "Executing sql:" args)
+  (pg/execute!
+    (heroku-url-spec (System/getenv "DATABASE_URL"))
+    args))
+
+
+;(def execute-sql!
+  ;(partial pg/execute! (heroku-url-spec (System/getenv "DATABASE_URL"))))
+
+(defn execute!
+  [query-or-command]
+  (execute-sql! (hsql/format query-or-command)))
+
+
+(defn insert!
+  [table values]
+  (execute! {:insert-into table
+             :values values}))
+
+;(hsqlh/values [{:a "this" :b 4}])
+
+;(-> (hsqlh/insert-into :migrations)
+    ;(hsqlh/values [{:a "this" :b 3}])
+    ;(hsql/format))
+
+
+
+;; get about the migration business
+
+(def migrations-path "server/postgres/migrations/")
+
+(defn sql-file?
+  [file]
+  (re-matches #".*\.sql" (str file)))
+
+(defn table-exists?
+  [table-name]
+  (->
+    (execute! {:select [:*]
+               :from [:information_schema.tables]
+               :where [:= :table_name (name table-name)]})
+    (not-empty)
+    (boolean)))
+
+;(table-exists? :migrations)
+;(table-exists? :fish)
+
+(defn remember-tx-migration! [name]
+  (insert! :migrations
+           [{:name name
+             :completed_at (System/currentTimeMillis)}]))
+
+;; Make sure we have a migrations table, which is basically just a list of filenames which have been
+;; transacted, as well as datetime
+(when-not (table-exists? :migrations)
+  (execute-sql!
+    ["CREATE TABLE migrations
+     (name VARCHAR(999) NOT NULL,
+      completed_at BIGINT NOT NULL);"]))
+
+(defn process-mig-file! [mig-file]
+  (let [mig-file (io/file mig-file)
+        name (.getName mig-file)]
+    (println "Processing migration file" name)
+    (execute-sql! [(slurp mig-file)])
+    (remember-tx-migration! name)))
+
+(defn migration-files []
+  (->> (.listFiles (io/file migrations-path))
+       (remove #(.isDirectory %))
+       (filter sql-file?)
+       (sort)))
+
+(defn remove-past-migrations
+  [mig-files]
+  (let [past-migrations
+        (->> 
+          (execute! {:select [:name]
+                     :from [:migrations]})
+          (map :migrations/name)
+          (set))]
+    (remove (comp past-migrations #(.getName %))
+            mig-files)))
+
+(defn new-migration-files
+  []
+  (remove-past-migrations (migration-files)))
+
+;(remove-past-migrations (migration-files))
+
+;(.getName (io/file "server/postgres/migrations/000000_initial.sql"))
+;(.getParent (io/file "server/postgres/migrations/000000_initial.sql"))
+
+(when-not (table-exists? :conversations)
+  (process-mig-file! "server/postgres/migrations/000000_initial.sql")
+  (remember-tx-migration! "000000_initial.sql"))
+
+(when-not (table-exists? :pwreset_tokens)
+  (process-mig-file! "server/postgres/migrations/000001_update_pwreset_table.sql")
+  (remember-tx-migration! "000001_update_pwreset_table.sql"))
+
+(def past-migrations
+  (execute! {:select [:*]
+             :from [:migrations]}))
+
+(let [mig-files (new-migration-files)]
+  (if (empty? mig-files)
+    (println "No new migrations to run")
+    (doseq [mig-file (new-migration-files)]
+      (process-mig-file! mig-file))))
+
+

ci/p027_rerecord_build.sh

@@ -6,7 +6,7 @@ export DOCKER_BUILDKIT=1
 # Sequential builds bound peak memory; inspect disk receipts when sizing the runner.
 df -h .
 docker build --target prod --build-arg NODE_ENV=production -t p027-server -f server/Dockerfile server
-docker build --build-context queue-rs=queue-rs -t p027-postgres -f server/Dockerfile-db server
+docker build -t p027-postgres -f server/Dockerfile-db server
 docker build -t p027-oidc-simulator oidc-simulator
 docker build -t p027-file-server --build-arg NODE_ENV=production \
   --build-arg AUTH_AUDIENCE=users --build-arg AUTH_CLIENT_ID=dev-client-id \