diff --git a/.github/workflows/cypress-tests.yml b/.github/workflows/cypress-tests.yml index bb7313c280..2272503842 100644 --- a/.github/workflows/cypress-tests.yml +++ b/.github/workflows/cypress-tests.yml @@ -85,6 +85,9 @@ jobs: # Build remaining services (caching OK for these) docker compose -f docker-compose.test.yml --env-file test.env build + - name: Apply and check test database migrations + run: bash ci/test-migrations.sh + - name: Start services run: | # Start all services in detached mode @@ -97,24 +100,6 @@ jobs: # Show running containers docker compose -f docker-compose.test.yml ps - - name: Run Database Migrations - env: - DATABASE_URL: postgres://postgres:PdwPNS2mDN73Vfbc@localhost:5432/polis-test - POSTGRES_DB: polis-test - POSTGRES_HOST: postgres:5432 - POSTGRES_PASSWORD: PdwPNS2mDN73Vfbc - POSTGRES_PORT: 5432 - POSTGRES_USER: postgres - run: | - echo "Installing postgres-client..." - sudo apt-get update && sudo apt-get install -y postgresql-client - - echo "Making migration script executable..." - chmod +x server/bin/run-migrations.sh - - echo "Running migrations..." - ./server/bin/run-migrations.sh - - name: Check service health run: | # Check if key services are responding diff --git a/.github/workflows/jest-server-test.yml b/.github/workflows/jest-server-test.yml index fe54f8593e..0b7122b224 100644 --- a/.github/workflows/jest-server-test.yml +++ b/.github/workflows/jest-server-test.yml @@ -97,6 +97,9 @@ jobs: docker compose -f docker-compose.test.yml --env-file test.env build \ postgres file-server ses-local oidc-simulator dynamodb + - name: Apply and check test database migrations + run: bash ci/test-migrations.sh + - name: Start services run: | # Start only required services in detached mode (exclude server + math-python) @@ -144,6 +147,13 @@ jobs: cd server npm run contract:check + - name: Check API migration readiness + working-directory: server + env: + DATABASE_URL: postgres://postgres:PdwPNS2mDN73Vfbc@localhost:5432/polis-test + DATABASE_SSL: "false" + run: node src/db/migrations.cjs + - name: Run server integration tests run: | cd server diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index 938a58c270..83ae72efe2 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -12,6 +12,11 @@ on: - 'delphi/requirements*.txt' - 'delphi/Dockerfile' - '.github/workflows/python-ci.yml' + - 'ci/test-migrations.sh' + - 'docker-compose.test.yml' + - 'server/Dockerfile-db' + - 'server/postgres/**' + - 'queue-rs/polis-migrate/**' # server/src so a new server-side wildcard runs the projection-gate sweep - 'server/src/**' # Representative payload tests execute the probe planner and independent gate. @@ -70,6 +75,9 @@ jobs: # Build all services in the test file (including delphi) docker compose -f docker-compose.test.yml --env-file .env build + - name: Apply and check test database migrations + run: bash ci/test-migrations.sh + - name: 4. Start all services run: | # Start all services (including delphi) in detached mode @@ -90,7 +98,7 @@ jobs: # The opt-in Postgres integration tests (require_polis_postgres) run here # against the compose `postgres` service, whose image bakes the polis - # migrations (server/postgres/migrations/*.sql via docker-entrypoint-initdb.d), + # migrations through polis-migrate during initialization and the explicit CI step, # so the votes / votes_latest_unique schema + on_vote_insert_update_unique_table # rule are already applied. The pytest step exports POLIS_TEST_POSTGRES_URL; # the cold-start generator under test is already baked into the delphi image diff --git a/.github/workflows/queue-rs-ci.yml b/.github/workflows/queue-rs-ci.yml index 2c9e166aed..0a0216ee64 100644 --- a/.github/workflows/queue-rs-ci.yml +++ b/.github/workflows/queue-rs-ci.yml @@ -9,6 +9,11 @@ on: pull_request: paths: - 'queue-rs/**' + - 'server/index.ts' + - 'server/src/db/migrations.cjs' + - 'server/Dockerfile-db' + - 'server/postgres/init-migrations.sh' + - 'scripts/after_install.sh' - 'file-server/nginx/**' - 'file-server/nginx.Dockerfile' - 'server/postgres/migrations/**' @@ -17,6 +22,11 @@ on: branches: [edge, stable] paths: - 'queue-rs/**' + - 'server/index.ts' + - 'server/src/db/migrations.cjs' + - 'server/Dockerfile-db' + - 'server/postgres/init-migrations.sh' + - 'scripts/after_install.sh' - 'file-server/nginx/**' - 'file-server/nginx.Dockerfile' - 'server/postgres/migrations/**' @@ -110,3 +120,40 @@ jobs: - name: nginx routing switch (off, absent/up/502/dead, bad names and settings, method and body gate, truncation limit) working-directory: . run: queue-rs/polis-api/conformance/nginx-routing.sh + + polis-migrate: + name: migration runner and startup refusal + runs-on: ubuntu-24.04 + timeout-minutes: 30 + env: + COMPOSE_PROJECT_NAME: polis-migrate-test-ci-${{ github.run_id }}-${{ github.run_attempt }} + POLIS_RECOVERY_PG_PORT: '55850' + RECOVERY_PG_PORT: '55850' + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + persist-credentials: false + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 + with: + node-version: '22.23.3' + - name: Build and lint the runner + working-directory: queue-rs + run: | + cargo fmt -p polis-migrate --check + cargo clippy --locked -p polis-migrate --all-targets -- -D warnings + cargo test --locked -p polis-migrate + cargo build --locked -p polis-migrate + - name: Compile the API entrypoint + working-directory: server + run: | + npm ci --ignore-scripts --no-audit --no-fund + npm run build + - name: Isolated real PostgreSQL proofs + run: | + docker compose -f queue-rs/polis-migrate/tests/compose.yml up -d --wait + python3 queue-rs/polis-migrate/tests/prove.py + - name: Prove the fresh database image and restart + run: bash queue-rs/polis-migrate/tests/fresh-image.sh + - name: Remove only the owned test project + if: always() + run: docker compose -f queue-rs/polis-migrate/tests/compose.yml down -v diff --git a/CHANGELOG.md b/CHANGELOG.md index 2995ba5a3f..ea736339c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,3 +31,20 @@ Changes which have been merged to `edge` but are not yet versioned on `stable` c * ... + +## Migration runner + +Deployments now apply pending numbered migrations through `polis-migrate` before +service replacement. The API refuses startup with pending or mismatched history. +Existing databases require one-time catalog-checked adoption. See +[upgrading](docs/upgrading.md) for the first CodeDeploy hook transition and the +release-wide coordinator hold. PostgreSQL 17+ is required. + +The explicit release manifest admits the supported legacy schema then applies +M19/M23/M24; M20/M21/M25/M26 stay outside the forward path. Deprecated M4/M5/M7 +are observation-only, never automatic destructive steps. Exact legacy type +alternatives, a catalog-only first-deploy report, per-deployment upgrade notes +and a source-to-release map are documented in +[migration upgrade notes](docs/migration-upgrade-notes.md). Historical SQL +checksums remain unchanged; no semantic version is invented for unversioned +historical releases. diff --git a/Makefile b/Makefile index 7fbf8ff6ea..23ff6b5650 100644 --- a/Makefile +++ b/Makefile @@ -177,8 +177,7 @@ refresh-devdb: ## Force dev DB mode (migrations), drop postgres_data volume, and # P-022 §C — poller recovery matrix (R01-R12) on a REAL Postgres # ---------------------------------------------------------------------------- # # Reuses docker-compose.test.yml's postgres service (built from -# server/Dockerfile-db, which bakes server/postgres/migrations/*.sql into -# docker-entrypoint-initdb.d) with docker-compose.recovery.yml overriding the +# server/Dockerfile-db, which runs polis-migrate during fresh initialization) with docker-compose.recovery.yml overriding the # host port and making the data directory a tmpfs, so every `up` re-runs initdb # with the real migrations and nothing survives teardown. # diff --git a/bin/run-migrations.clj b/bin/run-migrations.clj index 67bdb65d0d..72b875d1a4 100755 --- a/bin/run-migrations.clj +++ b/bin/run-migrations.clj @@ -1,143 +1,5 @@ #!/usr/bin/env bb - -(require '[babashka.pods :as pods] - '[babashka.deps :as deps] - '[clojure.pprint :as pp] - '[clojure.tools.cli :as cli] - '[clojure.java.io :as io] - '[clojure.string :as string]) - -(pods/load-pod 'org.babashka/postgresql "0.0.1") -(deps/add-deps '{:deps {honeysql/honeysql {:mvn/version "1.0.444"}}}) - -(require '[pod.babashka.postgresql :as pg] - '[honeysql.core :as hsql] - '[honeysql.helpers :as hsqlh]) - - - -(def db-url - (System/getenv "DATABASE_URL")) - -(defn heroku-url-spec [db-url] - (let [[_ user password host port db] (re-matches #"postgres://(?:(.+):(.*)@)?([^:]+)(?::(\d+))?/(.+)" db-url)] - {:dbtype "postgresql" - :host host - :dbname db - :port (or port 80) - :user user - :password password})) - -(defn execute-sql! [args] - (println "Executing sql:" args) - (pg/execute! - (heroku-url-spec (System/getenv "DATABASE_URL")) - args)) - - -;(def execute-sql! - ;(partial pg/execute! (heroku-url-spec (System/getenv "DATABASE_URL")))) - -(defn execute! - [query-or-command] - (execute-sql! (hsql/format query-or-command))) - - -(defn insert! - [table values] - (execute! {:insert-into table - :values values})) - -;(hsqlh/values [{:a "this" :b 4}]) - -;(-> (hsqlh/insert-into :migrations) - ;(hsqlh/values [{:a "this" :b 3}]) - ;(hsql/format)) - - - -;; get about the migration business - -(def migrations-path "server/postgres/migrations/") - -(defn sql-file? - [file] - (re-matches #".*\.sql" (str file))) - -(defn table-exists? - [table-name] - (-> - (execute! {:select [:*] - :from [:information_schema.tables] - :where [:= :table_name (name table-name)]}) - (not-empty) - (boolean))) - -;(table-exists? :migrations) -;(table-exists? :fish) - -(defn remember-tx-migration! [name] - (insert! :migrations - [{:name name - :completed_at (System/currentTimeMillis)}])) - -;; Make sure we have a migrations table, which is basically just a list of filenames which have been -;; transacted, as well as datetime -(when-not (table-exists? :migrations) - (execute-sql! - ["CREATE TABLE migrations - (name VARCHAR(999) NOT NULL, - completed_at BIGINT NOT NULL);"])) - -(defn process-mig-file! [mig-file] - (let [mig-file (io/file mig-file) - name (.getName mig-file)] - (println "Processing migration file" name) - (execute-sql! [(slurp mig-file)]) - (remember-tx-migration! name))) - -(defn migration-files [] - (->> (.listFiles (io/file migrations-path)) - (remove #(.isDirectory %)) - (filter sql-file?) - (sort))) - -(defn remove-past-migrations - [mig-files] - (let [past-migrations - (->> - (execute! {:select [:name] - :from [:migrations]}) - (map :migrations/name) - (set))] - (remove (comp past-migrations #(.getName %)) - mig-files))) - -(defn new-migration-files - [] - (remove-past-migrations (migration-files))) - -;(remove-past-migrations (migration-files)) - -;(.getName (io/file "server/postgres/migrations/000000_initial.sql")) -;(.getParent (io/file "server/postgres/migrations/000000_initial.sql")) - -(when-not (table-exists? :conversations) - (process-mig-file! "server/postgres/migrations/000000_initial.sql") - (remember-tx-migration! "000000_initial.sql")) - -(when-not (table-exists? :pwreset_tokens) - (process-mig-file! "server/postgres/migrations/000001_update_pwreset_table.sql") - (remember-tx-migration! "000001_update_pwreset_table.sql")) - -(def past-migrations - (execute! {:select [:*] - :from [:migrations]})) - -(let [mig-files (new-migration-files)] - (if (empty? mig-files) - (println "No new migrations to run") - (doseq [mig-file (new-migration-files)] - (process-mig-file! mig-file)))) - - +;; The 2021 ledger idea continues in polis-migrate. Do not replay all SQL here. +(require '[babashka.process :as process]) +(let [result @(process/process ["bash" "server/bin/run-migrations.sh"] {:inherit true})] + (System/exit (:exit result))) diff --git a/ci/p027_rerecord_build.sh b/ci/p027_rerecord_build.sh index 33eec4134f..1c4394ca63 100644 --- a/ci/p027_rerecord_build.sh +++ b/ci/p027_rerecord_build.sh @@ -6,7 +6,7 @@ export DOCKER_BUILDKIT=1 # Sequential builds bound peak memory; inspect disk receipts when sizing the runner. df -h . docker build --target prod --build-arg NODE_ENV=production -t p027-server -f server/Dockerfile server -docker build -t p027-postgres -f server/Dockerfile-db server +docker build --build-context queue-rs=queue-rs -t p027-postgres -f server/Dockerfile-db server docker build -t p027-oidc-simulator oidc-simulator docker build -t p027-file-server --build-arg NODE_ENV=production \ --build-arg AUTH_AUDIENCE=users --build-arg AUTH_CLIENT_ID=dev-client-id \ diff --git a/ci/test-migrations.sh b/ci/test-migrations.sh new file mode 100644 index 0000000000..fb92f902e8 --- /dev/null +++ b/ci/test-migrations.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# Disposable CI stack only. Use the runner baked into the Postgres image on +# fresh AND existing volumes before starting application services. No host +# Rust/psql installation, second history table or startup-check bypass. +# Optional Compose arguments (e.g. -f local-ports.yml) preserve test isolation. +set -euo pipefail +root=$(cd "$(dirname "$0")/.." && pwd) +cd "$root" +compose=(docker compose -f docker-compose.test.yml "$@" --env-file "${POLIS_TEST_ENV_FILE:-test.env}") +"${compose[@]}" up -d --wait --wait-timeout 120 postgres +"${compose[@]}" exec -T postgres sh -eu -c ' + export DATABASE_URL="host=/var/run/postgresql user=$POSTGRES_USER dbname=$POSTGRES_DB sslmode=disable" + export POLIS_MIGRATIONS_DIR=/migrations + polis-migrate apply + polis-migrate check +' diff --git a/delphi/tests/test_after_install_hook.py b/delphi/tests/test_after_install_hook.py index cceaa0a139..7111610e6a 100644 --- a/delphi/tests/test_after_install_hook.py +++ b/delphi/tests/test_after_install_hook.py @@ -1,4 +1,4 @@ -"""scripts/after_install.sh selects one branch per service type, and a queue +"""scripts/after_install.sh migrates before replacing any service, and a queue worker box (delphi-large, delphi-worker) starts only its polis-jobs.service. Runs the real deploy hook (and the stop hook) with every external command @@ -75,6 +75,7 @@ def _find(rel): ' *) echo \'{"username":"u","password":"p","dbname":"d"}\';;\n' 'esac\n'), "docker": RECORD + ( + 'if [ "$1" = "$FAKE_MIGRATION_FAILURE" ]; then exit 17; fi\n' 'if [ "$1" = ps ]; then\n' ' filter=""\n' ' while [ $# -gt 0 ]; do [ "$1" = --filter ] && filter="${2#name=}"; shift; done\n' @@ -124,8 +125,9 @@ def _run(tmp_path, script_path, rules, root, env): return proc, log -def _deploy(tmp_path, service_type, **kw): +def _deploy(tmp_path, service_type, migration_failure="", **kw): root, env = _box(tmp_path, service_type, **kw) + env["FAKE_MIGRATION_FAILURE"] = migration_failure rules = [("/usr/local/bin/docker-compose", "docker-compose", 4), ("/etc/app-info/", f"{root}/etc/app-info/", 2), ("/etc/systemd/system/", f"{root}/etc/systemd/system/", 0), @@ -206,17 +208,44 @@ def test_a_worker_box_that_does_not_match_fails_the_deploy(tmp_path, service_typ assert _calls(log, "systemctl") == [] -@pytest.mark.parametrize("service_type,stops", [("delphi", ["stop delphi"]), - ("delphi-large", []), ("delphi-worker", [])]) -def test_the_stop_hook_stops_no_worker_service(tmp_path, service_type, stops): +@pytest.mark.parametrize("service_type", ["server", "delphi", "math", "delphi-large", "delphi-worker", "unknown"]) +def test_the_stop_hook_leaves_every_service_running(tmp_path, service_type): root, env = _box(tmp_path, service_type) - (root / "opt/polis/polis").mkdir(parents=True) - # command -v needs the compose path to exist; point it at the fake. - rules = [("/usr/local/bin/docker-compose", str(tmp_path / "bin/docker-compose"), 2), - ("/etc/app-info/", f"{root}/etc/app-info/", 1), - ("/opt/polis/polis", f"{root}/opt/polis/polis", 1)] - proc, log = _run(tmp_path, STOP_PATH, rules, root, env) + # No path rewrite: the revised hook has no external calls or absolute paths. + proc, log = _run(tmp_path, STOP_PATH, [], root, env) assert proc.returncode == 0, proc.stdout + proc.stderr - assert [c for c in _calls(log, "docker-compose") if c.startswith("stop")] == stops + assert log == [] + assert "AfterInstall" in proc.stdout + + +@pytest.mark.parametrize("service_type,worker_class", [ + ("server", None), ("delphi", None), ("math", None), ("ollama", None), + ("delphi-large", "large"), ("delphi-worker", "delphi"), +]) +def test_migration_succeeds_before_any_service_replacement(tmp_path, service_type, worker_class): + proc, log = _deploy(tmp_path, service_type, worker_class=worker_class) + assert proc.returncode == 0, proc.stdout + proc.stderr + build = "docker build -t polis-migrate:deploy -f queue-rs/polis-migrate/Dockerfile ." + apply = "docker run --rm --network host --env-file .env polis-migrate:deploy apply" + assert log.count(build) == log.count(apply) == 1 + assert log.index(build) < log.index(apply) + replacements = [i for i, call in enumerate(log) if call.startswith(( + "docker-compose down", "docker rm ", "docker system prune", + "docker-compose up", "systemctl restart"))] + assert replacements + assert log.index(apply) < min(replacements) + + +@pytest.mark.parametrize("failure", ["build", "run"]) +@pytest.mark.parametrize("service_type,worker_class", [ + ("server", None), ("delphi", None), ("math", None), ("ollama", None), + ("delphi-large", "large"), ("delphi-worker", "delphi"), +]) +def test_migration_failure_preserves_every_running_service(tmp_path, service_type, worker_class, failure): + proc, log = _deploy(tmp_path, service_type, worker_class=worker_class, + migration_failure=failure) + assert proc.returncode == 17, proc.stdout + proc.stderr + assert _calls(log, "docker-compose") == [] assert _calls(log, "systemctl") == [] - assert "Unknown service type" not in proc.stdout + proc.stderr + docker = _calls(log, "docker") + assert [call.split()[0] for call in docker] == (["build"] if failure == "build" else ["build", "run"]) diff --git a/delphi/tests/test_before_install_hook.py b/delphi/tests/test_before_install_hook.py index a9dbbd3b2f..71a36998e4 100644 --- a/delphi/tests/test_before_install_hook.py +++ b/delphi/tests/test_before_install_hook.py @@ -1,20 +1,10 @@ -"""scripts/before_install.sh stops only the exact containers it names. +"""BeforeInstall leaves every healthy service running until AfterInstall migrates. -Docker's ``name`` filter is an unanchored regex, so ``--filter name=polis-math`` -also matched the Delphi box's ``polis-math-python-1``; the hook then ran -``docker stop polis-math-1``, which exists only on the math box, and the -production BeforeInstall failed with ``No such container: polis-math-1``. - -Runs the real hook against a fake ``docker`` on PATH that applies the filter -the way the daemon does (regex search against the name, with and without its -leading ``/``) and fails ``stop`` for a container that is not running. The hook -is located like ``test_compose_math_env.py`` locates after_install.sh: -$POLIS_CHECKOUT_DIR, else an ancestor of this file (CI copies it into the -checkout-shaped root). +Run the exact hook with forbidden external commands on PATH. A reintroduced +stop, removal, daemon restart or migration here must fail these controls. """ import os -import re import shutil import subprocess from pathlib import Path @@ -45,117 +35,23 @@ def _find_before_install(): pytest.mark.skipif(shutil.which("bash") is None, reason="bash not available"), ] -FAKE_DOCKER = r"""#!/bin/bash -# Fake docker: running containers come from $FAKE_DOCKER_RUNNING. -echo "$*" >> "$FAKE_DOCKER_LOG" -case "$1" in - ps) - filter="" - while [ $# -gt 0 ]; do - if [ "$1" = "--filter" ]; then filter="$2"; shift; fi - shift - done - regex="${filter#name=}" - for n in $FAKE_DOCKER_RUNNING; do - if printf '%s\n' "$n" | grep -Eq -- "$regex" || printf '%s\n' "/$n" | grep -Eq -- "$regex"; then - echo "id-$n" - fi - done - ;; - stop) - for n in $FAKE_DOCKER_RUNNING; do - if [ "$n" = "$2" ]; then echo "$2"; exit 0; fi - done - echo "Error response from daemon: No such container: $2" >&2 - exit 1 - ;; - *) - echo "fake docker: unexpected command: $*" >&2 - exit 2 - ;; -esac -""" - - -@pytest.fixture -def fake_docker(tmp_path): +@pytest.mark.parametrize("running", [ + "polis-math-python-1", "polis-delphi-1 polis-math-python-1", + "polis-math-python-large-1 polis-jobs", "polis-math-1", + "polis-server-1 polis-server-helper-1", "", +]) +def test_before_install_never_touches_running_services(tmp_path, running): bin_dir = tmp_path / "bin" bin_dir.mkdir() - docker = bin_dir / "docker" - docker.write_text(FAKE_DOCKER) - docker.chmod(0o755) - log = tmp_path / "docker.log" - log.write_text("") - - def run(*running, script=None): - env = dict(os.environ) - env["PATH"] = f"{bin_dir}{os.pathsep}{env['PATH']}" - env["FAKE_DOCKER_RUNNING"] = " ".join(running) - env["FAKE_DOCKER_LOG"] = str(log) - log.write_text("") - if script is None: - args = ["bash", str(BEFORE_INSTALL_PATH)] - else: - args = ["bash", "-c", script] - result = subprocess.run(args, env=env, capture_output=True, text=True, timeout=30) - stops = [line.split()[1] for line in log.read_text().splitlines() if line.startswith("stop ")] - return result, stops - - return run - - -def test_fake_docker_filters_by_substring_like_the_daemon(fake_docker): - # Guards the fake: an unanchored filter must reproduce the production failure. - result, _ = fake_docker( - "polis-math-python-1", - script="docker ps -q --filter name=polis-math", - ) - assert result.stdout.strip() == "id-polis-math-python-1" - - -def test_delphi_box_with_math_python_stops_nothing_math(fake_docker): - result, stops = fake_docker("polis-math-python-1") - assert result.returncode == 0, result.stderr - assert stops == [] - - -def test_delphi_box_stops_delphi_but_not_math_python(fake_docker): - result, stops = fake_docker("polis-delphi-1", "polis-math-python-1") - assert result.returncode == 0, result.stderr - assert stops == ["polis-delphi-1"] - - -def test_large_box_stops_nothing(fake_docker): - # A container no anchored filter names (here the former large poller's - # name, which no compose file defines any more) is left alone: the - # large box's queue worker is not this hook's to stop. - result, stops = fake_docker("polis-math-python-large-1") - assert result.returncode == 0, result.stderr - assert stops == [] - - -def test_math_box_stops_math(fake_docker): - result, stops = fake_docker("polis-math-1") - assert result.returncode == 0, result.stderr - assert stops == ["polis-math-1"] - - -def test_server_box_stops_server(fake_docker): - result, stops = fake_docker("polis-server-1", "polis-server-helper-1") - assert result.returncode == 0, result.stderr - assert stops == ["polis-server-1"] - - -def test_no_containers_is_a_clean_exit(fake_docker): - result, stops = fake_docker() - assert result.returncode == 0, result.stderr - assert stops == [] - - -def test_every_name_filter_is_anchored_to_the_container_it_stops(): - text = BEFORE_INSTALL_PATH.read_text() - filters = re.findall(r'--filter "name=([^"]*)"', text) - stopped = re.findall(r"docker stop (\S+)", text) - assert filters, "before_install.sh no longer filters by name" - assert "--filter name=" not in text and "--filter 'name=" not in text - assert [f"^/?{name}$" for name in stopped] == filters + log = tmp_path / "calls" + for name in ("docker", "docker-compose", "systemctl", "sudo", "polis-migrate"): + stub = bin_dir / name + stub.write_text('#!/bin/sh\necho "$0 $*" >> "$FAKE_LOG"\nexit 91\n') + stub.chmod(0o755) + env = {"PATH": f"{bin_dir}:/usr/bin:/bin", "FAKE_LOG": str(log), + "FAKE_DOCKER_RUNNING": running} + result = subprocess.run(["bash", str(BEFORE_INSTALL_PATH)], env=env, + capture_output=True, text=True, timeout=30) + assert result.returncode == 0, result.stdout + result.stderr + assert not log.exists(), "BeforeInstall must not invoke service or migration commands" + assert "AfterInstall" in result.stdout diff --git a/delphi/tests/test_compose_math_env.py b/delphi/tests/test_compose_math_env.py index 951058e458..a0ac58f32c 100644 --- a/delphi/tests/test_compose_math_env.py +++ b/delphi/tests/test_compose_math_env.py @@ -732,4 +732,6 @@ def test_the_stop_hook_stops_nothing_on_the_large_box_and_no_large_poller_anywhe roles = _stop_lines() assert roles.get("delphi-large", []) == [] assert not any("math-python-large" in l for lines in roles.values() for l in lines) - assert "delphi-large" in STOP_HOOK.read_text() + # Shutdown moved after the migration barrier in AfterInstall, for every role. + assert roles == {} + assert "AfterInstall" in STOP_HOOK.read_text() diff --git a/docker-compose.test.yml b/docker-compose.test.yml index f489a829f1..f02113ac25 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -82,9 +82,12 @@ services: - ${AUTH_CERTS_PATH:-~/.simulacrum/certs}:/root/.simulacrum/certs:ro restart: unless-stopped depends_on: - - oidc-simulator - - postgres - - dynamodb-init + oidc-simulator: + condition: service_started + postgres: + condition: service_healthy + dynamodb-init: + condition: service_completed_successfully extra_hosts: - "host.docker.internal:host-gateway" @@ -112,7 +115,8 @@ services: networks: - polis-test depends_on: - - postgres + postgres: + condition: service_healthy restart: unless-stopped extra_hosts: - "host.docker.internal:host-gateway" @@ -160,6 +164,8 @@ services: build: context: ./server dockerfile: Dockerfile-db + additional_contexts: + queue-rs: ./queue-rs labels: polis_tag: test environment: @@ -167,7 +173,8 @@ services: - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} - POSTGRES_USER=${POSTGRES_USER} healthcheck: - test: ["CMD-SHELL", "pg_isready -U postgres"] + # The init server is socket-only; TCP refuses until migrations finish. + test: ["CMD-SHELL", "pg_isready -h 127.0.0.1 -U $$POSTGRES_USER -d $$POSTGRES_DB"] interval: 5s timeout: 5s retries: 5 @@ -175,7 +182,7 @@ services: networks: - polis-test ports: - - 5432:5432 + - "${POLIS_RECOVERY_PG_PORT:-5432}:5432" restart: unless-stopped # PostgreSQL configuration for development/testing command: > @@ -262,8 +269,10 @@ services: networks: - "polis-test" depends_on: - - postgres - - dynamodb-init + postgres: + condition: service_healthy + dynamodb-init: + condition: service_completed_successfully command: tail -f /dev/null restart: unless-stopped diff --git a/docker-compose.yml b/docker-compose.yml index d87a06b89f..30a6822fc7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -385,6 +385,8 @@ services: # Dockerfile-db (default): migrations-based initialization # Dockerfile-pdb: production dump restoration (requires prodclone.dump) dockerfile: Dockerfile-${DB_INIT_MODE:-db} + additional_contexts: + queue-rs: ./queue-rs labels: polis_tag: ${TAG:-dev} environment: diff --git a/docs/migration-legacy-contract.md b/docs/migration-legacy-contract.md new file mode 100644 index 0000000000..6aec852d47 --- /dev/null +++ b/docs/migration-legacy-contract.md @@ -0,0 +1,175 @@ +# Supported legacy schema contract + +This release accepts the documented legacy schema without changing its data or +rewriting old migrations. The live schema is authoritative for the named legacy +variant; the existing bootstrap is a separately supported installation variant. +The immutable numbered files are execution history, not a claim that every +installation ran every file. Adoption predicates describe the supported states. + +The concrete admission changes are bounded: `worker_tasks.task_type` accepts +`text` or `varchar(99)`; renamed `pwreset_tokens.token` accepts `varchar(100)` +or `varchar(250)`; absent `conversations.branding_type` and +`math_ticks.caching_tick` are valid, but existing columns must have their known +integer/bigint types. Contributor agreement tables are outside this release's +adoption scope. M2 requires enforced owner/xid uniqueness and preserves an +optional enforced, nonpartial owner/uid unique index in either key order. +Other lengths, arbitrary text alternatives, invalid indexes and partial unique +owner/uid indexes remain refused. + +Six math payloads accept `json` alongside bootstrap `jsonb`: `math_bidtopid`, +`math_cache`, `math_exportstatus`, `math_main`, `math_profile`, `math_ptptstats`. +`math_report_correlationmatrix.data` remains `jsonb`; no observed legacy JSON +contract was established for it. This narrows an earlier broad helper. + +The file contract below records the rest of the reviewed live differences. +It does not add new global core-default, routine-body or ACL adoption checks. +These documentary contracts must not be mistaken for an exhaustive schema +certifier. Existing modern M3/M8–18 predicates continue to enforce the properties +introduced by those migrations; unsupported/partial schemas stop for review. + +## Named live attributes + +| Attribute | Legacy contract | Bootstrap alternative | +| --- | --- | --- | +| comments.uid | integer NOT NULL, default 0 | no default | +| comments.velocity | nullable real, default 1 | NOT NULL real | +| conversations.auth_needed_to_vote | default false | no default | +| conversations.auth_needed_to_write | default true | no default | +| conversations.auth_opt_fb | default true | no default | +| conversations.auth_opt_tw | default true | no default | +| conversations.auth_opt_allow_3rdparty | default true | no default | +| contexts/context_id, conversations/zid, courses/course_id, participant_metadata_answers/pmaid, participant_metadata_questions/pmqid, users/uid sequences | bigint sequence, maximum 9223372036854775807, integer owning column | integer sequence, maximum 2147483647 | + +Sequence configuration is not current sequence state. Never reset, narrow or +advance a live sequence to match bootstrap. Reviewed implementations of +`get_times_for_most_recent_visible_comments()`, `pid_auto()`, `tid_auto()`, +`random_string(integer)` and `random_polis_site_id()` are executable equivalents +of the public historical sources; formatting fingerprints differ. This release +never replaces those routines. Their exact historical source bindings are in +the deployment's private reconciliation record. + +Optional probe provisioning grants SELECT on comments, conversations, math_main, +math_ticks, participants and votes, plus public schema USAGE, to the configured +read-only probe role. It is separate from application and migration authority; +no role identity or grant is imposed on other deployments by this contract. +M19's actual migration session needs role creation/SET authority, public schema +USAGE/CREATE grant authority, conversations SELECT plus topic UPDATE and zid +REFERENCES grant authority. M23/M24 continue as the queue owner. Verify the +actual session; a superuser-only test is insufficient evidence for that session. + +## Other live attributes and scope + +| Attribute | Type | Nullability | +| --- | --- | --- | +| comments.curation | smallint | NOT NULL | +| participants_extended.encrypted_ip_address | character varying(9999) | nullable | +| participants_extended.encrypted_x_forwarded_for | character varying(9999) | nullable | +| suzinvites.modified | bigint | nullable | +| suzinvites.uid | integer | nullable | +| users.pwhash | character varying(128) | nullable | + +Existing `suzinvites.uid` references users.uid; no field/constraint is added, +dropped or backfilled during reconciliation. Legacy callers and generated +schema descriptions may observe these fields; fresh bootstrap does not acquire +them merely because the legacy contract documents them. Whole-tree generated +schema modernization is a separate change. + +The following complete research-triage list records the stable scope. These are +research dispositions, not claims that bootstrap or generated ORM declarations +were rewritten in this patch. The explicit adoption changes and separate legacy +file contract described above are implemented here; references below to canonical +files/ORM or generated-reader changes remain broader modernization obligations. +“Before +stable” means a file/adoption contract, never authorization for production DDL. +Ancillary objects are retained. No extension removal, name rewrite, course-invite +unique constraint addition, contributor rename or data cleanup is selected. + +| Difference | Scope | Contract disposition | +| --- | --- | --- | +| contributor | SET ASIDE | Defer ancillary file spelling repair for stable; preserve live correct table and working endpoint, scoped non-gating exclusion and graceful503 for absent schemas. No production operation. | +| branding | SYNC BEFORE STABLE | Canonical files omit branding_type to match production; reconcile required generated mappings, preserve exact downstream variants without DROP. | +| caching | SYNC BEFORE STABLE | Canonical files/ORM omit math_ticks.caching_tick to match production; preserve math_main active cursor and existing downstream variants. | +| course | SET ASIDE | Record production absence of invite uniqueness; set aside for narrow stable because M0 adoption does not test it. Later complete canonical files omit the invariant; no production constraint changes. | +| xids | SYNC BEFORE STABLE | Canonical files/adoption preserve both production XID unique invariants; revise prior M2 guard, no drop-owner/uid action. | +| worker_tasks.task_type text versus varchar(99) | SYNC BEFORE STABLE | Actual adoption refusal: align exact production type and preserve reviewed downstream variants. | +| pwreset_tokens.token varchar(100) versus varchar(250) | SYNC BEFORE STABLE | Actual adoption refusal: align exact production type and preserve reviewed downstream variants. | +| math_bidtopid.data json versus jsonb | SYNC BEFORE STABLE | Named JSON exception already admits this type. Record production JSON and supported JSONB variant; no conversion. | +| math_cache.data json versus jsonb | SYNC BEFORE STABLE | Named JSON exception already admits this type. Record production JSON and supported JSONB variant; no conversion. | +| math_exportstatus.data json versus jsonb | SYNC BEFORE STABLE | Named JSON exception already admits this type. Record production JSON and supported JSONB variant; no conversion. | +| math_main.data json versus jsonb | SYNC BEFORE STABLE | Named JSON exception already admits this type. Record production JSON and supported JSONB variant; no conversion. | +| math_profile.data json versus jsonb | SYNC BEFORE STABLE | Named JSON exception already admits this type. Record production JSON and supported JSONB variant; no conversion. | +| math_ptptstats.data json versus jsonb | SYNC BEFORE STABLE | Named JSON exception already admits this type. Record production JSON and supported JSONB variant; no conversion. | +| comments.uid default zero versus absent | SYNC BEFORE STABLE | Record exact live default/nullability in the file contract and reconcile affected generated readers. Existing M0 does not reject it. Defer behavior changes, backfill, or production DDL. | +| comments.velocity nullable versus NOT NULL | SYNC BEFORE STABLE | Record exact live default/nullability in the file contract and reconcile affected generated readers. Existing M0 does not reject it. Defer behavior changes, backfill, or production DDL. | +| conversations.auth_needed_to_vote default literal versus absent | SYNC BEFORE STABLE | Record exact live default/nullability in the file contract and reconcile affected generated readers. Existing M0 does not reject it. Defer behavior changes, backfill, or production DDL. | +| conversations.auth_needed_to_write default literal versus absent | SYNC BEFORE STABLE | Record exact live default/nullability in the file contract and reconcile affected generated readers. Existing M0 does not reject it. Defer behavior changes, backfill, or production DDL. | +| conversations.auth_opt_fb default literal versus absent | SYNC BEFORE STABLE | Record exact live default/nullability in the file contract and reconcile affected generated readers. Existing M0 does not reject it. Defer behavior changes, backfill, or production DDL. | +| conversations.auth_opt_tw default literal versus absent | SYNC BEFORE STABLE | Record exact live default/nullability in the file contract and reconcile affected generated readers. Existing M0 does not reject it. Defer behavior changes, backfill, or production DDL. | +| conversations.auth_opt_allow_3rdparty default literal versus absent | SYNC BEFORE STABLE | Record exact live default/nullability in the file contract and reconcile affected generated readers. Existing M0 does not reject it. Defer behavior changes, backfill, or production DDL. | +| contexts_context_id_seq | SYNC BEFORE STABLE | Record exact bigint sequence with integer owning column; preserve current state and consumers, no setval/ALTER. Existing adoption does not check its bounds. | +| conversations_zid_seq | SYNC BEFORE STABLE | Record exact bigint sequence with integer owning column; preserve current state and consumers, no setval/ALTER. Existing adoption does not check its bounds. | +| courses_course_id_seq | SYNC BEFORE STABLE | Record exact bigint sequence with integer owning column; preserve current state and consumers, no setval/ALTER. Existing adoption does not check its bounds. | +| participant_metadata_answers_pmaid_seq | SYNC BEFORE STABLE | Record exact bigint sequence with integer owning column; preserve current state and consumers, no setval/ALTER. Existing adoption does not check its bounds. | +| participant_metadata_questions_pmqid_seq | SYNC BEFORE STABLE | Record exact bigint sequence with integer owning column; preserve current state and consumers, no setval/ALTER. Existing adoption does not check its bounds. | +| users_uid_seq | SYNC BEFORE STABLE | Record exact bigint sequence with integer owning column; preserve current state and consumers, no setval/ALTER. Existing adoption does not check its bounds. | +| random_polis_site_id(integer) | SET ASIDE | Retain unused integer overload as ancillary legacy state; no current direct caller established. | +| get_times_for_most_recent_visible_comments() | SYNC BEFORE STABLE | Record reviewed production routine body/fingerprint for live trigger/caller contract. Executable equivalence established; no CREATE OR REPLACE and no existing body-hash adoption refusal. | +| pid_auto() | SYNC BEFORE STABLE | Record reviewed production routine body/fingerprint for live trigger/caller contract. Executable equivalence established; no CREATE OR REPLACE and no existing body-hash adoption refusal. | +| tid_auto() | SYNC BEFORE STABLE | Record reviewed production routine body/fingerprint for live trigger/caller contract. Executable equivalence established; no CREATE OR REPLACE and no existing body-hash adoption refusal. | +| random_string(integer) | SYNC BEFORE STABLE | Record reviewed production routine body/fingerprint for live trigger/caller contract. Executable equivalence established; no CREATE OR REPLACE and no existing body-hash adoption refusal. | +| random_polis_site_id() | SYNC BEFORE STABLE | Record reviewed production routine body/fingerprint for live trigger/caller contract. Executable equivalence established; no CREATE OR REPLACE and no existing body-hash adoption refusal. | +| comments | SYNC BEFORE STABLE | Bind existing optional probe provisioning contract; exact six SELECT/public-USAGE grants already have a source. No role/grant changes. | +| conversations | SYNC BEFORE STABLE | Bind existing optional probe provisioning contract; exact six SELECT/public-USAGE grants already have a source. No role/grant changes. | +| math_main | SYNC BEFORE STABLE | Bind existing optional probe provisioning contract; exact six SELECT/public-USAGE grants already have a source. No role/grant changes. | +| math_ticks | SYNC BEFORE STABLE | Bind existing optional probe provisioning contract; exact six SELECT/public-USAGE grants already have a source. No role/grant changes. | +| participants | SYNC BEFORE STABLE | Bind existing optional probe provisioning contract; exact six SELECT/public-USAGE grants already have a source. No role/grant changes. | +| votes | SYNC BEFORE STABLE | Bind existing optional probe provisioning contract; exact six SELECT/public-USAGE grants already have a source. No role/grant changes. | +| public | SYNC BEFORE STABLE | Bind existing optional probe provisioning contract; exact six SELECT/public-USAGE grants already have a source. No role/grant changes. | +| portable ownership and selected-release privileges | SYNC BEFORE STABLE | Use role parameters/capabilities, not hosted dbUser ownership in public bootstrap. Prove actual migration-session M19 grants/role authority and M23/M24 SET ROLE; reject partial conflicting installs. No role reassignment. | +| comments.curation | SYNC BEFORE STABLE | Record exact live/script-visible field in supported legacy file contract; reconcile generated declarations without changing live values. delphi/scripts/generate_cold_start_clojure.py:293–296 explicitly copies curation; client test hits are unrelated prose/UI. Fresh installs currently lack the script field. | +| conversations.dataset_explanation | SET ASIDE | No literal field reference at any scanned current pin; conversation-wide SELECT * and dynamic JSON remain indirect possibilities. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| conversations.is_curated | SET ASIDE | No literal field reference at the scanned current pins; generic conversation readers can expose it. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| conversations.need_suzinvite | SET ASIDE | No literal field reference at the scanned current pins; do not confuse auth handlers for suzinvites with this dormant flag. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| participants_extended.country_iso_code | SET ASIDE | No literal field reference in the scanned current pins. M7 removes country_code_iso (different spelling), NOT this field. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| participants_extended.encrypted_ip_address | SYNC BEFORE STABLE | Record exact live/script-visible field in supported legacy file contract; reconcile generated declarations without changing live values. server/src/participant.ts:291–300 and db/sql.ts:96–103 conditionally write/declare this field only for applicationName PolisWebServer. | +| participants_extended.encrypted_x_forwarded_for | SYNC BEFORE STABLE | Record exact live/script-visible field in supported legacy file contract; reconcile generated declarations without changing live values. server/src/participant.ts:291–300 and db/sql.ts:96–103 conditionally write/declare this field only for applicationName PolisWebServer. | +| suzinvites.modified | SYNC BEFORE STABLE | Record exact live/script-visible field in supported legacy file contract; reconcile generated declarations without changing live values. server/src/invites/suzinvites.ts:17,68 uses SELECT *; INSERTs at 40 and 167 omit modified. Whole-row consumers can observe it; bare modified inventory includes unrelated tables. | +| suzinvites.uid | SYNC BEFORE STABLE | Record exact live/script-visible field in supported legacy file contract; reconcile generated declarations without changing live values. server/src/invites/suzinvites.ts:17,68 SELECT *; normal INSERTs omit uid. Bare uid matches are broad; table-specific paths are the relevant indirect contract. | +| users.pwhash | SYNC BEFORE STABLE | Record exact live/script-visible field in supported legacy file contract; reconcile generated declarations without changing live values. bin/anonymize_users.sh: updates pwhash; generated Rust users row/catalog retains pwhash; M0 comments it out. No current password-login consumer established. | +| users.test | SET ASIDE | Bare test appears throughout test tooling and is not a users.test consumer; generic users SELECT * can expose it. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| conversation_invite_codes | SET ASIDE | No current literal SQL consumer established; whole-tree locations/zero-results and historical SQL pickaxe are retained. Generic queue/moderators hits are not proof of public.conversation_invite_codes usage. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| conversation_subscriptions | SET ASIDE | No current literal SQL consumer established; whole-tree locations/zero-results and historical SQL pickaxe are retained. Generic queue/moderators hits are not proof of public.conversation_subscriptions usage. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| error_reports | SET ASIDE | No current literal SQL consumer established; whole-tree locations/zero-results and historical SQL pickaxe are retained. Generic queue/moderators hits are not proof of public.error_reports usage. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| math_results_dev01 | SET ASIDE | No current literal SQL consumer established; whole-tree locations/zero-results and historical SQL pickaxe are retained. Generic queue/moderators hits are not proof of public.math_results_dev01 usage. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| minvites | SET ASIDE | No current literal SQL consumer established; whole-tree locations/zero-results and historical SQL pickaxe are retained. Generic queue/moderators hits are not proof of public.minvites usage. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| moderators | SET ASIDE | No current literal SQL consumer established; whole-tree locations/zero-results and historical SQL pickaxe are retained. Generic queue/moderators hits are not proof of public.moderators usage. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| nyt_users | SET ASIDE | No current literal SQL consumer established; whole-tree locations/zero-results and historical SQL pickaxe are retained. Generic queue/moderators hits are not proof of public.nyt_users usage. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| polismath_mod_claims | SET ASIDE | No current literal SQL consumer established; whole-tree locations/zero-results and historical SQL pickaxe are retained. Generic queue/moderators hits are not proof of public.polismath_mod_claims usage. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| queue | SET ASIDE | No current literal SQL consumer established; whole-tree locations/zero-results and historical SQL pickaxe are retained. Generic queue/moderators hits are not proof of public.queue usage. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| slack_participants_waiting_for_comments | SET ASIDE | No current literal SQL consumer established; whole-tree locations/zero-results and historical SQL pickaxe are retained. Generic queue/moderators hits are not proof of public.slack_participants_waiting_for_comments usage. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| slack_state_heap | SET ASIDE | No current literal SQL consumer established; whole-tree locations/zero-results and historical SQL pickaxe are retained. Generic queue/moderators hits are not proof of public.slack_state_heap usage. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| slack_state_stack | SET ASIDE | No current literal SQL consumer established; whole-tree locations/zero-results and historical SQL pickaxe are retained. Generic queue/moderators hits are not proof of public.slack_state_stack usage. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| slack_team_tokens | SET ASIDE | No current literal SQL consumer established; whole-tree locations/zero-results and historical SQL pickaxe are retained. Generic queue/moderators hits are not proof of public.slack_team_tokens usage. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| animals_id_auto | SET ASIDE | No current literal routine call established; votes_lastest_unique appears only as a bootstrap comment. All SQL snapshot bodies are available historically; current production body fingerprints are compared below. Dynamic/external SQL and trigger dependencies need the aggregate census. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| oid_auto | SET ASIDE | No current literal routine call established; votes_lastest_unique appears only as a bootstrap comment. All SQL snapshot bodies are available historically; current production body fingerprints are compared below. Dynamic/external SQL and trigger dependencies need the aggregate census. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| oid_auto_unlock | SET ASIDE | No current literal routine call established; votes_lastest_unique appears only as a bootstrap comment. All SQL snapshot bodies are available historically; current production body fingerprints are compared below. Dynamic/external SQL and trigger dependencies need the aggregate census. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| ptpt_id_auto | SET ASIDE | No current literal routine call established; votes_lastest_unique appears only as a bootstrap comment. All SQL snapshot bodies are available historically; current production body fingerprints are compared below. Dynamic/external SQL and trigger dependencies need the aggregate census. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| ptpt_id_auto_unlock | SET ASIDE | No current literal routine call established; votes_lastest_unique appears only as a bootstrap comment. All SQL snapshot bodies are available historically; current production body fingerprints are compared below. Dynamic/external SQL and trigger dependencies need the aggregate census. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| to_zinvite.integer | SET ASIDE | No current literal routine call established; votes_lastest_unique appears only as a bootstrap comment. All SQL snapshot bodies are available historically; current production body fingerprints are compared below. Dynamic/external SQL and trigger dependencies need the aggregate census. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| votes_foo.integer | SET ASIDE | No current literal routine call established; votes_lastest_unique appears only as a bootstrap comment. All SQL snapshot bodies are available historically; current production body fingerprints are compared below. Dynamic/external SQL and trigger dependencies need the aggregate census. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| votes_lastest_unique.integer | SET ASIDE | No current literal routine call established; votes_lastest_unique appears only as a bootstrap comment. All SQL snapshot bodies are available historically; current production body fingerprints are compared below. Dynamic/external SQL and trigger dependencies need the aggregate census. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| foobar | SET ASIDE | The four literal foobar hits are unrelated Clojure visualization/Rust child fixtures; no sequence user established. Historical dump contains it. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| animal_grp | SET ASIDE | No current literal consumer in tracked trees; historical animals_id_auto body references NEW.grp/enum_range. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| auth_tokens.auth_tokens_token_idx | SET ASIDE | Index planner use is implicit through auth_tokens queries; table-name inventory lists indirect consumers. No application can be declared independent solely because it does not name the index. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| participants.participants_uid_index | SET ASIDE | Index planner use is implicit through participants queries; table-name inventory lists indirect consumers. No application can be declared independent solely because it does not name the index. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| pwreset_tokens.pwreset_tokens_token_idx | SET ASIDE | Index planner use is implicit through pwreset_tokens queries; table-name inventory lists indirect consumers. No application can be declared independent solely because it does not name the index. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| votes.votes_zid_idx | SET ASIDE | Index planner use is implicit through votes queries; table-name inventory lists indirect consumers. No application can be declared independent solely because it does not name the index. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| postgis | SET ASIDE | server/Dockerfile-pdb:1 retains postgis image; bin/remove_postgis.sh is a destructive cleanup helper, NOT permission to run it. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| tablefunc | SET ASIDE | No literal current tree consumer; historical dump declares CREATE EXTENSION tablefunc. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| pg_stat_statements | SET ASIDE | server/src/ops/database.ts:4 explicitly says pg_stat_statements is NOT used (ruling R5: extension creation is a database change); the two unit-test hits do not establish a runtime consumer. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| unattributed_extension_members | SET ASIDE | Exact extension ownership remains unproven: 794 routine and 109 parent-object count mappings ready. Retain all candidates. No selected migration alters these objects; classification and any cleanup stay separate. | +| constraint_index_identity | SET ASIDE | Catalog generator and runner use semantic definitions. Exact alias-name consumers require a separate identifier search before any optional rename. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| social_settings.social_settings_uid_key | SET ASIDE | No current live uniqueness consumer established; generated declarations and historical usage alone do not gate queue migrations. Preserve existing unique constraint. | +| suzinvites.suzinvites_uid_fkey | SYNC BEFORE STABLE | Record existing live uniqueness/FK in file contract; no constraint creation or deletion. | +| conversations.conversations_zid_index | SET ASIDE | Redundant index variant does not block selected migrations. Retain current indexes, document optional shape; no rebuild. | +| users.users_uid_idx | SET ASIDE | Redundant index variant does not block selected migrations. Retain current indexes, document optional shape; no rebuild. | diff --git a/docs/migration-reconciliation.md b/docs/migration-reconciliation.md new file mode 100644 index 0000000000..9f99721148 --- /dev/null +++ b/docs/migration-reconciliation.md @@ -0,0 +1,70 @@ +# Reconcile an existing deployment + +Keep one record per database with the release being installed. This is a review +record for the normal `reconcile` → `apply` → `check` path; it is not a list of +host-specific skipped migrations. Never copy credentials, participant rows or +private identifiers into the public repository. + +1. Record the release commit and exact migration source hashes, database major + version, current application version, backup/restore evidence and installer + privileges. Confirm the previous application's stop hook: the first move to + deferred-stop hooks can still execute the old hook before migration refusal. +2. Collect a read-only catalog inventory through the deployment's existing DB + client. Preserve types, defaults, nullability, keys, index validity, routines, + triggers, grants and sequence configuration. Encode bigint values as strings + when transporting them through JavaScript JSON. Do not read sequence values + or vote/application rows as a shortcut to schema reconciliation. +3. Classify every logical migration: present, pending, partial, conflicting, + superseded by a known later contract, or outside the forward chain. Cite its + observed postconditions. Names, owners and physical column positions can + differ without changing the structure; compare key column names and actual + definitions. Do not count internal FK trigger OIDs as missing migrations. +4. Retain historical receipts separately. Filename aliases, duplicate legacy + timestamps and a current matching schema are different kinds of evidence. + Unknown legacy filenames, competing ledgers, nonempty undeclared vote data + and unknown routine implementations need reviewed resolution. Do not invent + execution dates or treat a latest filename as a complete prefix. +5. Prepare and test each needed forward repair on generated databases representing + the actual structural variant. Preserve existing rows, unrelated schemas, + grants and third-party dependants. Do not truncate or drop data to satisfy a + catalog predicate; do not change stored vote signs or overwrite a custom + routine merely to match a fresh installation. Older retirement SQL is not + authorization to delete retained deployment data. +6. Once all selected adoption postconditions are supported and reviewed, run + `polis-migrate reconcile --through NNNNNN` with the factual bound. A failure + must leave all prior history untouched. Then `apply` the ordered pending + migrations and run `check`. No history rows may be inserted manually. +7. Record the actual per-file outcomes and postconditions. M22 is resumable: + indexes can commit before their history row; inspect validity after a lost + connection and reuse valid results. Every other committed earlier file stays + applied if a later file fails. Do not run down scripts as automatic recovery. +8. Verify application health after service replacement. Detached Compose startup + alone is not a serving-health receipt. Keep migration readiness, process + readiness and application health as separate observations. + +Record fields: + +| Field | Evidence required | +| --- | --- | +| Release | Commit, SQL source hashes, release holds and pending PR composition | +| Observation | Catalog query version/hash, collection date, scope and transport | +| Per file | Logical identity, known aliases, state, exact postconditions and differences | +| History | Actual receipts and their provenance, or explicitly unknown | +| Preservation | Rows, custom objects, grants, active-writer compatibility and backup | +| Repair | Exact reviewed patch, generated regression evidence, failure recovery | +| Adoption | Explicit bound, atomic success or unchanged-history failure | +| Pending work | Ordered apply outcomes, concurrent-index progress where applicable | +| Completion | Runner check plus actual application health evidence | + +Fresh databases use `apply`, not reconciliation. Existing /1, /2 or /3 queues +must pass their recorded catalog contracts; install-table presence is not proof. +A partial legacy install needs compatible forward completion, not a fabricated +adoption. M21's release hold remains until its privilege, sequence and publisher +requirements are resolved. Convention declaration and any semantic sign operation +remain explicit operations; neither a catalog nor a migration merge supplies the +operator's declaration. + +This record does not by itself remediate unsupported historical variants, +retire destructive files, compose competing unreleased migration ledgers, or +make manual down scripts history-aware. Those release changes and their tests +must be complete before claiming an upgrade works across supported deployments. diff --git a/docs/migration-release-map.md b/docs/migration-release-map.md new file mode 100644 index 0000000000..2de64208f2 --- /dev/null +++ b/docs/migration-release-map.md @@ -0,0 +1,68 @@ +# Migration source to release map + +These 46 source variants map to releases by exact SQL bytes. Source membership +does not prove execution on any database. The only actual source version label +established by the historical research is `1.0`; subsequent semantic versions +were not consistently assigned. Strict/OIDC numbers below are the previously +reviewed retrospective proposals, **not published tags or a new version ruling**. +The current M19/M23/M24 bytes first ship at promotion PR #2994 (`5ded2e0a9`); +their semantic version is **UNASSIGNED**. A release owner must assign the real +release version before publishing one; this patch invents none. + +For an installation's execution record, use its receipts or record execution +as unknown. Never infer an apply date from this source table. The historical +version research is tracked separately from schema adoption. + +| Audited source variant | Role | Filename first present | Exact audited bytes first present | Disposition | +| --- | --- | --- | --- | --- | +| `000000_initial.sql` (#2997 reference; `2652134140cd`) | numbered_forward | declared 1.0 source (anchor) | `53817ae39`; strict 3.24.2 / OIDC 2.24.2 | SHIPPED_SOURCE_BYTES | +| `000001_update_pwreset_table.sql` (#2997 reference; `cb21278194c4`) | numbered_forward | declared 1.0 source (anchor) | declared 1.0 source (anchor) | SHIPPED_SOURCE_BYTES | +| `000002_add_xid_constraint.sql` (#2997 reference; `a27a8e63c79c`) | numbered_forward | declared 1.0 source (anchor) | declared 1.0 source (anchor) | SHIPPED_SOURCE_BYTES | +| `000003_add_origin_permanent_cookie_columns.sql` (#2997 reference; `0d7f27facfec`) | numbered_forward | declared 1.0 source (anchor) | declared 1.0 source (anchor) | SHIPPED_SOURCE_BYTES | +| `000004_drop_waitinglist_table.sql` (#2997 reference; `f2fc4184a965`) | numbered_forward | declared 1.0 source (anchor) | declared 1.0 source (anchor) | SHIPPED_SOURCE_BYTES | +| `000005_drop_slack_stripe_canvas.sql` (#2997 reference; `392e5b8aadb7`) | numbered_forward | declared 1.0 source (anchor) | declared 1.0 source (anchor) | SHIPPED_SOURCE_BYTES | +| `000006_update_votes_rule.sql` (#2997 reference; `8fb05b7b1b6a`) | numbered_forward | declared 1.0 source (anchor) | declared 1.0 source (anchor) | SHIPPED_SOURCE_BYTES | +| `000007_drop_geolocation_fields.sql` (#2997 reference; `f68b69b86112`) | numbered_forward | `8040579ef`; strict 1.1.0 / OIDC 1.1.0 | `8040579ef`; strict 1.1.0 / OIDC 1.1.0 | SHIPPED_SOURCE_BYTES | +| `000008_add_comment_priority.sql` (#2997 reference; `c867b53be3cc`) | numbered_forward | `8040579ef`; strict 1.1.0 / OIDC 1.1.0 | `8040579ef`; strict 1.1.0 / OIDC 1.1.0 | SHIPPED_SOURCE_BYTES | +| `000009_add_uuid_to_zinvites.sql` (#2997 reference; `43f36fe0b857`) | numbered_forward | `4ba8b0938`; strict 1.6.0 / OIDC 1.6.0 | `4ba8b0938`; strict 1.6.0 / OIDC 1.6.0 | SHIPPED_SOURCE_BYTES | +| `000010_create_oidc_user_mappings.sql` (#2997 reference; `450a3f69883a`) | numbered_forward | `3a7da1468`; strict 3.0.0 / OIDC 2.0.0 | `3a7da1468`; strict 3.0.0 / OIDC 2.0.0 | SHIPPED_SOURCE_BYTES | +| `000011_alter_suzinvites_xid_to_text.sql` (#2997 reference; `00a1eb2d9604`) | numbered_forward | `3a7da1468`; strict 3.0.0 / OIDC 2.0.0 | `3a7da1468`; strict 3.0.0 / OIDC 2.0.0 | SHIPPED_SOURCE_BYTES | +| `000012_create_topic_agenda_selections.sql` (#2997 reference; `cc513693124f`) | numbered_forward | `199569498`; strict 3.4.0 / OIDC 2.4.0 | `199569498`; strict 3.4.0 / OIDC 2.4.0 | SHIPPED_SOURCE_BYTES | +| `000013_create_treevite.sql` (#2997 reference; `4b8334f73246`) | numbered_forward | `b986fca0f`; strict 3.6.0 / OIDC 2.6.0 | `b986fca0f`; strict 3.6.0 / OIDC 2.6.0 | SHIPPED_SOURCE_BYTES | +| `000014_alter_reports_modlevel.sql` (#2997 reference; `c2af6d57af28`) | numbered_forward | `ee4405a44`; strict 3.7.2 / OIDC 2.7.2 | `ee4405a44`; strict 3.7.2 / OIDC 2.7.2 | SHIPPED_SOURCE_BYTES | +| `000015_add_xid_requirements.sql` (#2997 reference; `186c904addd0`) | numbered_forward | `a6d7215f7`; strict 3.19.0 / OIDC 2.19.0 | `06d6fa1af`; strict 3.23.0 / OIDC 2.23.0 | SHIPPED_SOURCE_BYTES | +| `000016_add_orig_id.sql` (#2997 reference; `6cdc0588c000`) | numbered_forward | `b13b316e9`; strict 3.21.0 / OIDC 2.21.0 | `b13b316e9`; strict 3.21.0 / OIDC 2.21.0 | SHIPPED_SOURCE_BYTES | +| `000017_create_byod_job_table.sql` (#2997 reference; `f27c03a1229f`) | numbered_forward | `b13b316e9`; strict 3.21.0 / OIDC 2.21.0 | `b13b316e9`; strict 3.21.0 / OIDC 2.21.0 | SHIPPED_SOURCE_BYTES | +| `000018_add_topics_enabled.sql` (#2997 reference; `a1e1c0572064`) | numbered_forward | `06d6fa1af`; strict 3.23.0 / OIDC 2.23.0 | `06d6fa1af`; strict 3.23.0 / OIDC 2.23.0 | SHIPPED_SOURCE_BYTES | +| `000019_create_polis_queue.sql` (#2997 reference; `fedfbcf9fc59`) | numbered_forward | `335418338`; strict 3.28.0 / OIDC 2.28.0 | `5ded2e0a9`; strict UNASSIGNED / OIDC UNASSIGNED | SHIPPED_SOURCE_BYTES | +| `000020_create_math_source_journal.sql` (PR#2739; `24c1dff07637`) | numbered_forward | NOT SHIPPED at checked endpoints | NOT SHIPPED at checked endpoints | PENDING_VARIANT_NOT_SHIPPED | +| `000021_create_polis_coordinator.sql` (#2997 reference; `d50f169ad7af`) | numbered_forward | `335418338`; strict 3.28.0 / OIDC 2.28.0 | `335418338`; strict 3.28.0 / OIDC 2.28.0 | SHIPPED_SOURCE_BYTES | +| `000022_add_poll_timestamp_indexes.sql` (#2997 reference; `14efc95b1478`) | numbered_forward | `2547b6ee5`; strict 3.30.0 / OIDC 2.30.0 | `2547b6ee5`; strict 3.30.0 / OIDC 2.30.0 | SHIPPED_SOURCE_BYTES | +| `000023_create_delphi_foundation.sql` (#2997 reference; `97437ea57d90`) | numbered_forward | `5ded2e0a9`; strict UNASSIGNED / OIDC UNASSIGNED | `5ded2e0a9`; strict UNASSIGNED / OIDC UNASSIGNED | SHIPPED_SOURCE_BYTES | +| `000024_create_polis_queue_large_class.sql` (#2997 reference; `68261afb81f2`) | numbered_forward | `5ded2e0a9`; strict UNASSIGNED / OIDC UNASSIGNED | `5ded2e0a9`; strict UNASSIGNED / OIDC UNASSIGNED | SHIPPED_SOURCE_BYTES | +| `000025_vote_convention.sql` (PR#2944; `cfff57e4f416`) | numbered_forward | NOT SHIPPED at checked endpoints | NOT SHIPPED at checked endpoints | PENDING_VARIANT_NOT_SHIPPED | +| `000026_create_polis_queue_retention.sql` (PR#2978; `ee29e37e94f5`) | numbered_forward | NOT SHIPPED at checked endpoints | NOT SHIPPED at checked endpoints | PENDING_VARIANT_NOT_SHIPPED | +| `000019_create_delphi_storage.sql` (PR#2600; `5808d85fe674`) | numbered_forward | NOT SHIPPED at checked endpoints | NOT SHIPPED at checked endpoints | PENDING_VARIANT_NOT_SHIPPED | +| `000012_create_topic_agenda_selections.sql` (PR#2110; `cc513693124f`) | numbered_forward | `199569498`; strict 3.4.0 / OIDC 2.4.0 | `199569498`; strict 3.4.0 / OIDC 2.4.0 | PENDING_VARIANT_BYTES_ALREADY_SHIPPED | +| `000000_initial.sql` (PR#2556; `183d22c0238a`) | numbered_forward | declared 1.0 source (anchor) | NOT SHIPPED at checked endpoints | PENDING_VARIANT_NOT_SHIPPED | +| `000026_create_polis_queue_retention.sql` (PR#2983; `0d1e357a72a9`) | numbered_forward | NOT SHIPPED at checked endpoints | NOT SHIPPED at checked endpoints | PENDING_VARIANT_NOT_SHIPPED | +| `000024_vote_sign_unflip.sql` (PR#2942; `3e67e0ab857d`) | held | NOT SHIPPED at checked endpoints | NOT SHIPPED at checked endpoints | PENDING_VARIANT_NOT_SHIPPED | +| `000023_vote_convention.sql` (PR#2942; `351628cfc1bc`) | fixture | NOT SHIPPED at checked endpoints | NOT SHIPPED at checked endpoints | PENDING_VARIANT_NOT_SHIPPED | +| `000025_drop_vote_convention.sql` (PR#2944; `9d747f6d0fe3`) | down | NOT SHIPPED at checked endpoints | NOT SHIPPED at checked endpoints | PENDING_VARIANT_NOT_SHIPPED | +| `000026_drop_polis_queue_retention.sql` (PR#2978; `747085bf1324`) | down | NOT SHIPPED at checked endpoints | NOT SHIPPED at checked endpoints | PENDING_VARIANT_NOT_SHIPPED | +| `000026_drop_polis_queue_retention.sql` (PR#2983; `063272749473`) | down | NOT SHIPPED at checked endpoints | NOT SHIPPED at checked endpoints | PENDING_VARIANT_NOT_SHIPPED | +| `db_000002.sql` (#2997 reference; `1f214c31e557`) | archive | declared 1.0 source (anchor) | declared 1.0 source (anchor) | SHIPPED_SOURCE_BYTES | +| `db_000004.sql` (#2997 reference; `3afacfa4c4e6`) | archive | declared 1.0 source (anchor) | declared 1.0 source (anchor) | SHIPPED_SOURCE_BYTES | +| `db_000006.sql` (#2997 reference; `4c6f61fa0a5b`) | archive | declared 1.0 source (anchor) | declared 1.0 source (anchor) | SHIPPED_SOURCE_BYTES | +| `db_000008.sql` (#2997 reference; `e5d4990d1dd7`) | archive | declared 1.0 source (anchor) | declared 1.0 source (anchor) | SHIPPED_SOURCE_BYTES | +| `db_000010.sql` (#2997 reference; `498344c75e9c`) | archive | declared 1.0 source (anchor) | declared 1.0 source (anchor) | SHIPPED_SOURCE_BYTES | +| `000019_drop_polis_queue.sql` (#2997 reference; `483de532876f`) | down | `335418338`; strict 3.28.0 / OIDC 2.28.0 | `335418338`; strict 3.28.0 / OIDC 2.28.0 | SHIPPED_SOURCE_BYTES | +| `000021_drop_polis_coordinator.sql` (#2997 reference; `f8547afa1e87`) | down | `335418338`; strict 3.28.0 / OIDC 2.28.0 | `335418338`; strict 3.28.0 / OIDC 2.28.0 | SHIPPED_SOURCE_BYTES | +| `000022_drop_poll_timestamp_indexes.sql` (#2997 reference; `fcf1f4695e48`) | down | `2547b6ee5`; strict 3.30.0 / OIDC 2.30.0 | `2547b6ee5`; strict 3.30.0 / OIDC 2.30.0 | SHIPPED_SOURCE_BYTES | +| `000023_drop_delphi_foundation.sql` (#2997 reference; `aa0a3d67766a`) | down | `5ded2e0a9`; strict UNASSIGNED / OIDC UNASSIGNED | `5ded2e0a9`; strict UNASSIGNED / OIDC UNASSIGNED | SHIPPED_SOURCE_BYTES | +| `000024_drop_polis_queue_large_class.sql` (#2997 reference; `08735f922a17`) | down | `5ded2e0a9`; strict UNASSIGNED / OIDC UNASSIGNED | `5ded2e0a9`; strict UNASSIGNED / OIDC UNASSIGNED | SHIPPED_SOURCE_BYTES | + + +The M20/M25/M26 PR-only variants, unflip fixture, archives and down scripts +are not in this release manifest. M21 is held. M4/M5/M7 are retained historical +sources and can only receive observed ADOPTED receipts in ordinary apply. diff --git a/docs/migration-upgrade-notes.md b/docs/migration-upgrade-notes.md new file mode 100644 index 0000000000..dc6b62c1ad --- /dev/null +++ b/docs/migration-upgrade-notes.md @@ -0,0 +1,100 @@ +# Migration runner upgrade notes + +The first selected legacy upgrade records the supported existing schema, then +applies only M19, M23 and M24. They create the job-system tables and routines. +They do not enable any worker or application feature flag. Production is never +changed merely to match old bootstrap files. + +## Choose the installation path + +| Starting state | Required path | Expected outcome | +| --- | --- | --- | +| Empty PostgreSQL 17+ | `apply`, then `check` | 20 executed files, three retirement ADOPTED receipts, 23 ready | +| Supported legacy schema through M18 plus valid M22, no queue/history | first-deploy report, `reconcile --through 000022`, `apply`, `check` | 20 ADOPTED; exactly M19/M23/M24 APPLIED | +| Supported legacy schema through M18, missing M22 | `reconcile --through 000018`, `apply`, `check` | 19 ADOPTED; M19, concurrent M22, M23/M24 APPLIED | +| Existing queue /1, /2 or /3 | exact installed catalog/receipts review, reconcile through actual installed version, then apply/check | compatible installed queue retained; only pending selected files execute | +| Modern valid runner history | `apply`, `check` | existing APPLIED/ADOPTED receipts retained; rerun executes zero | +| Legacy ledger with known filenames | bounded reconcile after catalog review | timestamps preserved, no fictional execution receipts | +| Retained M4/M5/M7 targets | stop for separately reviewed completion/retention plan | no automatic destructive migration or fake adoption | +| Partial/conflicting schema, unknown ledger/file, later queue variant | stop for explicit reviewed reconciliation | no blind replay, manual history insertion or source rewriting | + +The historical bootstrap and named legacy schema are distinct supported variants. +See [their exact contract](migration-legacy-contract.md). A successful limited +adoption contract is not an exhaustive certification of every custom object, +extension, historical routine implementation, data invariant or application path. + +## Deprecated removal files + +M4 historically removed `waitinglist`. M5 removed Slack OAuth/users/invites/bot +events, Stripe accounts/subscriptions, free-upgrade coupons, LTI users/context +memberships/OAuth credentials, Canvas callback/conversation tables, and the +`conversations.is_slack`, `conversations.lti_users_only`, `users.plan` columns. +M7 removed `geolocation_cache` and participants_extended's `country_code_iso`, +`encrypted_maxmind_response_city`, `ip_address`, `latitude`, `location`, +`longitude` and `x_forwarded_for` fields. + +All three files remain immutable historical evidence. Ordinary upgrades never +execute them. Absence of the exact named objects allows an ADOPTED receipt; +retained targets block without deletion. Other Slack tables, the differently +named `participants_extended.country_iso_code`, encrypted network fields, and +`facebook_users.location`/`twitter_users.location` are not removal targets. +A composite type's `location` attribute is not a participants_extended column. + +## Report-only first-deploy check + +`server/postgres/migrations/report/first-deploy.sql` is the first-deploy report. +Its SQL predicates are generated directly from the reviewed adoption sources, +with helper expressions inlined. It uses one bounded REPEATABLE READ READ ONLY +transaction followed by ROLLBACK, creates no temporary functions/tables, reads +no application rows, and returns only migration names, booleans and outcomes. +`python3 server/bin/build-migration-report.py --check` verifies its exact source +binding; regeneration is a reviewable file change. + +The operator sends the entire SQL file through **one connection** of the existing +server's database client, with existing connection/TLS settings. Do not deploy +the new runner or restart the app just to report. For a local or controlled +operator session, `psql -X -v ON_ERROR_STOP=1 -f ` is equivalent, using +the normal secret connection environment without a credential argument. On any +query error, ROLLBACK or close that same client before returning it to a pool. + +Expected initial-schema results: 20 `WOULD_ADOPT`, three `WOULD_APPLY`, and four +`OUTSIDE_RELEASE` entries. The report conservatively flags any existing queue +role for migration-session review rather than guessing inherited ADMIN/SET +authority. Existing ledgers/queues report review-required; they need the exact +normal reconciliation catalog verifier and a separate read-only examination. +A mismatch in any required predicate blocks the aggregate adoption forecast. +A runtime read-only role can report schema but cannot predict privileged DDL +success; check the intended migration session separately. + +The report proves no future lock acquisition, available capacity, successful +DDL or application health. Its snapshot expires; reconcile and apply recheck +their actual contracts when executed. M22 presence in the initial report is +required; missing M22 uses the explicit shorter-bound path above. + +## Privileges, interruption and deployment + +Use the intended migration session, including M19 role provisioning and grant +authority. Owning only the database or having normal app SELECT/INSERT access +is insufficient. M23/M24 SET ROLE to the queue owner. Never grant broad rights +to a runtime login just to satisfy startup; startup needs metadata SELECT. + +Each ordinary file and its history receipt commit together. If M23 fails after +M19 commits, M19 remains applied while all M23 changes and its receipt roll back. +Do not describe that as a rollback of the whole release. M22 is the documented +concurrent-index exception; interrupted valid indexes may persist and be reused. +Check invalid/conflicting index recovery in [migrations.md](migrations.md). +Two runners serialize using the database advisory lock; they do not double-apply. +Lost commit connections require history inspection, not an assumed rollback. + +On the first deployment, an old successful revision's stop hook may have already +stopped the old application before the new migration hook runs. New deferred-stop +hooks protect old containers only after that transition. A later API startup +failure may occur after old containers have been removed. This patch supplies +no automatic rollback, previous-image fallback or CodeDeploy health guarantee. +Verify actual HTTP/application health after replacement and preserve a reviewed +recovery procedure. A detached Compose launch is insufficient evidence. + +Keep a per-deployment [reconciliation record](migration-reconciliation.md), +including exact source hashes, schema report, actual session authority, observed +receipts, supported variants, backup evidence and health result. Do not publish +credentials, application records or real conversation/report identifiers. diff --git a/docs/migrations.md b/docs/migrations.md index 307609c77d..5eb63dc48b 100644 --- a/docs/migrations.md +++ b/docs/migrations.md @@ -1,40 +1,199 @@ +# Database migrations -# Database Migrations +Deploy the schema before replacing application services. Every deployment uses +`polis-migrate apply`; the API refuses startup if a required file is pending, +history is missing, or a recorded checksum disagrees with the release. +PostgreSQL 17 or newer is required. Keep a tested backup/restore procedure. -When we need to update the Polis database, we use SQL migration files. +Build once from the repository root (or use the release's migration image): -During initial provisioning of your Docker containers, all the migrations will be applied in order, and you won't need to think about this. -But if we update the database schema after your initial provisioning of your server via Docker, you'll need to manually apply each new SQL migration. +```sh +cargo build --locked --release --manifest-path queue-rs/Cargo.toml -p polis-migrate +export PATH="$PWD/queue-rs/target/release:$PATH" +# Set DATABASE_URL through your usual secret environment; never put it in argv. +polis-migrate apply +polis-migrate check +``` + +The image requires no local Rust installation: + +```sh +docker build -t polis-migrate:local -f queue-rs/polis-migrate/Dockerfile . +docker run --rm --network host --env-file .env polis-migrate:local apply +``` + +Use the database's actual network when it is inside Compose instead of the host +network. `POLIS_MIGRATIONS_DIR` or `--dir` selects the release's SQL directory. +TLS validates the certificate and hostname. The runner and API images include +a checksum-pinned public RDS CA bundle (official global truststore, retrieved +2026-10-09; SHA-256 `fe45bbebf92ad3e27a583bbb2ddd1553c521ed4d49af5514dc0a40372ea5395c`). `POLIS_MIGRATE_CA_FILE` supplies a +private PEM CA bundle (mount it into the container). Standalone Node startup +uses `NODE_EXTRA_CA_CERTS` for a private/RDS CA; standalone Rust uses +`POLIS_MIGRATE_CA_FILE`. For a disposable local +network only, a URL with `sslmode=disable` plus +`POLIS_MIGRATE_ALLOW_PLAINTEXT=local` permits Docker service DNS; loopback and +local sockets can use `sslmode=disable` without that variable. Never use this +local setting for a remote production database. -- Please note: **Backups are your responsibility.** These instructions assume - the data is disposable, and do not attempt to make backups. - - Pull requests are welcome if you'd like to see more guidance on this. - - Please submit an issue if you'd like to work on enabling backups through Docker Compose. -- Your database data is stored on a docker volume, which means that it will - persist even when you destroy all your docker containers. Be mindful of this. - - You can remove ALL volumes defined within a `docker-compose` file via: `docker compose --profile postgres down --volumes` - - You can remove ONE volume via `docker volume ls` and `docker volume rm ` -- SQL migrations can be found in [`server/postgres/migrations/`][] of this - repo. -- The path to the SQL file will be relative to its location in the docker - container filesystem, not your host system. +New Docker volumes run the same binary from `server/Dockerfile-db`. Existing +volumes do not rerun initialization: invoke `apply` before starting the new app. +The CodeDeploy hook invokes it before its container cleanup/start operations. +`server/bin/run-migrations.sh` delegates to the binary; the old Clojure entry +point delegates to that shell command. Neither replays every file anymore. -For example, if we add the migration file -`server/postgres/migrations/000001_update_pwreset_table.sql`, you'd run on your -host system: +## One-time adoption of an existing database + +An existing database without history is deliberately refused by `apply`. +Never replay `000000_initial.sql` on it. Use the known migration records and +inspect its schema to choose an explicit upper bound, then reconcile: ```sh -docker compose --profile postgres exec postgres psql --username postgres --dbname polis-dev --file=/docker-entrypoint-initdb.d/000001_update_pwreset_table.sql +# Example: the legacy schema through 000022, without the dormant queue. +polis-migrate reconcile --through 000022 +polis-migrate apply +polis-migrate check ``` -You can also run a local .sql file on a postgres container instance with this syntax: +The bound is not evidence of application. Every selected file must pass its +catalog postconditions in `server/postgres/migrations/adoption/`: enduring +initial tables/columns, renamed columns, unique and foreign-key constraints, +new columns/types/defaults, removed objects, the vote-update rule, and valid +index definitions. The exact [legacy variants](migration-legacy-contract.md) include six named +math `json` payloads and bounded type/column alternatives; unrelated types are refused. The missing queue migration 000019 is a recognized +hole **only when no queue/foundation tables or functions exist and no old +history row claims it**. The example adopts 000000–000018 and 000022 as ADOPTED; +`apply` then installs only 000019, 000023 and 000024. No vote data is inspected +or changed by reconciliation. + +Adoption is atomic: any failed postcondition leaves the old history untouched. +It records observation time and the checksum of the source being reconciled, +not a fictional historical execution date. If the 2021 `migrations(name, +completed_at)` table exists, its timestamps (including duplicates) are retained +in each row's `legacy_completed_at` array after all catalog checks pass. Unknown +or renamed legacy filenames stop for review; they are never silently replayed. +Already installed queues use their existing /1, /2 or /3 catalog verifiers, +including definitions/privileges and recorded installed catalogs. A matching +install row alone is insufficient. Select the actual upper version (for example +`--through 000024` for a previously initialized /3 Docker database). A partial +queue, later bound without a contract, unknown historical filename, or the +unreleased `schema_migrations` ledger stops for review. Never mark those by hand. + +## History and failure behavior + +`public.migrations` is the sole applied-history table. It records filename, +SHA-256 of the unmodified SQL source, APPLIED or ADOPTED, observation time/actor, +and any preserved legacy timestamps. Runtime roles need SELECT only, granted +on this metadata table; the deployment role needs the privileges required by +the actual migrations (including role administration for the queue). + +A database advisory lock serializes runners on the same connection across +per-file commits. A second runner waits up to five minutes, then reads the +committed history. Each ordinary file and its APPLIED row commit in one transaction. The two +000022 index builds are the explicit autocommit exception described below. +An optional historical outer BEGIN/COMMIT pair is removed; any other top-level +transaction control is rejected. Dollar-quoted function bodies stay intact. +SQL errors abort the file. Earlier successfully committed files stay applied. +Lock waits are bounded (five seconds for DDL), with five-minute statement and +transaction ceilings and a thirty-second idle-transaction limit. Existing SQL +may set tighter timeouts. No automatic destructive down migration runs. + +If the connection is lost at commit, the result can be unknown. Reconnect and +run `check`/`apply`: committed history is authoritative. Do not infer rollback +from a transport error. Restore changed historical files instead of editing +history to bypass checksum failures. + +For the exact released 000022 source, `apply` preflights both index identities, +then builds each missing watermark index with `CREATE INDEX CONCURRENTLY` in +autocommit, while retaining the same migration advisory lock. This is the normal +path on fresh and large populated databases. Valid existing indexes are checked +and preserved without rebuilding. The original SQL then rechecks both exact +index definitions in the transaction that records APPLIED. Its raw SQL checksum +is unchanged; changing that source requires review of this execution contract. + +An interruption can leave a valid first index, or an invalid index from a failed +concurrent build, without a migration history row. Rerunning `apply` reuses valid +indexes and builds only missing ones. It refuses invalid or conflicting objects +before building either index. Inspect an invalid index first; if it is the exact +interrupted watermark index, explicitly drop that index with `DROP INDEX +CONCURRENTLY public.` in autocommit, then rerun `apply`. The +runner never drops a preexisting index automatically. A differently defined +same-name object requires a reviewed resolution, not that drop instruction. +If history recording fails after both builds, both valid indexes remain; retry +records their checked state without rebuilding. Concurrent builds have the +same bounded statement/lock timeouts; a timeout is not a success receipt. + +## Release contents and new migrations + +`server/postgres/migrations/release.txt` explicitly selects every required file. +The Rust runner and Node startup gate use the same manifest; Docker initialization +uses that runner. A new top-level numbered SQL file must be selected or held, +otherwise startup/apply refuses it. Missing files, duplicate numbers or entries, +symlinks, and release/hold overlap refuse. Adding a file cannot silently extend +the release. Numbered source checksums remain unchanged. + +This release contains M0–M19 plus M22/M23/M24. M4/M5/M7 are deprecated +observation-only entries. They never execute through `apply`: their named +removed objects must already be absent, then an ADOPTED receipt is recorded. +Any retained target stops before apply mutates schema/history; arrange a separate +reviewed upgrade for that deployment. Existing valid APPLIED receipts are kept. +Fresh bootstrap already omits their targets and gets three ADOPTED receipts; +it executes 20 files. No historical execution date is invented. + +M20 (draft journal), M21 (held coordinator), M25 (vote convention) and M26 +(retention) are excluded from the forward path. M21 stays in `held.txt`; the +other files are not shipped by this branch. An added unclassified copy refuses. +The selected legacy upgrade adopts M0–M18/M22 and applies **M19 → M23 → M24**. +A deployment missing M22 instead builds its indexes concurrently between M19 +and M23. Archives, down scripts and unflip files never enter this manifest. + +Read [the release map](migration-release-map.md) for historical source/release +provenance and [upgrade notes](migration-upgrade-notes.md) for each supported +deployment shape. Semantic versions that were never assigned remain explicitly +unassigned; source shipment does not establish database execution. + +Merging a required migration means it runs at the next deployment. Review its +compatibility with the still-running previous application, locking, privileges, +data effects and reversal/restore plan at merge time. Update these public +upgrade records in that change. The manual sitting and client stage/unstage +steps are retired; no separate staging ceremony is required. + +## CI and local test stacks + +The migration-runner CI fresh-image step and the mm2/mm5 proof use exactly: ```sh -docker exec -i polis-dev-postgres-1 psql -U postgres -d polis-dev < server/postgres/migrations/000006_update_votes_rule.sql +COMPOSE_PROJECT_NAME=polis-migrate-test-my-owned-run \ +POLIS_RECOVERY_PG_PORT=55851 \ +bash queue-rs/polis-migrate/tests/fresh-image.sh ``` -where `polis-dev-postgres-1` is the name of the running container (see the output of `docker ps`), `postgres` is the db username and `polis-dev` is the database. +Choose a unique project and unused port on each shared machine. This entrypoint +builds `server/Dockerfile-db`, initializes a fresh database, checks exact receipt +names/checksums/statuses against `release.txt` and the M4/M5/M7 retirement policy, +then restarts and checks that the complete history is unchanged. Both the ready +count and expected receipt set derive from the selection. It removes only its +project volumes/containers and project-specific image tag, including on failure. +It does not replay production rows or prove a production deployment. + +After building `docker-compose.test.yml`, run `bash ci/test-migrations.sh` +before starting application services. It waits for Postgres initialization, +then runs `apply` and `check` using the binary and SQL packaged in that image. +It works on fresh and existing test volumes; it never creates history by hand +or bypasses the API startup check. Cypress, server integration and Delphi CI +all use this entrypoint. Server integration also runs the Node startup check +before loading its in-process test app. + +Set a unique `COMPOSE_PROJECT_NAME` and `POLIS_RECOVERY_PG_PORT` for a shared +local machine. `POLIS_TEST_ENV_FILE` chooses a test env file (default `test.env`); +optional Compose arguments such as `-f local-ports.yml` support isolated test +stacks. Use the same options when starting and removing your stack. + -You'd do this for each new file, in numeric order. +## Deployment reconciliation records - [`server/postgres/migrations/`]: /server/postgres/migrations +Use [the reconciliation record](migration-reconciliation.md) for each deployment. +A catalog match is an observation, not proof that a historical file ran. A failed +predicate must lead to a reviewed forward repair or a documented compatible +variant, never to manually inserting a migration row. Unknown historical variants remain review cases; this document does not authorize +a deployment that fails reconciliation. Run the [read-only first-deploy report](migration-upgrade-notes.md) +before the first transition; it does not replace application-health verification. diff --git a/docs/queue-substrate.md b/docs/queue-substrate.md index 4e3a9ea276..1d8c0c8aa6 100644 --- a/docs/queue-substrate.md +++ b/docs/queue-substrate.md @@ -44,19 +44,11 @@ executor role holds **no** direct read or write on any queue table. ## Applying the migration -A **fresh** container applies it automatically: the postgres image copies -`server/postgres/migrations/*.sql` into `/docker-entrypoint-initdb.d`, so -`make start` on a new volume comes up with the schema present and the flag off. - -An **existing** database needs the file applied by hand, through the checked -wrapper, which feeds it to psql exactly as [docs/migrations.md](migrations.md) -describes. Apply this file alone; never replay the migrations directory as an -upgrade mechanism. - -```sh -server/postgres/bin/apply-migration.sh --free-bytes 000019 -- \ - docker exec -i polis-dev-postgres-1 psql -U postgres -d polis-dev -``` +Both fresh databases and upgrades use [`polis-migrate apply`](migrations.md). +Fresh Docker volumes run it during initialization; deployments run it before +service replacement. Existing pre-runner databases reconcile once first. +Successful files are recorded and skipped on later deploys. Queue activation +remains controlled by the existing flags; applying schema does not enable it. ### What the apply locks, and the window it needs @@ -71,16 +63,16 @@ open transaction that already wrote a `conversations` row; when the timeout fires the transaction aborts and nothing is applied. The changed queue tables and the new objects are `ACCESS EXCLUSIVE` for the same span. So "additive and empty" is not "cannot block users": **apply in an idle or controlled writer -window** (producers paused, no open writer on `conversations`), and apply -000019 and 000023 as two separate steps, each with its own window. +window** (producers paused, no open writer on `conversations`), and account for both +000019 and 000023 in the deployment window; the runner gives each its own transaction. ### The wrapper: preflight and budgets -`server/postgres/bin/apply-migration.sh` is the one way to apply 000019 or -000023 to an existing database. It takes the migration number and, after `--`, -the psql command to run (the SQL goes on its stdin, so `docker exec -i … psql` -and a plain `psql` both work). It refuses, sending nothing, unless every -preflight check passes, and prints each one: +The former `server/postgres/bin/apply-migration.sh` is retained for historical +rehearsal/reversal tests. It is **not** the deployment entry point and does not +update the migration history. Its former first-install checks are recorded below +for reference. The runner now owns transactional application, bounded waits, +checksums and history; use the [migration guide](migrations.md). | check | what it requires | |---|---| @@ -391,18 +383,10 @@ Two of those tables deserve plain words: Schema ruling S1 (2026-10-05) approved it as direction: one datastore and typed contracts, flag off, DynamoDB running every job family until each is moved one -at a time. **Applying it to production is a separate, explicit step by the -owner**, 000019 first (it has never been applied there), then 000023, each -through the wrapper and in its own idle or controlled writer window, because -each holds `ShareRowExclusiveLock` on `conversations` until it commits -([what the apply locks](#what-the-apply-locks-and-the-window-it-needs); the -wrapper's preflight and budgets are -[described above](#the-wrapper-preflight-and-budgets)): - -```sh -server/postgres/bin/apply-migration.sh --free-bytes 000023 -- \ - docker exec -i polis-dev-postgres-1 psql -U postgres -d polis-dev -``` +at a time. **Required migrations now apply during deployment**, in numeric order through +`polis-migrate apply`; review and authorize their schema effects at merge time. +Both 000019 and 000023 still hold `ShareRowExclusiveLock` on conversations, bounded +by the runner's lock and transaction limits. See [migration operations](migrations.md). The applier must be able to `SET ROLE polis_queue_owner`; the file creates no role. It refuses, changing nothing, when 000019 is absent, when any diff --git a/docs/upgrading.md b/docs/upgrading.md index 976f65e0f2..7402e8417a 100644 --- a/docs/upgrading.md +++ b/docs/upgrading.md @@ -1,5 +1,34 @@ # Upgrade Guide +## Migration runner transition + +Use [the migration runner](migrations.md) before replacing services. Existing +databases need one catalog-checked `reconcile` before their first runner deploy; +new databases initialize through the runner. Missing or changed history prevents +API startup. Migrations 000019/23/24 now apply during deployment; the coordinator +000021 remains explicitly held in this release. This installs schema only, not +queue producers or workers. PostgreSQL 17+ is required. + +Read the [per-deployment upgrade paths and read-only report](migration-upgrade-notes.md) +before the first transition. One explicit release manifest selects the files; +M20/M21/M25/M26 are outside this forward release. Historical M4/M5/M7 never +execute automatically: absence of their removal targets permits ADOPTED receipts, +while retained targets stop for review without deletion. The +[named legacy contract](migration-legacy-contract.md) preserves supported live +variants, and the [release map](migration-release-map.md) distinguishes shipped +source from actual execution and unassigned semantic versions. + +CodeDeploy executes `ApplicationStop` from the **previous successful revision**. +Consequently the first rollout from the older hooks can still stop services +before the new migration hook runs. Complete one-time reconciliation before that +rollout and plan for that transition interruption; do not claim failure leaves +the old service up on the first transition. Subsequent releases defer shutdown +until migration success. A rollback to old hook code reinstates its old stopping +behavior. A failed migration does not trigger automatic down migrations. + +The sitting kit and image staging/unstaging are superseded by this ordinary +apply/check path. Keep backup and migration-specific recovery procedures. + ## Configuration Changes (Q1 2023) `polis.config.template.js` and `polis.config.js` files are removed and no longer used. diff --git a/queue-rs/Cargo.lock b/queue-rs/Cargo.lock index 8dc073d4f3..606e3f04ff 100644 --- a/queue-rs/Cargo.lock +++ b/queue-rs/Cargo.lock @@ -568,6 +568,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "polis-migrate" +version = "0.1.0" +dependencies = [ + "anyhow", + "native-tls", + "postgres", + "postgres-native-tls", + "sha2 0.10.9", +] + [[package]] name = "polis-queue-adapter" version = "0.1.0" diff --git a/queue-rs/Cargo.toml b/queue-rs/Cargo.toml index bfc1730620..6920223fc5 100644 --- a/queue-rs/Cargo.toml +++ b/queue-rs/Cargo.toml @@ -1,7 +1,7 @@ [workspace] # The root package (the polis-queue/1 adapter and the polis-jobs daemon) and the # polis-api route server share one lockfile, one toolchain and one CI job. -members = [".", "polis-api"] +members = [".", "polis-api", "polis-migrate"] resolver = "3" [package] diff --git a/queue-rs/polis-migrate/Cargo.toml b/queue-rs/polis-migrate/Cargo.toml new file mode 100644 index 0000000000..6403b1bb90 --- /dev/null +++ b/queue-rs/polis-migrate/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "polis-migrate" +version = "0.1.0" +edition = "2024" +rust-version = "1.88" +publish = false + +[dependencies] +anyhow = "1" +native-tls = "=0.2.14" +postgres = "0.19" +postgres-native-tls = "=0.5.3" +sha2 = "0.10" + +[lints.clippy] +unwrap_used = "deny" +expect_used = "deny" diff --git a/queue-rs/polis-migrate/Dockerfile b/queue-rs/polis-migrate/Dockerfile new file mode 100644 index 0000000000..c55dfb59fe --- /dev/null +++ b/queue-rs/polis-migrate/Dockerfile @@ -0,0 +1,20 @@ +# Build from repository root. One binary for deploys and manual/local upgrades. +FROM docker.io/library/rust:1.98.1-slim-bookworm AS build +RUN apt-get update && apt-get install -y --no-install-recommends pkg-config libssl-dev curl ca-certificates \ + && rm -rf /var/lib/apt/lists/* +# Public RDS truststore retrieved over HTTPS 2026-10-09; checksum pins its bytes. +RUN curl --fail --location --proto '=https' --tlsv1.2 https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem -o /rds-ca.pem \ + && echo 'fe45bbebf92ad3e27a583bbb2ddd1553c521ed4d49af5514dc0a40372ea5395c /rds-ca.pem' | sha256sum --check +WORKDIR /src +COPY queue-rs/ ./ +RUN cargo build --release --locked -p polis-migrate +FROM docker.io/library/debian:bookworm-slim +RUN apt-get update && apt-get install -y --no-install-recommends libssl3 ca-certificates \ + && rm -rf /var/lib/apt/lists/* +COPY --from=build /src/target/release/polis-migrate /usr/local/bin/polis-migrate +COPY server/postgres/migrations /migrations +COPY --from=build /rds-ca.pem /etc/polis/rds-ca.pem +ENV POLIS_MIGRATIONS_DIR=/migrations POLIS_MIGRATE_CA_FILE=/etc/polis/rds-ca.pem +USER 65534:65534 +ENTRYPOINT ["polis-migrate"] +CMD ["apply"] diff --git a/queue-rs/polis-migrate/Dockerfile.dockerignore b/queue-rs/polis-migrate/Dockerfile.dockerignore new file mode 100644 index 0000000000..2850efb53f --- /dev/null +++ b/queue-rs/polis-migrate/Dockerfile.dockerignore @@ -0,0 +1,8 @@ +** +!queue-rs/ +!queue-rs/** +queue-rs/target/ +!server/ +!server/postgres/ +!server/postgres/migrations/ +!server/postgres/migrations/** diff --git a/queue-rs/polis-migrate/src/indexes.rs b/queue-rs/polis-migrate/src/indexes.rs new file mode 100644 index 0000000000..f2127f0c9e --- /dev/null +++ b/queue-rs/polis-migrate/src/indexes.rs @@ -0,0 +1,66 @@ +//! Resumable, autocommit execution for the immutable watermark index migration. +//! The caller holds the database-wide migration lock across every statement. +use anyhow::{Context, Result, ensure}; +use postgres::Client; + +const INDEXES: [(&str, &str, &str); 2] = [ + ("votes_created_idx", "votes", "created"), + ("comments_modified_idx", "comments", "modified"), +]; + +fn state(client: &mut Client, name: &str, table: &str, column: &str) -> Result> { + let rows = client.query( + "SELECT x.indisvalid AND x.indisready AND x.indislive AS usable, + pg_get_indexdef(c.oid), c.relkind + FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace + LEFT JOIN pg_index x ON x.indexrelid=c.oid + WHERE n.nspname='public' AND c.relname=$1", + &[&name], + )?; + let Some(row) = rows.first() else { + return Ok(None); + }; + let definition: Option = row.get(1); + let expected = format!("CREATE INDEX {name} ON public.{table} USING btree ({column})"); + ensure!( + definition.as_deref() == Some(expected.as_str()), + "000022: conflicting object public.{name}; preserve it and resolve its definition before retrying" + ); + let usable: Option = row.get(0); + Ok(Some(usable == Some(true))) +} + +pub fn prepare(client: &mut Client) -> Result<()> { + // Preflight BOTH identities before changing either one. A conflicting or + // invalid preexisting index is never dropped or replaced automatically. + for (name, table, column) in INDEXES { + if let Some(usable) = state(client, name, table, column)? { + ensure!( + usable, + "000022: public.{name} is invalid/not ready; inspect the interrupted build, then explicitly DROP INDEX CONCURRENTLY public.{name} before retrying apply; history has not recorded 000022" + ); + } + } + for (name, table, column) in INDEXES { + // Recheck after the preceding build. Uncoordinated external DDL is not + // serialized by the runner's advisory lock; any collision fails closed. + if state(client, name, table, column)? == Some(true) { + continue; + } + ensure!( + state(client, name, table, column)?.is_none(), + "000022: concurrent catalog change for public.{name}; inspect before retrying" + ); + println!("BUILDING CONCURRENTLY public.{name}"); + client.batch_execute(&format!( + "CREATE INDEX CONCURRENTLY {name} ON public.{table} USING btree ({column})" + )).with_context(|| format!( + "000022: concurrent build interrupted for public.{name}; inspect validity before retrying; valid earlier indexes are retained and no 000022 history row was committed" + ))?; + ensure!( + state(client, name, table, column)? == Some(true), + "000022: index public.{name} did not reach the expected valid state" + ); + } + Ok(()) +} diff --git a/queue-rs/polis-migrate/src/lib.rs b/queue-rs/polis-migrate/src/lib.rs new file mode 100644 index 0000000000..266c5659d5 --- /dev/null +++ b/queue-rs/polis-migrate/src/lib.rs @@ -0,0 +1,578 @@ +//! One migration history, one connection and one database advisory lock. +//! SQL files remain immutable; only their optional outer BEGIN/COMMIT is removed. +mod indexes; +mod sql; +use anyhow::{Context, Result, bail, ensure}; +use postgres::{ + Client, GenericClient, + config::{Host, SslMode}, +}; +use sha2::{Digest, Sha256}; +use std::{collections::BTreeMap, fs, path::Path, time::Duration}; + +pub struct Migration { + pub name: String, + pub checksum: String, + pub body: String, + pub retired: bool, +} +// Fixed database-wide key, shared by apply/reconcile. A different DB has its own lock. +pub const LOCK: i64 = 0x506f6c69734d6967; +const HISTORY: &str = "CREATE TABLE public.migrations ( + name text PRIMARY KEY, checksum text NOT NULL CHECK(length(checksum)=64), + status text NOT NULL CHECK(status IN ('APPLIED','ADOPTED')), + recorded_at timestamptz NOT NULL DEFAULT clock_timestamp(), + recorded_by text NOT NULL DEFAULT session_user, + legacy_completed_at bigint[] NOT NULL DEFAULT '{}'); + REVOKE ALL ON public.migrations FROM PUBLIC; + GRANT SELECT ON public.migrations TO PUBLIC;"; + +pub fn load(dir: &Path) -> Result> { + let mut files = BTreeMap::new(); + let mut versions = std::collections::BTreeSet::new(); + let selected = manifest(dir, "release.txt")?; + let held = manifest(dir, "held.txt")?; + ensure!( + selected.is_disjoint(&held), + "release and held manifests overlap" + ); + for entry in fs::read_dir(dir)? { + let entry = entry?; + let name = entry + .file_name() + .into_string() + .map_err(|_| anyhow::anyhow!("non-UTF8 migration name"))?; + if !name.ends_with(".sql") { + continue; + } + ensure!( + entry.file_type()?.is_file(), + "migration must be a regular file: {name}" + ); + ensure!( + name.len() > 11 + && name.as_bytes()[..6].iter().all(u8::is_ascii_digit) + && name.as_bytes()[6] == b'_' + && name.as_bytes()[7..name.len() - 4] + .iter() + .all(|b| b.is_ascii_alphanumeric() || *b == b'_'), + "invalid migration name: {name}" + ); + ensure!( + versions.insert(name[..6].to_owned()), + "duplicate migration number: {name}" + ); + let source = fs::read_to_string(entry.path())?; + files.insert( + name.clone(), + Migration { + name: name.clone(), + checksum: format!("{:x}", Sha256::digest(source.as_bytes())), + body: sql::body(&source)?, + retired: matches!(&name[..6], "000004" | "000005" | "000007"), + }, + ); + } + for name in selected.iter().chain(&held) { + ensure!( + files.contains_key(name), + "manifest migration is missing: {name}" + ); + } + for name in files.keys() { + ensure!( + selected.contains(name) || held.contains(name), + "migration absent from release/held manifest: {name}" + ); + } + let migrations: Vec<_> = files + .into_values() + .filter(|m| selected.contains(&m.name)) + .collect(); + ensure!( + migrations + .first() + .is_some_and(|m| m.name == "000000_initial.sql"), + "missing initial migration" + ); + Ok(migrations) +} + +fn manifest(dir: &Path, file: &str) -> Result> { + let text = fs::read_to_string(dir.join(file)).with_context(|| format!("read {file}"))?; + let mut names = std::collections::BTreeSet::new(); + for name in text + .lines() + .filter(|s| !s.is_empty() && !s.starts_with('#')) + { + ensure!( + name.len() > 11 + && name.as_bytes()[..6].iter().all(u8::is_ascii_digit) + && name.as_bytes()[6] == b'_' + && name.ends_with(".sql") + && name.as_bytes()[7..name.len() - 4] + .iter() + .all(|b| b.is_ascii_alphanumeric() || *b == b'_'), + "invalid migration in {file}: {name}" + ); + ensure!( + names.insert(name.to_owned()), + "duplicate migration in {file}: {name}" + ); + } + Ok(names) +} + +// Retired historical DDL is never executable, even on a fresh installation. +// Check catalog absence directly; no temporary functions or user-data reads. +fn retired_absent(client: &mut impl GenericClient, m: &Migration) -> Result { + let (tables, columns): (&[&str], &[(&str, &str)]) = match &m.name[..6] { + "000004" => (&["waitinglist"], &[]), + "000005" => ( + &[ + "slack_oauth_access_tokens", + "slack_users", + "slack_user_invites", + "slack_bot_events", + "stripe_accounts", + "stripe_subscriptions", + "coupons_for_free_upgrades", + "lti_users", + "lti_context_memberships", + "canvas_assignment_callback_info", + "canvas_assignment_conversation_info", + "lti_oauthv1_credentials", + ], + &[ + ("conversations", "is_slack"), + ("conversations", "lti_users_only"), + ("users", "plan"), + ], + ), + "000007" => ( + &["geolocation_cache"], + &[ + ("participants_extended", "country_code_iso"), + ("participants_extended", "encrypted_maxmind_response_city"), + ("participants_extended", "ip_address"), + ("participants_extended", "latitude"), + ("participants_extended", "location"), + ("participants_extended", "longitude"), + ("participants_extended", "x_forwarded_for"), + ], + ), + _ => bail!("not a retired migration: {}", m.name), + }; + for table in tables { + if exists(client, &format!("public.{table}"))? { + return Ok(false); + } + } + for (table, column) in columns { + let present: bool = client.query_one("SELECT EXISTS(SELECT 1 FROM pg_attribute WHERE attrelid=to_regclass('public.' || $1) AND attname=$2 AND attnum>0 AND NOT attisdropped)", &[table, column])?.get(0); + if present { + return Ok(false); + } + } + Ok(true) +} + +pub fn connect(dsn: &str) -> Result { + let mut cfg: postgres::Config = dsn + .parse() + .map_err(|_| anyhow::anyhow!("invalid DATABASE_URL"))?; + cfg.connect_timeout(Duration::from_secs(10)) + .application_name("polis-migrate"); + ensure!( + !cfg.get_hosts().is_empty() && cfg.get_hostaddrs().is_empty(), + "DATABASE_URL must name a host and cannot use hostaddr" + ); + if cfg.get_ssl_mode() == SslMode::Disable { + // Docker-local tests may use service DNS; remote plaintext is never implicit. + ensure!( + std::env::var("POLIS_MIGRATE_ALLOW_PLAINTEXT").as_deref() == Ok("local") + || cfg.get_hosts().iter().all(|h| match h { + Host::Tcp(h) => h.parse::().is_ok_and(|a| a.is_loopback()), + #[cfg(unix)] + Host::Unix(p) => p.is_absolute(), + }), + "plaintext requires loopback/socket or explicit POLIS_MIGRATE_ALLOW_PLAINTEXT=local" + ); + cfg.connect(postgres::NoTls) + .context("connect to migration database") + } else { + // require encryption and validate hostname + certificate even with sslmode=require. + cfg.ssl_mode(SslMode::Require); + let mut tls = native_tls::TlsConnector::builder(); + if let Ok(path) = std::env::var("POLIS_MIGRATE_CA_FILE") { + let bytes = fs::read(path).context("read migration CA file")?; + // RDS and local CA bundles can contain several certificates. + for pem in String::from_utf8(bytes)?.split_inclusive("-----END CERTIFICATE-----") { + if pem.contains("-----BEGIN CERTIFICATE-----") { + tls.add_root_certificate(native_tls::Certificate::from_pem(pem.as_bytes())?); + } + } + } + cfg.connect(postgres_native_tls::MakeTlsConnector::new(tls.build()?)) + .context("connect to migration database (verified TLS)") + } +} + +fn setup(client: &mut Client) -> Result<()> { + let version: i32 = client + .query_one("SELECT current_setting('server_version_num')::integer", &[])? + .get(0); + ensure!(version >= 170000, "PostgreSQL 17 or newer is required"); + client.batch_execute("SET standard_conforming_strings=on; SET search_path=pg_catalog,public; SET statement_timeout='5min'; SET lock_timeout='5s'; SET idle_in_transaction_session_timeout='30s'; SET transaction_timeout='5min';")?; + Ok(()) +} +fn exists(client: &mut impl GenericClient, name: &str) -> Result { + Ok(client + .query_one("SELECT to_regclass($1) IS NOT NULL", &[&name])? + .get(0)) +} +fn history( + client: &mut impl GenericClient, + migrations: &[Migration], +) -> Result> { + ensure!( + exists(client, "public.migrations")?, + "migration history missing; run polis-migrate apply for a fresh database, or reconcile for an existing database (docs/migrations.md)" + ); + let rows = client + .query( + "SELECT name, checksum, status FROM public.migrations ORDER BY name", + &[], + ) + .context("unreconciled legacy history; run polis-migrate reconcile")?; + let mut result = BTreeMap::new(); + for row in rows { + let name: String = row.get(0); + let checksum: String = row.get(1); + let status: String = row.get(2); + ensure!( + matches!(status.as_str(), "APPLIED" | "ADOPTED"), + "unverified history: {name}" + ); + let m = migrations.iter().find(|m| m.name == name).ok_or_else(|| { + anyhow::anyhow!("history names migration absent from this release: {name}") + })?; + ensure!( + checksum == m.checksum, + "migration checksum mismatch: {name}; restore the released source; do not replay it" + ); + ensure!( + result.insert(name.clone(), checksum).is_none(), + "duplicate history: {name}" + ); + } + Ok(result) +} +pub fn check(client: &mut Client, migrations: &[Migration]) -> Result<()> { + setup(client)?; + let mut tx = client.build_transaction().read_only(true).start()?; + let applied = history(&mut tx, migrations)?; + let pending: Vec<_> = migrations + .iter() + .filter(|m| !applied.contains_key(&m.name)) + .map(|m| m.name.as_str()) + .collect(); + ensure!( + pending.is_empty(), + "pending migrations: {}; run polis-migrate apply before starting the server", + pending.join(", ") + ); + tx.commit()?; + println!("migration check: {} ready", migrations.len()); + Ok(()) +} +fn lock(client: &mut Client) -> Result<()> { + setup(client)?; + // A blocking SELECT pg_advisory_lock retains a snapshot while waiting. + // CREATE INDEX CONCURRENTLY in the holder can wait for that same snapshot, + // deadlocking two runners. Each try returns before the bounded client wait, + // so no server statement/snapshot is held by the waiting runner. + println!("waiting for migration lock"); + let deadline = std::time::Instant::now() + Duration::from_secs(300); + loop { + let acquired: bool = client + .query_one("SELECT pg_try_advisory_lock($1)", &[&LOCK])? + .get(0); + if acquired { + break; + } + ensure!( + std::time::Instant::now() < deadline, + "timed out waiting for migration lock; another runner is still active" + ); + std::thread::sleep(Duration::from_millis(100)); + } + println!("migration lock acquired"); + Ok(()) +} +fn record( + client: &mut impl GenericClient, + m: &Migration, + status: &str, + legacy: &[i64], +) -> Result<()> { + client.execute("INSERT INTO public.migrations(name,checksum,status,legacy_completed_at) VALUES($1,$2,$3,$4)", &[&m.name,&m.checksum,&status,&legacy])?; + Ok(()) +} +pub fn apply(client: &mut Client, migrations: &[Migration]) -> Result { + lock(client)?; + // Validate history and every pending retirement before CREATE HISTORY or + // unrelated forward DDL. A refusal leaves the database unchanged. + let prior = if exists(client, "public.migrations")? { + history(client, migrations)? + } else { + BTreeMap::new() + }; + for m in migrations + .iter() + .filter(|m| m.retired && !prior.contains_key(&m.name)) + { + ensure!( + retired_absent(client, m)?, + "retired migration {} still has removed objects; no changes made; explicit operator review required", + m.name + ); + } + // A no-history populated DB is never treated as a fresh install. Check ALL + // public relations, not only conversations, before allowing 000000. + if !exists(client, "public.migrations")? { + let populated: bool = client.query_one("SELECT EXISTS(SELECT 1 FROM pg_class WHERE relnamespace='public'::regnamespace AND relkind IN ('r','p','v','m','S','f')) OR EXISTS(SELECT 1 FROM pg_proc WHERE pronamespace='public'::regnamespace) OR EXISTS(SELECT 1 FROM pg_type WHERE typnamespace='public'::regnamespace AND typtype IN ('e','d'))", &[])?.get(0); + ensure!( + !populated, + "existing database has no history; run reconcile before apply; 000000 will not be replayed" + ); + client.batch_execute(HISTORY)?; + } + let applied = history(client, migrations)?; + let mut count = 0; + for m in migrations.iter().filter(|m| !applied.contains_key(&m.name)) { + if m.name.starts_with("000000_") { + let existing: bool = client.query_one("SELECT EXISTS(SELECT 1 FROM pg_class WHERE relnamespace='public'::regnamespace AND relkind IN ('r','p','v','m','S','f') AND relname <> 'migrations') OR EXISTS(SELECT 1 FROM pg_proc WHERE pronamespace='public'::regnamespace) OR EXISTS(SELECT 1 FROM pg_type WHERE typnamespace='public'::regnamespace AND typtype IN ('e','d'))", &[])?.get(0); + ensure!( + !existing, + "refusing initial migration on an existing schema; reconcile first" + ); + } + if m.name == "000022_add_poll_timestamp_indexes.sql" { + // Source-bound special execution; never silently reinterpret a new + // version of this SQL. The original body below still checks both + // definitions before its history row commits. + ensure!( + m.checksum == "14efc95b14787b52d70680ea06cfef020cd2224e82495258fd8eb324501b7b3e", + "000022 source changed; review the concurrent execution contract" + ); + indexes::prepare(client)?; + } + let mut tx = client.transaction()?; + tx.batch_execute("SET LOCAL standard_conforming_strings=on; SET LOCAL search_path=public,pg_catalog; SET LOCAL lock_timeout='5s'; SET LOCAL statement_timeout='5min'; SET LOCAL transaction_timeout='5min';")?; + let status = if m.retired { + ensure!( + retired_absent(&mut tx, m)?, + "retired migration {} has retained objects; refusing adoption", + m.name + ); + "ADOPTED" + } else { + tx.batch_execute(&m.body).with_context(|| { + format!("migration {} failed (transaction not committed)", m.name) + })?; + // Restore the login's context after queue scripts SET LOCAL ROLE. + tx.batch_execute("RESET ROLE; SET LOCAL search_path=pg_catalog,public")?; + "APPLIED" + }; + record(&mut tx, m, status, &[])?; + tx.commit().with_context(|| { + format!( + "commit outcome unknown for {}; reconnect and check history before retrying", + m.name + ) + })?; + println!("{status} {}", m.name); + if status == "APPLIED" { + count += 1; + } + } + client.query_one("SELECT pg_advisory_unlock($1)", &[&LOCK])?; + println!("applied {count} migration(s)"); + Ok(count) +} + +// Reuse the queue's existing catalog verifiers for older Docker installations. +// Only temporary catalog-reading functions are installed here, never queue DDL. +fn queue_version(tx: &mut impl GenericClient, migrations: &[Migration]) -> Result { + let present: bool = tx.query_one("SELECT EXISTS(SELECT 1 FROM pg_class WHERE relnamespace='public'::regnamespace AND (starts_with(relname,'polis_queue_') OR starts_with(relname,'delphi_'))) OR EXISTS(SELECT 1 FROM pg_proc WHERE pronamespace='public'::regnamespace AND (starts_with(proname,'pq_') OR starts_with(proname,'pd_')))",&[])?.get(0); + if !present { + return Ok(0); + } + ensure!( + exists(tx, "public.polis_queue_install")?, + "partial queue installation; no adoption committed" + ); + let mut definitions = 0; + for m in migrations + .iter() + .filter(|m| m.name.starts_with("000019_") || m.name.starts_with("000024_")) + { + for statement in sql::statements(&m.body)? { + if [ + "pq_catalog", + "pq_assert_catalog", + "pq_assert_signatures", + "pq_assert_functions", + "pd_state", + "pq3_state", + ] + .iter() + .any(|name| { + statement.starts_with(&format!("CREATE OR REPLACE FUNCTION pg_temp.{name}(")) + }) { + tx.batch_execute(&statement)?; + definitions += 1; + } + } + } + ensure!( + definitions == 7, + "queue adoption verifier definitions changed; review the catalog contract" + ); + if exists(tx, "public.polis_queue_large_class_install")? { + ensure!( + exists(tx, "public.delphi_foundation_install")?, + "partial queue /3 installation" + ); + let ok: bool=tx.query_one("SELECT (SELECT count(*)=1 FROM public.polis_queue_large_class_install) AND EXISTS(SELECT 1 FROM public.polis_queue_large_class_install WHERE singleton AND installed=pg_temp.pq3_state()) AND (SELECT count(*)=1 FROM public.delphi_foundation_install) AND (SELECT count(*)=1 AND bool_and(contract_version='polis-queue/3') FROM public.polis_queue_install)",&[])?.get(0); + ensure!(ok, "queue /3 catalog postconditions fail"); + Ok(24) + } else if exists(tx, "public.delphi_foundation_install")? { + let ok: bool=tx.query_one("SELECT (SELECT count(*)=1 FROM public.delphi_foundation_install) AND EXISTS(SELECT 1 FROM public.delphi_foundation_install WHERE singleton AND installed=pg_temp.pd_state()) AND (SELECT count(*)=1 AND bool_and(contract_version='polis-queue/2') FROM public.polis_queue_install)",&[])?.get(0); + ensure!(ok, "queue /2 catalog postconditions fail"); + Ok(23) + } else { + tx.batch_execute("SELECT pg_temp.pq_assert_catalog(false); SELECT pg_temp.pq_assert_signatures(false); SELECT pg_temp.pq_assert_functions(false)").context("queue /1 catalog postconditions fail")?; + ensure!( + tx.query_one("SELECT count(*)=1 FROM public.polis_queue_install", &[])? + .get::<_, bool>(0), + "queue /1 install record missing" + ); + Ok(19) + } +} + +/// Adoption is explicitly bounded by the operator, but every file is checked. +/// The missing 19 queue is a known hole in the pre-runner production baseline. +/// Existing queues use their catalog verifiers, never install-row presence alone. +pub fn reconcile( + client: &mut Client, + migrations: &[Migration], + dir: &Path, + through: &str, +) -> Result { + ensure!( + through.len() == 6 && through.bytes().all(|b| b.is_ascii_digit()), + "--through requires six digits" + ); + ensure!( + migrations.iter().any(|m| &m.name[..6] == through), + "unknown --through version" + ); + lock(client)?; + let mut tx = client.transaction()?; + let modern = tx.query_one("SELECT EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema='public' AND table_name='migrations' AND column_name='checksum')", &[])?.get::<_,bool>(0); + if modern { + history(&mut tx, migrations)?; + bail!("history is already reconciled; use apply/check"); + } + ensure!( + !exists(&mut tx, "public.schema_migrations")?, + "another schema_migrations ledger exists; resolve its unverified rows before reconciliation" + ); + let mut legacy: BTreeMap> = BTreeMap::new(); + if exists(&mut tx, "public.migrations")? { + for row in tx.query( + "SELECT name,completed_at FROM public.migrations ORDER BY name,completed_at", + &[], + )? { + let name: String = row.get(0); + ensure!( + migrations + .iter() + .any(|m| m.name == name && &m.name[..6] <= through), + "legacy history contains unknown/renamed/later migration {name}; resolve against its catalog, never replay blindly" + ); + legacy.entry(name).or_default().push(row.get(1)); + } + } + // Helpers are transaction-local and contain only catalog reads. + tx.batch_execute(&fs::read_to_string(dir.join("adoption/helpers.sql"))?)?; + let queue = queue_version(&mut tx, migrations)?; + ensure!( + queue <= through.parse::()?, + "queue is newer than --through; select its actual installed version" + ); + let mut adopted = Vec::new(); + for m in migrations.iter().filter(|m| &m.name[..6] <= through) { + let number = m.name[..6].parse::()?; + if matches!(number, 19 | 23 | 24) { + if queue >= number { + adopted.push(m); + } else { + ensure!( + !legacy.contains_key(&m.name), + "legacy queue history row was not verified: {}", + m.name + ); + } + continue; + } + let path = dir.join("adoption").join(&m.name); + let query = fs::read_to_string(path).with_context(|| { + format!( + "no adoption contract for {}; stop and review this existing installation", + m.name + ) + })?; + let ok: bool = tx + .query_one(&query, &[]) + .with_context(|| format!("catalog check for {}", m.name))? + .get(0); + ensure!( + ok, + "catalog postconditions fail for {}; no adoption rows committed; inspect schema before retrying", + m.name + ); + adopted.push(m); + } + for name in legacy.keys() { + ensure!( + adopted.iter().any(|m| m.name == *name), + "legacy row not verified: {name}" + ); + } + if exists(&mut tx, "public.migrations")? { + tx.batch_execute("DROP TABLE public.migrations")?; + } + tx.batch_execute(HISTORY)?; + for m in &adopted { + record( + &mut tx, + m, + "ADOPTED", + legacy.get(&m.name).map(Vec::as_slice).unwrap_or(&[]), + )?; + } + tx.commit()?; + client.query_one("SELECT pg_advisory_unlock($1)", &[&LOCK])?; + for m in &adopted { + println!("ADOPTED {}", m.name); + } + println!( + "adopted {} migration(s); no migration SQL replayed", + adopted.len() + ); + Ok(adopted.len()) +} diff --git a/queue-rs/polis-migrate/src/main.rs b/queue-rs/polis-migrate/src/main.rs new file mode 100644 index 0000000000..acf40964fc --- /dev/null +++ b/queue-rs/polis-migrate/src/main.rs @@ -0,0 +1,81 @@ +use anyhow::{Result, bail}; +use std::{env, path::PathBuf}; +fn run() -> Result<()> { + let mut args = env::args().skip(1); + let command = args.next().unwrap_or_default(); + if matches!(command.as_str(), "--help" | "-h") { + println!( + "polis-migrate apply|check|reconcile [--dir PATH] [--through NNNNNN]\nDATABASE_URL is read from the environment; reconcile requires --through. See docs/migrations.md." + ); + return Ok(()); + } + let mut dir = PathBuf::from( + env::var("POLIS_MIGRATIONS_DIR").unwrap_or_else(|_| "server/postgres/migrations".into()), + ); + let mut through = None; + while let Some(arg) = args.next() { + match arg.as_str() { + "--dir" => { + dir = args + .next() + .ok_or_else(|| anyhow::anyhow!("--dir needs a path"))? + .into() + } + "--through" => { + through = Some( + args.next() + .ok_or_else(|| anyhow::anyhow!("--through needs a version"))?, + ) + } + _ => bail!("unknown argument: {arg}"), + } + } + if !matches!(command.as_str(), "apply" | "check" | "reconcile") { + bail!("expected apply, check or reconcile (see --help)"); + } + if (command == "reconcile") != through.is_some() { + bail!("only reconcile requires --through NNNNNN"); + } + let migrations = polis_migrate::load(&dir)?; + let dsn = env::var("DATABASE_URL").map_err(|_| anyhow::anyhow!("DATABASE_URL is required"))?; + let mut client = polis_migrate::connect(&dsn)?; + match command.as_str() { + "apply" => { + polis_migrate::apply(&mut client, &migrations)?; + } + "check" => polis_migrate::check(&mut client, &migrations)?, + "reconcile" => { + polis_migrate::reconcile( + &mut client, + &migrations, + &dir, + through.as_deref().unwrap_or(""), + )?; + } + _ => unreachable!(), + } + Ok(()) +} +fn main() { + if let Err(error) = run() { + // Do not print connection strings, SQL statements or database error DETAIL + // (which may quote row values). Context + SQLSTATE are sufficient here. + for cause in error.chain() { + if let Some(pg) = cause.downcast_ref::() { + if let Some(db) = pg.as_db_error() { + eprintln!( + "database refused migration: SQLSTATE {} (inspect the named migration)", + db.code().code() + ); + } else { + eprintln!( + "database connection failed; check connectivity and TLS configuration" + ); + } + break; + } + eprintln!("{cause}"); + } + std::process::exit(1); + } +} diff --git a/queue-rs/polis-migrate/src/sql.rs b/queue-rs/polis-migrate/src/sql.rs new file mode 100644 index 0000000000..efcd023884 --- /dev/null +++ b/queue-rs/polis-migrate/src/sql.rs @@ -0,0 +1,282 @@ +//! Split only top-level SQL. Dollar bodies, quoted strings and comments are opaque. +//! This is deliberately not a general SQL rewriter: accept one optional outer +//! BEGIN/COMMIT pair and refuse every other transaction-control statement. +use anyhow::{Result, bail, ensure}; + +fn ident_start(c: char) -> bool { + c.is_ascii_alphabetic() || c == '_' || !c.is_ascii() +} + +fn ident_continue(c: char) -> bool { + ident_start(c) || c.is_ascii_digit() || c == '$' +} + +// PostgreSQL continues single-quoted strings across whitespace containing a +// newline, retaining the first segment's E-string escape mode. Line comments +// count as whitespace here; block comments terminate this lexical construct. +fn continued_string(bytes: &[u8], mut next: usize) -> Option { + let mut newline = false; + while next < bytes.len() { + match bytes[next] { + b'\n' | b'\r' => { + newline = true; + next += 1; + } + b' ' | b'\t' | b'\x0c' | b'\x0b' => next += 1, + b'-' if bytes[next..].starts_with(b"--") => { + while next < bytes.len() && !matches!(bytes[next], b'\n' | b'\r') { + next += 1; + } + } + b'\'' if newline => return Some(next + 1), + _ => return None, + } + } + None +} + +// Keep executable bytes alongside comment-free text used only for inspection. +// Removing comments from the executed source can change string-literal parsing. +fn lex(sql: &str) -> Result> { + let b = sql.as_bytes(); + let (mut i, mut part, mut raw, mut out) = (0, String::new(), String::new(), Vec::new()); + while i < b.len() { + let start = i; + if b[i..].starts_with(b"--") { + while i < b.len() && b[i] != b'\n' { + i += 1; + } + part.push(' '); + raw.push_str(&sql[start..i]); + continue; + } + if b[i..].starts_with(b"/*") { + let mut depth = 1; + i += 2; + while i < b.len() && depth > 0 { + if b[i..].starts_with(b"/*") { + depth += 1; + i += 2; + } else if b[i..].starts_with(b"*/") { + depth -= 1; + i += 2; + } else { + i += 1; + } + } + ensure!(depth == 0, "unterminated SQL comment"); + part.push(' '); + raw.push_str(&sql[start..i]); + continue; + } + if b[i] == b'\'' || b[i] == b'"' { + let quote = b[i]; + let escaped = quote == b'\'' + && i > 0 + && matches!(b[i - 1], b'e' | b'E') + && (i == 1 || !sql[..i - 1].chars().next_back().is_some_and(ident_continue)); + i += 1; + let mut closed = false; + while i < b.len() { + if escaped && b[i] == b'\\' { + i += 2; + } else if b[i] == quote { + i += 1; + if i < b.len() && b[i] == quote { + i += 1; + } else if quote == b'\'' + && let Some(next) = continued_string(b, i) + { + i = next; + } else { + closed = true; + break; + } + } else { + i += 1; + } + } + ensure!(closed && i <= b.len(), "unterminated SQL quote"); + } else if b[i] == b'$' { + let mut j = i + 1; + for c in sql[j..].chars() { + if ident_start(c) || (j > i + 1 && c.is_ascii_digit()) { + j += c.len_utf8(); + } else { + break; + } + } + if j < b.len() && b[j] == b'$' { + let tag = &sql[i..=j]; + let rest = &sql[j + 1..]; + let end = rest + .find(tag) + .ok_or_else(|| anyhow::anyhow!("unterminated dollar body"))?; + i = j + 1 + end + tag.len(); + } else { + i += 1; + } + } else if sql[i..].chars().next().is_some_and(ident_start) { + // '$' is legal inside an unquoted identifier. Consume the whole + // identifier before looking for a dollar-quoted body; PostgreSQL + // requires a delimiter between an identifier and such a body. + for c in sql[i..].chars() { + if !ident_continue(c) { + break; + } + i += c.len_utf8(); + } + } else if b[i] == b';' { + if !part.trim().is_empty() { + out.push((part.trim().to_owned(), raw.trim_start().to_owned())); + } + part.clear(); + raw.clear(); + i += 1; + continue; + } else { + // Work at UTF-8 character boundaries even in unquoted identifiers. + i += sql[i..].chars().next().map(char::len_utf8).unwrap_or(1); + } + part.push_str(&sql[start..i]); + raw.push_str(&sql[start..i]); + } + if !part.trim().is_empty() { + out.push((part.trim().to_owned(), raw.trim_start().to_owned())); + } + Ok(out) +} + +pub fn statements(sql: &str) -> Result> { + Ok(lex(sql)? + .into_iter() + .map(|(inspection, _)| inspection) + .collect()) +} + +pub fn body(sql: &str) -> Result { + let mut parts = lex(sql)?; + ensure!(!parts.is_empty(), "empty migration"); + if parts[0].0.eq_ignore_ascii_case("BEGIN") { + ensure!( + parts + .last() + .is_some_and(|s| s.0.eq_ignore_ascii_case("COMMIT")), + "BEGIN without final COMMIT" + ); + parts.remove(0); + parts.pop(); + } + for (part, _) in &parts { + let first = part + .split_whitespace() + .next() + .unwrap_or("") + .to_ascii_uppercase(); + if matches!( + first.as_str(), + "BEGIN" + | "START" + | "COMMIT" + | "END" + | "ROLLBACK" + | "ABORT" + | "SAVEPOINT" + | "RELEASE" + | "PREPARE" + ) || first.starts_with('\\') + { + bail!("migration contains transaction control or a psql command: {first}"); + } + } + Ok(parts + .into_iter() + .map(|(_, raw)| raw) + .collect::>() + .join(";\n") + + ";") +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn dollar_and_comment_bodies() -> Result<()> { + let s = "-- BEGIN;\nBEGIN; DO $q$ BEGIN RAISE NOTICE 'COMMIT;'; END $q$; /* a /* b */ c */ COMMIT;"; + assert_eq!(body(s)?, "DO $q$ BEGIN RAISE NOTICE 'COMMIT;'; END $q$;"); + Ok(()) + } + #[test] + fn nested_control_refused() { + for s in [ + "BEGIN; SELECT 1; COMMIT; COMMIT;", + "SELECT 1; ROLLBACK;", + "START TRANSACTION; SELECT 1;", + "BEGIN; SELECT 1;", + "SELECT 1; \\i file", + ] { + assert!(body(s).is_err(), "{s}"); + } + } + #[test] + fn quotes_and_unicode() -> Result<()> { + assert_eq!( + statements("SELECT E'a\\';b', 'c'';d', \"é;\"; SELECT '𝄞';")?.len(), + 2 + ); + Ok(()) + } + #[test] + fn incomplete_refused() { + for s in ["SELECT 'x", "DO $$ x", "/* unclosed", "-- empty"] { + assert!(body(s).is_err()); + } + } + #[test] + fn dollar_identifiers_do_not_quote_following_statements() -> Result<()> { + let source = + "CREATE TABLE a$tag$ (x int); COMMIT; CREATE TABLE b$tag$ (y int); SELECT 1/0;"; + assert_eq!(statements(source)?.len(), 4); + assert!(body(source).is_err()); + assert!(body("CREATE TABLE é$tag$ (x int); COMMIT; CREATE TABLE z$tag$ (y int);").is_err()); + assert_eq!( + statements("SELECT ordinary$identifier, other$$identifier;")?.len(), + 1 + ); + Ok(()) + } + #[test] + fn unicode_dollar_tags_preserve_function_bodies() -> Result<()> { + let source = "DO $тег_1$ BEGIN PERFORM 1; END $тег_1$; SELECT 2;"; + assert_eq!(statements(source)?.len(), 2); + assert!(body(source)?.contains("BEGIN PERFORM 1; END")); + Ok(()) + } + #[test] + fn executable_comments_are_preserved() -> Result<()> { + let source = "SELECT 'a' /* line\n /* nested */ end */ 'b';"; + assert_eq!(body(source)?, source); + assert!(body("BEGIN; SELECT 1; COMMIT/* trailing */;")?.contains("SELECT 1")); + assert!(body("SELECT 1; COMMIT/* trailing */;").is_err()); + Ok(()) + } + #[test] + fn escaped_string_continuations_keep_their_mode() -> Result<()> { + let source = "SELECT E'a' -- explanation\n'b\\'c'; SELECT 2;"; + assert_eq!(statements(source)?.len(), 2); + assert!(body(source)?.contains("'b\\'c'")); + Ok(()) + } + #[test] + fn typed_literals_after_identifiers_use_ordinary_string_mode() -> Result<()> { + // E is a prefix only at a token boundary. Dollar and non-ASCII characters + // are PostgreSQL identifier continuations too; use the same rule above. + for name in ["typed$E", "typedéE", "typedeE"] { + let source = format!("SELECT {name}'a\\'; SELECT 2;"); + assert_eq!(statements(&source)?.len(), 2, "{name}"); + assert!(body(&source).is_ok(), "{name}"); + } + assert_eq!(statements("SELECT E'a\\\\'; SELECT 2;")?.len(), 2); + Ok(()) + } +} diff --git a/queue-rs/polis-migrate/tests/adoption-cases.json b/queue-rs/polis-migrate/tests/adoption-cases.json new file mode 100644 index 0000000000..9faa38ef83 --- /dev/null +++ b/queue-rs/polis-migrate/tests/adoption-cases.json @@ -0,0 +1,1112 @@ +[ + { + "name": "canonical_3", + "migration": 3, + "mutation": "", + "expected": "t" + }, + { + "name": "default_added_participants_extended_permanent_cookie", + "migration": 3, + "mutation": "ALTER TABLE participants_extended ALTER COLUMN permanent_cookie SET DEFAULT 'constructed';", + "expected": "f" + }, + { + "name": "nullability_participants_extended_permanent_cookie", + "migration": 3, + "mutation": "ALTER TABLE participants_extended ALTER COLUMN permanent_cookie SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_added_participants_extended_origin", + "migration": 3, + "mutation": "ALTER TABLE participants_extended ALTER COLUMN origin SET DEFAULT 'constructed';", + "expected": "f" + }, + { + "name": "nullability_participants_extended_origin", + "migration": 3, + "mutation": "ALTER TABLE participants_extended ALTER COLUMN origin SET NOT NULL;", + "expected": "f" + }, + { + "name": "canonical_8", + "migration": 8, + "mutation": "", + "expected": "t" + }, + { + "name": "default_removed_conversations_importance_enabled", + "migration": 8, + "mutation": "ALTER TABLE conversations ALTER COLUMN importance_enabled DROP DEFAULT;", + "expected": "f" + }, + { + "name": "default_removed_votes_high_priority", + "migration": 8, + "mutation": "ALTER TABLE votes ALTER COLUMN high_priority DROP DEFAULT;", + "expected": "f" + }, + { + "name": "canonical_9", + "migration": 9, + "mutation": "", + "expected": "t" + }, + { + "name": "default_added_zinvites_uuid", + "migration": 9, + "mutation": "ALTER TABLE zinvites ALTER COLUMN uuid SET DEFAULT '00000000-0000-0000-0000-000000000001';", + "expected": "f" + }, + { + "name": "nullability_zinvites_uuid", + "migration": 9, + "mutation": "ALTER TABLE zinvites ALTER COLUMN uuid SET NOT NULL;", + "expected": "f" + }, + { + "name": "canonical_10", + "migration": 10, + "mutation": "", + "expected": "t" + }, + { + "name": "default_added_oidc_user_mappings_oidc_sub", + "migration": 10, + "mutation": "ALTER TABLE oidc_user_mappings ALTER COLUMN oidc_sub SET DEFAULT 'constructed';", + "expected": "f" + }, + { + "name": "default_added_oidc_user_mappings_uid", + "migration": 10, + "mutation": "ALTER TABLE oidc_user_mappings ALTER COLUMN uid SET DEFAULT 7;", + "expected": "f" + }, + { + "name": "default_removed_oidc_user_mappings_created", + "migration": 10, + "mutation": "ALTER TABLE oidc_user_mappings ALTER COLUMN created DROP DEFAULT;", + "expected": "f" + }, + { + "name": "nullability_oidc_user_mappings_created", + "migration": 10, + "mutation": "ALTER TABLE oidc_user_mappings ALTER COLUMN created SET NOT NULL;", + "expected": "f" + }, + { + "name": "canonical_11", + "migration": 11, + "mutation": "", + "expected": "t" + }, + { + "name": "canonical_12", + "migration": 12, + "mutation": "", + "expected": "t" + }, + { + "name": "default_added_topic_agenda_selections_zid", + "migration": 12, + "mutation": "ALTER TABLE topic_agenda_selections ALTER COLUMN zid SET DEFAULT 7;", + "expected": "f" + }, + { + "name": "default_added_topic_agenda_selections_pid", + "migration": 12, + "mutation": "ALTER TABLE topic_agenda_selections ALTER COLUMN pid SET DEFAULT 7;", + "expected": "f" + }, + { + "name": "default_removed_topic_agenda_selections_archetypal_selections", + "migration": 12, + "mutation": "ALTER TABLE topic_agenda_selections ALTER COLUMN archetypal_selections DROP DEFAULT;", + "expected": "f" + }, + { + "name": "default_added_topic_agenda_selections_delphi_job_id", + "migration": 12, + "mutation": "ALTER TABLE topic_agenda_selections ALTER COLUMN delphi_job_id SET DEFAULT 'constructed';", + "expected": "f" + }, + { + "name": "nullability_topic_agenda_selections_delphi_job_id", + "migration": 12, + "mutation": "ALTER TABLE topic_agenda_selections ALTER COLUMN delphi_job_id SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_removed_topic_agenda_selections_total_selections", + "migration": 12, + "mutation": "ALTER TABLE topic_agenda_selections ALTER COLUMN total_selections DROP DEFAULT;", + "expected": "f" + }, + { + "name": "default_removed_topic_agenda_selections_created_at", + "migration": 12, + "mutation": "ALTER TABLE topic_agenda_selections ALTER COLUMN created_at DROP DEFAULT;", + "expected": "f" + }, + { + "name": "nullability_topic_agenda_selections_created_at", + "migration": 12, + "mutation": "ALTER TABLE topic_agenda_selections ALTER COLUMN created_at SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_removed_topic_agenda_selections_updated_at", + "migration": 12, + "mutation": "ALTER TABLE topic_agenda_selections ALTER COLUMN updated_at DROP DEFAULT;", + "expected": "f" + }, + { + "name": "nullability_topic_agenda_selections_updated_at", + "migration": 12, + "mutation": "ALTER TABLE topic_agenda_selections ALTER COLUMN updated_at SET NOT NULL;", + "expected": "f" + }, + { + "name": "canonical_13", + "migration": 13, + "mutation": "", + "expected": "t" + }, + { + "name": "default_removed_conversations_treevite_enabled", + "migration": 13, + "mutation": "ALTER TABLE conversations ALTER COLUMN treevite_enabled DROP DEFAULT;", + "expected": "f" + }, + { + "name": "nullability_conversations_treevite_enabled", + "migration": 13, + "mutation": "ALTER TABLE conversations ALTER COLUMN treevite_enabled SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_removed_treevite_waves_id", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves ALTER COLUMN id DROP DEFAULT;", + "expected": "f" + }, + { + "name": "default_added_treevite_waves_zid", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves ALTER COLUMN zid SET DEFAULT 7;", + "expected": "f" + }, + { + "name": "default_added_treevite_waves_wave", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves ALTER COLUMN wave SET DEFAULT 7;", + "expected": "f" + }, + { + "name": "default_added_treevite_waves_parent_wave", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves ALTER COLUMN parent_wave SET DEFAULT 7;", + "expected": "f" + }, + { + "name": "nullability_treevite_waves_parent_wave", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves ALTER COLUMN parent_wave SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_added_treevite_waves_size", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves ALTER COLUMN size SET DEFAULT 7;", + "expected": "f" + }, + { + "name": "nullability_treevite_waves_size", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves ALTER COLUMN size SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_added_treevite_waves_invites_per_user", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves ALTER COLUMN invites_per_user SET DEFAULT 7;", + "expected": "f" + }, + { + "name": "default_removed_treevite_waves_owner_invites", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves ALTER COLUMN owner_invites DROP DEFAULT;", + "expected": "f" + }, + { + "name": "default_removed_treevite_waves_created_at", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves ALTER COLUMN created_at DROP DEFAULT;", + "expected": "f" + }, + { + "name": "nullability_treevite_waves_created_at", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves ALTER COLUMN created_at SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_removed_treevite_waves_updated_at", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves ALTER COLUMN updated_at DROP DEFAULT;", + "expected": "f" + }, + { + "name": "nullability_treevite_waves_updated_at", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves ALTER COLUMN updated_at SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_removed_treevite_invites_id", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER COLUMN id DROP DEFAULT;", + "expected": "f" + }, + { + "name": "default_added_treevite_invites_zid", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER COLUMN zid SET DEFAULT 7;", + "expected": "f" + }, + { + "name": "default_added_treevite_invites_wave_id", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER COLUMN wave_id SET DEFAULT 7;", + "expected": "f" + }, + { + "name": "default_added_treevite_invites_parent_invite_id", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER COLUMN parent_invite_id SET DEFAULT 7;", + "expected": "f" + }, + { + "name": "nullability_treevite_invites_parent_invite_id", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER COLUMN parent_invite_id SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_removed_treevite_invites_status", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER COLUMN status DROP DEFAULT;", + "expected": "f" + }, + { + "name": "default_added_treevite_invites_invite_owner_pid", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER COLUMN invite_owner_pid SET DEFAULT 7;", + "expected": "f" + }, + { + "name": "nullability_treevite_invites_invite_owner_pid", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER COLUMN invite_owner_pid SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_added_treevite_invites_invite_used_by_pid", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER COLUMN invite_used_by_pid SET DEFAULT 7;", + "expected": "f" + }, + { + "name": "nullability_treevite_invites_invite_used_by_pid", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER COLUMN invite_used_by_pid SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_added_treevite_invites_invite_used_at", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER COLUMN invite_used_at SET DEFAULT '2000-01-01';", + "expected": "f" + }, + { + "name": "nullability_treevite_invites_invite_used_at", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER COLUMN invite_used_at SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_removed_treevite_invites_created_at", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER COLUMN created_at DROP DEFAULT;", + "expected": "f" + }, + { + "name": "nullability_treevite_invites_created_at", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER COLUMN created_at SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_removed_treevite_invites_updated_at", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER COLUMN updated_at DROP DEFAULT;", + "expected": "f" + }, + { + "name": "nullability_treevite_invites_updated_at", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER COLUMN updated_at SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_removed_treevite_login_codes_id", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes ALTER COLUMN id DROP DEFAULT;", + "expected": "f" + }, + { + "name": "default_added_treevite_login_codes_zid", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes ALTER COLUMN zid SET DEFAULT 7;", + "expected": "f" + }, + { + "name": "default_added_treevite_login_codes_pid", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes ALTER COLUMN pid SET DEFAULT 7;", + "expected": "f" + }, + { + "name": "default_added_treevite_login_codes_login_code_hash", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes ALTER COLUMN login_code_hash SET DEFAULT 'constructed';", + "expected": "f" + }, + { + "name": "default_removed_treevite_login_codes_fp_kid", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes ALTER COLUMN fp_kid DROP DEFAULT;", + "expected": "f" + }, + { + "name": "default_removed_treevite_login_codes_revoked", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes ALTER COLUMN revoked DROP DEFAULT;", + "expected": "f" + }, + { + "name": "default_added_treevite_login_codes_expires_at", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes ALTER COLUMN expires_at SET DEFAULT '2000-01-01';", + "expected": "f" + }, + { + "name": "nullability_treevite_login_codes_expires_at", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes ALTER COLUMN expires_at SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_added_treevite_login_codes_last_used_at", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes ALTER COLUMN last_used_at SET DEFAULT '2000-01-01';", + "expected": "f" + }, + { + "name": "nullability_treevite_login_codes_last_used_at", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes ALTER COLUMN last_used_at SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_removed_treevite_login_codes_created_at", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes ALTER COLUMN created_at DROP DEFAULT;", + "expected": "f" + }, + { + "name": "nullability_treevite_login_codes_created_at", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes ALTER COLUMN created_at SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_removed_treevite_login_codes_updated_at", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes ALTER COLUMN updated_at DROP DEFAULT;", + "expected": "f" + }, + { + "name": "nullability_treevite_login_codes_updated_at", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes ALTER COLUMN updated_at SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_added_treevite_invites_invite_code", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER COLUMN invite_code SET DEFAULT 'constructed';", + "expected": "f" + }, + { + "name": "default_added_treevite_login_codes_login_code_fingerprint", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes ALTER COLUMN login_code_fingerprint SET DEFAULT 'constructed';", + "expected": "f" + }, + { + "name": "default_added_treevite_login_codes_login_code_lookup", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes ALTER COLUMN login_code_lookup SET DEFAULT 'constructed';", + "expected": "f" + }, + { + "name": "nullability_treevite_login_codes_login_code_lookup", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes ALTER COLUMN login_code_lookup SET NOT NULL;", + "expected": "f" + }, + { + "name": "canonical_14", + "migration": 14, + "mutation": "", + "expected": "t" + }, + { + "name": "default_removed_reports_mod_level", + "migration": 14, + "mutation": "ALTER TABLE reports ALTER COLUMN mod_level DROP DEFAULT;", + "expected": "f" + }, + { + "name": "canonical_15", + "migration": 15, + "mutation": "", + "expected": "t" + }, + { + "name": "default_removed_conversations_xid_required", + "migration": 15, + "mutation": "ALTER TABLE conversations ALTER COLUMN xid_required DROP DEFAULT;", + "expected": "f" + }, + { + "name": "default_added_xid_whitelist_zid", + "migration": 15, + "mutation": "ALTER TABLE xid_whitelist ALTER COLUMN zid SET DEFAULT 7;", + "expected": "f" + }, + { + "name": "nullability_xid_whitelist_zid", + "migration": 15, + "mutation": "ALTER TABLE xid_whitelist ALTER COLUMN zid SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_added_xids_zid", + "migration": 15, + "mutation": "ALTER TABLE xids ALTER COLUMN zid SET DEFAULT 7;", + "expected": "f" + }, + { + "name": "nullability_xids_zid", + "migration": 15, + "mutation": "ALTER TABLE xids ALTER COLUMN zid SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_added_xids_pid", + "migration": 15, + "mutation": "ALTER TABLE xids ALTER COLUMN pid SET DEFAULT 7;", + "expected": "f" + }, + { + "name": "nullability_xids_pid", + "migration": 15, + "mutation": "ALTER TABLE xids ALTER COLUMN pid SET NOT NULL;", + "expected": "f" + }, + { + "name": "canonical_16", + "migration": 16, + "mutation": "", + "expected": "t" + }, + { + "name": "default_added_comments_original_id", + "migration": 16, + "mutation": "ALTER TABLE comments ALTER COLUMN original_id SET DEFAULT '00000000-0000-0000-0000-000000000001';", + "expected": "f" + }, + { + "name": "nullability_comments_original_id", + "migration": 16, + "mutation": "ALTER TABLE comments ALTER COLUMN original_id SET NOT NULL;", + "expected": "f" + }, + { + "name": "canonical_17", + "migration": 17, + "mutation": "", + "expected": "t" + }, + { + "name": "default_removed_byod_import_jobs_id", + "migration": 17, + "mutation": "ALTER TABLE byod_import_jobs ALTER COLUMN id DROP DEFAULT;", + "expected": "f" + }, + { + "name": "default_added_byod_import_jobs_zid", + "migration": 17, + "mutation": "ALTER TABLE byod_import_jobs ALTER COLUMN zid SET DEFAULT 7;", + "expected": "f" + }, + { + "name": "default_added_byod_import_jobs_s3_key", + "migration": 17, + "mutation": "ALTER TABLE byod_import_jobs ALTER COLUMN s3_key SET DEFAULT 'constructed';", + "expected": "f" + }, + { + "name": "default_removed_byod_import_jobs_status", + "migration": 17, + "mutation": "ALTER TABLE byod_import_jobs ALTER COLUMN status DROP DEFAULT;", + "expected": "f" + }, + { + "name": "nullability_byod_import_jobs_status", + "migration": 17, + "mutation": "ALTER TABLE byod_import_jobs ALTER COLUMN status SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_removed_byod_import_jobs_stage", + "migration": 17, + "mutation": "ALTER TABLE byod_import_jobs ALTER COLUMN stage DROP DEFAULT;", + "expected": "f" + }, + { + "name": "nullability_byod_import_jobs_stage", + "migration": 17, + "mutation": "ALTER TABLE byod_import_jobs ALTER COLUMN stage SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_added_byod_import_jobs_error_message", + "migration": 17, + "mutation": "ALTER TABLE byod_import_jobs ALTER COLUMN error_message SET DEFAULT 'constructed';", + "expected": "f" + }, + { + "name": "nullability_byod_import_jobs_error_message", + "migration": 17, + "mutation": "ALTER TABLE byod_import_jobs ALTER COLUMN error_message SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_removed_byod_import_jobs_created_at", + "migration": 17, + "mutation": "ALTER TABLE byod_import_jobs ALTER COLUMN created_at DROP DEFAULT;", + "expected": "f" + }, + { + "name": "nullability_byod_import_jobs_created_at", + "migration": 17, + "mutation": "ALTER TABLE byod_import_jobs ALTER COLUMN created_at SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_removed_byod_import_jobs_updated_at", + "migration": 17, + "mutation": "ALTER TABLE byod_import_jobs ALTER COLUMN updated_at DROP DEFAULT;", + "expected": "f" + }, + { + "name": "nullability_byod_import_jobs_updated_at", + "migration": 17, + "mutation": "ALTER TABLE byod_import_jobs ALTER COLUMN updated_at SET NOT NULL;", + "expected": "f" + }, + { + "name": "canonical_18", + "migration": 18, + "mutation": "", + "expected": "t" + }, + { + "name": "default_removed_conversations_topics_enabled", + "migration": 18, + "mutation": "ALTER TABLE conversations ALTER COLUMN topics_enabled DROP DEFAULT;", + "expected": "f" + }, + { + "name": "missing_oidc_user_mappings_pkey", + "migration": 10, + "mutation": "ALTER TABLE oidc_user_mappings DROP CONSTRAINT oidc_user_mappings_pkey CASCADE;", + "expected": "f" + }, + { + "name": "missing_oidc_user_mappings_uid_fkey", + "migration": 10, + "mutation": "ALTER TABLE oidc_user_mappings DROP CONSTRAINT oidc_user_mappings_uid_fkey CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_oidc_user_mappings_uid_fkey", + "migration": 10, + "mutation": "ALTER TABLE oidc_user_mappings DROP CONSTRAINT oidc_user_mappings_uid_fkey CASCADE; ALTER TABLE oidc_user_mappings ADD CONSTRAINT oidc_user_mappings_uid_fkey FOREIGN KEY (uid) REFERENCES users(uid) ON DELETE CASCADE NOT VALID;", + "expected": "f" + }, + { + "name": "deferred_oidc_user_mappings_uid_fkey", + "migration": 10, + "mutation": "ALTER TABLE oidc_user_mappings ALTER CONSTRAINT oidc_user_mappings_uid_fkey DEFERRABLE INITIALLY DEFERRED;", + "expected": "f" + }, + { + "name": "missing_oidc_user_mappings_uid_key", + "migration": 10, + "mutation": "ALTER TABLE oidc_user_mappings DROP CONSTRAINT oidc_user_mappings_uid_key CASCADE;", + "expected": "f" + }, + { + "name": "disabled_triggers_oidc_user_mappings", + "migration": 10, + "mutation": "ALTER TABLE oidc_user_mappings DISABLE TRIGGER ALL;", + "expected": "f" + }, + { + "name": "missing_fk_conversation", + "migration": 12, + "mutation": "ALTER TABLE topic_agenda_selections DROP CONSTRAINT fk_conversation CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_fk_conversation", + "migration": 12, + "mutation": "ALTER TABLE topic_agenda_selections DROP CONSTRAINT fk_conversation CASCADE; ALTER TABLE topic_agenda_selections ADD CONSTRAINT fk_conversation FOREIGN KEY (zid) REFERENCES conversations(zid) ON DELETE CASCADE NOT VALID;", + "expected": "f" + }, + { + "name": "deferred_fk_conversation", + "migration": 12, + "mutation": "ALTER TABLE topic_agenda_selections ALTER CONSTRAINT fk_conversation DEFERRABLE INITIALLY DEFERRED;", + "expected": "f" + }, + { + "name": "missing_fk_participant", + "migration": 12, + "mutation": "ALTER TABLE topic_agenda_selections DROP CONSTRAINT fk_participant CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_fk_participant", + "migration": 12, + "mutation": "ALTER TABLE topic_agenda_selections DROP CONSTRAINT fk_participant CASCADE; ALTER TABLE topic_agenda_selections ADD CONSTRAINT fk_participant FOREIGN KEY (zid, pid) REFERENCES participants(zid, pid) ON DELETE CASCADE NOT VALID;", + "expected": "f" + }, + { + "name": "deferred_fk_participant", + "migration": 12, + "mutation": "ALTER TABLE topic_agenda_selections ALTER CONSTRAINT fk_participant DEFERRABLE INITIALLY DEFERRED;", + "expected": "f" + }, + { + "name": "missing_topic_agenda_selections_pkey", + "migration": 12, + "mutation": "ALTER TABLE topic_agenda_selections DROP CONSTRAINT topic_agenda_selections_pkey CASCADE;", + "expected": "f" + }, + { + "name": "disabled_triggers_topic_agenda_selections", + "migration": 12, + "mutation": "ALTER TABLE topic_agenda_selections DISABLE TRIGGER ALL;", + "expected": "f" + }, + { + "name": "missing_treevite_waves_invites_per_user_check", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves DROP CONSTRAINT treevite_waves_invites_per_user_check CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_treevite_waves_invites_per_user_check", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves DROP CONSTRAINT treevite_waves_invites_per_user_check CASCADE; ALTER TABLE treevite_waves ADD CONSTRAINT treevite_waves_invites_per_user_check CHECK ((invites_per_user >= 0)) NOT VALID;", + "expected": "f" + }, + { + "name": "missing_treevite_waves_not_both_zero", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves DROP CONSTRAINT treevite_waves_not_both_zero CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_treevite_waves_not_both_zero", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves DROP CONSTRAINT treevite_waves_not_both_zero CASCADE; ALTER TABLE treevite_waves ADD CONSTRAINT treevite_waves_not_both_zero CHECK (((invites_per_user > 0) OR (owner_invites > 0))) NOT VALID;", + "expected": "f" + }, + { + "name": "missing_treevite_waves_owner_invites_check", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves DROP CONSTRAINT treevite_waves_owner_invites_check CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_treevite_waves_owner_invites_check", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves DROP CONSTRAINT treevite_waves_owner_invites_check CASCADE; ALTER TABLE treevite_waves ADD CONSTRAINT treevite_waves_owner_invites_check CHECK ((owner_invites >= 0)) NOT VALID;", + "expected": "f" + }, + { + "name": "missing_treevite_waves_parent_wave_check", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves DROP CONSTRAINT treevite_waves_parent_wave_check CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_treevite_waves_parent_wave_check", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves DROP CONSTRAINT treevite_waves_parent_wave_check CASCADE; ALTER TABLE treevite_waves ADD CONSTRAINT treevite_waves_parent_wave_check CHECK (((parent_wave IS NULL) OR (parent_wave >= 0))) NOT VALID;", + "expected": "f" + }, + { + "name": "missing_treevite_waves_pkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves DROP CONSTRAINT treevite_waves_pkey CASCADE;", + "expected": "f" + }, + { + "name": "missing_treevite_waves_size_check", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves DROP CONSTRAINT treevite_waves_size_check CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_treevite_waves_size_check", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves DROP CONSTRAINT treevite_waves_size_check CASCADE; ALTER TABLE treevite_waves ADD CONSTRAINT treevite_waves_size_check CHECK (((size IS NULL) OR (size >= 0))) NOT VALID;", + "expected": "f" + }, + { + "name": "missing_treevite_waves_wave_check", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves DROP CONSTRAINT treevite_waves_wave_check CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_treevite_waves_wave_check", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves DROP CONSTRAINT treevite_waves_wave_check CASCADE; ALTER TABLE treevite_waves ADD CONSTRAINT treevite_waves_wave_check CHECK ((wave >= 1)) NOT VALID;", + "expected": "f" + }, + { + "name": "missing_treevite_waves_zid_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves DROP CONSTRAINT treevite_waves_zid_fkey CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_treevite_waves_zid_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves DROP CONSTRAINT treevite_waves_zid_fkey CASCADE; ALTER TABLE treevite_waves ADD CONSTRAINT treevite_waves_zid_fkey FOREIGN KEY (zid) REFERENCES conversations(zid) ON DELETE CASCADE NOT VALID;", + "expected": "f" + }, + { + "name": "deferred_treevite_waves_zid_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves ALTER CONSTRAINT treevite_waves_zid_fkey DEFERRABLE INITIALLY DEFERRED;", + "expected": "f" + }, + { + "name": "missing_treevite_waves_zid_wave_key", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves DROP CONSTRAINT treevite_waves_zid_wave_key CASCADE;", + "expected": "f" + }, + { + "name": "disabled_triggers_treevite_waves", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves DISABLE TRIGGER ALL;", + "expected": "f" + }, + { + "name": "missing_treevite_invites_code_unique", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites DROP CONSTRAINT treevite_invites_code_unique CASCADE;", + "expected": "f" + }, + { + "name": "missing_treevite_invites_owner_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites DROP CONSTRAINT treevite_invites_owner_fkey CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_treevite_invites_owner_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites DROP CONSTRAINT treevite_invites_owner_fkey CASCADE; ALTER TABLE treevite_invites ADD CONSTRAINT treevite_invites_owner_fkey FOREIGN KEY (zid, invite_owner_pid) REFERENCES participants(zid, pid) NOT VALID;", + "expected": "f" + }, + { + "name": "deferred_treevite_invites_owner_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER CONSTRAINT treevite_invites_owner_fkey DEFERRABLE INITIALLY DEFERRED;", + "expected": "f" + }, + { + "name": "missing_treevite_invites_parent_invite_id_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites DROP CONSTRAINT treevite_invites_parent_invite_id_fkey CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_treevite_invites_parent_invite_id_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites DROP CONSTRAINT treevite_invites_parent_invite_id_fkey CASCADE; ALTER TABLE treevite_invites ADD CONSTRAINT treevite_invites_parent_invite_id_fkey FOREIGN KEY (parent_invite_id) REFERENCES treevite_invites(id) ON DELETE SET NULL NOT VALID;", + "expected": "f" + }, + { + "name": "deferred_treevite_invites_parent_invite_id_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER CONSTRAINT treevite_invites_parent_invite_id_fkey DEFERRABLE INITIALLY DEFERRED;", + "expected": "f" + }, + { + "name": "missing_treevite_invites_pkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites DROP CONSTRAINT treevite_invites_pkey CASCADE;", + "expected": "f" + }, + { + "name": "missing_treevite_invites_status_check", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites DROP CONSTRAINT treevite_invites_status_check CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_treevite_invites_status_check", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites DROP CONSTRAINT treevite_invites_status_check CASCADE; ALTER TABLE treevite_invites ADD CONSTRAINT treevite_invites_status_check CHECK ((status = ANY (ARRAY[0, 1, 2, 3]))) NOT VALID;", + "expected": "f" + }, + { + "name": "missing_treevite_invites_used_by_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites DROP CONSTRAINT treevite_invites_used_by_fkey CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_treevite_invites_used_by_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites DROP CONSTRAINT treevite_invites_used_by_fkey CASCADE; ALTER TABLE treevite_invites ADD CONSTRAINT treevite_invites_used_by_fkey FOREIGN KEY (zid, invite_used_by_pid) REFERENCES participants(zid, pid) NOT VALID;", + "expected": "f" + }, + { + "name": "deferred_treevite_invites_used_by_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER CONSTRAINT treevite_invites_used_by_fkey DEFERRABLE INITIALLY DEFERRED;", + "expected": "f" + }, + { + "name": "missing_treevite_invites_wave_id_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites DROP CONSTRAINT treevite_invites_wave_id_fkey CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_treevite_invites_wave_id_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites DROP CONSTRAINT treevite_invites_wave_id_fkey CASCADE; ALTER TABLE treevite_invites ADD CONSTRAINT treevite_invites_wave_id_fkey FOREIGN KEY (wave_id) REFERENCES treevite_waves(id) ON DELETE CASCADE NOT VALID;", + "expected": "f" + }, + { + "name": "deferred_treevite_invites_wave_id_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER CONSTRAINT treevite_invites_wave_id_fkey DEFERRABLE INITIALLY DEFERRED;", + "expected": "f" + }, + { + "name": "missing_treevite_invites_zid_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites DROP CONSTRAINT treevite_invites_zid_fkey CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_treevite_invites_zid_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites DROP CONSTRAINT treevite_invites_zid_fkey CASCADE; ALTER TABLE treevite_invites ADD CONSTRAINT treevite_invites_zid_fkey FOREIGN KEY (zid) REFERENCES conversations(zid) ON DELETE CASCADE NOT VALID;", + "expected": "f" + }, + { + "name": "deferred_treevite_invites_zid_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER CONSTRAINT treevite_invites_zid_fkey DEFERRABLE INITIALLY DEFERRED;", + "expected": "f" + }, + { + "name": "disabled_triggers_treevite_invites", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites DISABLE TRIGGER ALL;", + "expected": "f" + }, + { + "name": "missing_treevite_login_codes_fp_unique", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes DROP CONSTRAINT treevite_login_codes_fp_unique CASCADE;", + "expected": "f" + }, + { + "name": "missing_treevite_login_codes_lookup_unique", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes DROP CONSTRAINT treevite_login_codes_lookup_unique CASCADE;", + "expected": "f" + }, + { + "name": "missing_treevite_login_codes_participant_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes DROP CONSTRAINT treevite_login_codes_participant_fkey CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_treevite_login_codes_participant_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes DROP CONSTRAINT treevite_login_codes_participant_fkey CASCADE; ALTER TABLE treevite_login_codes ADD CONSTRAINT treevite_login_codes_participant_fkey FOREIGN KEY (zid, pid) REFERENCES participants(zid, pid) ON DELETE CASCADE NOT VALID;", + "expected": "f" + }, + { + "name": "deferred_treevite_login_codes_participant_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes ALTER CONSTRAINT treevite_login_codes_participant_fkey DEFERRABLE INITIALLY DEFERRED;", + "expected": "f" + }, + { + "name": "missing_treevite_login_codes_pid_unique", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes DROP CONSTRAINT treevite_login_codes_pid_unique CASCADE;", + "expected": "f" + }, + { + "name": "missing_treevite_login_codes_pkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes DROP CONSTRAINT treevite_login_codes_pkey CASCADE;", + "expected": "f" + }, + { + "name": "disabled_triggers_treevite_login_codes", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes DISABLE TRIGGER ALL;", + "expected": "f" + }, + { + "name": "missing_xid_whitelist_zid_fkey", + "migration": 15, + "mutation": "ALTER TABLE xid_whitelist DROP CONSTRAINT xid_whitelist_zid_fkey CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_xid_whitelist_zid_fkey", + "migration": 15, + "mutation": "ALTER TABLE xid_whitelist DROP CONSTRAINT xid_whitelist_zid_fkey CASCADE; ALTER TABLE xid_whitelist ADD CONSTRAINT xid_whitelist_zid_fkey FOREIGN KEY (zid) REFERENCES conversations(zid) ON DELETE CASCADE NOT VALID;", + "expected": "f" + }, + { + "name": "deferred_xid_whitelist_zid_fkey", + "migration": 15, + "mutation": "ALTER TABLE xid_whitelist ALTER CONSTRAINT xid_whitelist_zid_fkey DEFERRABLE INITIALLY DEFERRED;", + "expected": "f" + }, + { + "name": "disabled_triggers_xid_whitelist", + "migration": 15, + "mutation": "ALTER TABLE xid_whitelist DISABLE TRIGGER ALL;", + "expected": "f" + }, + { + "name": "missing_xids_zid_fkey", + "migration": 15, + "mutation": "ALTER TABLE xids DROP CONSTRAINT xids_zid_fkey CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_xids_zid_fkey", + "migration": 15, + "mutation": "ALTER TABLE xids DROP CONSTRAINT xids_zid_fkey CASCADE; ALTER TABLE xids ADD CONSTRAINT xids_zid_fkey FOREIGN KEY (zid) REFERENCES conversations(zid) ON DELETE CASCADE NOT VALID;", + "expected": "f" + }, + { + "name": "deferred_xids_zid_fkey", + "migration": 15, + "mutation": "ALTER TABLE xids ALTER CONSTRAINT xids_zid_fkey DEFERRABLE INITIALLY DEFERRED;", + "expected": "f" + }, + { + "name": "missing_xids_pid_fkey", + "migration": 15, + "mutation": "ALTER TABLE xids DROP CONSTRAINT xids_pid_fkey CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_xids_pid_fkey", + "migration": 15, + "mutation": "ALTER TABLE xids DROP CONSTRAINT xids_pid_fkey CASCADE; ALTER TABLE xids ADD CONSTRAINT xids_pid_fkey FOREIGN KEY (zid, pid) REFERENCES participants(zid, pid) ON DELETE SET NULL NOT VALID;", + "expected": "f" + }, + { + "name": "deferred_xids_pid_fkey", + "migration": 15, + "mutation": "ALTER TABLE xids ALTER CONSTRAINT xids_pid_fkey DEFERRABLE INITIALLY DEFERRED;", + "expected": "f" + }, + { + "name": "disabled_triggers_xids", + "migration": 15, + "mutation": "ALTER TABLE xids DISABLE TRIGGER ALL;", + "expected": "f" + }, + { + "name": "missing_byod_import_jobs_pkey", + "migration": 17, + "mutation": "ALTER TABLE byod_import_jobs DROP CONSTRAINT byod_import_jobs_pkey CASCADE;", + "expected": "f" + }, + { + "name": "deferred_unique", + "migration": 10, + "mutation": "ALTER TABLE oidc_user_mappings DROP CONSTRAINT oidc_user_mappings_uid_key; ALTER TABLE oidc_user_mappings ADD CONSTRAINT generated_uid_unique UNIQUE(uid) DEFERRABLE INITIALLY DEFERRED;", + "expected": "f" + }, + { + "name": "m3_wrong_type", + "migration": 3, + "mutation": "ALTER TABLE participants_extended ALTER COLUMN origin TYPE text;", + "expected": "f" + }, + { + "name": "m11_preserved_default", + "migration": 11, + "mutation": "ALTER TABLE suzinvites ALTER COLUMN xid SET DEFAULT 'constructed';", + "expected": "t" + }, + { + "name": "renamed_equivalent_unique", + "migration": 10, + "mutation": "ALTER TABLE oidc_user_mappings RENAME CONSTRAINT oidc_user_mappings_uid_key TO generated_equivalent_key;", + "expected": "t" + }, + { + "name": "m3_generated_column", + "migration": 3, + "mutation": "ALTER TABLE participants_extended DROP COLUMN origin; ALTER TABLE participants_extended ADD COLUMN origin varchar(9999) GENERATED ALWAYS AS ('constructed'::varchar(9999)) STORED;", + "expected": "f" + }, + { + "name": "m17_identity_column", + "migration": 17, + "mutation": "ALTER TABLE byod_import_jobs ALTER COLUMN id DROP DEFAULT; ALTER TABLE byod_import_jobs ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY;", + "expected": "f" + } +] diff --git a/queue-rs/polis-migrate/tests/adoption.py b/queue-rs/polis-migrate/tests/adoption.py new file mode 100644 index 0000000000..07a08fa04c --- /dev/null +++ b/queue-rs/polis-migrate/tests/adoption.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +"""Persistent adoption-contract regression tests; generated DBs and no vote rows. + +Run after starting tests/compose.yml with unique COMPOSE_PROJECT_NAME and ports. +Uses the built polis-migrate in this checkout; no packet or original source needed. +""" +import json, pathlib, unittest +from prove import MIG, runner, sql + +class Adoption(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.db='adoption_contract_template' + sql('postgres',f'CREATE DATABASE {cls.db}') + for path in sorted(MIG.glob('*.sql')): + if int(path.name[:6]) <= 18 or path.name.startswith('000022_'): + sql(cls.db,path.read_text()) + sql(cls.db,"INSERT INTO users(hname) VALUES('constructed adoption sentinel')") + + def test_01_catalog_controls(self): + cases=json.loads(pathlib.Path(__file__).with_name('adoption-cases.json').read_text()) + helpers=(MIG/'adoption/helpers.sql').read_text() + for case in cases: + with self.subTest(case=case['name']): + predicate=next((MIG/'adoption').glob(f"{case['migration']:06d}_*.sql")).read_text() + out=sql(self.db,'BEGIN;\n'+case['mutation']+'\n'+helpers+'\n'+predicate+'\nROLLBACK;') + # psql also prints transaction/function command tags. + values=[line for line in out.splitlines() if line in ('t','f')] + self.assertEqual(values,[case['expected']]) + + def test_02_conflicts_leave_no_history(self): + cases=[(3,"ALTER TABLE participants_extended ALTER COLUMN origin SET DEFAULT 'constructed';"), + (13,'ALTER TABLE treevite_invites DISABLE TRIGGER ALL;'), + (17,'ALTER TABLE byod_import_jobs ALTER COLUMN id DROP DEFAULT;')] + for n,mutation in cases: + with self.subTest(migration=n): + db=f'adoption_contract_conflict_{n}' + sql('postgres',f'CREATE DATABASE {db} TEMPLATE {self.db}') + sql(db,mutation) + p=runner(db,'reconcile','--through','000022',ok=False) + self.assertIn(f'{n:06d}_',p.stderr) + self.assertEqual(sql(db,"SELECT to_regclass('public.migrations') IS NULL"),'t') + + def test_03_canonical_adoption(self): + db='adoption_contract_success' + sql('postgres',f'CREATE DATABASE {db} TEMPLATE {self.db}') + runner(db,'reconcile','--through','000022') + self.assertEqual(sql(db,"SELECT count(*) FROM migrations WHERE status='ADOPTED'"),'20') + self.assertEqual(sql(db,'SELECT hname FROM users'),'constructed adoption sentinel') + +if __name__=='__main__': unittest.main(verbosity=2) diff --git a/queue-rs/polis-migrate/tests/compose.yml b/queue-rs/polis-migrate/tests/compose.yml new file mode 100644 index 0000000000..fe26fd80db --- /dev/null +++ b/queue-rs/polis-migrate/tests/compose.yml @@ -0,0 +1,14 @@ +services: + postgres: + image: postgres:17-alpine + environment: + POSTGRES_HOST_AUTH_METHOD: trust + POSTGRES_USER: postgres + POSTGRES_DB: postgres + ports: + - "127.0.0.1:${POLIS_RECOVERY_PG_PORT:?set an owned port}:5432" + healthcheck: + test: [CMD-SHELL, "pg_isready -U postgres"] + interval: 1s + timeout: 3s + retries: 30 diff --git a/queue-rs/polis-migrate/tests/fresh-image.sh b/queue-rs/polis-migrate/tests/fresh-image.sh new file mode 100644 index 0000000000..80f4c41d97 --- /dev/null +++ b/queue-rs/polis-migrate/tests/fresh-image.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# CI and work boxes run this same build/bootstrap/restart proof. +set -euo pipefail +cd "$(dirname "$0")/../../.." +case "${COMPOSE_PROJECT_NAME:-}" in + polis-migrate-test-?*) ;; + *) echo 'Set an owned COMPOSE_PROJECT_NAME starting polis-migrate-test-' >&2; exit 2 ;; +esac +: "${POLIS_RECOVERY_PG_PORT:?set an owned port}" +export RECOVERY_PG_PORT="$POLIS_RECOVERY_PG_PORT" +export POLIS_MIGRATE_TEST_IMAGE="${COMPOSE_PROJECT_NAME}:fresh-image" +compose=(docker compose -f queue-rs/polis-migrate/tests/compose.yml -f queue-rs/polis-migrate/tests/image.yml) +cleanup() { + local status=$? + trap - EXIT + "${compose[@]}" down -v || status=1 + if docker image inspect "$POLIS_MIGRATE_TEST_IMAGE" >/dev/null 2>&1; then + docker image rm "$POLIS_MIGRATE_TEST_IMAGE" || status=1 + fi + exit "$status" +} +trap cleanup EXIT +python3 queue-rs/polis-migrate/tests/image-proof-test.py +"${compose[@]}" down -v +docker build --build-context queue-rs=queue-rs -t "$POLIS_MIGRATE_TEST_IMAGE" -f server/Dockerfile-db server +"${compose[@]}" up -d --wait --wait-timeout 120 +python3 queue-rs/polis-migrate/tests/image-proof.py diff --git a/queue-rs/polis-migrate/tests/image-proof-test.py b/queue-rs/polis-migrate/tests/image-proof-test.py new file mode 100644 index 0000000000..5b6865fb87 --- /dev/null +++ b/queue-rs/polis-migrate/tests/image-proof-test.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""Regression controls for fresh-image release evolution and corrupt receipts.""" +import importlib.util +import pathlib +import tempfile +import unittest + +spec = importlib.util.spec_from_file_location( + "image_proof", pathlib.Path(__file__).with_name("image-proof.py")) +proof = importlib.util.module_from_spec(spec) +spec.loader.exec_module(proof) + + +class ImageProofTests(unittest.TestCase): + def setUp(self): + self.expected = proof.expected_receipts(proof.MIGRATIONS) + self.rows = [{"name": name, **receipt} for name, receipt in self.expected.items()] + + def test_current_release_includes_adoption(self): + proof.assert_receipts(self.rows, self.expected) + self.assertEqual({row["name"][:6] for row in self.rows if row["status"] == "ADOPTED"}, + proof.RETIRED) + self.assertTrue(any(row["status"] == "APPLIED" for row in self.rows)) + + def test_same_count_wrong_status_refuses(self): + next(row for row in self.rows if row["status"] == "ADOPTED")["status"] = "APPLIED" + with self.assertRaises(AssertionError): + proof.assert_receipts(self.rows, self.expected) + + def test_same_count_wrong_name_refuses(self): + self.rows[0]["name"] = "999999_unselected.sql" + with self.assertRaises(AssertionError): + proof.assert_receipts(self.rows, self.expected) + + def test_same_count_wrong_checksum_refuses(self): + self.rows[0]["checksum"] = "0" * 64 + with self.assertRaises(AssertionError): + proof.assert_receipts(self.rows, self.expected) + + def test_missing_receipt_refuses(self): + with self.assertRaises(AssertionError): + proof.assert_receipts(self.rows[1:], self.expected) + + def test_extra_receipt_refuses(self): + with self.assertRaises(AssertionError): + proof.assert_receipts(self.rows + [{**self.rows[0], "name": "999999_extra.sql"}], + self.expected) + + def test_duplicate_receipt_refuses(self): + with self.assertRaises(AssertionError): + proof.assert_receipts(self.rows + [self.rows[0]], self.expected) + + def test_release_grows_and_shrinks_without_count_edits(self): + with tempfile.TemporaryDirectory() as tmp: + directory = pathlib.Path(tmp) + names = ["000000_initial.sql", "000004_drop_waitinglist_table.sql", + "000030_future.sql"] + for name in names: + (directory / name).write_text("-- synthetic SQL source\n") + for selected in (names[:2], names, names[:1]): + with self.subTest(selected=selected): + (directory / "release.txt").write_text( + "# synthetic release selection\n\n" + "\n".join(selected) + "\n") + expected = proof.expected_receipts(directory) + self.assertEqual(set(expected), set(selected)) + self.assertEqual(len(expected), len(selected)) + proof.assert_receipts([{"name": name, **receipt} + for name, receipt in expected.items()], expected) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/queue-rs/polis-migrate/tests/image-proof.py b/queue-rs/polis-migrate/tests/image-proof.py new file mode 100644 index 0000000000..76f6453aef --- /dev/null +++ b/queue-rs/polis-migrate/tests/image-proof.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""Check the selected fresh-image receipts and preserve them across restart.""" +import collections +import hashlib +import json +import os +import pathlib +import re +import subprocess + + +# Observation-only retirement policy in polis_migrate::load/retired_absent. +# These are migration identities, not a count of the current release. +RETIRED = {"000004", "000005", "000007"} +ROOT = pathlib.Path(__file__).resolve().parents[3] +MIGRATIONS = ROOT / "server/postgres/migrations" + + +def expected_receipts(directory): + names = [line for line in (directory / "release.txt").read_text().splitlines() + if line and not line.startswith("#")] + assert names and len(names) == len(set(names)), "empty/duplicate release selection" + assert all(re.fullmatch(r"[0-9]{6}_[A-Za-z0-9_]+\.sql", name) for name in names) + return { + name: {"status": "ADOPTED" if name[:6] in RETIRED else "APPLIED", + "checksum": hashlib.sha256((directory / name).read_bytes()).hexdigest()} + for name in names + } + + +def assert_receipts(rows, expected): + actual = {row["name"]: {"status": row["status"], "checksum": row["checksum"]} + for row in rows} + assert len(rows) == len(actual), "duplicate history names" + assert actual == expected, f"fresh history differs from release: {actual!r} != {expected!r}" + + +def main(): + assert os.environ.get("COMPOSE_PROJECT_NAME", "").startswith("polis-migrate-test-") + directory = pathlib.Path(__file__).resolve().parent + compose = ["docker", "compose", "-f", str(directory / "compose.yml"), + "-f", str(directory / "image.yml")] + + def run(*args): + return subprocess.check_output(compose + list(args), text=True).strip() + + def query(sql): + return run("exec", "-T", "postgres", "psql", "-X", "-v", "ON_ERROR_STOP=1", + "-U", "postgres", "-d", "postgres", "-Atc", sql) + + def history(): + return [json.loads(line) for line in + query("SELECT row_to_json(m) FROM migrations m ORDER BY name").splitlines()] + + expected = expected_receipts(MIGRATIONS) + before = history() + assert_receipts(before, expected) + assert query("SELECT to_regclass('public.polis_coordinator_install') IS NULL") == "t" + run("restart", "postgres") + run("up", "-d", "--wait", "--wait-timeout", "120") + assert history() == before, "restart changed migration receipts" + assert run("exec", "-T", "-e", + "DATABASE_URL=host=/var/run/postgresql user=postgres dbname=postgres sslmode=disable", + "-e", "POLIS_MIGRATIONS_DIR=/migrations", "postgres", "polis-migrate", "check" + ) == f"migration check: {len(expected)} ready" + counts = dict(sorted(collections.Counter(row["status"] for row in before).items())) + print(f"image proof: 4 checks PASS; {len(expected)} selected receipts; {counts}") + + +if __name__ == "__main__": + main() diff --git a/queue-rs/polis-migrate/tests/image.yml b/queue-rs/polis-migrate/tests/image.yml new file mode 100644 index 0000000000..dcaa565723 --- /dev/null +++ b/queue-rs/polis-migrate/tests/image.yml @@ -0,0 +1,7 @@ +services: + postgres: + image: ${POLIS_MIGRATE_TEST_IMAGE:?run fresh-image.sh with an owned project} + healthcheck: + # The initdb temporary server accepts socket connections before the runner + # has finished. TCP readiness waits for the final server after initialization. + test: [CMD-SHELL, "pg_isready -h 127.0.0.1 -U postgres"] diff --git a/queue-rs/polis-migrate/tests/indexes.py b/queue-rs/polis-migrate/tests/indexes.py new file mode 100644 index 0000000000..657ffd6660 --- /dev/null +++ b/queue-rs/polis-migrate/tests/indexes.py @@ -0,0 +1,84 @@ +"""Real generated Postgres runner controls. No vote rows are inserted.""" +import importlib.util,json,os,pathlib,shutil,subprocess,tempfile,time,unittest +ROOT=pathlib.Path(__file__).resolve().parents[3] +spec=importlib.util.spec_from_file_location('proof',ROOT/'queue-rs/polis-migrate/tests/prove.py');p=importlib.util.module_from_spec(spec);spec.loader.exec_module(p) +ORIGINAL=os.environ.get('ORIGINAL_RUNNER') +class IndexProof(unittest.TestCase): + def setUp(self): + self.tmp=tempfile.TemporaryDirectory();self.addCleanup(self.tmp.cleanup);self.d=pathlib.Path(self.tmp.name) + (self.d/'held.txt').write_text('') + self.m=p.MIG/'000022_add_poll_timestamp_indexes.sql';shutil.copy(self.m,self.d/self.m.name) + def db(self,name,extra=''): + db='idx_'+name;p.sql('postgres',f'CREATE DATABASE {db}') + (self.d/'000000_initial.sql').write_text('CREATE TABLE public.votes(created bigint); CREATE TABLE public.comments(modified bigint, other integer);'+extra) + return db + def runnew(self,db,*args,**kw): + (self.d/'release.txt').write_text(''.join(f.name+'\n' for f in sorted(self.d.glob('*.sql')))) + return p.runner(db,*args,dir=self.d,**kw) + def test_01_original_refuses_large_candidate_passes(self): + db=self.db('large','INSERT INTO public.comments SELECT x,0 FROM generate_series(1,100001) x;') + if ORIGINAL: + before=subprocess.run([ORIGINAL,'apply','--dir',str(self.d)],env=p.env(db),text=True,capture_output=True,timeout=60) + self.assertNotEqual(before.returncode,0);self.assertIn('000022',before.stderr) + self.assertEqual(p.sql(db,"SELECT count(*) FROM migrations WHERE name LIKE '000022%'"),'0') + self.assertIn('BUILDING CONCURRENTLY',self.runnew(db,'apply').stdout) + self.runnew(db,'check');self.assertEqual(p.sql(db,'SELECT count(*) FROM comments'),'100001') + self.assertEqual(p.sql(db,'SELECT count(*) FROM votes'),'0') + def test_02_partial_success_resumes_without_rebuild(self): + db=self.db('partial','CREATE INDEX votes_created_idx ON public.votes(created);') + # Bootstrap only first to observe its OID before the pending index runs. + m=self.d/self.m.name;m.rename(self.d/'index.pending') + self.runnew(db,'apply');oid=p.sql(db,"SELECT 'public.votes_created_idx'::regclass::oid") + (self.d/'index.pending').rename(m) + self.runnew(db,'apply');self.assertEqual(oid,p.sql(db,"SELECT 'public.votes_created_idx'::regclass::oid")) + self.assertIn('applied 0 migration(s)',self.runnew(db,'apply').stdout) + def test_03_collision_preflight_preserves_objects(self): + db=self.db('collision','CREATE INDEX comments_modified_idx ON public.comments(other);') + r=self.runnew(db,'apply',ok=False);self.assertIn('conflicting object',r.stderr) + self.assertEqual(p.sql(db,"SELECT to_regclass('public.votes_created_idx') IS NULL"),'t') + self.assertEqual(p.sql(db,"SELECT count(*) FROM migrations WHERE name LIKE '000022%'"),'0') + def test_04_history_failure_resumes_committed_indexes(self): + db=self.db('history',"CREATE FUNCTION public.refuse_index_record() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.name LIKE '000022%' THEN RAISE EXCEPTION 'generated history fault'; END IF; RETURN NEW; END $$; CREATE TRIGGER refuse_index_record BEFORE INSERT ON migrations FOR EACH ROW EXECUTE FUNCTION public.refuse_index_record();") + self.runnew(db,'apply',ok=False) + before=p.sql(db,"SELECT indexrelid FROM pg_index WHERE indrelid IN ('votes'::regclass,'comments'::regclass) ORDER BY indexrelid") + self.assertEqual(len(before.splitlines()),2) + self.assertEqual(p.sql(db,"SELECT count(*) FROM migrations WHERE name LIKE '000022%'"),'0') + p.sql(db,'DROP TRIGGER refuse_index_record ON migrations');self.runnew(db,'apply') + self.assertEqual(before,p.sql(db,"SELECT indexrelid FROM pg_index WHERE indrelid IN ('votes'::regclass,'comments'::regclass) ORDER BY indexrelid")) + def holder(self,db): + q=subprocess.Popen(p.COMPOSE+['exec','-T','postgres','psql','-X','-U','postgres','-d',db,'-Atq'],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True) + q.stdin.write("BEGIN; INSERT INTO comments VALUES(1,1); SELECT 'held';\n");q.stdin.flush();self.assertEqual(q.stdout.readline().strip(),'held');return q + def release(self,q): + q.stdin.write('COMMIT;\n\\q\n');q.stdin.flush();q.communicate(timeout=15) + def wait_index(self,db): + deadline=time.monotonic()+15 + while time.monotonic() /dev/null SERVICE_FROM_FILE=$(cat /etc/app-info/service_type.txt) echo "DEBUG: Service type read from /etc/app-info/service_type.txt: [$SERVICE_FROM_FILE]" +# Apply the release schema before stopping/replacing any healthy service. +# All roles use the same database lock; a second box waits, then applies nothing. +# Build the client with this checkout. No manual image staging or local Rust needed. +sudo docker build -t polis-migrate:deploy -f queue-rs/polis-migrate/Dockerfile . +sudo docker run --rm --network host --env-file .env polis-migrate:deploy apply + # Original Docker cleanup/start logic echo "Stopping and removing existing Docker containers..." sudo /usr/local/bin/docker-compose down || true diff --git a/scripts/application_stop.sh b/scripts/application_stop.sh index a16c56e718..a704ca08ca 100644 --- a/scripts/application_stop.sh +++ b/scripts/application_stop.sh @@ -1,83 +1,6 @@ #!/bin/bash -# This script runs during the ApplicationStop lifecycle event in CodeDeploy. -# It stops the relevant Docker containers based on the instance's role. - -set -e # Exit immediately if a command exits with a non-zero status. -set -x # Print commands and their arguments as they are executed. - -echo "Executing ApplicationStop hook..." - -# --- Configuration --- -# Directory where the docker-compose.yml file for the *current* deployment resides -# Adjust this path if your deployment process places files elsewhere -DEPLOY_DIR="/opt/polis/polis" -# File indicating the role of this instance (created by UserData/AfterInstall) -SERVICE_TYPE_FILE="/etc/app-info/service_type.txt" - -# --- Determine Service Type --- -if [ -f "$SERVICE_TYPE_FILE" ]; then - SERVICE_TYPE=$(cat "$SERVICE_TYPE_FILE") - echo "Detected service type: $SERVICE_TYPE" -else - echo "Warning: Service type file not found at $SERVICE_TYPE_FILE. Assuming nothing specific needs to be stopped by this script." - # Exit cleanly as we don't know what to stop, or maybe the instance role changed. - # CodeDeploy will likely proceed, and the AfterInstall script handles cleanup anyway. - exit 0 -fi - -# --- Stop Services based on Type --- - -# Check if the deployment directory exists (where docker-compose.yml should be) -if [ -d "$DEPLOY_DIR" ]; then - cd "$DEPLOY_DIR" - echo "Changed directory to $DEPLOY_DIR" - - # Check if docker-compose command exists - if ! command -v /usr/local/bin/docker-compose &> /dev/null; then - echo "Error: docker-compose command not found at /usr/local/bin/docker-compose. Cannot stop services." - # Exit with error because compose is expected if the directory exists and type isn't ollama - if [ "$SERVICE_TYPE" != "ollama" ]; then - exit 1 - fi - fi - - if [ "$SERVICE_TYPE" == "server" ]; then - echo "Stopping server-related services (server, nginx-proxy, file-server, client-participation-alpha)..." - # Stop services related to the 'server' type instance (as started in AfterInstall) - /usr/local/bin/docker-compose stop server nginx-proxy file-server client-participation-alpha || echo "Warning: Failed to stop server component(s), might already be stopped." - # Optional: Use 'down' if you want to remove networks etc. during stop, but 'stop' is usually sufficient here. - # /usr/local/bin/docker-compose down --remove-orphans server nginx-proxy file-server || echo "Warning..." - - elif [ "$SERVICE_TYPE" == "math" ]; then - # The Clojure math service is retired; AfterInstall starts nothing on a - # `math` box. Stop a container left by an earlier revision, if any. - echo "Service type 'math' is retired; stopping any leftover math container..." - docker stop polis-math-1 2>/dev/null || echo "No math container running." - - elif [ "$SERVICE_TYPE" == "delphi" ]; then - echo "Stopping delphi service..." - /usr/local/bin/docker-compose stop delphi || echo "Warning: Failed to stop delphi service, might already be stopped." - - elif [ "$SERVICE_TYPE" == "delphi-large" ]; then - # A queue worker box runs no compose service: its worker is the polis-jobs - # daemon (polis-jobs.service). Nothing is stopped here: stopping drains the - # running job for up to 900 s, longer than this hook may take, so - # AfterInstall restarts the unit instead (queued; the drain runs in systemd). - echo "Service type 'delphi-large': no compose service to stop (polis-jobs.service is restarted by AfterInstall)" - - elif [ "$SERVICE_TYPE" == "delphi-worker" ]; then - # The same for a Delphi queue worker box. - echo "Service type 'delphi-worker': no compose service to stop (polis-jobs.service is restarted by AfterInstall)" - - else - echo "Warning: Unknown service type '$SERVICE_TYPE' found in $SERVICE_TYPE_FILE. No specific services stopped." - # Avoid running a generic 'down' as it might affect unrelated containers if any exist - fi - -else - echo "Warning: Deployment directory $DEPLOY_DIR not found. Assuming no services need stopping." - # Exit cleanly if the directory isn't there, as nothing from this app could be running - exit 0 -fi - -echo "ApplicationStop hook finished successfully for service type: $SERVICE_TYPE." \ No newline at end of file +set -eu +# Keep the healthy revision running until AfterInstall has migrated successfully. +# CodeDeploy executes ApplicationStop from the PREVIOUS successful revision; see +# docs/upgrading.md for the one-time transition from the old stopping hook. +echo "Service shutdown is deferred until migrations succeed in AfterInstall." diff --git a/scripts/before_install.sh b/scripts/before_install.sh index 35e506accb..a704ca08ca 100644 --- a/scripts/before_install.sh +++ b/scripts/before_install.sh @@ -1,19 +1,6 @@ #!/bin/bash -set -e -set -x - -# Stop any existing Docker containers (if needed) -# Docker's `name` filter is an unanchored regex (a substring match), so -# `name=polis-math` also matches the Delphi box's `polis-math-python-1` and the -# hook then tried to stop a `polis-math-1` that only exists on the math box. -# Anchor every filter to the exact container name; the optional leading `/` -# covers daemons that match against the stored `/name` form. -if docker ps -q --filter "name=^/?polis-server-1$" | grep -q .; then - docker stop polis-server-1 -fi -if docker ps -q --filter "name=^/?polis-math-1$" | grep -q .; then - docker stop polis-math-1 -fi -if docker ps -q --filter "name=^/?polis-delphi-1$" | grep -q .; then - docker stop polis-delphi-1 -fi +set -eu +# Keep the healthy revision running until AfterInstall has migrated successfully. +# CodeDeploy executes ApplicationStop from the PREVIOUS successful revision; see +# docs/upgrading.md for the one-time transition from the old stopping hook. +echo "Service shutdown is deferred until migrations succeed in AfterInstall." diff --git a/server/Dockerfile b/server/Dockerfile index 0ef7bd96d4..b23a8774c1 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -18,7 +18,14 @@ ARG NODE_ENV WORKDIR /app # Install Python and necessary tools (including pip) -RUN apk add --no-cache python3 py3-pip +RUN apk add --no-cache python3 py3-pip ca-certificates + +# Verified startup migration checks use the same public RDS CA pin as the +# runner image. Deployments with another private CA can override this env. +RUN mkdir -p /etc/polis \ + && wget -O /etc/polis/rds-ca.pem https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem \ + && echo 'fe45bbebf92ad3e27a583bbb2ddd1553c521ed4d49af5514dc0a40372ea5395c /etc/polis/rds-ca.pem' | sha256sum -c - +ENV NODE_EXTRA_CA_CERTS=/etc/polis/rds-ca.pem # This package is needed at runtime RUN apk add libpq-dev diff --git a/server/Dockerfile-db b/server/Dockerfile-db index 3ba7e8a75c..3d1e57bda5 100644 --- a/server/Dockerfile-db +++ b/server/Dockerfile-db @@ -1,11 +1,12 @@ -# Example Usage: -# docker build -t polis-db:local -f Dockerfile-db . -# docker network create polis-net -# docker run --rm --name polis-db --network polis-net -p 5432:5432 \ -# -e POSTGRES_DB=polis-dev -e POSTGRES_PASSWORD=postgres polis-db:local - +# Build with --build-context queue-rs=queue-rs (Compose supplies it). +# New volumes use the same runner/history as every later deploy. +FROM docker.io/library/rust:1.98.1-alpine AS migrate-build +RUN apk add --no-cache musl-dev openssl-dev openssl-libs-static pkgconfig +WORKDIR /src +COPY --from=queue-rs . . +RUN cargo build --release --locked -p polis-migrate FROM postgres:17-alpine -# Used when no existing database on postgres volume, including first initialization. -# See: docs/deployment.md#database-migrations -# See: https://github.com/docker-library/docs/blob/master/postgres/README.md#initialization-scripts -COPY ./postgres/migrations/*.sql /docker-entrypoint-initdb.d/ +RUN apk add --no-cache libssl3 ca-certificates +COPY --from=migrate-build /src/target/release/polis-migrate /usr/local/bin/polis-migrate +COPY ./postgres/migrations /migrations +COPY ./postgres/init-migrations.sh /docker-entrypoint-initdb.d/00-migrations.sh diff --git a/server/README.md b/server/README.md index f800d84f87..78445edd8c 100644 --- a/server/README.md +++ b/server/README.md @@ -52,36 +52,19 @@ root folder of the polis project. To run everything but the API server in this f `docker compose -f docker-compose.yml -f docker-compose.dev.yml --profile postgres up math postgres file-server ses-local`. In this case the polis-dev database should be accessible at the default DATABASE_URL seen in server/example.env. -3\. Connect to the new database then run the migrations in its shell. You can skip this step if you built the -database with docker compose. - -```psql -\connect polis -\i postgres/migrations/000000_initial.sql -\i postgres/migrations/000001_update_pwreset_table.sql -\i postgres/migrations/000002_add_xid_constraint.sql -\i postgres/migrations/000003_add_origin_permanent_cookie_columns.sql -\i postgres/migrations/000004_drop_waitinglist_table.sql -\i postgres/migrations/000005_drop_slack_stripe_canvas.sql -\i postgres/migrations/000006_update_votes_rule.sql -\i postgres/migrations/000007_drop_geolocation_fields.sql -\i postgres/migrations/000008_add_comment_priority.sql -\i postgres/migrations/000009_add_uuid_to_zinvites.sql -``` - -You can also separately run `psql -d polis -f postgres/migrations/000000_initial.sql` and -`psql -d polis -f postgres/migrations/000001_update_pwreset_table.sql` etc. from the shell. - -Alternatively, you can use the provided migration script to run all migrations in sequence: +3\. Set `DATABASE_URL` in the environment and run the migration runner from the +repository root. A fresh Compose database does this during initialization. ```sh -# Using DATABASE_URL from environment -./bin/run-migrations.sh - -# Or providing the URL as argument -./bin/run-migrations.sh "postgres://username:password@localhost:5432/polis" +cargo build --locked --release --manifest-path queue-rs/Cargo.toml -p polis-migrate +queue-rs/target/release/polis-migrate apply +queue-rs/target/release/polis-migrate check ``` +Existing databases need catalog-checked reconciliation once. See +[database migrations](../docs/migrations.md) for the commands and release holds. +Do not replay the initial SQL or pass credentials as command-line arguments. + 4\. Update database connection settings in `.env`. Replace the username, password, and database_name in the DATABASE_URL ```sh diff --git a/server/__tests__/integration/conversation-stats.test.ts b/server/__tests__/integration/conversation-stats.test.ts index b6253b431a..52ee7399ad 100644 --- a/server/__tests__/integration/conversation-stats.test.ts +++ b/server/__tests__/integration/conversation-stats.test.ts @@ -9,6 +9,7 @@ import { submitVote, } from "../setup/api-test-helpers"; import { getPooledTestUser } from "../setup/test-user-helpers"; +import pg from "../../src/db/pg-query"; interface ConversationStats { voteTimes: number[]; @@ -85,8 +86,14 @@ describe("Conversation Stats API", () => { }); test("GET /api/v3/conversationStats - should accept until parameter", async () => { - // Get current time in milliseconds - const currentTimeMs = Date.now(); + // Use the same clock as the stored timestamps. A Docker VM can be ahead + // of the host, so Date.now() immediately after the vote may exclude it. + // The route uses a strict < cutoff; advance past the current DB millisecond. + const rows = (await pg.queryP_readOnly( + "SELECT now_as_millis() AS now", + [] + )) as Array<{ now: string }>; + const currentTimeMs = Number(rows[0].now) + 1; const response: Response = await agent.get( `/api/v3/conversationStats?conversation_id=${conversationId}&until=${currentTimeMs}` diff --git a/server/__tests__/unit/binBoundaries.test.ts b/server/__tests__/unit/binBoundaries.test.ts index 84fda1ae60..051908c71e 100644 --- a/server/__tests__/unit/binBoundaries.test.ts +++ b/server/__tests__/unit/binBoundaries.test.ts @@ -13,9 +13,10 @@ beforeEach(() => { afterEach(() => fs.rmSync(scratch, { recursive: true, force: true })); const publicEnvUrl = "postgresql://public:public-fixture@public.invalid/public"; -const publicArgUrl = "postgresql://public:other-fixture@other.invalid/public"; - -function migrationFixture(files: Record = {}) { +// The shell entrypoint delegates policy and SQL execution to polis-migrate. +// Its boundary keeps the DSN in the environment and preserves runner failures; +// real SQL/history/startup refusal is covered in polis-migrate/tests/prove.py. +function migrationFixture() { const script = path.join(scratch, "server/bin/run-migrations.sh"); const migrations = path.join(scratch, "server/postgres/migrations"); const fakeBin = path.join(scratch, "fake-bin"); @@ -23,129 +24,84 @@ function migrationFixture(files: Record = {}) { fs.mkdirSync(path.dirname(script), { recursive: true }); fs.mkdirSync(migrations, { recursive: true }); fs.mkdirSync(fakeBin); - // Byte-identical script relocated so it can see only public SQL fixtures. fs.copyFileSync(path.join(serverRoot, "bin/run-migrations.sh"), script); - for (const [name, contents] of Object.entries(files)) { - const file = path.join(migrations, name); - fs.mkdirSync(path.dirname(file), { recursive: true }); - fs.writeFileSync(file, contents); - } - const fakePsql = path.join(fakeBin, "psql"); - fs.writeFileSync( - fakePsql, - `#!${process.execPath} + const fakeRunner = path.join(fakeBin, "polis-migrate"); + fs.writeFileSync(fakeRunner, `#!${process.execPath} const fs = require('fs'); -const path = require('path'); -const [dsn, flag, file, extra] = process.argv.slice(2); -const allowed = ${JSON.stringify([publicEnvUrl, publicArgUrl])}; -if (!allowed.includes(dsn) || flag !== '-f' || extra || !file || - path.dirname(file) !== process.env.PUBLIC_MIGRATIONS) { - throw new Error('not an admitted public invocation'); -} -const contents = fs.readFileSync(file, 'utf8'); -if (!contents.startsWith('-- public fixture')) throw new Error('not public fixture SQL'); fs.appendFileSync(process.env.PUBLIC_TRACE, JSON.stringify({ - dsn, file: path.basename(file), password: process.env.PGPASSWORD, contents + args: process.argv.slice(2), dsn: process.env.DATABASE_URL }) + '\\n'); -process.exit(path.basename(file) === process.env.PUBLIC_FAIL ? 17 : 0); -` - ); - fs.chmodSync(fakePsql, 0o700); +process.exit(Number(process.env.PUBLIC_EXIT)); +`); + fs.chmodSync(fakeRunner, 0o700); return { - run(args: string[] = [], envUrl?: string, fail = "") { - const result = spawnSync("/bin/bash", [script, ...args], { + migrations, + fakeRunner, + run(options: { args?: string[]; code?: number; binary?: string; envUrl?: string } = {}) { + const result = spawnSync("/bin/bash", [script, ...(options.args || [])], { cwd: scratch, encoding: "utf8", timeout: 5000, env: { PATH: `${fakeBin}:/usr/bin:/bin`, - PUBLIC_MIGRATIONS: migrations, PUBLIC_TRACE: trace, - PUBLIC_FAIL: fail, - ...(envUrl === undefined ? {} : { DATABASE_URL: envUrl }), + PUBLIC_EXIT: String(options.code || 0), + DATABASE_URL: options.envUrl ?? publicEnvUrl, + ...(options.binary ? { POLIS_MIGRATE_BIN: options.binary } : {}), }, }); if (result.error) throw result.error; const calls = fs.existsSync(trace) - ? fs - .readFileSync(trace, "utf8") - .trim() - .split("\n") - .filter(Boolean) - .map((line) => JSON.parse(line)) + ? fs.readFileSync(trace, "utf8").trim().split("\n").map((line) => JSON.parse(line)) : []; return { ...result, calls }; }, }; } -const publicSql = "-- public fixture\nSELECT 'public value';\n"; - -test("migration runner requires a URL before invoking any psql process", () => { - const result = migrationFixture({ "001.sql": publicSql }).run(); - expect(result.status).toBe(1); - expect(result.stdout).toContain("DATABASE_URL is not set"); - expect(result.stdout).not.toContain("All migrations completed successfully"); - expect(result.calls).toEqual([]); +test("migration wrapper delegates once with the checkout's absolute source directory", () => { + const fixture = migrationFixture(); + const result = fixture.run(); + expect(result.status).toBe(0); + expect(result.calls).toEqual([{ args: ["apply", "--dir", fixture.migrations], dsn: publicEnvUrl }]); }); -test("migration runner sorts top-level SQL and ignores nested SQL and other extensions", () => { - const result = migrationFixture({ - "010-last.sql": publicSql, - "002-first.sql": publicSql, - "nested/001-ignored.sql": publicSql, - "003-ignored.txt": "not SQL", - }).run([], publicEnvUrl); +test("migration wrapper keeps the DSN out of argv and output", () => { + const result = migrationFixture().run(); expect(result.status).toBe(0); - expect(result.calls.map((call) => call.file)).toEqual([ - "002-first.sql", - "010-last.sql", - ]); - for (const call of result.calls) { - expect(call.dsn).toBe(publicEnvUrl); - expect(call.password).toBe("public-fixture"); - expect(call.contents).toBe(publicSql); - } - expect(result.stdout).toContain("All migrations completed successfully!"); + expect(result.calls).toHaveLength(1); + expect(result.calls[0].args.join(" ")).not.toContain(publicEnvUrl); + expect(result.stdout + result.stderr).not.toContain(publicEnvUrl); }); -test("explicit migration URL overrides the environment and binds matching password", () => { - const result = migrationFixture({ "001.sql": publicSql }).run( - [publicArgUrl], - publicEnvUrl - ); +test("migration wrapper supports an explicitly installed binary path containing spaces", () => { + const fixture = migrationFixture(); + const binary = path.join(scratch, "installed runner"); + fs.copyFileSync(fixture.fakeRunner, binary); + const result = fixture.run({ binary }); expect(result.status).toBe(0); expect(result.calls).toHaveLength(1); - expect(result.calls[0].dsn).toBe(publicArgUrl); - expect(result.calls[0].password).toBe("other-fixture"); }); -test("empty explicit URL preserves the environment URL", () => { - const result = migrationFixture({ "001.sql": publicSql }).run( - [""], - publicEnvUrl - ); - expect(result.status).toBe(0); - expect(result.calls[0].dsn).toBe(publicEnvUrl); +test("migration wrapper preserves argument boundaries for runner validation", () => { + const fixture = migrationFixture(); + const result = fixture.run({ args: ["--unknown-option", "value with spaces"], code: 2 }); + expect(result.status).toBe(2); + expect(result.calls[0].args).toEqual(["apply", "--dir", fixture.migrations, "--unknown-option", "value with spaces"]); }); -test("psql failure stops later migrations and cannot print completion", () => { - const result = migrationFixture({ - "001.sql": publicSql, - "002.sql": publicSql, - "003.sql": publicSql, - }).run([], publicEnvUrl, "002.sql"); +test("migration wrapper propagates runner failure without a success message or retry", () => { + const result = migrationFixture().run({ code: 17 }); expect(result.status).toBe(17); - expect(result.calls.map((call) => call.file)).toEqual(["001.sql", "002.sql"]); - expect(result.stdout).not.toContain("Applying migration: 003.sql"); - expect(result.stdout).not.toContain("All migrations completed successfully"); + expect(result.calls).toHaveLength(1); + expect(result.stdout).toBe(""); }); -test("empty migration directory completes without inventing a database call", () => { - const result = migrationFixture().run([], publicEnvUrl); - expect(result.status).toBe(0); +test("missing runner refuses without falling back to untracked SQL", () => { + const result = migrationFixture().run({ binary: path.join(scratch, "absent") }); + expect(result.status).not.toBe(0); // Bash 3/macOS: 1; Bash 5/Linux: 127. expect(result.calls).toEqual([]); - expect(result.stdout).toContain("All migrations completed successfully!"); + expect(result.stdout).toBe(""); }); function resetBoundary(databaseUrl: unknown) { diff --git a/server/bin/build-migration-report.py b/server/bin/build-migration-report.py new file mode 100644 index 0000000000..59c571d174 --- /dev/null +++ b/server/bin/build-migration-report.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 +"""Build the first-deploy catalog report by inlining the adoption SQL helpers. + +No database connection. --check compares bytes without changing any files. +The generated script executes in a read-only transaction, with no temp objects. +""" +import argparse +import hashlib +from pathlib import Path +import re + +MIG = Path(__file__).resolve().parents[1] / "postgres/migrations" + + +def tokens(sql): + # All helper/adoption sources use ordinary quoted strings, comments and $$. + return re.findall(r"--[^\n]*|/\*[\s\S]*?\*/|'(?:''|[^'])*'|[A-Za-z_][A-Za-z_0-9]*|\s+|.", sql) + + +def substitute(sql, arguments): + return ''.join('(' + arguments[t] + ')' if t in arguments else t for t in tokens(sql)) + + +def helpers(source): + result = {} + pattern = r'CREATE FUNCTION pg_temp\.(\w+)\((.*?)\)\s*RETURNS boolean LANGUAGE sql AS \$\$(.*?)\$\$;' + for name, signature, body in re.findall(pattern, source, re.S): + params = [] + for part in signature.split(','): + bits = part.strip().split() + params.append((bits[0], bits[3] if len(bits) == 4 and bits[2] == 'DEFAULT' else None)) + result[name] = (params, body.strip().removesuffix(';')) + assert len(result) == 6, 'review changed helper source before regenerating' + return result + + +def inline(query, definitions): + ts = tokens(query) + out = [] + i = 0 + while i < len(ts): + if ts[i] != 'pg_temp': + out.append(ts[i]); i += 1; continue + assert ts[i+1] == '.' and ts[i+2] in definitions and ts[i+3] == '(' + name = ts[i+2] + i += 4 + depth = 1 + args = [''] + while depth: + token = ts[i]; i += 1 + if token == '(': + depth += 1 + elif token == ')': + depth -= 1 + if not depth: + break + if token == ',' and depth == 1: + args.append('') + else: + args[-1] += token + params, body = definitions[name] + assert len(args) <= len(params) + values = {} + for index, (param, default) in enumerate(params): + value = args[index].strip() if index < len(args) else default + assert value is not None + values[param] = value + out.append('(' + substitute(body, values) + ')') + return ''.join(out) + + +def build(): + source = (MIG/'adoption/helpers.sql').read_text() + definitions = helpers(source) + files = sorted((MIG/'adoption').glob('0*.sql')) + def manifest(path): + lines = [s for s in path.read_text().splitlines() if s and not s.startswith('#')] + assert len(lines) == len(set(lines)), 'duplicate manifest entry' + assert all(re.fullmatch(r'\d{6}_[A-Za-z0-9_]+\.sql', s) for s in lines) + return set(lines) + selected = manifest(MIG/'release.txt') + held = manifest(MIG/'held.txt') + pending = {'000019_create_polis_queue.sql', '000023_create_delphi_foundation.sql', + '000024_create_polis_queue_large_class.sql'} + assert selected == {p.name for p in files} | pending, 'report scope differs from release selection' + assert held == {'000021_create_polis_coordinator.sql'}, 'review changed hold policy' + numbered = sorted(MIG.glob('*.sql')) + assert all(re.fullmatch(r'\d{6}_[A-Za-z0-9_]+\.sql', p.name) for p in numbered), 'invalid numbered source name' + assert len({p.name[:6] for p in numbered}) == len(numbered), 'duplicate migration number' + assert {p.name for p in numbered} == selected | held, 'unclassified/missing numbered SQL' + assert all(p.is_file() and not p.is_symlink() for p in numbered), 'nonregular numbered source' + bindings = [MIG/'adoption/helpers.sql', MIG/'release.txt', MIG/'held.txt'] + files + numbered + header = ['-- GENERATED by server/bin/build-migration-report.py; do not edit.', + '-- First deployment only: no existing ledger or queue. Other states refuse.', + '-- This is a catalog forecast, not DDL success or deployment-health proof.'] + for path in bindings: + header.append('-- sha256 ' + hashlib.sha256(path.read_bytes()).hexdigest() + ' ' + str(path.relative_to(MIG))) + # Each original predicate remains a scalar subquery. SQL helpers are inlined, + # never installed, and role/catalog identity fields never leave the database. + rows = [] + for path in files: + rows.append("SELECT '"+path.name+"'::text AS migration, ("+inline(path.read_text().strip().removesuffix(';'), definitions)+") AS catalog_match") + checks = '\nUNION ALL\n'.join(rows) + return '\n'.join(header)+'''\nBEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY; +SET LOCAL search_path=pg_catalog,public; +SET LOCAL statement_timeout='30s'; +SET LOCAL lock_timeout='2s'; +WITH checks AS ( +'''+checks+''' +), scope AS ( + SELECT to_regclass('public.migrations') IS NULL + AND to_regclass('public.schema_migrations') IS NULL + AND NOT EXISTS (SELECT 1 FROM pg_class WHERE relnamespace='public'::regnamespace + AND (starts_with(relname,'polis_queue_') OR starts_with(relname,'delphi_'))) + AND NOT EXISTS (SELECT 1 FROM pg_proc WHERE pronamespace='public'::regnamespace + AND (starts_with(proname,'pq_') OR starts_with(proname,'pd_'))) AS initial_state +), authority AS ( + SELECT + has_database_privilege(current_user,current_database(),'TEMP') + AND has_schema_privilege(current_user,'public','USAGE') + AND has_schema_privilege(current_user,'public','CREATE') AS reconcile_authority, + EXISTS (SELECT 1 FROM pg_roles WHERE rolname=current_user AND (rolsuper OR rolcreaterole)) AS create_roles, + NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname IN ('polis_queue_owner','polis_queue_executor')) AS queue_roles_absent, + has_schema_privilege(current_user,'public','USAGE WITH GRANT OPTION') + AND has_schema_privilege(current_user,'public','CREATE WITH GRANT OPTION') AS schema_grants, + EXISTS (SELECT 1 FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace + WHERE n.nspname='public' AND c.relname='conversations' + AND has_table_privilege(current_user,c.oid,'SELECT WITH GRANT OPTION') + AND has_column_privilege(current_user,c.oid,'topic','UPDATE WITH GRANT OPTION') + AND has_column_privilege(current_user,c.oid,'zid','REFERENCES WITH GRANT OPTION')) AS table_grants +), readiness AS ( + SELECT initial_state AND (SELECT bool_and(catalog_match) FROM checks) + AND current_setting('server_version_num')::integer >= 170000 AS can_reconcile, + create_roles AND queue_roles_absent AND schema_grants AND table_grants AS can_provision, + initial_state, reconcile_authority, create_roles, queue_roles_absent, schema_grants, table_grants + FROM scope CROSS JOIN authority +), output AS ( + SELECT migration, catalog_match, + CASE WHEN NOT initial_state THEN 'REVIEW_EXISTING_LEDGER_OR_QUEUE' + WHEN NOT catalog_match THEN 'CATALOG_MISMATCH' + WHEN NOT can_reconcile THEN 'BLOCKED_BY_OTHER_CATALOG_CHECK' + WHEN NOT reconcile_authority THEN 'REVIEW_MIGRATION_SESSION_AUTHORITY' + ELSE 'WOULD_ADOPT' END AS outcome + FROM checks CROSS JOIN readiness + UNION ALL + SELECT migration, NULL::boolean, + CASE WHEN NOT can_reconcile THEN 'BLOCKED_RECONCILIATION' + WHEN NOT reconcile_authority OR NOT can_provision THEN 'REVIEW_MIGRATION_SESSION_AUTHORITY' + ELSE 'WOULD_APPLY' END + FROM (VALUES ('000019_create_polis_queue.sql'),('000023_create_delphi_foundation.sql'), + ('000024_create_polis_queue_large_class.sql')) p(migration) CROSS JOIN readiness + UNION ALL + SELECT migration, NULL::boolean, 'OUTSIDE_RELEASE' + FROM (VALUES ('000020'),('000021'),('000025'),('000026')) p(migration) +) +SELECT migration, catalog_match, outcome FROM output ORDER BY migration; +ROLLBACK; +''' + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--check', action='store_true') + args = parser.parse_args() + output = MIG/'report/first-deploy.sql' + text = build() + if args.check: + assert output.read_text() == text, 'report is stale; regenerate and review' + print('PASS: report matches all source predicates and release metadata') + else: + output.parent.mkdir(exist_ok=True) + output.write_text(text) + print(output) + + +if __name__ == '__main__': + main() diff --git a/server/bin/run-migrations.sh b/server/bin/run-migrations.sh index 911ebe6680..34de794ca6 100755 --- a/server/bin/run-migrations.sh +++ b/server/bin/run-migrations.sh @@ -1,28 +1,5 @@ -#!/bin/bash -set -e - -# Use DATABASE_URL from environment or from argument if provided -if [ -n "$1" ]; then - DATABASE_URL="$1" -fi - -# Check if DATABASE_URL is set -if [ -z "$DATABASE_URL" ]; then - echo "Error: DATABASE_URL is not set. Please provide it as an environment variable or argument." - echo "Usage: $0 [DATABASE_URL]" - exit 1 -fi - -# Directory containing migration files -MIGRATIONS_DIR="$(dirname "$(dirname "$0")")/postgres/migrations" - -echo "Running migrations from $MIGRATIONS_DIR" - -# Get all migration files sorted by name (only from top-level directory) -for migration in $(find "$MIGRATIONS_DIR" -maxdepth 1 -name "*.sql" | sort); do - echo "Applying migration: $(basename "$migration")" - PGPASSWORD=$(echo "$DATABASE_URL" | sed -E 's/.*:([^:]+)@.*/\1/') \ - psql "$DATABASE_URL" -f "$migration" -done - -echo "All migrations completed successfully!" +#!/usr/bin/env bash +set -euo pipefail +# DATABASE_URL stays in the environment, never an argv or a traced shell command. +root=$(cd "$(dirname "$0")/../.." && pwd) +exec "${POLIS_MIGRATE_BIN:-polis-migrate}" apply --dir "$root/server/postgres/migrations" "$@" diff --git a/server/index.ts b/server/index.ts index dfba6599a7..35c5e36ee8 100644 --- a/server/index.ts +++ b/server/index.ts @@ -1,30 +1,25 @@ -/** - * Server entry point - * This file is responsible for starting the server after the app is configured - */ -import app from "./app"; -import Config from "./src/config"; -import { startNotificationLoop } from "./src/routes/notify"; -import logger from "./src/utils/logger"; +/** Check the release's migrations before importing the app or starting work. */ +import "dotenv/config"; +import { checkMigrations } from "./src/db/migrations.cjs"; -if (Config.nodeEnv === "production") { - // eslint-disable-next-line @typescript-eslint/no-unused-vars, @typescript-eslint/no-var-requires - const tracer = require("dd-trace").init(); -} - -/** - * Start the server on the configured port or a provided port - * @param {number} [port=Config.serverPort] - The port to listen on - * @returns {Object} The server instance - */ -function startServer(port = Config.serverPort) { - const server = app.listen(port); - logger.info(`Server started on port ${port}`); +async function startServer(port?: number) { + await checkMigrations(); + const { default: Config } = await import("./src/config"); + if (Config.nodeEnv === "production") { + // eslint-disable-next-line @typescript-eslint/no-var-requires + require("dd-trace").init(); + } + const { default: app } = await import("./app"); + const { startNotificationLoop } = await import("./src/routes/notify"); + const { default: logger } = await import("./src/utils/logger"); + startNotificationLoop(); + const server = app.listen(port ?? Config.serverPort); + logger.info(`Server started on port ${port ?? Config.serverPort}`); return server; } -startNotificationLoop(); -startServer(); - +startServer().catch((error) => { + process.stderr.write(`Server startup refused: ${error.message}\n`); + process.exit(1); +}); export { startServer }; -export default app; diff --git a/server/postgres/bin/apply-migration.sh b/server/postgres/bin/apply-migration.sh index e6ec532005..9c3c2e05f8 100755 --- a/server/postgres/bin/apply-migration.sh +++ b/server/postgres/bin/apply-migration.sh @@ -1,5 +1,8 @@ #!/usr/bin/env bash # +# HISTORICAL REHEARSAL HELPER ONLY. Deployments must use polis-migrate apply +# (docs/migrations.md). This script does not record applied migration history. +# # apply-migration.sh: the checked apply wrapper for the queue migrations # 000019 (polis-queue/1), 000023 (polis-queue/2, the Delphi job table) and # 000024 (polis-queue/3, the large worker class). diff --git a/server/postgres/init-migrations.sh b/server/postgres/init-migrations.sh new file mode 100644 index 0000000000..b70b0545ea --- /dev/null +++ b/server/postgres/init-migrations.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -eu +# Official postgres entrypoint sources executable/nonexecutable shell hooks only +# during fresh initialization; its temporary server listens on this local socket. +export DATABASE_URL="host=/var/run/postgresql user=$POSTGRES_USER dbname=$POSTGRES_DB sslmode=disable" +export POLIS_MIGRATIONS_DIR=/migrations +polis-migrate apply diff --git a/server/postgres/migrations/adoption/000000_initial.sql b/server/postgres/migrations/adoption/000000_initial.sql new file mode 100644 index 0000000000..3de3898aa1 --- /dev/null +++ b/server/postgres/migrations/adoption/000000_initial.sql @@ -0,0 +1,357 @@ +-- Named legacy variants are documented in docs/migration-legacy-contract.md. +-- Contributor agreement tables are outside this release adoption boundary. +-- Catalog postconditions only; this file never replays migration DDL. +SELECT NOT EXISTS (SELECT 1 FROM (VALUES + ('users','uid','integer'), + ('users','hname','character varying(746)'), + ('users','created','bigint'), + ('users','username','character varying(128)'), + ('users','email','character varying(256)'), + ('users','is_owner','boolean'), + ('users','zinvite','character varying(300)'), + ('users','oinvite','character varying(300)'), + ('users','tut','smallint'), + ('users','site_id','character varying(256)'), + ('users','site_owner','boolean'), + ('site_domain_whitelist','site_id','character varying(256)'), + ('site_domain_whitelist','domain_whitelist','character varying(999)'), + ('site_domain_whitelist','domain_whitelist_override_key','character varying(999)'), + ('site_domain_whitelist','modified','bigint'), + ('site_domain_whitelist','created','bigint'), + ('metrics','uid','integer'), + ('metrics','type','integer'), + ('metrics','dur','integer'), + ('metrics','hashedpc','integer'), + ('metrics','created','bigint'), + ('auth_tokens','token','character varying(32)'), + ('auth_tokens','uid','integer'), + ('auth_tokens','created','bigint'), + ('jianiuevyew','uid','integer'), + ('jianiuevyew','pwhash','character varying(128)'), + ('apikeysndvweifu','uid','integer'), + ('apikeysndvweifu','apikey','character varying(32)'), + ('apikeysndvweifu','created','bigint'), + ('courses','course_id','integer'), + ('courses','topic','character varying(1000)'), + ('courses','description','character varying(1000)'), + ('courses','owner','integer'), + ('courses','course_invite','character varying(32)'), + ('courses','created','bigint'), + ('conversations','zid','integer'), + ('conversations','topic','character varying(1000)'), + ('conversations','description','character varying(50000)'), + ('conversations','link_url','character varying(9999)'), + ('conversations','parent_url','character varying(9999)'), + ('conversations','upvotes','integer'), + ('conversations','participant_count','integer'), + ('conversations','is_anon','boolean'), + ('conversations','is_active','boolean'), + ('conversations','is_draft','boolean'), + ('conversations','is_public','boolean'), + ('conversations','is_data_open','boolean'), + ('conversations','profanity_filter','boolean'), + ('conversations','spam_filter','boolean'), + ('conversations','strict_moderation','boolean'), + ('conversations','prioritize_seed','boolean'), + ('conversations','vis_type','integer'), + ('conversations','write_type','integer'), + ('conversations','help_type','integer'), + ('conversations','write_hint_type','integer'), + ('conversations','style_btn','character varying(500)'), + ('conversations','socialbtn_type','integer'), + ('conversations','subscribe_type','integer'), + ('conversations','bgcolor','character varying(20)'), + ('conversations','help_bgcolor','character varying(20)'), + ('conversations','help_color','character varying(20)'), + ('conversations','email_domain','character varying(200)'), + ('conversations','use_xid_whitelist','boolean'), + ('conversations','owner','integer'), + ('conversations','org_id','integer'), + ('conversations','context','character varying(1000)'), + ('conversations','course_id','integer'), + ('conversations','owner_sees_participation_stats','boolean'), + ('conversations','auth_needed_to_vote','boolean'), + ('conversations','auth_needed_to_write','boolean'), + ('conversations','auth_opt_fb','boolean'), + ('conversations','auth_opt_tw','boolean'), + ('conversations','auth_opt_allow_3rdparty','boolean'), + ('conversations','modified','bigint'), + ('conversations','created','bigint'), + ('participant_metadata_questions','pmqid','integer'), + ('participant_metadata_questions','zid','integer'), + ('participant_metadata_questions','key','character varying(999)'), + ('participant_metadata_questions','alive','boolean'), + ('participant_metadata_questions','created','bigint'), + ('participant_metadata_answers','pmaid','integer'), + ('participant_metadata_answers','pmqid','integer'), + ('participant_metadata_answers','zid','integer'), + ('participant_metadata_answers','value','character varying(999)'), + ('participant_metadata_answers','alive','boolean'), + ('participant_metadata_answers','created','bigint'), + ('contexts','context_id','integer'), + ('contexts','name','character varying(300)'), + ('contexts','creator','integer'), + ('contexts','is_public','boolean'), + ('contexts','created','bigint'), + ('inviters','inviter_uid','integer'), + ('inviters','invited_email','character varying(999)'), + ('inviters','created','bigint'), + ('upvotes','uid','integer'), + ('upvotes','zid','integer'), + ('oinvites','oinvite','character varying(300)'), + ('oinvites','note','character varying(999)'), + ('oinvites','created','bigint'), + ('einvites','einvite','character varying(100)'), + ('einvites','email','character varying(999)'), + ('einvites','created','bigint'), + ('email_validations','email','character varying(999)'), + ('email_validations','created','bigint'), + ('zinvites','zid','integer'), + ('zinvites','zinvite','character varying(300)'), + ('zinvites','created','bigint'), + ('beta','name','character varying(999)'), + ('beta','email','character varying(200)'), + ('beta','organization','character varying(200)'), + ('beta','created','bigint'), + ('participants','pid','integer'), + ('participants','uid','integer'), + ('participants','zid','integer'), + ('participants','vote_count','integer'), + ('participants','last_interaction','bigint'), + ('participants','subscribed','integer'), + ('participants','last_notified','bigint'), + ('participants','nsli','smallint'), + ('participants','mod','integer'), + ('participants','created','bigint'), + ('participants_extended','uid','integer'), + ('participants_extended','zid','integer'), + ('participants_extended','referrer','character varying(9999)'), + ('participants_extended','parent_url','character varying(9999)'), + ('participants_extended','created','bigint'), + ('participants_extended','modified','bigint'), + ('participants_extended','subscribe_email','character varying(256)'), + ('participants_extended','show_translation_activated','boolean'), + ('participant_locations','zid','integer'), + ('participant_locations','uid','integer'), + ('participant_locations','pid','integer'), + ('participant_locations','lat','double precision'), + ('participant_locations','lng','double precision'), + ('participant_locations','created','bigint'), + ('participant_locations','source','integer'), + ('xids','uid','integer'), + ('xids','owner','integer'), + ('xids','xid','text'), + ('xids','x_profile_image_url','character varying(3000)'), + ('xids','x_name','character varying(746)'), + ('xids','x_email','character varying(256)'), + ('xids','created','bigint'), + ('xids','modified','bigint'), + ('xid_whitelist','owner','integer'), + ('xid_whitelist','xid','text'), + ('xid_whitelist','created','bigint'), + ('notification_tasks','zid','integer'), + ('notification_tasks','modified','bigint'), + ('participant_metadata_choices','zid','integer'), + ('participant_metadata_choices','pid','integer'), + ('participant_metadata_choices','pmqid','integer'), + ('participant_metadata_choices','pmaid','integer'), + ('participant_metadata_choices','alive','boolean'), + ('participant_metadata_choices','created','bigint'), + ('twitter_users','uid','integer'), + ('twitter_users','twitter_user_id','bigint'), + ('twitter_users','screen_name','character varying(999)'), + ('twitter_users','name','character varying(9999)'), + ('twitter_users','followers_count','integer'), + ('twitter_users','friends_count','integer'), + ('twitter_users','verified','boolean'), + ('twitter_users','profile_image_url_https','character varying(9999)'), + ('twitter_users','location','character varying(9999)'), + ('twitter_users','response','json'), + ('twitter_users','modified','bigint'), + ('twitter_users','created','bigint'), + ('facebook_users','uid','integer'), + ('facebook_users','fb_user_id','text'), + ('facebook_users','fb_name','character varying(9999)'), + ('facebook_users','fb_link','character varying(9999)'), + ('facebook_users','fb_public_profile','text'), + ('facebook_users','fb_login_status','text'), + ('facebook_users','fb_auth_response','text'), + ('facebook_users','fb_access_token','text'), + ('facebook_users','fb_granted_scopes','text'), + ('facebook_users','fb_location_id','character varying(100)'), + ('facebook_users','location','character varying(9999)'), + ('facebook_users','response','text'), + ('facebook_users','fb_friends_response','text'), + ('facebook_users','created','bigint'), + ('facebook_users','modified','bigint'), + ('social_settings','uid','integer'), + ('social_settings','polis_pic','character varying(3000)'), + ('facebook_friends','uid','integer'), + ('facebook_friends','friend','integer'), + ('suzinvites','owner','integer'), + ('suzinvites','zid','integer'), + ('suzinvites','created','bigint'), + ('suzinvites','suzinvite','character varying(32)'), + ('comments','tid','integer'), + ('comments','zid','integer'), + ('comments','pid','integer'), + ('comments','uid','integer'), + ('comments','created','bigint'), + ('comments','modified','bigint'), + ('comments','txt','character varying(1000)'), + ('comments','velocity','real'), + ('comments','mod','integer'), + ('comments','lang','character varying(10)'), + ('comments','lang_confidence','real'), + ('comments','active','boolean'), + ('comments','is_meta','boolean'), + ('comments','tweet_id','bigint'), + ('comments','quote_src_url','character varying(1000)'), + ('comments','anon','boolean'), + ('comments','is_seed','boolean'), + ('comment_translations','zid','integer'), + ('comment_translations','tid','integer'), + ('comment_translations','src','integer'), + ('comment_translations','txt','character varying(9999)'), + ('comment_translations','lang','character varying(10)'), + ('comment_translations','created','bigint'), + ('comment_translations','modified','bigint'), + ('conversation_translations','zid','integer'), + ('conversation_translations','src','integer'), + ('conversation_translations','topic','character varying(9999)'), + ('conversation_translations','description','character varying(9999)'), + ('conversation_translations','lang','character varying(10)'), + ('conversation_translations','created','bigint'), + ('conversation_translations','modified','bigint'), + ('reports','rid','bigint'), + ('reports','report_id','character varying(300)'), + ('reports','zid','integer'), + ('reports','created','bigint'), + ('reports','modified','bigint'), + ('reports','report_name','character varying(999)'), + ('reports','label_x_neg','character varying(999)'), + ('reports','label_x_pos','character varying(999)'), + ('reports','label_y_neg','character varying(999)'), + ('reports','label_y_pos','character varying(999)'), + ('reports','label_group_0','character varying(999)'), + ('reports','label_group_1','character varying(999)'), + ('reports','label_group_2','character varying(999)'), + ('reports','label_group_3','character varying(999)'), + ('reports','label_group_4','character varying(999)'), + ('reports','label_group_5','character varying(999)'), + ('reports','label_group_6','character varying(999)'), + ('reports','label_group_7','character varying(999)'), + ('reports','label_group_8','character varying(999)'), + ('reports','label_group_9','character varying(999)'), + ('report_comment_selections','zid','integer'), + ('report_comment_selections','rid','bigint'), + ('report_comment_selections','tid','integer'), + ('report_comment_selections','selection','smallint'), + ('report_comment_selections','modified','bigint'), + ('worker_tasks','created','bigint'), + ('worker_tasks','math_env','character varying(999)'), + ('worker_tasks','attempts','smallint'), + ('worker_tasks','task_data','jsonb'), + ('worker_tasks','task_type','text'), + ('worker_tasks','task_bucket','bigint'), + ('worker_tasks','finished_time','bigint'), + ('math_ticks','zid','integer'), + ('math_ticks','math_tick','bigint'), + ('math_ticks','math_env','character varying(999)'), + ('math_ticks','modified','bigint'), + ('math_main','zid','integer'), + ('math_main','math_env','character varying(999)'), + ('math_main','data','jsonb'), + ('math_main','last_vote_timestamp','bigint'), + ('math_main','caching_tick','bigint'), + ('math_main','math_tick','bigint'), + ('math_main','modified','bigint'), + ('math_profile','zid','integer'), + ('math_profile','math_env','character varying(999)'), + ('math_profile','data','jsonb'), + ('math_profile','modified','bigint'), + ('math_ptptstats','zid','integer'), + ('math_ptptstats','math_env','character varying(999)'), + ('math_ptptstats','math_tick','bigint'), + ('math_ptptstats','data','jsonb'), + ('math_ptptstats','modified','bigint'), + ('math_cache','zid','integer'), + ('math_cache','math_env','character varying(999)'), + ('math_cache','data','jsonb'), + ('math_cache','modified','bigint'), + ('math_bidtopid','zid','integer'), + ('math_bidtopid','math_env','character varying(999)'), + ('math_bidtopid','math_tick','bigint'), + ('math_bidtopid','data','jsonb'), + ('math_bidtopid','modified','bigint'), + ('math_exportstatus','zid','integer'), + ('math_exportstatus','math_env','character varying(999)'), + ('math_exportstatus','filename','character varying(9999)'), + ('math_exportstatus','data','jsonb'), + ('math_exportstatus','modified','bigint'), + ('math_report_correlationmatrix','rid','bigint'), + ('math_report_correlationmatrix','math_env','character varying(999)'), + ('math_report_correlationmatrix','data','jsonb'), + ('math_report_correlationmatrix','math_tick','bigint'), + ('math_report_correlationmatrix','modified','bigint'), + ('votes','zid','integer'), + ('votes','pid','integer'), + ('votes','tid','integer'), + ('votes','vote','smallint'), + ('votes','weight_x_32767','smallint'), + ('votes','created','bigint'), + ('votes_latest_unique','zid','integer'), + ('votes_latest_unique','pid','integer'), + ('votes_latest_unique','tid','integer'), + ('votes_latest_unique','vote','smallint'), + ('votes_latest_unique','weight_x_32767','smallint'), + ('votes_latest_unique','modified','bigint'), + ('crowd_mod','zid','integer'), + ('crowd_mod','pid','integer'), + ('crowd_mod','tid','integer'), + ('crowd_mod','created','bigint'), + ('crowd_mod','as_important','boolean'), + ('crowd_mod','as_factual','boolean'), + ('crowd_mod','as_feeling','boolean'), + ('crowd_mod','as_notmyfeeling','boolean'), + ('crowd_mod','as_notgoodidea','boolean'), + ('crowd_mod','as_notfact','boolean'), + ('crowd_mod','as_unsure','boolean'), + ('crowd_mod','as_spam','boolean'), + ('crowd_mod','as_abusive','boolean'), + ('crowd_mod','as_offtopic','boolean'), + ('event_ptpt_no_more_comments','zid','integer'), + ('event_ptpt_no_more_comments','pid','integer'), + ('event_ptpt_no_more_comments','votes_placed','smallint'), + ('event_ptpt_no_more_comments','created','bigint'), + ('stars','zid','integer'), + ('stars','pid','integer'), + ('stars','tid','integer'), + ('stars','starred','integer'), + ('stars','created','bigint'), + ('trashes','zid','integer'), + ('trashes','pid','integer'), + ('trashes','tid','integer'), + ('trashes','trashed','integer'), + ('trashes','created','bigint'), + ('permanentcookiezidjoins','zid','integer'), + ('permanentcookiezidjoins','cookie','character varying(32)'), + ('permanentcookiezidjoins','created','bigint'), + ('page_ids','site_id','character varying(100)'), + ('page_ids','page_id','character varying(100)'), + ('page_ids','zid','integer'), + ('demographic_data','uid','integer'), + ('demographic_data','fb_gender','integer'), + ('demographic_data','ms_birth_year_estimate_fb','integer'), + ('demographic_data','ms_gender_estimate_fb','integer'), + ('demographic_data','fb_timestamp','bigint'), + ('demographic_data','ms_fb_timestamp','bigint'), + ('demographic_data','ms_response','character varying(9999)'), + ('demographic_data','gender_guess','integer'), + ('demographic_data','birth_year_guess','integer')) AS expected(t,c,typ) WHERE NOT pg_temp.col(t,c,typ)) + AND NOT EXISTS (SELECT 1 FROM unnest(ARRAY['users','site_domain_whitelist','metrics','auth_tokens','jianiuevyew','apikeysndvweifu','courses','conversations','participant_metadata_questions','participant_metadata_answers','contexts','inviters','upvotes','oinvites','einvites','email_validations','zinvites','beta','participants','participants_extended','participant_locations','xids','xid_whitelist','notification_tasks','participant_metadata_choices','twitter_users','facebook_users','social_settings','facebook_friends','suzinvites','comments','comment_translations','conversation_translations','reports','report_comment_selections','worker_tasks','math_ticks','math_main','math_profile','math_ptptstats','math_cache','math_bidtopid','math_exportstatus','math_report_correlationmatrix','votes','votes_latest_unique','crowd_mod','event_ptpt_no_more_comments','stars','trashes','permanentcookiezidjoins','page_ids','demographic_data']) t WHERE to_regclass('public.'||t) IS NULL) + AND (pg_temp.col('pwreset_tokens','token','character varying(100)') OR pg_temp.col('password_reset_tokens','pwresettoken','character varying(100)')) + AND (pg_temp.absent_column('conversations','branding_type') OR pg_temp.col('conversations','branding_type','integer')) + AND (pg_temp.absent_column('math_ticks','caching_tick') OR pg_temp.col('math_ticks','caching_tick','bigint')) + AND to_regprocedure('public.now_as_millis()') IS NOT NULL + AND pg_temp.con('votes_latest_unique','UNIQUE (zid, pid, tid)'); diff --git a/server/postgres/migrations/adoption/000001_update_pwreset_table.sql b/server/postgres/migrations/adoption/000001_update_pwreset_table.sql new file mode 100644 index 0000000000..7752542f42 --- /dev/null +++ b/server/postgres/migrations/adoption/000001_update_pwreset_table.sql @@ -0,0 +1,4 @@ +-- Catalog postconditions only; this file never replays migration DDL. +SELECT to_regclass('public.password_reset_tokens') IS NULL + AND pg_temp.col('pwreset_tokens','token','character varying(100)') + AND pg_temp.absent_column('pwreset_tokens','pwresettoken'); diff --git a/server/postgres/migrations/adoption/000002_add_xid_constraint.sql b/server/postgres/migrations/adoption/000002_add_xid_constraint.sql new file mode 100644 index 0000000000..41e11d827e --- /dev/null +++ b/server/postgres/migrations/adoption/000002_add_xid_constraint.sql @@ -0,0 +1,14 @@ +-- Both the public bootstrap variant (owner,xid only) and the authoritative +-- legacy variant (also owner,uid) are supported. Never drop either invariant. +SELECT pg_temp.con_exact('xids','UNIQUE (owner, xid)') + -- If the optional legacy unique index exists, it must enforce the complete key. + AND NOT EXISTS ( + SELECT 1 FROM pg_index i + JOIN pg_attribute owner ON owner.attrelid=i.indrelid AND owner.attname='owner' + JOIN pg_attribute uid ON uid.attrelid=i.indrelid AND uid.attname='uid' + WHERE i.indrelid=to_regclass('public.xids') AND i.indisunique + AND i.indnkeyatts=2 + AND ARRAY[i.indkey[0],i.indkey[1]] @> ARRAY[owner.attnum,uid.attnum] + AND (NOT i.indisvalid OR NOT i.indisready OR NOT i.indislive + OR NOT i.indimmediate OR i.indpred IS NOT NULL) + ); diff --git a/server/postgres/migrations/adoption/000003_add_origin_permanent_cookie_columns.sql b/server/postgres/migrations/adoption/000003_add_origin_permanent_cookie_columns.sql new file mode 100644 index 0000000000..4f0a5cee75 --- /dev/null +++ b/server/postgres/migrations/adoption/000003_add_origin_permanent_cookie_columns.sql @@ -0,0 +1,3 @@ +-- Catalog postconditions only; this file never replays migration DDL. +SELECT pg_temp.col_exact('participants_extended','permanent_cookie','character varying(32)',false) + AND pg_temp.col_exact('participants_extended','origin','character varying(9999)',false); diff --git a/server/postgres/migrations/adoption/000004_drop_waitinglist_table.sql b/server/postgres/migrations/adoption/000004_drop_waitinglist_table.sql new file mode 100644 index 0000000000..61fac26e19 --- /dev/null +++ b/server/postgres/migrations/adoption/000004_drop_waitinglist_table.sql @@ -0,0 +1,2 @@ +-- Catalog postconditions only; this file never replays migration DDL. +SELECT to_regclass('public.waitinglist') IS NULL; diff --git a/server/postgres/migrations/adoption/000005_drop_slack_stripe_canvas.sql b/server/postgres/migrations/adoption/000005_drop_slack_stripe_canvas.sql new file mode 100644 index 0000000000..f0db14fc7d --- /dev/null +++ b/server/postgres/migrations/adoption/000005_drop_slack_stripe_canvas.sql @@ -0,0 +1,16 @@ +-- Catalog postconditions only; this file never replays migration DDL. +SELECT to_regclass('public.slack_oauth_access_tokens') IS NULL + AND to_regclass('public.slack_users') IS NULL + AND to_regclass('public.slack_user_invites') IS NULL + AND to_regclass('public.slack_bot_events') IS NULL + AND to_regclass('public.stripe_accounts') IS NULL + AND to_regclass('public.stripe_subscriptions') IS NULL + AND to_regclass('public.coupons_for_free_upgrades') IS NULL + AND to_regclass('public.lti_users') IS NULL + AND to_regclass('public.lti_context_memberships') IS NULL + AND to_regclass('public.canvas_assignment_callback_info') IS NULL + AND to_regclass('public.canvas_assignment_conversation_info') IS NULL + AND to_regclass('public.lti_oauthv1_credentials') IS NULL + AND pg_temp.absent_column('conversations','is_slack') + AND pg_temp.absent_column('conversations','lti_users_only') + AND pg_temp.absent_column('users','plan'); diff --git a/server/postgres/migrations/adoption/000006_update_votes_rule.sql b/server/postgres/migrations/adoption/000006_update_votes_rule.sql new file mode 100644 index 0000000000..b79f8b8243 --- /dev/null +++ b/server/postgres/migrations/adoption/000006_update_votes_rule.sql @@ -0,0 +1,2 @@ +-- Catalog postconditions only; this file never replays migration DDL. +SELECT EXISTS (SELECT 1 FROM pg_rewrite WHERE ev_class=to_regclass('public.votes') AND rulename='on_vote_insert_update_unique_table' AND ev_enabled='O' AND regexp_replace(pg_get_ruledef(oid), '\s+', ' ', 'g') = 'CREATE RULE on_vote_insert_update_unique_table AS ON INSERT TO public.votes DO INSERT INTO votes_latest_unique (zid, pid, tid, vote, weight_x_32767, modified) VALUES (new.zid, new.pid, new.tid, new.vote, new.weight_x_32767, new.created) ON CONFLICT(zid, pid, tid) DO UPDATE SET vote = excluded.vote, modified = excluded.modified;'); diff --git a/server/postgres/migrations/adoption/000007_drop_geolocation_fields.sql b/server/postgres/migrations/adoption/000007_drop_geolocation_fields.sql new file mode 100644 index 0000000000..c8f44df378 --- /dev/null +++ b/server/postgres/migrations/adoption/000007_drop_geolocation_fields.sql @@ -0,0 +1,9 @@ +-- Catalog postconditions only; this file never replays migration DDL. +SELECT to_regclass('public.geolocation_cache') IS NULL + AND pg_temp.absent_column('participants_extended','country_code_iso') + AND pg_temp.absent_column('participants_extended','encrypted_maxmind_response_city') + AND pg_temp.absent_column('participants_extended','ip_address') + AND pg_temp.absent_column('participants_extended','latitude') + AND pg_temp.absent_column('participants_extended','location') + AND pg_temp.absent_column('participants_extended','longitude') + AND pg_temp.absent_column('participants_extended','x_forwarded_for'); diff --git a/server/postgres/migrations/adoption/000008_add_comment_priority.sql b/server/postgres/migrations/adoption/000008_add_comment_priority.sql new file mode 100644 index 0000000000..11072158e0 --- /dev/null +++ b/server/postgres/migrations/adoption/000008_add_comment_priority.sql @@ -0,0 +1,3 @@ +-- Catalog postconditions only; this file never replays migration DDL. +SELECT pg_temp.col_exact('conversations','importance_enabled','boolean',true,'false') + AND pg_temp.col_exact('votes','high_priority','boolean',true,'false'); diff --git a/server/postgres/migrations/adoption/000009_add_uuid_to_zinvites.sql b/server/postgres/migrations/adoption/000009_add_uuid_to_zinvites.sql new file mode 100644 index 0000000000..adf006212d --- /dev/null +++ b/server/postgres/migrations/adoption/000009_add_uuid_to_zinvites.sql @@ -0,0 +1,2 @@ +-- Catalog postconditions only; this file never replays migration DDL. +SELECT pg_temp.col_exact('zinvites','uuid','uuid',false); diff --git a/server/postgres/migrations/adoption/000010_create_oidc_user_mappings.sql b/server/postgres/migrations/adoption/000010_create_oidc_user_mappings.sql new file mode 100644 index 0000000000..6ca36bcdb7 --- /dev/null +++ b/server/postgres/migrations/adoption/000010_create_oidc_user_mappings.sql @@ -0,0 +1,8 @@ +-- Catalog postconditions only; this file never replays migration DDL. +SELECT pg_temp.col_exact('oidc_user_mappings','oidc_sub','character varying(255)',true) + AND pg_temp.col_exact('oidc_user_mappings','uid','integer',true) + AND pg_temp.col_exact('oidc_user_mappings','created','bigint',false,'now_as_millis()') + AND pg_temp.con_exact('oidc_user_mappings','PRIMARY KEY (oidc_sub)') + AND pg_temp.con_exact('oidc_user_mappings','UNIQUE (uid)') + AND pg_temp.con_exact('oidc_user_mappings','FOREIGN KEY (uid) REFERENCES users(uid) ON DELETE CASCADE') + AND pg_temp.idx('idx_oidc_mappings_uid','CREATE INDEX idx_oidc_mappings_uid ON public.oidc_user_mappings USING btree (uid)'); diff --git a/server/postgres/migrations/adoption/000011_alter_suzinvites_xid_to_text.sql b/server/postgres/migrations/adoption/000011_alter_suzinvites_xid_to_text.sql new file mode 100644 index 0000000000..5235d98d42 --- /dev/null +++ b/server/postgres/migrations/adoption/000011_alter_suzinvites_xid_to_text.sql @@ -0,0 +1,2 @@ +-- Catalog postconditions only; this file never replays migration DDL. +SELECT pg_temp.col('suzinvites','xid','text',true); diff --git a/server/postgres/migrations/adoption/000012_create_topic_agenda_selections.sql b/server/postgres/migrations/adoption/000012_create_topic_agenda_selections.sql new file mode 100644 index 0000000000..62832a1743 --- /dev/null +++ b/server/postgres/migrations/adoption/000012_create_topic_agenda_selections.sql @@ -0,0 +1,15 @@ +-- Catalog postconditions only; this file never replays migration DDL. +SELECT pg_temp.col_exact('topic_agenda_selections','zid','integer',true) + AND pg_temp.col_exact('topic_agenda_selections','pid','integer',true) + AND pg_temp.col_exact('topic_agenda_selections','archetypal_selections','jsonb',true,'''[]''::jsonb') + AND pg_temp.col_exact('topic_agenda_selections','delphi_job_id','text',false) + AND pg_temp.col_exact('topic_agenda_selections','total_selections','integer',true,'0') + AND pg_temp.col_exact('topic_agenda_selections','created_at','timestamp with time zone',false,'CURRENT_TIMESTAMP') + AND pg_temp.col_exact('topic_agenda_selections','updated_at','timestamp with time zone',false,'CURRENT_TIMESTAMP') + AND pg_temp.con_exact('topic_agenda_selections','PRIMARY KEY (zid, pid)') + AND pg_temp.con_exact('topic_agenda_selections','FOREIGN KEY (zid) REFERENCES conversations(zid) ON DELETE CASCADE') + AND pg_temp.con_exact('topic_agenda_selections','FOREIGN KEY (zid, pid) REFERENCES participants(zid, pid) ON DELETE CASCADE') + AND pg_temp.idx('idx_topic_agenda_selections_zid','CREATE INDEX idx_topic_agenda_selections_zid ON public.topic_agenda_selections USING btree (zid)') + AND pg_temp.idx('idx_topic_agenda_selections_pid','CREATE INDEX idx_topic_agenda_selections_pid ON public.topic_agenda_selections USING btree (pid)') + AND pg_temp.idx('idx_topic_agenda_selections_delphi_job_id','CREATE INDEX idx_topic_agenda_selections_delphi_job_id ON public.topic_agenda_selections USING btree (delphi_job_id)') + AND pg_temp.idx('idx_topic_agenda_selections_created_at','CREATE INDEX idx_topic_agenda_selections_created_at ON public.topic_agenda_selections USING btree (created_at)'); diff --git a/server/postgres/migrations/adoption/000013_create_treevite.sql b/server/postgres/migrations/adoption/000013_create_treevite.sql new file mode 100644 index 0000000000..613b2b764f --- /dev/null +++ b/server/postgres/migrations/adoption/000013_create_treevite.sql @@ -0,0 +1,70 @@ +-- Catalog postconditions only; this file never replays migration DDL. +SELECT pg_temp.col_exact('conversations','treevite_enabled','boolean',false,'false') + AND pg_temp.col_exact('treevite_waves','id','bigint',true,'nextval(''treevite_waves_id_seq''::regclass)') + AND pg_temp.col_exact('treevite_waves','zid','integer',true) + AND pg_temp.col_exact('treevite_waves','wave','integer',true) + AND pg_temp.col_exact('treevite_waves','parent_wave','integer',false) + AND pg_temp.col_exact('treevite_waves','size','integer',false) + AND pg_temp.col_exact('treevite_waves','invites_per_user','integer',true) + AND pg_temp.col_exact('treevite_waves','owner_invites','integer',true,'0') + AND pg_temp.col_exact('treevite_waves','created_at','timestamp with time zone',false,'CURRENT_TIMESTAMP') + AND pg_temp.col_exact('treevite_waves','updated_at','timestamp with time zone',false,'CURRENT_TIMESTAMP') + AND pg_temp.con_exact('treevite_waves','PRIMARY KEY (id)') + AND pg_temp.col_exact('treevite_invites','id','bigint',true,'nextval(''treevite_invites_id_seq''::regclass)') + AND pg_temp.col_exact('treevite_invites','zid','integer',true) + AND pg_temp.col_exact('treevite_invites','wave_id','bigint',true) + AND pg_temp.col_exact('treevite_invites','parent_invite_id','bigint',false) + AND pg_temp.col_exact('treevite_invites','status','smallint',true,'0') + AND pg_temp.col_exact('treevite_invites','invite_owner_pid','integer',false) + AND pg_temp.col_exact('treevite_invites','invite_used_by_pid','integer',false) + AND pg_temp.col_exact('treevite_invites','invite_used_at','timestamp with time zone',false) + AND pg_temp.col_exact('treevite_invites','created_at','timestamp with time zone',false,'CURRENT_TIMESTAMP') + AND pg_temp.col_exact('treevite_invites','updated_at','timestamp with time zone',false,'CURRENT_TIMESTAMP') + AND pg_temp.con_exact('treevite_invites','PRIMARY KEY (id)') + AND pg_temp.col_exact('treevite_login_codes','id','bigint',true,'nextval(''treevite_login_codes_id_seq''::regclass)') + AND pg_temp.col_exact('treevite_login_codes','zid','integer',true) + AND pg_temp.col_exact('treevite_login_codes','pid','integer',true) + AND pg_temp.col_exact('treevite_login_codes','login_code_hash','text',true) + AND pg_temp.col_exact('treevite_login_codes','fp_kid','smallint',true,'1') + AND pg_temp.col_exact('treevite_login_codes','revoked','boolean',true,'false') + AND pg_temp.col_exact('treevite_login_codes','expires_at','timestamp with time zone',false) + AND pg_temp.col_exact('treevite_login_codes','last_used_at','timestamp with time zone',false) + AND pg_temp.col_exact('treevite_login_codes','created_at','timestamp with time zone',false,'CURRENT_TIMESTAMP') + AND pg_temp.col_exact('treevite_login_codes','updated_at','timestamp with time zone',false,'CURRENT_TIMESTAMP') + AND pg_temp.con_exact('treevite_login_codes','PRIMARY KEY (id)') + AND pg_temp.con_exact('treevite_waves','UNIQUE (zid, wave)') + AND pg_temp.con_exact('treevite_waves','FOREIGN KEY (zid) REFERENCES conversations(zid) ON DELETE CASCADE') + AND pg_temp.con_exact('treevite_invites','UNIQUE (zid, invite_code)') + AND pg_temp.con_exact('treevite_invites','FOREIGN KEY (zid) REFERENCES conversations(zid) ON DELETE CASCADE') + AND pg_temp.con_exact('treevite_invites','FOREIGN KEY (wave_id) REFERENCES treevite_waves(id) ON DELETE CASCADE') + AND pg_temp.con_exact('treevite_invites','FOREIGN KEY (parent_invite_id) REFERENCES treevite_invites(id) ON DELETE SET NULL') + AND pg_temp.con_exact('treevite_invites','FOREIGN KEY (zid, invite_owner_pid) REFERENCES participants(zid, pid)') + AND pg_temp.con_exact('treevite_invites','FOREIGN KEY (zid, invite_used_by_pid) REFERENCES participants(zid, pid)') + AND pg_temp.con_exact('treevite_login_codes','FOREIGN KEY (zid, pid) REFERENCES participants(zid, pid) ON DELETE CASCADE') + AND pg_temp.con_exact('treevite_login_codes','UNIQUE (zid, pid)') + AND pg_temp.con_exact('treevite_login_codes','UNIQUE (zid, login_code_fingerprint)') + AND pg_temp.con_exact('treevite_login_codes','UNIQUE (zid, login_code_lookup)') + AND pg_temp.idx('idx_treevite_waves_zid','CREATE INDEX idx_treevite_waves_zid ON public.treevite_waves USING btree (zid)') + AND pg_temp.idx('idx_treevite_waves_wave','CREATE INDEX idx_treevite_waves_wave ON public.treevite_waves USING btree (wave)') + AND pg_temp.idx('idx_treevite_waves_parent','CREATE INDEX idx_treevite_waves_parent ON public.treevite_waves USING btree (zid, parent_wave)') + AND pg_temp.idx('idx_treevite_invites_zid','CREATE INDEX idx_treevite_invites_zid ON public.treevite_invites USING btree (zid)') + AND pg_temp.idx('idx_treevite_invites_zid_status','CREATE INDEX idx_treevite_invites_zid_status ON public.treevite_invites USING btree (zid, status)') + AND pg_temp.idx('idx_treevite_invites_wave_id','CREATE INDEX idx_treevite_invites_wave_id ON public.treevite_invites USING btree (wave_id)') + AND pg_temp.idx('idx_treevite_invites_parent','CREATE INDEX idx_treevite_invites_parent ON public.treevite_invites USING btree (parent_invite_id)') + AND pg_temp.idx('idx_treevite_invites_owner_pid','CREATE INDEX idx_treevite_invites_owner_pid ON public.treevite_invites USING btree (invite_owner_pid)') + AND pg_temp.idx('idx_treevite_invites_used_by_pid','CREATE INDEX idx_treevite_invites_used_by_pid ON public.treevite_invites USING btree (invite_used_by_pid)') + AND pg_temp.idx('idx_treevite_invites_code','CREATE INDEX idx_treevite_invites_code ON public.treevite_invites USING btree (invite_code)') + AND pg_temp.idx('idx_treevite_login_codes_zid','CREATE INDEX idx_treevite_login_codes_zid ON public.treevite_login_codes USING btree (zid)') + AND pg_temp.idx('idx_treevite_login_codes_pid','CREATE INDEX idx_treevite_login_codes_pid ON public.treevite_login_codes USING btree (pid)') + AND pg_temp.idx('idx_treevite_login_codes_fp','CREATE INDEX idx_treevite_login_codes_fp ON public.treevite_login_codes USING btree (login_code_fingerprint)') + AND pg_temp.idx('idx_treevite_login_codes_lookup','CREATE INDEX idx_treevite_login_codes_lookup ON public.treevite_login_codes USING btree (zid, login_code_lookup)') + AND pg_temp.col_exact('treevite_invites','invite_code','character varying(64)',true) + AND pg_temp.col_exact('treevite_login_codes','login_code_fingerprint','character varying(128)',true) + AND pg_temp.col_exact('treevite_login_codes','login_code_lookup','character varying(128)',false) + AND pg_temp.con_exact('treevite_waves','CHECK (((invites_per_user > 0) OR (owner_invites > 0)))') + AND pg_temp.con_exact('treevite_waves','CHECK (((parent_wave IS NULL) OR (parent_wave >= 0)))') + AND pg_temp.con_exact('treevite_waves','CHECK (((size IS NULL) OR (size >= 0)))') + AND pg_temp.con_exact('treevite_waves','CHECK ((invites_per_user >= 0))') + AND pg_temp.con_exact('treevite_waves','CHECK ((owner_invites >= 0))') + AND pg_temp.con_exact('treevite_waves','CHECK ((wave >= 1))') + AND pg_temp.con_exact('treevite_invites','CHECK ((status = ANY (ARRAY[0, 1, 2, 3])))'); diff --git a/server/postgres/migrations/adoption/000014_alter_reports_modlevel.sql b/server/postgres/migrations/adoption/000014_alter_reports_modlevel.sql new file mode 100644 index 0000000000..762f142a17 --- /dev/null +++ b/server/postgres/migrations/adoption/000014_alter_reports_modlevel.sql @@ -0,0 +1,2 @@ +-- Catalog postconditions only; this file never replays migration DDL. +SELECT pg_temp.col_exact('reports','mod_level','smallint',true,'''-2''::integer'); diff --git a/server/postgres/migrations/adoption/000015_add_xid_requirements.sql b/server/postgres/migrations/adoption/000015_add_xid_requirements.sql new file mode 100644 index 0000000000..584bf1cca0 --- /dev/null +++ b/server/postgres/migrations/adoption/000015_add_xid_requirements.sql @@ -0,0 +1,15 @@ +-- Catalog postconditions only; this file never replays migration DDL. +SELECT pg_temp.col_exact('conversations','xid_required','boolean',true,'false') + AND pg_temp.col_exact('xid_whitelist','zid','integer',false) + AND pg_temp.col_exact('xids','zid','integer',false) + AND pg_temp.col_exact('xids','pid','integer',false) + AND pg_temp.con_exact('xid_whitelist','FOREIGN KEY (zid) REFERENCES conversations(zid) ON DELETE CASCADE') + AND pg_temp.con_exact('xids','FOREIGN KEY (zid) REFERENCES conversations(zid) ON DELETE CASCADE') + AND pg_temp.con_exact('xids','FOREIGN KEY (zid, pid) REFERENCES participants(zid, pid) ON DELETE SET NULL') + AND pg_temp.idx('idx_xid_whitelist_zid','CREATE INDEX idx_xid_whitelist_zid ON public.xid_whitelist USING btree (zid)') + AND pg_temp.idx('idx_xid_whitelist_xid','CREATE INDEX idx_xid_whitelist_xid ON public.xid_whitelist USING btree (xid)') + AND pg_temp.idx('idx_xids_zid','CREATE INDEX idx_xids_zid ON public.xids USING btree (zid)') + AND pg_temp.idx('idx_xids_xid','CREATE INDEX idx_xids_xid ON public.xids USING btree (xid)') + AND pg_temp.idx('idx_xids_pid','CREATE INDEX idx_xids_pid ON public.xids USING btree (pid)') + AND pg_temp.idx('idx_xids_zid_xid','CREATE INDEX idx_xids_zid_xid ON public.xids USING btree (zid, xid)') + AND pg_temp.idx('idx_xids_uid_zid','CREATE INDEX idx_xids_uid_zid ON public.xids USING btree (uid, zid)'); diff --git a/server/postgres/migrations/adoption/000016_add_orig_id.sql b/server/postgres/migrations/adoption/000016_add_orig_id.sql new file mode 100644 index 0000000000..c864197137 --- /dev/null +++ b/server/postgres/migrations/adoption/000016_add_orig_id.sql @@ -0,0 +1,2 @@ +-- Catalog postconditions only; this file never replays migration DDL. +SELECT pg_temp.col_exact('comments','original_id','uuid',false); diff --git a/server/postgres/migrations/adoption/000017_create_byod_job_table.sql b/server/postgres/migrations/adoption/000017_create_byod_job_table.sql new file mode 100644 index 0000000000..43ed6a91db --- /dev/null +++ b/server/postgres/migrations/adoption/000017_create_byod_job_table.sql @@ -0,0 +1,12 @@ +-- Catalog postconditions only; this file never replays migration DDL. +SELECT (SELECT array_agg(enumlabel::text ORDER BY enumsortorder) FROM pg_enum WHERE enumtypid=to_regtype('public.job_status')) = ARRAY['pending','processing','completed','failed'] + AND pg_temp.col_exact('byod_import_jobs','id','integer',true,'nextval(''byod_import_jobs_id_seq''::regclass)') + AND pg_temp.col_exact('byod_import_jobs','zid','integer',true) + AND pg_temp.col_exact('byod_import_jobs','s3_key','text',true) + AND pg_temp.col_exact('byod_import_jobs','status','job_status',false,'''pending''::job_status') + AND pg_temp.col_exact('byod_import_jobs','stage','text',false,'''init''::text') + AND pg_temp.col_exact('byod_import_jobs','error_message','text',false) + AND pg_temp.col_exact('byod_import_jobs','created_at','timestamp with time zone',false,'now()') + AND pg_temp.col_exact('byod_import_jobs','updated_at','timestamp with time zone',false,'now()') + AND pg_temp.con_exact('byod_import_jobs','PRIMARY KEY (id)') + AND pg_temp.idx('idx_byod_jobs_zid','CREATE INDEX idx_byod_jobs_zid ON public.byod_import_jobs USING btree (zid)'); diff --git a/server/postgres/migrations/adoption/000018_add_topics_enabled.sql b/server/postgres/migrations/adoption/000018_add_topics_enabled.sql new file mode 100644 index 0000000000..5a9299078b --- /dev/null +++ b/server/postgres/migrations/adoption/000018_add_topics_enabled.sql @@ -0,0 +1,2 @@ +-- Catalog postconditions only; this file never replays migration DDL. +SELECT pg_temp.col_exact('conversations','topics_enabled','boolean',true,'false'); diff --git a/server/postgres/migrations/adoption/000022_add_poll_timestamp_indexes.sql b/server/postgres/migrations/adoption/000022_add_poll_timestamp_indexes.sql new file mode 100644 index 0000000000..8ba522e217 --- /dev/null +++ b/server/postgres/migrations/adoption/000022_add_poll_timestamp_indexes.sql @@ -0,0 +1,3 @@ +-- Catalog postconditions only; this file never replays migration DDL. +SELECT pg_temp.idx('votes_created_idx','CREATE INDEX votes_created_idx ON public.votes USING btree (created)') + AND pg_temp.idx('comments_modified_idx','CREATE INDEX comments_modified_idx ON public.comments USING btree (modified)'); diff --git a/server/postgres/migrations/adoption/helpers.sql b/server/postgres/migrations/adoption/helpers.sql new file mode 100644 index 0000000000..de607fd80b --- /dev/null +++ b/server/postgres/migrations/adoption/helpers.sql @@ -0,0 +1,59 @@ +-- Reconciliation only. These pg_temp helpers vanish when the connection closes. +-- A nullability/default argument checks only when specified by that migration. +CREATE FUNCTION pg_temp.col(t text,c text,typ text,nn boolean DEFAULT NULL,def text DEFAULT NULL) +RETURNS boolean LANGUAGE sql AS $$ + SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||t) AND a.attname=c AND NOT a.attisdropped + AND (format_type(a.atttypid,a.atttypmod)=typ + -- Exact public bootstrap alternatives to the authoritative legacy contract. + OR (t='worker_tasks' AND c='task_type' AND typ='text' + AND format_type(a.atttypid,a.atttypmod)='character varying(99)') + OR (t IN ('pwreset_tokens','password_reset_tokens') + AND c IN ('token','pwresettoken') AND typ='character varying(100)' + AND format_type(a.atttypid,a.atttypmod)='character varying(250)') + -- Only historical math payloads have an established json equivalent. + -- Do not adopt arbitrary json columns in newer jsonb contracts. + OR (typ='jsonb' AND a.atttypid='json'::regtype AND c='data' + AND t IN ('math_main','math_profile','math_ptptstats','math_cache', + 'math_bidtopid','math_exportstatus'))) + AND (nn IS NULL OR a.attnotnull=nn) + AND (def IS NULL OR pg_get_expr(d.adbin,d.adrelid)=def)); +$$; +CREATE FUNCTION pg_temp.con(t text,definition text) RETURNS boolean LANGUAGE sql AS $$ + SELECT EXISTS(SELECT 1 FROM pg_constraint WHERE conrelid=to_regclass('public.'||t) + AND convalidated AND pg_get_constraintdef(oid)=definition); +$$; +CREATE FUNCTION pg_temp.idx(n text,definition text) RETURNS boolean LANGUAGE sql AS $$ + SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||n) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=definition); +$$; +CREATE FUNCTION pg_temp.absent_column(t text,c text) RETURNS boolean LANGUAGE sql AS $$ + SELECT to_regclass('public.'||t) IS NOT NULL AND NOT EXISTS(SELECT 1 FROM pg_attribute + WHERE attrelid=to_regclass('public.'||t) AND attname=c AND NOT attisdropped); +$$; + +-- Full contracts for newly created columns. NULL means no default, not a wildcard. +-- Keep col() unchanged for legacy/core and ALTER TYPE-only postconditions. +CREATE FUNCTION pg_temp.col_exact(t text,c text,typ text,nn boolean,def text DEFAULT NULL) +RETURNS boolean LANGUAGE sql AS $$ + SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||t) AND a.attname=c AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=typ AND a.attnotnull=nn + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM def); +$$; +-- Definition checks include actions/keys; catalog flags also require enforcement. +-- Preserve the legacy helper for contracts awaiting a separate policy decision. +CREATE FUNCTION pg_temp.con_exact(t text,definition text) RETURNS boolean LANGUAGE sql AS $$ + SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||t) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=definition + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O')); +$$; diff --git a/server/postgres/migrations/held.txt b/server/postgres/migrations/held.txt new file mode 100644 index 0000000000..b8cd53cfc9 --- /dev/null +++ b/server/postgres/migrations/held.txt @@ -0,0 +1,3 @@ +# Release-wide hold: the coordinator is dormant and not an API requirement. +# Moving this file into the required chain needs an explicit schema release. +000021_create_polis_coordinator.sql diff --git a/server/postgres/migrations/release.txt b/server/postgres/migrations/release.txt new file mode 100644 index 0000000000..d3717e0d34 --- /dev/null +++ b/server/postgres/migrations/release.txt @@ -0,0 +1,25 @@ +# Selected stable runner release. Unknown numbered files fail closed. +# M4/M5/M7 are retirement receipts only; their DDL never executes. +000000_initial.sql +000001_update_pwreset_table.sql +000002_add_xid_constraint.sql +000003_add_origin_permanent_cookie_columns.sql +000004_drop_waitinglist_table.sql +000005_drop_slack_stripe_canvas.sql +000006_update_votes_rule.sql +000007_drop_geolocation_fields.sql +000008_add_comment_priority.sql +000009_add_uuid_to_zinvites.sql +000010_create_oidc_user_mappings.sql +000011_alter_suzinvites_xid_to_text.sql +000012_create_topic_agenda_selections.sql +000013_create_treevite.sql +000014_alter_reports_modlevel.sql +000015_add_xid_requirements.sql +000016_add_orig_id.sql +000017_create_byod_job_table.sql +000018_add_topics_enabled.sql +000019_create_polis_queue.sql +000022_add_poll_timestamp_indexes.sql +000023_create_delphi_foundation.sql +000024_create_polis_queue_large_class.sql diff --git a/server/postgres/migrations/report/first-deploy.sql b/server/postgres/migrations/report/first-deploy.sql new file mode 100644 index 0000000000..5803309034 --- /dev/null +++ b/server/postgres/migrations/report/first-deploy.sql @@ -0,0 +1,1399 @@ +-- GENERATED by server/bin/build-migration-report.py; do not edit. +-- First deployment only: no existing ledger or queue. Other states refuse. +-- This is a catalog forecast, not DDL success or deployment-health proof. +-- sha256 8fa1066d9fc29c1df528b300199c619a2f9edecf1432926ad6a8146ebc650593 adoption/helpers.sql +-- sha256 0bbe8c3ac74a484580576f45d40dcd3ddae205bb0c63377ff8f053cecf979e0d release.txt +-- sha256 8a8b24fa47a25c613a8329aa79f413ac7461a93b12134045ae1cf2457e2ee3b7 held.txt +-- sha256 c6b5c71247d129964dd89ba560a4522caef1abbb9379ad1b3445d6ed96be805b adoption/000000_initial.sql +-- sha256 28ef33976beb9c7c9248ac800f29aab048f4410c2170ad12e3ade71d72e5ecb5 adoption/000001_update_pwreset_table.sql +-- sha256 c3393049eae0a663ecea42a4917a06b37521a8ff508d39b308f8fd9da5606bf9 adoption/000002_add_xid_constraint.sql +-- sha256 3b1c9a390cbf23882455f1f3d05119df4a306f9a04d18e8c0e2cff3374895b86 adoption/000003_add_origin_permanent_cookie_columns.sql +-- sha256 5fae81bb9ff86062125487df98c1742e7ce5c45466fbeed7f6f63cbf94b1933b adoption/000004_drop_waitinglist_table.sql +-- sha256 02c90a416148f29c0717c01bcc97e66d6e8083ab3a2fabc5dc6bc716e963447e adoption/000005_drop_slack_stripe_canvas.sql +-- sha256 42d43c346ea90d0504698df29f90dd2c7e57860338c633da11410aaf9aaf5148 adoption/000006_update_votes_rule.sql +-- sha256 5a4110278c60157f21e5cb920ece834f6d6f99e6d4fa34b9b55bf0c6e80587c0 adoption/000007_drop_geolocation_fields.sql +-- sha256 a10a1a1881044409d98efd6042b5b28933a4579bce32738be1c4bc268640f172 adoption/000008_add_comment_priority.sql +-- sha256 a5605b5a292f5b4d47849cb7f2b678211e3bcdc9722cf9e5eabe54f455a5aa7c adoption/000009_add_uuid_to_zinvites.sql +-- sha256 23d5179cd0b556a71fe8b1ce56d3ac9e614cdee59832ce7892a012f0193cfe66 adoption/000010_create_oidc_user_mappings.sql +-- sha256 c991ffd93cd9051302dd51ab8d5bd6025128fc9728330e29aa963306a9b80f3e adoption/000011_alter_suzinvites_xid_to_text.sql +-- sha256 c31bd6811efee0808da5abb32ba63c47deebe6c6e89baa1a80ddcacc9e250234 adoption/000012_create_topic_agenda_selections.sql +-- sha256 eed1d9e223322db2b234428d350b6548793e54b94315932658b693d8263a6001 adoption/000013_create_treevite.sql +-- sha256 6b99d93d2f90f03bee170305c4b1305f03636882a84cdd08e107a763f1fc0120 adoption/000014_alter_reports_modlevel.sql +-- sha256 30c8d693d4f0d523c68a375bf0160e71e574f0a155badc363e085c4688ff0625 adoption/000015_add_xid_requirements.sql +-- sha256 ffb57aff8be7bc81bf10899133fe2a243cb20b58eb6b73afb3b76a66d5632478 adoption/000016_add_orig_id.sql +-- sha256 b72fbd498c13aceb7973100331c58b55a468b7e4447645a3e5c93f79a1126fd6 adoption/000017_create_byod_job_table.sql +-- sha256 96bf4a25a5733ac3dce8a7f25a4f374b5ee9bed924e95e048f73a343d2ddf539 adoption/000018_add_topics_enabled.sql +-- sha256 9764fc94c4fe4ac19626381146a588da3f6b4d32cac53ab59ae87b401ea8af8a adoption/000022_add_poll_timestamp_indexes.sql +-- sha256 2652134140cd8796ee3ab64995d509206bab3ebe8a03bba0cf48038a32dc6cea 000000_initial.sql +-- sha256 cb21278194c4b7db510c12bf1a7761b3a905410a7b3769edeb0d1865b6ec047b 000001_update_pwreset_table.sql +-- sha256 a27a8e63c79cd055198fa201b1745c85e0cd0e0b75f228b7153571631e529c07 000002_add_xid_constraint.sql +-- sha256 0d7f27facfecff1327573d3e1a5f65a3277ea94a30bcd8692606f00a7e496d3d 000003_add_origin_permanent_cookie_columns.sql +-- sha256 f2fc4184a965ffad01913d0e71a806c6f7e2aecc5f59279c11b0cd3ed3a77528 000004_drop_waitinglist_table.sql +-- sha256 392e5b8aadb7a85767820ec97fcfb0853821d35089be3e8c3dfe70df24782839 000005_drop_slack_stripe_canvas.sql +-- sha256 8fb05b7b1b6a8b123cea6b2680054d8da2fe48585c51a2f9f108b8b174d85c60 000006_update_votes_rule.sql +-- sha256 f68b69b86112ad52e2cb79a9b631ea77a0285b24201b41c15d9c13257f7cd9cc 000007_drop_geolocation_fields.sql +-- sha256 c867b53be3cca6bce059b19ea9eb3950120fb5840a1476af409924ca55905261 000008_add_comment_priority.sql +-- sha256 43f36fe0b8578bc4b32761a2b9d7a5181a8159ff8b0c7bb0a1a73f802ec5f670 000009_add_uuid_to_zinvites.sql +-- sha256 450a3f69883aa56c3bc85994ab1baf5e208634d88435a156fe2a22738b1d9f9a 000010_create_oidc_user_mappings.sql +-- sha256 00a1eb2d9604804a56705eaac92bb6d37b6c696abdfd9bb079d4a1c494ff0ab6 000011_alter_suzinvites_xid_to_text.sql +-- sha256 cc513693124f031ce1d55e7e73b14b4e3c40a4070e65465adbc621e63b1008cf 000012_create_topic_agenda_selections.sql +-- sha256 4b8334f73246c69bc9ecadd4a2dc00c8cfb642351ba70ed4a9a58b882a4ac53f 000013_create_treevite.sql +-- sha256 c2af6d57af2866f1f259bb41b94dbf730eca499a4b18f092c4aeeebce5a95025 000014_alter_reports_modlevel.sql +-- sha256 186c904addd0c8a42c10057e6ae362fe2799cc076e877bef3ffa14e9852184e7 000015_add_xid_requirements.sql +-- sha256 6cdc0588c000fbbe89d578bc2f0aa60422b92b5320aba737d6d83f89d8d990db 000016_add_orig_id.sql +-- sha256 f27c03a1229f296fab21cee059b743c7137bf502e4178854e596a8efa4808b55 000017_create_byod_job_table.sql +-- sha256 a1e1c0572064d88877d87142bc4e40132b84672bbe84392e99f9f6f4c2ea107e 000018_add_topics_enabled.sql +-- sha256 fedfbcf9fc594c3e53193cacb76dfd35558d81a449d507fcacb3198f5e39beae 000019_create_polis_queue.sql +-- sha256 d50f169ad7afe12d14582a6a746c622d402ecafd8131aae246812263bf2d5e82 000021_create_polis_coordinator.sql +-- sha256 14efc95b14787b52d70680ea06cfef020cd2224e82495258fd8eb324501b7b3e 000022_add_poll_timestamp_indexes.sql +-- sha256 97437ea57d90c51cc664385ebb6e8ec3d8684ac0f8e7b0c60e84df1a010531f7 000023_create_delphi_foundation.sql +-- sha256 68261afb81f286bed45fdff6ab52e052392d1dd32abdb1e78579f72644116697 000024_create_polis_queue_large_class.sql +BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY; +SET LOCAL search_path=pg_catalog,public; +SET LOCAL statement_timeout='30s'; +SET LOCAL lock_timeout='2s'; +WITH checks AS ( +SELECT '000000_initial.sql'::text AS migration, (-- Named legacy variants are documented in docs/migration-legacy-contract.md. +-- Contributor agreement tables are outside this release adoption boundary. +-- Catalog postconditions only; this file never replays migration DDL. +SELECT NOT EXISTS (SELECT 1 FROM (VALUES + ('users','uid','integer'), + ('users','hname','character varying(746)'), + ('users','created','bigint'), + ('users','username','character varying(128)'), + ('users','email','character varying(256)'), + ('users','is_owner','boolean'), + ('users','zinvite','character varying(300)'), + ('users','oinvite','character varying(300)'), + ('users','tut','smallint'), + ('users','site_id','character varying(256)'), + ('users','site_owner','boolean'), + ('site_domain_whitelist','site_id','character varying(256)'), + ('site_domain_whitelist','domain_whitelist','character varying(999)'), + ('site_domain_whitelist','domain_whitelist_override_key','character varying(999)'), + ('site_domain_whitelist','modified','bigint'), + ('site_domain_whitelist','created','bigint'), + ('metrics','uid','integer'), + ('metrics','type','integer'), + ('metrics','dur','integer'), + ('metrics','hashedpc','integer'), + ('metrics','created','bigint'), + ('auth_tokens','token','character varying(32)'), + ('auth_tokens','uid','integer'), + ('auth_tokens','created','bigint'), + ('jianiuevyew','uid','integer'), + ('jianiuevyew','pwhash','character varying(128)'), + ('apikeysndvweifu','uid','integer'), + ('apikeysndvweifu','apikey','character varying(32)'), + ('apikeysndvweifu','created','bigint'), + ('courses','course_id','integer'), + ('courses','topic','character varying(1000)'), + ('courses','description','character varying(1000)'), + ('courses','owner','integer'), + ('courses','course_invite','character varying(32)'), + ('courses','created','bigint'), + ('conversations','zid','integer'), + ('conversations','topic','character varying(1000)'), + ('conversations','description','character varying(50000)'), + ('conversations','link_url','character varying(9999)'), + ('conversations','parent_url','character varying(9999)'), + ('conversations','upvotes','integer'), + ('conversations','participant_count','integer'), + ('conversations','is_anon','boolean'), + ('conversations','is_active','boolean'), + ('conversations','is_draft','boolean'), + ('conversations','is_public','boolean'), + ('conversations','is_data_open','boolean'), + ('conversations','profanity_filter','boolean'), + ('conversations','spam_filter','boolean'), + ('conversations','strict_moderation','boolean'), + ('conversations','prioritize_seed','boolean'), + ('conversations','vis_type','integer'), + ('conversations','write_type','integer'), + ('conversations','help_type','integer'), + ('conversations','write_hint_type','integer'), + ('conversations','style_btn','character varying(500)'), + ('conversations','socialbtn_type','integer'), + ('conversations','subscribe_type','integer'), + ('conversations','bgcolor','character varying(20)'), + ('conversations','help_bgcolor','character varying(20)'), + ('conversations','help_color','character varying(20)'), + ('conversations','email_domain','character varying(200)'), + ('conversations','use_xid_whitelist','boolean'), + ('conversations','owner','integer'), + ('conversations','org_id','integer'), + ('conversations','context','character varying(1000)'), + ('conversations','course_id','integer'), + ('conversations','owner_sees_participation_stats','boolean'), + ('conversations','auth_needed_to_vote','boolean'), + ('conversations','auth_needed_to_write','boolean'), + ('conversations','auth_opt_fb','boolean'), + ('conversations','auth_opt_tw','boolean'), + ('conversations','auth_opt_allow_3rdparty','boolean'), + ('conversations','modified','bigint'), + ('conversations','created','bigint'), + ('participant_metadata_questions','pmqid','integer'), + ('participant_metadata_questions','zid','integer'), + ('participant_metadata_questions','key','character varying(999)'), + ('participant_metadata_questions','alive','boolean'), + ('participant_metadata_questions','created','bigint'), + ('participant_metadata_answers','pmaid','integer'), + ('participant_metadata_answers','pmqid','integer'), + ('participant_metadata_answers','zid','integer'), + ('participant_metadata_answers','value','character varying(999)'), + ('participant_metadata_answers','alive','boolean'), + ('participant_metadata_answers','created','bigint'), + ('contexts','context_id','integer'), + ('contexts','name','character varying(300)'), + ('contexts','creator','integer'), + ('contexts','is_public','boolean'), + ('contexts','created','bigint'), + ('inviters','inviter_uid','integer'), + ('inviters','invited_email','character varying(999)'), + ('inviters','created','bigint'), + ('upvotes','uid','integer'), + ('upvotes','zid','integer'), + ('oinvites','oinvite','character varying(300)'), + ('oinvites','note','character varying(999)'), + ('oinvites','created','bigint'), + ('einvites','einvite','character varying(100)'), + ('einvites','email','character varying(999)'), + ('einvites','created','bigint'), + ('email_validations','email','character varying(999)'), + ('email_validations','created','bigint'), + ('zinvites','zid','integer'), + ('zinvites','zinvite','character varying(300)'), + ('zinvites','created','bigint'), + ('beta','name','character varying(999)'), + ('beta','email','character varying(200)'), + ('beta','organization','character varying(200)'), + ('beta','created','bigint'), + ('participants','pid','integer'), + ('participants','uid','integer'), + ('participants','zid','integer'), + ('participants','vote_count','integer'), + ('participants','last_interaction','bigint'), + ('participants','subscribed','integer'), + ('participants','last_notified','bigint'), + ('participants','nsli','smallint'), + ('participants','mod','integer'), + ('participants','created','bigint'), + ('participants_extended','uid','integer'), + ('participants_extended','zid','integer'), + ('participants_extended','referrer','character varying(9999)'), + ('participants_extended','parent_url','character varying(9999)'), + ('participants_extended','created','bigint'), + ('participants_extended','modified','bigint'), + ('participants_extended','subscribe_email','character varying(256)'), + ('participants_extended','show_translation_activated','boolean'), + ('participant_locations','zid','integer'), + ('participant_locations','uid','integer'), + ('participant_locations','pid','integer'), + ('participant_locations','lat','double precision'), + ('participant_locations','lng','double precision'), + ('participant_locations','created','bigint'), + ('participant_locations','source','integer'), + ('xids','uid','integer'), + ('xids','owner','integer'), + ('xids','xid','text'), + ('xids','x_profile_image_url','character varying(3000)'), + ('xids','x_name','character varying(746)'), + ('xids','x_email','character varying(256)'), + ('xids','created','bigint'), + ('xids','modified','bigint'), + ('xid_whitelist','owner','integer'), + ('xid_whitelist','xid','text'), + ('xid_whitelist','created','bigint'), + ('notification_tasks','zid','integer'), + ('notification_tasks','modified','bigint'), + ('participant_metadata_choices','zid','integer'), + ('participant_metadata_choices','pid','integer'), + ('participant_metadata_choices','pmqid','integer'), + ('participant_metadata_choices','pmaid','integer'), + ('participant_metadata_choices','alive','boolean'), + ('participant_metadata_choices','created','bigint'), + ('twitter_users','uid','integer'), + ('twitter_users','twitter_user_id','bigint'), + ('twitter_users','screen_name','character varying(999)'), + ('twitter_users','name','character varying(9999)'), + ('twitter_users','followers_count','integer'), + ('twitter_users','friends_count','integer'), + ('twitter_users','verified','boolean'), + ('twitter_users','profile_image_url_https','character varying(9999)'), + ('twitter_users','location','character varying(9999)'), + ('twitter_users','response','json'), + ('twitter_users','modified','bigint'), + ('twitter_users','created','bigint'), + ('facebook_users','uid','integer'), + ('facebook_users','fb_user_id','text'), + ('facebook_users','fb_name','character varying(9999)'), + ('facebook_users','fb_link','character varying(9999)'), + ('facebook_users','fb_public_profile','text'), + ('facebook_users','fb_login_status','text'), + ('facebook_users','fb_auth_response','text'), + ('facebook_users','fb_access_token','text'), + ('facebook_users','fb_granted_scopes','text'), + ('facebook_users','fb_location_id','character varying(100)'), + ('facebook_users','location','character varying(9999)'), + ('facebook_users','response','text'), + ('facebook_users','fb_friends_response','text'), + ('facebook_users','created','bigint'), + ('facebook_users','modified','bigint'), + ('social_settings','uid','integer'), + ('social_settings','polis_pic','character varying(3000)'), + ('facebook_friends','uid','integer'), + ('facebook_friends','friend','integer'), + ('suzinvites','owner','integer'), + ('suzinvites','zid','integer'), + ('suzinvites','created','bigint'), + ('suzinvites','suzinvite','character varying(32)'), + ('comments','tid','integer'), + ('comments','zid','integer'), + ('comments','pid','integer'), + ('comments','uid','integer'), + ('comments','created','bigint'), + ('comments','modified','bigint'), + ('comments','txt','character varying(1000)'), + ('comments','velocity','real'), + ('comments','mod','integer'), + ('comments','lang','character varying(10)'), + ('comments','lang_confidence','real'), + ('comments','active','boolean'), + ('comments','is_meta','boolean'), + ('comments','tweet_id','bigint'), + ('comments','quote_src_url','character varying(1000)'), + ('comments','anon','boolean'), + ('comments','is_seed','boolean'), + ('comment_translations','zid','integer'), + ('comment_translations','tid','integer'), + ('comment_translations','src','integer'), + ('comment_translations','txt','character varying(9999)'), + ('comment_translations','lang','character varying(10)'), + ('comment_translations','created','bigint'), + ('comment_translations','modified','bigint'), + ('conversation_translations','zid','integer'), + ('conversation_translations','src','integer'), + ('conversation_translations','topic','character varying(9999)'), + ('conversation_translations','description','character varying(9999)'), + ('conversation_translations','lang','character varying(10)'), + ('conversation_translations','created','bigint'), + ('conversation_translations','modified','bigint'), + ('reports','rid','bigint'), + ('reports','report_id','character varying(300)'), + ('reports','zid','integer'), + ('reports','created','bigint'), + ('reports','modified','bigint'), + ('reports','report_name','character varying(999)'), + ('reports','label_x_neg','character varying(999)'), + ('reports','label_x_pos','character varying(999)'), + ('reports','label_y_neg','character varying(999)'), + ('reports','label_y_pos','character varying(999)'), + ('reports','label_group_0','character varying(999)'), + ('reports','label_group_1','character varying(999)'), + ('reports','label_group_2','character varying(999)'), + ('reports','label_group_3','character varying(999)'), + ('reports','label_group_4','character varying(999)'), + ('reports','label_group_5','character varying(999)'), + ('reports','label_group_6','character varying(999)'), + ('reports','label_group_7','character varying(999)'), + ('reports','label_group_8','character varying(999)'), + ('reports','label_group_9','character varying(999)'), + ('report_comment_selections','zid','integer'), + ('report_comment_selections','rid','bigint'), + ('report_comment_selections','tid','integer'), + ('report_comment_selections','selection','smallint'), + ('report_comment_selections','modified','bigint'), + ('worker_tasks','created','bigint'), + ('worker_tasks','math_env','character varying(999)'), + ('worker_tasks','attempts','smallint'), + ('worker_tasks','task_data','jsonb'), + ('worker_tasks','task_type','text'), + ('worker_tasks','task_bucket','bigint'), + ('worker_tasks','finished_time','bigint'), + ('math_ticks','zid','integer'), + ('math_ticks','math_tick','bigint'), + ('math_ticks','math_env','character varying(999)'), + ('math_ticks','modified','bigint'), + ('math_main','zid','integer'), + ('math_main','math_env','character varying(999)'), + ('math_main','data','jsonb'), + ('math_main','last_vote_timestamp','bigint'), + ('math_main','caching_tick','bigint'), + ('math_main','math_tick','bigint'), + ('math_main','modified','bigint'), + ('math_profile','zid','integer'), + ('math_profile','math_env','character varying(999)'), + ('math_profile','data','jsonb'), + ('math_profile','modified','bigint'), + ('math_ptptstats','zid','integer'), + ('math_ptptstats','math_env','character varying(999)'), + ('math_ptptstats','math_tick','bigint'), + ('math_ptptstats','data','jsonb'), + ('math_ptptstats','modified','bigint'), + ('math_cache','zid','integer'), + ('math_cache','math_env','character varying(999)'), + ('math_cache','data','jsonb'), + ('math_cache','modified','bigint'), + ('math_bidtopid','zid','integer'), + ('math_bidtopid','math_env','character varying(999)'), + ('math_bidtopid','math_tick','bigint'), + ('math_bidtopid','data','jsonb'), + ('math_bidtopid','modified','bigint'), + ('math_exportstatus','zid','integer'), + ('math_exportstatus','math_env','character varying(999)'), + ('math_exportstatus','filename','character varying(9999)'), + ('math_exportstatus','data','jsonb'), + ('math_exportstatus','modified','bigint'), + ('math_report_correlationmatrix','rid','bigint'), + ('math_report_correlationmatrix','math_env','character varying(999)'), + ('math_report_correlationmatrix','data','jsonb'), + ('math_report_correlationmatrix','math_tick','bigint'), + ('math_report_correlationmatrix','modified','bigint'), + ('votes','zid','integer'), + ('votes','pid','integer'), + ('votes','tid','integer'), + ('votes','vote','smallint'), + ('votes','weight_x_32767','smallint'), + ('votes','created','bigint'), + ('votes_latest_unique','zid','integer'), + ('votes_latest_unique','pid','integer'), + ('votes_latest_unique','tid','integer'), + ('votes_latest_unique','vote','smallint'), + ('votes_latest_unique','weight_x_32767','smallint'), + ('votes_latest_unique','modified','bigint'), + ('crowd_mod','zid','integer'), + ('crowd_mod','pid','integer'), + ('crowd_mod','tid','integer'), + ('crowd_mod','created','bigint'), + ('crowd_mod','as_important','boolean'), + ('crowd_mod','as_factual','boolean'), + ('crowd_mod','as_feeling','boolean'), + ('crowd_mod','as_notmyfeeling','boolean'), + ('crowd_mod','as_notgoodidea','boolean'), + ('crowd_mod','as_notfact','boolean'), + ('crowd_mod','as_unsure','boolean'), + ('crowd_mod','as_spam','boolean'), + ('crowd_mod','as_abusive','boolean'), + ('crowd_mod','as_offtopic','boolean'), + ('event_ptpt_no_more_comments','zid','integer'), + ('event_ptpt_no_more_comments','pid','integer'), + ('event_ptpt_no_more_comments','votes_placed','smallint'), + ('event_ptpt_no_more_comments','created','bigint'), + ('stars','zid','integer'), + ('stars','pid','integer'), + ('stars','tid','integer'), + ('stars','starred','integer'), + ('stars','created','bigint'), + ('trashes','zid','integer'), + ('trashes','pid','integer'), + ('trashes','tid','integer'), + ('trashes','trashed','integer'), + ('trashes','created','bigint'), + ('permanentcookiezidjoins','zid','integer'), + ('permanentcookiezidjoins','cookie','character varying(32)'), + ('permanentcookiezidjoins','created','bigint'), + ('page_ids','site_id','character varying(100)'), + ('page_ids','page_id','character varying(100)'), + ('page_ids','zid','integer'), + ('demographic_data','uid','integer'), + ('demographic_data','fb_gender','integer'), + ('demographic_data','ms_birth_year_estimate_fb','integer'), + ('demographic_data','ms_gender_estimate_fb','integer'), + ('demographic_data','fb_timestamp','bigint'), + ('demographic_data','ms_fb_timestamp','bigint'), + ('demographic_data','ms_response','character varying(9999)'), + ('demographic_data','gender_guess','integer'), + ('demographic_data','birth_year_guess','integer')) AS expected(t,c,typ) WHERE NOT (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||(t)) AND a.attname=(c) AND NOT a.attisdropped + AND (format_type(a.atttypid,a.atttypmod)=(typ) + -- Exact public bootstrap alternatives to the authoritative legacy contract. + OR ((t)='worker_tasks' AND (c)='task_type' AND (typ)='text' + AND format_type(a.atttypid,a.atttypmod)='character varying(99)') + OR ((t) IN ('pwreset_tokens','password_reset_tokens') + AND (c) IN ('token','pwresettoken') AND (typ)='character varying(100)' + AND format_type(a.atttypid,a.atttypmod)='character varying(250)') + -- Only historical math payloads have an established json equivalent. + -- Do not adopt arbitrary json columns in newer jsonb contracts. + OR ((typ)='jsonb' AND a.atttypid='json'::regtype AND (c)='data' + AND (t) IN ('math_main','math_profile','math_ptptstats','math_cache', + 'math_bidtopid','math_exportstatus'))) + AND ((NULL) IS NULL OR a.attnotnull=(NULL)) + AND ((NULL) IS NULL OR pg_get_expr(d.adbin,d.adrelid)=(NULL))))) + AND NOT EXISTS (SELECT 1 FROM unnest(ARRAY['users','site_domain_whitelist','metrics','auth_tokens','jianiuevyew','apikeysndvweifu','courses','conversations','participant_metadata_questions','participant_metadata_answers','contexts','inviters','upvotes','oinvites','einvites','email_validations','zinvites','beta','participants','participants_extended','participant_locations','xids','xid_whitelist','notification_tasks','participant_metadata_choices','twitter_users','facebook_users','social_settings','facebook_friends','suzinvites','comments','comment_translations','conversation_translations','reports','report_comment_selections','worker_tasks','math_ticks','math_main','math_profile','math_ptptstats','math_cache','math_bidtopid','math_exportstatus','math_report_correlationmatrix','votes','votes_latest_unique','crowd_mod','event_ptpt_no_more_comments','stars','trashes','permanentcookiezidjoins','page_ids','demographic_data']) t WHERE to_regclass('public.'||t) IS NULL) + AND ((SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('pwreset_tokens')) AND a.attname=('token') AND NOT a.attisdropped + AND (format_type(a.atttypid,a.atttypmod)=('character varying(100)') + -- Exact public bootstrap alternatives to the authoritative legacy contract. + OR (('pwreset_tokens')='worker_tasks' AND ('token')='task_type' AND ('character varying(100)')='text' + AND format_type(a.atttypid,a.atttypmod)='character varying(99)') + OR (('pwreset_tokens') IN ('pwreset_tokens','password_reset_tokens') + AND ('token') IN ('token','pwresettoken') AND ('character varying(100)')='character varying(100)' + AND format_type(a.atttypid,a.atttypmod)='character varying(250)') + -- Only historical math payloads have an established json equivalent. + -- Do not adopt arbitrary json columns in newer jsonb contracts. + OR (('character varying(100)')='jsonb' AND a.atttypid='json'::regtype AND ('token')='data' + AND ('pwreset_tokens') IN ('math_main','math_profile','math_ptptstats','math_cache', + 'math_bidtopid','math_exportstatus'))) + AND ((NULL) IS NULL OR a.attnotnull=(NULL)) + AND ((NULL) IS NULL OR pg_get_expr(d.adbin,d.adrelid)=(NULL)))) OR (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('password_reset_tokens')) AND a.attname=('pwresettoken') AND NOT a.attisdropped + AND (format_type(a.atttypid,a.atttypmod)=('character varying(100)') + -- Exact public bootstrap alternatives to the authoritative legacy contract. + OR (('password_reset_tokens')='worker_tasks' AND ('pwresettoken')='task_type' AND ('character varying(100)')='text' + AND format_type(a.atttypid,a.atttypmod)='character varying(99)') + OR (('password_reset_tokens') IN ('pwreset_tokens','password_reset_tokens') + AND ('pwresettoken') IN ('token','pwresettoken') AND ('character varying(100)')='character varying(100)' + AND format_type(a.atttypid,a.atttypmod)='character varying(250)') + -- Only historical math payloads have an established json equivalent. + -- Do not adopt arbitrary json columns in newer jsonb contracts. + OR (('character varying(100)')='jsonb' AND a.atttypid='json'::regtype AND ('pwresettoken')='data' + AND ('password_reset_tokens') IN ('math_main','math_profile','math_ptptstats','math_cache', + 'math_bidtopid','math_exportstatus'))) + AND ((NULL) IS NULL OR a.attnotnull=(NULL)) + AND ((NULL) IS NULL OR pg_get_expr(d.adbin,d.adrelid)=(NULL))))) + AND ((SELECT to_regclass('public.'||('conversations')) IS NOT NULL AND NOT EXISTS(SELECT 1 FROM pg_attribute + WHERE attrelid=to_regclass('public.'||('conversations')) AND attname=('branding_type') AND NOT attisdropped)) OR (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('conversations')) AND a.attname=('branding_type') AND NOT a.attisdropped + AND (format_type(a.atttypid,a.atttypmod)=('integer') + -- Exact public bootstrap alternatives to the authoritative legacy contract. + OR (('conversations')='worker_tasks' AND ('branding_type')='task_type' AND ('integer')='text' + AND format_type(a.atttypid,a.atttypmod)='character varying(99)') + OR (('conversations') IN ('pwreset_tokens','password_reset_tokens') + AND ('branding_type') IN ('token','pwresettoken') AND ('integer')='character varying(100)' + AND format_type(a.atttypid,a.atttypmod)='character varying(250)') + -- Only historical math payloads have an established json equivalent. + -- Do not adopt arbitrary json columns in newer jsonb contracts. + OR (('integer')='jsonb' AND a.atttypid='json'::regtype AND ('branding_type')='data' + AND ('conversations') IN ('math_main','math_profile','math_ptptstats','math_cache', + 'math_bidtopid','math_exportstatus'))) + AND ((NULL) IS NULL OR a.attnotnull=(NULL)) + AND ((NULL) IS NULL OR pg_get_expr(d.adbin,d.adrelid)=(NULL))))) + AND ((SELECT to_regclass('public.'||('math_ticks')) IS NOT NULL AND NOT EXISTS(SELECT 1 FROM pg_attribute + WHERE attrelid=to_regclass('public.'||('math_ticks')) AND attname=('caching_tick') AND NOT attisdropped)) OR (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('math_ticks')) AND a.attname=('caching_tick') AND NOT a.attisdropped + AND (format_type(a.atttypid,a.atttypmod)=('bigint') + -- Exact public bootstrap alternatives to the authoritative legacy contract. + OR (('math_ticks')='worker_tasks' AND ('caching_tick')='task_type' AND ('bigint')='text' + AND format_type(a.atttypid,a.atttypmod)='character varying(99)') + OR (('math_ticks') IN ('pwreset_tokens','password_reset_tokens') + AND ('caching_tick') IN ('token','pwresettoken') AND ('bigint')='character varying(100)' + AND format_type(a.atttypid,a.atttypmod)='character varying(250)') + -- Only historical math payloads have an established json equivalent. + -- Do not adopt arbitrary json columns in newer jsonb contracts. + OR (('bigint')='jsonb' AND a.atttypid='json'::regtype AND ('caching_tick')='data' + AND ('math_ticks') IN ('math_main','math_profile','math_ptptstats','math_cache', + 'math_bidtopid','math_exportstatus'))) + AND ((NULL) IS NULL OR a.attnotnull=(NULL)) + AND ((NULL) IS NULL OR pg_get_expr(d.adbin,d.adrelid)=(NULL))))) + AND to_regprocedure('public.now_as_millis()') IS NOT NULL + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint WHERE conrelid=to_regclass('public.'||('votes_latest_unique')) + AND convalidated AND pg_get_constraintdef(oid)=('UNIQUE (zid, pid, tid)')))) AS catalog_match +UNION ALL +SELECT '000001_update_pwreset_table.sql'::text AS migration, (-- Catalog postconditions only; this file never replays migration DDL. +SELECT to_regclass('public.password_reset_tokens') IS NULL + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('pwreset_tokens')) AND a.attname=('token') AND NOT a.attisdropped + AND (format_type(a.atttypid,a.atttypmod)=('character varying(100)') + -- Exact public bootstrap alternatives to the authoritative legacy contract. + OR (('pwreset_tokens')='worker_tasks' AND ('token')='task_type' AND ('character varying(100)')='text' + AND format_type(a.atttypid,a.atttypmod)='character varying(99)') + OR (('pwreset_tokens') IN ('pwreset_tokens','password_reset_tokens') + AND ('token') IN ('token','pwresettoken') AND ('character varying(100)')='character varying(100)' + AND format_type(a.atttypid,a.atttypmod)='character varying(250)') + -- Only historical math payloads have an established json equivalent. + -- Do not adopt arbitrary json columns in newer jsonb contracts. + OR (('character varying(100)')='jsonb' AND a.atttypid='json'::regtype AND ('token')='data' + AND ('pwreset_tokens') IN ('math_main','math_profile','math_ptptstats','math_cache', + 'math_bidtopid','math_exportstatus'))) + AND ((NULL) IS NULL OR a.attnotnull=(NULL)) + AND ((NULL) IS NULL OR pg_get_expr(d.adbin,d.adrelid)=(NULL)))) + AND (SELECT to_regclass('public.'||('pwreset_tokens')) IS NOT NULL AND NOT EXISTS(SELECT 1 FROM pg_attribute + WHERE attrelid=to_regclass('public.'||('pwreset_tokens')) AND attname=('pwresettoken') AND NOT attisdropped))) AS catalog_match +UNION ALL +SELECT '000002_add_xid_constraint.sql'::text AS migration, (-- Both the public bootstrap variant (owner,xid only) and the authoritative +-- legacy variant (also owner,uid) are supported. Never drop either invariant. +SELECT (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('xids')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('UNIQUE (owner, xid)') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + -- If the optional legacy unique index exists, it must enforce the complete key. + AND NOT EXISTS ( + SELECT 1 FROM pg_index i + JOIN pg_attribute owner ON owner.attrelid=i.indrelid AND owner.attname='owner' + JOIN pg_attribute uid ON uid.attrelid=i.indrelid AND uid.attname='uid' + WHERE i.indrelid=to_regclass('public.xids') AND i.indisunique + AND i.indnkeyatts=2 + AND ARRAY[i.indkey[0],i.indkey[1]] @> ARRAY[owner.attnum,uid.attnum] + AND (NOT i.indisvalid OR NOT i.indisready OR NOT i.indislive + OR NOT i.indimmediate OR i.indpred IS NOT NULL) + )) AS catalog_match +UNION ALL +SELECT '000003_add_origin_permanent_cookie_columns.sql'::text AS migration, (-- Catalog postconditions only; this file never replays migration DDL. +SELECT (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('participants_extended')) AND a.attname=('permanent_cookie') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('character varying(32)') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('participants_extended')) AND a.attname=('origin') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('character varying(9999)') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL)))) AS catalog_match +UNION ALL +SELECT '000004_drop_waitinglist_table.sql'::text AS migration, (-- Catalog postconditions only; this file never replays migration DDL. +SELECT to_regclass('public.waitinglist') IS NULL) AS catalog_match +UNION ALL +SELECT '000005_drop_slack_stripe_canvas.sql'::text AS migration, (-- Catalog postconditions only; this file never replays migration DDL. +SELECT to_regclass('public.slack_oauth_access_tokens') IS NULL + AND to_regclass('public.slack_users') IS NULL + AND to_regclass('public.slack_user_invites') IS NULL + AND to_regclass('public.slack_bot_events') IS NULL + AND to_regclass('public.stripe_accounts') IS NULL + AND to_regclass('public.stripe_subscriptions') IS NULL + AND to_regclass('public.coupons_for_free_upgrades') IS NULL + AND to_regclass('public.lti_users') IS NULL + AND to_regclass('public.lti_context_memberships') IS NULL + AND to_regclass('public.canvas_assignment_callback_info') IS NULL + AND to_regclass('public.canvas_assignment_conversation_info') IS NULL + AND to_regclass('public.lti_oauthv1_credentials') IS NULL + AND (SELECT to_regclass('public.'||('conversations')) IS NOT NULL AND NOT EXISTS(SELECT 1 FROM pg_attribute + WHERE attrelid=to_regclass('public.'||('conversations')) AND attname=('is_slack') AND NOT attisdropped)) + AND (SELECT to_regclass('public.'||('conversations')) IS NOT NULL AND NOT EXISTS(SELECT 1 FROM pg_attribute + WHERE attrelid=to_regclass('public.'||('conversations')) AND attname=('lti_users_only') AND NOT attisdropped)) + AND (SELECT to_regclass('public.'||('users')) IS NOT NULL AND NOT EXISTS(SELECT 1 FROM pg_attribute + WHERE attrelid=to_regclass('public.'||('users')) AND attname=('plan') AND NOT attisdropped))) AS catalog_match +UNION ALL +SELECT '000006_update_votes_rule.sql'::text AS migration, (-- Catalog postconditions only; this file never replays migration DDL. +SELECT EXISTS (SELECT 1 FROM pg_rewrite WHERE ev_class=to_regclass('public.votes') AND rulename='on_vote_insert_update_unique_table' AND ev_enabled='O' AND regexp_replace(pg_get_ruledef(oid), '\s+', ' ', 'g') = 'CREATE RULE on_vote_insert_update_unique_table AS ON INSERT TO public.votes DO INSERT INTO votes_latest_unique (zid, pid, tid, vote, weight_x_32767, modified) VALUES (new.zid, new.pid, new.tid, new.vote, new.weight_x_32767, new.created) ON CONFLICT(zid, pid, tid) DO UPDATE SET vote = excluded.vote, modified = excluded.modified;')) AS catalog_match +UNION ALL +SELECT '000007_drop_geolocation_fields.sql'::text AS migration, (-- Catalog postconditions only; this file never replays migration DDL. +SELECT to_regclass('public.geolocation_cache') IS NULL + AND (SELECT to_regclass('public.'||('participants_extended')) IS NOT NULL AND NOT EXISTS(SELECT 1 FROM pg_attribute + WHERE attrelid=to_regclass('public.'||('participants_extended')) AND attname=('country_code_iso') AND NOT attisdropped)) + AND (SELECT to_regclass('public.'||('participants_extended')) IS NOT NULL AND NOT EXISTS(SELECT 1 FROM pg_attribute + WHERE attrelid=to_regclass('public.'||('participants_extended')) AND attname=('encrypted_maxmind_response_city') AND NOT attisdropped)) + AND (SELECT to_regclass('public.'||('participants_extended')) IS NOT NULL AND NOT EXISTS(SELECT 1 FROM pg_attribute + WHERE attrelid=to_regclass('public.'||('participants_extended')) AND attname=('ip_address') AND NOT attisdropped)) + AND (SELECT to_regclass('public.'||('participants_extended')) IS NOT NULL AND NOT EXISTS(SELECT 1 FROM pg_attribute + WHERE attrelid=to_regclass('public.'||('participants_extended')) AND attname=('latitude') AND NOT attisdropped)) + AND (SELECT to_regclass('public.'||('participants_extended')) IS NOT NULL AND NOT EXISTS(SELECT 1 FROM pg_attribute + WHERE attrelid=to_regclass('public.'||('participants_extended')) AND attname=('location') AND NOT attisdropped)) + AND (SELECT to_regclass('public.'||('participants_extended')) IS NOT NULL AND NOT EXISTS(SELECT 1 FROM pg_attribute + WHERE attrelid=to_regclass('public.'||('participants_extended')) AND attname=('longitude') AND NOT attisdropped)) + AND (SELECT to_regclass('public.'||('participants_extended')) IS NOT NULL AND NOT EXISTS(SELECT 1 FROM pg_attribute + WHERE attrelid=to_regclass('public.'||('participants_extended')) AND attname=('x_forwarded_for') AND NOT attisdropped))) AS catalog_match +UNION ALL +SELECT '000008_add_comment_priority.sql'::text AS migration, (-- Catalog postconditions only; this file never replays migration DDL. +SELECT (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('conversations')) AND a.attname=('importance_enabled') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('boolean') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('false'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('votes')) AND a.attname=('high_priority') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('boolean') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('false')))) AS catalog_match +UNION ALL +SELECT '000009_add_uuid_to_zinvites.sql'::text AS migration, (-- Catalog postconditions only; this file never replays migration DDL. +SELECT (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('zinvites')) AND a.attname=('uuid') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('uuid') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL)))) AS catalog_match +UNION ALL +SELECT '000010_create_oidc_user_mappings.sql'::text AS migration, (-- Catalog postconditions only; this file never replays migration DDL. +SELECT (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('oidc_user_mappings')) AND a.attname=('oidc_sub') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('character varying(255)') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('oidc_user_mappings')) AND a.attname=('uid') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('oidc_user_mappings')) AND a.attname=('created') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('bigint') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('now_as_millis()'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('oidc_user_mappings')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('PRIMARY KEY (oidc_sub)') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('oidc_user_mappings')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('UNIQUE (uid)') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('oidc_user_mappings')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('FOREIGN KEY (uid) REFERENCES users(uid) ON DELETE CASCADE') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_oidc_mappings_uid')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_oidc_mappings_uid ON public.oidc_user_mappings USING btree (uid)')))) AS catalog_match +UNION ALL +SELECT '000011_alter_suzinvites_xid_to_text.sql'::text AS migration, (-- Catalog postconditions only; this file never replays migration DDL. +SELECT (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('suzinvites')) AND a.attname=('xid') AND NOT a.attisdropped + AND (format_type(a.atttypid,a.atttypmod)=('text') + -- Exact public bootstrap alternatives to the authoritative legacy contract. + OR (('suzinvites')='worker_tasks' AND ('xid')='task_type' AND ('text')='text' + AND format_type(a.atttypid,a.atttypmod)='character varying(99)') + OR (('suzinvites') IN ('pwreset_tokens','password_reset_tokens') + AND ('xid') IN ('token','pwresettoken') AND ('text')='character varying(100)' + AND format_type(a.atttypid,a.atttypmod)='character varying(250)') + -- Only historical math payloads have an established json equivalent. + -- Do not adopt arbitrary json columns in newer jsonb contracts. + OR (('text')='jsonb' AND a.atttypid='json'::regtype AND ('xid')='data' + AND ('suzinvites') IN ('math_main','math_profile','math_ptptstats','math_cache', + 'math_bidtopid','math_exportstatus'))) + AND ((true) IS NULL OR a.attnotnull=(true)) + AND ((NULL) IS NULL OR pg_get_expr(d.adbin,d.adrelid)=(NULL))))) AS catalog_match +UNION ALL +SELECT '000012_create_topic_agenda_selections.sql'::text AS migration, (-- Catalog postconditions only; this file never replays migration DDL. +SELECT (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('topic_agenda_selections')) AND a.attname=('zid') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('topic_agenda_selections')) AND a.attname=('pid') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('topic_agenda_selections')) AND a.attname=('archetypal_selections') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('jsonb') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('''[]''::jsonb'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('topic_agenda_selections')) AND a.attname=('delphi_job_id') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('text') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('topic_agenda_selections')) AND a.attname=('total_selections') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('0'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('topic_agenda_selections')) AND a.attname=('created_at') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('timestamp with time zone') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('CURRENT_TIMESTAMP'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('topic_agenda_selections')) AND a.attname=('updated_at') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('timestamp with time zone') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('CURRENT_TIMESTAMP'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('topic_agenda_selections')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('PRIMARY KEY (zid, pid)') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('topic_agenda_selections')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('FOREIGN KEY (zid) REFERENCES conversations(zid) ON DELETE CASCADE') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('topic_agenda_selections')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('FOREIGN KEY (zid, pid) REFERENCES participants(zid, pid) ON DELETE CASCADE') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_topic_agenda_selections_zid')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_topic_agenda_selections_zid ON public.topic_agenda_selections USING btree (zid)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_topic_agenda_selections_pid')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_topic_agenda_selections_pid ON public.topic_agenda_selections USING btree (pid)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_topic_agenda_selections_delphi_job_id')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_topic_agenda_selections_delphi_job_id ON public.topic_agenda_selections USING btree (delphi_job_id)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_topic_agenda_selections_created_at')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_topic_agenda_selections_created_at ON public.topic_agenda_selections USING btree (created_at)')))) AS catalog_match +UNION ALL +SELECT '000013_create_treevite.sql'::text AS migration, (-- Catalog postconditions only; this file never replays migration DDL. +SELECT (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('conversations')) AND a.attname=('treevite_enabled') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('boolean') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('false'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_waves')) AND a.attname=('id') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('bigint') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('nextval(''treevite_waves_id_seq''::regclass)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_waves')) AND a.attname=('zid') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_waves')) AND a.attname=('wave') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_waves')) AND a.attname=('parent_wave') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_waves')) AND a.attname=('size') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_waves')) AND a.attname=('invites_per_user') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_waves')) AND a.attname=('owner_invites') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('0'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_waves')) AND a.attname=('created_at') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('timestamp with time zone') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('CURRENT_TIMESTAMP'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_waves')) AND a.attname=('updated_at') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('timestamp with time zone') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('CURRENT_TIMESTAMP'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_waves')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('PRIMARY KEY (id)') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_invites')) AND a.attname=('id') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('bigint') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('nextval(''treevite_invites_id_seq''::regclass)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_invites')) AND a.attname=('zid') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_invites')) AND a.attname=('wave_id') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('bigint') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_invites')) AND a.attname=('parent_invite_id') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('bigint') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_invites')) AND a.attname=('status') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('smallint') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('0'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_invites')) AND a.attname=('invite_owner_pid') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_invites')) AND a.attname=('invite_used_by_pid') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_invites')) AND a.attname=('invite_used_at') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('timestamp with time zone') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_invites')) AND a.attname=('created_at') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('timestamp with time zone') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('CURRENT_TIMESTAMP'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_invites')) AND a.attname=('updated_at') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('timestamp with time zone') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('CURRENT_TIMESTAMP'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_invites')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('PRIMARY KEY (id)') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_login_codes')) AND a.attname=('id') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('bigint') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('nextval(''treevite_login_codes_id_seq''::regclass)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_login_codes')) AND a.attname=('zid') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_login_codes')) AND a.attname=('pid') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_login_codes')) AND a.attname=('login_code_hash') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('text') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_login_codes')) AND a.attname=('fp_kid') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('smallint') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('1'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_login_codes')) AND a.attname=('revoked') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('boolean') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('false'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_login_codes')) AND a.attname=('expires_at') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('timestamp with time zone') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_login_codes')) AND a.attname=('last_used_at') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('timestamp with time zone') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_login_codes')) AND a.attname=('created_at') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('timestamp with time zone') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('CURRENT_TIMESTAMP'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_login_codes')) AND a.attname=('updated_at') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('timestamp with time zone') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('CURRENT_TIMESTAMP'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_login_codes')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('PRIMARY KEY (id)') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_waves')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('UNIQUE (zid, wave)') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_waves')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('FOREIGN KEY (zid) REFERENCES conversations(zid) ON DELETE CASCADE') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_invites')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('UNIQUE (zid, invite_code)') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_invites')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('FOREIGN KEY (zid) REFERENCES conversations(zid) ON DELETE CASCADE') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_invites')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('FOREIGN KEY (wave_id) REFERENCES treevite_waves(id) ON DELETE CASCADE') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_invites')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('FOREIGN KEY (parent_invite_id) REFERENCES treevite_invites(id) ON DELETE SET NULL') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_invites')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('FOREIGN KEY (zid, invite_owner_pid) REFERENCES participants(zid, pid)') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_invites')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('FOREIGN KEY (zid, invite_used_by_pid) REFERENCES participants(zid, pid)') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_login_codes')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('FOREIGN KEY (zid, pid) REFERENCES participants(zid, pid) ON DELETE CASCADE') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_login_codes')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('UNIQUE (zid, pid)') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_login_codes')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('UNIQUE (zid, login_code_fingerprint)') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_login_codes')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('UNIQUE (zid, login_code_lookup)') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_treevite_waves_zid')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_treevite_waves_zid ON public.treevite_waves USING btree (zid)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_treevite_waves_wave')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_treevite_waves_wave ON public.treevite_waves USING btree (wave)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_treevite_waves_parent')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_treevite_waves_parent ON public.treevite_waves USING btree (zid, parent_wave)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_treevite_invites_zid')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_treevite_invites_zid ON public.treevite_invites USING btree (zid)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_treevite_invites_zid_status')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_treevite_invites_zid_status ON public.treevite_invites USING btree (zid, status)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_treevite_invites_wave_id')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_treevite_invites_wave_id ON public.treevite_invites USING btree (wave_id)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_treevite_invites_parent')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_treevite_invites_parent ON public.treevite_invites USING btree (parent_invite_id)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_treevite_invites_owner_pid')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_treevite_invites_owner_pid ON public.treevite_invites USING btree (invite_owner_pid)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_treevite_invites_used_by_pid')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_treevite_invites_used_by_pid ON public.treevite_invites USING btree (invite_used_by_pid)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_treevite_invites_code')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_treevite_invites_code ON public.treevite_invites USING btree (invite_code)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_treevite_login_codes_zid')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_treevite_login_codes_zid ON public.treevite_login_codes USING btree (zid)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_treevite_login_codes_pid')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_treevite_login_codes_pid ON public.treevite_login_codes USING btree (pid)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_treevite_login_codes_fp')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_treevite_login_codes_fp ON public.treevite_login_codes USING btree (login_code_fingerprint)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_treevite_login_codes_lookup')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_treevite_login_codes_lookup ON public.treevite_login_codes USING btree (zid, login_code_lookup)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_invites')) AND a.attname=('invite_code') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('character varying(64)') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_login_codes')) AND a.attname=('login_code_fingerprint') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('character varying(128)') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_login_codes')) AND a.attname=('login_code_lookup') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('character varying(128)') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_waves')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('CHECK (((invites_per_user > 0) OR (owner_invites > 0)))') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_waves')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('CHECK (((parent_wave IS NULL) OR (parent_wave >= 0)))') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_waves')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('CHECK (((size IS NULL) OR (size >= 0)))') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_waves')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('CHECK ((invites_per_user >= 0))') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_waves')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('CHECK ((owner_invites >= 0))') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_waves')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('CHECK ((wave >= 1))') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_invites')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('CHECK ((status = ANY (ARRAY[0, 1, 2, 3])))') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O')))) AS catalog_match +UNION ALL +SELECT '000014_alter_reports_modlevel.sql'::text AS migration, (-- Catalog postconditions only; this file never replays migration DDL. +SELECT (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('reports')) AND a.attname=('mod_level') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('smallint') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('''-2''::integer')))) AS catalog_match +UNION ALL +SELECT '000015_add_xid_requirements.sql'::text AS migration, (-- Catalog postconditions only; this file never replays migration DDL. +SELECT (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('conversations')) AND a.attname=('xid_required') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('boolean') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('false'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('xid_whitelist')) AND a.attname=('zid') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('xids')) AND a.attname=('zid') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('xids')) AND a.attname=('pid') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('xid_whitelist')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('FOREIGN KEY (zid) REFERENCES conversations(zid) ON DELETE CASCADE') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('xids')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('FOREIGN KEY (zid) REFERENCES conversations(zid) ON DELETE CASCADE') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('xids')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('FOREIGN KEY (zid, pid) REFERENCES participants(zid, pid) ON DELETE SET NULL') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_xid_whitelist_zid')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_xid_whitelist_zid ON public.xid_whitelist USING btree (zid)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_xid_whitelist_xid')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_xid_whitelist_xid ON public.xid_whitelist USING btree (xid)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_xids_zid')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_xids_zid ON public.xids USING btree (zid)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_xids_xid')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_xids_xid ON public.xids USING btree (xid)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_xids_pid')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_xids_pid ON public.xids USING btree (pid)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_xids_zid_xid')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_xids_zid_xid ON public.xids USING btree (zid, xid)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_xids_uid_zid')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_xids_uid_zid ON public.xids USING btree (uid, zid)')))) AS catalog_match +UNION ALL +SELECT '000016_add_orig_id.sql'::text AS migration, (-- Catalog postconditions only; this file never replays migration DDL. +SELECT (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('comments')) AND a.attname=('original_id') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('uuid') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL)))) AS catalog_match +UNION ALL +SELECT '000017_create_byod_job_table.sql'::text AS migration, (-- Catalog postconditions only; this file never replays migration DDL. +SELECT (SELECT array_agg(enumlabel::text ORDER BY enumsortorder) FROM pg_enum WHERE enumtypid=to_regtype('public.job_status')) = ARRAY['pending','processing','completed','failed'] + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('byod_import_jobs')) AND a.attname=('id') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('nextval(''byod_import_jobs_id_seq''::regclass)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('byod_import_jobs')) AND a.attname=('zid') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('byod_import_jobs')) AND a.attname=('s3_key') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('text') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('byod_import_jobs')) AND a.attname=('status') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('job_status') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('''pending''::job_status'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('byod_import_jobs')) AND a.attname=('stage') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('text') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('''init''::text'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('byod_import_jobs')) AND a.attname=('error_message') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('text') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('byod_import_jobs')) AND a.attname=('created_at') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('timestamp with time zone') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('now()'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('byod_import_jobs')) AND a.attname=('updated_at') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('timestamp with time zone') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('now()'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('byod_import_jobs')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('PRIMARY KEY (id)') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_byod_jobs_zid')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_byod_jobs_zid ON public.byod_import_jobs USING btree (zid)')))) AS catalog_match +UNION ALL +SELECT '000018_add_topics_enabled.sql'::text AS migration, (-- Catalog postconditions only; this file never replays migration DDL. +SELECT (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('conversations')) AND a.attname=('topics_enabled') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('boolean') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('false')))) AS catalog_match +UNION ALL +SELECT '000022_add_poll_timestamp_indexes.sql'::text AS migration, (-- Catalog postconditions only; this file never replays migration DDL. +SELECT (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('votes_created_idx')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX votes_created_idx ON public.votes USING btree (created)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('comments_modified_idx')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX comments_modified_idx ON public.comments USING btree (modified)')))) AS catalog_match +), scope AS ( + SELECT to_regclass('public.migrations') IS NULL + AND to_regclass('public.schema_migrations') IS NULL + AND NOT EXISTS (SELECT 1 FROM pg_class WHERE relnamespace='public'::regnamespace + AND (starts_with(relname,'polis_queue_') OR starts_with(relname,'delphi_'))) + AND NOT EXISTS (SELECT 1 FROM pg_proc WHERE pronamespace='public'::regnamespace + AND (starts_with(proname,'pq_') OR starts_with(proname,'pd_'))) AS initial_state +), authority AS ( + SELECT + has_database_privilege(current_user,current_database(),'TEMP') + AND has_schema_privilege(current_user,'public','USAGE') + AND has_schema_privilege(current_user,'public','CREATE') AS reconcile_authority, + EXISTS (SELECT 1 FROM pg_roles WHERE rolname=current_user AND (rolsuper OR rolcreaterole)) AS create_roles, + NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname IN ('polis_queue_owner','polis_queue_executor')) AS queue_roles_absent, + has_schema_privilege(current_user,'public','USAGE WITH GRANT OPTION') + AND has_schema_privilege(current_user,'public','CREATE WITH GRANT OPTION') AS schema_grants, + EXISTS (SELECT 1 FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace + WHERE n.nspname='public' AND c.relname='conversations' + AND has_table_privilege(current_user,c.oid,'SELECT WITH GRANT OPTION') + AND has_column_privilege(current_user,c.oid,'topic','UPDATE WITH GRANT OPTION') + AND has_column_privilege(current_user,c.oid,'zid','REFERENCES WITH GRANT OPTION')) AS table_grants +), readiness AS ( + SELECT initial_state AND (SELECT bool_and(catalog_match) FROM checks) + AND current_setting('server_version_num')::integer >= 170000 AS can_reconcile, + create_roles AND queue_roles_absent AND schema_grants AND table_grants AS can_provision, + initial_state, reconcile_authority, create_roles, queue_roles_absent, schema_grants, table_grants + FROM scope CROSS JOIN authority +), output AS ( + SELECT migration, catalog_match, + CASE WHEN NOT initial_state THEN 'REVIEW_EXISTING_LEDGER_OR_QUEUE' + WHEN NOT catalog_match THEN 'CATALOG_MISMATCH' + WHEN NOT can_reconcile THEN 'BLOCKED_BY_OTHER_CATALOG_CHECK' + WHEN NOT reconcile_authority THEN 'REVIEW_MIGRATION_SESSION_AUTHORITY' + ELSE 'WOULD_ADOPT' END AS outcome + FROM checks CROSS JOIN readiness + UNION ALL + SELECT migration, NULL::boolean, + CASE WHEN NOT can_reconcile THEN 'BLOCKED_RECONCILIATION' + WHEN NOT reconcile_authority OR NOT can_provision THEN 'REVIEW_MIGRATION_SESSION_AUTHORITY' + ELSE 'WOULD_APPLY' END + FROM (VALUES ('000019_create_polis_queue.sql'),('000023_create_delphi_foundation.sql'), + ('000024_create_polis_queue_large_class.sql')) p(migration) CROSS JOIN readiness + UNION ALL + SELECT migration, NULL::boolean, 'OUTSIDE_RELEASE' + FROM (VALUES ('000020'),('000021'),('000025'),('000026')) p(migration) +) +SELECT migration, catalog_match, outcome FROM output ORDER BY migration; +ROLLBACK; diff --git a/server/src/db/migrations.cjs b/server/src/db/migrations.cjs new file mode 100644 index 0000000000..d7e6091776 --- /dev/null +++ b/server/src/db/migrations.cjs @@ -0,0 +1,62 @@ +/* eslint-disable no-restricted-properties -- Bootstrap must read its environment before config.ts and application modules load. */ +// Read-only startup gate. Uses the same raw SQL hashes and release-wide hold +// as polis-migrate; no application modules or background loops load before it. +const fs = require("node:fs"); +const path = require("node:path"); +const crypto = require("node:crypto"); +const { Client } = require("pg"); +const isTrue = require("boolean"); + +async function checkMigrations() { + const dir = process.env.POLIS_MIGRATIONS_DIR || path.resolve("postgres/migrations"); + function manifest(file) { + const names = new Set(); + for (const name of fs.readFileSync(path.join(dir, file), "utf8").split(/\r?\n/).filter((s) => s && !s.startsWith("#"))) { + if (name.trim() !== name || !/^\d{6}_[a-zA-Z0-9_]+\.sql$/.test(name) || names.has(name)) throw new Error(`Invalid or duplicate migration in ${file}: ${name}`); + names.add(name); + } + return names; + } + const selected = manifest("release.txt"); + const held = manifest("held.txt"); + if ([...selected].some((name) => held.has(name))) throw new Error("Release and held manifests overlap"); + const required = new Map(); + const versions = new Set(); + for (const name of fs.readdirSync(dir).sort()) { + if (!name.endsWith(".sql")) continue; + if (name.trim() !== name || !/^\d{6}_[a-zA-Z0-9_]+\.sql$/.test(name) || versions.has(name.slice(0, 6))) { + throw new Error(`Invalid or duplicate migration: ${name}`); + } + versions.add(name.slice(0, 6)); + if (!fs.lstatSync(path.join(dir, name)).isFile()) throw new Error(`Non-file migration: ${name}`); + if (held.has(name)) { held.delete(name); continue; } + if (!selected.delete(name)) throw new Error(`Migration absent from release/held manifest: ${name}`); + required.set(name, crypto.createHash("sha256").update(fs.readFileSync(path.join(dir, name))).digest("hex")); + } + if (held.size || selected.size || !required.has("000000_initial.sql")) throw new Error("Incomplete migration source directory"); + if (!process.env.DATABASE_URL) throw new Error("DATABASE_URL is required for migration check"); + const db = new Client({ connectionString: process.env.DATABASE_URL, + connectionTimeoutMillis: 10000, application_name: "polis-startup-migrations", + ssl: isTrue(process.env.DATABASE_SSL) ? { rejectUnauthorized: true } : undefined }); + try { + await db.connect(); + await db.query("BEGIN READ ONLY"); + await db.query("SET LOCAL statement_timeout='10s'; SET LOCAL lock_timeout='5s'"); + const { rows } = await db.query("SELECT name,checksum,status FROM public.migrations ORDER BY name"); + for (const row of rows) { + if (!required.has(row.name) || required.get(row.name) !== row.checksum || !["APPLIED", "ADOPTED"].includes(row.status)) { + throw new Error(`Migration history mismatch: ${row.name}`); + } + required.delete(row.name); + } + if (required.size) throw new Error(`Pending migrations: ${[...required.keys()].join(", ")}; run polis-migrate apply`); + await db.query("COMMIT"); + } catch (error) { + // Database DETAIL may contain data. Print only our own errors or SQLSTATE. + if (error.code) throw new Error(`Migration check failed (SQLSTATE ${error.code}); run polis-migrate check; reconcile legacy databases per docs/migrations.md`); + throw error; + } finally { await db.end(); } +} +module.exports = { checkMigrations }; +if (require.main === module) checkMigrations().then(() => process.stdout.write("migration check ready\n")) + .catch((error) => { process.stderr.write(`${error.message}\n`); process.exitCode = 1; });