diff --git a/.github/workflows/cypress-tests.yml b/.github/workflows/cypress-tests.yml index bb7313c280..2272503842 100644 --- a/.github/workflows/cypress-tests.yml +++ b/.github/workflows/cypress-tests.yml @@ -85,6 +85,9 @@ jobs: # Build remaining services (caching OK for these) docker compose -f docker-compose.test.yml --env-file test.env build + - name: Apply and check test database migrations + run: bash ci/test-migrations.sh + - name: Start services run: | # Start all services in detached mode @@ -97,24 +100,6 @@ jobs: # Show running containers docker compose -f docker-compose.test.yml ps - - name: Run Database Migrations - env: - DATABASE_URL: postgres://postgres:PdwPNS2mDN73Vfbc@localhost:5432/polis-test - POSTGRES_DB: polis-test - POSTGRES_HOST: postgres:5432 - POSTGRES_PASSWORD: PdwPNS2mDN73Vfbc - POSTGRES_PORT: 5432 - POSTGRES_USER: postgres - run: | - echo "Installing postgres-client..." - sudo apt-get update && sudo apt-get install -y postgresql-client - - echo "Making migration script executable..." - chmod +x server/bin/run-migrations.sh - - echo "Running migrations..." - ./server/bin/run-migrations.sh - - name: Check service health run: | # Check if key services are responding diff --git a/.github/workflows/jest-server-test.yml b/.github/workflows/jest-server-test.yml index fe54f8593e..0b7122b224 100644 --- a/.github/workflows/jest-server-test.yml +++ b/.github/workflows/jest-server-test.yml @@ -97,6 +97,9 @@ jobs: docker compose -f docker-compose.test.yml --env-file test.env build \ postgres file-server ses-local oidc-simulator dynamodb + - name: Apply and check test database migrations + run: bash ci/test-migrations.sh + - name: Start services run: | # Start only required services in detached mode (exclude server + math-python) @@ -144,6 +147,13 @@ jobs: cd server npm run contract:check + - name: Check API migration readiness + working-directory: server + env: + DATABASE_URL: postgres://postgres:PdwPNS2mDN73Vfbc@localhost:5432/polis-test + DATABASE_SSL: "false" + run: node src/db/migrations.cjs + - name: Run server integration tests run: | cd server diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index ccdd364b7e..39508bc708 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -17,6 +17,11 @@ on: - 'scripts/test-deploy-hooks.sh' - 'docker-compose*.yml' - '.github/workflows/python-ci.yml' + - 'ci/test-migrations.sh' + - 'docker-compose.test.yml' + - 'server/Dockerfile-db' + - 'server/postgres/**' + - 'queue-rs/polis-migrate/**' # server/src so a new server-side wildcard runs the projection-gate sweep - 'server/src/**' # Representative payload tests execute the probe planner and independent gate. @@ -75,9 +80,9 @@ jobs: # Build all services in the test file (including delphi) docker compose -f docker-compose.test.yml --env-file .env build - - name: 4. Start all services - run: | - # Start all services (including delphi) in detached mode + # Apply/check before starting application services on this volume. + bash ci/test-migrations.sh + # The 'delphi' container will start and run 'tail -f /dev/null' docker compose -f docker-compose.test.yml --env-file .env up -d @@ -95,8 +100,7 @@ jobs: # The opt-in Postgres integration tests (require_polis_postgres) run here # against the compose `postgres` service, whose image bakes the polis - # migrations (server/postgres/migrations/*.sql via docker-entrypoint-initdb.d), - # so the votes / votes_latest_unique schema + on_vote_insert_update_unique_table + # migrations through polis-migrate during initialization and the explicit CI step, # rule are already applied. The pytest step exports POLIS_TEST_POSTGRES_URL; # the cold-start generator under test is already baked into the delphi image # (Dockerfile `COPY scripts/ ./scripts/`), built from this checkout. diff --git a/.github/workflows/queue-rs-ci.yml b/.github/workflows/queue-rs-ci.yml index 2c9e166aed..1adcf1f13c 100644 --- a/.github/workflows/queue-rs-ci.yml +++ b/.github/workflows/queue-rs-ci.yml @@ -9,6 +9,17 @@ on: pull_request: paths: - 'queue-rs/**' + - 'delphi/scripts/job_graph_*.py' + - 'delphi/tests/job_graph/**' + - 'docs/job-graphs.md' + - 'server/__tests__/integration/queue-substrate.test.ts' + - 'server/jest.job-graphs.config.ts' + - 'server/package*.json' + - 'server/index.ts' + - 'server/src/db/migrations.cjs' + - 'server/Dockerfile-db' + - 'server/postgres/init-migrations.sh' + - 'scripts/after_install.sh' - 'file-server/nginx/**' - 'file-server/nginx.Dockerfile' - 'server/postgres/migrations/**' @@ -17,6 +28,17 @@ on: branches: [edge, stable] paths: - 'queue-rs/**' + - 'delphi/scripts/job_graph_*.py' + - 'delphi/tests/job_graph/**' + - 'docs/job-graphs.md' + - 'server/__tests__/integration/queue-substrate.test.ts' + - 'server/jest.job-graphs.config.ts' + - 'server/package*.json' + - 'server/index.ts' + - 'server/src/db/migrations.cjs' + - 'server/Dockerfile-db' + - 'server/postgres/init-migrations.sh' + - 'scripts/after_install.sh' - 'file-server/nginx/**' - 'file-server/nginx.Dockerfile' - 'server/postgres/migrations/**' @@ -110,3 +132,73 @@ jobs: - name: nginx routing switch (off, absent/up/502/dead, bad names and settings, method and body gate, truncation limit) working-directory: . run: queue-rs/polis-api/conformance/nginx-routing.sh + + polis-migrate: + name: migration runner and startup refusal + runs-on: ubuntu-24.04 + timeout-minutes: 30 + env: + COMPOSE_PROJECT_NAME: polis-migrate-test-ci-${{ github.run_id }}-${{ github.run_attempt }} + POLIS_RECOVERY_PG_PORT: '55850' + RECOVERY_PG_PORT: '55850' + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + persist-credentials: false + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 + with: + node-version: '22.23.3' + - name: Build and lint the runner + working-directory: queue-rs + run: | + cargo fmt -p polis-migrate --check + cargo clippy --locked -p polis-migrate --all-targets -- -D warnings + cargo test --locked -p polis-migrate + cargo build --locked -p polis-migrate + - name: Compile the API entrypoint + working-directory: server + run: | + npm ci --ignore-scripts --no-audit --no-fund + npm run build + - name: Isolated real PostgreSQL proofs + run: | + docker compose -f queue-rs/polis-migrate/tests/compose.yml up -d --wait + python3 server/bin/build-migration-report.py --check + python3 queue-rs/polis-migrate/tests/prove.py + python3 queue-rs/polis-migrate/tests/indexes.py + python3 queue-rs/polis-migrate/tests/adoption.py + python3 queue-rs/polis-migrate/tests/selection.py + python3 queue-rs/polis-migrate/tests/lexer.py + - name: Prove the fresh database image and restart + run: bash queue-rs/polis-migrate/tests/fresh-image.sh + - name: Remove only the owned test project + if: always() + run: docker compose -f queue-rs/polis-migrate/tests/compose.yml down -v + + job-graphs: + name: Per-step job graph core + runs-on: ubuntu-24.04 + timeout-minutes: 30 + env: + COMPOSE_PROJECT_NAME: polis-graph-test-ci-${{ github.run_id }}-${{ github.run_attempt }} + POLIS_RECOVERY_PG_PORT: '55467' + RECOVERY_PG_PORT: '55467' + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + - uses: actions/setup-node@v4 + with: + node-version: 24 + - name: Install Node adapter test dependencies + working-directory: server + run: npm ci --no-audit --no-fund + - name: Prove saved work, retries and complete publication + run: bash delphi/tests/job_graph/run.sh + - name: Retain generated proof receipts + if: always() + uses: actions/upload-artifact@v4 + with: + name: job-graph-proof + path: graph-proof/ + retention-days: 7 diff --git a/CHANGELOG.md b/CHANGELOG.md index 2995ba5a3f..ba2edcccd7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## Release B (unreleased) + +- Deploy reconciles the supported existing schema, applies selected migrations, + and checks readiness before replacing services; Release A hooks must already + be installed on each host. +- Select M19/M23/M24 and the trimmed M27 job graph core. Each step keeps its + result and inputs; dependencies wait, failed steps retry, and complete results + publish atomically. DynamoDB readers/importers remain outside this release. +- Refuse API startup when selected migration receipts are missing or changed. + ## 1.0 @@ -31,3 +41,20 @@ Changes which have been merged to `edge` but are not yet versioned on `stable` c * ... + +## Migration runner + +Deployments now apply pending numbered migrations through `polis-migrate` before +service replacement. The API refuses startup with pending or mismatched history. +Existing databases require one-time catalog-checked adoption. See +[upgrading](docs/upgrading.md) for the first CodeDeploy hook transition and the +release-wide coordinator hold. PostgreSQL 17+ is required. + +The explicit release manifest admits the supported legacy schema then applies +M19/M23/M24; M20/M21/M25/M26 stay outside the forward path. Deprecated M4/M5/M7 +are observation-only, never automatic destructive steps. Exact legacy type +alternatives, a catalog-only first-deploy report, per-deployment upgrade notes +and a source-to-release map are documented in +[migration upgrade notes](docs/migration-upgrade-notes.md). Historical SQL +checksums remain unchanged; no semantic version is invented for unversioned +historical releases. diff --git a/Makefile b/Makefile index 7fbf8ff6ea..23ff6b5650 100644 --- a/Makefile +++ b/Makefile @@ -177,8 +177,7 @@ refresh-devdb: ## Force dev DB mode (migrations), drop postgres_data volume, and # P-022 §C — poller recovery matrix (R01-R12) on a REAL Postgres # ---------------------------------------------------------------------------- # # Reuses docker-compose.test.yml's postgres service (built from -# server/Dockerfile-db, which bakes server/postgres/migrations/*.sql into -# docker-entrypoint-initdb.d) with docker-compose.recovery.yml overriding the +# server/Dockerfile-db, which runs polis-migrate during fresh initialization) with docker-compose.recovery.yml overriding the # host port and making the data directory a tmpfs, so every `up` re-runs initdb # with the real migrations and nothing survives teardown. # diff --git a/bin/run-migrations.clj b/bin/run-migrations.clj index 67bdb65d0d..72b875d1a4 100755 --- a/bin/run-migrations.clj +++ b/bin/run-migrations.clj @@ -1,143 +1,5 @@ #!/usr/bin/env bb - -(require '[babashka.pods :as pods] - '[babashka.deps :as deps] - '[clojure.pprint :as pp] - '[clojure.tools.cli :as cli] - '[clojure.java.io :as io] - '[clojure.string :as string]) - -(pods/load-pod 'org.babashka/postgresql "0.0.1") -(deps/add-deps '{:deps {honeysql/honeysql {:mvn/version "1.0.444"}}}) - -(require '[pod.babashka.postgresql :as pg] - '[honeysql.core :as hsql] - '[honeysql.helpers :as hsqlh]) - - - -(def db-url - (System/getenv "DATABASE_URL")) - -(defn heroku-url-spec [db-url] - (let [[_ user password host port db] (re-matches #"postgres://(?:(.+):(.*)@)?([^:]+)(?::(\d+))?/(.+)" db-url)] - {:dbtype "postgresql" - :host host - :dbname db - :port (or port 80) - :user user - :password password})) - -(defn execute-sql! [args] - (println "Executing sql:" args) - (pg/execute! - (heroku-url-spec (System/getenv "DATABASE_URL")) - args)) - - -;(def execute-sql! - ;(partial pg/execute! (heroku-url-spec (System/getenv "DATABASE_URL")))) - -(defn execute! - [query-or-command] - (execute-sql! (hsql/format query-or-command))) - - -(defn insert! - [table values] - (execute! {:insert-into table - :values values})) - -;(hsqlh/values [{:a "this" :b 4}]) - -;(-> (hsqlh/insert-into :migrations) - ;(hsqlh/values [{:a "this" :b 3}]) - ;(hsql/format)) - - - -;; get about the migration business - -(def migrations-path "server/postgres/migrations/") - -(defn sql-file? - [file] - (re-matches #".*\.sql" (str file))) - -(defn table-exists? - [table-name] - (-> - (execute! {:select [:*] - :from [:information_schema.tables] - :where [:= :table_name (name table-name)]}) - (not-empty) - (boolean))) - -;(table-exists? :migrations) -;(table-exists? :fish) - -(defn remember-tx-migration! [name] - (insert! :migrations - [{:name name - :completed_at (System/currentTimeMillis)}])) - -;; Make sure we have a migrations table, which is basically just a list of filenames which have been -;; transacted, as well as datetime -(when-not (table-exists? :migrations) - (execute-sql! - ["CREATE TABLE migrations - (name VARCHAR(999) NOT NULL, - completed_at BIGINT NOT NULL);"])) - -(defn process-mig-file! [mig-file] - (let [mig-file (io/file mig-file) - name (.getName mig-file)] - (println "Processing migration file" name) - (execute-sql! [(slurp mig-file)]) - (remember-tx-migration! name))) - -(defn migration-files [] - (->> (.listFiles (io/file migrations-path)) - (remove #(.isDirectory %)) - (filter sql-file?) - (sort))) - -(defn remove-past-migrations - [mig-files] - (let [past-migrations - (->> - (execute! {:select [:name] - :from [:migrations]}) - (map :migrations/name) - (set))] - (remove (comp past-migrations #(.getName %)) - mig-files))) - -(defn new-migration-files - [] - (remove-past-migrations (migration-files))) - -;(remove-past-migrations (migration-files)) - -;(.getName (io/file "server/postgres/migrations/000000_initial.sql")) -;(.getParent (io/file "server/postgres/migrations/000000_initial.sql")) - -(when-not (table-exists? :conversations) - (process-mig-file! "server/postgres/migrations/000000_initial.sql") - (remember-tx-migration! "000000_initial.sql")) - -(when-not (table-exists? :pwreset_tokens) - (process-mig-file! "server/postgres/migrations/000001_update_pwreset_table.sql") - (remember-tx-migration! "000001_update_pwreset_table.sql")) - -(def past-migrations - (execute! {:select [:*] - :from [:migrations]})) - -(let [mig-files (new-migration-files)] - (if (empty? mig-files) - (println "No new migrations to run") - (doseq [mig-file (new-migration-files)] - (process-mig-file! mig-file)))) - - +;; The 2021 ledger idea continues in polis-migrate. Do not replay all SQL here. +(require '[babashka.process :as process]) +(let [result @(process/process ["bash" "server/bin/run-migrations.sh"] {:inherit true})] + (System/exit (:exit result))) diff --git a/ci/p027_rerecord_build.sh b/ci/p027_rerecord_build.sh index 33eec4134f..1c4394ca63 100644 --- a/ci/p027_rerecord_build.sh +++ b/ci/p027_rerecord_build.sh @@ -6,7 +6,7 @@ export DOCKER_BUILDKIT=1 # Sequential builds bound peak memory; inspect disk receipts when sizing the runner. df -h . docker build --target prod --build-arg NODE_ENV=production -t p027-server -f server/Dockerfile server -docker build -t p027-postgres -f server/Dockerfile-db server +docker build --build-context queue-rs=queue-rs -t p027-postgres -f server/Dockerfile-db server docker build -t p027-oidc-simulator oidc-simulator docker build -t p027-file-server --build-arg NODE_ENV=production \ --build-arg AUTH_AUDIENCE=users --build-arg AUTH_CLIENT_ID=dev-client-id \ diff --git a/ci/test-migrations.sh b/ci/test-migrations.sh new file mode 100644 index 0000000000..fb92f902e8 --- /dev/null +++ b/ci/test-migrations.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# Disposable CI stack only. Use the runner baked into the Postgres image on +# fresh AND existing volumes before starting application services. No host +# Rust/psql installation, second history table or startup-check bypass. +# Optional Compose arguments (e.g. -f local-ports.yml) preserve test isolation. +set -euo pipefail +root=$(cd "$(dirname "$0")/.." && pwd) +cd "$root" +compose=(docker compose -f docker-compose.test.yml "$@" --env-file "${POLIS_TEST_ENV_FILE:-test.env}") +"${compose[@]}" up -d --wait --wait-timeout 120 postgres +"${compose[@]}" exec -T postgres sh -eu -c ' + export DATABASE_URL="host=/var/run/postgresql user=$POSTGRES_USER dbname=$POSTGRES_DB sslmode=disable" + export POLIS_MIGRATIONS_DIR=/migrations + polis-migrate apply + polis-migrate check +' diff --git a/delphi/scripts/job_graph_client.py b/delphi/scripts/job_graph_client.py new file mode 100644 index 0000000000..7ba517cf43 --- /dev/null +++ b/delphi/scripts/job_graph_client.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +"""Internal graph admission/status/result adapter. Caller supplies authorized namespace. + +A queue executor connection, never the main application/admin DB credential. +No HTTP endpoint or end-user authentication is implied by this internal adapter. +""" +import argparse +from contextlib import closing +import json +import os +import psycopg2 + + +class GraphClient: + def __init__(self, connection, environment): + self.connection = connection + self.environment = environment + + def _call(self, function, *arguments): + with self.connection: + with self.connection.cursor() as cursor: + cursor.execute('SELECT public.'+function+'('+','.join(['%s']*(len(arguments)+1))+')', (self.environment,)+arguments) + return cursor.fetchone()[0] + + def admit(self, zid, scope, request_key, specification, supersedes=None): + return self._call('pd_graph_admit',zid,scope,request_key,json.dumps(specification),supersedes) + + def status(self, graph_id): + return self._call('pd_graph_view',graph_id) + + def served(self, zid, scope): + return self._call('pd_graph_served',zid,scope) + + def publish(self, graph_id, job_id, expected_generation): + return self._call('pd_graph_publish',graph_id,job_id,expected_generation) + + +if __name__ == '__main__': + parser=argparse.ArgumentParser(description=__doc__) + parser.add_argument('--env',required=True) + sub=parser.add_subparsers(dest='command',required=True) + status=sub.add_parser('status'); status.add_argument('graph_id') + served=sub.add_parser('served'); served.add_argument('zid',type=int);served.add_argument('scope') + args=parser.parse_args() + with closing(psycopg2.connect(os.environ['QUEUE_DATABASE_URL'])) as connection: + client=GraphClient(connection,args.env) + result=client.status(args.graph_id) if args.command=='status' else client.served(args.zid,args.scope) + print(json.dumps(result,sort_keys=True,indent=2)) diff --git a/delphi/scripts/job_graph_stage.py b/delphi/scripts/job_graph_stage.py new file mode 100644 index 0000000000..30c0abb23a --- /dev/null +++ b/delphi/scripts/job_graph_stage.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +"""Small, offline reference graph stages. No live DB, provider, reset or serving writes. + +This adapter proves the artifact contract, not migration of Delphi's UMAP/LLM +pipeline. Its closed model IDs distinguish it from production model outputs. +""" +import hashlib +import json +import os +from pathlib import Path +import sys + + +def run(frame): + if frame['schema'] != 'polis-job-stage-frame/1': + raise ValueError('unsupported frame') + inp = frame['input'] + if hashlib.sha256(frame['input_json'].encode()).hexdigest() != frame['input_sha256'] or json.loads(frame['input_json']) != inp: + raise ValueError('resolved input digest mismatch') + declared = inp['declared'] + if inp['schema'] != 'polis-job-input/1' or declared['mode'] != 'full': + raise ValueError('unsupported execution mode') + if declared['code'] != hashlib.sha256(Path(__file__).read_bytes()).hexdigest() or declared['runtime'] != 'python-'+sys.version.split()[0]: + raise ValueError('worker code/runtime differs from declared provenance') + stage = frame['stage'] + for artifact in inp['artifacts'].values(): + if hashlib.sha256(artifact['payload'].encode()).hexdigest() != artifact['sha256']: + raise ValueError('artifact content mismatch') + if stage == 'graph_embed': + if declared['model'] != 'local-token-count/1': + raise ValueError('unsupported embedding model') + texts = declared['snapshot']['data']['texts'] + if not texts or len(texts) > 100 or any(not isinstance(t, str) or len(t)>4096 for t in texts): + raise ValueError('bounded text snapshot required') + vocabulary = sorted({word for text in texts for word in text.lower().split()}) + output = {'vocabulary': vocabulary, 'vectors': [[text.lower().split().count(w) for w in vocabulary] for text in texts]} + elif stage == 'graph_cluster': + if declared['model'] != 'local-nearest-centroid/1': + raise ValueError('unsupported cluster model') + vectors = json.loads(inp['artifacts']['embeddings']['payload'])['vectors'] + # Deterministic small reference Lloyd fit; independent of legacy resets. + centers = [list(map(float, v)) for v in (vectors[0], vectors[-1])] + labels = [] + for _ in range(20): + labels = [min(range(len(centers)), key=lambda k: sum((a-b)**2 for a,b in zip(v, centers[k]))) for v in vectors] + updated = [[sum(v[d] for v,l in zip(vectors, labels) if l==k)/labels.count(k) if k in labels else centers[k][d] for d in range(len(vectors[0]))] for k in range(len(centers))] + if updated == centers: + break + centers = updated + output = {'labels': labels, 'centers': centers} + elif stage == 'graph_narrative': + if declared['model'] != 'local-cluster-summary/1': + raise ValueError('unsupported narrative model; paid adapters need admission and budget integration') + clusters = json.loads(inp['artifacts']['clusters']['payload']) + output = {'text': f"{len(clusters['labels'])} statements in {len(set(clusters['labels']))} clusters.", 'labels': clusters['labels']} + else: + raise ValueError('unsupported stage') + payload = json.dumps(output, sort_keys=True, separators=(',', ':')) + return dict(schema='polis-job-artifact-manifest/1',job_id=frame['job_id'],run_id=frame['run_id'],attempt_id=frame['attempt_id'],stage=stage, + input_sha256=frame['input_sha256'],outcome='succeeded',output=dict(role='result',schema=stage+'/1',payload=payload,sha256=hashlib.sha256(payload.encode()).hexdigest())) + + +if __name__ == '__main__': + frame = json.loads(Path(os.environ['DELPHI_FRAME']).read_text()) + manifest = run(frame) + path = Path(os.environ['DELPHI_OUTPUT_MANIFEST']) + temporary = path.with_suffix('.tmp') + temporary.write_text(json.dumps(manifest, sort_keys=True, separators=(',', ':'))) + temporary.replace(path) diff --git a/delphi/tests/job_graph/adversarial.py b/delphi/tests/job_graph/adversarial.py new file mode 100644 index 0000000000..fca71c06fa --- /dev/null +++ b/delphi/tests/job_graph/adversarial.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +"""RPC-boundary and race controls, complement the real daemon scenario in prove.py.""" +import concurrent.futures +import copy +import importlib.util +import json +import os +from pathlib import Path +import subprocess as sp +import time +import uuid +from prove import * + + +def claim(cls='delphi',stage='graph_embed'): + return rpc('pd_graph_claim',"'proof'","1::smallint",lit(uuid.uuid4()),lit(uuid.uuid4()),'10',lit(cls),lit(stage)) + +def ident(c):return ["'proof'",lit(c['job_id']),lit(c['owner_id']),lit(c['attempt_id']),lit(c['lease_epoch'])] +def finish(c,manifest,seq): + raw=json.dumps(manifest,sort_keys=True,separators=(',',':')) + sha=hashlib.sha256(raw.encode()).hexdigest() + sql(f"INSERT INTO polis_queue_logs(env,attempt_id,seq,stream,line) VALUES('proof',{lit(c['attempt_id'])},{seq},'manifest',{lit(raw)})",True) + return rpc('pd_graph_finalize',*ident(c),"'unused'",lit(sha)),sha + +def main(): + # Existing output is unaffected by every rejection below. + served=rpc('pd_graph_served',"'proof'",'1',"'retry'") + only=spec();only['nodes']=only['nodes'][:1] + with concurrent.futures.ThreadPoolExecutor(2) as pool: + results=list(pool.map(lambda _:graph('concurrent',only),range(2))) + assert results[0]['graph_id']==results[1]['graph_id'] + record('P07_concurrent_idempotent_admission',responses=results) + g=results[0];n=nodes(g)['e'] + # The installed membership/edge triggers close owner-control bypasses too. + sql(f"UPDATE delphi_graphs SET sealed=false WHERE graph_id={lit(g['graph_id'])}",ok=False) + sql(f"UPDATE delphi_graph_nodes SET declared='{{}}' WHERE job_id={lit(n['job_id'])}",ok=False) + with concurrent.futures.ThreadPoolExecutor(2) as pool: + cs=list(pool.map(lambda _:claim(),range(2))) + assert sorted(c['outcome'] for c in cs)==['none','owned'];c=next(x for x in cs if x['outcome']=='owned') + # No process was spawned for this RPC control: exit proof is honest. + rpc('pq_end_attempt',*ident(c),"'confirm_exit'","NULL",'true') + module_spec=importlib.util.spec_from_file_location('stage',ROOT/'delphi/scripts/job_graph_stage.py') + module=importlib.util.module_from_spec(module_spec);module_spec.loader.exec_module(module) + frame=dict(schema='polis-job-stage-frame/1',job_id=c['job_id'],run_id=c['run_id'],attempt_id=c['attempt_id'],stage=c['stage'],input=c['graph_input'],input_sha256=c['graph_input_sha'],input_json=c['graph_input_wire']) + manifest=module.run(frame) + for i,(field,value) in enumerate([('input_sha256','0'*64),('run_id',str(uuid.uuid4())),('attempt_id',str(uuid.uuid4()))]): + bad=copy.deepcopy(manifest);bad[field]=value + assert finish(c,bad,i)[0]['outcome']=='invalid_output' + bad=copy.deepcopy(manifest);bad['output']['sha256']='0'*64 + assert finish(c,bad,3)[0]['outcome']=='invalid_output' + good,sha=finish(c,manifest,4);assert good['outcome']=='succeeded' + assert rpc('pd_graph_finalize',*ident(c),"'lost-reply'",lit(sha))['outcome']=='already_succeeded' + badid=ident(c);badid[2]=lit(uuid.uuid4()) + assert rpc('pd_graph_finalize',*badid,"'stale'",lit(sha))['outcome']=='fenced' + assert sql(f"SELECT count(*) FROM delphi_artifacts WHERE job_id={lit(c['job_id'])}")=='1' + sql(f"UPDATE delphi_artifacts SET run_id={lit(uuid.uuid4())} WHERE job_id={lit(c['job_id'])}",ok=False) + record('P08_P09_wrong_bindings_lost_finalize_ack_stale_owner_single_artifact') + rpc('pd_graph_reconcile',"'proof'") + # An uncommitted admitted graph is invisible to claims; after seal+commit + # only its root may run and both dependency edges are already visible. + s=spec();query=f"BEGIN;SET ROLE polis_queue_executor;SELECT pd_graph_admit('proof',1,'seal-race','first',{js(s)},NULL);SELECT pg_sleep(2);COMMIT;" + proc=sp.Popen(C+['exec','-T','postgres','psql','-XqAt','-v','ON_ERROR_STOP=1','-U','postgres','-d',DB],stdin=sp.PIPE,stdout=sp.PIPE,stderr=sp.PIPE,text=True) + proc.stdin.write(query);proc.stdin.close();time.sleep(.5) + assert claim()['outcome']=='none' + out=proc.stdout.read();err=proc.stderr.read();assert proc.wait()==0,err + race=json.loads(next(l for l in out.splitlines() if l.startswith('{')));rn=nodes(race) + c=claim();assert c['job_id']==rn['e']['job_id'];assert claim(stage='graph_cluster,graph_narrative')['outcome']=='none' + # A closed edge cannot be removed, nor a late dependency appended. + sql(f"DELETE FROM delphi_graph_edges WHERE consumer={lit(rn['c']['job_id'])}",ok=False) + sql(f"UPDATE delphi_jobs SET parent_job_id={lit(rn['n']['job_id'])} WHERE job_id={lit(rn['e']['job_id'])}",ok=False) + rpc('pq_fail',*ident(c),'true',"'negative-control'",'true') + record('P07_seal_claim_race_and_late_mutation_refusal') + # Durable provider intent, lost submission ACK, no duplicate execution. + provider=graph('provider-unknown',only);c=claim();assert c['job_id']==nodes(provider)['e']['job_id'] + request=str(uuid.uuid4()) + rpc('pd_provider_intent',*ident(c),lit(request),"'local-provider-fixture'","decode(repeat('a',64),'hex')") + rpc('pd_provider_update',*ident(c),lit(request),"'submission_unknown'",'NULL') + parked=rpc('pq_fail',*ident(c),'false',"'lost_provider_ack'",'true');assert parked['state']=='parked' + assert claim()['outcome']=='none' + assert sql(f"SELECT count(*) FROM delphi_provider_requests WHERE job_id={lit(c['job_id'])}")=='1' + record('P10_unknown_provider_intent_stays_parked_no_reclaim',job=c['job_id']) + # Expired ownership with an unconfirmed child cannot gain another owner. + stale=graph('stale-lease',only);c=claim();assert c['job_id']==nodes(stale)['e']['job_id'] + sql(f"UPDATE polis_queue_jobs SET locked_until=clock_timestamp()-interval '1 second' WHERE job_id={lit(c['job_id'])}") + rpc('pq_reap',"'proof'",'NULL','100',"'delphi'") + assert claim()['outcome']=='none' + assert rpc('pd_graph_finalize',*ident(c),"'stale'",lit('0'*64))['outcome']=='fenced' + record('P09_expired_lease_requires_exit_proof') + assert rpc('pd_graph_served',"'proof'",'1',"'retry'")==served + record('P12_old_served_bundle_survives_all_controls') + +if __name__=='__main__':main() diff --git a/delphi/tests/job_graph/child_fixture.py b/delphi/tests/job_graph/child_fixture.py new file mode 100644 index 0000000000..114c12eac0 --- /dev/null +++ b/delphi/tests/job_graph/child_fixture.py @@ -0,0 +1,30 @@ +"""Failure injection around the actual reference adapter; never shipped as a worker.""" +import importlib.util +import json +import os +from pathlib import Path +import sys +import time + +frame = json.loads(Path(os.environ['DELPHI_FRAME']).read_text()) +root = Path(os.environ['GRAPH_PROOF_ROOT']) +counter = root/'starts'/frame['job_id'] +counter.parent.mkdir(exist_ok=True) +count = int(counter.read_text())+1 if counter.exists() else 1 +counter.write_text(str(count)) +control = json.loads((root/'control.json').read_text()) +(root/'frames').mkdir(exist_ok=True) +(root/'frames'/f"{frame['job_id']}-{count}.json").write_text(json.dumps(frame,sort_keys=True)) +if frame['stage']==control.get('hold_stage'): + (root/'held-child.json').write_text(json.dumps(dict(pid=os.getpid(),job_id=frame['job_id']))) + while True: + time.sleep(.1) +if frame['stage']=='graph_cluster' and control.get('fail_clusters',0)>=count: + Path(os.environ['DELPHI_OUTPUT_MANIFEST']).with_name('uncommitted-staging.json').write_text('{"unpublished":true}') + sys.exit(1) +source = Path(os.environ['GRAPH_STAGE_SOURCE']) +spec = importlib.util.spec_from_file_location('graph_stage',source) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) +manifest = module.run(frame) +Path(os.environ['DELPHI_OUTPUT_MANIFEST']).write_text(json.dumps(manifest,sort_keys=True,separators=(',',':'))) diff --git a/delphi/tests/job_graph/compose.yml b/delphi/tests/job_graph/compose.yml new file mode 100644 index 0000000000..fe26fd80db --- /dev/null +++ b/delphi/tests/job_graph/compose.yml @@ -0,0 +1,14 @@ +services: + postgres: + image: postgres:17-alpine + environment: + POSTGRES_HOST_AUTH_METHOD: trust + POSTGRES_USER: postgres + POSTGRES_DB: postgres + ports: + - "127.0.0.1:${POLIS_RECOVERY_PG_PORT:?set an owned port}:5432" + healthcheck: + test: [CMD-SHELL, "pg_isready -U postgres"] + interval: 1s + timeout: 3s + retries: 30 diff --git a/delphi/tests/job_graph/installs.py b/delphi/tests/job_graph/installs.py new file mode 100644 index 0000000000..245d93c479 --- /dev/null +++ b/delphi/tests/job_graph/installs.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +"""Apply the core SQL on fresh local fixtures; no deployment runner or fake receipts.""" +import hashlib +import sys +from prove import ROOT, C, DB, OUT, record, sql, sp +MIG = ROOT / 'server/postgres/migrations' + +def query(db, text, ok=True): + p = sp.run(C + ['exec', '-T', 'postgres', 'psql', '-XqAt', '-v', + 'ON_ERROR_STOP=1', '-U', 'postgres', '-d', db], + input=text, text=True, capture_output=True) + if (p.returncode == 0) != ok: + raise AssertionError((db, p.stdout[-1000:], p.stderr[-2000:])) + return p.stdout.strip() if ok else p.stderr + +def baseline(db): + query('postgres', 'CREATE DATABASE ' + db) + paths = sorted(p for p in MIG.glob('*.sql') if p.name[:6].isdigit() + and int(p.name[:6]) <= 24) + assert [int(p.name[:6]) for p in paths] == [n for n in range(25) if n != 20] + for p in paths: + query(db, p.read_text()) + assert query(db, 'SELECT contract_version FROM polis_queue_install') == 'polis-queue/3' + +forward = (MIG / '000027_create_sealed_job_graphs.sql').read_text() +down = (MIG / 'down/000027_drop_sealed_job_graphs.sql').read_text() +if sys.argv[1:] == ['--bootstrap']: + baseline(DB) + query(DB, forward) + print('Applied unchanged 000000–000024 prerequisites (no 20), then trimmed M27') +else: + for line in (MIG / 'down/000027-files.sha256').read_text().splitlines(): + digest, name = line.split() + assert hashlib.sha256((MIG / name).read_bytes()).hexdigest() == digest + record('forward_and_down_seals_match') + db = DB + '_empty_down' + baseline(db) + query(db, forward) + # The down migration itself compares the complete /3 catalog to the baseline. + query(db, down) + assert query(db, 'SELECT contract_version FROM polis_queue_install') == 'polis-queue/3' + query(db, forward) + assert query(db, 'SELECT contract_version FROM polis_queue_install') == 'polis-queue/5' + assert 'nonempty graph contract' in sql(down, ok=False) + record('empty_forward_down_reapply_exact_catalog_nonempty_refusal') + # Never adopt the earlier unreleased M27 under the new seal. + assert 'queue /3 catalog drift' in query(DB, forward, ok=False) + record('reapply_requires_explicit_migration_management') diff --git a/delphi/tests/job_graph/process_failure.py b/delphi/tests/job_graph/process_failure.py new file mode 100644 index 0000000000..2b3c8954d6 --- /dev/null +++ b/delphi/tests/job_graph/process_failure.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +"""Kernel process kill while staging, followed by a retry on the same served scope.""" +import shutil +from prove import * + +def main(): + # Reuse the first proof's E through the actual served-bundle reader. + before=rpc('pd_graph_served',"'proof'",'1',"'retry'") + original=next(a for a in before['bundle']['artifacts'] if a['schema_version']=='graph_embed/1') + declared_sha=sql(f"SELECT pd_graph_hash(declared) FROM delphi_graph_nodes WHERE job_id={lit(original['job_id'])}") + s=spec();s['nodes']=s['nodes'][1:] + s['nodes'][0]['inputs']=[dict(artifact_id=original['artifact_id'],sha256=original['content_sha'],contract_sha256=declared_sha,role='embeddings')] + rpc('pd_graph_reconcile',"'proof'") + g=graph('retry',s,'kernel-rebuild');ns=nodes(g) + (OUT/'control.json').write_text(json.dumps(dict(hold_stage='graph_cluster'))) + worker=start('held-writer') + until=time.monotonic()+30 + while not (OUT/'held-child.json').exists() and time.monotonic()&2 + exit 2 +fi +trap '"${compose[@]}" down -v > "$proof_output/cleanup.log" 2>&1' EXIT +(cd queue-rs + cargo fmt --all --check + cargo build --locked --bins + cargo test --locked + cargo clippy --locked --all-targets -- -D warnings + cargo clippy --locked --all-targets --features jobs-integration -- -D warnings +) 2>&1 | tee "$proof_output/rust.log" +"${compose[@]}" up -d --wait +python3 delphi/tests/job_graph/installs.py --bootstrap 2>&1 | tee "$proof_output/install.log" +python3 delphi/tests/job_graph/prove.py 2>&1 | tee "$proof_output/proof.log" +GRAPH_PROOF_ROOT="$proof_output/process" python3 delphi/tests/job_graph/process_failure.py 2>&1 | tee "$proof_output/process.log" +GRAPH_PROOF_ROOT="$proof_output/adversarial" python3 delphi/tests/job_graph/adversarial.py 2>&1 | tee "$proof_output/adversarial.log" +GRAPH_PROOF_ROOT="$proof_output/installs" python3 delphi/tests/job_graph/installs.py 2>&1 | tee "$proof_output/installs.log" +# Same legacy compatibility command as queue-rs-ci, isolated in this owned server. +"${compose[@]}" exec -T postgres psql -U postgres -c 'CREATE DATABASE queue_acceptance' +export POLIS_JOBS_TEST_DATABASE_URL="postgresql://postgres@127.0.0.1:$POLIS_RECOVERY_PG_PORT/queue_acceptance" +export POLIS_JOBS_TEST_PYTHON=python3 +(cd queue-rs && cargo test --locked --features jobs-integration --test jobs_integration -- --test-threads 4) 2>&1 | tee "$proof_output/legacy.log" + +# Existing Node adapter gets an independent fresh /5 database: graph fixtures +# deliberately use fixed identifiers and must not perturb its serial sequences. +GRAPH_PROOF_DB=graph_node python3 delphi/tests/job_graph/installs.py --bootstrap 2>&1 | tee "$proof_output/node-install.log" +export DATABASE_URL="postgresql://postgres@127.0.0.1:$POLIS_RECOVERY_PG_PORT/graph_node" +export DATABASE_SSL=false +export NODE_ENV=test +(cd server && npx jest --config jest.job-graphs.config.ts --ci --runInBand) 2>&1 | tee "$proof_output/node.log" diff --git a/delphi/tests/test_after_install_hook.py b/delphi/tests/test_after_install_hook.py index c9f86a350c..65181cdc61 100644 --- a/delphi/tests/test_after_install_hook.py +++ b/delphi/tests/test_after_install_hook.py @@ -221,8 +221,34 @@ def test_the_stop_hook_leaves_every_service_running(tmp_path, service_type): assert "AfterInstall" in proc.stdout -def test_unknown_role_refuses_before_migration_or_replacement(tmp_path): - proc, log = _deploy(tmp_path, "unknown") - assert proc.returncode != 0 - assert not _calls(log, "docker") - assert not _calls(log, "docker-compose") +@pytest.mark.parametrize("service_type,worker_class", [ + ("server", None), ("delphi", None), ("math", None), + ("delphi-large", "large"), ("delphi-worker", "delphi"), +]) +def test_migration_succeeds_before_any_service_replacement(tmp_path, service_type, worker_class): + proc, log = _deploy(tmp_path, service_type, worker_class=worker_class) + assert proc.returncode == 0, proc.stdout + proc.stderr + build = "docker build -t polis-migrate:deploy -f queue-rs/polis-migrate/Dockerfile ." + apply = "docker run --rm --network host --env-file .env polis-migrate:deploy deploy" + assert log.count(build) == log.count(apply) == 1 + assert log.index(build) < log.index(apply) + replacements = [i for i, call in enumerate(log) if call.startswith(( + "docker-compose down", "docker rm ", "docker system prune", + "docker-compose up", "systemctl restart"))] + assert replacements or service_type == "math" + assert not replacements or log.index(apply) < min(replacements) + + +@pytest.mark.parametrize("failure", ["build", "run"]) +@pytest.mark.parametrize("service_type,worker_class", [ + ("server", None), ("delphi", None), ("math", None), + ("delphi-large", "large"), ("delphi-worker", "delphi"), +]) +def test_migration_failure_preserves_every_running_service(tmp_path, service_type, worker_class, failure): + proc, log = _deploy(tmp_path, service_type, worker_class=worker_class, + migration_failure=failure) + assert proc.returncode == 17, proc.stdout + proc.stderr + assert _calls(log, "docker-compose") == [] + assert _calls(log, "systemctl") == [] + docker = _calls(log, "docker") + assert [call.split()[0] for call in docker] == (["build"] if failure == "build" else ["build", "run"]) diff --git a/docker-compose.test.yml b/docker-compose.test.yml index f489a829f1..f02113ac25 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -82,9 +82,12 @@ services: - ${AUTH_CERTS_PATH:-~/.simulacrum/certs}:/root/.simulacrum/certs:ro restart: unless-stopped depends_on: - - oidc-simulator - - postgres - - dynamodb-init + oidc-simulator: + condition: service_started + postgres: + condition: service_healthy + dynamodb-init: + condition: service_completed_successfully extra_hosts: - "host.docker.internal:host-gateway" @@ -112,7 +115,8 @@ services: networks: - polis-test depends_on: - - postgres + postgres: + condition: service_healthy restart: unless-stopped extra_hosts: - "host.docker.internal:host-gateway" @@ -160,6 +164,8 @@ services: build: context: ./server dockerfile: Dockerfile-db + additional_contexts: + queue-rs: ./queue-rs labels: polis_tag: test environment: @@ -167,7 +173,8 @@ services: - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} - POSTGRES_USER=${POSTGRES_USER} healthcheck: - test: ["CMD-SHELL", "pg_isready -U postgres"] + # The init server is socket-only; TCP refuses until migrations finish. + test: ["CMD-SHELL", "pg_isready -h 127.0.0.1 -U $$POSTGRES_USER -d $$POSTGRES_DB"] interval: 5s timeout: 5s retries: 5 @@ -175,7 +182,7 @@ services: networks: - polis-test ports: - - 5432:5432 + - "${POLIS_RECOVERY_PG_PORT:-5432}:5432" restart: unless-stopped # PostgreSQL configuration for development/testing command: > @@ -262,8 +269,10 @@ services: networks: - "polis-test" depends_on: - - postgres - - dynamodb-init + postgres: + condition: service_healthy + dynamodb-init: + condition: service_completed_successfully command: tail -f /dev/null restart: unless-stopped diff --git a/docker-compose.yml b/docker-compose.yml index d87a06b89f..30a6822fc7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -385,6 +385,8 @@ services: # Dockerfile-db (default): migrations-based initialization # Dockerfile-pdb: production dump restoration (requires prodclone.dump) dockerfile: Dockerfile-${DB_INIT_MODE:-db} + additional_contexts: + queue-rs: ./queue-rs labels: polis_tag: ${TAG:-dev} environment: diff --git a/docs/job-graphs.md b/docs/job-graphs.md new file mode 100644 index 0000000000..0322b70662 --- /dev/null +++ b/docs/job-graphs.md @@ -0,0 +1,97 @@ +# Per-step jobs that keep completed work + +Each computation has its own run, recorded inputs and immutable saved result. +Dependants wait for their producers. A failed step retries using the same saved +inputs; completed steps do not rerun. Readers keep seeing the last complete +bundle until a complete replacement is published atomically. + +The reference example is embeddings → clusters → narrative. Its local adapters +use token counts, a small deterministic clustering fit and a cluster summary. +These exercise the real daemon and separate child processes. They are not the +production MiniLM, UMAP or provider pipeline; those adapters and report readers +are a separate integration. + +## Stored contract + +M27 extends M19/M23/M24 without changing those files. The queue contract is +`polis-queue/5`; `/4` remains reserved. Each step has one run and job; each retry +has a new attempt. Inputs record the data snapshot and digest, code digest, +model, runtime, seed, effective configuration and computation mode. This +adapter supports `full` only and refuses unsupported incremental requests. +Reusing embeddings while fully recomputing clusters is supported. + +Admission seals the complete graph in one transaction. Edges name producers +before their results exist, then bind once to immutable artifacts. Claims +include exact serialized resolved inputs and their digest; both the Rust daemon +and Python child verify them. Postgres binds results to the job, run and exited +attempt. Finalization checks ownership, lease and process-exit proof and is +idempotent after a lost reply. Failed attempt files are never served results. + +Publication uses a generation compare-and-swap to select a completed narrative +and its transitive artifacts. Old bundles are untouched by failures or worker +replacement. Completion of the embedding root alone does not release the graph +scope; durable reconciliation waits for all members and their exit proofs. +Class-depth demand excludes dependency-blocked work. + +`delphi/scripts/job_graph_client.py` provides internal admission, status, +publication and result reading. Callers must authorize the namespace before +using it; this is not a public HTTP API. Configure graph workers explicitly with +`POLIS_JOBS_STAGES=graph_embed,graph_cluster,graph_narrative`. Legacy and graph +stages require separate workers. Defaults continue to select legacy stages. +Existing workers without `/5` support must be replaced before applying M27. + +## Deliberately outside this core + +Scoped durable breakers and half-open probes, provider reconciliation and its +evidence table, and explicit superseding dead-branch redrive are deferred. +The sixth admission argument is retained for wire compatibility but must be +NULL. Retry exhaustion leaves the job dead and its dependants visibly blocked; +it never wipes completed results. The existing queue retry/dead-letter logic +and fail-closed provider-uncertainty checks remain. No paid-provider activation, +spend controls, new placement/scaling system or retention deletion is added. + +Limits are 32 nodes, four inputs per node, 100 texts, 1 MiB admission and +512 KiB per artifact. Each job has at most ten attempts. Memory declarations +are checked against class bounds (512 MiB Delphi, 2 GiB large); this is not OS +memory enforcement. Real adapters and larger result storage need their own +contracts. No artifact purge is enabled. + +## Apply and verify + +Release B restores the migration runner and selects M27 for fresh databases and +existing deployments. The deploy reconciles the supported legacy catalog before +applying pending files; the API checks their receipts before starting. +The earlier unreleased M27 +had different bytes: never adopt its old installation under this new checksum. +Rebuild disposable draft databases; released histories require a forward +migration, never a rewritten receipt. + +The empty down migration compares the restored catalog with its original `/3` +state. It refuses once graph data exists. A nonempty installation requires +compatible `/5` workers and readers; deleting results is not rollback. + +Hosted CI and mm5 use exactly the same entry: + +```sh +COMPOSE_PROJECT_NAME=polis-graph-test-local \ +POLIS_RECOVERY_PG_PORT=55467 RECOVERY_PG_PORT=55467 \ +bash delphi/tests/job_graph/run.sh +``` + +Run on a build box with Docker, the pinned Rust toolchain, Python 3 and Node 24; +first run `npm ci --no-audit --no-fund` in `server`. Choose an unused project and +port. The entry refuses existing project containers, starts only its own local +Postgres and removes its project on exit. It runs the graph scenarios, SQL +boundaries, empty down/reapply, Rust and existing Node/legacy queue tests. +Receipts go to `graph-proof/` or `GRAPH_PROOF_ROOT`. Fixtures contain generated +texts and zero vote rows. No provider calls are needed. + +## Release B integration + +The selected migration release now includes M27. After Release A has succeeded +on every host, the deploy hook runs `polis-migrate deploy`: reconcile a supported +pre-runner catalog through M22, then apply M19/M23/M24/M27 and check readiness, +before replacing services. A modern M24 history upgrades by applying M27 only; +existing receipt bytes are preserved. An unledgered or old-draft M27 catalog is +not silently adopted or re-sealed. The API refuses pending or changed receipts. +DynamoDB readers/importers and M28/M29 are outside this release. diff --git a/docs/migration-legacy-contract.md b/docs/migration-legacy-contract.md new file mode 100644 index 0000000000..6aec852d47 --- /dev/null +++ b/docs/migration-legacy-contract.md @@ -0,0 +1,175 @@ +# Supported legacy schema contract + +This release accepts the documented legacy schema without changing its data or +rewriting old migrations. The live schema is authoritative for the named legacy +variant; the existing bootstrap is a separately supported installation variant. +The immutable numbered files are execution history, not a claim that every +installation ran every file. Adoption predicates describe the supported states. + +The concrete admission changes are bounded: `worker_tasks.task_type` accepts +`text` or `varchar(99)`; renamed `pwreset_tokens.token` accepts `varchar(100)` +or `varchar(250)`; absent `conversations.branding_type` and +`math_ticks.caching_tick` are valid, but existing columns must have their known +integer/bigint types. Contributor agreement tables are outside this release's +adoption scope. M2 requires enforced owner/xid uniqueness and preserves an +optional enforced, nonpartial owner/uid unique index in either key order. +Other lengths, arbitrary text alternatives, invalid indexes and partial unique +owner/uid indexes remain refused. + +Six math payloads accept `json` alongside bootstrap `jsonb`: `math_bidtopid`, +`math_cache`, `math_exportstatus`, `math_main`, `math_profile`, `math_ptptstats`. +`math_report_correlationmatrix.data` remains `jsonb`; no observed legacy JSON +contract was established for it. This narrows an earlier broad helper. + +The file contract below records the rest of the reviewed live differences. +It does not add new global core-default, routine-body or ACL adoption checks. +These documentary contracts must not be mistaken for an exhaustive schema +certifier. Existing modern M3/M8–18 predicates continue to enforce the properties +introduced by those migrations; unsupported/partial schemas stop for review. + +## Named live attributes + +| Attribute | Legacy contract | Bootstrap alternative | +| --- | --- | --- | +| comments.uid | integer NOT NULL, default 0 | no default | +| comments.velocity | nullable real, default 1 | NOT NULL real | +| conversations.auth_needed_to_vote | default false | no default | +| conversations.auth_needed_to_write | default true | no default | +| conversations.auth_opt_fb | default true | no default | +| conversations.auth_opt_tw | default true | no default | +| conversations.auth_opt_allow_3rdparty | default true | no default | +| contexts/context_id, conversations/zid, courses/course_id, participant_metadata_answers/pmaid, participant_metadata_questions/pmqid, users/uid sequences | bigint sequence, maximum 9223372036854775807, integer owning column | integer sequence, maximum 2147483647 | + +Sequence configuration is not current sequence state. Never reset, narrow or +advance a live sequence to match bootstrap. Reviewed implementations of +`get_times_for_most_recent_visible_comments()`, `pid_auto()`, `tid_auto()`, +`random_string(integer)` and `random_polis_site_id()` are executable equivalents +of the public historical sources; formatting fingerprints differ. This release +never replaces those routines. Their exact historical source bindings are in +the deployment's private reconciliation record. + +Optional probe provisioning grants SELECT on comments, conversations, math_main, +math_ticks, participants and votes, plus public schema USAGE, to the configured +read-only probe role. It is separate from application and migration authority; +no role identity or grant is imposed on other deployments by this contract. +M19's actual migration session needs role creation/SET authority, public schema +USAGE/CREATE grant authority, conversations SELECT plus topic UPDATE and zid +REFERENCES grant authority. M23/M24 continue as the queue owner. Verify the +actual session; a superuser-only test is insufficient evidence for that session. + +## Other live attributes and scope + +| Attribute | Type | Nullability | +| --- | --- | --- | +| comments.curation | smallint | NOT NULL | +| participants_extended.encrypted_ip_address | character varying(9999) | nullable | +| participants_extended.encrypted_x_forwarded_for | character varying(9999) | nullable | +| suzinvites.modified | bigint | nullable | +| suzinvites.uid | integer | nullable | +| users.pwhash | character varying(128) | nullable | + +Existing `suzinvites.uid` references users.uid; no field/constraint is added, +dropped or backfilled during reconciliation. Legacy callers and generated +schema descriptions may observe these fields; fresh bootstrap does not acquire +them merely because the legacy contract documents them. Whole-tree generated +schema modernization is a separate change. + +The following complete research-triage list records the stable scope. These are +research dispositions, not claims that bootstrap or generated ORM declarations +were rewritten in this patch. The explicit adoption changes and separate legacy +file contract described above are implemented here; references below to canonical +files/ORM or generated-reader changes remain broader modernization obligations. +“Before +stable” means a file/adoption contract, never authorization for production DDL. +Ancillary objects are retained. No extension removal, name rewrite, course-invite +unique constraint addition, contributor rename or data cleanup is selected. + +| Difference | Scope | Contract disposition | +| --- | --- | --- | +| contributor | SET ASIDE | Defer ancillary file spelling repair for stable; preserve live correct table and working endpoint, scoped non-gating exclusion and graceful503 for absent schemas. No production operation. | +| branding | SYNC BEFORE STABLE | Canonical files omit branding_type to match production; reconcile required generated mappings, preserve exact downstream variants without DROP. | +| caching | SYNC BEFORE STABLE | Canonical files/ORM omit math_ticks.caching_tick to match production; preserve math_main active cursor and existing downstream variants. | +| course | SET ASIDE | Record production absence of invite uniqueness; set aside for narrow stable because M0 adoption does not test it. Later complete canonical files omit the invariant; no production constraint changes. | +| xids | SYNC BEFORE STABLE | Canonical files/adoption preserve both production XID unique invariants; revise prior M2 guard, no drop-owner/uid action. | +| worker_tasks.task_type text versus varchar(99) | SYNC BEFORE STABLE | Actual adoption refusal: align exact production type and preserve reviewed downstream variants. | +| pwreset_tokens.token varchar(100) versus varchar(250) | SYNC BEFORE STABLE | Actual adoption refusal: align exact production type and preserve reviewed downstream variants. | +| math_bidtopid.data json versus jsonb | SYNC BEFORE STABLE | Named JSON exception already admits this type. Record production JSON and supported JSONB variant; no conversion. | +| math_cache.data json versus jsonb | SYNC BEFORE STABLE | Named JSON exception already admits this type. Record production JSON and supported JSONB variant; no conversion. | +| math_exportstatus.data json versus jsonb | SYNC BEFORE STABLE | Named JSON exception already admits this type. Record production JSON and supported JSONB variant; no conversion. | +| math_main.data json versus jsonb | SYNC BEFORE STABLE | Named JSON exception already admits this type. Record production JSON and supported JSONB variant; no conversion. | +| math_profile.data json versus jsonb | SYNC BEFORE STABLE | Named JSON exception already admits this type. Record production JSON and supported JSONB variant; no conversion. | +| math_ptptstats.data json versus jsonb | SYNC BEFORE STABLE | Named JSON exception already admits this type. Record production JSON and supported JSONB variant; no conversion. | +| comments.uid default zero versus absent | SYNC BEFORE STABLE | Record exact live default/nullability in the file contract and reconcile affected generated readers. Existing M0 does not reject it. Defer behavior changes, backfill, or production DDL. | +| comments.velocity nullable versus NOT NULL | SYNC BEFORE STABLE | Record exact live default/nullability in the file contract and reconcile affected generated readers. Existing M0 does not reject it. Defer behavior changes, backfill, or production DDL. | +| conversations.auth_needed_to_vote default literal versus absent | SYNC BEFORE STABLE | Record exact live default/nullability in the file contract and reconcile affected generated readers. Existing M0 does not reject it. Defer behavior changes, backfill, or production DDL. | +| conversations.auth_needed_to_write default literal versus absent | SYNC BEFORE STABLE | Record exact live default/nullability in the file contract and reconcile affected generated readers. Existing M0 does not reject it. Defer behavior changes, backfill, or production DDL. | +| conversations.auth_opt_fb default literal versus absent | SYNC BEFORE STABLE | Record exact live default/nullability in the file contract and reconcile affected generated readers. Existing M0 does not reject it. Defer behavior changes, backfill, or production DDL. | +| conversations.auth_opt_tw default literal versus absent | SYNC BEFORE STABLE | Record exact live default/nullability in the file contract and reconcile affected generated readers. Existing M0 does not reject it. Defer behavior changes, backfill, or production DDL. | +| conversations.auth_opt_allow_3rdparty default literal versus absent | SYNC BEFORE STABLE | Record exact live default/nullability in the file contract and reconcile affected generated readers. Existing M0 does not reject it. Defer behavior changes, backfill, or production DDL. | +| contexts_context_id_seq | SYNC BEFORE STABLE | Record exact bigint sequence with integer owning column; preserve current state and consumers, no setval/ALTER. Existing adoption does not check its bounds. | +| conversations_zid_seq | SYNC BEFORE STABLE | Record exact bigint sequence with integer owning column; preserve current state and consumers, no setval/ALTER. Existing adoption does not check its bounds. | +| courses_course_id_seq | SYNC BEFORE STABLE | Record exact bigint sequence with integer owning column; preserve current state and consumers, no setval/ALTER. Existing adoption does not check its bounds. | +| participant_metadata_answers_pmaid_seq | SYNC BEFORE STABLE | Record exact bigint sequence with integer owning column; preserve current state and consumers, no setval/ALTER. Existing adoption does not check its bounds. | +| participant_metadata_questions_pmqid_seq | SYNC BEFORE STABLE | Record exact bigint sequence with integer owning column; preserve current state and consumers, no setval/ALTER. Existing adoption does not check its bounds. | +| users_uid_seq | SYNC BEFORE STABLE | Record exact bigint sequence with integer owning column; preserve current state and consumers, no setval/ALTER. Existing adoption does not check its bounds. | +| random_polis_site_id(integer) | SET ASIDE | Retain unused integer overload as ancillary legacy state; no current direct caller established. | +| get_times_for_most_recent_visible_comments() | SYNC BEFORE STABLE | Record reviewed production routine body/fingerprint for live trigger/caller contract. Executable equivalence established; no CREATE OR REPLACE and no existing body-hash adoption refusal. | +| pid_auto() | SYNC BEFORE STABLE | Record reviewed production routine body/fingerprint for live trigger/caller contract. Executable equivalence established; no CREATE OR REPLACE and no existing body-hash adoption refusal. | +| tid_auto() | SYNC BEFORE STABLE | Record reviewed production routine body/fingerprint for live trigger/caller contract. Executable equivalence established; no CREATE OR REPLACE and no existing body-hash adoption refusal. | +| random_string(integer) | SYNC BEFORE STABLE | Record reviewed production routine body/fingerprint for live trigger/caller contract. Executable equivalence established; no CREATE OR REPLACE and no existing body-hash adoption refusal. | +| random_polis_site_id() | SYNC BEFORE STABLE | Record reviewed production routine body/fingerprint for live trigger/caller contract. Executable equivalence established; no CREATE OR REPLACE and no existing body-hash adoption refusal. | +| comments | SYNC BEFORE STABLE | Bind existing optional probe provisioning contract; exact six SELECT/public-USAGE grants already have a source. No role/grant changes. | +| conversations | SYNC BEFORE STABLE | Bind existing optional probe provisioning contract; exact six SELECT/public-USAGE grants already have a source. No role/grant changes. | +| math_main | SYNC BEFORE STABLE | Bind existing optional probe provisioning contract; exact six SELECT/public-USAGE grants already have a source. No role/grant changes. | +| math_ticks | SYNC BEFORE STABLE | Bind existing optional probe provisioning contract; exact six SELECT/public-USAGE grants already have a source. No role/grant changes. | +| participants | SYNC BEFORE STABLE | Bind existing optional probe provisioning contract; exact six SELECT/public-USAGE grants already have a source. No role/grant changes. | +| votes | SYNC BEFORE STABLE | Bind existing optional probe provisioning contract; exact six SELECT/public-USAGE grants already have a source. No role/grant changes. | +| public | SYNC BEFORE STABLE | Bind existing optional probe provisioning contract; exact six SELECT/public-USAGE grants already have a source. No role/grant changes. | +| portable ownership and selected-release privileges | SYNC BEFORE STABLE | Use role parameters/capabilities, not hosted dbUser ownership in public bootstrap. Prove actual migration-session M19 grants/role authority and M23/M24 SET ROLE; reject partial conflicting installs. No role reassignment. | +| comments.curation | SYNC BEFORE STABLE | Record exact live/script-visible field in supported legacy file contract; reconcile generated declarations without changing live values. delphi/scripts/generate_cold_start_clojure.py:293–296 explicitly copies curation; client test hits are unrelated prose/UI. Fresh installs currently lack the script field. | +| conversations.dataset_explanation | SET ASIDE | No literal field reference at any scanned current pin; conversation-wide SELECT * and dynamic JSON remain indirect possibilities. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| conversations.is_curated | SET ASIDE | No literal field reference at the scanned current pins; generic conversation readers can expose it. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| conversations.need_suzinvite | SET ASIDE | No literal field reference at the scanned current pins; do not confuse auth handlers for suzinvites with this dormant flag. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| participants_extended.country_iso_code | SET ASIDE | No literal field reference in the scanned current pins. M7 removes country_code_iso (different spelling), NOT this field. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| participants_extended.encrypted_ip_address | SYNC BEFORE STABLE | Record exact live/script-visible field in supported legacy file contract; reconcile generated declarations without changing live values. server/src/participant.ts:291–300 and db/sql.ts:96–103 conditionally write/declare this field only for applicationName PolisWebServer. | +| participants_extended.encrypted_x_forwarded_for | SYNC BEFORE STABLE | Record exact live/script-visible field in supported legacy file contract; reconcile generated declarations without changing live values. server/src/participant.ts:291–300 and db/sql.ts:96–103 conditionally write/declare this field only for applicationName PolisWebServer. | +| suzinvites.modified | SYNC BEFORE STABLE | Record exact live/script-visible field in supported legacy file contract; reconcile generated declarations without changing live values. server/src/invites/suzinvites.ts:17,68 uses SELECT *; INSERTs at 40 and 167 omit modified. Whole-row consumers can observe it; bare modified inventory includes unrelated tables. | +| suzinvites.uid | SYNC BEFORE STABLE | Record exact live/script-visible field in supported legacy file contract; reconcile generated declarations without changing live values. server/src/invites/suzinvites.ts:17,68 SELECT *; normal INSERTs omit uid. Bare uid matches are broad; table-specific paths are the relevant indirect contract. | +| users.pwhash | SYNC BEFORE STABLE | Record exact live/script-visible field in supported legacy file contract; reconcile generated declarations without changing live values. bin/anonymize_users.sh: updates pwhash; generated Rust users row/catalog retains pwhash; M0 comments it out. No current password-login consumer established. | +| users.test | SET ASIDE | Bare test appears throughout test tooling and is not a users.test consumer; generic users SELECT * can expose it. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| conversation_invite_codes | SET ASIDE | No current literal SQL consumer established; whole-tree locations/zero-results and historical SQL pickaxe are retained. Generic queue/moderators hits are not proof of public.conversation_invite_codes usage. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| conversation_subscriptions | SET ASIDE | No current literal SQL consumer established; whole-tree locations/zero-results and historical SQL pickaxe are retained. Generic queue/moderators hits are not proof of public.conversation_subscriptions usage. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| error_reports | SET ASIDE | No current literal SQL consumer established; whole-tree locations/zero-results and historical SQL pickaxe are retained. Generic queue/moderators hits are not proof of public.error_reports usage. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| math_results_dev01 | SET ASIDE | No current literal SQL consumer established; whole-tree locations/zero-results and historical SQL pickaxe are retained. Generic queue/moderators hits are not proof of public.math_results_dev01 usage. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| minvites | SET ASIDE | No current literal SQL consumer established; whole-tree locations/zero-results and historical SQL pickaxe are retained. Generic queue/moderators hits are not proof of public.minvites usage. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| moderators | SET ASIDE | No current literal SQL consumer established; whole-tree locations/zero-results and historical SQL pickaxe are retained. Generic queue/moderators hits are not proof of public.moderators usage. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| nyt_users | SET ASIDE | No current literal SQL consumer established; whole-tree locations/zero-results and historical SQL pickaxe are retained. Generic queue/moderators hits are not proof of public.nyt_users usage. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| polismath_mod_claims | SET ASIDE | No current literal SQL consumer established; whole-tree locations/zero-results and historical SQL pickaxe are retained. Generic queue/moderators hits are not proof of public.polismath_mod_claims usage. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| queue | SET ASIDE | No current literal SQL consumer established; whole-tree locations/zero-results and historical SQL pickaxe are retained. Generic queue/moderators hits are not proof of public.queue usage. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| slack_participants_waiting_for_comments | SET ASIDE | No current literal SQL consumer established; whole-tree locations/zero-results and historical SQL pickaxe are retained. Generic queue/moderators hits are not proof of public.slack_participants_waiting_for_comments usage. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| slack_state_heap | SET ASIDE | No current literal SQL consumer established; whole-tree locations/zero-results and historical SQL pickaxe are retained. Generic queue/moderators hits are not proof of public.slack_state_heap usage. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| slack_state_stack | SET ASIDE | No current literal SQL consumer established; whole-tree locations/zero-results and historical SQL pickaxe are retained. Generic queue/moderators hits are not proof of public.slack_state_stack usage. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| slack_team_tokens | SET ASIDE | No current literal SQL consumer established; whole-tree locations/zero-results and historical SQL pickaxe are retained. Generic queue/moderators hits are not proof of public.slack_team_tokens usage. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| animals_id_auto | SET ASIDE | No current literal routine call established; votes_lastest_unique appears only as a bootstrap comment. All SQL snapshot bodies are available historically; current production body fingerprints are compared below. Dynamic/external SQL and trigger dependencies need the aggregate census. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| oid_auto | SET ASIDE | No current literal routine call established; votes_lastest_unique appears only as a bootstrap comment. All SQL snapshot bodies are available historically; current production body fingerprints are compared below. Dynamic/external SQL and trigger dependencies need the aggregate census. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| oid_auto_unlock | SET ASIDE | No current literal routine call established; votes_lastest_unique appears only as a bootstrap comment. All SQL snapshot bodies are available historically; current production body fingerprints are compared below. Dynamic/external SQL and trigger dependencies need the aggregate census. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| ptpt_id_auto | SET ASIDE | No current literal routine call established; votes_lastest_unique appears only as a bootstrap comment. All SQL snapshot bodies are available historically; current production body fingerprints are compared below. Dynamic/external SQL and trigger dependencies need the aggregate census. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| ptpt_id_auto_unlock | SET ASIDE | No current literal routine call established; votes_lastest_unique appears only as a bootstrap comment. All SQL snapshot bodies are available historically; current production body fingerprints are compared below. Dynamic/external SQL and trigger dependencies need the aggregate census. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| to_zinvite.integer | SET ASIDE | No current literal routine call established; votes_lastest_unique appears only as a bootstrap comment. All SQL snapshot bodies are available historically; current production body fingerprints are compared below. Dynamic/external SQL and trigger dependencies need the aggregate census. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| votes_foo.integer | SET ASIDE | No current literal routine call established; votes_lastest_unique appears only as a bootstrap comment. All SQL snapshot bodies are available historically; current production body fingerprints are compared below. Dynamic/external SQL and trigger dependencies need the aggregate census. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| votes_lastest_unique.integer | SET ASIDE | No current literal routine call established; votes_lastest_unique appears only as a bootstrap comment. All SQL snapshot bodies are available historically; current production body fingerprints are compared below. Dynamic/external SQL and trigger dependencies need the aggregate census. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| foobar | SET ASIDE | The four literal foobar hits are unrelated Clojure visualization/Rust child fixtures; no sequence user established. Historical dump contains it. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| animal_grp | SET ASIDE | No current literal consumer in tracked trees; historical animals_id_auto body references NEW.grp/enum_range. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| auth_tokens.auth_tokens_token_idx | SET ASIDE | Index planner use is implicit through auth_tokens queries; table-name inventory lists indirect consumers. No application can be declared independent solely because it does not name the index. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| participants.participants_uid_index | SET ASIDE | Index planner use is implicit through participants queries; table-name inventory lists indirect consumers. No application can be declared independent solely because it does not name the index. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| pwreset_tokens.pwreset_tokens_token_idx | SET ASIDE | Index planner use is implicit through pwreset_tokens queries; table-name inventory lists indirect consumers. No application can be declared independent solely because it does not name the index. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| votes.votes_zid_idx | SET ASIDE | Index planner use is implicit through votes queries; table-name inventory lists indirect consumers. No application can be declared independent solely because it does not name the index. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| postgis | SET ASIDE | server/Dockerfile-pdb:1 retains postgis image; bin/remove_postgis.sh is a destructive cleanup helper, NOT permission to run it. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| tablefunc | SET ASIDE | No literal current tree consumer; historical dump declares CREATE EXTENSION tablefunc. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| pg_stat_statements | SET ASIDE | server/src/ops/database.ts:4 explicitly says pg_stat_statements is NOT used (ruling R5: extension creation is a database change); the two unit-test hits do not establish a runtime consumer. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| unattributed_extension_members | SET ASIDE | Exact extension ownership remains unproven: 794 routine and 109 parent-object count mappings ready. Retain all candidates. No selected migration alters these objects; classification and any cleanup stay separate. | +| constraint_index_identity | SET ASIDE | Catalog generator and runner use semantic definitions. Exact alias-name consumers require a separate identifier search before any optional rename. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| social_settings.social_settings_uid_key | SET ASIDE | No current live uniqueness consumer established; generated declarations and historical usage alone do not gate queue migrations. Preserve existing unique constraint. | +| suzinvites.suzinvites_uid_fkey | SYNC BEFORE STABLE | Record existing live uniqueness/FK in file contract; no constraint creation or deletion. | +| conversations.conversations_zid_index | SET ASIDE | Redundant index variant does not block selected migrations. Retain current indexes, document optional shape; no rebuild. | +| users.users_uid_idx | SET ASIDE | Redundant index variant does not block selected migrations. Retain current indexes, document optional shape; no rebuild. | diff --git a/docs/migration-reconciliation.md b/docs/migration-reconciliation.md new file mode 100644 index 0000000000..9f99721148 --- /dev/null +++ b/docs/migration-reconciliation.md @@ -0,0 +1,70 @@ +# Reconcile an existing deployment + +Keep one record per database with the release being installed. This is a review +record for the normal `reconcile` → `apply` → `check` path; it is not a list of +host-specific skipped migrations. Never copy credentials, participant rows or +private identifiers into the public repository. + +1. Record the release commit and exact migration source hashes, database major + version, current application version, backup/restore evidence and installer + privileges. Confirm the previous application's stop hook: the first move to + deferred-stop hooks can still execute the old hook before migration refusal. +2. Collect a read-only catalog inventory through the deployment's existing DB + client. Preserve types, defaults, nullability, keys, index validity, routines, + triggers, grants and sequence configuration. Encode bigint values as strings + when transporting them through JavaScript JSON. Do not read sequence values + or vote/application rows as a shortcut to schema reconciliation. +3. Classify every logical migration: present, pending, partial, conflicting, + superseded by a known later contract, or outside the forward chain. Cite its + observed postconditions. Names, owners and physical column positions can + differ without changing the structure; compare key column names and actual + definitions. Do not count internal FK trigger OIDs as missing migrations. +4. Retain historical receipts separately. Filename aliases, duplicate legacy + timestamps and a current matching schema are different kinds of evidence. + Unknown legacy filenames, competing ledgers, nonempty undeclared vote data + and unknown routine implementations need reviewed resolution. Do not invent + execution dates or treat a latest filename as a complete prefix. +5. Prepare and test each needed forward repair on generated databases representing + the actual structural variant. Preserve existing rows, unrelated schemas, + grants and third-party dependants. Do not truncate or drop data to satisfy a + catalog predicate; do not change stored vote signs or overwrite a custom + routine merely to match a fresh installation. Older retirement SQL is not + authorization to delete retained deployment data. +6. Once all selected adoption postconditions are supported and reviewed, run + `polis-migrate reconcile --through NNNNNN` with the factual bound. A failure + must leave all prior history untouched. Then `apply` the ordered pending + migrations and run `check`. No history rows may be inserted manually. +7. Record the actual per-file outcomes and postconditions. M22 is resumable: + indexes can commit before their history row; inspect validity after a lost + connection and reuse valid results. Every other committed earlier file stays + applied if a later file fails. Do not run down scripts as automatic recovery. +8. Verify application health after service replacement. Detached Compose startup + alone is not a serving-health receipt. Keep migration readiness, process + readiness and application health as separate observations. + +Record fields: + +| Field | Evidence required | +| --- | --- | +| Release | Commit, SQL source hashes, release holds and pending PR composition | +| Observation | Catalog query version/hash, collection date, scope and transport | +| Per file | Logical identity, known aliases, state, exact postconditions and differences | +| History | Actual receipts and their provenance, or explicitly unknown | +| Preservation | Rows, custom objects, grants, active-writer compatibility and backup | +| Repair | Exact reviewed patch, generated regression evidence, failure recovery | +| Adoption | Explicit bound, atomic success or unchanged-history failure | +| Pending work | Ordered apply outcomes, concurrent-index progress where applicable | +| Completion | Runner check plus actual application health evidence | + +Fresh databases use `apply`, not reconciliation. Existing /1, /2 or /3 queues +must pass their recorded catalog contracts; install-table presence is not proof. +A partial legacy install needs compatible forward completion, not a fabricated +adoption. M21's release hold remains until its privilege, sequence and publisher +requirements are resolved. Convention declaration and any semantic sign operation +remain explicit operations; neither a catalog nor a migration merge supplies the +operator's declaration. + +This record does not by itself remediate unsupported historical variants, +retire destructive files, compose competing unreleased migration ledgers, or +make manual down scripts history-aware. Those release changes and their tests +must be complete before claiming an upgrade works across supported deployments. diff --git a/docs/migration-release-map.md b/docs/migration-release-map.md new file mode 100644 index 0000000000..28b207a658 --- /dev/null +++ b/docs/migration-release-map.md @@ -0,0 +1,77 @@ +# Migration source to release map + +These 46 source variants map to releases by exact SQL bytes. Source membership +does not prove execution on any database. The only actual source version label +established by the historical research is `1.0`; subsequent semantic versions +were not consistently assigned. Strict/OIDC numbers below are the previously +reviewed retrospective proposals, **not published tags or a new version ruling**. +The current M19/M23/M24/M27 bytes first ship at promotion PR #2994 (`5ded2e0a9`); +their semantic version is **UNASSIGNED**. A release owner must assign the real +release version before publishing one; this patch invents none. + +For an installation's execution record, use its receipts or record execution +as unknown. Never infer an apply date from this source table. The historical +version research is tracked separately from schema adoption. + +| Audited source variant | Role | Filename first present | Exact audited bytes first present | Disposition | +| --- | --- | --- | --- | --- | +| `000000_initial.sql` (#2997 reference; `2652134140cd`) | numbered_forward | declared 1.0 source (anchor) | `53817ae39`; strict 3.24.2 / OIDC 2.24.2 | SHIPPED_SOURCE_BYTES | +| `000001_update_pwreset_table.sql` (#2997 reference; `cb21278194c4`) | numbered_forward | declared 1.0 source (anchor) | declared 1.0 source (anchor) | SHIPPED_SOURCE_BYTES | +| `000002_add_xid_constraint.sql` (#2997 reference; `a27a8e63c79c`) | numbered_forward | declared 1.0 source (anchor) | declared 1.0 source (anchor) | SHIPPED_SOURCE_BYTES | +| `000003_add_origin_permanent_cookie_columns.sql` (#2997 reference; `0d7f27facfec`) | numbered_forward | declared 1.0 source (anchor) | declared 1.0 source (anchor) | SHIPPED_SOURCE_BYTES | +| `000004_drop_waitinglist_table.sql` (#2997 reference; `f2fc4184a965`) | numbered_forward | declared 1.0 source (anchor) | declared 1.0 source (anchor) | SHIPPED_SOURCE_BYTES | +| `000005_drop_slack_stripe_canvas.sql` (#2997 reference; `392e5b8aadb7`) | numbered_forward | declared 1.0 source (anchor) | declared 1.0 source (anchor) | SHIPPED_SOURCE_BYTES | +| `000006_update_votes_rule.sql` (#2997 reference; `8fb05b7b1b6a`) | numbered_forward | declared 1.0 source (anchor) | declared 1.0 source (anchor) | SHIPPED_SOURCE_BYTES | +| `000007_drop_geolocation_fields.sql` (#2997 reference; `f68b69b86112`) | numbered_forward | `8040579ef`; strict 1.1.0 / OIDC 1.1.0 | `8040579ef`; strict 1.1.0 / OIDC 1.1.0 | SHIPPED_SOURCE_BYTES | +| `000008_add_comment_priority.sql` (#2997 reference; `c867b53be3cc`) | numbered_forward | `8040579ef`; strict 1.1.0 / OIDC 1.1.0 | `8040579ef`; strict 1.1.0 / OIDC 1.1.0 | SHIPPED_SOURCE_BYTES | +| `000009_add_uuid_to_zinvites.sql` (#2997 reference; `43f36fe0b857`) | numbered_forward | `4ba8b0938`; strict 1.6.0 / OIDC 1.6.0 | `4ba8b0938`; strict 1.6.0 / OIDC 1.6.0 | SHIPPED_SOURCE_BYTES | +| `000010_create_oidc_user_mappings.sql` (#2997 reference; `450a3f69883a`) | numbered_forward | `3a7da1468`; strict 3.0.0 / OIDC 2.0.0 | `3a7da1468`; strict 3.0.0 / OIDC 2.0.0 | SHIPPED_SOURCE_BYTES | +| `000011_alter_suzinvites_xid_to_text.sql` (#2997 reference; `00a1eb2d9604`) | numbered_forward | `3a7da1468`; strict 3.0.0 / OIDC 2.0.0 | `3a7da1468`; strict 3.0.0 / OIDC 2.0.0 | SHIPPED_SOURCE_BYTES | +| `000012_create_topic_agenda_selections.sql` (#2997 reference; `cc513693124f`) | numbered_forward | `199569498`; strict 3.4.0 / OIDC 2.4.0 | `199569498`; strict 3.4.0 / OIDC 2.4.0 | SHIPPED_SOURCE_BYTES | +| `000013_create_treevite.sql` (#2997 reference; `4b8334f73246`) | numbered_forward | `b986fca0f`; strict 3.6.0 / OIDC 2.6.0 | `b986fca0f`; strict 3.6.0 / OIDC 2.6.0 | SHIPPED_SOURCE_BYTES | +| `000014_alter_reports_modlevel.sql` (#2997 reference; `c2af6d57af28`) | numbered_forward | `ee4405a44`; strict 3.7.2 / OIDC 2.7.2 | `ee4405a44`; strict 3.7.2 / OIDC 2.7.2 | SHIPPED_SOURCE_BYTES | +| `000015_add_xid_requirements.sql` (#2997 reference; `186c904addd0`) | numbered_forward | `a6d7215f7`; strict 3.19.0 / OIDC 2.19.0 | `06d6fa1af`; strict 3.23.0 / OIDC 2.23.0 | SHIPPED_SOURCE_BYTES | +| `000016_add_orig_id.sql` (#2997 reference; `6cdc0588c000`) | numbered_forward | `b13b316e9`; strict 3.21.0 / OIDC 2.21.0 | `b13b316e9`; strict 3.21.0 / OIDC 2.21.0 | SHIPPED_SOURCE_BYTES | +| `000017_create_byod_job_table.sql` (#2997 reference; `f27c03a1229f`) | numbered_forward | `b13b316e9`; strict 3.21.0 / OIDC 2.21.0 | `b13b316e9`; strict 3.21.0 / OIDC 2.21.0 | SHIPPED_SOURCE_BYTES | +| `000018_add_topics_enabled.sql` (#2997 reference; `a1e1c0572064`) | numbered_forward | `06d6fa1af`; strict 3.23.0 / OIDC 2.23.0 | `06d6fa1af`; strict 3.23.0 / OIDC 2.23.0 | SHIPPED_SOURCE_BYTES | +| `000019_create_polis_queue.sql` (#2997 reference; `fedfbcf9fc59`) | numbered_forward | `335418338`; strict 3.28.0 / OIDC 2.28.0 | `5ded2e0a9`; strict UNASSIGNED / OIDC UNASSIGNED | SHIPPED_SOURCE_BYTES | +| `000020_create_math_source_journal.sql` (PR#2739; `24c1dff07637`) | numbered_forward | NOT SHIPPED at checked endpoints | NOT SHIPPED at checked endpoints | PENDING_VARIANT_NOT_SHIPPED | +| `000021_create_polis_coordinator.sql` (#2997 reference; `d50f169ad7af`) | numbered_forward | `335418338`; strict 3.28.0 / OIDC 2.28.0 | `335418338`; strict 3.28.0 / OIDC 2.28.0 | SHIPPED_SOURCE_BYTES | +| `000022_add_poll_timestamp_indexes.sql` (#2997 reference; `14efc95b1478`) | numbered_forward | `2547b6ee5`; strict 3.30.0 / OIDC 2.30.0 | `2547b6ee5`; strict 3.30.0 / OIDC 2.30.0 | SHIPPED_SOURCE_BYTES | +| `000023_create_delphi_foundation.sql` (#2997 reference; `97437ea57d90`) | numbered_forward | `5ded2e0a9`; strict UNASSIGNED / OIDC UNASSIGNED | `5ded2e0a9`; strict UNASSIGNED / OIDC UNASSIGNED | SHIPPED_SOURCE_BYTES | +| `000024_create_polis_queue_large_class.sql` (#2997 reference; `68261afb81f2`) | numbered_forward | `5ded2e0a9`; strict UNASSIGNED / OIDC UNASSIGNED | `5ded2e0a9`; strict UNASSIGNED / OIDC UNASSIGNED | SHIPPED_SOURCE_BYTES | +| `000025_vote_convention.sql` (PR#2944; `cfff57e4f416`) | numbered_forward | NOT SHIPPED at checked endpoints | NOT SHIPPED at checked endpoints | PENDING_VARIANT_NOT_SHIPPED | +| `000026_create_polis_queue_retention.sql` (PR#2978; `ee29e37e94f5`) | numbered_forward | NOT SHIPPED at checked endpoints | NOT SHIPPED at checked endpoints | PENDING_VARIANT_NOT_SHIPPED | +| `000019_create_delphi_storage.sql` (PR#2600; `5808d85fe674`) | numbered_forward | NOT SHIPPED at checked endpoints | NOT SHIPPED at checked endpoints | PENDING_VARIANT_NOT_SHIPPED | +| `000012_create_topic_agenda_selections.sql` (PR#2110; `cc513693124f`) | numbered_forward | `199569498`; strict 3.4.0 / OIDC 2.4.0 | `199569498`; strict 3.4.0 / OIDC 2.4.0 | PENDING_VARIANT_BYTES_ALREADY_SHIPPED | +| `000000_initial.sql` (PR#2556; `183d22c0238a`) | numbered_forward | declared 1.0 source (anchor) | NOT SHIPPED at checked endpoints | PENDING_VARIANT_NOT_SHIPPED | +| `000026_create_polis_queue_retention.sql` (PR#2983; `0d1e357a72a9`) | numbered_forward | NOT SHIPPED at checked endpoints | NOT SHIPPED at checked endpoints | PENDING_VARIANT_NOT_SHIPPED | +| `000024_vote_sign_unflip.sql` (PR#2942; `3e67e0ab857d`) | held | NOT SHIPPED at checked endpoints | NOT SHIPPED at checked endpoints | PENDING_VARIANT_NOT_SHIPPED | +| `000023_vote_convention.sql` (PR#2942; `351628cfc1bc`) | fixture | NOT SHIPPED at checked endpoints | NOT SHIPPED at checked endpoints | PENDING_VARIANT_NOT_SHIPPED | +| `000025_drop_vote_convention.sql` (PR#2944; `9d747f6d0fe3`) | down | NOT SHIPPED at checked endpoints | NOT SHIPPED at checked endpoints | PENDING_VARIANT_NOT_SHIPPED | +| `000026_drop_polis_queue_retention.sql` (PR#2978; `747085bf1324`) | down | NOT SHIPPED at checked endpoints | NOT SHIPPED at checked endpoints | PENDING_VARIANT_NOT_SHIPPED | +| `000026_drop_polis_queue_retention.sql` (PR#2983; `063272749473`) | down | NOT SHIPPED at checked endpoints | NOT SHIPPED at checked endpoints | PENDING_VARIANT_NOT_SHIPPED | +| `db_000002.sql` (#2997 reference; `1f214c31e557`) | archive | declared 1.0 source (anchor) | declared 1.0 source (anchor) | SHIPPED_SOURCE_BYTES | +| `db_000004.sql` (#2997 reference; `3afacfa4c4e6`) | archive | declared 1.0 source (anchor) | declared 1.0 source (anchor) | SHIPPED_SOURCE_BYTES | +| `db_000006.sql` (#2997 reference; `4c6f61fa0a5b`) | archive | declared 1.0 source (anchor) | declared 1.0 source (anchor) | SHIPPED_SOURCE_BYTES | +| `db_000008.sql` (#2997 reference; `e5d4990d1dd7`) | archive | declared 1.0 source (anchor) | declared 1.0 source (anchor) | SHIPPED_SOURCE_BYTES | +| `db_000010.sql` (#2997 reference; `498344c75e9c`) | archive | declared 1.0 source (anchor) | declared 1.0 source (anchor) | SHIPPED_SOURCE_BYTES | +| `000019_drop_polis_queue.sql` (#2997 reference; `483de532876f`) | down | `335418338`; strict 3.28.0 / OIDC 2.28.0 | `335418338`; strict 3.28.0 / OIDC 2.28.0 | SHIPPED_SOURCE_BYTES | +| `000021_drop_polis_coordinator.sql` (#2997 reference; `f8547afa1e87`) | down | `335418338`; strict 3.28.0 / OIDC 2.28.0 | `335418338`; strict 3.28.0 / OIDC 2.28.0 | SHIPPED_SOURCE_BYTES | +| `000022_drop_poll_timestamp_indexes.sql` (#2997 reference; `fcf1f4695e48`) | down | `2547b6ee5`; strict 3.30.0 / OIDC 2.30.0 | `2547b6ee5`; strict 3.30.0 / OIDC 2.30.0 | SHIPPED_SOURCE_BYTES | +| `000023_drop_delphi_foundation.sql` (#2997 reference; `aa0a3d67766a`) | down | `5ded2e0a9`; strict UNASSIGNED / OIDC UNASSIGNED | `5ded2e0a9`; strict UNASSIGNED / OIDC UNASSIGNED | SHIPPED_SOURCE_BYTES | +| `000024_drop_polis_queue_large_class.sql` (#2997 reference; `08735f922a17`) | down | `5ded2e0a9`; strict UNASSIGNED / OIDC UNASSIGNED | `5ded2e0a9`; strict UNASSIGNED / OIDC UNASSIGNED | SHIPPED_SOURCE_BYTES | + + +The M20/M25/M26 PR-only variants, unflip fixture, archives and down scripts +are not in this release manifest. M21 is held. M4/M5/M7 are retained historical +sources and can only receive observed ADOPTED receipts in ordinary apply. + +## Release B addition + +M27 (`000027_create_sealed_job_graphs.sql`, SHA-256 +`fbbf948e4316010dd96344ae91542006c38299e52e0a0eb281371c039ab37775`) +is selected after M19/M23/M24. It adds the approved per-step graph core. +Its strict release version remains UNASSIGNED until the release decision; +this source selection is not evidence of a production application. +M28/M29 and the DynamoDB readers/importer are not part of this release. diff --git a/docs/migration-upgrade-notes.md b/docs/migration-upgrade-notes.md new file mode 100644 index 0000000000..37be4635b5 --- /dev/null +++ b/docs/migration-upgrade-notes.md @@ -0,0 +1,100 @@ +# Migration runner upgrade notes + +The first selected legacy upgrade records the supported existing schema, then +applies only M19, M23, M24 and M27. They create the job-system tables and routines. +They do not enable any worker or application feature flag. Production is never +changed merely to match old bootstrap files. + +## Choose the installation path + +| Starting state | Required path | Expected outcome | +| --- | --- | --- | +| Empty PostgreSQL 17+ | `apply`, then `check` | 21 executed files, three retirement ADOPTED receipts, 24 ready | +| Supported legacy schema through M18 plus valid M22, no queue/history | first-deploy report, `reconcile --through 000022`, `apply`, `check` | 20 ADOPTED; exactly M19/M23/M24/M27 APPLIED | +| Supported legacy schema through M18, missing M22 | `reconcile --through 000018`, `apply`, `check` | 19 ADOPTED; M19, concurrent M22, M23/M24/M27 APPLIED | +| Existing queue /1, /2 or /3 | exact installed catalog/receipts review, reconcile through actual installed version, then apply/check | compatible installed queue retained; only pending selected files execute | +| Modern valid runner history | `apply`, `check` | existing APPLIED/ADOPTED receipts retained; rerun executes zero | +| Legacy ledger with known filenames | bounded reconcile after catalog review | timestamps preserved, no fictional execution receipts | +| Retained M4/M5/M7 targets | stop for separately reviewed completion/retention plan | no automatic destructive migration or fake adoption | +| Partial/conflicting schema, unknown ledger/file, later queue variant | stop for explicit reviewed reconciliation | no blind replay, manual history insertion or source rewriting | + +The historical bootstrap and named legacy schema are distinct supported variants. +See [their exact contract](migration-legacy-contract.md). A successful limited +adoption contract is not an exhaustive certification of every custom object, +extension, historical routine implementation, data invariant or application path. + +## Deprecated removal files + +M4 historically removed `waitinglist`. M5 removed Slack OAuth/users/invites/bot +events, Stripe accounts/subscriptions, free-upgrade coupons, LTI users/context +memberships/OAuth credentials, Canvas callback/conversation tables, and the +`conversations.is_slack`, `conversations.lti_users_only`, `users.plan` columns. +M7 removed `geolocation_cache` and participants_extended's `country_code_iso`, +`encrypted_maxmind_response_city`, `ip_address`, `latitude`, `location`, +`longitude` and `x_forwarded_for` fields. + +All three files remain immutable historical evidence. Ordinary upgrades never +execute them. Absence of the exact named objects allows an ADOPTED receipt; +retained targets block without deletion. Other Slack tables, the differently +named `participants_extended.country_iso_code`, encrypted network fields, and +`facebook_users.location`/`twitter_users.location` are not removal targets. +A composite type's `location` attribute is not a participants_extended column. + +## Report-only first-deploy check + +`server/postgres/migrations/report/first-deploy.sql` is the first-deploy report. +Its SQL predicates are generated directly from the reviewed adoption sources, +with helper expressions inlined. It uses one bounded REPEATABLE READ READ ONLY +transaction followed by ROLLBACK, creates no temporary functions/tables, reads +no application rows, and returns only migration names, booleans and outcomes. +`python3 server/bin/build-migration-report.py --check` verifies its exact source +binding; regeneration is a reviewable file change. + +The operator sends the entire SQL file through **one connection** of the existing +server's database client, with existing connection/TLS settings. Do not deploy +the new runner or restart the app just to report. For a local or controlled +operator session, `psql -X -v ON_ERROR_STOP=1 -f ` is equivalent, using +the normal secret connection environment without a credential argument. On any +query error, ROLLBACK or close that same client before returning it to a pool. + +Expected initial-schema results: 20 `WOULD_ADOPT`, four `WOULD_APPLY`, and four +`OUTSIDE_RELEASE` entries. The report conservatively flags any existing queue +role for migration-session review rather than guessing inherited ADMIN/SET +authority. Existing ledgers/queues report review-required; they need the exact +normal reconciliation catalog verifier and a separate read-only examination. +A mismatch in any required predicate blocks the aggregate adoption forecast. +A runtime read-only role can report schema but cannot predict privileged DDL +success; check the intended migration session separately. + +The report proves no future lock acquisition, available capacity, successful +DDL or application health. Its snapshot expires; reconcile and apply recheck +their actual contracts when executed. M22 presence in the initial report is +required; missing M22 uses the explicit shorter-bound path above. + +## Privileges, interruption and deployment + +Use the intended migration session, including M19 role provisioning and grant +authority. Owning only the database or having normal app SELECT/INSERT access +is insufficient. M23/M24/M27 SET ROLE to the queue owner. Never grant broad rights +to a runtime login just to satisfy startup; startup needs metadata SELECT. + +Each ordinary file and its history receipt commit together. If M23 fails after +M19 commits, M19 remains applied while all M23 changes and its receipt roll back. +Do not describe that as a rollback of the whole release. M22 is the documented +concurrent-index exception; interrupted valid indexes may persist and be reused. +Check invalid/conflicting index recovery in [migrations.md](migrations.md). +Two runners serialize using the database advisory lock; they do not double-apply. +Lost commit connections require history inspection, not an assumed rollback. + +On the first deployment, an old successful revision's stop hook may have already +stopped the old application before the new migration hook runs. New deferred-stop +hooks protect old containers only after that transition. A later API startup +failure may occur after old containers have been removed. This patch supplies +no automatic rollback, previous-image fallback or CodeDeploy health guarantee. +Verify actual HTTP/application health after replacement and preserve a reviewed +recovery procedure. A detached Compose launch is insufficient evidence. + +Keep a per-deployment [reconciliation record](migration-reconciliation.md), +including exact source hashes, schema report, actual session authority, observed +receipts, supported variants, backup evidence and health result. Do not publish +credentials, application records or real conversation/report identifiers. diff --git a/docs/migrations.md b/docs/migrations.md index 307609c77d..4cc8f7c590 100644 --- a/docs/migrations.md +++ b/docs/migrations.md @@ -1,40 +1,201 @@ +# Database migrations -# Database Migrations +Deploy the schema before replacing application services. Every deployment uses +`polis-migrate deploy`; the API refuses startup if a required file is pending, +history is missing, or a recorded checksum disagrees with the release. +PostgreSQL 17 or newer is required. Keep a tested backup/restore procedure. -When we need to update the Polis database, we use SQL migration files. +Build once from the repository root (or use the release's migration image): -During initial provisioning of your Docker containers, all the migrations will be applied in order, and you won't need to think about this. -But if we update the database schema after your initial provisioning of your server via Docker, you'll need to manually apply each new SQL migration. +```sh +cargo build --locked --release --manifest-path queue-rs/Cargo.toml -p polis-migrate +export PATH="$PWD/queue-rs/target/release:$PATH" +# Set DATABASE_URL through your usual secret environment; never put it in argv. +polis-migrate apply +polis-migrate check +``` + +The image requires no local Rust installation: + +```sh +docker build -t polis-migrate:local -f queue-rs/polis-migrate/Dockerfile . +docker run --rm --network host --env-file .env polis-migrate:local apply +``` + +Use the database's actual network when it is inside Compose instead of the host +network. `POLIS_MIGRATIONS_DIR` or `--dir` selects the release's SQL directory. +TLS validates the certificate and hostname. The runner and API images include +a checksum-pinned public RDS CA bundle (official global truststore, retrieved +2026-10-09; SHA-256 `fe45bbebf92ad3e27a583bbb2ddd1553c521ed4d49af5514dc0a40372ea5395c`). `POLIS_MIGRATE_CA_FILE` supplies a +private PEM CA bundle (mount it into the container). Standalone Node startup +uses `NODE_EXTRA_CA_CERTS` for a private/RDS CA; standalone Rust uses +`POLIS_MIGRATE_CA_FILE`. For a disposable local +network only, a URL with `sslmode=disable` plus +`POLIS_MIGRATE_ALLOW_PLAINTEXT=local` permits Docker service DNS; loopback and +local sockets can use `sslmode=disable` without that variable. Never use this +local setting for a remote production database. -- Please note: **Backups are your responsibility.** These instructions assume - the data is disposable, and do not attempt to make backups. - - Pull requests are welcome if you'd like to see more guidance on this. - - Please submit an issue if you'd like to work on enabling backups through Docker Compose. -- Your database data is stored on a docker volume, which means that it will - persist even when you destroy all your docker containers. Be mindful of this. - - You can remove ALL volumes defined within a `docker-compose` file via: `docker compose --profile postgres down --volumes` - - You can remove ONE volume via `docker volume ls` and `docker volume rm ` -- SQL migrations can be found in [`server/postgres/migrations/`][] of this - repo. -- The path to the SQL file will be relative to its location in the docker - container filesystem, not your host system. +New Docker volumes run the same binary from `server/Dockerfile-db`. Existing +volumes do not rerun initialization: invoke `apply` before starting the new app. +The CodeDeploy hook invokes `deploy` (catalog-checked first adoption, apply, check) +before replacing containers. See the mandatory two-release transition in +[the upgrade guide](upgrading.md). Direct `apply` still refuses unadopted databases. +`server/bin/run-migrations.sh` delegates to the binary; the old Clojure entry +point delegates to that shell command. Neither replays every file anymore. -For example, if we add the migration file -`server/postgres/migrations/000001_update_pwreset_table.sql`, you'd run on your -host system: +## One-time adoption of an existing database + +An existing database without history is deliberately refused by `apply`. +Never replay `000000_initial.sql` on it. Use the known migration records and +inspect its schema to choose an explicit upper bound, then reconcile: ```sh -docker compose --profile postgres exec postgres psql --username postgres --dbname polis-dev --file=/docker-entrypoint-initdb.d/000001_update_pwreset_table.sql +# Example: the legacy schema through 000022, without the dormant queue. +polis-migrate reconcile --through 000022 +polis-migrate apply +polis-migrate check ``` -You can also run a local .sql file on a postgres container instance with this syntax: +The bound is not evidence of application. Every selected file must pass its +catalog postconditions in `server/postgres/migrations/adoption/`: enduring +initial tables/columns, renamed columns, unique and foreign-key constraints, +new columns/types/defaults, removed objects, the vote-update rule, and valid +index definitions. The exact [legacy variants](migration-legacy-contract.md) include six named +math `json` payloads and bounded type/column alternatives; unrelated types are refused. The missing queue migration 000019 is a recognized +hole **only when no queue/foundation tables or functions exist and no old +history row claims it**. The example adopts 000000–000018 and 000022 as ADOPTED; +`apply` then installs only 000019, 000023, 000024 and 000027. No vote data is inspected +or changed by reconciliation. + +Adoption is atomic: any failed postcondition leaves the old history untouched. +It records observation time and the checksum of the source being reconciled, +not a fictional historical execution date. If the 2021 `migrations(name, +completed_at)` table exists, its timestamps (including duplicates) are retained +in each row's `legacy_completed_at` array after all catalog checks pass. Unknown +or renamed legacy filenames stop for review; they are never silently replayed. +Already installed queues use their existing /1, /2 or /3 catalog verifiers, +including definitions/privileges and recorded installed catalogs. A matching +install row alone is insufficient. Select the actual upper version (for example +`--through 000024` for a previously initialized /3 Docker database). A partial +queue, later bound without a contract, unknown historical filename, or the +unreleased `schema_migrations` ledger stops for review. Never mark those by hand. + +## History and failure behavior + +`public.migrations` is the sole applied-history table. It records filename, +SHA-256 of the unmodified SQL source, APPLIED or ADOPTED, observation time/actor, +and any preserved legacy timestamps. Runtime roles need SELECT only, granted +on this metadata table; the deployment role needs the privileges required by +the actual migrations (including role administration for the queue). + +A database advisory lock serializes runners on the same connection across +per-file commits. A second runner waits up to five minutes, then reads the +committed history. Each ordinary file and its APPLIED row commit in one transaction. The two +000022 index builds are the explicit autocommit exception described below. +An optional historical outer BEGIN/COMMIT pair is removed; any other top-level +transaction control is rejected. Dollar-quoted function bodies stay intact. +SQL errors abort the file. Earlier successfully committed files stay applied. +Lock waits are bounded (five seconds for DDL), with five-minute statement and +transaction ceilings and a thirty-second idle-transaction limit. Existing SQL +may set tighter timeouts. No automatic destructive down migration runs. + +If the connection is lost at commit, the result can be unknown. Reconnect and +run `check`/`apply`: committed history is authoritative. Do not infer rollback +from a transport error. Restore changed historical files instead of editing +history to bypass checksum failures. + +For the exact released 000022 source, `apply` preflights both index identities, +then builds each missing watermark index with `CREATE INDEX CONCURRENTLY` in +autocommit, while retaining the same migration advisory lock. This is the normal +path on fresh and large populated databases. Valid existing indexes are checked +and preserved without rebuilding. The original SQL then rechecks both exact +index definitions in the transaction that records APPLIED. Its raw SQL checksum +is unchanged; changing that source requires review of this execution contract. + +An interruption can leave a valid first index, or an invalid index from a failed +concurrent build, without a migration history row. Rerunning `apply` reuses valid +indexes and builds only missing ones. It refuses invalid or conflicting objects +before building either index. Inspect an invalid index first; if it is the exact +interrupted watermark index, explicitly drop that index with `DROP INDEX +CONCURRENTLY public.` in autocommit, then rerun `apply`. The +runner never drops a preexisting index automatically. A differently defined +same-name object requires a reviewed resolution, not that drop instruction. +If history recording fails after both builds, both valid indexes remain; retry +records their checked state without rebuilding. Concurrent builds have the +same bounded statement/lock timeouts; a timeout is not a success receipt. + +## Release contents and new migrations + +`server/postgres/migrations/release.txt` explicitly selects every required file. +The Rust runner and Node startup gate use the same manifest; Docker initialization +uses that runner. A new top-level numbered SQL file must be selected or held, +otherwise startup/apply refuses it. Missing files, duplicate numbers or entries, +symlinks, and release/hold overlap refuse. Adding a file cannot silently extend +the release. Numbered source checksums remain unchanged. + +This release contains M0–M19 plus M22/M23/M24/M27. M4/M5/M7 are deprecated +observation-only entries. They never execute through `apply`: their named +removed objects must already be absent, then an ADOPTED receipt is recorded. +Any retained target stops before apply mutates schema/history; arrange a separate +reviewed upgrade for that deployment. Existing valid APPLIED receipts are kept. +Fresh bootstrap already omits their targets and gets three ADOPTED receipts; +it executes 21 files. No historical execution date is invented. + +M20 (draft journal), M21 (held coordinator), M25 (vote convention) and M26 +(retention) are excluded from the forward path. M21 stays in `held.txt`; the +other files are not shipped by this branch. An added unclassified copy refuses. +The selected legacy upgrade adopts M0–M18/M22 and applies **M19 → M23 → M24 → M27**. +A deployment missing M22 instead builds its indexes concurrently between M19 +and M23. Archives, down scripts and unflip files never enter this manifest. + +Read [the release map](migration-release-map.md) for historical source/release +provenance and [upgrade notes](migration-upgrade-notes.md) for each supported +deployment shape. Semantic versions that were never assigned remain explicitly +unassigned; source shipment does not establish database execution. + +Merging a required migration means it runs at the next deployment. Review its +compatibility with the still-running previous application, locking, privileges, +data effects and reversal/restore plan at merge time. Update these public +upgrade records in that change. The manual sitting and client stage/unstage +steps are retired; no separate staging ceremony is required. + +## CI and local test stacks + +The migration-runner CI fresh-image step and the mm2/mm5 proof use exactly: ```sh -docker exec -i polis-dev-postgres-1 psql -U postgres -d polis-dev < server/postgres/migrations/000006_update_votes_rule.sql +COMPOSE_PROJECT_NAME=polis-migrate-test-my-owned-run \ +POLIS_RECOVERY_PG_PORT=55851 \ +bash queue-rs/polis-migrate/tests/fresh-image.sh ``` -where `polis-dev-postgres-1` is the name of the running container (see the output of `docker ps`), `postgres` is the db username and `polis-dev` is the database. +Choose a unique project and unused port on each shared machine. This entrypoint +builds `server/Dockerfile-db`, initializes a fresh database, checks exact receipt +names/checksums/statuses against `release.txt` and the M4/M5/M7 retirement policy, +then restarts and checks that the complete history is unchanged. Both the ready +count and expected receipt set derive from the selection. It removes only its +project volumes/containers and project-specific image tag, including on failure. +It does not replay production rows or prove a production deployment. + +After building `docker-compose.test.yml`, run `bash ci/test-migrations.sh` +before starting application services. It waits for Postgres initialization, +then runs `apply` and `check` using the binary and SQL packaged in that image. +It works on fresh and existing test volumes; it never creates history by hand +or bypasses the API startup check. Cypress, server integration and Delphi CI +all use this entrypoint. Server integration also runs the Node startup check +before loading its in-process test app. + +Set a unique `COMPOSE_PROJECT_NAME` and `POLIS_RECOVERY_PG_PORT` for a shared +local machine. `POLIS_TEST_ENV_FILE` chooses a test env file (default `test.env`); +optional Compose arguments such as `-f local-ports.yml` support isolated test +stacks. Use the same options when starting and removing your stack. + -You'd do this for each new file, in numeric order. +## Deployment reconciliation records - [`server/postgres/migrations/`]: /server/postgres/migrations +Use [the reconciliation record](migration-reconciliation.md) for each deployment. +A catalog match is an observation, not proof that a historical file ran. A failed +predicate must lead to a reviewed forward repair or a documented compatible +variant, never to manually inserting a migration row. Unknown historical variants remain review cases; this document does not authorize +a deployment that fails reconciliation. Run the [read-only first-deploy report](migration-upgrade-notes.md) +before the first transition; it does not replace application-health verification. diff --git a/docs/queue-substrate.md b/docs/queue-substrate.md index 4e3a9ea276..1d8c0c8aa6 100644 --- a/docs/queue-substrate.md +++ b/docs/queue-substrate.md @@ -44,19 +44,11 @@ executor role holds **no** direct read or write on any queue table. ## Applying the migration -A **fresh** container applies it automatically: the postgres image copies -`server/postgres/migrations/*.sql` into `/docker-entrypoint-initdb.d`, so -`make start` on a new volume comes up with the schema present and the flag off. - -An **existing** database needs the file applied by hand, through the checked -wrapper, which feeds it to psql exactly as [docs/migrations.md](migrations.md) -describes. Apply this file alone; never replay the migrations directory as an -upgrade mechanism. - -```sh -server/postgres/bin/apply-migration.sh --free-bytes 000019 -- \ - docker exec -i polis-dev-postgres-1 psql -U postgres -d polis-dev -``` +Both fresh databases and upgrades use [`polis-migrate apply`](migrations.md). +Fresh Docker volumes run it during initialization; deployments run it before +service replacement. Existing pre-runner databases reconcile once first. +Successful files are recorded and skipped on later deploys. Queue activation +remains controlled by the existing flags; applying schema does not enable it. ### What the apply locks, and the window it needs @@ -71,16 +63,16 @@ open transaction that already wrote a `conversations` row; when the timeout fires the transaction aborts and nothing is applied. The changed queue tables and the new objects are `ACCESS EXCLUSIVE` for the same span. So "additive and empty" is not "cannot block users": **apply in an idle or controlled writer -window** (producers paused, no open writer on `conversations`), and apply -000019 and 000023 as two separate steps, each with its own window. +window** (producers paused, no open writer on `conversations`), and account for both +000019 and 000023 in the deployment window; the runner gives each its own transaction. ### The wrapper: preflight and budgets -`server/postgres/bin/apply-migration.sh` is the one way to apply 000019 or -000023 to an existing database. It takes the migration number and, after `--`, -the psql command to run (the SQL goes on its stdin, so `docker exec -i … psql` -and a plain `psql` both work). It refuses, sending nothing, unless every -preflight check passes, and prints each one: +The former `server/postgres/bin/apply-migration.sh` is retained for historical +rehearsal/reversal tests. It is **not** the deployment entry point and does not +update the migration history. Its former first-install checks are recorded below +for reference. The runner now owns transactional application, bounded waits, +checksums and history; use the [migration guide](migrations.md). | check | what it requires | |---|---| @@ -391,18 +383,10 @@ Two of those tables deserve plain words: Schema ruling S1 (2026-10-05) approved it as direction: one datastore and typed contracts, flag off, DynamoDB running every job family until each is moved one -at a time. **Applying it to production is a separate, explicit step by the -owner**, 000019 first (it has never been applied there), then 000023, each -through the wrapper and in its own idle or controlled writer window, because -each holds `ShareRowExclusiveLock` on `conversations` until it commits -([what the apply locks](#what-the-apply-locks-and-the-window-it-needs); the -wrapper's preflight and budgets are -[described above](#the-wrapper-preflight-and-budgets)): - -```sh -server/postgres/bin/apply-migration.sh --free-bytes 000023 -- \ - docker exec -i polis-dev-postgres-1 psql -U postgres -d polis-dev -``` +at a time. **Required migrations now apply during deployment**, in numeric order through +`polis-migrate apply`; review and authorize their schema effects at merge time. +Both 000019 and 000023 still hold `ShareRowExclusiveLock` on conversations, bounded +by the runner's lock and transaction limits. See [migration operations](migrations.md). The applier must be able to `SET ROLE polis_queue_owner`; the file creates no role. It refuses, changing nothing, when 000019 is absent, when any diff --git a/docs/upgrading.md b/docs/upgrading.md index 787e645389..837cdb0ac5 100644 --- a/docs/upgrading.md +++ b/docs/upgrading.md @@ -1,16 +1,45 @@ # Upgrade Guide -## Safe deployment hooks (release A) +## Migration runner transition -This release changes deployment hooks only; it does not introduce the migration -runner or an API migration startup check. Install it successfully on every box -before enabling the runner in release B. CodeDeploy uses the PREVIOUS successful -revision's ApplicationStop, so this first transition can still interrupt service. -New hooks defer replacement until AfterInstall and validate the server's HTTP and -database routes. A failure in ValidateService fails the deployment; it does not -automatically restore containers or undo schema changes. Delphi/worker validation -checks process/container liveness, not real-time math correctness. +Use [the migration runner](migrations.md) before replacing services. Existing +databases need one catalog-checked `reconcile` before their first runner deploy; +new databases initialize through the runner. Missing or changed history prevents +API startup. Migrations 000019/23/24/27 now apply during deployment; the coordinator +000021 remains explicitly held in this release. This installs schema only, not +queue producers or workers. PostgreSQL 17+ is required. +Read the [per-deployment upgrade paths and read-only report](migration-upgrade-notes.md) +before the first transition. One explicit release manifest selects the files; +M20/M21/M25/M26 are outside this forward release. Historical M4/M5/M7 never +execute automatically: absence of their removal targets permits ADOPTED receipts, +while retained targets stop for review without deletion. The +[named legacy contract](migration-legacy-contract.md) preserves supported live +variants, and the [release map](migration-release-map.md) distinguishes shipped +source from actual execution and unassigned semantic versions. + +Install **release A** (safe hooks, no runner or startup check) successfully on +EVERY deployment-group instance before this release B. CodeDeploy executes +ApplicationStop from the previous successful revision. A's first transition may +still interrupt service; B's migration refusal preserves A's running services. +An instance that missed A must not receive B. Rolling back to pre-A hooks removes +this protection; restore A before retrying B. + +AfterInstall builds the runner, invokes `polis-migrate deploy`, and only after +success replaces the role's services. Under one database advisory lock, deploy +reconciles the catalog through 000022 if history is absent/legacy on an existing +database, applies the selected pending migrations, and checks readiness. A fresh +database goes directly to apply; modern history is checked rather than adopted +again. Any mismatch stops. No error is used as a signal to retry reconciliation. +This release's production-shaped transition adopts 20 files and applies only +M19/M23/M24/M27. Existing queue catalogs newer than the adoption bound still require +their separately reviewed explicit reconciliation before this deploy. + +ValidateService checks the server's HTTP and database routes, plus container +liveness; Delphi/worker checks prove container/process liveness only. Validation +failure fails deployment, not an automatic container rollback or schema undo. +Successful earlier migrations may remain after a later failure. Preserve backup +and migration-specific recovery procedures. No automatic down migration runs. ## Configuration Changes (Q1 2023) diff --git a/queue-rs/Cargo.lock b/queue-rs/Cargo.lock index 8dc073d4f3..606e3f04ff 100644 --- a/queue-rs/Cargo.lock +++ b/queue-rs/Cargo.lock @@ -568,6 +568,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "polis-migrate" +version = "0.1.0" +dependencies = [ + "anyhow", + "native-tls", + "postgres", + "postgres-native-tls", + "sha2 0.10.9", +] + [[package]] name = "polis-queue-adapter" version = "0.1.0" diff --git a/queue-rs/Cargo.toml b/queue-rs/Cargo.toml index bfc1730620..6920223fc5 100644 --- a/queue-rs/Cargo.toml +++ b/queue-rs/Cargo.toml @@ -1,7 +1,7 @@ [workspace] # The root package (the polis-queue/1 adapter and the polis-jobs daemon) and the # polis-api route server share one lockfile, one toolchain and one CI job. -members = [".", "polis-api"] +members = [".", "polis-api", "polis-migrate"] resolver = "3" [package] diff --git a/queue-rs/polis-migrate/Cargo.toml b/queue-rs/polis-migrate/Cargo.toml new file mode 100644 index 0000000000..6403b1bb90 --- /dev/null +++ b/queue-rs/polis-migrate/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "polis-migrate" +version = "0.1.0" +edition = "2024" +rust-version = "1.88" +publish = false + +[dependencies] +anyhow = "1" +native-tls = "=0.2.14" +postgres = "0.19" +postgres-native-tls = "=0.5.3" +sha2 = "0.10" + +[lints.clippy] +unwrap_used = "deny" +expect_used = "deny" diff --git a/queue-rs/polis-migrate/Dockerfile b/queue-rs/polis-migrate/Dockerfile new file mode 100644 index 0000000000..c55dfb59fe --- /dev/null +++ b/queue-rs/polis-migrate/Dockerfile @@ -0,0 +1,20 @@ +# Build from repository root. One binary for deploys and manual/local upgrades. +FROM docker.io/library/rust:1.98.1-slim-bookworm AS build +RUN apt-get update && apt-get install -y --no-install-recommends pkg-config libssl-dev curl ca-certificates \ + && rm -rf /var/lib/apt/lists/* +# Public RDS truststore retrieved over HTTPS 2026-10-09; checksum pins its bytes. +RUN curl --fail --location --proto '=https' --tlsv1.2 https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem -o /rds-ca.pem \ + && echo 'fe45bbebf92ad3e27a583bbb2ddd1553c521ed4d49af5514dc0a40372ea5395c /rds-ca.pem' | sha256sum --check +WORKDIR /src +COPY queue-rs/ ./ +RUN cargo build --release --locked -p polis-migrate +FROM docker.io/library/debian:bookworm-slim +RUN apt-get update && apt-get install -y --no-install-recommends libssl3 ca-certificates \ + && rm -rf /var/lib/apt/lists/* +COPY --from=build /src/target/release/polis-migrate /usr/local/bin/polis-migrate +COPY server/postgres/migrations /migrations +COPY --from=build /rds-ca.pem /etc/polis/rds-ca.pem +ENV POLIS_MIGRATIONS_DIR=/migrations POLIS_MIGRATE_CA_FILE=/etc/polis/rds-ca.pem +USER 65534:65534 +ENTRYPOINT ["polis-migrate"] +CMD ["apply"] diff --git a/queue-rs/polis-migrate/Dockerfile.dockerignore b/queue-rs/polis-migrate/Dockerfile.dockerignore new file mode 100644 index 0000000000..2850efb53f --- /dev/null +++ b/queue-rs/polis-migrate/Dockerfile.dockerignore @@ -0,0 +1,8 @@ +** +!queue-rs/ +!queue-rs/** +queue-rs/target/ +!server/ +!server/postgres/ +!server/postgres/migrations/ +!server/postgres/migrations/** diff --git a/queue-rs/polis-migrate/src/indexes.rs b/queue-rs/polis-migrate/src/indexes.rs new file mode 100644 index 0000000000..f2127f0c9e --- /dev/null +++ b/queue-rs/polis-migrate/src/indexes.rs @@ -0,0 +1,66 @@ +//! Resumable, autocommit execution for the immutable watermark index migration. +//! The caller holds the database-wide migration lock across every statement. +use anyhow::{Context, Result, ensure}; +use postgres::Client; + +const INDEXES: [(&str, &str, &str); 2] = [ + ("votes_created_idx", "votes", "created"), + ("comments_modified_idx", "comments", "modified"), +]; + +fn state(client: &mut Client, name: &str, table: &str, column: &str) -> Result> { + let rows = client.query( + "SELECT x.indisvalid AND x.indisready AND x.indislive AS usable, + pg_get_indexdef(c.oid), c.relkind + FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace + LEFT JOIN pg_index x ON x.indexrelid=c.oid + WHERE n.nspname='public' AND c.relname=$1", + &[&name], + )?; + let Some(row) = rows.first() else { + return Ok(None); + }; + let definition: Option = row.get(1); + let expected = format!("CREATE INDEX {name} ON public.{table} USING btree ({column})"); + ensure!( + definition.as_deref() == Some(expected.as_str()), + "000022: conflicting object public.{name}; preserve it and resolve its definition before retrying" + ); + let usable: Option = row.get(0); + Ok(Some(usable == Some(true))) +} + +pub fn prepare(client: &mut Client) -> Result<()> { + // Preflight BOTH identities before changing either one. A conflicting or + // invalid preexisting index is never dropped or replaced automatically. + for (name, table, column) in INDEXES { + if let Some(usable) = state(client, name, table, column)? { + ensure!( + usable, + "000022: public.{name} is invalid/not ready; inspect the interrupted build, then explicitly DROP INDEX CONCURRENTLY public.{name} before retrying apply; history has not recorded 000022" + ); + } + } + for (name, table, column) in INDEXES { + // Recheck after the preceding build. Uncoordinated external DDL is not + // serialized by the runner's advisory lock; any collision fails closed. + if state(client, name, table, column)? == Some(true) { + continue; + } + ensure!( + state(client, name, table, column)?.is_none(), + "000022: concurrent catalog change for public.{name}; inspect before retrying" + ); + println!("BUILDING CONCURRENTLY public.{name}"); + client.batch_execute(&format!( + "CREATE INDEX CONCURRENTLY {name} ON public.{table} USING btree ({column})" + )).with_context(|| format!( + "000022: concurrent build interrupted for public.{name}; inspect validity before retrying; valid earlier indexes are retained and no 000022 history row was committed" + ))?; + ensure!( + state(client, name, table, column)? == Some(true), + "000022: index public.{name} did not reach the expected valid state" + ); + } + Ok(()) +} diff --git a/queue-rs/polis-migrate/src/lib.rs b/queue-rs/polis-migrate/src/lib.rs new file mode 100644 index 0000000000..f40e4b6b54 --- /dev/null +++ b/queue-rs/polis-migrate/src/lib.rs @@ -0,0 +1,617 @@ +//! One migration history, one connection and one database advisory lock. +//! SQL files remain immutable; only their optional outer BEGIN/COMMIT is removed. +mod indexes; +mod sql; +use anyhow::{Context, Result, bail, ensure}; +use postgres::{ + Client, GenericClient, + config::{Host, SslMode}, +}; +use sha2::{Digest, Sha256}; +use std::{collections::BTreeMap, fs, path::Path, time::Duration}; + +pub struct Migration { + pub name: String, + pub checksum: String, + pub body: String, + pub retired: bool, +} +// Fixed database-wide key, shared by apply/reconcile. A different DB has its own lock. +pub const LOCK: i64 = 0x506f6c69734d6967; +const HISTORY: &str = "CREATE TABLE public.migrations ( + name text PRIMARY KEY, checksum text NOT NULL CHECK(length(checksum)=64), + status text NOT NULL CHECK(status IN ('APPLIED','ADOPTED')), + recorded_at timestamptz NOT NULL DEFAULT clock_timestamp(), + recorded_by text NOT NULL DEFAULT session_user, + legacy_completed_at bigint[] NOT NULL DEFAULT '{}'); + REVOKE ALL ON public.migrations FROM PUBLIC; + GRANT SELECT ON public.migrations TO PUBLIC;"; + +pub fn load(dir: &Path) -> Result> { + let mut files = BTreeMap::new(); + let mut versions = std::collections::BTreeSet::new(); + let selected = manifest(dir, "release.txt")?; + let held = manifest(dir, "held.txt")?; + ensure!( + selected.is_disjoint(&held), + "release and held manifests overlap" + ); + for entry in fs::read_dir(dir)? { + let entry = entry?; + let name = entry + .file_name() + .into_string() + .map_err(|_| anyhow::anyhow!("non-UTF8 migration name"))?; + if !name.ends_with(".sql") { + continue; + } + ensure!( + entry.file_type()?.is_file(), + "migration must be a regular file: {name}" + ); + ensure!( + name.len() > 11 + && name.as_bytes()[..6].iter().all(u8::is_ascii_digit) + && name.as_bytes()[6] == b'_' + && name.as_bytes()[7..name.len() - 4] + .iter() + .all(|b| b.is_ascii_alphanumeric() || *b == b'_'), + "invalid migration name: {name}" + ); + ensure!( + versions.insert(name[..6].to_owned()), + "duplicate migration number: {name}" + ); + let source = fs::read_to_string(entry.path())?; + files.insert( + name.clone(), + Migration { + name: name.clone(), + checksum: format!("{:x}", Sha256::digest(source.as_bytes())), + body: sql::body(&source)?, + retired: matches!(&name[..6], "000004" | "000005" | "000007"), + }, + ); + } + for name in selected.iter().chain(&held) { + ensure!( + files.contains_key(name), + "manifest migration is missing: {name}" + ); + } + for name in files.keys() { + ensure!( + selected.contains(name) || held.contains(name), + "migration absent from release/held manifest: {name}" + ); + } + let migrations: Vec<_> = files + .into_values() + .filter(|m| selected.contains(&m.name)) + .collect(); + ensure!( + migrations + .first() + .is_some_and(|m| m.name == "000000_initial.sql"), + "missing initial migration" + ); + Ok(migrations) +} + +fn manifest(dir: &Path, file: &str) -> Result> { + let text = fs::read_to_string(dir.join(file)).with_context(|| format!("read {file}"))?; + let mut names = std::collections::BTreeSet::new(); + for name in text + .lines() + .filter(|s| !s.is_empty() && !s.starts_with('#')) + { + ensure!( + name.len() > 11 + && name.as_bytes()[..6].iter().all(u8::is_ascii_digit) + && name.as_bytes()[6] == b'_' + && name.ends_with(".sql") + && name.as_bytes()[7..name.len() - 4] + .iter() + .all(|b| b.is_ascii_alphanumeric() || *b == b'_'), + "invalid migration in {file}: {name}" + ); + ensure!( + names.insert(name.to_owned()), + "duplicate migration in {file}: {name}" + ); + } + Ok(names) +} + +// Retired historical DDL is never executable, even on a fresh installation. +// Check catalog absence directly; no temporary functions or user-data reads. +fn retired_absent(client: &mut impl GenericClient, m: &Migration) -> Result { + let (tables, columns): (&[&str], &[(&str, &str)]) = match &m.name[..6] { + "000004" => (&["waitinglist"], &[]), + "000005" => ( + &[ + "slack_oauth_access_tokens", + "slack_users", + "slack_user_invites", + "slack_bot_events", + "stripe_accounts", + "stripe_subscriptions", + "coupons_for_free_upgrades", + "lti_users", + "lti_context_memberships", + "canvas_assignment_callback_info", + "canvas_assignment_conversation_info", + "lti_oauthv1_credentials", + ], + &[ + ("conversations", "is_slack"), + ("conversations", "lti_users_only"), + ("users", "plan"), + ], + ), + "000007" => ( + &["geolocation_cache"], + &[ + ("participants_extended", "country_code_iso"), + ("participants_extended", "encrypted_maxmind_response_city"), + ("participants_extended", "ip_address"), + ("participants_extended", "latitude"), + ("participants_extended", "location"), + ("participants_extended", "longitude"), + ("participants_extended", "x_forwarded_for"), + ], + ), + _ => bail!("not a retired migration: {}", m.name), + }; + for table in tables { + if exists(client, &format!("public.{table}"))? { + return Ok(false); + } + } + for (table, column) in columns { + let present: bool = client.query_one("SELECT EXISTS(SELECT 1 FROM pg_attribute WHERE attrelid=to_regclass('public.' || $1) AND attname=$2 AND attnum>0 AND NOT attisdropped)", &[table, column])?.get(0); + if present { + return Ok(false); + } + } + Ok(true) +} + +pub fn connect(dsn: &str) -> Result { + let mut cfg: postgres::Config = dsn + .parse() + .map_err(|_| anyhow::anyhow!("invalid DATABASE_URL"))?; + cfg.connect_timeout(Duration::from_secs(10)) + .application_name("polis-migrate"); + ensure!( + !cfg.get_hosts().is_empty() && cfg.get_hostaddrs().is_empty(), + "DATABASE_URL must name a host and cannot use hostaddr" + ); + if cfg.get_ssl_mode() == SslMode::Disable { + // Docker-local tests may use service DNS; remote plaintext is never implicit. + ensure!( + std::env::var("POLIS_MIGRATE_ALLOW_PLAINTEXT").as_deref() == Ok("local") + || cfg.get_hosts().iter().all(|h| match h { + Host::Tcp(h) => h.parse::().is_ok_and(|a| a.is_loopback()), + #[cfg(unix)] + Host::Unix(p) => p.is_absolute(), + }), + "plaintext requires loopback/socket or explicit POLIS_MIGRATE_ALLOW_PLAINTEXT=local" + ); + cfg.connect(postgres::NoTls) + .context("connect to migration database") + } else { + // require encryption and validate hostname + certificate even with sslmode=require. + cfg.ssl_mode(SslMode::Require); + let mut tls = native_tls::TlsConnector::builder(); + if let Ok(path) = std::env::var("POLIS_MIGRATE_CA_FILE") { + let bytes = fs::read(path).context("read migration CA file")?; + // RDS and local CA bundles can contain several certificates. + for pem in String::from_utf8(bytes)?.split_inclusive("-----END CERTIFICATE-----") { + if pem.contains("-----BEGIN CERTIFICATE-----") { + tls.add_root_certificate(native_tls::Certificate::from_pem(pem.as_bytes())?); + } + } + } + cfg.connect(postgres_native_tls::MakeTlsConnector::new(tls.build()?)) + .context("connect to migration database (verified TLS)") + } +} + +fn setup(client: &mut Client) -> Result<()> { + let version: i32 = client + .query_one("SELECT current_setting('server_version_num')::integer", &[])? + .get(0); + ensure!(version >= 170000, "PostgreSQL 17 or newer is required"); + client.batch_execute("SET standard_conforming_strings=on; SET search_path=pg_catalog,public; SET statement_timeout='5min'; SET lock_timeout='5s'; SET idle_in_transaction_session_timeout='30s'; SET transaction_timeout='5min';")?; + Ok(()) +} +fn exists(client: &mut impl GenericClient, name: &str) -> Result { + Ok(client + .query_one("SELECT to_regclass($1) IS NOT NULL", &[&name])? + .get(0)) +} +fn history( + client: &mut impl GenericClient, + migrations: &[Migration], +) -> Result> { + ensure!( + exists(client, "public.migrations")?, + "migration history missing; run polis-migrate apply for a fresh database, or reconcile for an existing database (docs/migrations.md)" + ); + let rows = client + .query( + "SELECT name, checksum, status FROM public.migrations ORDER BY name", + &[], + ) + .context("unreconciled legacy history; run polis-migrate reconcile")?; + let mut result = BTreeMap::new(); + for row in rows { + let name: String = row.get(0); + let checksum: String = row.get(1); + let status: String = row.get(2); + ensure!( + matches!(status.as_str(), "APPLIED" | "ADOPTED"), + "unverified history: {name}" + ); + let m = migrations.iter().find(|m| m.name == name).ok_or_else(|| { + anyhow::anyhow!("history names migration absent from this release: {name}") + })?; + ensure!( + checksum == m.checksum, + "migration checksum mismatch: {name}; restore the released source; do not replay it" + ); + ensure!( + result.insert(name.clone(), checksum).is_none(), + "duplicate history: {name}" + ); + } + Ok(result) +} +pub fn check(client: &mut Client, migrations: &[Migration]) -> Result<()> { + setup(client)?; + let mut tx = client.build_transaction().read_only(true).start()?; + let applied = history(&mut tx, migrations)?; + let pending: Vec<_> = migrations + .iter() + .filter(|m| !applied.contains_key(&m.name)) + .map(|m| m.name.as_str()) + .collect(); + ensure!( + pending.is_empty(), + "pending migrations: {}; run polis-migrate apply before starting the server", + pending.join(", ") + ); + tx.commit()?; + println!("migration check: {} ready", migrations.len()); + Ok(()) +} +fn lock(client: &mut Client) -> Result<()> { + setup(client)?; + // A blocking SELECT pg_advisory_lock retains a snapshot while waiting. + // CREATE INDEX CONCURRENTLY in the holder can wait for that same snapshot, + // deadlocking two runners. Each try returns before the bounded client wait, + // so no server statement/snapshot is held by the waiting runner. + println!("waiting for migration lock"); + let deadline = std::time::Instant::now() + Duration::from_secs(300); + loop { + let acquired: bool = client + .query_one("SELECT pg_try_advisory_lock($1)", &[&LOCK])? + .get(0); + if acquired { + break; + } + ensure!( + std::time::Instant::now() < deadline, + "timed out waiting for migration lock; another runner is still active" + ); + std::thread::sleep(Duration::from_millis(100)); + } + println!("migration lock acquired"); + Ok(()) +} +fn record( + client: &mut impl GenericClient, + m: &Migration, + status: &str, + legacy: &[i64], +) -> Result<()> { + client.execute("INSERT INTO public.migrations(name,checksum,status,legacy_completed_at) VALUES($1,$2,$3,$4)", &[&m.name,&m.checksum,&status,&legacy])?; + Ok(()) +} +pub fn apply(client: &mut Client, migrations: &[Migration]) -> Result { + lock(client)?; + let count = apply_locked(client, migrations)?; + client.query_one("SELECT pg_advisory_unlock($1)", &[&LOCK])?; + Ok(count) +} + +fn apply_locked(client: &mut Client, migrations: &[Migration]) -> Result { + // Validate history and every pending retirement before CREATE HISTORY or + // unrelated forward DDL. A refusal leaves the database unchanged. + let prior = if exists(client, "public.migrations")? { + history(client, migrations)? + } else { + BTreeMap::new() + }; + for m in migrations + .iter() + .filter(|m| m.retired && !prior.contains_key(&m.name)) + { + ensure!( + retired_absent(client, m)?, + "retired migration {} still has removed objects; no changes made; explicit operator review required", + m.name + ); + } + // A no-history populated DB is never treated as a fresh install. Check ALL + // public relations, not only conversations, before allowing 000000. + if !exists(client, "public.migrations")? { + let populated: bool = client.query_one("SELECT EXISTS(SELECT 1 FROM pg_class WHERE relnamespace='public'::regnamespace AND relkind IN ('r','p','v','m','S','f')) OR EXISTS(SELECT 1 FROM pg_proc WHERE pronamespace='public'::regnamespace) OR EXISTS(SELECT 1 FROM pg_type WHERE typnamespace='public'::regnamespace AND typtype IN ('e','d'))", &[])?.get(0); + ensure!( + !populated, + "existing database has no history; run reconcile before apply; 000000 will not be replayed" + ); + client.batch_execute(HISTORY)?; + } + let applied = history(client, migrations)?; + let mut count = 0; + for m in migrations.iter().filter(|m| !applied.contains_key(&m.name)) { + if m.name.starts_with("000000_") { + let existing: bool = client.query_one("SELECT EXISTS(SELECT 1 FROM pg_class WHERE relnamespace='public'::regnamespace AND relkind IN ('r','p','v','m','S','f') AND relname <> 'migrations') OR EXISTS(SELECT 1 FROM pg_proc WHERE pronamespace='public'::regnamespace) OR EXISTS(SELECT 1 FROM pg_type WHERE typnamespace='public'::regnamespace AND typtype IN ('e','d'))", &[])?.get(0); + ensure!( + !existing, + "refusing initial migration on an existing schema; reconcile first" + ); + } + if m.name == "000022_add_poll_timestamp_indexes.sql" { + // Source-bound special execution; never silently reinterpret a new + // version of this SQL. The original body below still checks both + // definitions before its history row commits. + ensure!( + m.checksum == "14efc95b14787b52d70680ea06cfef020cd2224e82495258fd8eb324501b7b3e", + "000022 source changed; review the concurrent execution contract" + ); + indexes::prepare(client)?; + } + let mut tx = client.transaction()?; + tx.batch_execute("SET LOCAL standard_conforming_strings=on; SET LOCAL search_path=public,pg_catalog; SET LOCAL lock_timeout='5s'; SET LOCAL statement_timeout='5min'; SET LOCAL transaction_timeout='5min';")?; + let status = if m.retired { + ensure!( + retired_absent(&mut tx, m)?, + "retired migration {} has retained objects; refusing adoption", + m.name + ); + "ADOPTED" + } else { + tx.batch_execute(&m.body).with_context(|| { + format!("migration {} failed (transaction not committed)", m.name) + })?; + // Restore the login's context after queue scripts SET LOCAL ROLE. + tx.batch_execute("RESET ROLE; SET LOCAL search_path=pg_catalog,public")?; + "APPLIED" + }; + record(&mut tx, m, status, &[])?; + tx.commit().with_context(|| { + format!( + "commit outcome unknown for {}; reconnect and check history before retrying", + m.name + ) + })?; + println!("{status} {}", m.name); + if status == "APPLIED" { + count += 1; + } + } + println!("applied {count} migration(s)"); + Ok(count) +} + +// Reuse the queue's existing catalog verifiers for older Docker installations. +// Only temporary catalog-reading functions are installed here, never queue DDL. +fn queue_version(tx: &mut impl GenericClient, migrations: &[Migration]) -> Result { + let present: bool = tx.query_one("SELECT EXISTS(SELECT 1 FROM pg_class WHERE relnamespace='public'::regnamespace AND (starts_with(relname,'polis_queue_') OR starts_with(relname,'delphi_'))) OR EXISTS(SELECT 1 FROM pg_proc WHERE pronamespace='public'::regnamespace AND (starts_with(proname,'pq_') OR starts_with(proname,'pd_')))",&[])?.get(0); + if !present { + return Ok(0); + } + ensure!( + exists(tx, "public.polis_queue_install")?, + "partial queue installation; no adoption committed" + ); + let mut definitions = 0; + for m in migrations + .iter() + .filter(|m| m.name.starts_with("000019_") || m.name.starts_with("000024_")) + { + for statement in sql::statements(&m.body)? { + if [ + "pq_catalog", + "pq_assert_catalog", + "pq_assert_signatures", + "pq_assert_functions", + "pd_state", + "pq3_state", + ] + .iter() + .any(|name| { + statement.starts_with(&format!("CREATE OR REPLACE FUNCTION pg_temp.{name}(")) + }) { + tx.batch_execute(&statement)?; + definitions += 1; + } + } + } + ensure!( + definitions == 7, + "queue adoption verifier definitions changed; review the catalog contract" + ); + if exists(tx, "public.polis_queue_large_class_install")? { + ensure!( + exists(tx, "public.delphi_foundation_install")?, + "partial queue /3 installation" + ); + let ok: bool=tx.query_one("SELECT (SELECT count(*)=1 FROM public.polis_queue_large_class_install) AND EXISTS(SELECT 1 FROM public.polis_queue_large_class_install WHERE singleton AND installed=pg_temp.pq3_state()) AND (SELECT count(*)=1 FROM public.delphi_foundation_install) AND (SELECT count(*)=1 AND bool_and(contract_version='polis-queue/3') FROM public.polis_queue_install)",&[])?.get(0); + ensure!(ok, "queue /3 catalog postconditions fail"); + Ok(24) + } else if exists(tx, "public.delphi_foundation_install")? { + let ok: bool=tx.query_one("SELECT (SELECT count(*)=1 FROM public.delphi_foundation_install) AND EXISTS(SELECT 1 FROM public.delphi_foundation_install WHERE singleton AND installed=pg_temp.pd_state()) AND (SELECT count(*)=1 AND bool_and(contract_version='polis-queue/2') FROM public.polis_queue_install)",&[])?.get(0); + ensure!(ok, "queue /2 catalog postconditions fail"); + Ok(23) + } else { + tx.batch_execute("SELECT pg_temp.pq_assert_catalog(false); SELECT pg_temp.pq_assert_signatures(false); SELECT pg_temp.pq_assert_functions(false)").context("queue /1 catalog postconditions fail")?; + ensure!( + tx.query_one("SELECT count(*)=1 FROM public.polis_queue_install", &[])? + .get::<_, bool>(0), + "queue /1 install record missing" + ); + Ok(19) + } +} + +/// Adoption is explicitly bounded by the operator, but every file is checked. +/// The missing 19 queue is a known hole in the pre-runner production baseline. +/// Existing queues use their catalog verifiers, never install-row presence alone. +pub fn reconcile( + client: &mut Client, + migrations: &[Migration], + dir: &Path, + through: &str, +) -> Result { + ensure!( + through.len() == 6 && through.bytes().all(|b| b.is_ascii_digit()), + "--through requires six digits" + ); + ensure!( + migrations.iter().any(|m| &m.name[..6] == through), + "unknown --through version" + ); + lock(client)?; + let count = reconcile_locked(client, migrations, dir, through)?; + client.query_one("SELECT pg_advisory_unlock($1)", &[&LOCK])?; + Ok(count) +} + +fn reconcile_locked( + client: &mut Client, + migrations: &[Migration], + dir: &Path, + through: &str, +) -> Result { + let mut tx = client.transaction()?; + let modern = tx.query_one("SELECT EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema='public' AND table_name='migrations' AND column_name='checksum')", &[])?.get::<_,bool>(0); + if modern { + history(&mut tx, migrations)?; + bail!("history is already reconciled; use apply/check"); + } + ensure!( + !exists(&mut tx, "public.schema_migrations")?, + "another schema_migrations ledger exists; resolve its unverified rows before reconciliation" + ); + let mut legacy: BTreeMap> = BTreeMap::new(); + if exists(&mut tx, "public.migrations")? { + for row in tx.query( + "SELECT name,completed_at FROM public.migrations ORDER BY name,completed_at", + &[], + )? { + let name: String = row.get(0); + ensure!( + migrations + .iter() + .any(|m| m.name == name && &m.name[..6] <= through), + "legacy history contains unknown/renamed/later migration {name}; resolve against its catalog, never replay blindly" + ); + legacy.entry(name).or_default().push(row.get(1)); + } + } + // Helpers are transaction-local and contain only catalog reads. + tx.batch_execute(&fs::read_to_string(dir.join("adoption/helpers.sql"))?)?; + let queue = queue_version(&mut tx, migrations)?; + ensure!( + queue <= through.parse::()?, + "queue is newer than --through; select its actual installed version" + ); + let mut adopted = Vec::new(); + for m in migrations.iter().filter(|m| &m.name[..6] <= through) { + let number = m.name[..6].parse::()?; + if matches!(number, 19 | 23 | 24) { + if queue >= number { + adopted.push(m); + } else { + ensure!( + !legacy.contains_key(&m.name), + "legacy queue history row was not verified: {}", + m.name + ); + } + continue; + } + let path = dir.join("adoption").join(&m.name); + let query = fs::read_to_string(path).with_context(|| { + format!( + "no adoption contract for {}; stop and review this existing installation", + m.name + ) + })?; + let ok: bool = tx + .query_one(&query, &[]) + .with_context(|| format!("catalog check for {}", m.name))? + .get(0); + ensure!( + ok, + "catalog postconditions fail for {}; no adoption rows committed; inspect schema before retrying", + m.name + ); + adopted.push(m); + } + for name in legacy.keys() { + ensure!( + adopted.iter().any(|m| m.name == *name), + "legacy row not verified: {name}" + ); + } + if exists(&mut tx, "public.migrations")? { + tx.batch_execute("DROP TABLE public.migrations")?; + } + tx.batch_execute(HISTORY)?; + for m in &adopted { + record( + &mut tx, + m, + "ADOPTED", + legacy.get(&m.name).map(Vec::as_slice).unwrap_or(&[]), + )?; + } + tx.commit()?; + for m in &adopted { + println!("ADOPTED {}", m.name); + } + println!( + "adopted {} migration(s); no migration SQL replayed", + adopted.len() + ); + Ok(adopted.len()) +} + +/// Release deployment: classify history while holding the same lock as apply. +/// The 000022 bound is this release's reviewed legacy contract, not a guess +/// from a failed apply. Later/partial/unknown schemas must still pass reconcile. +pub fn deploy(client: &mut Client, migrations: &[Migration], dir: &Path) -> Result<()> { + lock(client)?; + let modern: bool = client.query_one("SELECT EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema='public' AND table_name='migrations' AND column_name='checksum')", &[])?.get(0); + let populated: bool = client.query_one("SELECT EXISTS(SELECT 1 FROM pg_class WHERE relnamespace='public'::regnamespace AND relkind IN ('r','p','v','m','S','f')) OR EXISTS(SELECT 1 FROM pg_proc WHERE pronamespace='public'::regnamespace) OR EXISTS(SELECT 1 FROM pg_type WHERE typnamespace='public'::regnamespace AND typtype IN ('e','d'))", &[])?.get(0); + ensure!( + !exists(client, "public.schema_migrations")?, + "another schema_migrations ledger exists; resolve it before deployment" + ); + if !modern && populated { + ensure!( + migrations.iter().any(|m| m.name.starts_with("000022_")), + "deploy requires the reviewed 000022 adoption bound" + ); + reconcile_locked(client, migrations, dir, "000022")?; + } + apply_locked(client, migrations)?; + check(client, migrations)?; + client.query_one("SELECT pg_advisory_unlock($1)", &[&LOCK])?; + Ok(()) +} diff --git a/queue-rs/polis-migrate/src/main.rs b/queue-rs/polis-migrate/src/main.rs new file mode 100644 index 0000000000..eccd9ed0b6 --- /dev/null +++ b/queue-rs/polis-migrate/src/main.rs @@ -0,0 +1,82 @@ +use anyhow::{Result, bail}; +use std::{env, path::PathBuf}; +fn run() -> Result<()> { + let mut args = env::args().skip(1); + let command = args.next().unwrap_or_default(); + if matches!(command.as_str(), "--help" | "-h") { + println!( + "polis-migrate deploy|apply|check|reconcile [--dir PATH] [--through NNNNNN]\nDATABASE_URL is read from the environment; reconcile requires --through. See docs/migrations.md." + ); + return Ok(()); + } + let mut dir = PathBuf::from( + env::var("POLIS_MIGRATIONS_DIR").unwrap_or_else(|_| "server/postgres/migrations".into()), + ); + let mut through = None; + while let Some(arg) = args.next() { + match arg.as_str() { + "--dir" => { + dir = args + .next() + .ok_or_else(|| anyhow::anyhow!("--dir needs a path"))? + .into() + } + "--through" => { + through = Some( + args.next() + .ok_or_else(|| anyhow::anyhow!("--through needs a version"))?, + ) + } + _ => bail!("unknown argument: {arg}"), + } + } + if !matches!(command.as_str(), "deploy" | "apply" | "check" | "reconcile") { + bail!("expected deploy, apply, check or reconcile (see --help)"); + } + if (command == "reconcile") != through.is_some() { + bail!("only reconcile requires --through NNNNNN"); + } + let migrations = polis_migrate::load(&dir)?; + let dsn = env::var("DATABASE_URL").map_err(|_| anyhow::anyhow!("DATABASE_URL is required"))?; + let mut client = polis_migrate::connect(&dsn)?; + match command.as_str() { + "deploy" => polis_migrate::deploy(&mut client, &migrations, &dir)?, + "apply" => { + polis_migrate::apply(&mut client, &migrations)?; + } + "check" => polis_migrate::check(&mut client, &migrations)?, + "reconcile" => { + polis_migrate::reconcile( + &mut client, + &migrations, + &dir, + through.as_deref().unwrap_or(""), + )?; + } + _ => unreachable!(), + } + Ok(()) +} +fn main() { + if let Err(error) = run() { + // Do not print connection strings, SQL statements or database error DETAIL + // (which may quote row values). Context + SQLSTATE are sufficient here. + for cause in error.chain() { + if let Some(pg) = cause.downcast_ref::() { + if let Some(db) = pg.as_db_error() { + eprintln!( + "database refused migration: SQLSTATE {} (inspect the named migration)", + db.code().code() + ); + } else { + eprintln!( + "database connection failed; check connectivity and TLS configuration" + ); + } + break; + } + eprintln!("{cause}"); + } + std::process::exit(1); + } +} diff --git a/queue-rs/polis-migrate/src/sql.rs b/queue-rs/polis-migrate/src/sql.rs new file mode 100644 index 0000000000..efcd023884 --- /dev/null +++ b/queue-rs/polis-migrate/src/sql.rs @@ -0,0 +1,282 @@ +//! Split only top-level SQL. Dollar bodies, quoted strings and comments are opaque. +//! This is deliberately not a general SQL rewriter: accept one optional outer +//! BEGIN/COMMIT pair and refuse every other transaction-control statement. +use anyhow::{Result, bail, ensure}; + +fn ident_start(c: char) -> bool { + c.is_ascii_alphabetic() || c == '_' || !c.is_ascii() +} + +fn ident_continue(c: char) -> bool { + ident_start(c) || c.is_ascii_digit() || c == '$' +} + +// PostgreSQL continues single-quoted strings across whitespace containing a +// newline, retaining the first segment's E-string escape mode. Line comments +// count as whitespace here; block comments terminate this lexical construct. +fn continued_string(bytes: &[u8], mut next: usize) -> Option { + let mut newline = false; + while next < bytes.len() { + match bytes[next] { + b'\n' | b'\r' => { + newline = true; + next += 1; + } + b' ' | b'\t' | b'\x0c' | b'\x0b' => next += 1, + b'-' if bytes[next..].starts_with(b"--") => { + while next < bytes.len() && !matches!(bytes[next], b'\n' | b'\r') { + next += 1; + } + } + b'\'' if newline => return Some(next + 1), + _ => return None, + } + } + None +} + +// Keep executable bytes alongside comment-free text used only for inspection. +// Removing comments from the executed source can change string-literal parsing. +fn lex(sql: &str) -> Result> { + let b = sql.as_bytes(); + let (mut i, mut part, mut raw, mut out) = (0, String::new(), String::new(), Vec::new()); + while i < b.len() { + let start = i; + if b[i..].starts_with(b"--") { + while i < b.len() && b[i] != b'\n' { + i += 1; + } + part.push(' '); + raw.push_str(&sql[start..i]); + continue; + } + if b[i..].starts_with(b"/*") { + let mut depth = 1; + i += 2; + while i < b.len() && depth > 0 { + if b[i..].starts_with(b"/*") { + depth += 1; + i += 2; + } else if b[i..].starts_with(b"*/") { + depth -= 1; + i += 2; + } else { + i += 1; + } + } + ensure!(depth == 0, "unterminated SQL comment"); + part.push(' '); + raw.push_str(&sql[start..i]); + continue; + } + if b[i] == b'\'' || b[i] == b'"' { + let quote = b[i]; + let escaped = quote == b'\'' + && i > 0 + && matches!(b[i - 1], b'e' | b'E') + && (i == 1 || !sql[..i - 1].chars().next_back().is_some_and(ident_continue)); + i += 1; + let mut closed = false; + while i < b.len() { + if escaped && b[i] == b'\\' { + i += 2; + } else if b[i] == quote { + i += 1; + if i < b.len() && b[i] == quote { + i += 1; + } else if quote == b'\'' + && let Some(next) = continued_string(b, i) + { + i = next; + } else { + closed = true; + break; + } + } else { + i += 1; + } + } + ensure!(closed && i <= b.len(), "unterminated SQL quote"); + } else if b[i] == b'$' { + let mut j = i + 1; + for c in sql[j..].chars() { + if ident_start(c) || (j > i + 1 && c.is_ascii_digit()) { + j += c.len_utf8(); + } else { + break; + } + } + if j < b.len() && b[j] == b'$' { + let tag = &sql[i..=j]; + let rest = &sql[j + 1..]; + let end = rest + .find(tag) + .ok_or_else(|| anyhow::anyhow!("unterminated dollar body"))?; + i = j + 1 + end + tag.len(); + } else { + i += 1; + } + } else if sql[i..].chars().next().is_some_and(ident_start) { + // '$' is legal inside an unquoted identifier. Consume the whole + // identifier before looking for a dollar-quoted body; PostgreSQL + // requires a delimiter between an identifier and such a body. + for c in sql[i..].chars() { + if !ident_continue(c) { + break; + } + i += c.len_utf8(); + } + } else if b[i] == b';' { + if !part.trim().is_empty() { + out.push((part.trim().to_owned(), raw.trim_start().to_owned())); + } + part.clear(); + raw.clear(); + i += 1; + continue; + } else { + // Work at UTF-8 character boundaries even in unquoted identifiers. + i += sql[i..].chars().next().map(char::len_utf8).unwrap_or(1); + } + part.push_str(&sql[start..i]); + raw.push_str(&sql[start..i]); + } + if !part.trim().is_empty() { + out.push((part.trim().to_owned(), raw.trim_start().to_owned())); + } + Ok(out) +} + +pub fn statements(sql: &str) -> Result> { + Ok(lex(sql)? + .into_iter() + .map(|(inspection, _)| inspection) + .collect()) +} + +pub fn body(sql: &str) -> Result { + let mut parts = lex(sql)?; + ensure!(!parts.is_empty(), "empty migration"); + if parts[0].0.eq_ignore_ascii_case("BEGIN") { + ensure!( + parts + .last() + .is_some_and(|s| s.0.eq_ignore_ascii_case("COMMIT")), + "BEGIN without final COMMIT" + ); + parts.remove(0); + parts.pop(); + } + for (part, _) in &parts { + let first = part + .split_whitespace() + .next() + .unwrap_or("") + .to_ascii_uppercase(); + if matches!( + first.as_str(), + "BEGIN" + | "START" + | "COMMIT" + | "END" + | "ROLLBACK" + | "ABORT" + | "SAVEPOINT" + | "RELEASE" + | "PREPARE" + ) || first.starts_with('\\') + { + bail!("migration contains transaction control or a psql command: {first}"); + } + } + Ok(parts + .into_iter() + .map(|(_, raw)| raw) + .collect::>() + .join(";\n") + + ";") +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn dollar_and_comment_bodies() -> Result<()> { + let s = "-- BEGIN;\nBEGIN; DO $q$ BEGIN RAISE NOTICE 'COMMIT;'; END $q$; /* a /* b */ c */ COMMIT;"; + assert_eq!(body(s)?, "DO $q$ BEGIN RAISE NOTICE 'COMMIT;'; END $q$;"); + Ok(()) + } + #[test] + fn nested_control_refused() { + for s in [ + "BEGIN; SELECT 1; COMMIT; COMMIT;", + "SELECT 1; ROLLBACK;", + "START TRANSACTION; SELECT 1;", + "BEGIN; SELECT 1;", + "SELECT 1; \\i file", + ] { + assert!(body(s).is_err(), "{s}"); + } + } + #[test] + fn quotes_and_unicode() -> Result<()> { + assert_eq!( + statements("SELECT E'a\\';b', 'c'';d', \"é;\"; SELECT '𝄞';")?.len(), + 2 + ); + Ok(()) + } + #[test] + fn incomplete_refused() { + for s in ["SELECT 'x", "DO $$ x", "/* unclosed", "-- empty"] { + assert!(body(s).is_err()); + } + } + #[test] + fn dollar_identifiers_do_not_quote_following_statements() -> Result<()> { + let source = + "CREATE TABLE a$tag$ (x int); COMMIT; CREATE TABLE b$tag$ (y int); SELECT 1/0;"; + assert_eq!(statements(source)?.len(), 4); + assert!(body(source).is_err()); + assert!(body("CREATE TABLE é$tag$ (x int); COMMIT; CREATE TABLE z$tag$ (y int);").is_err()); + assert_eq!( + statements("SELECT ordinary$identifier, other$$identifier;")?.len(), + 1 + ); + Ok(()) + } + #[test] + fn unicode_dollar_tags_preserve_function_bodies() -> Result<()> { + let source = "DO $тег_1$ BEGIN PERFORM 1; END $тег_1$; SELECT 2;"; + assert_eq!(statements(source)?.len(), 2); + assert!(body(source)?.contains("BEGIN PERFORM 1; END")); + Ok(()) + } + #[test] + fn executable_comments_are_preserved() -> Result<()> { + let source = "SELECT 'a' /* line\n /* nested */ end */ 'b';"; + assert_eq!(body(source)?, source); + assert!(body("BEGIN; SELECT 1; COMMIT/* trailing */;")?.contains("SELECT 1")); + assert!(body("SELECT 1; COMMIT/* trailing */;").is_err()); + Ok(()) + } + #[test] + fn escaped_string_continuations_keep_their_mode() -> Result<()> { + let source = "SELECT E'a' -- explanation\n'b\\'c'; SELECT 2;"; + assert_eq!(statements(source)?.len(), 2); + assert!(body(source)?.contains("'b\\'c'")); + Ok(()) + } + #[test] + fn typed_literals_after_identifiers_use_ordinary_string_mode() -> Result<()> { + // E is a prefix only at a token boundary. Dollar and non-ASCII characters + // are PostgreSQL identifier continuations too; use the same rule above. + for name in ["typed$E", "typedéE", "typedeE"] { + let source = format!("SELECT {name}'a\\'; SELECT 2;"); + assert_eq!(statements(&source)?.len(), 2, "{name}"); + assert!(body(&source).is_ok(), "{name}"); + } + assert_eq!(statements("SELECT E'a\\\\'; SELECT 2;")?.len(), 2); + Ok(()) + } +} diff --git a/queue-rs/polis-migrate/tests/adoption-cases.json b/queue-rs/polis-migrate/tests/adoption-cases.json new file mode 100644 index 0000000000..9faa38ef83 --- /dev/null +++ b/queue-rs/polis-migrate/tests/adoption-cases.json @@ -0,0 +1,1112 @@ +[ + { + "name": "canonical_3", + "migration": 3, + "mutation": "", + "expected": "t" + }, + { + "name": "default_added_participants_extended_permanent_cookie", + "migration": 3, + "mutation": "ALTER TABLE participants_extended ALTER COLUMN permanent_cookie SET DEFAULT 'constructed';", + "expected": "f" + }, + { + "name": "nullability_participants_extended_permanent_cookie", + "migration": 3, + "mutation": "ALTER TABLE participants_extended ALTER COLUMN permanent_cookie SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_added_participants_extended_origin", + "migration": 3, + "mutation": "ALTER TABLE participants_extended ALTER COLUMN origin SET DEFAULT 'constructed';", + "expected": "f" + }, + { + "name": "nullability_participants_extended_origin", + "migration": 3, + "mutation": "ALTER TABLE participants_extended ALTER COLUMN origin SET NOT NULL;", + "expected": "f" + }, + { + "name": "canonical_8", + "migration": 8, + "mutation": "", + "expected": "t" + }, + { + "name": "default_removed_conversations_importance_enabled", + "migration": 8, + "mutation": "ALTER TABLE conversations ALTER COLUMN importance_enabled DROP DEFAULT;", + "expected": "f" + }, + { + "name": "default_removed_votes_high_priority", + "migration": 8, + "mutation": "ALTER TABLE votes ALTER COLUMN high_priority DROP DEFAULT;", + "expected": "f" + }, + { + "name": "canonical_9", + "migration": 9, + "mutation": "", + "expected": "t" + }, + { + "name": "default_added_zinvites_uuid", + "migration": 9, + "mutation": "ALTER TABLE zinvites ALTER COLUMN uuid SET DEFAULT '00000000-0000-0000-0000-000000000001';", + "expected": "f" + }, + { + "name": "nullability_zinvites_uuid", + "migration": 9, + "mutation": "ALTER TABLE zinvites ALTER COLUMN uuid SET NOT NULL;", + "expected": "f" + }, + { + "name": "canonical_10", + "migration": 10, + "mutation": "", + "expected": "t" + }, + { + "name": "default_added_oidc_user_mappings_oidc_sub", + "migration": 10, + "mutation": "ALTER TABLE oidc_user_mappings ALTER COLUMN oidc_sub SET DEFAULT 'constructed';", + "expected": "f" + }, + { + "name": "default_added_oidc_user_mappings_uid", + "migration": 10, + "mutation": "ALTER TABLE oidc_user_mappings ALTER COLUMN uid SET DEFAULT 7;", + "expected": "f" + }, + { + "name": "default_removed_oidc_user_mappings_created", + "migration": 10, + "mutation": "ALTER TABLE oidc_user_mappings ALTER COLUMN created DROP DEFAULT;", + "expected": "f" + }, + { + "name": "nullability_oidc_user_mappings_created", + "migration": 10, + "mutation": "ALTER TABLE oidc_user_mappings ALTER COLUMN created SET NOT NULL;", + "expected": "f" + }, + { + "name": "canonical_11", + "migration": 11, + "mutation": "", + "expected": "t" + }, + { + "name": "canonical_12", + "migration": 12, + "mutation": "", + "expected": "t" + }, + { + "name": "default_added_topic_agenda_selections_zid", + "migration": 12, + "mutation": "ALTER TABLE topic_agenda_selections ALTER COLUMN zid SET DEFAULT 7;", + "expected": "f" + }, + { + "name": "default_added_topic_agenda_selections_pid", + "migration": 12, + "mutation": "ALTER TABLE topic_agenda_selections ALTER COLUMN pid SET DEFAULT 7;", + "expected": "f" + }, + { + "name": "default_removed_topic_agenda_selections_archetypal_selections", + "migration": 12, + "mutation": "ALTER TABLE topic_agenda_selections ALTER COLUMN archetypal_selections DROP DEFAULT;", + "expected": "f" + }, + { + "name": "default_added_topic_agenda_selections_delphi_job_id", + "migration": 12, + "mutation": "ALTER TABLE topic_agenda_selections ALTER COLUMN delphi_job_id SET DEFAULT 'constructed';", + "expected": "f" + }, + { + "name": "nullability_topic_agenda_selections_delphi_job_id", + "migration": 12, + "mutation": "ALTER TABLE topic_agenda_selections ALTER COLUMN delphi_job_id SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_removed_topic_agenda_selections_total_selections", + "migration": 12, + "mutation": "ALTER TABLE topic_agenda_selections ALTER COLUMN total_selections DROP DEFAULT;", + "expected": "f" + }, + { + "name": "default_removed_topic_agenda_selections_created_at", + "migration": 12, + "mutation": "ALTER TABLE topic_agenda_selections ALTER COLUMN created_at DROP DEFAULT;", + "expected": "f" + }, + { + "name": "nullability_topic_agenda_selections_created_at", + "migration": 12, + "mutation": "ALTER TABLE topic_agenda_selections ALTER COLUMN created_at SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_removed_topic_agenda_selections_updated_at", + "migration": 12, + "mutation": "ALTER TABLE topic_agenda_selections ALTER COLUMN updated_at DROP DEFAULT;", + "expected": "f" + }, + { + "name": "nullability_topic_agenda_selections_updated_at", + "migration": 12, + "mutation": "ALTER TABLE topic_agenda_selections ALTER COLUMN updated_at SET NOT NULL;", + "expected": "f" + }, + { + "name": "canonical_13", + "migration": 13, + "mutation": "", + "expected": "t" + }, + { + "name": "default_removed_conversations_treevite_enabled", + "migration": 13, + "mutation": "ALTER TABLE conversations ALTER COLUMN treevite_enabled DROP DEFAULT;", + "expected": "f" + }, + { + "name": "nullability_conversations_treevite_enabled", + "migration": 13, + "mutation": "ALTER TABLE conversations ALTER COLUMN treevite_enabled SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_removed_treevite_waves_id", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves ALTER COLUMN id DROP DEFAULT;", + "expected": "f" + }, + { + "name": "default_added_treevite_waves_zid", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves ALTER COLUMN zid SET DEFAULT 7;", + "expected": "f" + }, + { + "name": "default_added_treevite_waves_wave", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves ALTER COLUMN wave SET DEFAULT 7;", + "expected": "f" + }, + { + "name": "default_added_treevite_waves_parent_wave", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves ALTER COLUMN parent_wave SET DEFAULT 7;", + "expected": "f" + }, + { + "name": "nullability_treevite_waves_parent_wave", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves ALTER COLUMN parent_wave SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_added_treevite_waves_size", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves ALTER COLUMN size SET DEFAULT 7;", + "expected": "f" + }, + { + "name": "nullability_treevite_waves_size", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves ALTER COLUMN size SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_added_treevite_waves_invites_per_user", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves ALTER COLUMN invites_per_user SET DEFAULT 7;", + "expected": "f" + }, + { + "name": "default_removed_treevite_waves_owner_invites", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves ALTER COLUMN owner_invites DROP DEFAULT;", + "expected": "f" + }, + { + "name": "default_removed_treevite_waves_created_at", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves ALTER COLUMN created_at DROP DEFAULT;", + "expected": "f" + }, + { + "name": "nullability_treevite_waves_created_at", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves ALTER COLUMN created_at SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_removed_treevite_waves_updated_at", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves ALTER COLUMN updated_at DROP DEFAULT;", + "expected": "f" + }, + { + "name": "nullability_treevite_waves_updated_at", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves ALTER COLUMN updated_at SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_removed_treevite_invites_id", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER COLUMN id DROP DEFAULT;", + "expected": "f" + }, + { + "name": "default_added_treevite_invites_zid", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER COLUMN zid SET DEFAULT 7;", + "expected": "f" + }, + { + "name": "default_added_treevite_invites_wave_id", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER COLUMN wave_id SET DEFAULT 7;", + "expected": "f" + }, + { + "name": "default_added_treevite_invites_parent_invite_id", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER COLUMN parent_invite_id SET DEFAULT 7;", + "expected": "f" + }, + { + "name": "nullability_treevite_invites_parent_invite_id", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER COLUMN parent_invite_id SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_removed_treevite_invites_status", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER COLUMN status DROP DEFAULT;", + "expected": "f" + }, + { + "name": "default_added_treevite_invites_invite_owner_pid", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER COLUMN invite_owner_pid SET DEFAULT 7;", + "expected": "f" + }, + { + "name": "nullability_treevite_invites_invite_owner_pid", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER COLUMN invite_owner_pid SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_added_treevite_invites_invite_used_by_pid", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER COLUMN invite_used_by_pid SET DEFAULT 7;", + "expected": "f" + }, + { + "name": "nullability_treevite_invites_invite_used_by_pid", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER COLUMN invite_used_by_pid SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_added_treevite_invites_invite_used_at", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER COLUMN invite_used_at SET DEFAULT '2000-01-01';", + "expected": "f" + }, + { + "name": "nullability_treevite_invites_invite_used_at", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER COLUMN invite_used_at SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_removed_treevite_invites_created_at", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER COLUMN created_at DROP DEFAULT;", + "expected": "f" + }, + { + "name": "nullability_treevite_invites_created_at", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER COLUMN created_at SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_removed_treevite_invites_updated_at", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER COLUMN updated_at DROP DEFAULT;", + "expected": "f" + }, + { + "name": "nullability_treevite_invites_updated_at", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER COLUMN updated_at SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_removed_treevite_login_codes_id", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes ALTER COLUMN id DROP DEFAULT;", + "expected": "f" + }, + { + "name": "default_added_treevite_login_codes_zid", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes ALTER COLUMN zid SET DEFAULT 7;", + "expected": "f" + }, + { + "name": "default_added_treevite_login_codes_pid", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes ALTER COLUMN pid SET DEFAULT 7;", + "expected": "f" + }, + { + "name": "default_added_treevite_login_codes_login_code_hash", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes ALTER COLUMN login_code_hash SET DEFAULT 'constructed';", + "expected": "f" + }, + { + "name": "default_removed_treevite_login_codes_fp_kid", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes ALTER COLUMN fp_kid DROP DEFAULT;", + "expected": "f" + }, + { + "name": "default_removed_treevite_login_codes_revoked", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes ALTER COLUMN revoked DROP DEFAULT;", + "expected": "f" + }, + { + "name": "default_added_treevite_login_codes_expires_at", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes ALTER COLUMN expires_at SET DEFAULT '2000-01-01';", + "expected": "f" + }, + { + "name": "nullability_treevite_login_codes_expires_at", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes ALTER COLUMN expires_at SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_added_treevite_login_codes_last_used_at", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes ALTER COLUMN last_used_at SET DEFAULT '2000-01-01';", + "expected": "f" + }, + { + "name": "nullability_treevite_login_codes_last_used_at", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes ALTER COLUMN last_used_at SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_removed_treevite_login_codes_created_at", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes ALTER COLUMN created_at DROP DEFAULT;", + "expected": "f" + }, + { + "name": "nullability_treevite_login_codes_created_at", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes ALTER COLUMN created_at SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_removed_treevite_login_codes_updated_at", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes ALTER COLUMN updated_at DROP DEFAULT;", + "expected": "f" + }, + { + "name": "nullability_treevite_login_codes_updated_at", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes ALTER COLUMN updated_at SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_added_treevite_invites_invite_code", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER COLUMN invite_code SET DEFAULT 'constructed';", + "expected": "f" + }, + { + "name": "default_added_treevite_login_codes_login_code_fingerprint", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes ALTER COLUMN login_code_fingerprint SET DEFAULT 'constructed';", + "expected": "f" + }, + { + "name": "default_added_treevite_login_codes_login_code_lookup", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes ALTER COLUMN login_code_lookup SET DEFAULT 'constructed';", + "expected": "f" + }, + { + "name": "nullability_treevite_login_codes_login_code_lookup", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes ALTER COLUMN login_code_lookup SET NOT NULL;", + "expected": "f" + }, + { + "name": "canonical_14", + "migration": 14, + "mutation": "", + "expected": "t" + }, + { + "name": "default_removed_reports_mod_level", + "migration": 14, + "mutation": "ALTER TABLE reports ALTER COLUMN mod_level DROP DEFAULT;", + "expected": "f" + }, + { + "name": "canonical_15", + "migration": 15, + "mutation": "", + "expected": "t" + }, + { + "name": "default_removed_conversations_xid_required", + "migration": 15, + "mutation": "ALTER TABLE conversations ALTER COLUMN xid_required DROP DEFAULT;", + "expected": "f" + }, + { + "name": "default_added_xid_whitelist_zid", + "migration": 15, + "mutation": "ALTER TABLE xid_whitelist ALTER COLUMN zid SET DEFAULT 7;", + "expected": "f" + }, + { + "name": "nullability_xid_whitelist_zid", + "migration": 15, + "mutation": "ALTER TABLE xid_whitelist ALTER COLUMN zid SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_added_xids_zid", + "migration": 15, + "mutation": "ALTER TABLE xids ALTER COLUMN zid SET DEFAULT 7;", + "expected": "f" + }, + { + "name": "nullability_xids_zid", + "migration": 15, + "mutation": "ALTER TABLE xids ALTER COLUMN zid SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_added_xids_pid", + "migration": 15, + "mutation": "ALTER TABLE xids ALTER COLUMN pid SET DEFAULT 7;", + "expected": "f" + }, + { + "name": "nullability_xids_pid", + "migration": 15, + "mutation": "ALTER TABLE xids ALTER COLUMN pid SET NOT NULL;", + "expected": "f" + }, + { + "name": "canonical_16", + "migration": 16, + "mutation": "", + "expected": "t" + }, + { + "name": "default_added_comments_original_id", + "migration": 16, + "mutation": "ALTER TABLE comments ALTER COLUMN original_id SET DEFAULT '00000000-0000-0000-0000-000000000001';", + "expected": "f" + }, + { + "name": "nullability_comments_original_id", + "migration": 16, + "mutation": "ALTER TABLE comments ALTER COLUMN original_id SET NOT NULL;", + "expected": "f" + }, + { + "name": "canonical_17", + "migration": 17, + "mutation": "", + "expected": "t" + }, + { + "name": "default_removed_byod_import_jobs_id", + "migration": 17, + "mutation": "ALTER TABLE byod_import_jobs ALTER COLUMN id DROP DEFAULT;", + "expected": "f" + }, + { + "name": "default_added_byod_import_jobs_zid", + "migration": 17, + "mutation": "ALTER TABLE byod_import_jobs ALTER COLUMN zid SET DEFAULT 7;", + "expected": "f" + }, + { + "name": "default_added_byod_import_jobs_s3_key", + "migration": 17, + "mutation": "ALTER TABLE byod_import_jobs ALTER COLUMN s3_key SET DEFAULT 'constructed';", + "expected": "f" + }, + { + "name": "default_removed_byod_import_jobs_status", + "migration": 17, + "mutation": "ALTER TABLE byod_import_jobs ALTER COLUMN status DROP DEFAULT;", + "expected": "f" + }, + { + "name": "nullability_byod_import_jobs_status", + "migration": 17, + "mutation": "ALTER TABLE byod_import_jobs ALTER COLUMN status SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_removed_byod_import_jobs_stage", + "migration": 17, + "mutation": "ALTER TABLE byod_import_jobs ALTER COLUMN stage DROP DEFAULT;", + "expected": "f" + }, + { + "name": "nullability_byod_import_jobs_stage", + "migration": 17, + "mutation": "ALTER TABLE byod_import_jobs ALTER COLUMN stage SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_added_byod_import_jobs_error_message", + "migration": 17, + "mutation": "ALTER TABLE byod_import_jobs ALTER COLUMN error_message SET DEFAULT 'constructed';", + "expected": "f" + }, + { + "name": "nullability_byod_import_jobs_error_message", + "migration": 17, + "mutation": "ALTER TABLE byod_import_jobs ALTER COLUMN error_message SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_removed_byod_import_jobs_created_at", + "migration": 17, + "mutation": "ALTER TABLE byod_import_jobs ALTER COLUMN created_at DROP DEFAULT;", + "expected": "f" + }, + { + "name": "nullability_byod_import_jobs_created_at", + "migration": 17, + "mutation": "ALTER TABLE byod_import_jobs ALTER COLUMN created_at SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_removed_byod_import_jobs_updated_at", + "migration": 17, + "mutation": "ALTER TABLE byod_import_jobs ALTER COLUMN updated_at DROP DEFAULT;", + "expected": "f" + }, + { + "name": "nullability_byod_import_jobs_updated_at", + "migration": 17, + "mutation": "ALTER TABLE byod_import_jobs ALTER COLUMN updated_at SET NOT NULL;", + "expected": "f" + }, + { + "name": "canonical_18", + "migration": 18, + "mutation": "", + "expected": "t" + }, + { + "name": "default_removed_conversations_topics_enabled", + "migration": 18, + "mutation": "ALTER TABLE conversations ALTER COLUMN topics_enabled DROP DEFAULT;", + "expected": "f" + }, + { + "name": "missing_oidc_user_mappings_pkey", + "migration": 10, + "mutation": "ALTER TABLE oidc_user_mappings DROP CONSTRAINT oidc_user_mappings_pkey CASCADE;", + "expected": "f" + }, + { + "name": "missing_oidc_user_mappings_uid_fkey", + "migration": 10, + "mutation": "ALTER TABLE oidc_user_mappings DROP CONSTRAINT oidc_user_mappings_uid_fkey CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_oidc_user_mappings_uid_fkey", + "migration": 10, + "mutation": "ALTER TABLE oidc_user_mappings DROP CONSTRAINT oidc_user_mappings_uid_fkey CASCADE; ALTER TABLE oidc_user_mappings ADD CONSTRAINT oidc_user_mappings_uid_fkey FOREIGN KEY (uid) REFERENCES users(uid) ON DELETE CASCADE NOT VALID;", + "expected": "f" + }, + { + "name": "deferred_oidc_user_mappings_uid_fkey", + "migration": 10, + "mutation": "ALTER TABLE oidc_user_mappings ALTER CONSTRAINT oidc_user_mappings_uid_fkey DEFERRABLE INITIALLY DEFERRED;", + "expected": "f" + }, + { + "name": "missing_oidc_user_mappings_uid_key", + "migration": 10, + "mutation": "ALTER TABLE oidc_user_mappings DROP CONSTRAINT oidc_user_mappings_uid_key CASCADE;", + "expected": "f" + }, + { + "name": "disabled_triggers_oidc_user_mappings", + "migration": 10, + "mutation": "ALTER TABLE oidc_user_mappings DISABLE TRIGGER ALL;", + "expected": "f" + }, + { + "name": "missing_fk_conversation", + "migration": 12, + "mutation": "ALTER TABLE topic_agenda_selections DROP CONSTRAINT fk_conversation CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_fk_conversation", + "migration": 12, + "mutation": "ALTER TABLE topic_agenda_selections DROP CONSTRAINT fk_conversation CASCADE; ALTER TABLE topic_agenda_selections ADD CONSTRAINT fk_conversation FOREIGN KEY (zid) REFERENCES conversations(zid) ON DELETE CASCADE NOT VALID;", + "expected": "f" + }, + { + "name": "deferred_fk_conversation", + "migration": 12, + "mutation": "ALTER TABLE topic_agenda_selections ALTER CONSTRAINT fk_conversation DEFERRABLE INITIALLY DEFERRED;", + "expected": "f" + }, + { + "name": "missing_fk_participant", + "migration": 12, + "mutation": "ALTER TABLE topic_agenda_selections DROP CONSTRAINT fk_participant CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_fk_participant", + "migration": 12, + "mutation": "ALTER TABLE topic_agenda_selections DROP CONSTRAINT fk_participant CASCADE; ALTER TABLE topic_agenda_selections ADD CONSTRAINT fk_participant FOREIGN KEY (zid, pid) REFERENCES participants(zid, pid) ON DELETE CASCADE NOT VALID;", + "expected": "f" + }, + { + "name": "deferred_fk_participant", + "migration": 12, + "mutation": "ALTER TABLE topic_agenda_selections ALTER CONSTRAINT fk_participant DEFERRABLE INITIALLY DEFERRED;", + "expected": "f" + }, + { + "name": "missing_topic_agenda_selections_pkey", + "migration": 12, + "mutation": "ALTER TABLE topic_agenda_selections DROP CONSTRAINT topic_agenda_selections_pkey CASCADE;", + "expected": "f" + }, + { + "name": "disabled_triggers_topic_agenda_selections", + "migration": 12, + "mutation": "ALTER TABLE topic_agenda_selections DISABLE TRIGGER ALL;", + "expected": "f" + }, + { + "name": "missing_treevite_waves_invites_per_user_check", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves DROP CONSTRAINT treevite_waves_invites_per_user_check CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_treevite_waves_invites_per_user_check", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves DROP CONSTRAINT treevite_waves_invites_per_user_check CASCADE; ALTER TABLE treevite_waves ADD CONSTRAINT treevite_waves_invites_per_user_check CHECK ((invites_per_user >= 0)) NOT VALID;", + "expected": "f" + }, + { + "name": "missing_treevite_waves_not_both_zero", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves DROP CONSTRAINT treevite_waves_not_both_zero CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_treevite_waves_not_both_zero", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves DROP CONSTRAINT treevite_waves_not_both_zero CASCADE; ALTER TABLE treevite_waves ADD CONSTRAINT treevite_waves_not_both_zero CHECK (((invites_per_user > 0) OR (owner_invites > 0))) NOT VALID;", + "expected": "f" + }, + { + "name": "missing_treevite_waves_owner_invites_check", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves DROP CONSTRAINT treevite_waves_owner_invites_check CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_treevite_waves_owner_invites_check", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves DROP CONSTRAINT treevite_waves_owner_invites_check CASCADE; ALTER TABLE treevite_waves ADD CONSTRAINT treevite_waves_owner_invites_check CHECK ((owner_invites >= 0)) NOT VALID;", + "expected": "f" + }, + { + "name": "missing_treevite_waves_parent_wave_check", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves DROP CONSTRAINT treevite_waves_parent_wave_check CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_treevite_waves_parent_wave_check", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves DROP CONSTRAINT treevite_waves_parent_wave_check CASCADE; ALTER TABLE treevite_waves ADD CONSTRAINT treevite_waves_parent_wave_check CHECK (((parent_wave IS NULL) OR (parent_wave >= 0))) NOT VALID;", + "expected": "f" + }, + { + "name": "missing_treevite_waves_pkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves DROP CONSTRAINT treevite_waves_pkey CASCADE;", + "expected": "f" + }, + { + "name": "missing_treevite_waves_size_check", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves DROP CONSTRAINT treevite_waves_size_check CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_treevite_waves_size_check", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves DROP CONSTRAINT treevite_waves_size_check CASCADE; ALTER TABLE treevite_waves ADD CONSTRAINT treevite_waves_size_check CHECK (((size IS NULL) OR (size >= 0))) NOT VALID;", + "expected": "f" + }, + { + "name": "missing_treevite_waves_wave_check", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves DROP CONSTRAINT treevite_waves_wave_check CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_treevite_waves_wave_check", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves DROP CONSTRAINT treevite_waves_wave_check CASCADE; ALTER TABLE treevite_waves ADD CONSTRAINT treevite_waves_wave_check CHECK ((wave >= 1)) NOT VALID;", + "expected": "f" + }, + { + "name": "missing_treevite_waves_zid_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves DROP CONSTRAINT treevite_waves_zid_fkey CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_treevite_waves_zid_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves DROP CONSTRAINT treevite_waves_zid_fkey CASCADE; ALTER TABLE treevite_waves ADD CONSTRAINT treevite_waves_zid_fkey FOREIGN KEY (zid) REFERENCES conversations(zid) ON DELETE CASCADE NOT VALID;", + "expected": "f" + }, + { + "name": "deferred_treevite_waves_zid_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves ALTER CONSTRAINT treevite_waves_zid_fkey DEFERRABLE INITIALLY DEFERRED;", + "expected": "f" + }, + { + "name": "missing_treevite_waves_zid_wave_key", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves DROP CONSTRAINT treevite_waves_zid_wave_key CASCADE;", + "expected": "f" + }, + { + "name": "disabled_triggers_treevite_waves", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves DISABLE TRIGGER ALL;", + "expected": "f" + }, + { + "name": "missing_treevite_invites_code_unique", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites DROP CONSTRAINT treevite_invites_code_unique CASCADE;", + "expected": "f" + }, + { + "name": "missing_treevite_invites_owner_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites DROP CONSTRAINT treevite_invites_owner_fkey CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_treevite_invites_owner_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites DROP CONSTRAINT treevite_invites_owner_fkey CASCADE; ALTER TABLE treevite_invites ADD CONSTRAINT treevite_invites_owner_fkey FOREIGN KEY (zid, invite_owner_pid) REFERENCES participants(zid, pid) NOT VALID;", + "expected": "f" + }, + { + "name": "deferred_treevite_invites_owner_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER CONSTRAINT treevite_invites_owner_fkey DEFERRABLE INITIALLY DEFERRED;", + "expected": "f" + }, + { + "name": "missing_treevite_invites_parent_invite_id_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites DROP CONSTRAINT treevite_invites_parent_invite_id_fkey CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_treevite_invites_parent_invite_id_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites DROP CONSTRAINT treevite_invites_parent_invite_id_fkey CASCADE; ALTER TABLE treevite_invites ADD CONSTRAINT treevite_invites_parent_invite_id_fkey FOREIGN KEY (parent_invite_id) REFERENCES treevite_invites(id) ON DELETE SET NULL NOT VALID;", + "expected": "f" + }, + { + "name": "deferred_treevite_invites_parent_invite_id_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER CONSTRAINT treevite_invites_parent_invite_id_fkey DEFERRABLE INITIALLY DEFERRED;", + "expected": "f" + }, + { + "name": "missing_treevite_invites_pkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites DROP CONSTRAINT treevite_invites_pkey CASCADE;", + "expected": "f" + }, + { + "name": "missing_treevite_invites_status_check", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites DROP CONSTRAINT treevite_invites_status_check CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_treevite_invites_status_check", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites DROP CONSTRAINT treevite_invites_status_check CASCADE; ALTER TABLE treevite_invites ADD CONSTRAINT treevite_invites_status_check CHECK ((status = ANY (ARRAY[0, 1, 2, 3]))) NOT VALID;", + "expected": "f" + }, + { + "name": "missing_treevite_invites_used_by_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites DROP CONSTRAINT treevite_invites_used_by_fkey CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_treevite_invites_used_by_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites DROP CONSTRAINT treevite_invites_used_by_fkey CASCADE; ALTER TABLE treevite_invites ADD CONSTRAINT treevite_invites_used_by_fkey FOREIGN KEY (zid, invite_used_by_pid) REFERENCES participants(zid, pid) NOT VALID;", + "expected": "f" + }, + { + "name": "deferred_treevite_invites_used_by_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER CONSTRAINT treevite_invites_used_by_fkey DEFERRABLE INITIALLY DEFERRED;", + "expected": "f" + }, + { + "name": "missing_treevite_invites_wave_id_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites DROP CONSTRAINT treevite_invites_wave_id_fkey CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_treevite_invites_wave_id_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites DROP CONSTRAINT treevite_invites_wave_id_fkey CASCADE; ALTER TABLE treevite_invites ADD CONSTRAINT treevite_invites_wave_id_fkey FOREIGN KEY (wave_id) REFERENCES treevite_waves(id) ON DELETE CASCADE NOT VALID;", + "expected": "f" + }, + { + "name": "deferred_treevite_invites_wave_id_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER CONSTRAINT treevite_invites_wave_id_fkey DEFERRABLE INITIALLY DEFERRED;", + "expected": "f" + }, + { + "name": "missing_treevite_invites_zid_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites DROP CONSTRAINT treevite_invites_zid_fkey CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_treevite_invites_zid_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites DROP CONSTRAINT treevite_invites_zid_fkey CASCADE; ALTER TABLE treevite_invites ADD CONSTRAINT treevite_invites_zid_fkey FOREIGN KEY (zid) REFERENCES conversations(zid) ON DELETE CASCADE NOT VALID;", + "expected": "f" + }, + { + "name": "deferred_treevite_invites_zid_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER CONSTRAINT treevite_invites_zid_fkey DEFERRABLE INITIALLY DEFERRED;", + "expected": "f" + }, + { + "name": "disabled_triggers_treevite_invites", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites DISABLE TRIGGER ALL;", + "expected": "f" + }, + { + "name": "missing_treevite_login_codes_fp_unique", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes DROP CONSTRAINT treevite_login_codes_fp_unique CASCADE;", + "expected": "f" + }, + { + "name": "missing_treevite_login_codes_lookup_unique", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes DROP CONSTRAINT treevite_login_codes_lookup_unique CASCADE;", + "expected": "f" + }, + { + "name": "missing_treevite_login_codes_participant_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes DROP CONSTRAINT treevite_login_codes_participant_fkey CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_treevite_login_codes_participant_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes DROP CONSTRAINT treevite_login_codes_participant_fkey CASCADE; ALTER TABLE treevite_login_codes ADD CONSTRAINT treevite_login_codes_participant_fkey FOREIGN KEY (zid, pid) REFERENCES participants(zid, pid) ON DELETE CASCADE NOT VALID;", + "expected": "f" + }, + { + "name": "deferred_treevite_login_codes_participant_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes ALTER CONSTRAINT treevite_login_codes_participant_fkey DEFERRABLE INITIALLY DEFERRED;", + "expected": "f" + }, + { + "name": "missing_treevite_login_codes_pid_unique", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes DROP CONSTRAINT treevite_login_codes_pid_unique CASCADE;", + "expected": "f" + }, + { + "name": "missing_treevite_login_codes_pkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes DROP CONSTRAINT treevite_login_codes_pkey CASCADE;", + "expected": "f" + }, + { + "name": "disabled_triggers_treevite_login_codes", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes DISABLE TRIGGER ALL;", + "expected": "f" + }, + { + "name": "missing_xid_whitelist_zid_fkey", + "migration": 15, + "mutation": "ALTER TABLE xid_whitelist DROP CONSTRAINT xid_whitelist_zid_fkey CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_xid_whitelist_zid_fkey", + "migration": 15, + "mutation": "ALTER TABLE xid_whitelist DROP CONSTRAINT xid_whitelist_zid_fkey CASCADE; ALTER TABLE xid_whitelist ADD CONSTRAINT xid_whitelist_zid_fkey FOREIGN KEY (zid) REFERENCES conversations(zid) ON DELETE CASCADE NOT VALID;", + "expected": "f" + }, + { + "name": "deferred_xid_whitelist_zid_fkey", + "migration": 15, + "mutation": "ALTER TABLE xid_whitelist ALTER CONSTRAINT xid_whitelist_zid_fkey DEFERRABLE INITIALLY DEFERRED;", + "expected": "f" + }, + { + "name": "disabled_triggers_xid_whitelist", + "migration": 15, + "mutation": "ALTER TABLE xid_whitelist DISABLE TRIGGER ALL;", + "expected": "f" + }, + { + "name": "missing_xids_zid_fkey", + "migration": 15, + "mutation": "ALTER TABLE xids DROP CONSTRAINT xids_zid_fkey CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_xids_zid_fkey", + "migration": 15, + "mutation": "ALTER TABLE xids DROP CONSTRAINT xids_zid_fkey CASCADE; ALTER TABLE xids ADD CONSTRAINT xids_zid_fkey FOREIGN KEY (zid) REFERENCES conversations(zid) ON DELETE CASCADE NOT VALID;", + "expected": "f" + }, + { + "name": "deferred_xids_zid_fkey", + "migration": 15, + "mutation": "ALTER TABLE xids ALTER CONSTRAINT xids_zid_fkey DEFERRABLE INITIALLY DEFERRED;", + "expected": "f" + }, + { + "name": "missing_xids_pid_fkey", + "migration": 15, + "mutation": "ALTER TABLE xids DROP CONSTRAINT xids_pid_fkey CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_xids_pid_fkey", + "migration": 15, + "mutation": "ALTER TABLE xids DROP CONSTRAINT xids_pid_fkey CASCADE; ALTER TABLE xids ADD CONSTRAINT xids_pid_fkey FOREIGN KEY (zid, pid) REFERENCES participants(zid, pid) ON DELETE SET NULL NOT VALID;", + "expected": "f" + }, + { + "name": "deferred_xids_pid_fkey", + "migration": 15, + "mutation": "ALTER TABLE xids ALTER CONSTRAINT xids_pid_fkey DEFERRABLE INITIALLY DEFERRED;", + "expected": "f" + }, + { + "name": "disabled_triggers_xids", + "migration": 15, + "mutation": "ALTER TABLE xids DISABLE TRIGGER ALL;", + "expected": "f" + }, + { + "name": "missing_byod_import_jobs_pkey", + "migration": 17, + "mutation": "ALTER TABLE byod_import_jobs DROP CONSTRAINT byod_import_jobs_pkey CASCADE;", + "expected": "f" + }, + { + "name": "deferred_unique", + "migration": 10, + "mutation": "ALTER TABLE oidc_user_mappings DROP CONSTRAINT oidc_user_mappings_uid_key; ALTER TABLE oidc_user_mappings ADD CONSTRAINT generated_uid_unique UNIQUE(uid) DEFERRABLE INITIALLY DEFERRED;", + "expected": "f" + }, + { + "name": "m3_wrong_type", + "migration": 3, + "mutation": "ALTER TABLE participants_extended ALTER COLUMN origin TYPE text;", + "expected": "f" + }, + { + "name": "m11_preserved_default", + "migration": 11, + "mutation": "ALTER TABLE suzinvites ALTER COLUMN xid SET DEFAULT 'constructed';", + "expected": "t" + }, + { + "name": "renamed_equivalent_unique", + "migration": 10, + "mutation": "ALTER TABLE oidc_user_mappings RENAME CONSTRAINT oidc_user_mappings_uid_key TO generated_equivalent_key;", + "expected": "t" + }, + { + "name": "m3_generated_column", + "migration": 3, + "mutation": "ALTER TABLE participants_extended DROP COLUMN origin; ALTER TABLE participants_extended ADD COLUMN origin varchar(9999) GENERATED ALWAYS AS ('constructed'::varchar(9999)) STORED;", + "expected": "f" + }, + { + "name": "m17_identity_column", + "migration": 17, + "mutation": "ALTER TABLE byod_import_jobs ALTER COLUMN id DROP DEFAULT; ALTER TABLE byod_import_jobs ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY;", + "expected": "f" + } +] diff --git a/queue-rs/polis-migrate/tests/adoption.py b/queue-rs/polis-migrate/tests/adoption.py new file mode 100644 index 0000000000..07a08fa04c --- /dev/null +++ b/queue-rs/polis-migrate/tests/adoption.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +"""Persistent adoption-contract regression tests; generated DBs and no vote rows. + +Run after starting tests/compose.yml with unique COMPOSE_PROJECT_NAME and ports. +Uses the built polis-migrate in this checkout; no packet or original source needed. +""" +import json, pathlib, unittest +from prove import MIG, runner, sql + +class Adoption(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.db='adoption_contract_template' + sql('postgres',f'CREATE DATABASE {cls.db}') + for path in sorted(MIG.glob('*.sql')): + if int(path.name[:6]) <= 18 or path.name.startswith('000022_'): + sql(cls.db,path.read_text()) + sql(cls.db,"INSERT INTO users(hname) VALUES('constructed adoption sentinel')") + + def test_01_catalog_controls(self): + cases=json.loads(pathlib.Path(__file__).with_name('adoption-cases.json').read_text()) + helpers=(MIG/'adoption/helpers.sql').read_text() + for case in cases: + with self.subTest(case=case['name']): + predicate=next((MIG/'adoption').glob(f"{case['migration']:06d}_*.sql")).read_text() + out=sql(self.db,'BEGIN;\n'+case['mutation']+'\n'+helpers+'\n'+predicate+'\nROLLBACK;') + # psql also prints transaction/function command tags. + values=[line for line in out.splitlines() if line in ('t','f')] + self.assertEqual(values,[case['expected']]) + + def test_02_conflicts_leave_no_history(self): + cases=[(3,"ALTER TABLE participants_extended ALTER COLUMN origin SET DEFAULT 'constructed';"), + (13,'ALTER TABLE treevite_invites DISABLE TRIGGER ALL;'), + (17,'ALTER TABLE byod_import_jobs ALTER COLUMN id DROP DEFAULT;')] + for n,mutation in cases: + with self.subTest(migration=n): + db=f'adoption_contract_conflict_{n}' + sql('postgres',f'CREATE DATABASE {db} TEMPLATE {self.db}') + sql(db,mutation) + p=runner(db,'reconcile','--through','000022',ok=False) + self.assertIn(f'{n:06d}_',p.stderr) + self.assertEqual(sql(db,"SELECT to_regclass('public.migrations') IS NULL"),'t') + + def test_03_canonical_adoption(self): + db='adoption_contract_success' + sql('postgres',f'CREATE DATABASE {db} TEMPLATE {self.db}') + runner(db,'reconcile','--through','000022') + self.assertEqual(sql(db,"SELECT count(*) FROM migrations WHERE status='ADOPTED'"),'20') + self.assertEqual(sql(db,'SELECT hname FROM users'),'constructed adoption sentinel') + +if __name__=='__main__': unittest.main(verbosity=2) diff --git a/queue-rs/polis-migrate/tests/compose.yml b/queue-rs/polis-migrate/tests/compose.yml new file mode 100644 index 0000000000..fe26fd80db --- /dev/null +++ b/queue-rs/polis-migrate/tests/compose.yml @@ -0,0 +1,14 @@ +services: + postgres: + image: postgres:17-alpine + environment: + POSTGRES_HOST_AUTH_METHOD: trust + POSTGRES_USER: postgres + POSTGRES_DB: postgres + ports: + - "127.0.0.1:${POLIS_RECOVERY_PG_PORT:?set an owned port}:5432" + healthcheck: + test: [CMD-SHELL, "pg_isready -U postgres"] + interval: 1s + timeout: 3s + retries: 30 diff --git a/queue-rs/polis-migrate/tests/fresh-image.sh b/queue-rs/polis-migrate/tests/fresh-image.sh new file mode 100644 index 0000000000..80f4c41d97 --- /dev/null +++ b/queue-rs/polis-migrate/tests/fresh-image.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# CI and work boxes run this same build/bootstrap/restart proof. +set -euo pipefail +cd "$(dirname "$0")/../../.." +case "${COMPOSE_PROJECT_NAME:-}" in + polis-migrate-test-?*) ;; + *) echo 'Set an owned COMPOSE_PROJECT_NAME starting polis-migrate-test-' >&2; exit 2 ;; +esac +: "${POLIS_RECOVERY_PG_PORT:?set an owned port}" +export RECOVERY_PG_PORT="$POLIS_RECOVERY_PG_PORT" +export POLIS_MIGRATE_TEST_IMAGE="${COMPOSE_PROJECT_NAME}:fresh-image" +compose=(docker compose -f queue-rs/polis-migrate/tests/compose.yml -f queue-rs/polis-migrate/tests/image.yml) +cleanup() { + local status=$? + trap - EXIT + "${compose[@]}" down -v || status=1 + if docker image inspect "$POLIS_MIGRATE_TEST_IMAGE" >/dev/null 2>&1; then + docker image rm "$POLIS_MIGRATE_TEST_IMAGE" || status=1 + fi + exit "$status" +} +trap cleanup EXIT +python3 queue-rs/polis-migrate/tests/image-proof-test.py +"${compose[@]}" down -v +docker build --build-context queue-rs=queue-rs -t "$POLIS_MIGRATE_TEST_IMAGE" -f server/Dockerfile-db server +"${compose[@]}" up -d --wait --wait-timeout 120 +python3 queue-rs/polis-migrate/tests/image-proof.py diff --git a/queue-rs/polis-migrate/tests/image-proof-test.py b/queue-rs/polis-migrate/tests/image-proof-test.py new file mode 100644 index 0000000000..be4181fe9c --- /dev/null +++ b/queue-rs/polis-migrate/tests/image-proof-test.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""Regression controls for fresh-image release evolution and corrupt receipts.""" +import importlib.util +import pathlib +import tempfile +import unittest + +spec = importlib.util.spec_from_file_location( + "image_proof", pathlib.Path(__file__).with_name("image-proof.py")) +proof = importlib.util.module_from_spec(spec) +spec.loader.exec_module(proof) + + +class ImageProofTests(unittest.TestCase): + def setUp(self): + self.expected = proof.expected_receipts(proof.MIGRATIONS) + self.rows = [{"name": name, **receipt} for name, receipt in self.expected.items()] + + def test_current_release_includes_adoption(self): + proof.assert_receipts(self.rows, self.expected) + self.assertEqual({row["name"][:6] for row in self.rows if row["status"] == "ADOPTED"}, + proof.RETIRED) + self.assertTrue(any(row["status"] == "APPLIED" for row in self.rows)) + + def test_same_count_wrong_status_refuses(self): + next(row for row in self.rows if row["status"] == "ADOPTED")["status"] = "APPLIED" + with self.assertRaises(AssertionError): + proof.assert_receipts(self.rows, self.expected) + + def test_same_count_wrong_name_refuses(self): + self.rows[0]["name"] = "999999_unselected.sql" + with self.assertRaises(AssertionError): + proof.assert_receipts(self.rows, self.expected) + + def test_same_count_wrong_checksum_refuses(self): + self.rows[0]["checksum"] = "0" * 64 + with self.assertRaises(AssertionError): + proof.assert_receipts(self.rows, self.expected) + + def test_missing_receipt_refuses(self): + with self.assertRaises(AssertionError): + proof.assert_receipts(self.rows[1:], self.expected) + + def test_extra_receipt_refuses(self): + with self.assertRaises(AssertionError): + proof.assert_receipts(self.rows + [{**self.rows[0], "name": "999999_extra.sql"}], + self.expected) + + def test_duplicate_receipt_refuses(self): + with self.assertRaises(AssertionError): + proof.assert_receipts(self.rows + [self.rows[0]], self.expected) + + def test_release_grows_and_shrinks_without_count_edits(self): + with tempfile.TemporaryDirectory() as tmp: + directory = pathlib.Path(tmp) + names = ["000000_initial.sql", "000004_drop_waitinglist_table.sql", + "000030_future.sql"] + for name in names: + (directory / name).write_text("-- constructed SQL source\n") + for selected in (names[:2], names, names[:1]): + with self.subTest(selected=selected): + (directory / "release.txt").write_text( + "# constructed release selection\n\n" + "\n".join(selected) + "\n") + expected = proof.expected_receipts(directory) + self.assertEqual(set(expected), set(selected)) + self.assertEqual(len(expected), len(selected)) + proof.assert_receipts([{"name": name, **receipt} + for name, receipt in expected.items()], expected) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/queue-rs/polis-migrate/tests/image-proof.py b/queue-rs/polis-migrate/tests/image-proof.py new file mode 100644 index 0000000000..76f6453aef --- /dev/null +++ b/queue-rs/polis-migrate/tests/image-proof.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""Check the selected fresh-image receipts and preserve them across restart.""" +import collections +import hashlib +import json +import os +import pathlib +import re +import subprocess + + +# Observation-only retirement policy in polis_migrate::load/retired_absent. +# These are migration identities, not a count of the current release. +RETIRED = {"000004", "000005", "000007"} +ROOT = pathlib.Path(__file__).resolve().parents[3] +MIGRATIONS = ROOT / "server/postgres/migrations" + + +def expected_receipts(directory): + names = [line for line in (directory / "release.txt").read_text().splitlines() + if line and not line.startswith("#")] + assert names and len(names) == len(set(names)), "empty/duplicate release selection" + assert all(re.fullmatch(r"[0-9]{6}_[A-Za-z0-9_]+\.sql", name) for name in names) + return { + name: {"status": "ADOPTED" if name[:6] in RETIRED else "APPLIED", + "checksum": hashlib.sha256((directory / name).read_bytes()).hexdigest()} + for name in names + } + + +def assert_receipts(rows, expected): + actual = {row["name"]: {"status": row["status"], "checksum": row["checksum"]} + for row in rows} + assert len(rows) == len(actual), "duplicate history names" + assert actual == expected, f"fresh history differs from release: {actual!r} != {expected!r}" + + +def main(): + assert os.environ.get("COMPOSE_PROJECT_NAME", "").startswith("polis-migrate-test-") + directory = pathlib.Path(__file__).resolve().parent + compose = ["docker", "compose", "-f", str(directory / "compose.yml"), + "-f", str(directory / "image.yml")] + + def run(*args): + return subprocess.check_output(compose + list(args), text=True).strip() + + def query(sql): + return run("exec", "-T", "postgres", "psql", "-X", "-v", "ON_ERROR_STOP=1", + "-U", "postgres", "-d", "postgres", "-Atc", sql) + + def history(): + return [json.loads(line) for line in + query("SELECT row_to_json(m) FROM migrations m ORDER BY name").splitlines()] + + expected = expected_receipts(MIGRATIONS) + before = history() + assert_receipts(before, expected) + assert query("SELECT to_regclass('public.polis_coordinator_install') IS NULL") == "t" + run("restart", "postgres") + run("up", "-d", "--wait", "--wait-timeout", "120") + assert history() == before, "restart changed migration receipts" + assert run("exec", "-T", "-e", + "DATABASE_URL=host=/var/run/postgresql user=postgres dbname=postgres sslmode=disable", + "-e", "POLIS_MIGRATIONS_DIR=/migrations", "postgres", "polis-migrate", "check" + ) == f"migration check: {len(expected)} ready" + counts = dict(sorted(collections.Counter(row["status"] for row in before).items())) + print(f"image proof: 4 checks PASS; {len(expected)} selected receipts; {counts}") + + +if __name__ == "__main__": + main() diff --git a/queue-rs/polis-migrate/tests/image.yml b/queue-rs/polis-migrate/tests/image.yml new file mode 100644 index 0000000000..dcaa565723 --- /dev/null +++ b/queue-rs/polis-migrate/tests/image.yml @@ -0,0 +1,7 @@ +services: + postgres: + image: ${POLIS_MIGRATE_TEST_IMAGE:?run fresh-image.sh with an owned project} + healthcheck: + # The initdb temporary server accepts socket connections before the runner + # has finished. TCP readiness waits for the final server after initialization. + test: [CMD-SHELL, "pg_isready -h 127.0.0.1 -U postgres"] diff --git a/queue-rs/polis-migrate/tests/indexes.py b/queue-rs/polis-migrate/tests/indexes.py new file mode 100644 index 0000000000..657ffd6660 --- /dev/null +++ b/queue-rs/polis-migrate/tests/indexes.py @@ -0,0 +1,84 @@ +"""Real generated Postgres runner controls. No vote rows are inserted.""" +import importlib.util,json,os,pathlib,shutil,subprocess,tempfile,time,unittest +ROOT=pathlib.Path(__file__).resolve().parents[3] +spec=importlib.util.spec_from_file_location('proof',ROOT/'queue-rs/polis-migrate/tests/prove.py');p=importlib.util.module_from_spec(spec);spec.loader.exec_module(p) +ORIGINAL=os.environ.get('ORIGINAL_RUNNER') +class IndexProof(unittest.TestCase): + def setUp(self): + self.tmp=tempfile.TemporaryDirectory();self.addCleanup(self.tmp.cleanup);self.d=pathlib.Path(self.tmp.name) + (self.d/'held.txt').write_text('') + self.m=p.MIG/'000022_add_poll_timestamp_indexes.sql';shutil.copy(self.m,self.d/self.m.name) + def db(self,name,extra=''): + db='idx_'+name;p.sql('postgres',f'CREATE DATABASE {db}') + (self.d/'000000_initial.sql').write_text('CREATE TABLE public.votes(created bigint); CREATE TABLE public.comments(modified bigint, other integer);'+extra) + return db + def runnew(self,db,*args,**kw): + (self.d/'release.txt').write_text(''.join(f.name+'\n' for f in sorted(self.d.glob('*.sql')))) + return p.runner(db,*args,dir=self.d,**kw) + def test_01_original_refuses_large_candidate_passes(self): + db=self.db('large','INSERT INTO public.comments SELECT x,0 FROM generate_series(1,100001) x;') + if ORIGINAL: + before=subprocess.run([ORIGINAL,'apply','--dir',str(self.d)],env=p.env(db),text=True,capture_output=True,timeout=60) + self.assertNotEqual(before.returncode,0);self.assertIn('000022',before.stderr) + self.assertEqual(p.sql(db,"SELECT count(*) FROM migrations WHERE name LIKE '000022%'"),'0') + self.assertIn('BUILDING CONCURRENTLY',self.runnew(db,'apply').stdout) + self.runnew(db,'check');self.assertEqual(p.sql(db,'SELECT count(*) FROM comments'),'100001') + self.assertEqual(p.sql(db,'SELECT count(*) FROM votes'),'0') + def test_02_partial_success_resumes_without_rebuild(self): + db=self.db('partial','CREATE INDEX votes_created_idx ON public.votes(created);') + # Bootstrap only first to observe its OID before the pending index runs. + m=self.d/self.m.name;m.rename(self.d/'index.pending') + self.runnew(db,'apply');oid=p.sql(db,"SELECT 'public.votes_created_idx'::regclass::oid") + (self.d/'index.pending').rename(m) + self.runnew(db,'apply');self.assertEqual(oid,p.sql(db,"SELECT 'public.votes_created_idx'::regclass::oid")) + self.assertIn('applied 0 migration(s)',self.runnew(db,'apply').stdout) + def test_03_collision_preflight_preserves_objects(self): + db=self.db('collision','CREATE INDEX comments_modified_idx ON public.comments(other);') + r=self.runnew(db,'apply',ok=False);self.assertIn('conflicting object',r.stderr) + self.assertEqual(p.sql(db,"SELECT to_regclass('public.votes_created_idx') IS NULL"),'t') + self.assertEqual(p.sql(db,"SELECT count(*) FROM migrations WHERE name LIKE '000022%'"),'0') + def test_04_history_failure_resumes_committed_indexes(self): + db=self.db('history',"CREATE FUNCTION public.refuse_index_record() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.name LIKE '000022%' THEN RAISE EXCEPTION 'generated history fault'; END IF; RETURN NEW; END $$; CREATE TRIGGER refuse_index_record BEFORE INSERT ON migrations FOR EACH ROW EXECUTE FUNCTION public.refuse_index_record();") + self.runnew(db,'apply',ok=False) + before=p.sql(db,"SELECT indexrelid FROM pg_index WHERE indrelid IN ('votes'::regclass,'comments'::regclass) ORDER BY indexrelid") + self.assertEqual(len(before.splitlines()),2) + self.assertEqual(p.sql(db,"SELECT count(*) FROM migrations WHERE name LIKE '000022%'"),'0') + p.sql(db,'DROP TRIGGER refuse_index_record ON migrations');self.runnew(db,'apply') + self.assertEqual(before,p.sql(db,"SELECT indexrelid FROM pg_index WHERE indrelid IN ('votes'::regclass,'comments'::regclass) ORDER BY indexrelid")) + def holder(self,db): + q=subprocess.Popen(p.COMPOSE+['exec','-T','postgres','psql','-X','-U','postgres','-d',db,'-Atq'],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True) + q.stdin.write("BEGIN; INSERT INTO comments VALUES(1,1); SELECT 'held';\n");q.stdin.flush();self.assertEqual(q.stdout.readline().strip(),'held');return q + def release(self,q): + q.stdin.write('COMMIT;\n\\q\n');q.stdin.flush();q.communicate(timeout=15) + def wait_index(self,db): + deadline=time.monotonic()+15 + while time.monotonic()) -> Result { + if claim.stage.starts_with("graph_") { + return Ok( + json!({"schema":"polis-job-stage-frame/1", "env":claim.env, "zid":adm.zid, + "job_id":claim.job_id,"run_id":claim.run_id,"attempt_id":claim.attempt_id, + "lease_epoch":claim.lease_epoch,"stage":claim.stage,"input":adm.inputs, + "input_sha256":adm.config["input_sha256"],"input_json":adm.config["input_wire"]}), + ); + } let config = if claim.stage == "math_rebuild" { MathConfig::from_admission(adm)?.to_json() } else { @@ -331,6 +339,9 @@ pub fn command_args( app.join("umap_narrative/803_check_batch_status.py"), vec![format!("--job-id={}", claim.job_id)], ), + ("graph_embed" | "graph_cluster" | "graph_narrative", "run") => { + (app.join("scripts/job_graph_stage.py"), vec![]) + } ("math_rebuild", "run") => (app.join("scripts/math_poller.py"), vec!["--job".to_owned()]), (stage, phase) => bail!("no command for stage {stage} phase {phase}"), }) diff --git a/queue-rs/src/jobs/claim.rs b/queue-rs/src/jobs/claim.rs index a57007472b..6f379eff4a 100644 --- a/queue-rs/src/jobs/claim.rs +++ b/queue-rs/src/jobs/claim.rs @@ -62,7 +62,14 @@ fn claim_one( json!(cfg.lease_seconds), json!(cfg.worker_class.name()), ]; - let reply = match rpc.call("pq_claim", &args)? { + let graph = cfg.stages.iter().any(|s| s.starts_with("graph_")); + let mut args = args.to_vec(); + if graph { + args.push(json!( + cfg.stages.iter().cloned().collect::>().join(",") + )); + } + let reply = match rpc.call(if graph { "pd_graph_claim" } else { "pq_claim" }, &args)? { Completion::Committed(r) => r, Completion::Unknown(r) => { // Never reissue an uncertain claim: renew the exact token. @@ -329,6 +336,10 @@ pub fn run(cfg: Config) -> i32 { } } } + if cfg.stages.iter().any(|s| s.starts_with("graph_")) && ctx.contract() != "polis-queue/5" { + line("graph stages require polis-queue/5"); + return EXIT_CONTRACT; + } recover(&ctx, &mut rpc); let mut listener: Option = rpc.listener().ok(); let mut jobs: Vec> = vec![]; @@ -387,6 +398,11 @@ pub fn run(cfg: Config) -> i32 { if Instant::now() >= next_reap { next_reap = Instant::now() + cfg.reap_interval; retry_pending(&ctx, &mut rpc); + if ctx.contract() == "polis-queue/5" + && let Err(e) = rpc.committed("pd_graph_reconcile", &[json!(cfg.env)]) + { + line(&format!("graph release reconciliation failed: {e}")); + } if let Err(e) = reaper.tick(&mut rpc, cfg.worker_class.name(), &owner, &ctx.counters) { db_failures += 1; line(&format!("reap failed: {e}")); diff --git a/queue-rs/src/jobs/config.rs b/queue-rs/src/jobs/config.rs index c8ac054829..22ee8a5fcb 100644 --- a/queue-rs/src/jobs/config.rs +++ b/queue-rs/src/jobs/config.rs @@ -6,6 +6,7 @@ use std::{collections::BTreeSet, path::PathBuf, time::Duration}; /// `polis_queue_jobs_stage_check`). pub const KNOWN_STAGES: [&str; 2] = ["delphi_full_pipeline", "delphi_narrative"]; /// The stages of class `large`, which `polis-queue/3` admits (000024). +pub const GRAPH_STAGES: [&str; 3] = ["graph_embed", "graph_cluster", "graph_narrative"]; pub const LARGE_STAGES: [&str; 1] = ["math_rebuild"]; /// The worker class the daemon claims as (`POLIS_JOBS_WORKER_CLASS`). Each @@ -47,8 +48,8 @@ impl WorkerClass { /// worker of this class may start on. pub fn contracts(self) -> &'static [&'static str] { match self { - Self::Delphi => &["polis-queue/2", "polis-queue/3"], - Self::Large => &["polis-queue/3"], + Self::Delphi => &["polis-queue/2", "polis-queue/3", "polis-queue/5"], + Self::Large => &["polis-queue/3", "polis-queue/5"], } } @@ -201,9 +202,11 @@ pub fn load Option>(get: F) -> Result .map(str::to_owned) .collect(); if stages.is_empty() - || stages - .iter() - .any(|s| !worker_class.stages().contains(&s.as_str())) + || stages.iter().any(|s| { + !worker_class.stages().contains(&s.as_str()) + && !(GRAPH_STAGES.contains(&s.as_str()) + && (worker_class == WorkerClass::Delphi || s == "graph_cluster")) + }) { return err(format!( "POLIS_JOBS_STAGES must be a non-empty subset of {} (the stages of class {})", @@ -211,6 +214,11 @@ pub fn load Option>(get: F) -> Result worker_class.name() )); } + if stages.iter().any(|s| s.starts_with("graph_")) + && stages.iter().any(|s| !s.starts_with("graph_")) + { + return err("graph and legacy stages require separate workers"); + } let lease = number(&get, "POLIS_JOBS_LEASE_SECONDS", 120, 10, 900)?; let heartbeat = number(&get, "POLIS_JOBS_HEARTBEAT_SECONDS", 30, 1, 900)?; if heartbeat * 3 >= lease { diff --git a/queue-rs/src/jobs/graph.rs b/queue-rs/src/jobs/graph.rs new file mode 100644 index 0000000000..fb823a4a10 --- /dev/null +++ b/queue-rs/src/jobs/graph.rs @@ -0,0 +1,112 @@ +//! Versioned sealed-graph boundary. SQL is final authority for content binding. +use super::{ + child::Admission, + manifest::{Invalid, Manifest, Outcome}, + sha256_hex, +}; +use anyhow::{Result, ensure}; +use serde_json::{Value, json}; +pub fn admission(reply: &Value) -> Result { + ensure!( + reply["schema_version"] == "polis-queue/5" + && reply["graph_input"]["schema"] == "polis-job-input/1", + "graph_input_schema" + ); + let sha = reply["graph_input_sha"].as_str().unwrap_or_default(); + ensure!( + sha.len() == 64 && sha.bytes().all(|c| c.is_ascii_hexdigit()), + "graph_input_digest" + ); + let wire = reply["graph_input_wire"] + .as_str() + .ok_or_else(|| anyhow::anyhow!("graph_input_wire"))?; + ensure!( + sha256_hex(wire.as_bytes()) == sha + && serde_json::from_str::(wire)? == reply["graph_input"], + "graph_input_digest" + ); + Ok(Admission { + zid: reply["zid"] + .as_i64() + .ok_or_else(|| anyhow::anyhow!("graph_zid"))?, + report_id: None, + config: json!({"input_sha256":sha,"input_wire":wire}), + inputs: reply["graph_input"].clone(), + }) +} +pub fn validate( + bytes: Option<&[u8]>, + job: &str, + attempt: &str, + stage: &str, +) -> Result { + let b = bytes.ok_or(Invalid::Missing)?; + if b.len() > super::manifest::MAX_BYTES { + return Err(Invalid::TooLarge); + } + let text = std::str::from_utf8(b).map_err(|_| Invalid::NotUtf8)?; + let m: Value = serde_json::from_str(text).map_err(|_| Invalid::NotJson)?; + let keys = [ + "schema", + "job_id", + "run_id", + "attempt_id", + "stage", + "input_sha256", + "outcome", + "output", + ]; + if m.as_object() + .is_none_or(|o| o.len() != keys.len() || keys.iter().any(|k| !o.contains_key(*k))) + || m["schema"] != "polis-job-artifact-manifest/1" + || m["job_id"] != job + || m["attempt_id"] != attempt + || m["stage"] != stage + || m["outcome"] != "succeeded" + || m["output"]["role"] != "result" + || m["output"]["schema"] != format!("{stage}/1") + || m["output"]["payload"] + .as_str() + .is_none_or(|p| p.len() > 524288 || m["output"]["sha256"] != sha256_hex(p.as_bytes())) + { + return Err(Invalid::Field("graph_manifest")); + } + Ok(Manifest { + text: text.into(), + sha256: sha256_hex(b), + outcome: Outcome::Succeeded, + phase: "run".into(), + recheck_after: None, + batch_id: None, + batch_ids: vec![], + tokens_in: None, + tokens_out: None, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn resolved_bytes_are_checked_before_dispatch() { + let wire = r#"{"schema":"polis-job-input/1","declared":{},"artifacts":{}}"#; + let mut r = json!({"schema_version":"polis-queue/5","graph_input":serde_json::from_str::(wire).unwrap_or_else(|e| panic!("{e}")), + "graph_input_wire":wire,"graph_input_sha":sha256_hex(wire.as_bytes()),"zid":1}); + assert!(admission(&r).is_ok()); + r["graph_input"]["declared"] = json!({"tampered":true}); + assert!(admission(&r).is_err()); + r["schema_version"] = json!("polis-queue/3"); + assert!(admission(&r).is_err()); + } + #[test] + fn content_mismatch_and_legacy_receipts_refuse() { + let mut m = json!({"schema":"polis-job-artifact-manifest/1","job_id":"j","run_id":"r","attempt_id":"a", + "stage":"graph_embed","input_sha256":"0".repeat(64),"outcome":"succeeded", + "output":{"role":"result","schema":"graph_embed/1","payload":"{}","sha256":sha256_hex(b"{}")}}); + assert!(validate(Some(m.to_string().as_bytes()), "j", "a", "graph_embed").is_ok()); + m["output"]["payload"] = json!("changed"); + assert!(validate(Some(m.to_string().as_bytes()), "j", "a", "graph_embed").is_err()); + m["schema"] = json!("polis-jobs.output-manifest/1"); + assert!(validate(Some(m.to_string().as_bytes()), "j", "a", "graph_embed").is_err()); + } +} diff --git a/queue-rs/src/jobs/mod.rs b/queue-rs/src/jobs/mod.rs index dc6567ad03..cecda34f6a 100644 --- a/queue-rs/src/jobs/mod.rs +++ b/queue-rs/src/jobs/mod.rs @@ -8,6 +8,7 @@ pub mod child; pub mod claim; pub mod config; +pub mod graph; pub mod journal; pub mod lease; pub mod logs; diff --git a/queue-rs/src/jobs/rpc.rs b/queue-rs/src/jobs/rpc.rs index 54fba284bf..df2c3de82d 100644 --- a/queue-rs/src/jobs/rpc.rs +++ b/queue-rs/src/jobs/rpc.rs @@ -23,7 +23,7 @@ const TABLES: [&str; 5] = [ /// The installed contracts the daemon knows: `polis-queue/2` (000023) and /// `polis-queue/3` (000024, the large class). Which one a worker may start /// on is the class's say (`WorkerClass::contracts`). -pub const CONTRACTS: [&str; 2] = ["polis-queue/2", "polis-queue/3"]; +pub const CONTRACTS: [&str; 3] = ["polis-queue/2", "polis-queue/3", "polis-queue/5"]; pub struct Rpc { connector: Arc, diff --git a/queue-rs/src/jobs/task.rs b/queue-rs/src/jobs/task.rs index d34ea63cae..de48b8d87a 100644 --- a/queue-rs/src/jobs/task.rs +++ b/queue-rs/src/jobs/task.rs @@ -338,10 +338,16 @@ pub fn run(ctx: Arc, claim: Claim, reply: Value) { let cfg = ctx.cfg.clone(); let mut rpc = Rpc::new(ctx.connector.clone(), &claim.env); let started = Instant::now(); - let admission = match child::decode_admission( - reply["input"]["uri"].as_str().unwrap_or_default(), - reply["input"]["sha256"].as_str().unwrap_or_default(), - ) { + let graph = claim.stage.starts_with("graph_"); + let admission_result = if graph { + super::graph::admission(&reply) + } else { + child::decode_admission( + reply["input"]["uri"].as_str().unwrap_or_default(), + reply["input"]["sha256"].as_str().unwrap_or_default(), + ) + }; + let admission = match admission_result { Ok(a) => a, Err(_) => return refuse_without_child(&ctx, &mut rpc, &claim, true, "frame_invalid"), }; @@ -612,7 +618,12 @@ pub fn run(ctx: Arc, claim: Claim, reply: Value) { }, }; let manifest_bytes = std::fs::read(&manifest_path).ok(); - let manifest: Result = manifest::validate( + let validate = if graph { + super::graph::validate + } else { + manifest::validate + }; + let manifest: Result = validate( manifest_bytes.as_deref(), &claim.job_id, &claim.attempt_id, @@ -735,7 +746,11 @@ pub fn run(ctx: Arc, claim: Claim, reply: Value) { (true, Ok(_)) => { let fin = terminal( &mut rpc, - "pq_finalize", + if graph { + "pd_graph_finalize" + } else { + "pq_finalize" + }, &with(id.clone(), &[json!(manifest_uri), json!(m.sha256)]), ); match fin { diff --git a/queue-rs/src/lib.rs b/queue-rs/src/lib.rs index 81752f60fd..e9df6b8ced 100644 --- a/queue-rs/src/lib.rs +++ b/queue-rs/src/lib.rs @@ -65,6 +65,11 @@ fn signature(name: &str) -> Result<&'static [&'static str]> { /// and the four-argument reaper can never fall back to a `/1` form. pub fn signature_v2(name: &str) -> Result<&'static [&'static str]> { Ok(match name { + "pd_graph_claim" => &[ + "text", "smallint", "uuid", "uuid", "integer", "text", "text", + ], + "pd_graph_finalize" => &["text", "uuid", "uuid", "uuid", "bigint", "text", "text"], + "pd_graph_reconcile" => &["text"], "pq_claim" => &["text", "smallint", "uuid", "uuid", "integer", "text"], "pq_class_depth" => &["text", "text"], "pq_heartbeat" => &["text", "uuid", "uuid", "uuid", "bigint", "integer"], diff --git a/queue-rs/tests/jobs_integration.rs b/queue-rs/tests/jobs_integration.rs index 2597cb0d34..d992268a5d 100644 --- a/queue-rs/tests/jobs_integration.rs +++ b/queue-rs/tests/jobs_integration.rs @@ -94,6 +94,9 @@ fn ensure_templates() { .filter(|p| { p.extension().is_some_and(|e| e == "sql") && p.file_name().unwrap().to_string_lossy().as_bytes()[0].is_ascii_digit() + && p.file_name().unwrap().to_string_lossy()[..6] + .parse::() + .is_ok_and(|n| n <= 24) }) .collect(); chain.sort(); diff --git a/scripts/after_install.sh b/scripts/after_install.sh index 13db753c0f..ced1bfe0b5 100644 --- a/scripts/after_install.sh +++ b/scripts/after_install.sh @@ -87,6 +87,12 @@ case "$SERVICE_FROM_FILE" in *) echo "Error: Unknown service type: [$SERVICE_FROM_FILE]"; exit 1 ;; esac +# Apply the release schema before stopping/replacing any healthy service. +# All roles use the same database lock; a second box waits, then applies nothing. +# Build the client with this checkout. No manual image staging or local Rust needed. +sudo docker build -t polis-migrate:deploy -f queue-rs/polis-migrate/Dockerfile . +sudo docker run --rm --network host --env-file .env polis-migrate:deploy deploy + # Validate configuration before replacing any running container. Compose replaces # only this Compose project below; never remove unrelated containers or prune here. sudo /usr/local/bin/docker-compose config --quiet diff --git a/server/Dockerfile b/server/Dockerfile index 0ef7bd96d4..b23a8774c1 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -18,7 +18,14 @@ ARG NODE_ENV WORKDIR /app # Install Python and necessary tools (including pip) -RUN apk add --no-cache python3 py3-pip +RUN apk add --no-cache python3 py3-pip ca-certificates + +# Verified startup migration checks use the same public RDS CA pin as the +# runner image. Deployments with another private CA can override this env. +RUN mkdir -p /etc/polis \ + && wget -O /etc/polis/rds-ca.pem https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem \ + && echo 'fe45bbebf92ad3e27a583bbb2ddd1553c521ed4d49af5514dc0a40372ea5395c /etc/polis/rds-ca.pem' | sha256sum -c - +ENV NODE_EXTRA_CA_CERTS=/etc/polis/rds-ca.pem # This package is needed at runtime RUN apk add libpq-dev diff --git a/server/Dockerfile-db b/server/Dockerfile-db index 3ba7e8a75c..3d1e57bda5 100644 --- a/server/Dockerfile-db +++ b/server/Dockerfile-db @@ -1,11 +1,12 @@ -# Example Usage: -# docker build -t polis-db:local -f Dockerfile-db . -# docker network create polis-net -# docker run --rm --name polis-db --network polis-net -p 5432:5432 \ -# -e POSTGRES_DB=polis-dev -e POSTGRES_PASSWORD=postgres polis-db:local - +# Build with --build-context queue-rs=queue-rs (Compose supplies it). +# New volumes use the same runner/history as every later deploy. +FROM docker.io/library/rust:1.98.1-alpine AS migrate-build +RUN apk add --no-cache musl-dev openssl-dev openssl-libs-static pkgconfig +WORKDIR /src +COPY --from=queue-rs . . +RUN cargo build --release --locked -p polis-migrate FROM postgres:17-alpine -# Used when no existing database on postgres volume, including first initialization. -# See: docs/deployment.md#database-migrations -# See: https://github.com/docker-library/docs/blob/master/postgres/README.md#initialization-scripts -COPY ./postgres/migrations/*.sql /docker-entrypoint-initdb.d/ +RUN apk add --no-cache libssl3 ca-certificates +COPY --from=migrate-build /src/target/release/polis-migrate /usr/local/bin/polis-migrate +COPY ./postgres/migrations /migrations +COPY ./postgres/init-migrations.sh /docker-entrypoint-initdb.d/00-migrations.sh diff --git a/server/README.md b/server/README.md index f800d84f87..78445edd8c 100644 --- a/server/README.md +++ b/server/README.md @@ -52,36 +52,19 @@ root folder of the polis project. To run everything but the API server in this f `docker compose -f docker-compose.yml -f docker-compose.dev.yml --profile postgres up math postgres file-server ses-local`. In this case the polis-dev database should be accessible at the default DATABASE_URL seen in server/example.env. -3\. Connect to the new database then run the migrations in its shell. You can skip this step if you built the -database with docker compose. - -```psql -\connect polis -\i postgres/migrations/000000_initial.sql -\i postgres/migrations/000001_update_pwreset_table.sql -\i postgres/migrations/000002_add_xid_constraint.sql -\i postgres/migrations/000003_add_origin_permanent_cookie_columns.sql -\i postgres/migrations/000004_drop_waitinglist_table.sql -\i postgres/migrations/000005_drop_slack_stripe_canvas.sql -\i postgres/migrations/000006_update_votes_rule.sql -\i postgres/migrations/000007_drop_geolocation_fields.sql -\i postgres/migrations/000008_add_comment_priority.sql -\i postgres/migrations/000009_add_uuid_to_zinvites.sql -``` - -You can also separately run `psql -d polis -f postgres/migrations/000000_initial.sql` and -`psql -d polis -f postgres/migrations/000001_update_pwreset_table.sql` etc. from the shell. - -Alternatively, you can use the provided migration script to run all migrations in sequence: +3\. Set `DATABASE_URL` in the environment and run the migration runner from the +repository root. A fresh Compose database does this during initialization. ```sh -# Using DATABASE_URL from environment -./bin/run-migrations.sh - -# Or providing the URL as argument -./bin/run-migrations.sh "postgres://username:password@localhost:5432/polis" +cargo build --locked --release --manifest-path queue-rs/Cargo.toml -p polis-migrate +queue-rs/target/release/polis-migrate apply +queue-rs/target/release/polis-migrate check ``` +Existing databases need catalog-checked reconciliation once. See +[database migrations](../docs/migrations.md) for the commands and release holds. +Do not replay the initial SQL or pass credentials as command-line arguments. + 4\. Update database connection settings in `.env`. Replace the username, password, and database_name in the DATABASE_URL ```sh diff --git a/server/__tests__/integration/conversation-stats.test.ts b/server/__tests__/integration/conversation-stats.test.ts index b6253b431a..52ee7399ad 100644 --- a/server/__tests__/integration/conversation-stats.test.ts +++ b/server/__tests__/integration/conversation-stats.test.ts @@ -9,6 +9,7 @@ import { submitVote, } from "../setup/api-test-helpers"; import { getPooledTestUser } from "../setup/test-user-helpers"; +import pg from "../../src/db/pg-query"; interface ConversationStats { voteTimes: number[]; @@ -85,8 +86,14 @@ describe("Conversation Stats API", () => { }); test("GET /api/v3/conversationStats - should accept until parameter", async () => { - // Get current time in milliseconds - const currentTimeMs = Date.now(); + // Use the same clock as the stored timestamps. A Docker VM can be ahead + // of the host, so Date.now() immediately after the vote may exclude it. + // The route uses a strict < cutoff; advance past the current DB millisecond. + const rows = (await pg.queryP_readOnly( + "SELECT now_as_millis() AS now", + [] + )) as Array<{ now: string }>; + const currentTimeMs = Number(rows[0].now) + 1; const response: Response = await agent.get( `/api/v3/conversationStats?conversation_id=${conversationId}&until=${currentTimeMs}` diff --git a/server/__tests__/integration/delphi-job-table.test.ts b/server/__tests__/integration/delphi-job-table.test.ts index 327ea6cd33..8350433fef 100644 --- a/server/__tests__/integration/delphi-job-table.test.ts +++ b/server/__tests__/integration/delphi-job-table.test.ts @@ -1,6 +1,8 @@ /** * The Delphi job table: migration 000023, contract polis-queue/2, and the * large worker class on top of it: migration 000024, contract polis-queue/3. + * Migration 000027 adds graph stages under the install contract polis-queue/5; + * existing RPC wire versions and worker classes remain unchanged. * * The migrations reach the test database the way every migration does: the * postgres image applies server/postgres/migrations/*.sql once at initdb. So @@ -66,13 +68,13 @@ async function one(text: string, values?: unknown[]): Promise { return Object.values(result.rows[0] ?? {})[0] as T; } -describe("the Delphi job table (000023) under the large class (000024)", () => { - it("records contract polis-queue/3 in the queue's install row", async () => { +describe("the Delphi job table (000023) with large class (000024) and graph stages (000027)", () => { + it("records contract polis-queue/5 in the queue's install row", async () => { expect( await one( "SELECT contract_version FROM public.polis_queue_install", ), - ).toBe("polis-queue/3"); + ).toBe("polis-queue/5"); }); it.each(JOB_TABLES)( @@ -101,14 +103,14 @@ describe("the Delphi job table (000023) under the large class (000024)", () => { }, ); - it("admits exactly the daemon's two stages and the rebuild beside noop, and holds no job of any", async () => { + it("admits exactly noop, both daemon stages, rebuild and the three graph stages, and holds no job of any", async () => { expect( await one( "SELECT pg_get_constraintdef(oid) FROM pg_constraint " + "WHERE conrelid = 'public.polis_queue_jobs'::regclass AND conname = 'polis_queue_jobs_stage_check'", ), ).toBe( - "CHECK ((stage = ANY (ARRAY['noop'::text, 'delphi_full_pipeline'::text, 'delphi_narrative'::text, 'math_rebuild'::text])))", + "CHECK ((stage = ANY (ARRAY['noop'::text, 'delphi_full_pipeline'::text, 'delphi_narrative'::text, 'math_rebuild'::text, 'graph_embed'::text, 'graph_cluster'::text, 'graph_narrative'::text])))", ); expect( await one( @@ -117,7 +119,7 @@ describe("the Delphi job table (000023) under the large class (000024)", () => { ).toBe(0); }); - it("admits exactly the classes noop, delphi and large, and binds the rebuild stage to the large class", async () => { + it("admits exactly noop, delphi and large, with large required for rebuild and optional for graph clustering", async () => { expect( await one( "SELECT pg_get_constraintdef(oid) FROM pg_constraint " + @@ -132,7 +134,7 @@ describe("the Delphi job table (000023) under the large class (000024)", () => { "WHERE conrelid = 'public.polis_queue_jobs'::regclass AND conname = 'pq_stage_large'", ), ).toBe( - "CHECK (((stage = 'math_rebuild'::text) = (worker_class = 'large'::text)))", + "CHECK ((((stage = 'math_rebuild'::text) AND (worker_class = 'large'::text)) OR ((stage = 'graph_cluster'::text) AND (worker_class = ANY (ARRAY['delphi'::text, 'large'::text]))) OR ((stage <> ALL (ARRAY['math_rebuild'::text, 'graph_cluster'::text])) AND (worker_class <> 'large'::text))))", ); }); diff --git a/server/__tests__/integration/queue-substrate.test.ts b/server/__tests__/integration/queue-substrate.test.ts index 4960a56ac2..02eda17d44 100644 --- a/server/__tests__/integration/queue-substrate.test.ts +++ b/server/__tests__/integration/queue-substrate.test.ts @@ -827,6 +827,11 @@ describeProvisioned( it("replays for a superuser against the real conversations schema, or refuses once 000023 is in place", async () => { requireProvisioning(); if (foundationInstalled) { + // A newer additive contract must remain untouched by an old replay. + const installedBeforeReplay = await sql( + mainPool, + "SELECT contract_version FROM public.polis_queue_install" + ); await expect(runMigration(mainPool)).rejects.toThrow( /queue catalog drift/ ); @@ -835,7 +840,7 @@ describeProvisioned( mainPool, "SELECT contract_version FROM public.polis_queue_install" ) - ).toBe(largeClassInstalled ? "polis-queue/3" : "polis-queue/2"); + ).toBe(installedBeforeReplay); return; } await runMigration(mainPool); diff --git a/server/__tests__/unit/binBoundaries.test.ts b/server/__tests__/unit/binBoundaries.test.ts index 84fda1ae60..051908c71e 100644 --- a/server/__tests__/unit/binBoundaries.test.ts +++ b/server/__tests__/unit/binBoundaries.test.ts @@ -13,9 +13,10 @@ beforeEach(() => { afterEach(() => fs.rmSync(scratch, { recursive: true, force: true })); const publicEnvUrl = "postgresql://public:public-fixture@public.invalid/public"; -const publicArgUrl = "postgresql://public:other-fixture@other.invalid/public"; - -function migrationFixture(files: Record = {}) { +// The shell entrypoint delegates policy and SQL execution to polis-migrate. +// Its boundary keeps the DSN in the environment and preserves runner failures; +// real SQL/history/startup refusal is covered in polis-migrate/tests/prove.py. +function migrationFixture() { const script = path.join(scratch, "server/bin/run-migrations.sh"); const migrations = path.join(scratch, "server/postgres/migrations"); const fakeBin = path.join(scratch, "fake-bin"); @@ -23,129 +24,84 @@ function migrationFixture(files: Record = {}) { fs.mkdirSync(path.dirname(script), { recursive: true }); fs.mkdirSync(migrations, { recursive: true }); fs.mkdirSync(fakeBin); - // Byte-identical script relocated so it can see only public SQL fixtures. fs.copyFileSync(path.join(serverRoot, "bin/run-migrations.sh"), script); - for (const [name, contents] of Object.entries(files)) { - const file = path.join(migrations, name); - fs.mkdirSync(path.dirname(file), { recursive: true }); - fs.writeFileSync(file, contents); - } - const fakePsql = path.join(fakeBin, "psql"); - fs.writeFileSync( - fakePsql, - `#!${process.execPath} + const fakeRunner = path.join(fakeBin, "polis-migrate"); + fs.writeFileSync(fakeRunner, `#!${process.execPath} const fs = require('fs'); -const path = require('path'); -const [dsn, flag, file, extra] = process.argv.slice(2); -const allowed = ${JSON.stringify([publicEnvUrl, publicArgUrl])}; -if (!allowed.includes(dsn) || flag !== '-f' || extra || !file || - path.dirname(file) !== process.env.PUBLIC_MIGRATIONS) { - throw new Error('not an admitted public invocation'); -} -const contents = fs.readFileSync(file, 'utf8'); -if (!contents.startsWith('-- public fixture')) throw new Error('not public fixture SQL'); fs.appendFileSync(process.env.PUBLIC_TRACE, JSON.stringify({ - dsn, file: path.basename(file), password: process.env.PGPASSWORD, contents + args: process.argv.slice(2), dsn: process.env.DATABASE_URL }) + '\\n'); -process.exit(path.basename(file) === process.env.PUBLIC_FAIL ? 17 : 0); -` - ); - fs.chmodSync(fakePsql, 0o700); +process.exit(Number(process.env.PUBLIC_EXIT)); +`); + fs.chmodSync(fakeRunner, 0o700); return { - run(args: string[] = [], envUrl?: string, fail = "") { - const result = spawnSync("/bin/bash", [script, ...args], { + migrations, + fakeRunner, + run(options: { args?: string[]; code?: number; binary?: string; envUrl?: string } = {}) { + const result = spawnSync("/bin/bash", [script, ...(options.args || [])], { cwd: scratch, encoding: "utf8", timeout: 5000, env: { PATH: `${fakeBin}:/usr/bin:/bin`, - PUBLIC_MIGRATIONS: migrations, PUBLIC_TRACE: trace, - PUBLIC_FAIL: fail, - ...(envUrl === undefined ? {} : { DATABASE_URL: envUrl }), + PUBLIC_EXIT: String(options.code || 0), + DATABASE_URL: options.envUrl ?? publicEnvUrl, + ...(options.binary ? { POLIS_MIGRATE_BIN: options.binary } : {}), }, }); if (result.error) throw result.error; const calls = fs.existsSync(trace) - ? fs - .readFileSync(trace, "utf8") - .trim() - .split("\n") - .filter(Boolean) - .map((line) => JSON.parse(line)) + ? fs.readFileSync(trace, "utf8").trim().split("\n").map((line) => JSON.parse(line)) : []; return { ...result, calls }; }, }; } -const publicSql = "-- public fixture\nSELECT 'public value';\n"; - -test("migration runner requires a URL before invoking any psql process", () => { - const result = migrationFixture({ "001.sql": publicSql }).run(); - expect(result.status).toBe(1); - expect(result.stdout).toContain("DATABASE_URL is not set"); - expect(result.stdout).not.toContain("All migrations completed successfully"); - expect(result.calls).toEqual([]); +test("migration wrapper delegates once with the checkout's absolute source directory", () => { + const fixture = migrationFixture(); + const result = fixture.run(); + expect(result.status).toBe(0); + expect(result.calls).toEqual([{ args: ["apply", "--dir", fixture.migrations], dsn: publicEnvUrl }]); }); -test("migration runner sorts top-level SQL and ignores nested SQL and other extensions", () => { - const result = migrationFixture({ - "010-last.sql": publicSql, - "002-first.sql": publicSql, - "nested/001-ignored.sql": publicSql, - "003-ignored.txt": "not SQL", - }).run([], publicEnvUrl); +test("migration wrapper keeps the DSN out of argv and output", () => { + const result = migrationFixture().run(); expect(result.status).toBe(0); - expect(result.calls.map((call) => call.file)).toEqual([ - "002-first.sql", - "010-last.sql", - ]); - for (const call of result.calls) { - expect(call.dsn).toBe(publicEnvUrl); - expect(call.password).toBe("public-fixture"); - expect(call.contents).toBe(publicSql); - } - expect(result.stdout).toContain("All migrations completed successfully!"); + expect(result.calls).toHaveLength(1); + expect(result.calls[0].args.join(" ")).not.toContain(publicEnvUrl); + expect(result.stdout + result.stderr).not.toContain(publicEnvUrl); }); -test("explicit migration URL overrides the environment and binds matching password", () => { - const result = migrationFixture({ "001.sql": publicSql }).run( - [publicArgUrl], - publicEnvUrl - ); +test("migration wrapper supports an explicitly installed binary path containing spaces", () => { + const fixture = migrationFixture(); + const binary = path.join(scratch, "installed runner"); + fs.copyFileSync(fixture.fakeRunner, binary); + const result = fixture.run({ binary }); expect(result.status).toBe(0); expect(result.calls).toHaveLength(1); - expect(result.calls[0].dsn).toBe(publicArgUrl); - expect(result.calls[0].password).toBe("other-fixture"); }); -test("empty explicit URL preserves the environment URL", () => { - const result = migrationFixture({ "001.sql": publicSql }).run( - [""], - publicEnvUrl - ); - expect(result.status).toBe(0); - expect(result.calls[0].dsn).toBe(publicEnvUrl); +test("migration wrapper preserves argument boundaries for runner validation", () => { + const fixture = migrationFixture(); + const result = fixture.run({ args: ["--unknown-option", "value with spaces"], code: 2 }); + expect(result.status).toBe(2); + expect(result.calls[0].args).toEqual(["apply", "--dir", fixture.migrations, "--unknown-option", "value with spaces"]); }); -test("psql failure stops later migrations and cannot print completion", () => { - const result = migrationFixture({ - "001.sql": publicSql, - "002.sql": publicSql, - "003.sql": publicSql, - }).run([], publicEnvUrl, "002.sql"); +test("migration wrapper propagates runner failure without a success message or retry", () => { + const result = migrationFixture().run({ code: 17 }); expect(result.status).toBe(17); - expect(result.calls.map((call) => call.file)).toEqual(["001.sql", "002.sql"]); - expect(result.stdout).not.toContain("Applying migration: 003.sql"); - expect(result.stdout).not.toContain("All migrations completed successfully"); + expect(result.calls).toHaveLength(1); + expect(result.stdout).toBe(""); }); -test("empty migration directory completes without inventing a database call", () => { - const result = migrationFixture().run([], publicEnvUrl); - expect(result.status).toBe(0); +test("missing runner refuses without falling back to untracked SQL", () => { + const result = migrationFixture().run({ binary: path.join(scratch, "absent") }); + expect(result.status).not.toBe(0); // Bash 3/macOS: 1; Bash 5/Linux: 127. expect(result.calls).toEqual([]); - expect(result.stdout).toContain("All migrations completed successfully!"); + expect(result.stdout).toBe(""); }); function resetBoundary(databaseUrl: unknown) { diff --git a/server/bin/build-migration-report.py b/server/bin/build-migration-report.py new file mode 100644 index 0000000000..1acf91ce4e --- /dev/null +++ b/server/bin/build-migration-report.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +"""Build the first-deploy catalog report by inlining the adoption SQL helpers. + +No database connection. --check compares bytes without changing any files. +The generated script executes in a read-only transaction, with no temp objects. +""" +import argparse +import hashlib +from pathlib import Path +import re + +MIG = Path(__file__).resolve().parents[1] / "postgres/migrations" + + +def tokens(sql): + # All helper/adoption sources use ordinary quoted strings, comments and $$. + return re.findall(r"--[^\n]*|/\*[\s\S]*?\*/|'(?:''|[^'])*'|[A-Za-z_][A-Za-z_0-9]*|\s+|.", sql) + + +def substitute(sql, arguments): + return ''.join('(' + arguments[t] + ')' if t in arguments else t for t in tokens(sql)) + + +def helpers(source): + result = {} + pattern = r'CREATE FUNCTION pg_temp\.(\w+)\((.*?)\)\s*RETURNS boolean LANGUAGE sql AS \$\$(.*?)\$\$;' + for name, signature, body in re.findall(pattern, source, re.S): + params = [] + for part in signature.split(','): + bits = part.strip().split() + params.append((bits[0], bits[3] if len(bits) == 4 and bits[2] == 'DEFAULT' else None)) + result[name] = (params, body.strip().removesuffix(';')) + assert len(result) == 6, 'review changed helper source before regenerating' + return result + + +def inline(query, definitions): + ts = tokens(query) + out = [] + i = 0 + while i < len(ts): + if ts[i] != 'pg_temp': + out.append(ts[i]); i += 1; continue + assert ts[i+1] == '.' and ts[i+2] in definitions and ts[i+3] == '(' + name = ts[i+2] + i += 4 + depth = 1 + args = [''] + while depth: + token = ts[i]; i += 1 + if token == '(': + depth += 1 + elif token == ')': + depth -= 1 + if not depth: + break + if token == ',' and depth == 1: + args.append('') + else: + args[-1] += token + params, body = definitions[name] + assert len(args) <= len(params) + values = {} + for index, (param, default) in enumerate(params): + value = args[index].strip() if index < len(args) else default + assert value is not None + values[param] = value + out.append('(' + substitute(body, values) + ')') + return ''.join(out) + + +def build(): + source = (MIG/'adoption/helpers.sql').read_text() + definitions = helpers(source) + files = sorted((MIG/'adoption').glob('0*.sql')) + def manifest(path): + lines = [s for s in path.read_text().splitlines() if s and not s.startswith('#')] + assert len(lines) == len(set(lines)), 'duplicate manifest entry' + assert all(re.fullmatch(r'\d{6}_[A-Za-z0-9_]+\.sql', s) for s in lines) + return set(lines) + selected = manifest(MIG/'release.txt') + held = manifest(MIG/'held.txt') + pending = {'000019_create_polis_queue.sql', '000023_create_delphi_foundation.sql', + '000024_create_polis_queue_large_class.sql', '000027_create_sealed_job_graphs.sql'} + assert selected == {p.name for p in files} | pending, 'report scope differs from release selection' + assert held == {'000021_create_polis_coordinator.sql'}, 'review changed hold policy' + numbered = sorted(MIG.glob('*.sql')) + assert all(re.fullmatch(r'\d{6}_[A-Za-z0-9_]+\.sql', p.name) for p in numbered), 'invalid numbered source name' + assert len({p.name[:6] for p in numbered}) == len(numbered), 'duplicate migration number' + assert {p.name for p in numbered} == selected | held, 'unclassified/missing numbered SQL' + assert all(p.is_file() and not p.is_symlink() for p in numbered), 'nonregular numbered source' + bindings = [MIG/'adoption/helpers.sql', MIG/'release.txt', MIG/'held.txt'] + files + numbered + header = ['-- GENERATED by server/bin/build-migration-report.py; do not edit.', + '-- First deployment only: no existing ledger or queue. Other states refuse.', + '-- This is a catalog forecast, not DDL success or deployment-health proof.'] + for path in bindings: + header.append('-- sha256 ' + hashlib.sha256(path.read_bytes()).hexdigest() + ' ' + str(path.relative_to(MIG))) + # Each original predicate remains a scalar subquery. SQL helpers are inlined, + # never installed, and role/catalog identity fields never leave the database. + rows = [] + for path in files: + rows.append("SELECT '"+path.name+"'::text AS migration, ("+inline(path.read_text().strip().removesuffix(';'), definitions)+") AS catalog_match") + checks = '\nUNION ALL\n'.join(rows) + return '\n'.join(header)+'''\nBEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY; +SET LOCAL search_path=pg_catalog,public; +SET LOCAL statement_timeout='30s'; +SET LOCAL lock_timeout='2s'; +WITH checks AS ( +'''+checks+''' +), scope AS ( + SELECT to_regclass('public.migrations') IS NULL + AND to_regclass('public.schema_migrations') IS NULL + AND NOT EXISTS (SELECT 1 FROM pg_class WHERE relnamespace='public'::regnamespace + AND (starts_with(relname,'polis_queue_') OR starts_with(relname,'delphi_'))) + AND NOT EXISTS (SELECT 1 FROM pg_proc WHERE pronamespace='public'::regnamespace + AND (starts_with(proname,'pq_') OR starts_with(proname,'pd_'))) AS initial_state +), authority AS ( + SELECT + has_database_privilege(current_user,current_database(),'TEMP') + AND has_schema_privilege(current_user,'public','USAGE') + AND has_schema_privilege(current_user,'public','CREATE') AS reconcile_authority, + EXISTS (SELECT 1 FROM pg_roles WHERE rolname=current_user AND (rolsuper OR rolcreaterole)) AS create_roles, + NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname IN ('polis_queue_owner','polis_queue_executor')) AS queue_roles_absent, + has_schema_privilege(current_user,'public','USAGE WITH GRANT OPTION') + AND has_schema_privilege(current_user,'public','CREATE WITH GRANT OPTION') AS schema_grants, + EXISTS (SELECT 1 FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace + WHERE n.nspname='public' AND c.relname='conversations' + AND has_table_privilege(current_user,c.oid,'SELECT WITH GRANT OPTION') + AND has_column_privilege(current_user,c.oid,'topic','UPDATE WITH GRANT OPTION') + AND has_column_privilege(current_user,c.oid,'zid','REFERENCES WITH GRANT OPTION')) AS table_grants +), readiness AS ( + SELECT initial_state AND (SELECT bool_and(catalog_match) FROM checks) + AND current_setting('server_version_num')::integer >= 170000 AS can_reconcile, + create_roles AND queue_roles_absent AND schema_grants AND table_grants AS can_provision, + initial_state, reconcile_authority, create_roles, queue_roles_absent, schema_grants, table_grants + FROM scope CROSS JOIN authority +), output AS ( + SELECT migration, catalog_match, + CASE WHEN NOT initial_state THEN 'REVIEW_EXISTING_LEDGER_OR_QUEUE' + WHEN NOT catalog_match THEN 'CATALOG_MISMATCH' + WHEN NOT can_reconcile THEN 'BLOCKED_BY_OTHER_CATALOG_CHECK' + WHEN NOT reconcile_authority THEN 'REVIEW_MIGRATION_SESSION_AUTHORITY' + ELSE 'WOULD_ADOPT' END AS outcome + FROM checks CROSS JOIN readiness + UNION ALL + SELECT migration, NULL::boolean, + CASE WHEN NOT can_reconcile THEN 'BLOCKED_RECONCILIATION' + WHEN NOT reconcile_authority OR NOT can_provision THEN 'REVIEW_MIGRATION_SESSION_AUTHORITY' + ELSE 'WOULD_APPLY' END + FROM (VALUES ('000019_create_polis_queue.sql'),('000023_create_delphi_foundation.sql'), + ('000024_create_polis_queue_large_class.sql'), + ('000027_create_sealed_job_graphs.sql')) p(migration) CROSS JOIN readiness + UNION ALL + SELECT migration, NULL::boolean, 'OUTSIDE_RELEASE' + FROM (VALUES ('000020'),('000021'),('000025'),('000026')) p(migration) +) +SELECT migration, catalog_match, outcome FROM output ORDER BY migration; +ROLLBACK; +''' + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--check', action='store_true') + args = parser.parse_args() + output = MIG/'report/first-deploy.sql' + text = build() + if args.check: + assert output.read_text() == text, 'report is stale; regenerate and review' + print('PASS: report matches all source predicates and release metadata') + else: + output.parent.mkdir(exist_ok=True) + output.write_text(text) + print(output) + + +if __name__ == '__main__': + main() diff --git a/server/bin/run-migrations.sh b/server/bin/run-migrations.sh index 911ebe6680..34de794ca6 100755 --- a/server/bin/run-migrations.sh +++ b/server/bin/run-migrations.sh @@ -1,28 +1,5 @@ -#!/bin/bash -set -e - -# Use DATABASE_URL from environment or from argument if provided -if [ -n "$1" ]; then - DATABASE_URL="$1" -fi - -# Check if DATABASE_URL is set -if [ -z "$DATABASE_URL" ]; then - echo "Error: DATABASE_URL is not set. Please provide it as an environment variable or argument." - echo "Usage: $0 [DATABASE_URL]" - exit 1 -fi - -# Directory containing migration files -MIGRATIONS_DIR="$(dirname "$(dirname "$0")")/postgres/migrations" - -echo "Running migrations from $MIGRATIONS_DIR" - -# Get all migration files sorted by name (only from top-level directory) -for migration in $(find "$MIGRATIONS_DIR" -maxdepth 1 -name "*.sql" | sort); do - echo "Applying migration: $(basename "$migration")" - PGPASSWORD=$(echo "$DATABASE_URL" | sed -E 's/.*:([^:]+)@.*/\1/') \ - psql "$DATABASE_URL" -f "$migration" -done - -echo "All migrations completed successfully!" +#!/usr/bin/env bash +set -euo pipefail +# DATABASE_URL stays in the environment, never an argv or a traced shell command. +root=$(cd "$(dirname "$0")/../.." && pwd) +exec "${POLIS_MIGRATE_BIN:-polis-migrate}" apply --dir "$root/server/postgres/migrations" "$@" diff --git a/server/index.ts b/server/index.ts index dfba6599a7..35c5e36ee8 100644 --- a/server/index.ts +++ b/server/index.ts @@ -1,30 +1,25 @@ -/** - * Server entry point - * This file is responsible for starting the server after the app is configured - */ -import app from "./app"; -import Config from "./src/config"; -import { startNotificationLoop } from "./src/routes/notify"; -import logger from "./src/utils/logger"; +/** Check the release's migrations before importing the app or starting work. */ +import "dotenv/config"; +import { checkMigrations } from "./src/db/migrations.cjs"; -if (Config.nodeEnv === "production") { - // eslint-disable-next-line @typescript-eslint/no-unused-vars, @typescript-eslint/no-var-requires - const tracer = require("dd-trace").init(); -} - -/** - * Start the server on the configured port or a provided port - * @param {number} [port=Config.serverPort] - The port to listen on - * @returns {Object} The server instance - */ -function startServer(port = Config.serverPort) { - const server = app.listen(port); - logger.info(`Server started on port ${port}`); +async function startServer(port?: number) { + await checkMigrations(); + const { default: Config } = await import("./src/config"); + if (Config.nodeEnv === "production") { + // eslint-disable-next-line @typescript-eslint/no-var-requires + require("dd-trace").init(); + } + const { default: app } = await import("./app"); + const { startNotificationLoop } = await import("./src/routes/notify"); + const { default: logger } = await import("./src/utils/logger"); + startNotificationLoop(); + const server = app.listen(port ?? Config.serverPort); + logger.info(`Server started on port ${port ?? Config.serverPort}`); return server; } -startNotificationLoop(); -startServer(); - +startServer().catch((error) => { + process.stderr.write(`Server startup refused: ${error.message}\n`); + process.exit(1); +}); export { startServer }; -export default app; diff --git a/server/jest.job-graphs.config.ts b/server/jest.job-graphs.config.ts new file mode 100644 index 0000000000..bd568d0350 --- /dev/null +++ b/server/jest.job-graphs.config.ts @@ -0,0 +1,12 @@ +// Database-only compatibility checks in the job-graph CI entry. These tests own +// their database/role fixtures; they do not need the full HTTP/OIDC test stack. +import base from './jest.config'; +export default { + ...base, + testMatch: ['**/__tests__/integration/queue-substrate.test.ts'], + setupFilesAfterEnv: [], + globalSetup: undefined, + globalTeardown: undefined, + collectCoverage: false, + reporters: ['default'], +}; diff --git a/server/postgres/bin/apply-migration.sh b/server/postgres/bin/apply-migration.sh index e6ec532005..9c3c2e05f8 100755 --- a/server/postgres/bin/apply-migration.sh +++ b/server/postgres/bin/apply-migration.sh @@ -1,5 +1,8 @@ #!/usr/bin/env bash # +# HISTORICAL REHEARSAL HELPER ONLY. Deployments must use polis-migrate apply +# (docs/migrations.md). This script does not record applied migration history. +# # apply-migration.sh: the checked apply wrapper for the queue migrations # 000019 (polis-queue/1), 000023 (polis-queue/2, the Delphi job table) and # 000024 (polis-queue/3, the large worker class). diff --git a/server/postgres/init-migrations.sh b/server/postgres/init-migrations.sh new file mode 100644 index 0000000000..b70b0545ea --- /dev/null +++ b/server/postgres/init-migrations.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -eu +# Official postgres entrypoint sources executable/nonexecutable shell hooks only +# during fresh initialization; its temporary server listens on this local socket. +export DATABASE_URL="host=/var/run/postgresql user=$POSTGRES_USER dbname=$POSTGRES_DB sslmode=disable" +export POLIS_MIGRATIONS_DIR=/migrations +polis-migrate apply diff --git a/server/postgres/migrations/000027_create_sealed_job_graphs.sql b/server/postgres/migrations/000027_create_sealed_job_graphs.sql new file mode 100644 index 0000000000..1118176e42 --- /dev/null +++ b/server/postgres/migrations/000027_create_sealed_job_graphs.sql @@ -0,0 +1,528 @@ +-- Per-step runs, immutable results and inputs, dependency waiting, atomic publication. +-- Contract /5; /4 belongs to held retention. No provider remediation or scoped breakers. +-- Forward-only compatibility with the original M19/M23/M24 checksums. +BEGIN; +SET LOCAL lock_timeout='5s'; +SET LOCAL ROLE polis_queue_owner; +SET LOCAL search_path=pg_catalog,pg_temp; +CREATE OR REPLACE FUNCTION pg_temp.pq_catalog(p_table oid) RETURNS jsonb +LANGUAGE sql SET search_path=pg_catalog,pg_temp AS $catalog$ +SELECT jsonb_build_object( + 'columns',(SELECT jsonb_agg(jsonb_build_array(a.attname,format_type(a.atttypid,a.atttypmod),a.attnotnull,a.attidentity,a.attgenerated,co.collname,pg_get_expr(d.adbin,d.adrelid),NULLIF(a.attacl::text,'{}')) ORDER BY a.attnum) + FROM pg_attribute a LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum LEFT JOIN pg_collation co ON co.oid=a.attcollation + WHERE a.attrelid=c.oid AND a.attnum>0 AND NOT a.attisdropped), + 'constraints',(SELECT jsonb_agg(jsonb_build_array(conname,pg_get_constraintdef(oid),convalidated,connoinherit) ORDER BY conname) FROM pg_constraint WHERE conrelid=c.oid), + 'indexes',(SELECT jsonb_agg(jsonb_build_array(ic.relname,pg_get_indexdef(i.indexrelid),i.indisvalid,i.indisready) ORDER BY ic.relname) FROM pg_index i JOIN pg_class ic ON ic.oid=i.indexrelid WHERE i.indrelid=c.oid), + 'triggers',(SELECT jsonb_agg(jsonb_build_array(t.tgname,pg_get_triggerdef(t.oid),t.tgenabled) ORDER BY t.tgname) FROM pg_trigger t WHERE t.tgrelid=c.oid AND NOT t.tgisinternal), + 'owner',pg_get_userbyid(c.relowner),'kind',c.relkind,'rls',c.relrowsecurity,'force_rls',c.relforcerowsecurity,'options',c.reloptions, + 'acl',(SELECT jsonb_agg(jsonb_build_array(CASE WHEN x.grantee=0 THEN 'PUBLIC' ELSE pg_get_userbyid(x.grantee) END,x.privilege_type,x.is_grantable) ORDER BY x.grantee=0,pg_get_userbyid(x.grantee),x.privilege_type,x.is_grantable) FROM aclexplode(COALESCE(c.relacl,acldefault('r',c.relowner))) x) + ) FROM pg_class c WHERE c.oid=p_table +$catalog$; +CREATE OR REPLACE FUNCTION pg_temp.pd_state() RETURNS jsonb LANGUAGE sql SET search_path=pg_catalog,pg_temp AS $s$ +SELECT jsonb_build_object('tables',(SELECT jsonb_object_agg(c.relname,pg_temp.pq_catalog(c.oid)) FROM pg_class c + WHERE c.relnamespace='public'::regnamespace AND c.relkind='r' AND (starts_with(c.relname,'polis_queue_') OR starts_with(c.relname,'delphi_')) AND c.relname<>'delphi_foundation_install'), + 'functions',(SELECT jsonb_object_agg(p.oid::regprocedure::text,jsonb_build_array(pg_get_functiondef(p.oid),p.proacl::text,pg_get_userbyid(p.proowner))) + FROM pg_proc p WHERE p.pronamespace='public'::regnamespace AND (starts_with(p.proname,'pq_') OR starts_with(p.proname,'pd_')))) +$s$; +-- The /3 state: the same, less this file's own install table. +CREATE OR REPLACE FUNCTION pg_temp.pq3_state() RETURNS jsonb LANGUAGE sql SET search_path=pg_catalog,pg_temp AS $s$ +SELECT jsonb_build_object('tables',(SELECT jsonb_object_agg(c.relname,pg_temp.pq_catalog(c.oid)) FROM pg_class c + WHERE c.relnamespace='public'::regnamespace AND c.relkind='r' AND (starts_with(c.relname,'polis_queue_') OR starts_with(c.relname,'delphi_')) + AND c.relname NOT IN ('delphi_foundation_install','polis_queue_large_class_install')), + 'functions',(SELECT jsonb_object_agg(p.oid::regprocedure::text,jsonb_build_array(pg_get_functiondef(p.oid),p.proacl::text,pg_get_userbyid(p.proowner))) + FROM pg_proc p WHERE p.pronamespace='public'::regnamespace AND (starts_with(p.proname,'pq_') OR starts_with(p.proname,'pd_')))) +$s$; + +DO $$ BEGIN + IF NOT EXISTS(SELECT 1 FROM public.polis_queue_large_class_install WHERE installed=pg_temp.pq3_state()) + THEN RAISE EXCEPTION 'queue /3 catalog drift'; END IF; +END $$; +DO $$ BEGIN PERFORM set_config('graph.baseline',pg_temp.pq3_state()::text,true); END $$; +CREATE TABLE public.delphi_graph_install(singleton boolean PRIMARY KEY CHECK(singleton), baseline jsonb NOT NULL); +INSERT INTO public.delphi_graph_install VALUES(true,current_setting('graph.baseline')::jsonb); +ALTER TABLE public.polis_queue_install DROP CONSTRAINT polis_queue_install_contract_version_check; +UPDATE public.polis_queue_install SET contract_version='polis-queue/5'; +ALTER TABLE public.polis_queue_install ADD CHECK(contract_version='polis-queue/5'); +ALTER TABLE public.polis_queue_install ALTER COLUMN contract_version SET DEFAULT 'polis-queue/5'; +ALTER TABLE public.polis_queue_runs DROP CONSTRAINT polis_queue_runs_contract_version_check; +ALTER TABLE public.polis_queue_runs ADD CHECK(contract_version IN ('polis-queue/1','polis-queue/2','polis-queue/3','polis-queue/5')); +ALTER TABLE public.polis_queue_jobs DROP CONSTRAINT polis_queue_jobs_stage_check; +ALTER TABLE public.polis_queue_jobs ADD CHECK(stage IN ('noop','delphi_full_pipeline','delphi_narrative','math_rebuild','graph_embed','graph_cluster','graph_narrative')); +ALTER TABLE public.polis_queue_jobs DROP CONSTRAINT pq_stage_large; +ALTER TABLE public.polis_queue_jobs ADD CONSTRAINT pq_stage_large CHECK( + (stage='math_rebuild' AND worker_class='large') OR + (stage='graph_cluster' AND worker_class IN ('delphi','large')) OR + (stage NOT IN ('math_rebuild','graph_cluster') AND worker_class<>'large')); + +CREATE TABLE public.delphi_graphs( + env text NOT NULL, graph_id uuid NOT NULL, zid integer NOT NULL REFERENCES public.conversations(zid), + scope_key text NOT NULL CHECK(length(scope_key) BETWEEN 1 AND 128), request_key text NOT NULL, + request jsonb NOT NULL, root_job_id uuid NOT NULL, sealed boolean NOT NULL DEFAULT false, + supersedes uuid, created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY(env,graph_id), UNIQUE(env,scope_key,request_key), UNIQUE(env,zid,graph_id), + FOREIGN KEY(env,supersedes) REFERENCES public.delphi_graphs(env,graph_id) +); +CREATE TABLE public.delphi_graph_nodes( + env text NOT NULL, zid integer NOT NULL, graph_id uuid NOT NULL, job_id uuid NOT NULL, + run_id uuid NOT NULL, node_key text NOT NULL CHECK(node_key ~ '^[a-z][a-z0-9_]{0,31}$'), + declared jsonb NOT NULL, resolved jsonb, resolved_sha text, + PRIMARY KEY(env,job_id), UNIQUE(env,run_id), UNIQUE(env,job_id,run_id), UNIQUE(env,graph_id,node_key), + FOREIGN KEY(env,zid,graph_id) REFERENCES public.delphi_graphs(env,zid,graph_id), + FOREIGN KEY(env,job_id) REFERENCES public.delphi_jobs(env,job_id), + FOREIGN KEY(env,run_id) REFERENCES public.polis_queue_runs(env,run_id), + CHECK((resolved IS NULL)=(resolved_sha IS NULL)) +); +CREATE TABLE public.delphi_artifacts( + env text NOT NULL, artifact_id uuid NOT NULL DEFAULT gen_random_uuid(), job_id uuid NOT NULL, + run_id uuid NOT NULL, attempt_id uuid NOT NULL, output_role text NOT NULL CHECK(output_role='result'), + schema_version text NOT NULL, payload text NOT NULL CHECK(octet_length(payload)<=524288), + content_sha text NOT NULL CHECK(content_sha ~ '^[0-9a-f]{64}$'), + byte_count integer NOT NULL CHECK(byte_count>=0), created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY(env,artifact_id), UNIQUE(env,job_id,output_role), + FOREIGN KEY(env,job_id) REFERENCES public.delphi_graph_nodes(env,job_id), + FOREIGN KEY(env,job_id,run_id) REFERENCES public.delphi_graph_nodes(env,job_id,run_id), + FOREIGN KEY(env,job_id,attempt_id) REFERENCES public.polis_queue_attempts(env,job_id,attempt_id), + CHECK(content_sha=encode(sha256(convert_to(payload,'UTF8')),'hex')), + CHECK(byte_count=octet_length(payload)) +); +CREATE TABLE public.delphi_graph_edges( + env text NOT NULL, consumer uuid NOT NULL, producer uuid NOT NULL, input_role text NOT NULL, + expected_sha text CHECK(expected_sha ~ '^[0-9a-f]{64}$'), artifact_id uuid, + PRIMARY KEY(env,consumer,input_role), CHECK(consumer<>producer), + FOREIGN KEY(env,consumer) REFERENCES public.delphi_graph_nodes(env,job_id), + FOREIGN KEY(env,producer) REFERENCES public.delphi_graph_nodes(env,job_id), + FOREIGN KEY(env,artifact_id) REFERENCES public.delphi_artifacts(env,artifact_id) +); +CREATE INDEX delphi_graph_edges_producer ON public.delphi_graph_edges(env,producer); +CREATE TABLE public.delphi_graph_served( + env text NOT NULL, zid integer NOT NULL REFERENCES public.conversations(zid), scope_key text NOT NULL, + generation bigint NOT NULL CHECK(generation>0), artifact_id uuid NOT NULL, + PRIMARY KEY(env,zid,scope_key), FOREIGN KEY(env,artifact_id) REFERENCES public.delphi_artifacts(env,artifact_id) +); +CREATE FUNCTION public.pd_graph_hash(j jsonb) RETURNS text LANGUAGE sql IMMUTABLE + SET search_path=pg_catalog,pg_temp AS $$ SELECT encode(sha256(convert_to(j::text,'UTF8')),'hex') $$; +CREATE FUNCTION public.pd_graph_immutable() RETURNS trigger LANGUAGE plpgsql SET search_path=pg_catalog,pg_temp AS $$ +BEGIN RAISE EXCEPTION 'immutable graph result'; END $$; +CREATE TRIGGER graph_artifact_immutable BEFORE UPDATE OR DELETE ON public.delphi_artifacts FOR EACH ROW EXECUTE FUNCTION public.pd_graph_immutable(); +CREATE FUNCTION public.pd_graph_node_guard() RETURNS trigger LANGUAGE plpgsql SET search_path=pg_catalog,pg_temp AS $$ +BEGIN + IF TG_OP='DELETE' THEN RAISE EXCEPTION 'immutable graph node'; END IF; + IF TG_OP='UPDATE' AND (to_jsonb(NEW)-'resolved'-'resolved_sha' IS DISTINCT FROM to_jsonb(OLD)-'resolved'-'resolved_sha' + OR OLD.resolved IS NOT NULL) THEN RAISE EXCEPTION 'immutable graph input'; END IF; + IF TG_OP='INSERT' AND EXISTS(SELECT 1 FROM public.delphi_graphs WHERE env=NEW.env AND graph_id=NEW.graph_id AND sealed) + THEN RAISE EXCEPTION 'sealed graph'; END IF; + RETURN NEW; +END $$; +CREATE TRIGGER graph_node_guard BEFORE INSERT OR UPDATE OR DELETE ON public.delphi_graph_nodes FOR EACH ROW EXECUTE FUNCTION public.pd_graph_node_guard(); +CREATE FUNCTION public.pd_graph_edge_guard() RETURNS trigger LANGUAGE plpgsql SET search_path=pg_catalog,pg_temp AS $$ +#variable_conflict use_column +DECLARE c public.delphi_graph_nodes; p public.delphi_graph_nodes; +BEGIN + IF TG_OP='DELETE' THEN RAISE EXCEPTION 'immutable graph edge'; END IF; + SELECT * INTO STRICT c FROM public.delphi_graph_nodes WHERE env=NEW.env AND job_id=NEW.consumer; + SELECT * INTO STRICT p FROM public.delphi_graph_nodes WHERE env=NEW.env AND job_id=NEW.producer; + IF c.zid<>p.zid THEN RAISE EXCEPTION 'dependency namespace'; END IF; + IF TG_OP='UPDATE' THEN + IF to_jsonb(NEW)-'artifact_id' IS DISTINCT FROM to_jsonb(OLD)-'artifact_id' OR OLD.artifact_id IS NOT NULL + THEN RAISE EXCEPTION 'immutable dependency'; END IF; + ELSE + IF EXISTS(SELECT 1 FROM public.delphi_graphs WHERE env=c.env AND graph_id=c.graph_id AND sealed) + THEN RAISE EXCEPTION 'sealed graph'; END IF; + IF EXISTS(WITH RECURSIVE up(id) AS (SELECT NEW.producer UNION SELECT e.producer FROM public.delphi_graph_edges e JOIN up ON e.consumer=up.id WHERE e.env=NEW.env) SELECT 1 FROM up WHERE id=NEW.consumer) + THEN RAISE EXCEPTION 'dependency cycle'; END IF; + END IF; + IF NEW.artifact_id IS NOT NULL AND NOT EXISTS(SELECT 1 FROM public.delphi_artifacts a + WHERE a.env=NEW.env AND a.artifact_id=NEW.artifact_id AND a.job_id=NEW.producer + AND (NEW.expected_sha IS NULL OR a.content_sha=NEW.expected_sha)) THEN RAISE EXCEPTION 'artifact binding'; END IF; + RETURN NEW; +END $$; +CREATE TRIGGER graph_edge_guard BEFORE INSERT OR UPDATE OR DELETE ON public.delphi_graph_edges FOR EACH ROW EXECUTE FUNCTION public.pd_graph_edge_guard(); +CREATE FUNCTION public.pd_graph_parent_guard() RETURNS trigger LANGUAGE plpgsql SET search_path=pg_catalog,pg_temp AS $$ +BEGIN + IF TG_OP='UPDATE' AND (NEW.parent_job_id IS DISTINCT FROM OLD.parent_job_id OR NEW.run_id IS DISTINCT FROM OLD.run_id OR NEW.env<>OLD.env OR NEW.zid<>OLD.zid) + AND EXISTS(SELECT 1 FROM public.delphi_graph_nodes WHERE env=OLD.env AND job_id=OLD.job_id) + THEN RAISE EXCEPTION 'immutable graph membership'; END IF; + IF TG_OP='INSERT' AND NEW.parent_job_id IS NOT NULL AND EXISTS( + SELECT 1 FROM public.delphi_graph_nodes n JOIN public.delphi_graphs g USING(env,graph_id) WHERE n.env=NEW.env AND n.job_id=NEW.parent_job_id AND g.sealed) + THEN RAISE EXCEPTION 'late child in sealed graph'; END IF; + RETURN NEW; +END $$; +CREATE TRIGGER graph_parent_guard BEFORE INSERT OR UPDATE ON public.delphi_jobs FOR EACH ROW EXECUTE FUNCTION public.pd_graph_parent_guard(); + +CREATE OR REPLACE FUNCTION public.pd_queue_binding() RETURNS trigger +LANGUAGE plpgsql SECURITY DEFINER SET search_path=pg_catalog,pg_temp AS $$ +#variable_conflict use_column +DECLARE q public.polis_queue_jobs; j public.delphi_jobs; r public.polis_queue_runs; +BEGIN + SELECT * INTO q FROM public.polis_queue_jobs WHERE env=NEW.env AND job_id=NEW.job_id; + IF EXISTS(SELECT 1 FROM public.delphi_graph_nodes gn WHERE gn.env=q.env AND gn.run_id=q.run_id) AND (SELECT count(*) FROM public.polis_queue_jobs jq WHERE jq.env=q.env AND jq.run_id=q.run_id)<>1 THEN RAISE EXCEPTION 'one computational job per run'; END IF; + IF q.stage LIKE 'graph_%' THEN + IF NOT EXISTS(SELECT 1 FROM public.delphi_graph_nodes n JOIN public.polis_queue_runs r USING(env,run_id) JOIN public.delphi_jobs d ON d.env=n.env AND d.job_id=n.job_id WHERE n.env=q.env AND n.job_id=q.job_id AND n.run_id=q.run_id AND r.contract_version='polis-queue/5' AND d.run_id=q.run_id AND d.zid=r.zid AND d.kind=substring(q.stage from 7)) THEN RAISE EXCEPTION 'invalid graph execution binding'; END IF; + RETURN NULL; + END IF; + IF q.stage='noop' THEN RETURN NULL; END IF; + SELECT * INTO j FROM public.delphi_jobs WHERE env=q.env AND job_id=q.job_id; + SELECT * INTO r FROM public.polis_queue_runs WHERE env=q.env AND run_id=q.run_id; + IF j.job_id IS NULL OR j.zid<>r.zid OR j.origin<>'queued' OR j.run_id IS DISTINCT FROM q.run_id + OR r.contract_version<>(CASE WHEN q.stage='math_rebuild' THEN 'polis-queue/3' ELSE 'polis-queue/2' END) + OR (q.stage='delphi_full_pipeline' AND j.kind<>'full_pipeline') + OR (q.stage='delphi_narrative' AND j.kind<>'narrative') + OR (q.stage='math_rebuild' AND (j.kind<>'math_rebuild' OR q.worker_class<>'large' OR j.report_id IS NOT NULL)) + THEN RAISE EXCEPTION 'invalid logical execution binding'; END IF; + RETURN NULL; +END $$; +CREATE OR REPLACE FUNCTION public.pq_result(p_outcome text, j public.polis_queue_jobs, p_published boolean DEFAULT false) +RETURNS jsonb LANGUAGE sql VOLATILE SET search_path=pg_catalog,pg_temp SET TimeZone='UTC' AS $$ + SELECT jsonb_build_object('schema_version',CASE WHEN j.stage IS NULL OR j.stage='noop' THEN 'polis-queue/1' WHEN j.stage LIKE 'graph_%' THEN 'polis-queue/5' WHEN j.stage='math_rebuild' THEN 'polis-queue/3' ELSE 'polis-queue/2' END,'outcome',p_outcome, + 'env',j.env,'job_id',j.job_id,'run_id',j.run_id, + 'attempt_id',COALESCE(j.attempt_id,j.terminal_attempt_id),'owner_id',j.owner_id, + 'lease_epoch',j.lease_epoch::text,'version',j.version::text,'mgmt_version',j.mgmt_version::text,'locked_until',j.locked_until, + 'state',j.state,'output_sha256',j.output_sha256,'published',COALESCE(p_published,false), + 'stage',j.stage,'stage_instance',j.stage_instance,'attempt_count',j.attempt_count,'max_attempts',j.max_attempts,'parked_attempt_count',j.parked_attempt_count, + 'eligible_at',j.eligible_at,'first_parked_at',j.first_parked_at,'last_error_code',j.last_error_code, + 'input',(SELECT jsonb_build_object('uri',r.input_uri,'sha256',r.input_sha256,'config_sha256',r.config_sha256,'code_image_digest',r.code_image_digest) FROM public.polis_queue_runs r WHERE r.env=j.env AND r.run_id=j.run_id)) +$$; +CREATE OR REPLACE FUNCTION public.pq_claim(p_env text,p_priority smallint,p_owner uuid,p_attempt uuid,p_lease_seconds integer,p_worker_class text) +RETURNS jsonb LANGUAGE plpgsql SECURITY DEFINER SET search_path=pg_catalog,pg_temp AS $$ +#variable_conflict use_column +DECLARE j public.polis_queue_jobs; +BEGIN + IF p_owner IS NULL OR p_attempt IS NULL OR p_lease_seconds IS NULL OR p_lease_seconds NOT BETWEEN 10 AND 900 + OR p_worker_class IS NULL OR p_worker_class NOT IN ('delphi','large') THEN RAISE EXCEPTION 'invalid claim'; END IF; + WITH candidate AS ( + SELECT q.env,q.job_id FROM public.polis_queue_jobs q JOIN public.delphi_jobs d ON d.env=q.env AND d.job_id=q.job_id + WHERE q.env=p_env AND q.priority=p_priority AND q.worker_class=p_worker_class + AND q.stage NOT LIKE 'graph_%' + AND q.state IN ('queued','retry_wait') AND q.eligible_at<=statement_timestamp() + AND q.attempt_count-q.parked_attempt_count'succeeded' OR p.output_manifest_digest IS DISTINCT FROM i.producer_output_digest)) + -- An expired lease is not evidence that the previous process stopped. + AND NOT EXISTS(SELECT 1 FROM public.polis_queue_attempts a WHERE a.env=q.env AND a.job_id=q.job_id AND a.process_exit_confirmed_at IS NULL) + AND NOT EXISTS(SELECT 1 FROM public.delphi_provider_requests pr WHERE pr.env=q.env AND pr.job_id=q.job_id AND pr.state IN ('intent','submission_unknown')) + ORDER BY q.eligible_at,q.created_at,q.job_id FOR UPDATE OF q SKIP LOCKED LIMIT 1 + ) UPDATE public.polis_queue_jobs q SET state='running',owner_id=p_owner,attempt_id=p_attempt, + locked_until=clock_timestamp()+make_interval(secs=>p_lease_seconds),lease_epoch=q.lease_epoch+1, + version=q.version+1,mgmt_version=q.mgmt_version+1,attempt_count=q.attempt_count+1,updated_at=clock_timestamp() + FROM candidate c WHERE q.env=c.env AND q.job_id=c.job_id RETURNING q.* INTO j; + IF NOT FOUND THEN RETURN jsonb_build_object('schema_version',CASE p_worker_class WHEN 'large' THEN 'polis-queue/3' ELSE 'polis-queue/2' END,'outcome','none'); END IF; + INSERT INTO public.polis_queue_attempts(env,attempt_id,job_id,owner_id,lease_epoch,outcome) + VALUES(j.env,j.attempt_id,j.job_id,j.owner_id,j.lease_epoch,'running'); + RETURN public.pq_result('owned',j); +END $$; +CREATE FUNCTION public.pd_graph_seal_guard() RETURNS trigger LANGUAGE plpgsql SET search_path=pg_catalog,pg_temp AS $$ +BEGIN + IF TG_OP='DELETE' THEN RAISE EXCEPTION 'immutable graph'; END IF; + IF OLD.sealed OR NOT NEW.sealed OR to_jsonb(OLD)-'sealed' IS DISTINCT FROM to_jsonb(NEW)-'sealed' + THEN RAISE EXCEPTION 'immutable sealed graph'; END IF; + RETURN NEW; +END $$; +CREATE TRIGGER graph_seal_guard BEFORE UPDATE OR DELETE ON public.delphi_graphs FOR EACH ROW EXECUTE FUNCTION public.pd_graph_seal_guard(); + +-- The admission JSON is a closed, bounded local-artifact contract. Every node +-- names its exact snapshot, code/model/runtime/config/seed and full-fit mode. +CREATE FUNCTION public.pd_graph_admit(p_env text,p_zid integer,p_scope text,p_key text,p_spec jsonb,p_supersedes uuid DEFAULT NULL) +RETURNS jsonb LANGUAGE plpgsql SECURITY DEFINER SET search_path=pg_catalog,pg_temp AS $$ +#variable_conflict use_column +DECLARE g public.delphi_graphs; gid uuid=gen_random_uuid(); root uuid; + n jsonb; e jsonb; jid uuid; rid uuid; prod uuid; aid uuid; decl jsonb; reply jsonb; + v_stage text; cls text; +BEGIN + IF p_supersedes IS NOT NULL THEN RAISE EXCEPTION 'superseding dead branches is not supported'; END IF; + IF p_env IS NULL OR p_env !~ '^[a-z0-9_-]{1,64}$' OR p_scope IS NULL OR length(p_scope) NOT BETWEEN 1 AND 128 + OR p_key IS NULL OR length(p_key) NOT BETWEEN 1 AND 128 OR p_spec->>'schema' IS DISTINCT FROM 'polis-job-graph/1' + OR jsonb_typeof(p_spec->'nodes') IS DISTINCT FROM 'array' OR jsonb_array_length(p_spec->'nodes') NOT BETWEEN 1 AND 32 + OR p_spec-'schema'-'nodes'<>'{}'::jsonb OR octet_length(p_spec::text)>1048576 THEN RAISE EXCEPTION 'invalid graph'; END IF; + PERFORM 1 FROM public.conversations WHERE zid=p_zid FOR KEY SHARE; + IF NOT FOUND THEN RAISE EXCEPTION 'unknown conversation'; END IF; + PERFORM pg_advisory_xact_lock(hashtextextended(jsonb_build_array('pd:scope',p_env,p_scope)::text,0)); + SELECT * INTO g FROM public.delphi_graphs WHERE env=p_env AND scope_key=p_scope AND request_key=p_key; + IF FOUND THEN + IF g.zid<>p_zid OR g.request IS DISTINCT FROM p_spec OR g.supersedes IS DISTINCT FROM p_supersedes THEN RAISE EXCEPTION 'graph request conflict'; END IF; + RETURN jsonb_build_object('outcome','existing','graph_id',g.graph_id,'root_job_id',g.root_job_id); + END IF; + IF EXISTS(SELECT 1 FROM public.delphi_job_guards WHERE env=p_env AND scope_key=p_scope) THEN RAISE EXCEPTION 'scope busy'; END IF; + root=gen_random_uuid(); + INSERT INTO public.delphi_graphs(env,graph_id,zid,scope_key,request_key,request,root_job_id,supersedes) + VALUES(p_env,gid,p_zid,p_scope,p_key,p_spec,root,p_supersedes); + FOR n IN SELECT value FROM jsonb_array_elements(p_spec->'nodes') LOOP + v_stage=n->>'stage'; cls=n->>'class'; decl=n->'declared'; + IF n-'key'-'stage'-'class'-'declared'-'inputs'-'max_attempts'<>'{}'::jsonb + OR n->>'key' IS NULL OR v_stage IS NULL OR v_stage NOT IN ('graph_embed','graph_cluster','graph_narrative') + OR cls IS NULL OR NOT ((v_stage='graph_cluster' AND cls IN ('delphi','large')) OR (v_stage<>'graph_cluster' AND cls='delphi')) + OR jsonb_typeof(decl) IS DISTINCT FROM 'object' + OR decl-'snapshot'-'code'-'model'-'runtime'-'seed'-'config'-'mode'-'memory_bytes'-'work_units'<>'{}'::jsonb + OR decl->>'mode' IS DISTINCT FROM 'full' + OR decl->>'code' IS NULL OR decl->>'code' !~ '^[0-9a-f]{40,64}$' + OR decl->>'model' IS DISTINCT FROM (CASE v_stage WHEN 'graph_embed' THEN 'local-token-count/1' WHEN 'graph_cluster' THEN 'local-nearest-centroid/1' WHEN 'graph_narrative' THEN 'local-cluster-summary/1' END) OR COALESCE(decl->>'runtime','')='' + OR jsonb_typeof(decl->'seed') IS DISTINCT FROM 'number' OR jsonb_typeof(decl->'config') IS DISTINCT FROM 'object' + OR jsonb_typeof(decl->'snapshot') IS DISTINCT FROM 'object' + OR jsonb_typeof(decl->'snapshot'->'data'->'texts') IS DISTINCT FROM 'array' + OR jsonb_array_length(decl->'snapshot'->'data'->'texts') NOT BETWEEN 1 AND 100 + OR (decl->'snapshot'->'data')-'texts'<>'{}'::jsonb + OR public.pd_graph_hash(decl->'snapshot'->'data') IS DISTINCT FROM decl->'snapshot'->>'sha256' + OR (decl->'snapshot')-'data'-'sha256'<>'{}'::jsonb + OR COALESCE((decl->>'memory_bytes')::bigint,0) NOT BETWEEN 1 AND (CASE cls WHEN 'delphi' THEN 536870912 ELSE 2147483648 END) + OR COALESCE((decl->>'work_units')::integer,0) NOT BETWEEN 1 AND 10000 + OR jsonb_typeof(n->'inputs') IS DISTINCT FROM 'array' OR jsonb_array_length(n->'inputs')>4 + OR COALESCE((n->>'max_attempts')::integer,0) NOT BETWEEN 1 AND 10 + THEN RAISE EXCEPTION 'invalid stage contract (incremental not supported)'; END IF; + IF v_stage='graph_embed' AND jsonb_array_length(n->'inputs')<>0 OR v_stage<>'graph_embed' AND jsonb_array_length(n->'inputs')<>1 + THEN RAISE EXCEPTION 'stage input arity'; END IF; + jid=CASE WHEN NOT EXISTS(SELECT 1 FROM public.delphi_graph_nodes WHERE env=p_env AND graph_id=gid) THEN root ELSE gen_random_uuid() END; + rid=gen_random_uuid(); + reply=public.pq_enqueue(p_env,p_zid,'graph:'||p_scope||':'||(n->>'key'),'graph-admission',p_key, + public.pd_graph_hash(n),rid,jid,'graph://'||gid::text||'/'||(n->>'key'),public.pd_graph_hash(decl),public.pd_graph_hash(decl->'config'), + decl->>'code',1::smallint,(n->>'max_attempts')::integer); + IF reply->>'outcome'<>'enqueued' THEN RAISE EXCEPTION 'graph product conflict'; END IF; + UPDATE public.polis_queue_runs SET contract_version='polis-queue/5' WHERE env=p_env AND run_id=rid; + INSERT INTO public.delphi_jobs(job_id,env,zid,kind,parent_job_id,run_id,origin,replayable,reuse_eligible,status,config_effective,code_version,model_versions) + VALUES(jid,p_env,p_zid,substring(v_stage from 7),CASE WHEN jid<>root THEN root END,rid,'queued',true,true,'queued',decl->'config',decl->>'code',jsonb_build_object('exact',decl->>'model')); + INSERT INTO public.delphi_graph_nodes VALUES(p_env,p_zid,gid,jid,rid,n->>'key',decl,NULL,NULL); + UPDATE public.polis_queue_jobs SET stage=n->>'stage',worker_class=cls WHERE env=p_env AND job_id=jid; + END LOOP; + FOR n IN SELECT value FROM jsonb_array_elements(p_spec->'nodes') LOOP + SELECT job_id INTO STRICT jid FROM public.delphi_graph_nodes WHERE env=p_env AND graph_id=gid AND node_key=n->>'key'; + FOR e IN SELECT value FROM jsonb_array_elements(n->'inputs') LOOP + aid=NULL; + IF e-'node'-'artifact_id'-'sha256'-'contract_sha256'-'role'<>'{}'::jsonb + OR e->>'role' IS DISTINCT FROM (CASE n->>'stage' WHEN 'graph_cluster' THEN 'embeddings' WHEN 'graph_narrative' THEN 'clusters' END) + OR (e ? 'node')=(e ? 'artifact_id') THEN RAISE EXCEPTION 'invalid input role or reference'; END IF; + IF e ? 'node' THEN + SELECT job_id INTO STRICT prod FROM public.delphi_graph_nodes WHERE env=p_env AND graph_id=gid AND node_key=e->>'node'; + ELSE + aid=(e->>'artifact_id')::uuid; + SELECT a.job_id INTO STRICT prod FROM public.delphi_artifacts a JOIN public.delphi_graph_nodes pn USING(env,job_id) + WHERE a.env=p_env AND a.artifact_id=aid AND pn.zid=p_zid AND a.content_sha=e->>'sha256' AND public.pd_graph_hash(pn.declared)=e->>'contract_sha256'; + END IF; + IF NOT EXISTS(SELECT 1 FROM public.polis_queue_jobs qp WHERE qp.env=p_env AND qp.job_id=prod AND qp.stage=CASE n->>'stage' WHEN 'graph_cluster' THEN 'graph_embed' WHEN 'graph_narrative' THEN 'graph_cluster' END) + THEN RAISE EXCEPTION 'input stage mismatch'; END IF; + IF (SELECT gn.declared->'snapshot'->>'sha256' FROM public.delphi_graph_nodes gn WHERE gn.env=p_env AND gn.job_id=prod) IS DISTINCT FROM n->'declared'->'snapshot'->>'sha256' THEN RAISE EXCEPTION 'upstream snapshot mismatch'; END IF; + INSERT INTO public.delphi_graph_edges VALUES(p_env,jid,prod,e->>'role',e->>'sha256',aid); + END LOOP; + END LOOP; + UPDATE public.delphi_graphs SET sealed=true WHERE env=p_env AND graph_id=gid; + INSERT INTO public.delphi_job_guards VALUES(p_env,p_scope,p_zid,root,public.pd_graph_hash(p_spec)); + RETURN jsonb_build_object('outcome','enqueued','graph_id',gid,'root_job_id',root); +END $$; + +CREATE FUNCTION public.pd_graph_readiness(p_env text,p_job uuid) RETURNS jsonb +LANGUAGE sql STABLE SECURITY DEFINER SET search_path=pg_catalog,pg_temp AS $$ + SELECT jsonb_build_object('schema','polis-job-readiness/1','job_id',p_job,'state',q.state, + 'blockers',COALESCE((SELECT jsonb_agg(jsonb_build_object('producer',e.producer,'reason',CASE + WHEN p.state='dead' THEN 'dependency_dead' WHEN p.state='cancelled' THEN 'dependency_cancelled' + WHEN p.state<>'succeeded' THEN 'waiting_for_input' + WHEN a.artifact_id IS NULL THEN 'unresolved_output' + ELSE 'digest_mismatch' END) ORDER BY e.input_role) + FROM public.delphi_graph_edges e JOIN public.polis_queue_jobs p ON p.env=e.env AND p.job_id=e.producer + LEFT JOIN public.delphi_artifacts a ON a.env=e.env AND a.job_id=e.producer + WHERE e.env=p_env AND e.consumer=p_job AND (p.state<>'succeeded' OR a.artifact_id IS NULL OR (e.expected_sha IS NOT NULL AND e.expected_sha<>a.content_sha))),'[]'::jsonb)) + FROM public.polis_queue_jobs q JOIN public.delphi_graph_nodes n USING(env,job_id) WHERE q.env=p_env AND q.job_id=p_job +$$; +CREATE FUNCTION public.pd_graph_claim(p_env text,p_priority smallint,p_owner uuid,p_attempt uuid,p_lease integer,p_class text,p_stages text) +RETURNS jsonb LANGUAGE plpgsql SECURITY DEFINER SET search_path=pg_catalog,pg_temp AS $$ +#variable_conflict use_column +DECLARE j public.polis_queue_jobs; n public.delphi_graph_nodes; inputs jsonb; v_resolved jsonb; +BEGIN + IF p_owner IS NULL OR p_attempt IS NULL OR p_lease IS NULL OR p_lease NOT BETWEEN 10 AND 900 + OR p_class IS NULL OR p_class NOT IN ('delphi','large') THEN RAISE EXCEPTION 'invalid graph claim'; END IF; + SELECT q.* INTO j FROM public.polis_queue_jobs q JOIN public.delphi_graph_nodes n USING(env,job_id) + JOIN public.delphi_graphs g USING(env,graph_id) + WHERE q.env=p_env AND q.priority=p_priority AND q.worker_class=p_class AND q.stage=ANY(string_to_array(p_stages,',')) + AND g.sealed AND q.state IN ('queued','retry_wait') AND q.eligible_at<=statement_timestamp() + AND q.attempt_count-q.parked_attempt_count'blockers'='[]'::jsonb + AND NOT EXISTS(SELECT 1 FROM public.polis_queue_attempts a WHERE a.env=q.env AND a.job_id=q.job_id AND a.process_exit_confirmed_at IS NULL) + AND NOT EXISTS(SELECT 1 FROM public.delphi_provider_requests pr WHERE pr.env=q.env AND pr.job_id=q.job_id AND pr.state IN ('intent','submission_unknown','submitted')) + ORDER BY q.eligible_at,q.created_at,q.job_id FOR UPDATE OF q SKIP LOCKED LIMIT 1; + IF NOT FOUND THEN RETURN jsonb_build_object('schema_version','polis-queue/5','outcome','none'); END IF; + UPDATE public.delphi_graph_edges e SET artifact_id=a.artifact_id FROM public.delphi_artifacts a + WHERE e.env=p_env AND e.consumer=j.job_id AND e.artifact_id IS NULL AND a.env=e.env AND a.job_id=e.producer + AND (e.expected_sha IS NULL OR e.expected_sha=a.content_sha); + SELECT * INTO STRICT n FROM public.delphi_graph_nodes WHERE env=p_env AND job_id=j.job_id; + IF n.resolved IS NULL THEN + SELECT COALESCE(jsonb_object_agg(e.input_role,jsonb_build_object('artifact_id',a.artifact_id,'producer',a.job_id,'run_id',a.run_id, + 'attempt_id',a.attempt_id,'sha256',a.content_sha,'schema',a.schema_version,'payload',a.payload)),'{}'::jsonb) + INTO inputs FROM public.delphi_graph_edges e JOIN public.delphi_artifacts a USING(env,artifact_id) WHERE e.env=p_env AND e.consumer=j.job_id; + v_resolved=jsonb_build_object('schema','polis-job-input/1','declared',n.declared,'artifacts',inputs); + UPDATE public.delphi_graph_nodes SET resolved=v_resolved,resolved_sha=public.pd_graph_hash(v_resolved) WHERE env=p_env AND job_id=j.job_id; + n.resolved=v_resolved; n.resolved_sha=public.pd_graph_hash(v_resolved); + END IF; + UPDATE public.polis_queue_jobs SET state='running',owner_id=p_owner,attempt_id=p_attempt, + locked_until=clock_timestamp()+make_interval(secs=>p_lease),lease_epoch=lease_epoch+1, + version=version+1,mgmt_version=mgmt_version+1,attempt_count=attempt_count+1,updated_at=clock_timestamp() + WHERE env=p_env AND job_id=j.job_id RETURNING * INTO j; + INSERT INTO public.polis_queue_attempts(env,attempt_id,job_id,owner_id,lease_epoch,outcome) + VALUES(p_env,p_attempt,j.job_id,p_owner,j.lease_epoch,'running'); + RETURN public.pq_result('owned',j)||jsonb_build_object('graph_input',n.resolved,'graph_input_wire',n.resolved::text,'graph_input_sha',n.resolved_sha,'zid',n.zid); +END $$; + +CREATE FUNCTION public.pd_graph_finalize(p_env text,p_job uuid,p_owner uuid,p_attempt uuid,p_epoch bigint,p_uri text,p_sha text) +RETURNS jsonb LANGUAGE plpgsql SECURITY DEFINER SET search_path=pg_catalog,pg_temp AS $$ +#variable_conflict use_column +DECLARE j public.polis_queue_jobs; a public.polis_queue_attempts; n public.delphi_graph_nodes; m jsonb; raw text; outdoc jsonb; +BEGIN + j=public.pd_lock(p_env,p_job); + SELECT * INTO a FROM public.polis_queue_attempts WHERE env=p_env AND attempt_id=p_attempt AND job_id=p_job; + IF j.state='succeeded' AND j.terminal_attempt_id=p_attempt AND a.owner_id=p_owner AND a.lease_epoch=p_epoch THEN + RETURN public.pq_result(CASE WHEN a.output_sha256=p_sha THEN 'already_succeeded' ELSE 'invalid_output' END,j); + END IF; + IF j.stage NOT LIKE 'graph_%' OR NOT public.pq_owns(j,p_owner,p_attempt,p_epoch) THEN RETURN public.pq_result('fenced',j); END IF; + IF a.process_exit_confirmed_at IS NULL THEN RAISE EXCEPTION 'process exit proof required'; END IF; + SELECT * INTO STRICT n FROM public.delphi_graph_nodes WHERE env=p_env AND job_id=p_job; + SELECT line INTO raw FROM public.polis_queue_logs WHERE env=p_env AND attempt_id=p_attempt AND stream='manifest' + AND encode(sha256(convert_to(line,'UTF8')),'hex')=p_sha ORDER BY seq DESC LIMIT 1; + IF raw IS NULL THEN RETURN public.pq_result('invalid_output',j); END IF; + m=raw::jsonb; outdoc=m->'output'; + IF m->>'schema' IS DISTINCT FROM 'polis-job-artifact-manifest/1' + OR m->>'job_id' IS DISTINCT FROM p_job::text OR m->>'run_id' IS DISTINCT FROM j.run_id::text + OR m->>'attempt_id' IS DISTINCT FROM p_attempt::text OR m->>'stage' IS DISTINCT FROM j.stage + OR m->>'input_sha256' IS DISTINCT FROM n.resolved_sha OR m->>'outcome' IS DISTINCT FROM 'succeeded' + OR m-'schema'-'job_id'-'run_id'-'attempt_id'-'stage'-'input_sha256'-'outcome'-'output'<>'{}'::jsonb + OR outdoc-'role'-'schema'-'payload'-'sha256'<>'{}'::jsonb + OR outdoc->>'role' IS DISTINCT FROM 'result' OR outdoc->>'schema' IS DISTINCT FROM j.stage||'/1' + OR jsonb_typeof(outdoc->'payload') IS DISTINCT FROM 'string' OR octet_length(outdoc->>'payload')>524288 + OR outdoc->>'sha256' IS DISTINCT FROM encode(sha256(convert_to(outdoc->>'payload','UTF8')),'hex') + THEN RETURN public.pq_result('invalid_output',j); END IF; + IF EXISTS(SELECT 1 FROM public.delphi_provider_requests WHERE env=p_env AND job_id=p_job AND state IN ('intent','submission_unknown','submitted')) + THEN RAISE EXCEPTION 'provider request unresolved'; END IF; + INSERT INTO public.delphi_artifacts(env,job_id,run_id,attempt_id,output_role,schema_version,payload,content_sha,byte_count) + VALUES(p_env,p_job,j.run_id,p_attempt,'result',outdoc->>'schema',outdoc->>'payload',outdoc->>'sha256',octet_length(outdoc->>'payload')); + UPDATE public.polis_queue_attempts SET outcome='succeeded',ended_at=clock_timestamp(),output_sha256=p_sha WHERE env=p_env AND attempt_id=p_attempt; + UPDATE public.polis_queue_jobs SET state='succeeded',terminal_attempt_id=p_attempt,owner_id=NULL,attempt_id=NULL,locked_until=NULL, + output_sha256=p_sha,version=version+1,mgmt_version=mgmt_version+1,updated_at=clock_timestamp() WHERE env=p_env AND job_id=p_job RETURNING * INTO j; + UPDATE public.delphi_jobs SET output_manifest_digest=decode(p_sha,'hex') WHERE env=p_env AND job_id=p_job; + UPDATE public.polis_queue_runs SET state='succeeded',expected_output_uri='pg-artifact://'||p_job::text,expected_output_sha256=p_sha,output_sha256=p_sha WHERE env=p_env AND run_id=j.run_id; + RETURN public.pq_result('succeeded',j,false); +END $$; +CREATE FUNCTION public.pd_graph_reconcile(p_env text) RETURNS jsonb +LANGUAGE plpgsql SECURITY DEFINER SET search_path=pg_catalog,pg_temp AS $$ +#variable_conflict use_column +DECLARE g record; released integer=0; +BEGIN + FOR g IN SELECT d.scope_key FROM public.delphi_job_guards d JOIN public.delphi_graphs g ON g.env=d.env AND g.root_job_id=d.root_job_id + WHERE d.env=p_env AND NOT EXISTS(SELECT 1 FROM public.delphi_graph_nodes gn JOIN public.polis_queue_jobs jq USING(env,job_id) WHERE gn.env=g.env AND gn.graph_id=g.graph_id AND jq.state NOT IN ('succeeded','dead','cancelled')) AND NOT EXISTS(SELECT 1 FROM public.delphi_graph_nodes gn JOIN public.polis_queue_attempts a USING(env,job_id) WHERE gn.env=g.env AND gn.graph_id=g.graph_id AND a.process_exit_confirmed_at IS NULL) AND NOT EXISTS(SELECT 1 FROM public.delphi_graph_nodes gn JOIN public.delphi_provider_requests pr USING(env,job_id) WHERE gn.env=g.env AND gn.graph_id=g.graph_id AND pr.state IN ('intent','submission_unknown','submitted')) ORDER BY d.scope_key LIMIT 100 LOOP + IF public.pd_release_scope(p_env,g.scope_key) THEN released=released+1; END IF; + END LOOP; + RETURN jsonb_build_object('released',released); +END $$; +CREATE FUNCTION public.pd_graph_view(p_env text,p_graph uuid) RETURNS jsonb +LANGUAGE sql STABLE SECURITY DEFINER SET search_path=pg_catalog,pg_temp AS $$ + SELECT jsonb_build_object('schema','polis-job-graph-status/1','graph_id',g.graph_id,'supersedes',g.supersedes,'sealed',g.sealed, + 'guard_held',EXISTS(SELECT 1 FROM public.delphi_job_guards WHERE env=g.env AND root_job_id=g.root_job_id), + 'nodes',(SELECT jsonb_agg(jsonb_build_object('key',n.node_key,'job_id',n.job_id,'run_id',n.run_id,'declared_sha',public.pd_graph_hash(n.declared), + 'readiness',public.pd_graph_readiness(n.env,n.job_id),'attempts',q.attempt_count,'mgmt_version',q.mgmt_version::text, + 'input_sha256',n.resolved_sha,'provider_requests',(SELECT COALESCE(jsonb_agg(jsonb_build_object('request_id',pr.request_id,'state',pr.state,'batch_id',pr.provider_batch_id)),'[]'::jsonb) FROM public.delphi_provider_requests pr WHERE pr.env=n.env AND pr.job_id=n.job_id),'artifact',(SELECT to_jsonb(a) FROM public.delphi_artifacts a WHERE a.env=n.env AND a.job_id=n.job_id)) ORDER BY n.node_key) + FROM public.delphi_graph_nodes n JOIN public.polis_queue_jobs q USING(env,job_id) WHERE n.env=g.env AND n.graph_id=g.graph_id)) + FROM public.delphi_graphs g WHERE g.env=p_env AND g.graph_id=p_graph +$$; +CREATE FUNCTION public.pd_graph_bundle(p_env text,p_artifact uuid) RETURNS jsonb +LANGUAGE sql STABLE SECURITY DEFINER SET search_path=pg_catalog,pg_temp AS $$ + WITH RECURSIVE upstream(id) AS ( + SELECT p_artifact UNION SELECT e.artifact_id FROM upstream u JOIN public.delphi_artifacts a ON a.env=p_env AND a.artifact_id=u.id + JOIN public.delphi_graph_edges e ON e.env=a.env AND e.consumer=a.job_id WHERE e.artifact_id IS NOT NULL) + SELECT jsonb_build_object('schema','polis-job-bundle/1','root',p_artifact, + 'artifacts',jsonb_agg(to_jsonb(a) ORDER BY a.artifact_id)) FROM upstream u JOIN public.delphi_artifacts a ON a.env=p_env AND a.artifact_id=u.id +$$; +CREATE FUNCTION public.pd_graph_publish(p_env text,p_graph uuid,p_job uuid,p_expected bigint) RETURNS jsonb +LANGUAGE plpgsql SECURITY DEFINER SET search_path=pg_catalog,pg_temp AS $$ +#variable_conflict use_column +DECLARE g public.delphi_graphs; aid uuid; current_generation bigint; +BEGIN + SELECT * INTO STRICT g FROM public.delphi_graphs WHERE env=p_env AND graph_id=p_graph AND sealed; + PERFORM pg_advisory_xact_lock(hashtextextended(jsonb_build_array('pd:scope',p_env,g.scope_key)::text,0)); + SELECT a.artifact_id INTO STRICT aid FROM public.delphi_graph_nodes n JOIN public.delphi_artifacts a USING(env,job_id) + JOIN public.polis_queue_jobs q USING(env,job_id) WHERE n.env=p_env AND n.graph_id=p_graph AND n.job_id=p_job AND q.stage='graph_narrative' AND q.state='succeeded'; + IF EXISTS(SELECT 1 FROM public.delphi_graph_nodes n JOIN public.polis_queue_jobs q USING(env,job_id) WHERE n.env=p_env AND n.graph_id=p_graph AND q.state<>'succeeded') + OR EXISTS(SELECT 1 FROM public.delphi_graphs x WHERE x.env=p_env AND x.zid=g.zid AND x.scope_key=g.scope_key AND x.created_at>g.created_at) THEN RAISE EXCEPTION 'incomplete or superseded bundle'; END IF; + SELECT generation INTO current_generation FROM public.delphi_graph_served WHERE env=p_env AND zid=g.zid AND scope_key=g.scope_key FOR UPDATE; + IF COALESCE(current_generation,0) IS DISTINCT FROM p_expected THEN RETURN jsonb_build_object('outcome','conflict'); END IF; + INSERT INTO public.delphi_graph_served VALUES(p_env,g.zid,g.scope_key,p_expected+1,aid) + ON CONFLICT(env,zid,scope_key) DO UPDATE SET generation=EXCLUDED.generation,artifact_id=EXCLUDED.artifact_id; + RETURN jsonb_build_object('outcome','published','generation',p_expected+1,'bundle',public.pd_graph_bundle(p_env,aid)); +END $$; +CREATE FUNCTION public.pd_graph_served(p_env text,p_zid integer,p_scope text) RETURNS jsonb +LANGUAGE sql STABLE SECURITY DEFINER SET search_path=pg_catalog,pg_temp AS $$ + SELECT jsonb_build_object('generation',generation,'bundle',public.pd_graph_bundle(env,artifact_id)) + FROM public.delphi_graph_served WHERE env=p_env AND zid=p_zid AND scope_key=p_scope +$$; +CREATE FUNCTION public.pd_graph_depth(p_env text,p_class text) RETURNS jsonb +LANGUAGE sql STABLE SECURITY DEFINER SET search_path=pg_catalog,pg_temp AS $$ + SELECT jsonb_build_object('schema','polis-job-depth/1', + 'runnable',count(*) FILTER(WHERE state IN ('queued','retry_wait') AND eligible_at<=statement_timestamp() AND public.pd_graph_readiness(q.env,q.job_id)->'blockers'='[]'::jsonb AND q.attempt_count-q.parked_attempt_count'blockers'<>'[]'::jsonb), + 'running',count(*) FILTER(WHERE state='running'),'dead',count(*) FILTER(WHERE state='dead')) + FROM public.polis_queue_jobs q WHERE env=p_env AND worker_class=p_class AND stage LIKE 'graph_%' +$$; +CREATE OR REPLACE FUNCTION public.pd_finalize(p_env text,p_job uuid,p_owner uuid,p_attempt uuid,p_epoch bigint,p_uri text,p_sha text) +RETURNS jsonb LANGUAGE plpgsql SET search_path=pg_catalog,pg_temp SET TimeZone='UTC' AS $$ +DECLARE j public.polis_queue_jobs; a public.polis_queue_attempts; r public.polis_queue_runs; raw text; m jsonb; +BEGIN + j=public.pd_lock(p_env,p_job); + IF j.stage LIKE 'graph_%' THEN RAISE EXCEPTION 'graph manifest required'; END IF; + SELECT * INTO a FROM public.polis_queue_attempts WHERE env=p_env AND job_id=p_job AND attempt_id=p_attempt; + SELECT * INTO r FROM public.polis_queue_runs WHERE env=p_env AND run_id=j.run_id; + IF j.state='succeeded' AND j.terminal_attempt_id=p_attempt AND a.owner_id=p_owner AND a.lease_epoch=p_epoch THEN + RETURN public.pq_result(CASE WHEN a.output_sha256=p_sha AND r.expected_output_uri=p_uri THEN 'already_succeeded' ELSE 'invalid_output' END,j); + END IF; + IF NOT public.pq_owns(j,p_owner,p_attempt,p_epoch) THEN RETURN public.pq_result('fenced',j); END IF; + IF a.process_exit_confirmed_at IS NULL THEN RAISE EXCEPTION 'process exit proof required'; END IF; + IF p_uri IS NULL OR p_uri !~ '^file://.+' OR length(p_uri)>2048 OR p_sha IS NULL OR p_sha !~ '^[0-9a-f]{64}$' + THEN RETURN public.pq_result('invalid_output',j); END IF; + SELECT line INTO raw FROM public.polis_queue_logs WHERE env=p_env AND attempt_id=p_attempt AND stream='manifest' + AND encode(sha256(convert_to(line,'UTF8')),'hex')=p_sha ORDER BY seq DESC LIMIT 1; + IF raw IS NULL THEN RETURN public.pq_result('invalid_output',j); END IF; + BEGIN m=raw::jsonb; EXCEPTION WHEN invalid_text_representation THEN RETURN public.pq_result('invalid_output',j); END; + IF m->>'schema' IS DISTINCT FROM 'polis-jobs.output-manifest/1' + OR m->>'job_id' IS DISTINCT FROM p_job::text OR m->>'attempt_id' IS DISTINCT FROM p_attempt::text + OR m->>'stage' IS DISTINCT FROM j.stage OR m->>'outcome' IS DISTINCT FROM 'succeeded' + OR COALESCE(m->>'phase','') NOT IN ('submit','recheck','run') + OR jsonb_typeof(m->'outputs') IS DISTINCT FROM 'array' + OR jsonb_typeof(m->'inputs') IS DISTINCT FROM 'object' + OR jsonb_typeof(m->'models') IS DISTINCT FROM 'object' + OR jsonb_typeof(m->'cost') IS DISTINCT FROM 'object' + OR (m ? 'artifacts' AND m->'artifacts'<>'[]'::jsonb) + THEN RETURN public.pq_result('invalid_output',j); END IF; + IF EXISTS(SELECT 1 FROM jsonb_array_elements(m->'outputs') o WHERE o->>'store' IS DISTINCT FROM 'dynamodb' + OR COALESCE(o->>'table','')='' OR NOT (o ? 'keys' OR o ? 'key_prefix') OR COALESCE(o->>'rows','') !~ '^[0-9]+$') + THEN RETURN public.pq_result('invalid_output',j); END IF; + IF EXISTS(SELECT 1 FROM public.delphi_provider_requests WHERE env=p_env AND job_id=p_job AND state IN ('intent','submission_unknown','submitted')) + THEN RAISE EXCEPTION 'provider request unresolved'; END IF; + UPDATE public.polis_queue_attempts SET outcome='succeeded',ended_at=clock_timestamp(),output_sha256=p_sha WHERE env=p_env AND attempt_id=p_attempt; + UPDATE public.polis_queue_jobs SET state='succeeded',terminal_attempt_id=p_attempt,owner_id=NULL,attempt_id=NULL,locked_until=NULL, + output_sha256=p_sha,version=version+1,mgmt_version=mgmt_version+1,updated_at=clock_timestamp() WHERE env=p_env AND job_id=p_job RETURNING * INTO j; + UPDATE public.delphi_jobs SET output_manifest_digest=decode(p_sha,'hex') WHERE env=p_env AND job_id=p_job; + UPDATE public.polis_queue_runs SET state='succeeded',expected_output_uri=p_uri,expected_output_sha256=p_sha,output_sha256=p_sha WHERE env=p_env AND run_id=j.run_id; + RETURN public.pq_result('succeeded',j,false); +END $$; +-- Explicit operator remediation, never granted to an ordinary executor. +-- Preserve the existing class-depth wire; graph blocked work is not scale-out demand. +CREATE OR REPLACE FUNCTION public.pq_class_depth(p_env text,p_worker_class text) +RETURNS jsonb LANGUAGE plpgsql STABLE SECURITY DEFINER SET search_path=pg_catalog,pg_temp SET TimeZone='UTC' AS $$ +DECLARE c record; +BEGIN + IF p_env IS NULL OR p_env='' OR p_worker_class IS NULL OR p_worker_class NOT IN ('delphi','large') + THEN RAISE EXCEPTION 'invalid class depth read'; END IF; + SELECT count(*) FILTER (WHERE q.state IN ('queued','retry_wait') AND (q.stage NOT LIKE 'graph_%' OR (q.eligible_at<=statement_timestamp() AND q.attempt_count-q.parked_attempt_count'blockers'='[]'::jsonb AND NOT EXISTS(SELECT 1 FROM public.polis_queue_attempts a WHERE a.env=q.env AND a.job_id=q.job_id AND a.process_exit_confirmed_at IS NULL) AND NOT EXISTS(SELECT 1 FROM public.delphi_provider_requests pr WHERE pr.env=q.env AND pr.job_id=q.job_id AND pr.state IN ('intent','submission_unknown','submitted'))))) AS queued, + count(*) FILTER (WHERE q.state='running') AS leased, + count(*) FILTER (WHERE q.state='parked') AS parked, + count(*) FILTER (WHERE q.state='dead') AS dead, + min(q.created_at) FILTER (WHERE q.state NOT IN ('succeeded','dead','cancelled')) AS oldest + INTO c FROM public.polis_queue_jobs q WHERE q.env=p_env AND q.worker_class=p_worker_class; + RETURN jsonb_build_object('schema_version','polis-queue/3','outcome','class_depth','env',p_env,'worker_class',p_worker_class, + 'queued',c.queued,'leased',c.leased,'parked',c.parked,'dead',c.dead,'oldest_unresolved_created_at',c.oldest); +END $$; +-- No table access or internal helper privilege crosses the executor boundary. +DO $$ DECLARE r record; BEGIN + FOR r IN SELECT oid::regclass AS name FROM pg_class WHERE relnamespace='public'::regnamespace AND (relname LIKE 'delphi_graph_%' OR relname='delphi_graphs' OR relname='delphi_artifacts') AND relkind='r' LOOP + EXECUTE format('REVOKE ALL ON %s FROM PUBLIC,polis_queue_executor',r.name); + END LOOP; + FOR r IN SELECT oid::regprocedure AS name FROM pg_proc WHERE pronamespace='public'::regnamespace AND proname LIKE 'pd_graph_%' LOOP + EXECUTE format('REVOKE ALL ON FUNCTION %s FROM PUBLIC,polis_queue_executor',r.name); + END LOOP; +END $$; +GRANT EXECUTE ON FUNCTION public.pd_graph_admit(text,integer,text,text,jsonb,uuid),public.pd_graph_claim(text,smallint,uuid,uuid,integer,text,text), + public.pd_graph_finalize(text,uuid,uuid,uuid,bigint,text,text),public.pd_graph_reconcile(text), + public.pd_graph_view(text,uuid),public.pd_graph_readiness(text,uuid),public.pd_graph_depth(text,text), + public.pd_graph_publish(text,uuid,uuid,bigint),public.pd_graph_served(text,integer,text) TO polis_queue_executor; +COMMIT; diff --git a/server/postgres/migrations/adoption/000000_initial.sql b/server/postgres/migrations/adoption/000000_initial.sql new file mode 100644 index 0000000000..3de3898aa1 --- /dev/null +++ b/server/postgres/migrations/adoption/000000_initial.sql @@ -0,0 +1,357 @@ +-- Named legacy variants are documented in docs/migration-legacy-contract.md. +-- Contributor agreement tables are outside this release adoption boundary. +-- Catalog postconditions only; this file never replays migration DDL. +SELECT NOT EXISTS (SELECT 1 FROM (VALUES + ('users','uid','integer'), + ('users','hname','character varying(746)'), + ('users','created','bigint'), + ('users','username','character varying(128)'), + ('users','email','character varying(256)'), + ('users','is_owner','boolean'), + ('users','zinvite','character varying(300)'), + ('users','oinvite','character varying(300)'), + ('users','tut','smallint'), + ('users','site_id','character varying(256)'), + ('users','site_owner','boolean'), + ('site_domain_whitelist','site_id','character varying(256)'), + ('site_domain_whitelist','domain_whitelist','character varying(999)'), + ('site_domain_whitelist','domain_whitelist_override_key','character varying(999)'), + ('site_domain_whitelist','modified','bigint'), + ('site_domain_whitelist','created','bigint'), + ('metrics','uid','integer'), + ('metrics','type','integer'), + ('metrics','dur','integer'), + ('metrics','hashedpc','integer'), + ('metrics','created','bigint'), + ('auth_tokens','token','character varying(32)'), + ('auth_tokens','uid','integer'), + ('auth_tokens','created','bigint'), + ('jianiuevyew','uid','integer'), + ('jianiuevyew','pwhash','character varying(128)'), + ('apikeysndvweifu','uid','integer'), + ('apikeysndvweifu','apikey','character varying(32)'), + ('apikeysndvweifu','created','bigint'), + ('courses','course_id','integer'), + ('courses','topic','character varying(1000)'), + ('courses','description','character varying(1000)'), + ('courses','owner','integer'), + ('courses','course_invite','character varying(32)'), + ('courses','created','bigint'), + ('conversations','zid','integer'), + ('conversations','topic','character varying(1000)'), + ('conversations','description','character varying(50000)'), + ('conversations','link_url','character varying(9999)'), + ('conversations','parent_url','character varying(9999)'), + ('conversations','upvotes','integer'), + ('conversations','participant_count','integer'), + ('conversations','is_anon','boolean'), + ('conversations','is_active','boolean'), + ('conversations','is_draft','boolean'), + ('conversations','is_public','boolean'), + ('conversations','is_data_open','boolean'), + ('conversations','profanity_filter','boolean'), + ('conversations','spam_filter','boolean'), + ('conversations','strict_moderation','boolean'), + ('conversations','prioritize_seed','boolean'), + ('conversations','vis_type','integer'), + ('conversations','write_type','integer'), + ('conversations','help_type','integer'), + ('conversations','write_hint_type','integer'), + ('conversations','style_btn','character varying(500)'), + ('conversations','socialbtn_type','integer'), + ('conversations','subscribe_type','integer'), + ('conversations','bgcolor','character varying(20)'), + ('conversations','help_bgcolor','character varying(20)'), + ('conversations','help_color','character varying(20)'), + ('conversations','email_domain','character varying(200)'), + ('conversations','use_xid_whitelist','boolean'), + ('conversations','owner','integer'), + ('conversations','org_id','integer'), + ('conversations','context','character varying(1000)'), + ('conversations','course_id','integer'), + ('conversations','owner_sees_participation_stats','boolean'), + ('conversations','auth_needed_to_vote','boolean'), + ('conversations','auth_needed_to_write','boolean'), + ('conversations','auth_opt_fb','boolean'), + ('conversations','auth_opt_tw','boolean'), + ('conversations','auth_opt_allow_3rdparty','boolean'), + ('conversations','modified','bigint'), + ('conversations','created','bigint'), + ('participant_metadata_questions','pmqid','integer'), + ('participant_metadata_questions','zid','integer'), + ('participant_metadata_questions','key','character varying(999)'), + ('participant_metadata_questions','alive','boolean'), + ('participant_metadata_questions','created','bigint'), + ('participant_metadata_answers','pmaid','integer'), + ('participant_metadata_answers','pmqid','integer'), + ('participant_metadata_answers','zid','integer'), + ('participant_metadata_answers','value','character varying(999)'), + ('participant_metadata_answers','alive','boolean'), + ('participant_metadata_answers','created','bigint'), + ('contexts','context_id','integer'), + ('contexts','name','character varying(300)'), + ('contexts','creator','integer'), + ('contexts','is_public','boolean'), + ('contexts','created','bigint'), + ('inviters','inviter_uid','integer'), + ('inviters','invited_email','character varying(999)'), + ('inviters','created','bigint'), + ('upvotes','uid','integer'), + ('upvotes','zid','integer'), + ('oinvites','oinvite','character varying(300)'), + ('oinvites','note','character varying(999)'), + ('oinvites','created','bigint'), + ('einvites','einvite','character varying(100)'), + ('einvites','email','character varying(999)'), + ('einvites','created','bigint'), + ('email_validations','email','character varying(999)'), + ('email_validations','created','bigint'), + ('zinvites','zid','integer'), + ('zinvites','zinvite','character varying(300)'), + ('zinvites','created','bigint'), + ('beta','name','character varying(999)'), + ('beta','email','character varying(200)'), + ('beta','organization','character varying(200)'), + ('beta','created','bigint'), + ('participants','pid','integer'), + ('participants','uid','integer'), + ('participants','zid','integer'), + ('participants','vote_count','integer'), + ('participants','last_interaction','bigint'), + ('participants','subscribed','integer'), + ('participants','last_notified','bigint'), + ('participants','nsli','smallint'), + ('participants','mod','integer'), + ('participants','created','bigint'), + ('participants_extended','uid','integer'), + ('participants_extended','zid','integer'), + ('participants_extended','referrer','character varying(9999)'), + ('participants_extended','parent_url','character varying(9999)'), + ('participants_extended','created','bigint'), + ('participants_extended','modified','bigint'), + ('participants_extended','subscribe_email','character varying(256)'), + ('participants_extended','show_translation_activated','boolean'), + ('participant_locations','zid','integer'), + ('participant_locations','uid','integer'), + ('participant_locations','pid','integer'), + ('participant_locations','lat','double precision'), + ('participant_locations','lng','double precision'), + ('participant_locations','created','bigint'), + ('participant_locations','source','integer'), + ('xids','uid','integer'), + ('xids','owner','integer'), + ('xids','xid','text'), + ('xids','x_profile_image_url','character varying(3000)'), + ('xids','x_name','character varying(746)'), + ('xids','x_email','character varying(256)'), + ('xids','created','bigint'), + ('xids','modified','bigint'), + ('xid_whitelist','owner','integer'), + ('xid_whitelist','xid','text'), + ('xid_whitelist','created','bigint'), + ('notification_tasks','zid','integer'), + ('notification_tasks','modified','bigint'), + ('participant_metadata_choices','zid','integer'), + ('participant_metadata_choices','pid','integer'), + ('participant_metadata_choices','pmqid','integer'), + ('participant_metadata_choices','pmaid','integer'), + ('participant_metadata_choices','alive','boolean'), + ('participant_metadata_choices','created','bigint'), + ('twitter_users','uid','integer'), + ('twitter_users','twitter_user_id','bigint'), + ('twitter_users','screen_name','character varying(999)'), + ('twitter_users','name','character varying(9999)'), + ('twitter_users','followers_count','integer'), + ('twitter_users','friends_count','integer'), + ('twitter_users','verified','boolean'), + ('twitter_users','profile_image_url_https','character varying(9999)'), + ('twitter_users','location','character varying(9999)'), + ('twitter_users','response','json'), + ('twitter_users','modified','bigint'), + ('twitter_users','created','bigint'), + ('facebook_users','uid','integer'), + ('facebook_users','fb_user_id','text'), + ('facebook_users','fb_name','character varying(9999)'), + ('facebook_users','fb_link','character varying(9999)'), + ('facebook_users','fb_public_profile','text'), + ('facebook_users','fb_login_status','text'), + ('facebook_users','fb_auth_response','text'), + ('facebook_users','fb_access_token','text'), + ('facebook_users','fb_granted_scopes','text'), + ('facebook_users','fb_location_id','character varying(100)'), + ('facebook_users','location','character varying(9999)'), + ('facebook_users','response','text'), + ('facebook_users','fb_friends_response','text'), + ('facebook_users','created','bigint'), + ('facebook_users','modified','bigint'), + ('social_settings','uid','integer'), + ('social_settings','polis_pic','character varying(3000)'), + ('facebook_friends','uid','integer'), + ('facebook_friends','friend','integer'), + ('suzinvites','owner','integer'), + ('suzinvites','zid','integer'), + ('suzinvites','created','bigint'), + ('suzinvites','suzinvite','character varying(32)'), + ('comments','tid','integer'), + ('comments','zid','integer'), + ('comments','pid','integer'), + ('comments','uid','integer'), + ('comments','created','bigint'), + ('comments','modified','bigint'), + ('comments','txt','character varying(1000)'), + ('comments','velocity','real'), + ('comments','mod','integer'), + ('comments','lang','character varying(10)'), + ('comments','lang_confidence','real'), + ('comments','active','boolean'), + ('comments','is_meta','boolean'), + ('comments','tweet_id','bigint'), + ('comments','quote_src_url','character varying(1000)'), + ('comments','anon','boolean'), + ('comments','is_seed','boolean'), + ('comment_translations','zid','integer'), + ('comment_translations','tid','integer'), + ('comment_translations','src','integer'), + ('comment_translations','txt','character varying(9999)'), + ('comment_translations','lang','character varying(10)'), + ('comment_translations','created','bigint'), + ('comment_translations','modified','bigint'), + ('conversation_translations','zid','integer'), + ('conversation_translations','src','integer'), + ('conversation_translations','topic','character varying(9999)'), + ('conversation_translations','description','character varying(9999)'), + ('conversation_translations','lang','character varying(10)'), + ('conversation_translations','created','bigint'), + ('conversation_translations','modified','bigint'), + ('reports','rid','bigint'), + ('reports','report_id','character varying(300)'), + ('reports','zid','integer'), + ('reports','created','bigint'), + ('reports','modified','bigint'), + ('reports','report_name','character varying(999)'), + ('reports','label_x_neg','character varying(999)'), + ('reports','label_x_pos','character varying(999)'), + ('reports','label_y_neg','character varying(999)'), + ('reports','label_y_pos','character varying(999)'), + ('reports','label_group_0','character varying(999)'), + ('reports','label_group_1','character varying(999)'), + ('reports','label_group_2','character varying(999)'), + ('reports','label_group_3','character varying(999)'), + ('reports','label_group_4','character varying(999)'), + ('reports','label_group_5','character varying(999)'), + ('reports','label_group_6','character varying(999)'), + ('reports','label_group_7','character varying(999)'), + ('reports','label_group_8','character varying(999)'), + ('reports','label_group_9','character varying(999)'), + ('report_comment_selections','zid','integer'), + ('report_comment_selections','rid','bigint'), + ('report_comment_selections','tid','integer'), + ('report_comment_selections','selection','smallint'), + ('report_comment_selections','modified','bigint'), + ('worker_tasks','created','bigint'), + ('worker_tasks','math_env','character varying(999)'), + ('worker_tasks','attempts','smallint'), + ('worker_tasks','task_data','jsonb'), + ('worker_tasks','task_type','text'), + ('worker_tasks','task_bucket','bigint'), + ('worker_tasks','finished_time','bigint'), + ('math_ticks','zid','integer'), + ('math_ticks','math_tick','bigint'), + ('math_ticks','math_env','character varying(999)'), + ('math_ticks','modified','bigint'), + ('math_main','zid','integer'), + ('math_main','math_env','character varying(999)'), + ('math_main','data','jsonb'), + ('math_main','last_vote_timestamp','bigint'), + ('math_main','caching_tick','bigint'), + ('math_main','math_tick','bigint'), + ('math_main','modified','bigint'), + ('math_profile','zid','integer'), + ('math_profile','math_env','character varying(999)'), + ('math_profile','data','jsonb'), + ('math_profile','modified','bigint'), + ('math_ptptstats','zid','integer'), + ('math_ptptstats','math_env','character varying(999)'), + ('math_ptptstats','math_tick','bigint'), + ('math_ptptstats','data','jsonb'), + ('math_ptptstats','modified','bigint'), + ('math_cache','zid','integer'), + ('math_cache','math_env','character varying(999)'), + ('math_cache','data','jsonb'), + ('math_cache','modified','bigint'), + ('math_bidtopid','zid','integer'), + ('math_bidtopid','math_env','character varying(999)'), + ('math_bidtopid','math_tick','bigint'), + ('math_bidtopid','data','jsonb'), + ('math_bidtopid','modified','bigint'), + ('math_exportstatus','zid','integer'), + ('math_exportstatus','math_env','character varying(999)'), + ('math_exportstatus','filename','character varying(9999)'), + ('math_exportstatus','data','jsonb'), + ('math_exportstatus','modified','bigint'), + ('math_report_correlationmatrix','rid','bigint'), + ('math_report_correlationmatrix','math_env','character varying(999)'), + ('math_report_correlationmatrix','data','jsonb'), + ('math_report_correlationmatrix','math_tick','bigint'), + ('math_report_correlationmatrix','modified','bigint'), + ('votes','zid','integer'), + ('votes','pid','integer'), + ('votes','tid','integer'), + ('votes','vote','smallint'), + ('votes','weight_x_32767','smallint'), + ('votes','created','bigint'), + ('votes_latest_unique','zid','integer'), + ('votes_latest_unique','pid','integer'), + ('votes_latest_unique','tid','integer'), + ('votes_latest_unique','vote','smallint'), + ('votes_latest_unique','weight_x_32767','smallint'), + ('votes_latest_unique','modified','bigint'), + ('crowd_mod','zid','integer'), + ('crowd_mod','pid','integer'), + ('crowd_mod','tid','integer'), + ('crowd_mod','created','bigint'), + ('crowd_mod','as_important','boolean'), + ('crowd_mod','as_factual','boolean'), + ('crowd_mod','as_feeling','boolean'), + ('crowd_mod','as_notmyfeeling','boolean'), + ('crowd_mod','as_notgoodidea','boolean'), + ('crowd_mod','as_notfact','boolean'), + ('crowd_mod','as_unsure','boolean'), + ('crowd_mod','as_spam','boolean'), + ('crowd_mod','as_abusive','boolean'), + ('crowd_mod','as_offtopic','boolean'), + ('event_ptpt_no_more_comments','zid','integer'), + ('event_ptpt_no_more_comments','pid','integer'), + ('event_ptpt_no_more_comments','votes_placed','smallint'), + ('event_ptpt_no_more_comments','created','bigint'), + ('stars','zid','integer'), + ('stars','pid','integer'), + ('stars','tid','integer'), + ('stars','starred','integer'), + ('stars','created','bigint'), + ('trashes','zid','integer'), + ('trashes','pid','integer'), + ('trashes','tid','integer'), + ('trashes','trashed','integer'), + ('trashes','created','bigint'), + ('permanentcookiezidjoins','zid','integer'), + ('permanentcookiezidjoins','cookie','character varying(32)'), + ('permanentcookiezidjoins','created','bigint'), + ('page_ids','site_id','character varying(100)'), + ('page_ids','page_id','character varying(100)'), + ('page_ids','zid','integer'), + ('demographic_data','uid','integer'), + ('demographic_data','fb_gender','integer'), + ('demographic_data','ms_birth_year_estimate_fb','integer'), + ('demographic_data','ms_gender_estimate_fb','integer'), + ('demographic_data','fb_timestamp','bigint'), + ('demographic_data','ms_fb_timestamp','bigint'), + ('demographic_data','ms_response','character varying(9999)'), + ('demographic_data','gender_guess','integer'), + ('demographic_data','birth_year_guess','integer')) AS expected(t,c,typ) WHERE NOT pg_temp.col(t,c,typ)) + AND NOT EXISTS (SELECT 1 FROM unnest(ARRAY['users','site_domain_whitelist','metrics','auth_tokens','jianiuevyew','apikeysndvweifu','courses','conversations','participant_metadata_questions','participant_metadata_answers','contexts','inviters','upvotes','oinvites','einvites','email_validations','zinvites','beta','participants','participants_extended','participant_locations','xids','xid_whitelist','notification_tasks','participant_metadata_choices','twitter_users','facebook_users','social_settings','facebook_friends','suzinvites','comments','comment_translations','conversation_translations','reports','report_comment_selections','worker_tasks','math_ticks','math_main','math_profile','math_ptptstats','math_cache','math_bidtopid','math_exportstatus','math_report_correlationmatrix','votes','votes_latest_unique','crowd_mod','event_ptpt_no_more_comments','stars','trashes','permanentcookiezidjoins','page_ids','demographic_data']) t WHERE to_regclass('public.'||t) IS NULL) + AND (pg_temp.col('pwreset_tokens','token','character varying(100)') OR pg_temp.col('password_reset_tokens','pwresettoken','character varying(100)')) + AND (pg_temp.absent_column('conversations','branding_type') OR pg_temp.col('conversations','branding_type','integer')) + AND (pg_temp.absent_column('math_ticks','caching_tick') OR pg_temp.col('math_ticks','caching_tick','bigint')) + AND to_regprocedure('public.now_as_millis()') IS NOT NULL + AND pg_temp.con('votes_latest_unique','UNIQUE (zid, pid, tid)'); diff --git a/server/postgres/migrations/adoption/000001_update_pwreset_table.sql b/server/postgres/migrations/adoption/000001_update_pwreset_table.sql new file mode 100644 index 0000000000..7752542f42 --- /dev/null +++ b/server/postgres/migrations/adoption/000001_update_pwreset_table.sql @@ -0,0 +1,4 @@ +-- Catalog postconditions only; this file never replays migration DDL. +SELECT to_regclass('public.password_reset_tokens') IS NULL + AND pg_temp.col('pwreset_tokens','token','character varying(100)') + AND pg_temp.absent_column('pwreset_tokens','pwresettoken'); diff --git a/server/postgres/migrations/adoption/000002_add_xid_constraint.sql b/server/postgres/migrations/adoption/000002_add_xid_constraint.sql new file mode 100644 index 0000000000..41e11d827e --- /dev/null +++ b/server/postgres/migrations/adoption/000002_add_xid_constraint.sql @@ -0,0 +1,14 @@ +-- Both the public bootstrap variant (owner,xid only) and the authoritative +-- legacy variant (also owner,uid) are supported. Never drop either invariant. +SELECT pg_temp.con_exact('xids','UNIQUE (owner, xid)') + -- If the optional legacy unique index exists, it must enforce the complete key. + AND NOT EXISTS ( + SELECT 1 FROM pg_index i + JOIN pg_attribute owner ON owner.attrelid=i.indrelid AND owner.attname='owner' + JOIN pg_attribute uid ON uid.attrelid=i.indrelid AND uid.attname='uid' + WHERE i.indrelid=to_regclass('public.xids') AND i.indisunique + AND i.indnkeyatts=2 + AND ARRAY[i.indkey[0],i.indkey[1]] @> ARRAY[owner.attnum,uid.attnum] + AND (NOT i.indisvalid OR NOT i.indisready OR NOT i.indislive + OR NOT i.indimmediate OR i.indpred IS NOT NULL) + ); diff --git a/server/postgres/migrations/adoption/000003_add_origin_permanent_cookie_columns.sql b/server/postgres/migrations/adoption/000003_add_origin_permanent_cookie_columns.sql new file mode 100644 index 0000000000..4f0a5cee75 --- /dev/null +++ b/server/postgres/migrations/adoption/000003_add_origin_permanent_cookie_columns.sql @@ -0,0 +1,3 @@ +-- Catalog postconditions only; this file never replays migration DDL. +SELECT pg_temp.col_exact('participants_extended','permanent_cookie','character varying(32)',false) + AND pg_temp.col_exact('participants_extended','origin','character varying(9999)',false); diff --git a/server/postgres/migrations/adoption/000004_drop_waitinglist_table.sql b/server/postgres/migrations/adoption/000004_drop_waitinglist_table.sql new file mode 100644 index 0000000000..61fac26e19 --- /dev/null +++ b/server/postgres/migrations/adoption/000004_drop_waitinglist_table.sql @@ -0,0 +1,2 @@ +-- Catalog postconditions only; this file never replays migration DDL. +SELECT to_regclass('public.waitinglist') IS NULL; diff --git a/server/postgres/migrations/adoption/000005_drop_slack_stripe_canvas.sql b/server/postgres/migrations/adoption/000005_drop_slack_stripe_canvas.sql new file mode 100644 index 0000000000..f0db14fc7d --- /dev/null +++ b/server/postgres/migrations/adoption/000005_drop_slack_stripe_canvas.sql @@ -0,0 +1,16 @@ +-- Catalog postconditions only; this file never replays migration DDL. +SELECT to_regclass('public.slack_oauth_access_tokens') IS NULL + AND to_regclass('public.slack_users') IS NULL + AND to_regclass('public.slack_user_invites') IS NULL + AND to_regclass('public.slack_bot_events') IS NULL + AND to_regclass('public.stripe_accounts') IS NULL + AND to_regclass('public.stripe_subscriptions') IS NULL + AND to_regclass('public.coupons_for_free_upgrades') IS NULL + AND to_regclass('public.lti_users') IS NULL + AND to_regclass('public.lti_context_memberships') IS NULL + AND to_regclass('public.canvas_assignment_callback_info') IS NULL + AND to_regclass('public.canvas_assignment_conversation_info') IS NULL + AND to_regclass('public.lti_oauthv1_credentials') IS NULL + AND pg_temp.absent_column('conversations','is_slack') + AND pg_temp.absent_column('conversations','lti_users_only') + AND pg_temp.absent_column('users','plan'); diff --git a/server/postgres/migrations/adoption/000006_update_votes_rule.sql b/server/postgres/migrations/adoption/000006_update_votes_rule.sql new file mode 100644 index 0000000000..b79f8b8243 --- /dev/null +++ b/server/postgres/migrations/adoption/000006_update_votes_rule.sql @@ -0,0 +1,2 @@ +-- Catalog postconditions only; this file never replays migration DDL. +SELECT EXISTS (SELECT 1 FROM pg_rewrite WHERE ev_class=to_regclass('public.votes') AND rulename='on_vote_insert_update_unique_table' AND ev_enabled='O' AND regexp_replace(pg_get_ruledef(oid), '\s+', ' ', 'g') = 'CREATE RULE on_vote_insert_update_unique_table AS ON INSERT TO public.votes DO INSERT INTO votes_latest_unique (zid, pid, tid, vote, weight_x_32767, modified) VALUES (new.zid, new.pid, new.tid, new.vote, new.weight_x_32767, new.created) ON CONFLICT(zid, pid, tid) DO UPDATE SET vote = excluded.vote, modified = excluded.modified;'); diff --git a/server/postgres/migrations/adoption/000007_drop_geolocation_fields.sql b/server/postgres/migrations/adoption/000007_drop_geolocation_fields.sql new file mode 100644 index 0000000000..c8f44df378 --- /dev/null +++ b/server/postgres/migrations/adoption/000007_drop_geolocation_fields.sql @@ -0,0 +1,9 @@ +-- Catalog postconditions only; this file never replays migration DDL. +SELECT to_regclass('public.geolocation_cache') IS NULL + AND pg_temp.absent_column('participants_extended','country_code_iso') + AND pg_temp.absent_column('participants_extended','encrypted_maxmind_response_city') + AND pg_temp.absent_column('participants_extended','ip_address') + AND pg_temp.absent_column('participants_extended','latitude') + AND pg_temp.absent_column('participants_extended','location') + AND pg_temp.absent_column('participants_extended','longitude') + AND pg_temp.absent_column('participants_extended','x_forwarded_for'); diff --git a/server/postgres/migrations/adoption/000008_add_comment_priority.sql b/server/postgres/migrations/adoption/000008_add_comment_priority.sql new file mode 100644 index 0000000000..11072158e0 --- /dev/null +++ b/server/postgres/migrations/adoption/000008_add_comment_priority.sql @@ -0,0 +1,3 @@ +-- Catalog postconditions only; this file never replays migration DDL. +SELECT pg_temp.col_exact('conversations','importance_enabled','boolean',true,'false') + AND pg_temp.col_exact('votes','high_priority','boolean',true,'false'); diff --git a/server/postgres/migrations/adoption/000009_add_uuid_to_zinvites.sql b/server/postgres/migrations/adoption/000009_add_uuid_to_zinvites.sql new file mode 100644 index 0000000000..adf006212d --- /dev/null +++ b/server/postgres/migrations/adoption/000009_add_uuid_to_zinvites.sql @@ -0,0 +1,2 @@ +-- Catalog postconditions only; this file never replays migration DDL. +SELECT pg_temp.col_exact('zinvites','uuid','uuid',false); diff --git a/server/postgres/migrations/adoption/000010_create_oidc_user_mappings.sql b/server/postgres/migrations/adoption/000010_create_oidc_user_mappings.sql new file mode 100644 index 0000000000..6ca36bcdb7 --- /dev/null +++ b/server/postgres/migrations/adoption/000010_create_oidc_user_mappings.sql @@ -0,0 +1,8 @@ +-- Catalog postconditions only; this file never replays migration DDL. +SELECT pg_temp.col_exact('oidc_user_mappings','oidc_sub','character varying(255)',true) + AND pg_temp.col_exact('oidc_user_mappings','uid','integer',true) + AND pg_temp.col_exact('oidc_user_mappings','created','bigint',false,'now_as_millis()') + AND pg_temp.con_exact('oidc_user_mappings','PRIMARY KEY (oidc_sub)') + AND pg_temp.con_exact('oidc_user_mappings','UNIQUE (uid)') + AND pg_temp.con_exact('oidc_user_mappings','FOREIGN KEY (uid) REFERENCES users(uid) ON DELETE CASCADE') + AND pg_temp.idx('idx_oidc_mappings_uid','CREATE INDEX idx_oidc_mappings_uid ON public.oidc_user_mappings USING btree (uid)'); diff --git a/server/postgres/migrations/adoption/000011_alter_suzinvites_xid_to_text.sql b/server/postgres/migrations/adoption/000011_alter_suzinvites_xid_to_text.sql new file mode 100644 index 0000000000..5235d98d42 --- /dev/null +++ b/server/postgres/migrations/adoption/000011_alter_suzinvites_xid_to_text.sql @@ -0,0 +1,2 @@ +-- Catalog postconditions only; this file never replays migration DDL. +SELECT pg_temp.col('suzinvites','xid','text',true); diff --git a/server/postgres/migrations/adoption/000012_create_topic_agenda_selections.sql b/server/postgres/migrations/adoption/000012_create_topic_agenda_selections.sql new file mode 100644 index 0000000000..62832a1743 --- /dev/null +++ b/server/postgres/migrations/adoption/000012_create_topic_agenda_selections.sql @@ -0,0 +1,15 @@ +-- Catalog postconditions only; this file never replays migration DDL. +SELECT pg_temp.col_exact('topic_agenda_selections','zid','integer',true) + AND pg_temp.col_exact('topic_agenda_selections','pid','integer',true) + AND pg_temp.col_exact('topic_agenda_selections','archetypal_selections','jsonb',true,'''[]''::jsonb') + AND pg_temp.col_exact('topic_agenda_selections','delphi_job_id','text',false) + AND pg_temp.col_exact('topic_agenda_selections','total_selections','integer',true,'0') + AND pg_temp.col_exact('topic_agenda_selections','created_at','timestamp with time zone',false,'CURRENT_TIMESTAMP') + AND pg_temp.col_exact('topic_agenda_selections','updated_at','timestamp with time zone',false,'CURRENT_TIMESTAMP') + AND pg_temp.con_exact('topic_agenda_selections','PRIMARY KEY (zid, pid)') + AND pg_temp.con_exact('topic_agenda_selections','FOREIGN KEY (zid) REFERENCES conversations(zid) ON DELETE CASCADE') + AND pg_temp.con_exact('topic_agenda_selections','FOREIGN KEY (zid, pid) REFERENCES participants(zid, pid) ON DELETE CASCADE') + AND pg_temp.idx('idx_topic_agenda_selections_zid','CREATE INDEX idx_topic_agenda_selections_zid ON public.topic_agenda_selections USING btree (zid)') + AND pg_temp.idx('idx_topic_agenda_selections_pid','CREATE INDEX idx_topic_agenda_selections_pid ON public.topic_agenda_selections USING btree (pid)') + AND pg_temp.idx('idx_topic_agenda_selections_delphi_job_id','CREATE INDEX idx_topic_agenda_selections_delphi_job_id ON public.topic_agenda_selections USING btree (delphi_job_id)') + AND pg_temp.idx('idx_topic_agenda_selections_created_at','CREATE INDEX idx_topic_agenda_selections_created_at ON public.topic_agenda_selections USING btree (created_at)'); diff --git a/server/postgres/migrations/adoption/000013_create_treevite.sql b/server/postgres/migrations/adoption/000013_create_treevite.sql new file mode 100644 index 0000000000..613b2b764f --- /dev/null +++ b/server/postgres/migrations/adoption/000013_create_treevite.sql @@ -0,0 +1,70 @@ +-- Catalog postconditions only; this file never replays migration DDL. +SELECT pg_temp.col_exact('conversations','treevite_enabled','boolean',false,'false') + AND pg_temp.col_exact('treevite_waves','id','bigint',true,'nextval(''treevite_waves_id_seq''::regclass)') + AND pg_temp.col_exact('treevite_waves','zid','integer',true) + AND pg_temp.col_exact('treevite_waves','wave','integer',true) + AND pg_temp.col_exact('treevite_waves','parent_wave','integer',false) + AND pg_temp.col_exact('treevite_waves','size','integer',false) + AND pg_temp.col_exact('treevite_waves','invites_per_user','integer',true) + AND pg_temp.col_exact('treevite_waves','owner_invites','integer',true,'0') + AND pg_temp.col_exact('treevite_waves','created_at','timestamp with time zone',false,'CURRENT_TIMESTAMP') + AND pg_temp.col_exact('treevite_waves','updated_at','timestamp with time zone',false,'CURRENT_TIMESTAMP') + AND pg_temp.con_exact('treevite_waves','PRIMARY KEY (id)') + AND pg_temp.col_exact('treevite_invites','id','bigint',true,'nextval(''treevite_invites_id_seq''::regclass)') + AND pg_temp.col_exact('treevite_invites','zid','integer',true) + AND pg_temp.col_exact('treevite_invites','wave_id','bigint',true) + AND pg_temp.col_exact('treevite_invites','parent_invite_id','bigint',false) + AND pg_temp.col_exact('treevite_invites','status','smallint',true,'0') + AND pg_temp.col_exact('treevite_invites','invite_owner_pid','integer',false) + AND pg_temp.col_exact('treevite_invites','invite_used_by_pid','integer',false) + AND pg_temp.col_exact('treevite_invites','invite_used_at','timestamp with time zone',false) + AND pg_temp.col_exact('treevite_invites','created_at','timestamp with time zone',false,'CURRENT_TIMESTAMP') + AND pg_temp.col_exact('treevite_invites','updated_at','timestamp with time zone',false,'CURRENT_TIMESTAMP') + AND pg_temp.con_exact('treevite_invites','PRIMARY KEY (id)') + AND pg_temp.col_exact('treevite_login_codes','id','bigint',true,'nextval(''treevite_login_codes_id_seq''::regclass)') + AND pg_temp.col_exact('treevite_login_codes','zid','integer',true) + AND pg_temp.col_exact('treevite_login_codes','pid','integer',true) + AND pg_temp.col_exact('treevite_login_codes','login_code_hash','text',true) + AND pg_temp.col_exact('treevite_login_codes','fp_kid','smallint',true,'1') + AND pg_temp.col_exact('treevite_login_codes','revoked','boolean',true,'false') + AND pg_temp.col_exact('treevite_login_codes','expires_at','timestamp with time zone',false) + AND pg_temp.col_exact('treevite_login_codes','last_used_at','timestamp with time zone',false) + AND pg_temp.col_exact('treevite_login_codes','created_at','timestamp with time zone',false,'CURRENT_TIMESTAMP') + AND pg_temp.col_exact('treevite_login_codes','updated_at','timestamp with time zone',false,'CURRENT_TIMESTAMP') + AND pg_temp.con_exact('treevite_login_codes','PRIMARY KEY (id)') + AND pg_temp.con_exact('treevite_waves','UNIQUE (zid, wave)') + AND pg_temp.con_exact('treevite_waves','FOREIGN KEY (zid) REFERENCES conversations(zid) ON DELETE CASCADE') + AND pg_temp.con_exact('treevite_invites','UNIQUE (zid, invite_code)') + AND pg_temp.con_exact('treevite_invites','FOREIGN KEY (zid) REFERENCES conversations(zid) ON DELETE CASCADE') + AND pg_temp.con_exact('treevite_invites','FOREIGN KEY (wave_id) REFERENCES treevite_waves(id) ON DELETE CASCADE') + AND pg_temp.con_exact('treevite_invites','FOREIGN KEY (parent_invite_id) REFERENCES treevite_invites(id) ON DELETE SET NULL') + AND pg_temp.con_exact('treevite_invites','FOREIGN KEY (zid, invite_owner_pid) REFERENCES participants(zid, pid)') + AND pg_temp.con_exact('treevite_invites','FOREIGN KEY (zid, invite_used_by_pid) REFERENCES participants(zid, pid)') + AND pg_temp.con_exact('treevite_login_codes','FOREIGN KEY (zid, pid) REFERENCES participants(zid, pid) ON DELETE CASCADE') + AND pg_temp.con_exact('treevite_login_codes','UNIQUE (zid, pid)') + AND pg_temp.con_exact('treevite_login_codes','UNIQUE (zid, login_code_fingerprint)') + AND pg_temp.con_exact('treevite_login_codes','UNIQUE (zid, login_code_lookup)') + AND pg_temp.idx('idx_treevite_waves_zid','CREATE INDEX idx_treevite_waves_zid ON public.treevite_waves USING btree (zid)') + AND pg_temp.idx('idx_treevite_waves_wave','CREATE INDEX idx_treevite_waves_wave ON public.treevite_waves USING btree (wave)') + AND pg_temp.idx('idx_treevite_waves_parent','CREATE INDEX idx_treevite_waves_parent ON public.treevite_waves USING btree (zid, parent_wave)') + AND pg_temp.idx('idx_treevite_invites_zid','CREATE INDEX idx_treevite_invites_zid ON public.treevite_invites USING btree (zid)') + AND pg_temp.idx('idx_treevite_invites_zid_status','CREATE INDEX idx_treevite_invites_zid_status ON public.treevite_invites USING btree (zid, status)') + AND pg_temp.idx('idx_treevite_invites_wave_id','CREATE INDEX idx_treevite_invites_wave_id ON public.treevite_invites USING btree (wave_id)') + AND pg_temp.idx('idx_treevite_invites_parent','CREATE INDEX idx_treevite_invites_parent ON public.treevite_invites USING btree (parent_invite_id)') + AND pg_temp.idx('idx_treevite_invites_owner_pid','CREATE INDEX idx_treevite_invites_owner_pid ON public.treevite_invites USING btree (invite_owner_pid)') + AND pg_temp.idx('idx_treevite_invites_used_by_pid','CREATE INDEX idx_treevite_invites_used_by_pid ON public.treevite_invites USING btree (invite_used_by_pid)') + AND pg_temp.idx('idx_treevite_invites_code','CREATE INDEX idx_treevite_invites_code ON public.treevite_invites USING btree (invite_code)') + AND pg_temp.idx('idx_treevite_login_codes_zid','CREATE INDEX idx_treevite_login_codes_zid ON public.treevite_login_codes USING btree (zid)') + AND pg_temp.idx('idx_treevite_login_codes_pid','CREATE INDEX idx_treevite_login_codes_pid ON public.treevite_login_codes USING btree (pid)') + AND pg_temp.idx('idx_treevite_login_codes_fp','CREATE INDEX idx_treevite_login_codes_fp ON public.treevite_login_codes USING btree (login_code_fingerprint)') + AND pg_temp.idx('idx_treevite_login_codes_lookup','CREATE INDEX idx_treevite_login_codes_lookup ON public.treevite_login_codes USING btree (zid, login_code_lookup)') + AND pg_temp.col_exact('treevite_invites','invite_code','character varying(64)',true) + AND pg_temp.col_exact('treevite_login_codes','login_code_fingerprint','character varying(128)',true) + AND pg_temp.col_exact('treevite_login_codes','login_code_lookup','character varying(128)',false) + AND pg_temp.con_exact('treevite_waves','CHECK (((invites_per_user > 0) OR (owner_invites > 0)))') + AND pg_temp.con_exact('treevite_waves','CHECK (((parent_wave IS NULL) OR (parent_wave >= 0)))') + AND pg_temp.con_exact('treevite_waves','CHECK (((size IS NULL) OR (size >= 0)))') + AND pg_temp.con_exact('treevite_waves','CHECK ((invites_per_user >= 0))') + AND pg_temp.con_exact('treevite_waves','CHECK ((owner_invites >= 0))') + AND pg_temp.con_exact('treevite_waves','CHECK ((wave >= 1))') + AND pg_temp.con_exact('treevite_invites','CHECK ((status = ANY (ARRAY[0, 1, 2, 3])))'); diff --git a/server/postgres/migrations/adoption/000014_alter_reports_modlevel.sql b/server/postgres/migrations/adoption/000014_alter_reports_modlevel.sql new file mode 100644 index 0000000000..762f142a17 --- /dev/null +++ b/server/postgres/migrations/adoption/000014_alter_reports_modlevel.sql @@ -0,0 +1,2 @@ +-- Catalog postconditions only; this file never replays migration DDL. +SELECT pg_temp.col_exact('reports','mod_level','smallint',true,'''-2''::integer'); diff --git a/server/postgres/migrations/adoption/000015_add_xid_requirements.sql b/server/postgres/migrations/adoption/000015_add_xid_requirements.sql new file mode 100644 index 0000000000..584bf1cca0 --- /dev/null +++ b/server/postgres/migrations/adoption/000015_add_xid_requirements.sql @@ -0,0 +1,15 @@ +-- Catalog postconditions only; this file never replays migration DDL. +SELECT pg_temp.col_exact('conversations','xid_required','boolean',true,'false') + AND pg_temp.col_exact('xid_whitelist','zid','integer',false) + AND pg_temp.col_exact('xids','zid','integer',false) + AND pg_temp.col_exact('xids','pid','integer',false) + AND pg_temp.con_exact('xid_whitelist','FOREIGN KEY (zid) REFERENCES conversations(zid) ON DELETE CASCADE') + AND pg_temp.con_exact('xids','FOREIGN KEY (zid) REFERENCES conversations(zid) ON DELETE CASCADE') + AND pg_temp.con_exact('xids','FOREIGN KEY (zid, pid) REFERENCES participants(zid, pid) ON DELETE SET NULL') + AND pg_temp.idx('idx_xid_whitelist_zid','CREATE INDEX idx_xid_whitelist_zid ON public.xid_whitelist USING btree (zid)') + AND pg_temp.idx('idx_xid_whitelist_xid','CREATE INDEX idx_xid_whitelist_xid ON public.xid_whitelist USING btree (xid)') + AND pg_temp.idx('idx_xids_zid','CREATE INDEX idx_xids_zid ON public.xids USING btree (zid)') + AND pg_temp.idx('idx_xids_xid','CREATE INDEX idx_xids_xid ON public.xids USING btree (xid)') + AND pg_temp.idx('idx_xids_pid','CREATE INDEX idx_xids_pid ON public.xids USING btree (pid)') + AND pg_temp.idx('idx_xids_zid_xid','CREATE INDEX idx_xids_zid_xid ON public.xids USING btree (zid, xid)') + AND pg_temp.idx('idx_xids_uid_zid','CREATE INDEX idx_xids_uid_zid ON public.xids USING btree (uid, zid)'); diff --git a/server/postgres/migrations/adoption/000016_add_orig_id.sql b/server/postgres/migrations/adoption/000016_add_orig_id.sql new file mode 100644 index 0000000000..c864197137 --- /dev/null +++ b/server/postgres/migrations/adoption/000016_add_orig_id.sql @@ -0,0 +1,2 @@ +-- Catalog postconditions only; this file never replays migration DDL. +SELECT pg_temp.col_exact('comments','original_id','uuid',false); diff --git a/server/postgres/migrations/adoption/000017_create_byod_job_table.sql b/server/postgres/migrations/adoption/000017_create_byod_job_table.sql new file mode 100644 index 0000000000..43ed6a91db --- /dev/null +++ b/server/postgres/migrations/adoption/000017_create_byod_job_table.sql @@ -0,0 +1,12 @@ +-- Catalog postconditions only; this file never replays migration DDL. +SELECT (SELECT array_agg(enumlabel::text ORDER BY enumsortorder) FROM pg_enum WHERE enumtypid=to_regtype('public.job_status')) = ARRAY['pending','processing','completed','failed'] + AND pg_temp.col_exact('byod_import_jobs','id','integer',true,'nextval(''byod_import_jobs_id_seq''::regclass)') + AND pg_temp.col_exact('byod_import_jobs','zid','integer',true) + AND pg_temp.col_exact('byod_import_jobs','s3_key','text',true) + AND pg_temp.col_exact('byod_import_jobs','status','job_status',false,'''pending''::job_status') + AND pg_temp.col_exact('byod_import_jobs','stage','text',false,'''init''::text') + AND pg_temp.col_exact('byod_import_jobs','error_message','text',false) + AND pg_temp.col_exact('byod_import_jobs','created_at','timestamp with time zone',false,'now()') + AND pg_temp.col_exact('byod_import_jobs','updated_at','timestamp with time zone',false,'now()') + AND pg_temp.con_exact('byod_import_jobs','PRIMARY KEY (id)') + AND pg_temp.idx('idx_byod_jobs_zid','CREATE INDEX idx_byod_jobs_zid ON public.byod_import_jobs USING btree (zid)'); diff --git a/server/postgres/migrations/adoption/000018_add_topics_enabled.sql b/server/postgres/migrations/adoption/000018_add_topics_enabled.sql new file mode 100644 index 0000000000..5a9299078b --- /dev/null +++ b/server/postgres/migrations/adoption/000018_add_topics_enabled.sql @@ -0,0 +1,2 @@ +-- Catalog postconditions only; this file never replays migration DDL. +SELECT pg_temp.col_exact('conversations','topics_enabled','boolean',true,'false'); diff --git a/server/postgres/migrations/adoption/000022_add_poll_timestamp_indexes.sql b/server/postgres/migrations/adoption/000022_add_poll_timestamp_indexes.sql new file mode 100644 index 0000000000..8ba522e217 --- /dev/null +++ b/server/postgres/migrations/adoption/000022_add_poll_timestamp_indexes.sql @@ -0,0 +1,3 @@ +-- Catalog postconditions only; this file never replays migration DDL. +SELECT pg_temp.idx('votes_created_idx','CREATE INDEX votes_created_idx ON public.votes USING btree (created)') + AND pg_temp.idx('comments_modified_idx','CREATE INDEX comments_modified_idx ON public.comments USING btree (modified)'); diff --git a/server/postgres/migrations/adoption/helpers.sql b/server/postgres/migrations/adoption/helpers.sql new file mode 100644 index 0000000000..de607fd80b --- /dev/null +++ b/server/postgres/migrations/adoption/helpers.sql @@ -0,0 +1,59 @@ +-- Reconciliation only. These pg_temp helpers vanish when the connection closes. +-- A nullability/default argument checks only when specified by that migration. +CREATE FUNCTION pg_temp.col(t text,c text,typ text,nn boolean DEFAULT NULL,def text DEFAULT NULL) +RETURNS boolean LANGUAGE sql AS $$ + SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||t) AND a.attname=c AND NOT a.attisdropped + AND (format_type(a.atttypid,a.atttypmod)=typ + -- Exact public bootstrap alternatives to the authoritative legacy contract. + OR (t='worker_tasks' AND c='task_type' AND typ='text' + AND format_type(a.atttypid,a.atttypmod)='character varying(99)') + OR (t IN ('pwreset_tokens','password_reset_tokens') + AND c IN ('token','pwresettoken') AND typ='character varying(100)' + AND format_type(a.atttypid,a.atttypmod)='character varying(250)') + -- Only historical math payloads have an established json equivalent. + -- Do not adopt arbitrary json columns in newer jsonb contracts. + OR (typ='jsonb' AND a.atttypid='json'::regtype AND c='data' + AND t IN ('math_main','math_profile','math_ptptstats','math_cache', + 'math_bidtopid','math_exportstatus'))) + AND (nn IS NULL OR a.attnotnull=nn) + AND (def IS NULL OR pg_get_expr(d.adbin,d.adrelid)=def)); +$$; +CREATE FUNCTION pg_temp.con(t text,definition text) RETURNS boolean LANGUAGE sql AS $$ + SELECT EXISTS(SELECT 1 FROM pg_constraint WHERE conrelid=to_regclass('public.'||t) + AND convalidated AND pg_get_constraintdef(oid)=definition); +$$; +CREATE FUNCTION pg_temp.idx(n text,definition text) RETURNS boolean LANGUAGE sql AS $$ + SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||n) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=definition); +$$; +CREATE FUNCTION pg_temp.absent_column(t text,c text) RETURNS boolean LANGUAGE sql AS $$ + SELECT to_regclass('public.'||t) IS NOT NULL AND NOT EXISTS(SELECT 1 FROM pg_attribute + WHERE attrelid=to_regclass('public.'||t) AND attname=c AND NOT attisdropped); +$$; + +-- Full contracts for newly created columns. NULL means no default, not a wildcard. +-- Keep col() unchanged for legacy/core and ALTER TYPE-only postconditions. +CREATE FUNCTION pg_temp.col_exact(t text,c text,typ text,nn boolean,def text DEFAULT NULL) +RETURNS boolean LANGUAGE sql AS $$ + SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||t) AND a.attname=c AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=typ AND a.attnotnull=nn + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM def); +$$; +-- Definition checks include actions/keys; catalog flags also require enforcement. +-- Preserve the legacy helper for contracts awaiting a separate policy decision. +CREATE FUNCTION pg_temp.con_exact(t text,definition text) RETURNS boolean LANGUAGE sql AS $$ + SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||t) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=definition + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O')); +$$; diff --git a/server/postgres/migrations/down/000027-files.sha256 b/server/postgres/migrations/down/000027-files.sha256 new file mode 100644 index 0000000000..9b80b8fc3c --- /dev/null +++ b/server/postgres/migrations/down/000027-files.sha256 @@ -0,0 +1,2 @@ +fbbf948e4316010dd96344ae91542006c38299e52e0a0eb281371c039ab37775 000027_create_sealed_job_graphs.sql +cb2c16f421329a8f049d3d66a0ddbb7c76abaa9f7d2cae049642a98677694170 down/000027_drop_sealed_job_graphs.sql diff --git a/server/postgres/migrations/down/000027_drop_sealed_job_graphs.sql b/server/postgres/migrations/down/000027_drop_sealed_job_graphs.sql new file mode 100644 index 0000000000..56de2ebb5f --- /dev/null +++ b/server/postgres/migrations/down/000027_drop_sealed_job_graphs.sql @@ -0,0 +1,62 @@ +-- Empty draft reversal only. Runner history is never edited here. +BEGIN; +SET LOCAL ROLE polis_queue_owner; +SET LOCAL search_path=pg_catalog,pg_temp; +SET LOCAL lock_timeout='5s'; +CREATE OR REPLACE FUNCTION pg_temp.pq_catalog(p_table oid) RETURNS jsonb +LANGUAGE sql SET search_path=pg_catalog,pg_temp AS $catalog$ +SELECT jsonb_build_object( + 'columns',(SELECT jsonb_agg(jsonb_build_array(a.attname,format_type(a.atttypid,a.atttypmod),a.attnotnull,a.attidentity,a.attgenerated,co.collname,pg_get_expr(d.adbin,d.adrelid),NULLIF(a.attacl::text,'{}')) ORDER BY a.attnum) + FROM pg_attribute a LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum LEFT JOIN pg_collation co ON co.oid=a.attcollation + WHERE a.attrelid=c.oid AND a.attnum>0 AND NOT a.attisdropped), + 'constraints',(SELECT jsonb_agg(jsonb_build_array(conname,pg_get_constraintdef(oid),convalidated,connoinherit) ORDER BY conname) FROM pg_constraint WHERE conrelid=c.oid), + 'indexes',(SELECT jsonb_agg(jsonb_build_array(ic.relname,pg_get_indexdef(i.indexrelid),i.indisvalid,i.indisready) ORDER BY ic.relname) FROM pg_index i JOIN pg_class ic ON ic.oid=i.indexrelid WHERE i.indrelid=c.oid), + 'triggers',(SELECT jsonb_agg(jsonb_build_array(t.tgname,pg_get_triggerdef(t.oid),t.tgenabled) ORDER BY t.tgname) FROM pg_trigger t WHERE t.tgrelid=c.oid AND NOT t.tgisinternal), + 'owner',pg_get_userbyid(c.relowner),'kind',c.relkind,'rls',c.relrowsecurity,'force_rls',c.relforcerowsecurity,'options',c.reloptions, + 'acl',(SELECT jsonb_agg(jsonb_build_array(CASE WHEN x.grantee=0 THEN 'PUBLIC' ELSE pg_get_userbyid(x.grantee) END,x.privilege_type,x.is_grantable) ORDER BY x.grantee=0,pg_get_userbyid(x.grantee),x.privilege_type,x.is_grantable) FROM aclexplode(COALESCE(c.relacl,acldefault('r',c.relowner))) x) + ) FROM pg_class c WHERE c.oid=p_table +$catalog$; +CREATE OR REPLACE FUNCTION pg_temp.pd_state() RETURNS jsonb LANGUAGE sql SET search_path=pg_catalog,pg_temp AS $s$ +SELECT jsonb_build_object('tables',(SELECT jsonb_object_agg(c.relname,pg_temp.pq_catalog(c.oid)) FROM pg_class c + WHERE c.relnamespace='public'::regnamespace AND c.relkind='r' AND (starts_with(c.relname,'polis_queue_') OR starts_with(c.relname,'delphi_')) AND c.relname<>'delphi_foundation_install'), + 'functions',(SELECT jsonb_object_agg(p.oid::regprocedure::text,jsonb_build_array(pg_get_functiondef(p.oid),p.proacl::text,pg_get_userbyid(p.proowner))) + FROM pg_proc p WHERE p.pronamespace='public'::regnamespace AND (starts_with(p.proname,'pq_') OR starts_with(p.proname,'pd_')))) +$s$; +-- The /3 state: the same, less this file's own install table. +CREATE OR REPLACE FUNCTION pg_temp.pq3_state() RETURNS jsonb LANGUAGE sql SET search_path=pg_catalog,pg_temp AS $s$ +SELECT jsonb_build_object('tables',(SELECT jsonb_object_agg(c.relname,pg_temp.pq_catalog(c.oid)) FROM pg_class c + WHERE c.relnamespace='public'::regnamespace AND c.relkind='r' AND (starts_with(c.relname,'polis_queue_') OR starts_with(c.relname,'delphi_')) + AND c.relname NOT IN ('delphi_foundation_install','polis_queue_large_class_install')), + 'functions',(SELECT jsonb_object_agg(p.oid::regprocedure::text,jsonb_build_array(pg_get_functiondef(p.oid),p.proacl::text,pg_get_userbyid(p.proowner))) + FROM pg_proc p WHERE p.pronamespace='public'::regnamespace AND (starts_with(p.proname,'pq_') OR starts_with(p.proname,'pd_')))) +$s$; + + +DO $$ DECLARE r record; baseline jsonb; BEGIN + LOCK TABLE public.delphi_graphs IN ACCESS EXCLUSIVE MODE; + IF EXISTS(SELECT 1 FROM public.delphi_graphs) + THEN RAISE EXCEPTION 'nonempty graph contract: retain schema and use compatible workers'; END IF; + SELECT i.baseline||'{}'::jsonb INTO STRICT baseline FROM public.delphi_graph_install i; + -- Drop only draft objects. Restore original function definitions and ACLs below. + DROP TRIGGER graph_parent_guard ON public.delphi_jobs; + DROP TABLE public.delphi_graph_served,public.delphi_graph_edges,public.delphi_artifacts, + public.delphi_graph_nodes,public.delphi_graphs,public.delphi_graph_install CASCADE; + FOR r IN SELECT oid::regprocedure AS name FROM pg_proc WHERE pronamespace='public'::regnamespace AND proname LIKE 'pd_graph_%' LOOP + EXECUTE format('DROP FUNCTION %s CASCADE',r.name); + END LOOP; + ALTER TABLE public.polis_queue_install DROP CONSTRAINT polis_queue_install_contract_version_check; + UPDATE public.polis_queue_install SET contract_version='polis-queue/3'; + ALTER TABLE public.polis_queue_install ADD CONSTRAINT polis_queue_install_contract_version_check CHECK(contract_version='polis-queue/3'); + ALTER TABLE public.polis_queue_install ALTER COLUMN contract_version SET DEFAULT 'polis-queue/3'; + ALTER TABLE public.polis_queue_runs DROP CONSTRAINT polis_queue_runs_contract_version_check; + ALTER TABLE public.polis_queue_runs ADD CONSTRAINT polis_queue_runs_contract_version_check CHECK(contract_version IN ('polis-queue/1','polis-queue/2','polis-queue/3')); + ALTER TABLE public.polis_queue_jobs DROP CONSTRAINT polis_queue_jobs_stage_check; + ALTER TABLE public.polis_queue_jobs ADD CONSTRAINT polis_queue_jobs_stage_check CHECK(stage IN ('noop','delphi_full_pipeline','delphi_narrative','math_rebuild')); + ALTER TABLE public.polis_queue_jobs DROP CONSTRAINT pq_stage_large; + ALTER TABLE public.polis_queue_jobs ADD CONSTRAINT pq_stage_large CHECK((stage='math_rebuild')=(worker_class='large')); + FOR r IN SELECT key,value FROM jsonb_each(baseline->'functions') LOOP + EXECUTE r.value->>0; + END LOOP; + IF baseline IS DISTINCT FROM pg_temp.pq3_state() THEN RAISE EXCEPTION 'graph down did not restore exact /3 catalog'; END IF; +END $$; +COMMIT; diff --git a/server/postgres/migrations/held.txt b/server/postgres/migrations/held.txt new file mode 100644 index 0000000000..b8cd53cfc9 --- /dev/null +++ b/server/postgres/migrations/held.txt @@ -0,0 +1,3 @@ +# Release-wide hold: the coordinator is dormant and not an API requirement. +# Moving this file into the required chain needs an explicit schema release. +000021_create_polis_coordinator.sql diff --git a/server/postgres/migrations/release.txt b/server/postgres/migrations/release.txt new file mode 100644 index 0000000000..8b46a0c527 --- /dev/null +++ b/server/postgres/migrations/release.txt @@ -0,0 +1,26 @@ +# Selected stable runner release. Unknown numbered files fail closed. +# M4/M5/M7 are retirement receipts only; their DDL never executes. +000000_initial.sql +000001_update_pwreset_table.sql +000002_add_xid_constraint.sql +000003_add_origin_permanent_cookie_columns.sql +000004_drop_waitinglist_table.sql +000005_drop_slack_stripe_canvas.sql +000006_update_votes_rule.sql +000007_drop_geolocation_fields.sql +000008_add_comment_priority.sql +000009_add_uuid_to_zinvites.sql +000010_create_oidc_user_mappings.sql +000011_alter_suzinvites_xid_to_text.sql +000012_create_topic_agenda_selections.sql +000013_create_treevite.sql +000014_alter_reports_modlevel.sql +000015_add_xid_requirements.sql +000016_add_orig_id.sql +000017_create_byod_job_table.sql +000018_add_topics_enabled.sql +000019_create_polis_queue.sql +000022_add_poll_timestamp_indexes.sql +000023_create_delphi_foundation.sql +000024_create_polis_queue_large_class.sql +000027_create_sealed_job_graphs.sql diff --git a/server/postgres/migrations/report/first-deploy.sql b/server/postgres/migrations/report/first-deploy.sql new file mode 100644 index 0000000000..3398618f8d --- /dev/null +++ b/server/postgres/migrations/report/first-deploy.sql @@ -0,0 +1,1401 @@ +-- GENERATED by server/bin/build-migration-report.py; do not edit. +-- First deployment only: no existing ledger or queue. Other states refuse. +-- This is a catalog forecast, not DDL success or deployment-health proof. +-- sha256 8fa1066d9fc29c1df528b300199c619a2f9edecf1432926ad6a8146ebc650593 adoption/helpers.sql +-- sha256 491da3bcc32d863925f93a0cfcd5a5f777987089f696b796efc087f94819dcfc release.txt +-- sha256 8a8b24fa47a25c613a8329aa79f413ac7461a93b12134045ae1cf2457e2ee3b7 held.txt +-- sha256 c6b5c71247d129964dd89ba560a4522caef1abbb9379ad1b3445d6ed96be805b adoption/000000_initial.sql +-- sha256 28ef33976beb9c7c9248ac800f29aab048f4410c2170ad12e3ade71d72e5ecb5 adoption/000001_update_pwreset_table.sql +-- sha256 c3393049eae0a663ecea42a4917a06b37521a8ff508d39b308f8fd9da5606bf9 adoption/000002_add_xid_constraint.sql +-- sha256 3b1c9a390cbf23882455f1f3d05119df4a306f9a04d18e8c0e2cff3374895b86 adoption/000003_add_origin_permanent_cookie_columns.sql +-- sha256 5fae81bb9ff86062125487df98c1742e7ce5c45466fbeed7f6f63cbf94b1933b adoption/000004_drop_waitinglist_table.sql +-- sha256 02c90a416148f29c0717c01bcc97e66d6e8083ab3a2fabc5dc6bc716e963447e adoption/000005_drop_slack_stripe_canvas.sql +-- sha256 42d43c346ea90d0504698df29f90dd2c7e57860338c633da11410aaf9aaf5148 adoption/000006_update_votes_rule.sql +-- sha256 5a4110278c60157f21e5cb920ece834f6d6f99e6d4fa34b9b55bf0c6e80587c0 adoption/000007_drop_geolocation_fields.sql +-- sha256 a10a1a1881044409d98efd6042b5b28933a4579bce32738be1c4bc268640f172 adoption/000008_add_comment_priority.sql +-- sha256 a5605b5a292f5b4d47849cb7f2b678211e3bcdc9722cf9e5eabe54f455a5aa7c adoption/000009_add_uuid_to_zinvites.sql +-- sha256 23d5179cd0b556a71fe8b1ce56d3ac9e614cdee59832ce7892a012f0193cfe66 adoption/000010_create_oidc_user_mappings.sql +-- sha256 c991ffd93cd9051302dd51ab8d5bd6025128fc9728330e29aa963306a9b80f3e adoption/000011_alter_suzinvites_xid_to_text.sql +-- sha256 c31bd6811efee0808da5abb32ba63c47deebe6c6e89baa1a80ddcacc9e250234 adoption/000012_create_topic_agenda_selections.sql +-- sha256 eed1d9e223322db2b234428d350b6548793e54b94315932658b693d8263a6001 adoption/000013_create_treevite.sql +-- sha256 6b99d93d2f90f03bee170305c4b1305f03636882a84cdd08e107a763f1fc0120 adoption/000014_alter_reports_modlevel.sql +-- sha256 30c8d693d4f0d523c68a375bf0160e71e574f0a155badc363e085c4688ff0625 adoption/000015_add_xid_requirements.sql +-- sha256 ffb57aff8be7bc81bf10899133fe2a243cb20b58eb6b73afb3b76a66d5632478 adoption/000016_add_orig_id.sql +-- sha256 b72fbd498c13aceb7973100331c58b55a468b7e4447645a3e5c93f79a1126fd6 adoption/000017_create_byod_job_table.sql +-- sha256 96bf4a25a5733ac3dce8a7f25a4f374b5ee9bed924e95e048f73a343d2ddf539 adoption/000018_add_topics_enabled.sql +-- sha256 9764fc94c4fe4ac19626381146a588da3f6b4d32cac53ab59ae87b401ea8af8a adoption/000022_add_poll_timestamp_indexes.sql +-- sha256 2652134140cd8796ee3ab64995d509206bab3ebe8a03bba0cf48038a32dc6cea 000000_initial.sql +-- sha256 cb21278194c4b7db510c12bf1a7761b3a905410a7b3769edeb0d1865b6ec047b 000001_update_pwreset_table.sql +-- sha256 a27a8e63c79cd055198fa201b1745c85e0cd0e0b75f228b7153571631e529c07 000002_add_xid_constraint.sql +-- sha256 0d7f27facfecff1327573d3e1a5f65a3277ea94a30bcd8692606f00a7e496d3d 000003_add_origin_permanent_cookie_columns.sql +-- sha256 f2fc4184a965ffad01913d0e71a806c6f7e2aecc5f59279c11b0cd3ed3a77528 000004_drop_waitinglist_table.sql +-- sha256 392e5b8aadb7a85767820ec97fcfb0853821d35089be3e8c3dfe70df24782839 000005_drop_slack_stripe_canvas.sql +-- sha256 8fb05b7b1b6a8b123cea6b2680054d8da2fe48585c51a2f9f108b8b174d85c60 000006_update_votes_rule.sql +-- sha256 f68b69b86112ad52e2cb79a9b631ea77a0285b24201b41c15d9c13257f7cd9cc 000007_drop_geolocation_fields.sql +-- sha256 c867b53be3cca6bce059b19ea9eb3950120fb5840a1476af409924ca55905261 000008_add_comment_priority.sql +-- sha256 43f36fe0b8578bc4b32761a2b9d7a5181a8159ff8b0c7bb0a1a73f802ec5f670 000009_add_uuid_to_zinvites.sql +-- sha256 450a3f69883aa56c3bc85994ab1baf5e208634d88435a156fe2a22738b1d9f9a 000010_create_oidc_user_mappings.sql +-- sha256 00a1eb2d9604804a56705eaac92bb6d37b6c696abdfd9bb079d4a1c494ff0ab6 000011_alter_suzinvites_xid_to_text.sql +-- sha256 cc513693124f031ce1d55e7e73b14b4e3c40a4070e65465adbc621e63b1008cf 000012_create_topic_agenda_selections.sql +-- sha256 4b8334f73246c69bc9ecadd4a2dc00c8cfb642351ba70ed4a9a58b882a4ac53f 000013_create_treevite.sql +-- sha256 c2af6d57af2866f1f259bb41b94dbf730eca499a4b18f092c4aeeebce5a95025 000014_alter_reports_modlevel.sql +-- sha256 186c904addd0c8a42c10057e6ae362fe2799cc076e877bef3ffa14e9852184e7 000015_add_xid_requirements.sql +-- sha256 6cdc0588c000fbbe89d578bc2f0aa60422b92b5320aba737d6d83f89d8d990db 000016_add_orig_id.sql +-- sha256 f27c03a1229f296fab21cee059b743c7137bf502e4178854e596a8efa4808b55 000017_create_byod_job_table.sql +-- sha256 a1e1c0572064d88877d87142bc4e40132b84672bbe84392e99f9f6f4c2ea107e 000018_add_topics_enabled.sql +-- sha256 fedfbcf9fc594c3e53193cacb76dfd35558d81a449d507fcacb3198f5e39beae 000019_create_polis_queue.sql +-- sha256 d50f169ad7afe12d14582a6a746c622d402ecafd8131aae246812263bf2d5e82 000021_create_polis_coordinator.sql +-- sha256 14efc95b14787b52d70680ea06cfef020cd2224e82495258fd8eb324501b7b3e 000022_add_poll_timestamp_indexes.sql +-- sha256 97437ea57d90c51cc664385ebb6e8ec3d8684ac0f8e7b0c60e84df1a010531f7 000023_create_delphi_foundation.sql +-- sha256 68261afb81f286bed45fdff6ab52e052392d1dd32abdb1e78579f72644116697 000024_create_polis_queue_large_class.sql +-- sha256 fbbf948e4316010dd96344ae91542006c38299e52e0a0eb281371c039ab37775 000027_create_sealed_job_graphs.sql +BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY; +SET LOCAL search_path=pg_catalog,public; +SET LOCAL statement_timeout='30s'; +SET LOCAL lock_timeout='2s'; +WITH checks AS ( +SELECT '000000_initial.sql'::text AS migration, (-- Named legacy variants are documented in docs/migration-legacy-contract.md. +-- Contributor agreement tables are outside this release adoption boundary. +-- Catalog postconditions only; this file never replays migration DDL. +SELECT NOT EXISTS (SELECT 1 FROM (VALUES + ('users','uid','integer'), + ('users','hname','character varying(746)'), + ('users','created','bigint'), + ('users','username','character varying(128)'), + ('users','email','character varying(256)'), + ('users','is_owner','boolean'), + ('users','zinvite','character varying(300)'), + ('users','oinvite','character varying(300)'), + ('users','tut','smallint'), + ('users','site_id','character varying(256)'), + ('users','site_owner','boolean'), + ('site_domain_whitelist','site_id','character varying(256)'), + ('site_domain_whitelist','domain_whitelist','character varying(999)'), + ('site_domain_whitelist','domain_whitelist_override_key','character varying(999)'), + ('site_domain_whitelist','modified','bigint'), + ('site_domain_whitelist','created','bigint'), + ('metrics','uid','integer'), + ('metrics','type','integer'), + ('metrics','dur','integer'), + ('metrics','hashedpc','integer'), + ('metrics','created','bigint'), + ('auth_tokens','token','character varying(32)'), + ('auth_tokens','uid','integer'), + ('auth_tokens','created','bigint'), + ('jianiuevyew','uid','integer'), + ('jianiuevyew','pwhash','character varying(128)'), + ('apikeysndvweifu','uid','integer'), + ('apikeysndvweifu','apikey','character varying(32)'), + ('apikeysndvweifu','created','bigint'), + ('courses','course_id','integer'), + ('courses','topic','character varying(1000)'), + ('courses','description','character varying(1000)'), + ('courses','owner','integer'), + ('courses','course_invite','character varying(32)'), + ('courses','created','bigint'), + ('conversations','zid','integer'), + ('conversations','topic','character varying(1000)'), + ('conversations','description','character varying(50000)'), + ('conversations','link_url','character varying(9999)'), + ('conversations','parent_url','character varying(9999)'), + ('conversations','upvotes','integer'), + ('conversations','participant_count','integer'), + ('conversations','is_anon','boolean'), + ('conversations','is_active','boolean'), + ('conversations','is_draft','boolean'), + ('conversations','is_public','boolean'), + ('conversations','is_data_open','boolean'), + ('conversations','profanity_filter','boolean'), + ('conversations','spam_filter','boolean'), + ('conversations','strict_moderation','boolean'), + ('conversations','prioritize_seed','boolean'), + ('conversations','vis_type','integer'), + ('conversations','write_type','integer'), + ('conversations','help_type','integer'), + ('conversations','write_hint_type','integer'), + ('conversations','style_btn','character varying(500)'), + ('conversations','socialbtn_type','integer'), + ('conversations','subscribe_type','integer'), + ('conversations','bgcolor','character varying(20)'), + ('conversations','help_bgcolor','character varying(20)'), + ('conversations','help_color','character varying(20)'), + ('conversations','email_domain','character varying(200)'), + ('conversations','use_xid_whitelist','boolean'), + ('conversations','owner','integer'), + ('conversations','org_id','integer'), + ('conversations','context','character varying(1000)'), + ('conversations','course_id','integer'), + ('conversations','owner_sees_participation_stats','boolean'), + ('conversations','auth_needed_to_vote','boolean'), + ('conversations','auth_needed_to_write','boolean'), + ('conversations','auth_opt_fb','boolean'), + ('conversations','auth_opt_tw','boolean'), + ('conversations','auth_opt_allow_3rdparty','boolean'), + ('conversations','modified','bigint'), + ('conversations','created','bigint'), + ('participant_metadata_questions','pmqid','integer'), + ('participant_metadata_questions','zid','integer'), + ('participant_metadata_questions','key','character varying(999)'), + ('participant_metadata_questions','alive','boolean'), + ('participant_metadata_questions','created','bigint'), + ('participant_metadata_answers','pmaid','integer'), + ('participant_metadata_answers','pmqid','integer'), + ('participant_metadata_answers','zid','integer'), + ('participant_metadata_answers','value','character varying(999)'), + ('participant_metadata_answers','alive','boolean'), + ('participant_metadata_answers','created','bigint'), + ('contexts','context_id','integer'), + ('contexts','name','character varying(300)'), + ('contexts','creator','integer'), + ('contexts','is_public','boolean'), + ('contexts','created','bigint'), + ('inviters','inviter_uid','integer'), + ('inviters','invited_email','character varying(999)'), + ('inviters','created','bigint'), + ('upvotes','uid','integer'), + ('upvotes','zid','integer'), + ('oinvites','oinvite','character varying(300)'), + ('oinvites','note','character varying(999)'), + ('oinvites','created','bigint'), + ('einvites','einvite','character varying(100)'), + ('einvites','email','character varying(999)'), + ('einvites','created','bigint'), + ('email_validations','email','character varying(999)'), + ('email_validations','created','bigint'), + ('zinvites','zid','integer'), + ('zinvites','zinvite','character varying(300)'), + ('zinvites','created','bigint'), + ('beta','name','character varying(999)'), + ('beta','email','character varying(200)'), + ('beta','organization','character varying(200)'), + ('beta','created','bigint'), + ('participants','pid','integer'), + ('participants','uid','integer'), + ('participants','zid','integer'), + ('participants','vote_count','integer'), + ('participants','last_interaction','bigint'), + ('participants','subscribed','integer'), + ('participants','last_notified','bigint'), + ('participants','nsli','smallint'), + ('participants','mod','integer'), + ('participants','created','bigint'), + ('participants_extended','uid','integer'), + ('participants_extended','zid','integer'), + ('participants_extended','referrer','character varying(9999)'), + ('participants_extended','parent_url','character varying(9999)'), + ('participants_extended','created','bigint'), + ('participants_extended','modified','bigint'), + ('participants_extended','subscribe_email','character varying(256)'), + ('participants_extended','show_translation_activated','boolean'), + ('participant_locations','zid','integer'), + ('participant_locations','uid','integer'), + ('participant_locations','pid','integer'), + ('participant_locations','lat','double precision'), + ('participant_locations','lng','double precision'), + ('participant_locations','created','bigint'), + ('participant_locations','source','integer'), + ('xids','uid','integer'), + ('xids','owner','integer'), + ('xids','xid','text'), + ('xids','x_profile_image_url','character varying(3000)'), + ('xids','x_name','character varying(746)'), + ('xids','x_email','character varying(256)'), + ('xids','created','bigint'), + ('xids','modified','bigint'), + ('xid_whitelist','owner','integer'), + ('xid_whitelist','xid','text'), + ('xid_whitelist','created','bigint'), + ('notification_tasks','zid','integer'), + ('notification_tasks','modified','bigint'), + ('participant_metadata_choices','zid','integer'), + ('participant_metadata_choices','pid','integer'), + ('participant_metadata_choices','pmqid','integer'), + ('participant_metadata_choices','pmaid','integer'), + ('participant_metadata_choices','alive','boolean'), + ('participant_metadata_choices','created','bigint'), + ('twitter_users','uid','integer'), + ('twitter_users','twitter_user_id','bigint'), + ('twitter_users','screen_name','character varying(999)'), + ('twitter_users','name','character varying(9999)'), + ('twitter_users','followers_count','integer'), + ('twitter_users','friends_count','integer'), + ('twitter_users','verified','boolean'), + ('twitter_users','profile_image_url_https','character varying(9999)'), + ('twitter_users','location','character varying(9999)'), + ('twitter_users','response','json'), + ('twitter_users','modified','bigint'), + ('twitter_users','created','bigint'), + ('facebook_users','uid','integer'), + ('facebook_users','fb_user_id','text'), + ('facebook_users','fb_name','character varying(9999)'), + ('facebook_users','fb_link','character varying(9999)'), + ('facebook_users','fb_public_profile','text'), + ('facebook_users','fb_login_status','text'), + ('facebook_users','fb_auth_response','text'), + ('facebook_users','fb_access_token','text'), + ('facebook_users','fb_granted_scopes','text'), + ('facebook_users','fb_location_id','character varying(100)'), + ('facebook_users','location','character varying(9999)'), + ('facebook_users','response','text'), + ('facebook_users','fb_friends_response','text'), + ('facebook_users','created','bigint'), + ('facebook_users','modified','bigint'), + ('social_settings','uid','integer'), + ('social_settings','polis_pic','character varying(3000)'), + ('facebook_friends','uid','integer'), + ('facebook_friends','friend','integer'), + ('suzinvites','owner','integer'), + ('suzinvites','zid','integer'), + ('suzinvites','created','bigint'), + ('suzinvites','suzinvite','character varying(32)'), + ('comments','tid','integer'), + ('comments','zid','integer'), + ('comments','pid','integer'), + ('comments','uid','integer'), + ('comments','created','bigint'), + ('comments','modified','bigint'), + ('comments','txt','character varying(1000)'), + ('comments','velocity','real'), + ('comments','mod','integer'), + ('comments','lang','character varying(10)'), + ('comments','lang_confidence','real'), + ('comments','active','boolean'), + ('comments','is_meta','boolean'), + ('comments','tweet_id','bigint'), + ('comments','quote_src_url','character varying(1000)'), + ('comments','anon','boolean'), + ('comments','is_seed','boolean'), + ('comment_translations','zid','integer'), + ('comment_translations','tid','integer'), + ('comment_translations','src','integer'), + ('comment_translations','txt','character varying(9999)'), + ('comment_translations','lang','character varying(10)'), + ('comment_translations','created','bigint'), + ('comment_translations','modified','bigint'), + ('conversation_translations','zid','integer'), + ('conversation_translations','src','integer'), + ('conversation_translations','topic','character varying(9999)'), + ('conversation_translations','description','character varying(9999)'), + ('conversation_translations','lang','character varying(10)'), + ('conversation_translations','created','bigint'), + ('conversation_translations','modified','bigint'), + ('reports','rid','bigint'), + ('reports','report_id','character varying(300)'), + ('reports','zid','integer'), + ('reports','created','bigint'), + ('reports','modified','bigint'), + ('reports','report_name','character varying(999)'), + ('reports','label_x_neg','character varying(999)'), + ('reports','label_x_pos','character varying(999)'), + ('reports','label_y_neg','character varying(999)'), + ('reports','label_y_pos','character varying(999)'), + ('reports','label_group_0','character varying(999)'), + ('reports','label_group_1','character varying(999)'), + ('reports','label_group_2','character varying(999)'), + ('reports','label_group_3','character varying(999)'), + ('reports','label_group_4','character varying(999)'), + ('reports','label_group_5','character varying(999)'), + ('reports','label_group_6','character varying(999)'), + ('reports','label_group_7','character varying(999)'), + ('reports','label_group_8','character varying(999)'), + ('reports','label_group_9','character varying(999)'), + ('report_comment_selections','zid','integer'), + ('report_comment_selections','rid','bigint'), + ('report_comment_selections','tid','integer'), + ('report_comment_selections','selection','smallint'), + ('report_comment_selections','modified','bigint'), + ('worker_tasks','created','bigint'), + ('worker_tasks','math_env','character varying(999)'), + ('worker_tasks','attempts','smallint'), + ('worker_tasks','task_data','jsonb'), + ('worker_tasks','task_type','text'), + ('worker_tasks','task_bucket','bigint'), + ('worker_tasks','finished_time','bigint'), + ('math_ticks','zid','integer'), + ('math_ticks','math_tick','bigint'), + ('math_ticks','math_env','character varying(999)'), + ('math_ticks','modified','bigint'), + ('math_main','zid','integer'), + ('math_main','math_env','character varying(999)'), + ('math_main','data','jsonb'), + ('math_main','last_vote_timestamp','bigint'), + ('math_main','caching_tick','bigint'), + ('math_main','math_tick','bigint'), + ('math_main','modified','bigint'), + ('math_profile','zid','integer'), + ('math_profile','math_env','character varying(999)'), + ('math_profile','data','jsonb'), + ('math_profile','modified','bigint'), + ('math_ptptstats','zid','integer'), + ('math_ptptstats','math_env','character varying(999)'), + ('math_ptptstats','math_tick','bigint'), + ('math_ptptstats','data','jsonb'), + ('math_ptptstats','modified','bigint'), + ('math_cache','zid','integer'), + ('math_cache','math_env','character varying(999)'), + ('math_cache','data','jsonb'), + ('math_cache','modified','bigint'), + ('math_bidtopid','zid','integer'), + ('math_bidtopid','math_env','character varying(999)'), + ('math_bidtopid','math_tick','bigint'), + ('math_bidtopid','data','jsonb'), + ('math_bidtopid','modified','bigint'), + ('math_exportstatus','zid','integer'), + ('math_exportstatus','math_env','character varying(999)'), + ('math_exportstatus','filename','character varying(9999)'), + ('math_exportstatus','data','jsonb'), + ('math_exportstatus','modified','bigint'), + ('math_report_correlationmatrix','rid','bigint'), + ('math_report_correlationmatrix','math_env','character varying(999)'), + ('math_report_correlationmatrix','data','jsonb'), + ('math_report_correlationmatrix','math_tick','bigint'), + ('math_report_correlationmatrix','modified','bigint'), + ('votes','zid','integer'), + ('votes','pid','integer'), + ('votes','tid','integer'), + ('votes','vote','smallint'), + ('votes','weight_x_32767','smallint'), + ('votes','created','bigint'), + ('votes_latest_unique','zid','integer'), + ('votes_latest_unique','pid','integer'), + ('votes_latest_unique','tid','integer'), + ('votes_latest_unique','vote','smallint'), + ('votes_latest_unique','weight_x_32767','smallint'), + ('votes_latest_unique','modified','bigint'), + ('crowd_mod','zid','integer'), + ('crowd_mod','pid','integer'), + ('crowd_mod','tid','integer'), + ('crowd_mod','created','bigint'), + ('crowd_mod','as_important','boolean'), + ('crowd_mod','as_factual','boolean'), + ('crowd_mod','as_feeling','boolean'), + ('crowd_mod','as_notmyfeeling','boolean'), + ('crowd_mod','as_notgoodidea','boolean'), + ('crowd_mod','as_notfact','boolean'), + ('crowd_mod','as_unsure','boolean'), + ('crowd_mod','as_spam','boolean'), + ('crowd_mod','as_abusive','boolean'), + ('crowd_mod','as_offtopic','boolean'), + ('event_ptpt_no_more_comments','zid','integer'), + ('event_ptpt_no_more_comments','pid','integer'), + ('event_ptpt_no_more_comments','votes_placed','smallint'), + ('event_ptpt_no_more_comments','created','bigint'), + ('stars','zid','integer'), + ('stars','pid','integer'), + ('stars','tid','integer'), + ('stars','starred','integer'), + ('stars','created','bigint'), + ('trashes','zid','integer'), + ('trashes','pid','integer'), + ('trashes','tid','integer'), + ('trashes','trashed','integer'), + ('trashes','created','bigint'), + ('permanentcookiezidjoins','zid','integer'), + ('permanentcookiezidjoins','cookie','character varying(32)'), + ('permanentcookiezidjoins','created','bigint'), + ('page_ids','site_id','character varying(100)'), + ('page_ids','page_id','character varying(100)'), + ('page_ids','zid','integer'), + ('demographic_data','uid','integer'), + ('demographic_data','fb_gender','integer'), + ('demographic_data','ms_birth_year_estimate_fb','integer'), + ('demographic_data','ms_gender_estimate_fb','integer'), + ('demographic_data','fb_timestamp','bigint'), + ('demographic_data','ms_fb_timestamp','bigint'), + ('demographic_data','ms_response','character varying(9999)'), + ('demographic_data','gender_guess','integer'), + ('demographic_data','birth_year_guess','integer')) AS expected(t,c,typ) WHERE NOT (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||(t)) AND a.attname=(c) AND NOT a.attisdropped + AND (format_type(a.atttypid,a.atttypmod)=(typ) + -- Exact public bootstrap alternatives to the authoritative legacy contract. + OR ((t)='worker_tasks' AND (c)='task_type' AND (typ)='text' + AND format_type(a.atttypid,a.atttypmod)='character varying(99)') + OR ((t) IN ('pwreset_tokens','password_reset_tokens') + AND (c) IN ('token','pwresettoken') AND (typ)='character varying(100)' + AND format_type(a.atttypid,a.atttypmod)='character varying(250)') + -- Only historical math payloads have an established json equivalent. + -- Do not adopt arbitrary json columns in newer jsonb contracts. + OR ((typ)='jsonb' AND a.atttypid='json'::regtype AND (c)='data' + AND (t) IN ('math_main','math_profile','math_ptptstats','math_cache', + 'math_bidtopid','math_exportstatus'))) + AND ((NULL) IS NULL OR a.attnotnull=(NULL)) + AND ((NULL) IS NULL OR pg_get_expr(d.adbin,d.adrelid)=(NULL))))) + AND NOT EXISTS (SELECT 1 FROM unnest(ARRAY['users','site_domain_whitelist','metrics','auth_tokens','jianiuevyew','apikeysndvweifu','courses','conversations','participant_metadata_questions','participant_metadata_answers','contexts','inviters','upvotes','oinvites','einvites','email_validations','zinvites','beta','participants','participants_extended','participant_locations','xids','xid_whitelist','notification_tasks','participant_metadata_choices','twitter_users','facebook_users','social_settings','facebook_friends','suzinvites','comments','comment_translations','conversation_translations','reports','report_comment_selections','worker_tasks','math_ticks','math_main','math_profile','math_ptptstats','math_cache','math_bidtopid','math_exportstatus','math_report_correlationmatrix','votes','votes_latest_unique','crowd_mod','event_ptpt_no_more_comments','stars','trashes','permanentcookiezidjoins','page_ids','demographic_data']) t WHERE to_regclass('public.'||t) IS NULL) + AND ((SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('pwreset_tokens')) AND a.attname=('token') AND NOT a.attisdropped + AND (format_type(a.atttypid,a.atttypmod)=('character varying(100)') + -- Exact public bootstrap alternatives to the authoritative legacy contract. + OR (('pwreset_tokens')='worker_tasks' AND ('token')='task_type' AND ('character varying(100)')='text' + AND format_type(a.atttypid,a.atttypmod)='character varying(99)') + OR (('pwreset_tokens') IN ('pwreset_tokens','password_reset_tokens') + AND ('token') IN ('token','pwresettoken') AND ('character varying(100)')='character varying(100)' + AND format_type(a.atttypid,a.atttypmod)='character varying(250)') + -- Only historical math payloads have an established json equivalent. + -- Do not adopt arbitrary json columns in newer jsonb contracts. + OR (('character varying(100)')='jsonb' AND a.atttypid='json'::regtype AND ('token')='data' + AND ('pwreset_tokens') IN ('math_main','math_profile','math_ptptstats','math_cache', + 'math_bidtopid','math_exportstatus'))) + AND ((NULL) IS NULL OR a.attnotnull=(NULL)) + AND ((NULL) IS NULL OR pg_get_expr(d.adbin,d.adrelid)=(NULL)))) OR (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('password_reset_tokens')) AND a.attname=('pwresettoken') AND NOT a.attisdropped + AND (format_type(a.atttypid,a.atttypmod)=('character varying(100)') + -- Exact public bootstrap alternatives to the authoritative legacy contract. + OR (('password_reset_tokens')='worker_tasks' AND ('pwresettoken')='task_type' AND ('character varying(100)')='text' + AND format_type(a.atttypid,a.atttypmod)='character varying(99)') + OR (('password_reset_tokens') IN ('pwreset_tokens','password_reset_tokens') + AND ('pwresettoken') IN ('token','pwresettoken') AND ('character varying(100)')='character varying(100)' + AND format_type(a.atttypid,a.atttypmod)='character varying(250)') + -- Only historical math payloads have an established json equivalent. + -- Do not adopt arbitrary json columns in newer jsonb contracts. + OR (('character varying(100)')='jsonb' AND a.atttypid='json'::regtype AND ('pwresettoken')='data' + AND ('password_reset_tokens') IN ('math_main','math_profile','math_ptptstats','math_cache', + 'math_bidtopid','math_exportstatus'))) + AND ((NULL) IS NULL OR a.attnotnull=(NULL)) + AND ((NULL) IS NULL OR pg_get_expr(d.adbin,d.adrelid)=(NULL))))) + AND ((SELECT to_regclass('public.'||('conversations')) IS NOT NULL AND NOT EXISTS(SELECT 1 FROM pg_attribute + WHERE attrelid=to_regclass('public.'||('conversations')) AND attname=('branding_type') AND NOT attisdropped)) OR (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('conversations')) AND a.attname=('branding_type') AND NOT a.attisdropped + AND (format_type(a.atttypid,a.atttypmod)=('integer') + -- Exact public bootstrap alternatives to the authoritative legacy contract. + OR (('conversations')='worker_tasks' AND ('branding_type')='task_type' AND ('integer')='text' + AND format_type(a.atttypid,a.atttypmod)='character varying(99)') + OR (('conversations') IN ('pwreset_tokens','password_reset_tokens') + AND ('branding_type') IN ('token','pwresettoken') AND ('integer')='character varying(100)' + AND format_type(a.atttypid,a.atttypmod)='character varying(250)') + -- Only historical math payloads have an established json equivalent. + -- Do not adopt arbitrary json columns in newer jsonb contracts. + OR (('integer')='jsonb' AND a.atttypid='json'::regtype AND ('branding_type')='data' + AND ('conversations') IN ('math_main','math_profile','math_ptptstats','math_cache', + 'math_bidtopid','math_exportstatus'))) + AND ((NULL) IS NULL OR a.attnotnull=(NULL)) + AND ((NULL) IS NULL OR pg_get_expr(d.adbin,d.adrelid)=(NULL))))) + AND ((SELECT to_regclass('public.'||('math_ticks')) IS NOT NULL AND NOT EXISTS(SELECT 1 FROM pg_attribute + WHERE attrelid=to_regclass('public.'||('math_ticks')) AND attname=('caching_tick') AND NOT attisdropped)) OR (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('math_ticks')) AND a.attname=('caching_tick') AND NOT a.attisdropped + AND (format_type(a.atttypid,a.atttypmod)=('bigint') + -- Exact public bootstrap alternatives to the authoritative legacy contract. + OR (('math_ticks')='worker_tasks' AND ('caching_tick')='task_type' AND ('bigint')='text' + AND format_type(a.atttypid,a.atttypmod)='character varying(99)') + OR (('math_ticks') IN ('pwreset_tokens','password_reset_tokens') + AND ('caching_tick') IN ('token','pwresettoken') AND ('bigint')='character varying(100)' + AND format_type(a.atttypid,a.atttypmod)='character varying(250)') + -- Only historical math payloads have an established json equivalent. + -- Do not adopt arbitrary json columns in newer jsonb contracts. + OR (('bigint')='jsonb' AND a.atttypid='json'::regtype AND ('caching_tick')='data' + AND ('math_ticks') IN ('math_main','math_profile','math_ptptstats','math_cache', + 'math_bidtopid','math_exportstatus'))) + AND ((NULL) IS NULL OR a.attnotnull=(NULL)) + AND ((NULL) IS NULL OR pg_get_expr(d.adbin,d.adrelid)=(NULL))))) + AND to_regprocedure('public.now_as_millis()') IS NOT NULL + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint WHERE conrelid=to_regclass('public.'||('votes_latest_unique')) + AND convalidated AND pg_get_constraintdef(oid)=('UNIQUE (zid, pid, tid)')))) AS catalog_match +UNION ALL +SELECT '000001_update_pwreset_table.sql'::text AS migration, (-- Catalog postconditions only; this file never replays migration DDL. +SELECT to_regclass('public.password_reset_tokens') IS NULL + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('pwreset_tokens')) AND a.attname=('token') AND NOT a.attisdropped + AND (format_type(a.atttypid,a.atttypmod)=('character varying(100)') + -- Exact public bootstrap alternatives to the authoritative legacy contract. + OR (('pwreset_tokens')='worker_tasks' AND ('token')='task_type' AND ('character varying(100)')='text' + AND format_type(a.atttypid,a.atttypmod)='character varying(99)') + OR (('pwreset_tokens') IN ('pwreset_tokens','password_reset_tokens') + AND ('token') IN ('token','pwresettoken') AND ('character varying(100)')='character varying(100)' + AND format_type(a.atttypid,a.atttypmod)='character varying(250)') + -- Only historical math payloads have an established json equivalent. + -- Do not adopt arbitrary json columns in newer jsonb contracts. + OR (('character varying(100)')='jsonb' AND a.atttypid='json'::regtype AND ('token')='data' + AND ('pwreset_tokens') IN ('math_main','math_profile','math_ptptstats','math_cache', + 'math_bidtopid','math_exportstatus'))) + AND ((NULL) IS NULL OR a.attnotnull=(NULL)) + AND ((NULL) IS NULL OR pg_get_expr(d.adbin,d.adrelid)=(NULL)))) + AND (SELECT to_regclass('public.'||('pwreset_tokens')) IS NOT NULL AND NOT EXISTS(SELECT 1 FROM pg_attribute + WHERE attrelid=to_regclass('public.'||('pwreset_tokens')) AND attname=('pwresettoken') AND NOT attisdropped))) AS catalog_match +UNION ALL +SELECT '000002_add_xid_constraint.sql'::text AS migration, (-- Both the public bootstrap variant (owner,xid only) and the authoritative +-- legacy variant (also owner,uid) are supported. Never drop either invariant. +SELECT (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('xids')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('UNIQUE (owner, xid)') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + -- If the optional legacy unique index exists, it must enforce the complete key. + AND NOT EXISTS ( + SELECT 1 FROM pg_index i + JOIN pg_attribute owner ON owner.attrelid=i.indrelid AND owner.attname='owner' + JOIN pg_attribute uid ON uid.attrelid=i.indrelid AND uid.attname='uid' + WHERE i.indrelid=to_regclass('public.xids') AND i.indisunique + AND i.indnkeyatts=2 + AND ARRAY[i.indkey[0],i.indkey[1]] @> ARRAY[owner.attnum,uid.attnum] + AND (NOT i.indisvalid OR NOT i.indisready OR NOT i.indislive + OR NOT i.indimmediate OR i.indpred IS NOT NULL) + )) AS catalog_match +UNION ALL +SELECT '000003_add_origin_permanent_cookie_columns.sql'::text AS migration, (-- Catalog postconditions only; this file never replays migration DDL. +SELECT (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('participants_extended')) AND a.attname=('permanent_cookie') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('character varying(32)') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('participants_extended')) AND a.attname=('origin') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('character varying(9999)') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL)))) AS catalog_match +UNION ALL +SELECT '000004_drop_waitinglist_table.sql'::text AS migration, (-- Catalog postconditions only; this file never replays migration DDL. +SELECT to_regclass('public.waitinglist') IS NULL) AS catalog_match +UNION ALL +SELECT '000005_drop_slack_stripe_canvas.sql'::text AS migration, (-- Catalog postconditions only; this file never replays migration DDL. +SELECT to_regclass('public.slack_oauth_access_tokens') IS NULL + AND to_regclass('public.slack_users') IS NULL + AND to_regclass('public.slack_user_invites') IS NULL + AND to_regclass('public.slack_bot_events') IS NULL + AND to_regclass('public.stripe_accounts') IS NULL + AND to_regclass('public.stripe_subscriptions') IS NULL + AND to_regclass('public.coupons_for_free_upgrades') IS NULL + AND to_regclass('public.lti_users') IS NULL + AND to_regclass('public.lti_context_memberships') IS NULL + AND to_regclass('public.canvas_assignment_callback_info') IS NULL + AND to_regclass('public.canvas_assignment_conversation_info') IS NULL + AND to_regclass('public.lti_oauthv1_credentials') IS NULL + AND (SELECT to_regclass('public.'||('conversations')) IS NOT NULL AND NOT EXISTS(SELECT 1 FROM pg_attribute + WHERE attrelid=to_regclass('public.'||('conversations')) AND attname=('is_slack') AND NOT attisdropped)) + AND (SELECT to_regclass('public.'||('conversations')) IS NOT NULL AND NOT EXISTS(SELECT 1 FROM pg_attribute + WHERE attrelid=to_regclass('public.'||('conversations')) AND attname=('lti_users_only') AND NOT attisdropped)) + AND (SELECT to_regclass('public.'||('users')) IS NOT NULL AND NOT EXISTS(SELECT 1 FROM pg_attribute + WHERE attrelid=to_regclass('public.'||('users')) AND attname=('plan') AND NOT attisdropped))) AS catalog_match +UNION ALL +SELECT '000006_update_votes_rule.sql'::text AS migration, (-- Catalog postconditions only; this file never replays migration DDL. +SELECT EXISTS (SELECT 1 FROM pg_rewrite WHERE ev_class=to_regclass('public.votes') AND rulename='on_vote_insert_update_unique_table' AND ev_enabled='O' AND regexp_replace(pg_get_ruledef(oid), '\s+', ' ', 'g') = 'CREATE RULE on_vote_insert_update_unique_table AS ON INSERT TO public.votes DO INSERT INTO votes_latest_unique (zid, pid, tid, vote, weight_x_32767, modified) VALUES (new.zid, new.pid, new.tid, new.vote, new.weight_x_32767, new.created) ON CONFLICT(zid, pid, tid) DO UPDATE SET vote = excluded.vote, modified = excluded.modified;')) AS catalog_match +UNION ALL +SELECT '000007_drop_geolocation_fields.sql'::text AS migration, (-- Catalog postconditions only; this file never replays migration DDL. +SELECT to_regclass('public.geolocation_cache') IS NULL + AND (SELECT to_regclass('public.'||('participants_extended')) IS NOT NULL AND NOT EXISTS(SELECT 1 FROM pg_attribute + WHERE attrelid=to_regclass('public.'||('participants_extended')) AND attname=('country_code_iso') AND NOT attisdropped)) + AND (SELECT to_regclass('public.'||('participants_extended')) IS NOT NULL AND NOT EXISTS(SELECT 1 FROM pg_attribute + WHERE attrelid=to_regclass('public.'||('participants_extended')) AND attname=('encrypted_maxmind_response_city') AND NOT attisdropped)) + AND (SELECT to_regclass('public.'||('participants_extended')) IS NOT NULL AND NOT EXISTS(SELECT 1 FROM pg_attribute + WHERE attrelid=to_regclass('public.'||('participants_extended')) AND attname=('ip_address') AND NOT attisdropped)) + AND (SELECT to_regclass('public.'||('participants_extended')) IS NOT NULL AND NOT EXISTS(SELECT 1 FROM pg_attribute + WHERE attrelid=to_regclass('public.'||('participants_extended')) AND attname=('latitude') AND NOT attisdropped)) + AND (SELECT to_regclass('public.'||('participants_extended')) IS NOT NULL AND NOT EXISTS(SELECT 1 FROM pg_attribute + WHERE attrelid=to_regclass('public.'||('participants_extended')) AND attname=('location') AND NOT attisdropped)) + AND (SELECT to_regclass('public.'||('participants_extended')) IS NOT NULL AND NOT EXISTS(SELECT 1 FROM pg_attribute + WHERE attrelid=to_regclass('public.'||('participants_extended')) AND attname=('longitude') AND NOT attisdropped)) + AND (SELECT to_regclass('public.'||('participants_extended')) IS NOT NULL AND NOT EXISTS(SELECT 1 FROM pg_attribute + WHERE attrelid=to_regclass('public.'||('participants_extended')) AND attname=('x_forwarded_for') AND NOT attisdropped))) AS catalog_match +UNION ALL +SELECT '000008_add_comment_priority.sql'::text AS migration, (-- Catalog postconditions only; this file never replays migration DDL. +SELECT (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('conversations')) AND a.attname=('importance_enabled') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('boolean') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('false'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('votes')) AND a.attname=('high_priority') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('boolean') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('false')))) AS catalog_match +UNION ALL +SELECT '000009_add_uuid_to_zinvites.sql'::text AS migration, (-- Catalog postconditions only; this file never replays migration DDL. +SELECT (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('zinvites')) AND a.attname=('uuid') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('uuid') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL)))) AS catalog_match +UNION ALL +SELECT '000010_create_oidc_user_mappings.sql'::text AS migration, (-- Catalog postconditions only; this file never replays migration DDL. +SELECT (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('oidc_user_mappings')) AND a.attname=('oidc_sub') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('character varying(255)') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('oidc_user_mappings')) AND a.attname=('uid') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('oidc_user_mappings')) AND a.attname=('created') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('bigint') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('now_as_millis()'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('oidc_user_mappings')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('PRIMARY KEY (oidc_sub)') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('oidc_user_mappings')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('UNIQUE (uid)') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('oidc_user_mappings')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('FOREIGN KEY (uid) REFERENCES users(uid) ON DELETE CASCADE') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_oidc_mappings_uid')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_oidc_mappings_uid ON public.oidc_user_mappings USING btree (uid)')))) AS catalog_match +UNION ALL +SELECT '000011_alter_suzinvites_xid_to_text.sql'::text AS migration, (-- Catalog postconditions only; this file never replays migration DDL. +SELECT (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('suzinvites')) AND a.attname=('xid') AND NOT a.attisdropped + AND (format_type(a.atttypid,a.atttypmod)=('text') + -- Exact public bootstrap alternatives to the authoritative legacy contract. + OR (('suzinvites')='worker_tasks' AND ('xid')='task_type' AND ('text')='text' + AND format_type(a.atttypid,a.atttypmod)='character varying(99)') + OR (('suzinvites') IN ('pwreset_tokens','password_reset_tokens') + AND ('xid') IN ('token','pwresettoken') AND ('text')='character varying(100)' + AND format_type(a.atttypid,a.atttypmod)='character varying(250)') + -- Only historical math payloads have an established json equivalent. + -- Do not adopt arbitrary json columns in newer jsonb contracts. + OR (('text')='jsonb' AND a.atttypid='json'::regtype AND ('xid')='data' + AND ('suzinvites') IN ('math_main','math_profile','math_ptptstats','math_cache', + 'math_bidtopid','math_exportstatus'))) + AND ((true) IS NULL OR a.attnotnull=(true)) + AND ((NULL) IS NULL OR pg_get_expr(d.adbin,d.adrelid)=(NULL))))) AS catalog_match +UNION ALL +SELECT '000012_create_topic_agenda_selections.sql'::text AS migration, (-- Catalog postconditions only; this file never replays migration DDL. +SELECT (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('topic_agenda_selections')) AND a.attname=('zid') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('topic_agenda_selections')) AND a.attname=('pid') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('topic_agenda_selections')) AND a.attname=('archetypal_selections') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('jsonb') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('''[]''::jsonb'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('topic_agenda_selections')) AND a.attname=('delphi_job_id') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('text') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('topic_agenda_selections')) AND a.attname=('total_selections') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('0'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('topic_agenda_selections')) AND a.attname=('created_at') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('timestamp with time zone') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('CURRENT_TIMESTAMP'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('topic_agenda_selections')) AND a.attname=('updated_at') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('timestamp with time zone') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('CURRENT_TIMESTAMP'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('topic_agenda_selections')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('PRIMARY KEY (zid, pid)') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('topic_agenda_selections')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('FOREIGN KEY (zid) REFERENCES conversations(zid) ON DELETE CASCADE') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('topic_agenda_selections')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('FOREIGN KEY (zid, pid) REFERENCES participants(zid, pid) ON DELETE CASCADE') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_topic_agenda_selections_zid')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_topic_agenda_selections_zid ON public.topic_agenda_selections USING btree (zid)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_topic_agenda_selections_pid')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_topic_agenda_selections_pid ON public.topic_agenda_selections USING btree (pid)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_topic_agenda_selections_delphi_job_id')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_topic_agenda_selections_delphi_job_id ON public.topic_agenda_selections USING btree (delphi_job_id)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_topic_agenda_selections_created_at')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_topic_agenda_selections_created_at ON public.topic_agenda_selections USING btree (created_at)')))) AS catalog_match +UNION ALL +SELECT '000013_create_treevite.sql'::text AS migration, (-- Catalog postconditions only; this file never replays migration DDL. +SELECT (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('conversations')) AND a.attname=('treevite_enabled') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('boolean') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('false'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_waves')) AND a.attname=('id') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('bigint') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('nextval(''treevite_waves_id_seq''::regclass)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_waves')) AND a.attname=('zid') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_waves')) AND a.attname=('wave') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_waves')) AND a.attname=('parent_wave') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_waves')) AND a.attname=('size') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_waves')) AND a.attname=('invites_per_user') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_waves')) AND a.attname=('owner_invites') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('0'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_waves')) AND a.attname=('created_at') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('timestamp with time zone') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('CURRENT_TIMESTAMP'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_waves')) AND a.attname=('updated_at') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('timestamp with time zone') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('CURRENT_TIMESTAMP'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_waves')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('PRIMARY KEY (id)') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_invites')) AND a.attname=('id') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('bigint') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('nextval(''treevite_invites_id_seq''::regclass)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_invites')) AND a.attname=('zid') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_invites')) AND a.attname=('wave_id') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('bigint') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_invites')) AND a.attname=('parent_invite_id') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('bigint') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_invites')) AND a.attname=('status') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('smallint') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('0'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_invites')) AND a.attname=('invite_owner_pid') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_invites')) AND a.attname=('invite_used_by_pid') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_invites')) AND a.attname=('invite_used_at') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('timestamp with time zone') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_invites')) AND a.attname=('created_at') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('timestamp with time zone') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('CURRENT_TIMESTAMP'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_invites')) AND a.attname=('updated_at') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('timestamp with time zone') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('CURRENT_TIMESTAMP'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_invites')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('PRIMARY KEY (id)') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_login_codes')) AND a.attname=('id') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('bigint') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('nextval(''treevite_login_codes_id_seq''::regclass)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_login_codes')) AND a.attname=('zid') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_login_codes')) AND a.attname=('pid') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_login_codes')) AND a.attname=('login_code_hash') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('text') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_login_codes')) AND a.attname=('fp_kid') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('smallint') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('1'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_login_codes')) AND a.attname=('revoked') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('boolean') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('false'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_login_codes')) AND a.attname=('expires_at') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('timestamp with time zone') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_login_codes')) AND a.attname=('last_used_at') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('timestamp with time zone') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_login_codes')) AND a.attname=('created_at') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('timestamp with time zone') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('CURRENT_TIMESTAMP'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_login_codes')) AND a.attname=('updated_at') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('timestamp with time zone') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('CURRENT_TIMESTAMP'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_login_codes')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('PRIMARY KEY (id)') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_waves')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('UNIQUE (zid, wave)') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_waves')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('FOREIGN KEY (zid) REFERENCES conversations(zid) ON DELETE CASCADE') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_invites')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('UNIQUE (zid, invite_code)') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_invites')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('FOREIGN KEY (zid) REFERENCES conversations(zid) ON DELETE CASCADE') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_invites')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('FOREIGN KEY (wave_id) REFERENCES treevite_waves(id) ON DELETE CASCADE') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_invites')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('FOREIGN KEY (parent_invite_id) REFERENCES treevite_invites(id) ON DELETE SET NULL') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_invites')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('FOREIGN KEY (zid, invite_owner_pid) REFERENCES participants(zid, pid)') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_invites')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('FOREIGN KEY (zid, invite_used_by_pid) REFERENCES participants(zid, pid)') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_login_codes')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('FOREIGN KEY (zid, pid) REFERENCES participants(zid, pid) ON DELETE CASCADE') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_login_codes')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('UNIQUE (zid, pid)') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_login_codes')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('UNIQUE (zid, login_code_fingerprint)') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_login_codes')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('UNIQUE (zid, login_code_lookup)') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_treevite_waves_zid')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_treevite_waves_zid ON public.treevite_waves USING btree (zid)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_treevite_waves_wave')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_treevite_waves_wave ON public.treevite_waves USING btree (wave)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_treevite_waves_parent')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_treevite_waves_parent ON public.treevite_waves USING btree (zid, parent_wave)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_treevite_invites_zid')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_treevite_invites_zid ON public.treevite_invites USING btree (zid)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_treevite_invites_zid_status')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_treevite_invites_zid_status ON public.treevite_invites USING btree (zid, status)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_treevite_invites_wave_id')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_treevite_invites_wave_id ON public.treevite_invites USING btree (wave_id)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_treevite_invites_parent')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_treevite_invites_parent ON public.treevite_invites USING btree (parent_invite_id)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_treevite_invites_owner_pid')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_treevite_invites_owner_pid ON public.treevite_invites USING btree (invite_owner_pid)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_treevite_invites_used_by_pid')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_treevite_invites_used_by_pid ON public.treevite_invites USING btree (invite_used_by_pid)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_treevite_invites_code')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_treevite_invites_code ON public.treevite_invites USING btree (invite_code)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_treevite_login_codes_zid')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_treevite_login_codes_zid ON public.treevite_login_codes USING btree (zid)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_treevite_login_codes_pid')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_treevite_login_codes_pid ON public.treevite_login_codes USING btree (pid)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_treevite_login_codes_fp')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_treevite_login_codes_fp ON public.treevite_login_codes USING btree (login_code_fingerprint)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_treevite_login_codes_lookup')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_treevite_login_codes_lookup ON public.treevite_login_codes USING btree (zid, login_code_lookup)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_invites')) AND a.attname=('invite_code') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('character varying(64)') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_login_codes')) AND a.attname=('login_code_fingerprint') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('character varying(128)') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_login_codes')) AND a.attname=('login_code_lookup') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('character varying(128)') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_waves')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('CHECK (((invites_per_user > 0) OR (owner_invites > 0)))') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_waves')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('CHECK (((parent_wave IS NULL) OR (parent_wave >= 0)))') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_waves')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('CHECK (((size IS NULL) OR (size >= 0)))') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_waves')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('CHECK ((invites_per_user >= 0))') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_waves')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('CHECK ((owner_invites >= 0))') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_waves')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('CHECK ((wave >= 1))') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_invites')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('CHECK ((status = ANY (ARRAY[0, 1, 2, 3])))') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O')))) AS catalog_match +UNION ALL +SELECT '000014_alter_reports_modlevel.sql'::text AS migration, (-- Catalog postconditions only; this file never replays migration DDL. +SELECT (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('reports')) AND a.attname=('mod_level') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('smallint') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('''-2''::integer')))) AS catalog_match +UNION ALL +SELECT '000015_add_xid_requirements.sql'::text AS migration, (-- Catalog postconditions only; this file never replays migration DDL. +SELECT (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('conversations')) AND a.attname=('xid_required') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('boolean') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('false'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('xid_whitelist')) AND a.attname=('zid') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('xids')) AND a.attname=('zid') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('xids')) AND a.attname=('pid') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('xid_whitelist')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('FOREIGN KEY (zid) REFERENCES conversations(zid) ON DELETE CASCADE') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('xids')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('FOREIGN KEY (zid) REFERENCES conversations(zid) ON DELETE CASCADE') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('xids')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('FOREIGN KEY (zid, pid) REFERENCES participants(zid, pid) ON DELETE SET NULL') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_xid_whitelist_zid')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_xid_whitelist_zid ON public.xid_whitelist USING btree (zid)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_xid_whitelist_xid')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_xid_whitelist_xid ON public.xid_whitelist USING btree (xid)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_xids_zid')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_xids_zid ON public.xids USING btree (zid)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_xids_xid')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_xids_xid ON public.xids USING btree (xid)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_xids_pid')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_xids_pid ON public.xids USING btree (pid)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_xids_zid_xid')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_xids_zid_xid ON public.xids USING btree (zid, xid)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_xids_uid_zid')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_xids_uid_zid ON public.xids USING btree (uid, zid)')))) AS catalog_match +UNION ALL +SELECT '000016_add_orig_id.sql'::text AS migration, (-- Catalog postconditions only; this file never replays migration DDL. +SELECT (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('comments')) AND a.attname=('original_id') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('uuid') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL)))) AS catalog_match +UNION ALL +SELECT '000017_create_byod_job_table.sql'::text AS migration, (-- Catalog postconditions only; this file never replays migration DDL. +SELECT (SELECT array_agg(enumlabel::text ORDER BY enumsortorder) FROM pg_enum WHERE enumtypid=to_regtype('public.job_status')) = ARRAY['pending','processing','completed','failed'] + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('byod_import_jobs')) AND a.attname=('id') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('nextval(''byod_import_jobs_id_seq''::regclass)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('byod_import_jobs')) AND a.attname=('zid') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('byod_import_jobs')) AND a.attname=('s3_key') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('text') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('byod_import_jobs')) AND a.attname=('status') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('job_status') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('''pending''::job_status'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('byod_import_jobs')) AND a.attname=('stage') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('text') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('''init''::text'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('byod_import_jobs')) AND a.attname=('error_message') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('text') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('byod_import_jobs')) AND a.attname=('created_at') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('timestamp with time zone') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('now()'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('byod_import_jobs')) AND a.attname=('updated_at') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('timestamp with time zone') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('now()'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('byod_import_jobs')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('PRIMARY KEY (id)') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_byod_jobs_zid')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_byod_jobs_zid ON public.byod_import_jobs USING btree (zid)')))) AS catalog_match +UNION ALL +SELECT '000018_add_topics_enabled.sql'::text AS migration, (-- Catalog postconditions only; this file never replays migration DDL. +SELECT (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('conversations')) AND a.attname=('topics_enabled') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('boolean') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('false')))) AS catalog_match +UNION ALL +SELECT '000022_add_poll_timestamp_indexes.sql'::text AS migration, (-- Catalog postconditions only; this file never replays migration DDL. +SELECT (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('votes_created_idx')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX votes_created_idx ON public.votes USING btree (created)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('comments_modified_idx')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX comments_modified_idx ON public.comments USING btree (modified)')))) AS catalog_match +), scope AS ( + SELECT to_regclass('public.migrations') IS NULL + AND to_regclass('public.schema_migrations') IS NULL + AND NOT EXISTS (SELECT 1 FROM pg_class WHERE relnamespace='public'::regnamespace + AND (starts_with(relname,'polis_queue_') OR starts_with(relname,'delphi_'))) + AND NOT EXISTS (SELECT 1 FROM pg_proc WHERE pronamespace='public'::regnamespace + AND (starts_with(proname,'pq_') OR starts_with(proname,'pd_'))) AS initial_state +), authority AS ( + SELECT + has_database_privilege(current_user,current_database(),'TEMP') + AND has_schema_privilege(current_user,'public','USAGE') + AND has_schema_privilege(current_user,'public','CREATE') AS reconcile_authority, + EXISTS (SELECT 1 FROM pg_roles WHERE rolname=current_user AND (rolsuper OR rolcreaterole)) AS create_roles, + NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname IN ('polis_queue_owner','polis_queue_executor')) AS queue_roles_absent, + has_schema_privilege(current_user,'public','USAGE WITH GRANT OPTION') + AND has_schema_privilege(current_user,'public','CREATE WITH GRANT OPTION') AS schema_grants, + EXISTS (SELECT 1 FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace + WHERE n.nspname='public' AND c.relname='conversations' + AND has_table_privilege(current_user,c.oid,'SELECT WITH GRANT OPTION') + AND has_column_privilege(current_user,c.oid,'topic','UPDATE WITH GRANT OPTION') + AND has_column_privilege(current_user,c.oid,'zid','REFERENCES WITH GRANT OPTION')) AS table_grants +), readiness AS ( + SELECT initial_state AND (SELECT bool_and(catalog_match) FROM checks) + AND current_setting('server_version_num')::integer >= 170000 AS can_reconcile, + create_roles AND queue_roles_absent AND schema_grants AND table_grants AS can_provision, + initial_state, reconcile_authority, create_roles, queue_roles_absent, schema_grants, table_grants + FROM scope CROSS JOIN authority +), output AS ( + SELECT migration, catalog_match, + CASE WHEN NOT initial_state THEN 'REVIEW_EXISTING_LEDGER_OR_QUEUE' + WHEN NOT catalog_match THEN 'CATALOG_MISMATCH' + WHEN NOT can_reconcile THEN 'BLOCKED_BY_OTHER_CATALOG_CHECK' + WHEN NOT reconcile_authority THEN 'REVIEW_MIGRATION_SESSION_AUTHORITY' + ELSE 'WOULD_ADOPT' END AS outcome + FROM checks CROSS JOIN readiness + UNION ALL + SELECT migration, NULL::boolean, + CASE WHEN NOT can_reconcile THEN 'BLOCKED_RECONCILIATION' + WHEN NOT reconcile_authority OR NOT can_provision THEN 'REVIEW_MIGRATION_SESSION_AUTHORITY' + ELSE 'WOULD_APPLY' END + FROM (VALUES ('000019_create_polis_queue.sql'),('000023_create_delphi_foundation.sql'), + ('000024_create_polis_queue_large_class.sql'), + ('000027_create_sealed_job_graphs.sql')) p(migration) CROSS JOIN readiness + UNION ALL + SELECT migration, NULL::boolean, 'OUTSIDE_RELEASE' + FROM (VALUES ('000020'),('000021'),('000025'),('000026')) p(migration) +) +SELECT migration, catalog_match, outcome FROM output ORDER BY migration; +ROLLBACK; diff --git a/server/src/db/migrations.cjs b/server/src/db/migrations.cjs new file mode 100644 index 0000000000..d7e6091776 --- /dev/null +++ b/server/src/db/migrations.cjs @@ -0,0 +1,62 @@ +/* eslint-disable no-restricted-properties -- Bootstrap must read its environment before config.ts and application modules load. */ +// Read-only startup gate. Uses the same raw SQL hashes and release-wide hold +// as polis-migrate; no application modules or background loops load before it. +const fs = require("node:fs"); +const path = require("node:path"); +const crypto = require("node:crypto"); +const { Client } = require("pg"); +const isTrue = require("boolean"); + +async function checkMigrations() { + const dir = process.env.POLIS_MIGRATIONS_DIR || path.resolve("postgres/migrations"); + function manifest(file) { + const names = new Set(); + for (const name of fs.readFileSync(path.join(dir, file), "utf8").split(/\r?\n/).filter((s) => s && !s.startsWith("#"))) { + if (name.trim() !== name || !/^\d{6}_[a-zA-Z0-9_]+\.sql$/.test(name) || names.has(name)) throw new Error(`Invalid or duplicate migration in ${file}: ${name}`); + names.add(name); + } + return names; + } + const selected = manifest("release.txt"); + const held = manifest("held.txt"); + if ([...selected].some((name) => held.has(name))) throw new Error("Release and held manifests overlap"); + const required = new Map(); + const versions = new Set(); + for (const name of fs.readdirSync(dir).sort()) { + if (!name.endsWith(".sql")) continue; + if (name.trim() !== name || !/^\d{6}_[a-zA-Z0-9_]+\.sql$/.test(name) || versions.has(name.slice(0, 6))) { + throw new Error(`Invalid or duplicate migration: ${name}`); + } + versions.add(name.slice(0, 6)); + if (!fs.lstatSync(path.join(dir, name)).isFile()) throw new Error(`Non-file migration: ${name}`); + if (held.has(name)) { held.delete(name); continue; } + if (!selected.delete(name)) throw new Error(`Migration absent from release/held manifest: ${name}`); + required.set(name, crypto.createHash("sha256").update(fs.readFileSync(path.join(dir, name))).digest("hex")); + } + if (held.size || selected.size || !required.has("000000_initial.sql")) throw new Error("Incomplete migration source directory"); + if (!process.env.DATABASE_URL) throw new Error("DATABASE_URL is required for migration check"); + const db = new Client({ connectionString: process.env.DATABASE_URL, + connectionTimeoutMillis: 10000, application_name: "polis-startup-migrations", + ssl: isTrue(process.env.DATABASE_SSL) ? { rejectUnauthorized: true } : undefined }); + try { + await db.connect(); + await db.query("BEGIN READ ONLY"); + await db.query("SET LOCAL statement_timeout='10s'; SET LOCAL lock_timeout='5s'"); + const { rows } = await db.query("SELECT name,checksum,status FROM public.migrations ORDER BY name"); + for (const row of rows) { + if (!required.has(row.name) || required.get(row.name) !== row.checksum || !["APPLIED", "ADOPTED"].includes(row.status)) { + throw new Error(`Migration history mismatch: ${row.name}`); + } + required.delete(row.name); + } + if (required.size) throw new Error(`Pending migrations: ${[...required.keys()].join(", ")}; run polis-migrate apply`); + await db.query("COMMIT"); + } catch (error) { + // Database DETAIL may contain data. Print only our own errors or SQLSTATE. + if (error.code) throw new Error(`Migration check failed (SQLSTATE ${error.code}); run polis-migrate check; reconcile legacy databases per docs/migrations.md`); + throw error; + } finally { await db.end(); } +} +module.exports = { checkMigrations }; +if (require.main === module) checkMigrations().then(() => process.stdout.write("migration check ready\n")) + .catch((error) => { process.stderr.write(`${error.message}\n`); process.exitCode = 1; });