diff --git a/.github/workflows/cypress-tests.yml b/.github/workflows/cypress-tests.yml index bb7313c280..2272503842 100644 --- a/.github/workflows/cypress-tests.yml +++ b/.github/workflows/cypress-tests.yml @@ -85,6 +85,9 @@ jobs: # Build remaining services (caching OK for these) docker compose -f docker-compose.test.yml --env-file test.env build + - name: Apply and check test database migrations + run: bash ci/test-migrations.sh + - name: Start services run: | # Start all services in detached mode @@ -97,24 +100,6 @@ jobs: # Show running containers docker compose -f docker-compose.test.yml ps - - name: Run Database Migrations - env: - DATABASE_URL: postgres://postgres:PdwPNS2mDN73Vfbc@localhost:5432/polis-test - POSTGRES_DB: polis-test - POSTGRES_HOST: postgres:5432 - POSTGRES_PASSWORD: PdwPNS2mDN73Vfbc - POSTGRES_PORT: 5432 - POSTGRES_USER: postgres - run: | - echo "Installing postgres-client..." - sudo apt-get update && sudo apt-get install -y postgresql-client - - echo "Making migration script executable..." - chmod +x server/bin/run-migrations.sh - - echo "Running migrations..." - ./server/bin/run-migrations.sh - - name: Check service health run: | # Check if key services are responding diff --git a/.github/workflows/delphi-characterization.yml b/.github/workflows/delphi-characterization.yml index 78d3d6349c..a657e47def 100644 --- a/.github/workflows/delphi-characterization.yml +++ b/.github/workflows/delphi-characterization.yml @@ -64,6 +64,8 @@ jobs: run: | python -m pip install --quiet pyyaml python -m unittest tests.test_delphi_storage_codec -v + python -m unittest tests.test_delphi_postgres_results -v + python -m unittest tests.test_delphi_result_resource -v - name: Codec golden test (Node reads the files Python wrote, writes the cross file) working-directory: server diff --git a/.github/workflows/dynamo-removal.yml b/.github/workflows/dynamo-removal.yml new file mode 100644 index 0000000000..0618553f7f --- /dev/null +++ b/.github/workflows/dynamo-removal.yml @@ -0,0 +1,63 @@ +name: Dynamo removal + +on: + pull_request: + paths: + - 'delphi/**' + - 'queue-rs/**' + - 'server/**' + - 'client-report/**' + - 'scripts/test-dynamo-removal.sh' + - 'scripts/prove-dynamo-report.sh' + - 'ci/dynamo-removal/**' + - '.github/workflows/dynamo-removal.yml' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: dynamo-removal-${{ github.ref }} + cancel-in-progress: true + +jobs: + postgres-report: + runs-on: ubuntu-24.04 + timeout-minutes: 150 + env: + COMPOSE_PROJECT_NAME: polis-graph-test-dynamo-${{ github.run_id }}-${{ github.run_attempt }} + POLIS_RECOVERY_PG_PORT: '55449' + RECOVERY_PG_PORT: '55449' + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: '3.12' + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: '22' + - name: Queue toolchain + working-directory: queue-rs + run: rustup show active-toolchain + - name: Install uv + run: python -m pip install uv==0.9.2 + - name: Queue, Postgres results, importer and report with DynamoDB stopped + run: bash scripts/test-dynamo-removal.sh + - name: Retain proof logs and report screenshots + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: dynamo-removal-proof + retention-days: 7 + include-hidden-files: true + path: | + .dynamo-proof/${{ env.COMPOSE_PROJECT_NAME }}/*.log + .dynamo-proof/${{ env.COMPOSE_PROJECT_NAME }}/demo-result.json + .dynamo-proof/${{ env.COMPOSE_PROJECT_NAME }}/sql-source-sha256.json + .dynamo-proof/${{ env.COMPOSE_PROJECT_NAME }}/install/results.json + .dynamo-proof/${{ env.COMPOSE_PROJECT_NAME }}/results-sql/results.json + .dynamo-proof/${{ env.COMPOSE_PROJECT_NAME }}/report/*.log + .dynamo-proof/${{ env.COMPOSE_PROJECT_NAME }}/report/browser.json + .dynamo-proof/${{ env.COMPOSE_PROJECT_NAME }}/report/*.png diff --git a/.github/workflows/jest-server-test.yml b/.github/workflows/jest-server-test.yml index fe54f8593e..0b7122b224 100644 --- a/.github/workflows/jest-server-test.yml +++ b/.github/workflows/jest-server-test.yml @@ -97,6 +97,9 @@ jobs: docker compose -f docker-compose.test.yml --env-file test.env build \ postgres file-server ses-local oidc-simulator dynamodb + - name: Apply and check test database migrations + run: bash ci/test-migrations.sh + - name: Start services run: | # Start only required services in detached mode (exclude server + math-python) @@ -144,6 +147,13 @@ jobs: cd server npm run contract:check + - name: Check API migration readiness + working-directory: server + env: + DATABASE_URL: postgres://postgres:PdwPNS2mDN73Vfbc@localhost:5432/polis-test + DATABASE_SSL: "false" + run: node src/db/migrations.cjs + - name: Run server integration tests run: | cd server diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index ccdd364b7e..39508bc708 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -17,6 +17,11 @@ on: - 'scripts/test-deploy-hooks.sh' - 'docker-compose*.yml' - '.github/workflows/python-ci.yml' + - 'ci/test-migrations.sh' + - 'docker-compose.test.yml' + - 'server/Dockerfile-db' + - 'server/postgres/**' + - 'queue-rs/polis-migrate/**' # server/src so a new server-side wildcard runs the projection-gate sweep - 'server/src/**' # Representative payload tests execute the probe planner and independent gate. @@ -75,9 +80,9 @@ jobs: # Build all services in the test file (including delphi) docker compose -f docker-compose.test.yml --env-file .env build - - name: 4. Start all services - run: | - # Start all services (including delphi) in detached mode + # Apply/check before starting application services on this volume. + bash ci/test-migrations.sh + # The 'delphi' container will start and run 'tail -f /dev/null' docker compose -f docker-compose.test.yml --env-file .env up -d @@ -95,8 +100,7 @@ jobs: # The opt-in Postgres integration tests (require_polis_postgres) run here # against the compose `postgres` service, whose image bakes the polis - # migrations (server/postgres/migrations/*.sql via docker-entrypoint-initdb.d), - # so the votes / votes_latest_unique schema + on_vote_insert_update_unique_table + # migrations through polis-migrate during initialization and the explicit CI step, # rule are already applied. The pytest step exports POLIS_TEST_POSTGRES_URL; # the cold-start generator under test is already baked into the delphi image # (Dockerfile `COPY scripts/ ./scripts/`), built from this checkout. diff --git a/.github/workflows/queue-rs-ci.yml b/.github/workflows/queue-rs-ci.yml index 2c9e166aed..1adcf1f13c 100644 --- a/.github/workflows/queue-rs-ci.yml +++ b/.github/workflows/queue-rs-ci.yml @@ -9,6 +9,17 @@ on: pull_request: paths: - 'queue-rs/**' + - 'delphi/scripts/job_graph_*.py' + - 'delphi/tests/job_graph/**' + - 'docs/job-graphs.md' + - 'server/__tests__/integration/queue-substrate.test.ts' + - 'server/jest.job-graphs.config.ts' + - 'server/package*.json' + - 'server/index.ts' + - 'server/src/db/migrations.cjs' + - 'server/Dockerfile-db' + - 'server/postgres/init-migrations.sh' + - 'scripts/after_install.sh' - 'file-server/nginx/**' - 'file-server/nginx.Dockerfile' - 'server/postgres/migrations/**' @@ -17,6 +28,17 @@ on: branches: [edge, stable] paths: - 'queue-rs/**' + - 'delphi/scripts/job_graph_*.py' + - 'delphi/tests/job_graph/**' + - 'docs/job-graphs.md' + - 'server/__tests__/integration/queue-substrate.test.ts' + - 'server/jest.job-graphs.config.ts' + - 'server/package*.json' + - 'server/index.ts' + - 'server/src/db/migrations.cjs' + - 'server/Dockerfile-db' + - 'server/postgres/init-migrations.sh' + - 'scripts/after_install.sh' - 'file-server/nginx/**' - 'file-server/nginx.Dockerfile' - 'server/postgres/migrations/**' @@ -110,3 +132,73 @@ jobs: - name: nginx routing switch (off, absent/up/502/dead, bad names and settings, method and body gate, truncation limit) working-directory: . run: queue-rs/polis-api/conformance/nginx-routing.sh + + polis-migrate: + name: migration runner and startup refusal + runs-on: ubuntu-24.04 + timeout-minutes: 30 + env: + COMPOSE_PROJECT_NAME: polis-migrate-test-ci-${{ github.run_id }}-${{ github.run_attempt }} + POLIS_RECOVERY_PG_PORT: '55850' + RECOVERY_PG_PORT: '55850' + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + persist-credentials: false + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 + with: + node-version: '22.23.3' + - name: Build and lint the runner + working-directory: queue-rs + run: | + cargo fmt -p polis-migrate --check + cargo clippy --locked -p polis-migrate --all-targets -- -D warnings + cargo test --locked -p polis-migrate + cargo build --locked -p polis-migrate + - name: Compile the API entrypoint + working-directory: server + run: | + npm ci --ignore-scripts --no-audit --no-fund + npm run build + - name: Isolated real PostgreSQL proofs + run: | + docker compose -f queue-rs/polis-migrate/tests/compose.yml up -d --wait + python3 server/bin/build-migration-report.py --check + python3 queue-rs/polis-migrate/tests/prove.py + python3 queue-rs/polis-migrate/tests/indexes.py + python3 queue-rs/polis-migrate/tests/adoption.py + python3 queue-rs/polis-migrate/tests/selection.py + python3 queue-rs/polis-migrate/tests/lexer.py + - name: Prove the fresh database image and restart + run: bash queue-rs/polis-migrate/tests/fresh-image.sh + - name: Remove only the owned test project + if: always() + run: docker compose -f queue-rs/polis-migrate/tests/compose.yml down -v + + job-graphs: + name: Per-step job graph core + runs-on: ubuntu-24.04 + timeout-minutes: 30 + env: + COMPOSE_PROJECT_NAME: polis-graph-test-ci-${{ github.run_id }}-${{ github.run_attempt }} + POLIS_RECOVERY_PG_PORT: '55467' + RECOVERY_PG_PORT: '55467' + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + - uses: actions/setup-node@v4 + with: + node-version: 24 + - name: Install Node adapter test dependencies + working-directory: server + run: npm ci --no-audit --no-fund + - name: Prove saved work, retries and complete publication + run: bash delphi/tests/job_graph/run.sh + - name: Retain generated proof receipts + if: always() + uses: actions/upload-artifact@v4 + with: + name: job-graph-proof + path: graph-proof/ + retention-days: 7 diff --git a/CHANGELOG.md b/CHANGELOG.md index 2995ba5a3f..ba2edcccd7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## Release B (unreleased) + +- Deploy reconciles the supported existing schema, applies selected migrations, + and checks readiness before replacing services; Release A hooks must already + be installed on each host. +- Select M19/M23/M24 and the trimmed M27 job graph core. Each step keeps its + result and inputs; dependencies wait, failed steps retry, and complete results + publish atomically. DynamoDB readers/importers remain outside this release. +- Refuse API startup when selected migration receipts are missing or changed. + ## 1.0 @@ -31,3 +41,20 @@ Changes which have been merged to `edge` but are not yet versioned on `stable` c * ... + +## Migration runner + +Deployments now apply pending numbered migrations through `polis-migrate` before +service replacement. The API refuses startup with pending or mismatched history. +Existing databases require one-time catalog-checked adoption. See +[upgrading](docs/upgrading.md) for the first CodeDeploy hook transition and the +release-wide coordinator hold. PostgreSQL 17+ is required. + +The explicit release manifest admits the supported legacy schema then applies +M19/M23/M24; M20/M21/M25/M26 stay outside the forward path. Deprecated M4/M5/M7 +are observation-only, never automatic destructive steps. Exact legacy type +alternatives, a catalog-only first-deploy report, per-deployment upgrade notes +and a source-to-release map are documented in +[migration upgrade notes](docs/migration-upgrade-notes.md). Historical SQL +checksums remain unchanged; no semantic version is invented for unversioned +historical releases. diff --git a/Makefile b/Makefile index 7fbf8ff6ea..23ff6b5650 100644 --- a/Makefile +++ b/Makefile @@ -177,8 +177,7 @@ refresh-devdb: ## Force dev DB mode (migrations), drop postgres_data volume, and # P-022 §C — poller recovery matrix (R01-R12) on a REAL Postgres # ---------------------------------------------------------------------------- # # Reuses docker-compose.test.yml's postgres service (built from -# server/Dockerfile-db, which bakes server/postgres/migrations/*.sql into -# docker-entrypoint-initdb.d) with docker-compose.recovery.yml overriding the +# server/Dockerfile-db, which runs polis-migrate during fresh initialization) with docker-compose.recovery.yml overriding the # host port and making the data directory a tmpfs, so every `up` re-runs initdb # with the real migrations and nothing survives teardown. # diff --git a/bin/run-migrations.clj b/bin/run-migrations.clj index 67bdb65d0d..72b875d1a4 100755 --- a/bin/run-migrations.clj +++ b/bin/run-migrations.clj @@ -1,143 +1,5 @@ #!/usr/bin/env bb - -(require '[babashka.pods :as pods] - '[babashka.deps :as deps] - '[clojure.pprint :as pp] - '[clojure.tools.cli :as cli] - '[clojure.java.io :as io] - '[clojure.string :as string]) - -(pods/load-pod 'org.babashka/postgresql "0.0.1") -(deps/add-deps '{:deps {honeysql/honeysql {:mvn/version "1.0.444"}}}) - -(require '[pod.babashka.postgresql :as pg] - '[honeysql.core :as hsql] - '[honeysql.helpers :as hsqlh]) - - - -(def db-url - (System/getenv "DATABASE_URL")) - -(defn heroku-url-spec [db-url] - (let [[_ user password host port db] (re-matches #"postgres://(?:(.+):(.*)@)?([^:]+)(?::(\d+))?/(.+)" db-url)] - {:dbtype "postgresql" - :host host - :dbname db - :port (or port 80) - :user user - :password password})) - -(defn execute-sql! [args] - (println "Executing sql:" args) - (pg/execute! - (heroku-url-spec (System/getenv "DATABASE_URL")) - args)) - - -;(def execute-sql! - ;(partial pg/execute! (heroku-url-spec (System/getenv "DATABASE_URL")))) - -(defn execute! - [query-or-command] - (execute-sql! (hsql/format query-or-command))) - - -(defn insert! - [table values] - (execute! {:insert-into table - :values values})) - -;(hsqlh/values [{:a "this" :b 4}]) - -;(-> (hsqlh/insert-into :migrations) - ;(hsqlh/values [{:a "this" :b 3}]) - ;(hsql/format)) - - - -;; get about the migration business - -(def migrations-path "server/postgres/migrations/") - -(defn sql-file? - [file] - (re-matches #".*\.sql" (str file))) - -(defn table-exists? - [table-name] - (-> - (execute! {:select [:*] - :from [:information_schema.tables] - :where [:= :table_name (name table-name)]}) - (not-empty) - (boolean))) - -;(table-exists? :migrations) -;(table-exists? :fish) - -(defn remember-tx-migration! [name] - (insert! :migrations - [{:name name - :completed_at (System/currentTimeMillis)}])) - -;; Make sure we have a migrations table, which is basically just a list of filenames which have been -;; transacted, as well as datetime -(when-not (table-exists? :migrations) - (execute-sql! - ["CREATE TABLE migrations - (name VARCHAR(999) NOT NULL, - completed_at BIGINT NOT NULL);"])) - -(defn process-mig-file! [mig-file] - (let [mig-file (io/file mig-file) - name (.getName mig-file)] - (println "Processing migration file" name) - (execute-sql! [(slurp mig-file)]) - (remember-tx-migration! name))) - -(defn migration-files [] - (->> (.listFiles (io/file migrations-path)) - (remove #(.isDirectory %)) - (filter sql-file?) - (sort))) - -(defn remove-past-migrations - [mig-files] - (let [past-migrations - (->> - (execute! {:select [:name] - :from [:migrations]}) - (map :migrations/name) - (set))] - (remove (comp past-migrations #(.getName %)) - mig-files))) - -(defn new-migration-files - [] - (remove-past-migrations (migration-files))) - -;(remove-past-migrations (migration-files)) - -;(.getName (io/file "server/postgres/migrations/000000_initial.sql")) -;(.getParent (io/file "server/postgres/migrations/000000_initial.sql")) - -(when-not (table-exists? :conversations) - (process-mig-file! "server/postgres/migrations/000000_initial.sql") - (remember-tx-migration! "000000_initial.sql")) - -(when-not (table-exists? :pwreset_tokens) - (process-mig-file! "server/postgres/migrations/000001_update_pwreset_table.sql") - (remember-tx-migration! "000001_update_pwreset_table.sql")) - -(def past-migrations - (execute! {:select [:*] - :from [:migrations]})) - -(let [mig-files (new-migration-files)] - (if (empty? mig-files) - (println "No new migrations to run") - (doseq [mig-file (new-migration-files)] - (process-mig-file! mig-file)))) - - +;; The 2021 ledger idea continues in polis-migrate. Do not replay all SQL here. +(require '[babashka.process :as process]) +(let [result @(process/process ["bash" "server/bin/run-migrations.sh"] {:inherit true})] + (System/exit (:exit result))) diff --git a/ci/dynamo-removal/report-nginx.conf b/ci/dynamo-removal/report-nginx.conf new file mode 100644 index 0000000000..ccd01eb882 --- /dev/null +++ b/ci/dynamo-removal/report-nginx.conf @@ -0,0 +1,10 @@ +events {} +http { + include /etc/nginx/mime.types; + server { + listen 8080; + root /report; + location /api/ { proxy_pass http://astra-dynamo1424-server:5000; proxy_read_timeout 180s; } + location / { try_files $uri /index_report.html; } + } +} diff --git a/ci/dynamo-removal/report-proof.cjs b/ci/dynamo-removal/report-proof.cjs new file mode 100644 index 0000000000..94c680d20b --- /dev/null +++ b/ci/dynamo-removal/report-proof.cjs @@ -0,0 +1,78 @@ +/* Read-only browser proof: use generated local reports, never a production URL. */ +const fs=require('node:fs'); +const path=require('node:path'); +const assert=require('node:assert/strict'); +const {assertFixtureReports,assertReportCoverage}=require('./report-quality.cjs'); +const playwright=process.env.DYNAMO_PROOF_PLAYWRIGHT; +assert(playwright,'DYNAMO_PROOF_PLAYWRIGHT must point to an installed local Playwright package'); +const {chromium}=require(playwright); +const base=process.env.DYNAMO_PROOF_REPORT_URL; +assert(/^http:\/\/(127\.0\.0\.1|localhost):\d+$/.test(base),'local browser proof URL required'); +const rid=process.env.DYNAMO_PROOF_REPORT_ID; +assert(/^rlocal[a-z0-9]+$/.test(rid),'generated local report ID required'); +const output=process.env.DYNAMO_PROOF_OUTPUT; +assert(output,'DYNAMO_PROOF_OUTPUT required');fs.mkdirSync(output,{recursive:true}); +(async()=>{ + const browser=await chromium.launch({headless:true});const receipts=[]; + try { + for(const route of ['report','topicStats','topicReport']){ + const page=await browser.newPage({viewport:{width:1440,height:1100}}); + const errors=[];const responses=[];let narrativeEvidence; + page.on('pageerror',error=>errors.push(error.message)); + page.on('response',async response=>{ + if(!response.url().includes('/api/'))return; + let body;try{body=await response.text()}catch{body='unavailable'} + responses.push({url:response.url(),status:response.status(),body}); + }); + await page.goto(`${base}/${route}/${rid}`,{waitUntil:'networkidle'}); + if(route==='report') { + const response=await page.request.get(`${base}/api/v3/delphi/visualizations?report_id=${encodeURIComponent(rid)}`); + assert(response.ok(),'visualization metadata endpoint succeeded'); + const metadata=await response.json();assert.equal(metadata.status,'success'); + assert(metadata.jobs?.length>0,'actual queue metadata returned with DynamoDB unavailable'); + assert(metadata.jobs.every(job=>job.workLive===false),'published completed graph has no live work'); + responses.push({url:response.url(),status:response.status(),body:JSON.stringify(metadata)}); + } + if(route==='topicStats')await page.getByText('Group Consensus',{exact:false}).first().waitFor({timeout:30000}); + if(route==='topicReport'){ + const selector=page.locator('select'); + await selector.first().waitFor({timeout:30000}); + const options=await selector.first().locator('option').evaluateAll(options=>options.map(o=>({value:o.value,text:o.textContent}))); + const sourceResponse=await page.request.get(`${base}/api/v3/delphi/reports?report_id=${encodeURIComponent(rid)}`); + assert(sourceResponse.ok(),'narrative source endpoint succeeded'); + const source=await sourceResponse.json();assert.equal(source.status,'success'); + const checked=assertFixtureReports(source.reports); + const topicResponse=await page.request.get(`${base}/api/v3/delphi?report_id=${encodeURIComponent(rid)}`); + assert(topicResponse.ok(),'topic source endpoint succeeded'); + const topics=await topicResponse.json();assert.equal(topics.status,'success'); + const checkedTopics=Object.values(topics.runs).flatMap(run=>Object.values(run.topics_by_layer).flatMap(Object.values)).length; + assertReportCoverage(source.reports,topics.runs); + const named=options.find(o=>o.value && source.reports?.[o.value]?.report_data); + assert(named,'at least one available narrative section maps to the rendered selector'); + const stored=source.reports[named.value]; + const {clauses}=checked[named.value]; + await selector.first().selectOption(named.value); + await page.waitForLoadState('networkidle'); + await page.locator('.topic-text-content .paragraph').first().waitFor({timeout:30000}); + const rendered=await page.locator('.topic-text-content').innerText(); + for(const clause of clauses)assert(rendered.includes(clause),'stored narrative clause renders exactly'); + narrativeEvidence={section:named.value,model:stored.model,job_id:stored.job_id,clauses:clauses.length,metadata:stored.metadata,checkedSections:Object.keys(checked),checkedTopics}; + } + const body=await page.locator('body').innerText(); + await page.screenshot({path:path.join(output,route+'.png'),fullPage:true}); + await page.screenshot({path:path.join(output,route+'-viewport.png')}); + const record={route,body,errors,responses,narrativeEvidence};receipts.push(record); + fs.writeFileSync(path.join(output,'browser.json'),JSON.stringify(receipts,null,2)); + assert.equal(errors.length,0,route+': '+errors.join('; ')); + assert(responses.length>0,route+': API requests observed'); + assert(responses.every(r=>r.status<400),route+': '+responses.filter(r=>r.status>=400).map(r=>r.url)); + for(const response of responses){let payload;try{payload=JSON.parse(response.body)}catch{continue}assert.notEqual(payload.status,'error',response.url+': application error')} + // Existing date-display behavior is outside this storage/queue proof. + // Preserve it with the default-backend route and view recordings. + if(route==='report')assert(/people voted/.test(body),route+': participant summary rendered'); + if(route==='topicReport')assert(body.length>300,route+': narrative content rendered'); + console.log('PASS',route,'API responses',responses.length); + await page.close(); + } + } finally {await browser.close()} +})().catch(error=>{console.error(error);process.exitCode=1}); diff --git a/ci/dynamo-removal/report-quality.cjs b/ci/dynamo-removal/report-quality.cjs new file mode 100644 index 0000000000..383196285d --- /dev/null +++ b/ci/dynamo-removal/report-quality.cjs @@ -0,0 +1,38 @@ +const assert = require('node:assert/strict'); + +function assertReportCoverage(reports, runs) { + const topics = Object.values(runs || {}).flatMap(run => Object.values(run.topics_by_layer || {}).flatMap(Object.values)); + assert(topics.length > 0, 'topic coverage requires served topics'); + const expected = topics.map(topic => { + assert(/^.+#\d+#\d+$/.test(topic.topic_key), 'valid topic key required'); + return topic.topic_key.replaceAll('#', '_'); + }); + assert.equal(new Set(expected).size, expected.length, 'duplicate topic section'); + const jobs = new Set(topics.map(topic => topic.topic_key.split('#')[0])); + assert.equal(jobs.size, 1, 'one coherent served narrative job required'); + const [job] = jobs; + expected.push(...['groups', 'group_informed_consensus', 'uncertainty'].map(name => `${job}_global_${name}`)); + const actual = Object.entries(reports || {}).map(([key, report]) => { + assert.equal(report.section, key, 'duplicate or mismatched report section'); + assert.equal(report.job_id, job, 'report belongs to served narrative job'); + return key; + }); + assert.deepEqual(actual.sort(), expected.sort(), 'complete topic and global section coverage'); + return expected.length; +} + +function assertFixtureReports(reports) { + assert(reports && Object.keys(reports).length > 0, 'served reports required'); + const checked = {}; + for (const [section, stored] of Object.entries(reports)) { + const document = typeof stored.report_data === 'string' ? JSON.parse(stored.report_data) : stored.report_data; + assert.equal(stored.model, 'local-narrative-fixture/1'); + assert.equal(stored.metadata?.provider_fixture, true); + assert.equal(document.provider_fixture, true); + const clauses = document.paragraphs.flatMap(p => p.sentences.flatMap(s => s.clauses.map(c => c.text))); + assert.deepEqual(clauses, ['Fixed narrative stand-in for queue, Postgres storage and report rendering proof. No LLM provider was called.']); + checked[section] = { document, clauses }; + } + return checked; +} +module.exports = { assertFixtureReports, assertReportCoverage }; diff --git a/ci/dynamo-removal/report-quality.test.cjs b/ci/dynamo-removal/report-quality.test.cjs new file mode 100644 index 0000000000..9e2cca314d --- /dev/null +++ b/ci/dynamo-removal/report-quality.test.cjs @@ -0,0 +1,33 @@ +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const { assertReportCoverage } = require('./report-quality.cjs'); +const coverageFixture = () => { + const runs = { current: { topics_by_layer: { 0: { + 0: { topic_key: 'generated#0#0' }, 1: { topic_key: 'generated#0#1' } + } } } }; + const sections = ['generated_0_0', 'generated_0_1', 'generated_global_groups', + 'generated_global_group_informed_consensus', 'generated_global_uncertainty']; + return { runs, reports: Object.fromEntries(sections.map(section => [section, { section, job_id: 'generated' }])) }; +}; +test('requires all topic and global sections from one served job', () => { + const { reports, runs } = coverageFixture(); + assert.equal(assertReportCoverage(reports, runs), 5); +}); +test('rejects a missing topic or global even when remaining reports are valid', () => { + for (const missing of ['generated_0_1', 'generated_global_uncertainty']) { + const { reports, runs } = coverageFixture(); delete reports[missing]; + assert.throws(() => assertReportCoverage(reports, runs), /complete topic and global/); + } +}); +test('rejects duplicate topic keys or duplicate report identities', () => { + const { reports, runs } = coverageFixture(); + runs.current.topics_by_layer[0][1].topic_key = 'generated#0#0'; + assert.throws(() => assertReportCoverage(reports, runs), /duplicate topic/); + const fresh = coverageFixture(); + fresh.reports.generated_0_1.section = 'generated_0_0'; + assert.throws(() => assertReportCoverage(fresh.reports, fresh.runs), /duplicate or mismatched/); +}); +test('rejects reports from a different job', () => { + const { reports, runs } = coverageFixture(); reports.generated_0_0.job_id = 'other-job'; + assert.throws(() => assertReportCoverage(reports, runs), /served narrative job/); +}); diff --git a/ci/p027_rerecord_build.sh b/ci/p027_rerecord_build.sh index 33eec4134f..1c4394ca63 100644 --- a/ci/p027_rerecord_build.sh +++ b/ci/p027_rerecord_build.sh @@ -6,7 +6,7 @@ export DOCKER_BUILDKIT=1 # Sequential builds bound peak memory; inspect disk receipts when sizing the runner. df -h . docker build --target prod --build-arg NODE_ENV=production -t p027-server -f server/Dockerfile server -docker build -t p027-postgres -f server/Dockerfile-db server +docker build --build-context queue-rs=queue-rs -t p027-postgres -f server/Dockerfile-db server docker build -t p027-oidc-simulator oidc-simulator docker build -t p027-file-server --build-arg NODE_ENV=production \ --build-arg AUTH_AUDIENCE=users --build-arg AUTH_CLIENT_ID=dev-client-id \ diff --git a/ci/test-migrations.sh b/ci/test-migrations.sh new file mode 100644 index 0000000000..fb92f902e8 --- /dev/null +++ b/ci/test-migrations.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# Disposable CI stack only. Use the runner baked into the Postgres image on +# fresh AND existing volumes before starting application services. No host +# Rust/psql installation, second history table or startup-check bypass. +# Optional Compose arguments (e.g. -f local-ports.yml) preserve test isolation. +set -euo pipefail +root=$(cd "$(dirname "$0")/.." && pwd) +cd "$root" +compose=(docker compose -f docker-compose.test.yml "$@" --env-file "${POLIS_TEST_ENV_FILE:-test.env}") +"${compose[@]}" up -d --wait --wait-timeout 120 postgres +"${compose[@]}" exec -T postgres sh -eu -c ' + export DATABASE_URL="host=/var/run/postgresql user=$POSTGRES_USER dbname=$POSTGRES_DB sslmode=disable" + export POLIS_MIGRATIONS_DIR=/migrations + polis-migrate apply + polis-migrate check +' diff --git a/delphi/docs/LEGACY_DYNAMO_IMPORT.md b/delphi/docs/LEGACY_DYNAMO_IMPORT.md new file mode 100644 index 0000000000..6e160e7789 --- /dev/null +++ b/delphi/docs/LEGACY_DYNAMO_IMPORT.md @@ -0,0 +1,60 @@ +# Import a local DynamoDB export + +`scripts/import_dynamo_export.py` moves a bounded, checksummed export into a +normal Postgres graph job. The worker verifies source bytes, codec version, +worker code, importer code and runtime; normal queue finalization stores its +immutable artifact and M28 result rows together. No historical job executes. + +The export command requires an explicit loopback endpoint and only uses dummy +local credentials. It never falls back to an AWS endpoint. Specify every family +to export with repeated `--family` arguments. The exporter reads all scan pages +and writes the frozen `delphi-storage-codec/1` files and `SHA256SUMS`. +Keep the source quiescent during export. Consistent scan pages do not provide +a transaction snapshot across a whole table or all families; checksums verify +the completed export but cannot detect changes between scan pages. + +```sh +python scripts/import_dynamo_export.py export-local /tmp/generated-export \ + --endpoint http://127.0.0.1:8000 \ + --family Delphi_CommentEmbeddings --family Delphi_NarrativeReports \ + --family Delphi_JobQueue --family Delphi_JobActiveGuard +python scripts/import_dynamo_export.py preview /tmp/generated-export \ + --zid "$DEMO_ZID" --report-id "$DEMO_REPORT" +python scripts/import_dynamo_export.py enqueue /tmp/generated-export \ + --zid "$DEMO_ZID" --report-id "$DEMO_REPORT" --env local-demo --scope import-demo +``` + +For enqueue, `QUEUE_DATABASE_URL` must use a queue executor login. +`DATABASE_URL` supplies read access to verify that every explicitly named +report belongs to the selected conversation. Preview is offline; it checks +explicit source bindings but cannot verify the live reports mapping. Neither +command remaps conversation IDs or report IDs. Mixed-conversation exports fail. + +The `delphi` worker claims `graph_narrative` and runs the declared +`legacy-dynamo-export/1` model. Repeating the same enqueue returns the same +graph. Source, code or runtime changes produce a new request; an active scope +still prevents overlapping admission. Admission does not publish the result. +After successful execution, publication uses the ordinary explicit +`pd_graph_publish` generation check, so a failed import cannot replace a report. +The approved core contract does not support replacing a failed graph with a +superseding branch; that capability is deferred to #1436. The importer exposes +ordinary admission, status verification and publication only. + +Every source row is counted as one of: + +- Imported result rows in the 18 M28 result families. +- Archived T15/T16 job and guard rows in `legacy_control_files`, with exact + codec bytes on the immutable artifact. They are never converted to current + jobs, active scopes, leases or provider submissions. +- Quarantined result rows containing NUL strings/keys, which JSONB cannot + represent. Their exact codec bytes and `postgres-jsonb-nul` reason remain in + `quarantine` on the immutable artifact. Binary zero bytes are valid base64 + codec data and are imported normally. + +Unknown families, duplicate/noncanonical keys, missing/changed/unlisted files, +wrong conversation/report bindings and oversized artifacts fail before enqueue. +The initial importer is bounded to 450,000 serialized output bytes and the +queue's existing input limit. It refuses larger exports without truncation; +large archives need a separate artifact transport. It processes result values +as tagged codec values, preserving decimal numbers, sets, binary values and +JSON stored as strings. It does not read, write or reinterpret stored votes. diff --git a/delphi/docs/MATH_REBUILD_QUEUE.md b/delphi/docs/MATH_REBUILD_QUEUE.md new file mode 100644 index 0000000000..80a551f971 --- /dev/null +++ b/delphi/docs/MATH_REBUILD_QUEUE.md @@ -0,0 +1,41 @@ +# Rebuild one conversation through the queue + +The capacity poller already admits oversized conversations as `math_rebuild` +jobs. The operator command admits a conversation of any size through the same +Postgres RPC and large worker. It reads conversation counts and timestamps, +estimates memory with the configured poller model, and submits a typed frame. + +Set `DATABASE_URL` to the conversation database, `MATH_CAPACITY_QUEUE_DSN` to +a restricted executor-member login, and `MATH_CAPACITY_QUEUE_ENV` to the worker's +queue namespace. Do not place database credentials in arguments or reports. + +From `delphi/`: + +```sh +PYTHONPATH=. python scripts/enqueue_math_rebuild.py \ + --zid "$DEMO_ZID" --staged-label demo-staged --target-label demo-math \ + --source-commit "$MATH_POLLER_SOURCE_COMMIT" --dry-run +``` + +Remove `--dry-run` to enqueue. Repeating admission while the job is active +returns the existing job. A poisoned scope or an active scope with conflicting +configuration returns exit 1 without creating a job; malformed configuration +or a failed database operation returns exit 2. The JSON outcome and job ID +identify the conflict for the operator. + +The daemon must have `POLIS_JOBS_WORKER_CLASS=large`, +`POLIS_JOBS_STAGES=math_rebuild`, the matching `QUEUE_ENV`, a conversation +`DATABASE_URL`, and a known `MATH_POLLER_MEMORY_LIMIT_MB` (or cgroup limit). +Its `MATH_POLLER_SOURCE_COMMIT` must equal the admission's commit. Do not set +the retired resident worker's `MATH_CAPACITY_CLASS=large`; the daemon starts +`math_poller.py --job` for each claimed job. The child rechecks capacity, +obtains the existing single-writer lock, and rebuilds the one conversation. + +Results land in `math_main`, `math_bidtopid`, and `math_ptptstats` under the +staged label. The child does not publish the target label. The existing +capacity promotion loop requires the queue's successful finalization and a +manifest matching the staged fingerprint before promotion. An operator +admission alone does not register a conversation in a resident capacity router; +use the staged label for local inspection, or let ordinary capacity routing +track and promote its own admission. `prod`, `python`, and identical staged and +target labels are refused for staged writes. diff --git a/delphi/polismath/database/dynamodb.py b/delphi/polismath/database/dynamodb.py index 8ce59e6c4f..0aab6691d4 100644 --- a/delphi/polismath/database/dynamodb.py +++ b/delphi/polismath/database/dynamodb.py @@ -7,6 +7,7 @@ """ import boto3 +from polismath.delphi_storage.resource import result_resource import time import os import logging @@ -67,7 +68,7 @@ def initialize(self) -> None: kwargs['aws_access_key_id'] = self.aws_access_key_id kwargs['aws_secret_access_key'] = self.aws_secret_access_key - self.dynamodb = boto3.resource('dynamodb', **kwargs) + self.dynamodb = result_resource('dynamodb', **kwargs) # Create tables if they don't exist self._ensure_tables_exist() diff --git a/delphi/polismath/delphi_storage/legacy_import.py b/delphi/polismath/delphi_storage/legacy_import.py new file mode 100644 index 0000000000..151066b125 --- /dev/null +++ b/delphi/polismath/delphi_storage/legacy_import.py @@ -0,0 +1,239 @@ +"""Bounded, lossless codec/1 export import through normal fenced graph execution. + +This never resumes historical jobs or guards. Their exact exports remain an +immutable archive in the import artifact. JSONB-incompatible NUL rows remain +in a separate quarantine with their original canonical bytes and a reason. +""" +from __future__ import annotations + +import hashlib +import json +import os +import stat +from pathlib import Path +import sys +from urllib.parse import urlsplit + +from .codec import FAMILIES, decode_family, encode_family, encode_item, header + +MODEL = "legacy-dynamo-export/1" +CONTROL_FAMILIES = frozenset({"Delphi_JobQueue", "Delphi_JobActiveGuard"}) +MAX_BYTES = 450_000 # Leave room inside the graph's 512 KiB output envelope. +MAX_MANIFEST_BYTES = 16_384 # Twenty known family names and SHA-256 digests. + + +def digest(data): + return hashlib.sha256(data).hexdigest() + + +def code_digest(): + return digest(Path(__file__).read_bytes()) + + +def codec_digest(): + return digest(Path(__file__).with_name("codec.py").read_bytes()) + + +def json_bytes(value): + return json.dumps(value, sort_keys=True, separators=(",", ":"), + ensure_ascii=True, allow_nan=False).encode("utf-8") + + +def inventory(files): + return {family: digest(wire.encode("utf-8")) for family, wire in sorted(files.items())} + + +def read_regular_file(path, limit): + """Do not follow symlinks, block on FIFOs, or trust a pre-read size alone.""" + flags = os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK + with os.fdopen(os.open(path, flags), "rb") as source: + metadata = os.fstat(source.fileno()) + if not stat.S_ISREG(metadata.st_mode): + raise ValueError("export input must be a regular file") + if metadata.st_size > limit: + raise ValueError("export file exceeds byte limit") + raw = source.read(limit + 1) + if len(raw) > limit: + raise ValueError("export file exceeds byte limit") + return raw + + +def read_export(directory): + """Read an explicit manifest; reject missing, changed, extra or foreign files.""" + directory = Path(directory) + expected = {} + for line in read_regular_file(directory / "SHA256SUMS", MAX_MANIFEST_BYTES).decode("utf-8").splitlines(): + sha, filename = line.split(" ", 1) + family = filename.removesuffix(".jsonl") + if (filename != family + ".jsonl" or family not in FAMILIES + or family in expected or len(sha) != 64 + or any(c not in "0123456789abcdef" for c in sha)): + raise ValueError("invalid export manifest") + expected[family] = sha + actual = {p.name for p in directory.glob("*.jsonl")} + if not expected or actual != {f + ".jsonl" for f in expected}: + raise ValueError("export inventory mismatch") + files = {} + remaining = MAX_BYTES + for family, sha in sorted(expected.items()): + raw = read_regular_file(directory / (family + ".jsonl"), remaining) + remaining -= len(raw) + if digest(raw) != sha or decode_family(raw)[0] != family: + raise ValueError("export digest or family mismatch") + files[family] = raw.decode("utf-8") + return files + + +def export_local(client, directory, families): + """Canonicalize every page from an explicitly local DynamoDB client.""" + directory = Path(directory) + if directory.exists() and any(directory.iterdir()): + raise ValueError("export destination must be empty") + directory.mkdir(parents=True, exist_ok=True) + files = {} + total_bytes = 0 + for family in sorted(set(families)): + if family not in FAMILIES: + raise ValueError("unknown export family") + total_bytes += len(header(family).encode("utf-8")) + 1 + if total_bytes > MAX_BYTES: + raise ValueError("export exceeds bounded inline import") + items, start = [], None + while True: + arguments = dict(TableName=family, ConsistentRead=True) + if start is not None: + arguments["ExclusiveStartKey"] = start + reply = client.scan(**arguments) + for item in reply.get("Items", []): + # Canonical sorting cannot change row byte sizes. Check each + # row before retaining it or fetching another scan page. + total_bytes += len(encode_item(family, item).encode("utf-8")) + 1 + if total_bytes > MAX_BYTES: + raise ValueError("export exceeds bounded inline import") + items.append(item) + start = reply.get("LastEvaluatedKey") + if not start: + break + raw = encode_family(family, items) + files[family] = raw.decode("utf-8") + (directory / (family + ".jsonl")).write_bytes(raw) + if not files: + raise ValueError("no families selected") + (directory / "SHA256SUMS").write_text("".join( + f"{sha} {family}.jsonl\n" for family, sha in inventory(files).items())) + return inventory(files) + + +def local_client(endpoint): + parsed = urlsplit(endpoint) + if (parsed.scheme != "http" or parsed.hostname not in {"127.0.0.1", "::1", "localhost"} + or parsed.username or parsed.password or parsed.path not in {"", "/"} + or parsed.query or parsed.fragment): + raise ValueError("export requires an explicit loopback DynamoDB endpoint") + import boto3 + from botocore.config import Config + return boto3.client("dynamodb", endpoint_url=endpoint, region_name="us-east-1", + aws_access_key_id="local", aws_secret_access_key="local", + config=Config(connect_timeout=5, read_timeout=30, retries={"max_attempts": 1})) + + +def has_nul(value): + if isinstance(value, str): + return "\0" in value + if isinstance(value, dict): + return any(has_nul(k) or has_nul(v) for k, v in value.items()) + if isinstance(value, (list, set, tuple)): + return any(has_nul(v) for v in value) + return False + + +def check_binding(item, zid, report_ids): + def scalar(name): + value = item[name] + return value.get("S", value.get("N")) + for name in ("conversation_id", "zid"): + if name in item and scalar(name) != str(zid): + raise ValueError("source conversation mismatch") + for name, separator in (("zid_tick", ":"), ("zid_tick_gid", ":"), ("zid_topic_jobid", "#")): + if name in item and (scalar(name) or "").split(separator, 1)[0] != str(zid): + raise ValueError("source composite conversation mismatch") + for name in ("report_id", "rid_section_model"): + if name in item and (scalar(name) or "").split("#", 1)[0] not in report_ids: + raise ValueError("source report requires explicit binding") + + +def validate_report_mapping(connection, zid, report_ids): + """Validate explicit report bindings against the local application database.""" + with connection.cursor() as cursor: + cursor.execute("SELECT report_id FROM reports WHERE zid=%s AND report_id=ANY(%s)", + (zid, list(report_ids))) + actual = {row[0] for row in cursor.fetchall()} + if actual != set(report_ids): + raise ValueError("source report is not attached to the requested conversation") + + +def import_output(files, zid, report_ids): + if type(zid) is not int or zid <= 0 or not files or set(files) - set(FAMILIES): + raise ValueError("invalid import identity or family set") + output = dict(model=MODEL, source_sha256=digest(json_bytes(inventory(files))), + source_inventory=inventory(files), legacy_control_files={}, quarantine={}, counts={}) + results = {} + for family, wire in sorted(files.items()): + actual, rows = decode_family(wire.encode("utf-8")) + if actual != family: + raise ValueError("source family mismatch") + good, bad = [], [] + for row in rows: + check_binding(row, zid, report_ids) + (bad if has_nul(row) else good).append(row) + if family in CONTROL_FAMILIES: + output["legacy_control_files"][family] = wire + output["counts"][family] = dict(source=len(rows), archived=len(rows), imported=0, quarantined=0) + else: + results[family] = encode_family(family, good).decode("utf-8") + if bad: + output["quarantine"][family] = dict(reason="postgres-jsonb-nul", + codec_wire=encode_family(family, bad).decode("utf-8"), row_count=len(bad)) + output["counts"][family] = dict(source=len(rows), archived=0, + imported=len(good), quarantined=len(bad)) + if results: + output["family_files"] = results + if len(json_bytes(output)) > MAX_BYTES: + raise ValueError("import artifact exceeds bounded inline import; no rows imported") + return output + + +def build_spec(files, zid, report_ids, worker_path=None): + """Import one checksummed export as an ordinary one-node graph.""" + output = import_output(files, zid, report_ids) + worker = Path(worker_path) if worker_path else Path(__file__).resolve().parents[2] / "scripts/job_graph_stage.py" + # This single-key ASCII object has exactly PostgreSQL jsonb's text spelling. + snapshot = {"texts": ["legacy import"]} + snapshot_sha = digest(json.dumps(snapshot).encode("utf-8")) + config = dict(family_files=files, source_sha256=output["source_sha256"], + report_ids=sorted(set(report_ids)), importer_sha256=code_digest(), + codec_sha256=codec_digest()) + spec = dict(schema="polis-job-graph/1", nodes=[dict( + key="legacy_import", stage="graph_narrative", **{"class": "delphi"}, + declared=dict(snapshot=dict(data=snapshot, sha256=snapshot_sha), + code=digest(worker.read_bytes()), model=MODEL, + runtime="python-" + sys.version.split()[0], seed=0, config=config, + mode="full", memory_bytes=64 * 1024 * 1024, work_units=1), + inputs=[], max_attempts=3)]) + if len(json_bytes(spec)) > 900_000: + raise ValueError("import admission exceeds graph input bound") + return spec + + +def execute_import(frame): + declared = frame["input"]["declared"] + config = declared["config"] + if (frame["stage"] != "graph_narrative" or declared["model"] != MODEL + or frame["input"]["artifacts"] + or config.get("importer_sha256") != code_digest() + or config.get("codec_sha256") != codec_digest()): + raise ValueError("legacy import provenance mismatch") + output = import_output(config["family_files"], frame["zid"], config["report_ids"]) + if config["source_sha256"] != output["source_sha256"]: + raise ValueError("legacy import source digest mismatch") + return output diff --git a/delphi/polismath/delphi_storage/postgres.py b/delphi/polismath/delphi_storage/postgres.py new file mode 100644 index 0000000000..d11d18b43f --- /dev/null +++ b/delphi/polismath/delphi_storage/postgres.py @@ -0,0 +1,83 @@ +"""Immutable run-bound PostgreSQL results using the frozen storage codec. + +The caller owns the connection and transaction. Methods never commit, allowing +an importer to stage, seal and finalize atomically. Normal graph children emit +``family_files`` with :func:`family_files`; the fenced artifact insert stores +those bytes in the same transaction as successful queue finalization. +""" +from __future__ import annotations + +from typing import Any, Iterable, Mapping + +from .codec import ( + CODEC_VERSION, FAMILIES, CodecError, decode_family, encode_family, + header, item_from_python, to_python, +) +import json + +RESULT_FAMILIES = frozenset(FAMILIES) - {"Delphi_JobQueue", "Delphi_JobActiveGuard"} + + +def family_files(families: Mapping[str, Iterable[dict[str, Any]]]) -> dict[str, str]: + """Encode native Python rows; floats must first follow the existing Decimal writer conversion.""" + result = {} + for family, items in families.items(): + if family not in RESULT_FAMILIES: + raise CodecError(f"not a result family: {family}") + result[family] = encode_family(family, (item_from_python(x) for x in items)).decode("utf-8") + return result + + +def decode_rows(family: str, rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Validate tagged JSONB values with codec/1 before returning Python types.""" + from .codec import dumps + _, decoded = decode_family((header(family) + "\n" + "".join(dumps(row) + "\n" for row in rows)).encode("utf-8")) + return [{name: to_python(value) for name, value in row.items()} for row in decoded] + + +class PostgresResultReader: + def __init__(self, connection: Any, env: str): + self.connection, self.env = connection, env + + def _call(self, name: str, casts: str, args: tuple[Any, ...]) -> Any: + # Names and casts are internal constants; all external values bind. + with self.connection.cursor() as cursor: + cursor.execute(f"SELECT public.{name}({casts})", (self.env, *args)) + value = cursor.fetchone()[0] + return json.loads(value) if isinstance(value, str) else value + + def read_artifact_family(self, artifact_id: str, family: str) -> list[dict[str, Any]]: + rows = self._call("pd_result_artifact_family", "%s::text,%s::uuid,%s::text", (artifact_id, family)) + return decode_rows(family, rows) + + def read_served_family(self, zid: int, scope: str, family: str) -> list[dict[str, Any]] | None: + rows = self._call("pd_result_served", "%s::text,%s::integer,%s::text,%s::text", (zid, scope, family)) + return None if rows is None else decode_rows(family, rows) + + def read_served_bundle(self, zid: int, scope: str) -> dict[str, Any] | None: + reply = self._call("pd_result_served_bundle", "%s::text,%s::integer,%s::text", (zid, scope)) + if reply is None: + return None + return {"generation": reply["generation"], "families": { + family: decode_rows(family, rows) for family, rows in reply["families"].items() + }} + + +class PostgresResultStore(PostgresResultReader): + def __init__(self, connection: Any, env: str, job_id: str, owner_id: str, + attempt_id: str, lease_epoch: int): + super().__init__(connection, env) + self.token = (job_id, owner_id, attempt_id, lease_epoch) + + def write_family(self, family: str, items: Iterable[dict[str, Any]]) -> dict[str, Any]: + wire = family_files({family: items})[family] + return self.write_family_wire(family, wire) + + def write_family_wire(self, family: str, wire: str) -> dict[str, Any]: + actual_family, _ = decode_family(wire.encode("utf-8")) + if family != actual_family or family not in RESULT_FAMILIES: + raise CodecError("result family mismatch") + return self._call("pd_result_put_family", "%s::text,%s::uuid,%s::uuid,%s::uuid,%s::bigint,%s::text,%s::text", (*self.token, family, wire)) + + def finish(self) -> dict[str, Any]: + return self._call("pd_result_seal", "%s::text,%s::uuid,%s::uuid,%s::uuid,%s::bigint", self.token) diff --git a/delphi/polismath/delphi_storage/resource.py b/delphi/polismath/delphi_storage/resource.py new file mode 100644 index 0000000000..e4ee5aad64 --- /dev/null +++ b/delphi/polismath/delphi_storage/resource.py @@ -0,0 +1,199 @@ +"""Read-only compatibility surface for published PostgreSQL result families. + +Selection is explicit; a PostgreSQL error never opens a DynamoDB connection. +Legacy writers must move through graph finalization instead of mutating results. +""" +from __future__ import annotations +import os +import json +import re +from decimal import Decimal +from types import SimpleNamespace +from typing import Any +from .codec import FAMILIES, canonical_number, decode_family, dumps, item_from_python +from .postgres import RESULT_FAMILIES, decode_rows + + +def result_resource(service_name='dynamodb', **kwargs): + backend = os.environ.get('DELPHI_RESULT_BACKEND', 'dynamodb') + if backend not in ('dynamodb', 'postgres'): + raise ValueError('invalid DELPHI_RESULT_BACKEND') + if service_name != 'dynamodb' or backend == 'dynamodb': + import boto3 + return boto3.resource(service_name, **kwargs) + return PostgresResource() + + +def _clauses(expression, names, values): + if expression is None: + return [] + if not isinstance(expression, str): + node=expression.get_expression() + op=node['operator']; args=node['values'] + if op=='AND': + return _clauses(args[0],names,values)+_clauses(args[1],names,values) + if op in ('=', 'begins_with'): + return [(op,args[0].name,args[1])] + raise ValueError(f'unsupported result condition {op}') + result=[] + for part in re.split(r'\s+AND\s+',expression,flags=re.I): + equal=re.fullmatch(r'\s*([#\w]+)\s*=\s*(:\w+)\s*',part) + prefix=re.fullmatch(r'\s*begins_with\(\s*([#\w]+)\s*,\s*(:\w+)\s*\)\s*',part) + match=equal or prefix + if match is None: + raise ValueError(f'unsupported result expression {part}') + result.append(('=' if equal else 'begins_with',names.get(match[1],match[1]),values[match[2]])) + return result + + +def _tag(value): + if isinstance(value,str):return 'S',value + if isinstance(value,bool):return 'BOOL',value + if isinstance(value,(int,Decimal)):return 'N',canonical_number(str(value)) + raise ValueError('result filter requires string, boolean or exact numeric value') + + +class PostgresResource: + def __init__(self, connection=None): + self.env=os.environ.get('DELPHI_RESULT_ENV') + if not self.env: + raise ValueError('DELPHI_RESULT_ENV is required for Postgres results') + self.connection=connection + self.meta=SimpleNamespace(client=self) + self.tables=SimpleNamespace(all=lambda: [self.Table(f) for f in sorted(RESULT_FAMILIES)]) + + def _connect(self): + if self.connection is None: + import psycopg2 + dsn=os.environ.get('DELPHI_RESULT_DATABASE_URL') or os.environ.get('DATABASE_URL') + if not dsn: + raise ValueError('DELPHI_RESULT_DATABASE_URL or DATABASE_URL is required') + self.connection=psycopg2.connect(dsn,application_name='delphi-pg-results/1') + self.connection.autocommit=True + return self.connection + + def Table(self,name): + if name not in RESULT_FAMILIES and name != 'Delphi_JobQueue': + raise ValueError(f'{name} is not a published result family; use the queue graph API') + return PostgresTable(self,name) + + def list_tables(self,**kwargs):return {'TableNames':sorted(RESULT_FAMILIES)} + def describe_table(self,TableName,**kwargs): + self.Table(TableName) + with self._connect().cursor() as cursor: + cursor.execute('SELECT 1 FROM public.delphi_result_current_rows LIMIT 0') + return {'Table':{'TableName':TableName,'TableStatus':'ACTIVE'}} + def batch_get_item(self,RequestItems,**kwargs): + response={} + for family,request in RequestItems.items(): + table=self.Table(family) + response[family]=[item for key in request['Keys'] if (item:=table.get_item(Key=key).get('Item')) is not None] + return {'Responses':response,'UnprocessedKeys':{}} + + +class PostgresTable: + def __init__(self,resource,name): + self.resource,self.name=resource,name + self.table_name=name + self.meta=resource.meta + + def load(self):return self.resource.describe_table(TableName=self.name) + def get_item(self,**kwargs): + rows=self._read(kwargs,True) + if not rows['Items']:return {} + item=rows['Items'][0] + if self.name=='Delphi_JobQueue' and not item.get('archived'): + with self.resource._connect().cursor() as cursor: + cursor.execute('SELECT public.pq_job_status(%s::text,%s::uuid)',[self.resource.env,item['job_id']]) + status=cursor.fetchall()[0][0] + attempt=(status or {}).get('attempt_id') + entries=[] + if attempt: + cursor.execute("""SELECT to_char(ts AT TIME ZONE 'UTC','YYYY-MM-DD"T"HH24:MI:SS.US"Z"'), + CASE stream WHEN 'stderr' THEN 'ERROR' ELSE 'INFO' END,line + FROM public.pq_attempt_logs(%s::text,%s::uuid,NULL,1000) + WHERE stream IN ('stdout','stderr')""",[self.resource.env,attempt]) + entries=[dict(timestamp=timestamp,level=level,message=line) for timestamp,level,line in cursor.fetchall()] + item={**item,'logs':json.dumps({'entries':entries}),'log_attempt_id':attempt} + return {'Item':item} + def query(self,**kwargs):return self._read(kwargs) + def scan(self,**kwargs):return self._read(kwargs) + def _read(self,params,get=False): + conditions=_clauses(params.get('KeyConditionExpression'),params.get('ExpressionAttributeNames',{}),params.get('ExpressionAttributeValues',{})) + conditions += [('=',k,v) for k,v in params.get('Key',{}).items()] + binds=[self.resource.env,self.name] + where=['env=%s','family=%s'] + scope=os.environ.get('DELPHI_RESULT_SCOPE') + if scope:where.append('scope_key=%s');binds.append(scope) + for op,key,value in conditions: + tag,wire=_tag(value) + if op=='begins_with': + if tag!='S':raise ValueError('begins_with requires a string') + where.append('starts_with(item->%s->>%s,%s)');binds.extend([key,tag,wire]) + else: + where.append('item->%s = %s::jsonb');binds.extend([key,dumps({tag:wire})]) + with self.resource._connect().cursor() as cursor: + if self.name == 'Delphi_JobQueue': + cursor.execute('SELECT row_to_json(j) FROM public.delphi_result_jobs j WHERE env=%s'+(' AND scope_key=%s' if scope else ''), [self.resource.env,scope] if scope else [self.resource.env]) + jobs=[row[0] for row in cursor.fetchall()] + active_ids={job['job_id'] for job in jobs} + archive_where='env=%s AND family=%s' + archive_binds=[self.resource.env,self.name] + if scope:archive_where+=' AND scope_key=%s';archive_binds.append(scope) + cursor.execute('SELECT zid,scope_key,generation,codec_wire FROM public.delphi_result_legacy_controls WHERE '+archive_where+' ORDER BY zid,scope_key',archive_binds) + records=[] + for zid,archive_scope,generation,wire in cursor.fetchall(): + family,_=decode_family(wire.encode('utf-8')) + archived=[json.loads(line) for line in wire.splitlines()[1:]] + if family!=self.name:raise ValueError('legacy control family mismatch') + for tagged in archived: + if tagged['job_id']['S'] not in active_ids: + records.append((zid,archive_scope,generation,{**tagged,'archived':{'BOOL':True}})) + records += [(job.get('conversation_id','queue'),'queue',0,item_from_python(job)) for job in jobs] + else: + cursor.execute('SELECT zid,scope_key,generation,item FROM public.delphi_result_current_rows WHERE '+' AND '.join(where)+' ORDER BY zid,scope_key,item_key::text',binds) + records=cursor.fetchall() + generations={dumps([str(zid),scope]):str(generation) for zid,scope,generation,_ in records} + start=params.get('ExclusiveStartKey') + if start and start.get('_polis_pg_generations')!=generations: + raise ValueError('result generation changed; restart pagination') + items=[];seen={} + for zid,scope,generation,tagged in records: + # decode_family validates sort order; one-row decoding retains every AV type. + item=decode_rows(self.name,[tagged])[0] + if not all(item.get(k)==v if op=='=' else isinstance(item.get(k),str) and item[k].startswith(v) for op,k,v in conditions):continue + key=dumps([item_from_python(item)[name] for name,_ in FAMILIES[self.name]['key']]) + if key in seen: + if seen[key]!=item:raise ValueError('ambiguous result scopes; set DELPHI_RESULT_SCOPE') + continue + seen[key]=item;items.append(item) + keys=[key for key,_ in FAMILIES[self.name]['key']] + index_order={'ConversationIndex':'created_at','StatusCreatedIndex':'created_at','ReportIdTimestampIndex':'timestamp','zid-created_at-index':'created_at'} + index=params.get('IndexName') + if index and index not in index_order:raise ValueError('unsupported result index') + ordering=([index_order[index]] if index else [])+keys + if index:items=[item for item in items if item.get(index_order[index]) is not None] + items.sort(key=lambda item:tuple(item[key] for key in ordering)) + if params.get('ScanIndexForward') is False:items.reverse() + if start: + startkey={k:v for k,v in start.items() if k!='_polis_pg_generations'} + position=next((i for i,item in enumerate(items) if all(item.get(k)==v for k,v in startkey.items())),None) + if position is None:raise ValueError('stale result cursor') + items=items[position+1:] + limit=params.get('Limit',1000) + if not isinstance(limit,int) or limit<1:raise ValueError('invalid result limit') + page=items[:limit] + filters=_clauses(params.get('FilterExpression'),params.get('ExpressionAttributeNames',{}),params.get('ExpressionAttributeValues',{})) + filtered=[item for item in page if all(item.get(k)==v if op=='=' else isinstance(item.get(k),str) and item[k].startswith(v) for op,k,v in filters)] + result={'Items':filtered,'Count':len(filtered),'ScannedCount':len(page)} + if len(items)>limit: + result['LastEvaluatedKey']={**{key:page[-1][key] for key in keys},'_polis_pg_generations':generations} + if params.get('ProjectionExpression'): + projected=[params.get('ExpressionAttributeNames',{}).get(k.strip(),k.strip()) for k in params['ProjectionExpression'].split(',')] + result['Items']=[{k:v for k,v in item.items() if k in projected} for item in filtered] + return result + + def __getattr__(self,name): + if name in ('put_item','update_item','delete_item','batch_writer','delete'): + raise RuntimeError('Published Delphi results are immutable; submit a new graph run') + raise AttributeError(name) diff --git a/delphi/polismath/job_child/census.py b/delphi/polismath/job_child/census.py index 6e9285bad8..c0324c3196 100644 --- a/delphi/polismath/job_child/census.py +++ b/delphi/polismath/job_child/census.py @@ -172,11 +172,11 @@ def default_pg_query() -> PgQuery: def default_dynamodb(region: Optional[str] = None): """The DynamoDB resource configured the way run_delphi.py's layer discovery does it.""" - import boto3 + from polismath.delphi_storage.resource import result_resource raw = os.environ.get("DYNAMODB_ENDPOINT") endpoint = raw if raw and raw.strip() else None if endpoint: - return boto3.resource("dynamodb", endpoint_url=endpoint, region_name="us-east-1", + return result_resource("dynamodb", endpoint_url=endpoint, region_name="us-east-1", aws_access_key_id="dummy", aws_secret_access_key="dummy") - return boto3.resource("dynamodb", region_name=region or os.environ.get("AWS_REGION", "us-east-1")) + return result_resource("dynamodb", region_name=region or os.environ.get("AWS_REGION", "us-east-1")) diff --git a/delphi/run_delphi.py b/delphi/run_delphi.py index e882f1c258..d0f33fba50 100644 --- a/delphi/run_delphi.py +++ b/delphi/run_delphi.py @@ -263,6 +263,7 @@ def main(): # First, determine available layers from DynamoDB try: import boto3 + from polismath.delphi_storage.resource import result_resource from boto3.dynamodb.conditions import Key raw_endpoint = os.environ.get('DYNAMODB_ENDPOINT') @@ -270,13 +271,13 @@ def main(): # Using dummy credentials for local, IAM role for AWS if endpoint_url: - dynamodb = boto3.resource('dynamodb', + dynamodb = result_resource('dynamodb', endpoint_url=endpoint_url, region_name='us-east-1', aws_access_key_id='dummy', aws_secret_access_key='dummy') else: - dynamodb = boto3.resource('dynamodb', region_name=args.region) + dynamodb = result_resource('dynamodb', region_name=args.region) table = dynamodb.Table('Delphi_CommentHierarchicalClusterAssignments') diff --git a/delphi/scripts/delphi_cli.py b/delphi/scripts/delphi_cli.py index 2b100789b9..e703420142 100755 --- a/delphi/scripts/delphi_cli.py +++ b/delphi/scripts/delphi_cli.py @@ -9,6 +9,7 @@ import argparse import sys import boto3 +from polismath.delphi_storage.resource import result_resource import json import uuid import os @@ -61,7 +62,7 @@ def setup_dynamodb(endpoint_url=None, region='us-east-1'): os.environ.setdefault('AWS_ACCESS_KEY_ID', 'fakeMyKeyId') os.environ.setdefault('AWS_SECRET_ACCESS_KEY', 'fakeSecretAccessKey') - return boto3.resource('dynamodb', endpoint_url=endpoint_url, region_name=region) + return result_resource('dynamodb', endpoint_url=endpoint_url, region_name=region) def submit_job(dynamodb, zid, job_type='FULL_PIPELINE', priority=50, max_votes=None, batch_size=None, # For FULL_PIPELINE/PCA diff --git a/delphi/scripts/delphi_graph_stages.py b/delphi/scripts/delphi_graph_stages.py new file mode 100644 index 0000000000..012e3f4f3f --- /dev/null +++ b/delphi/scripts/delphi_graph_stages.py @@ -0,0 +1,238 @@ +"""Bounded numerical Delphi graph adapters; all durable writes are fenced by SQL. + +Narratives and topic names use explicitly labelled fixed stand-ins for plumbing proof. +Demo snapshots with 5–2000 texts are admitted by this initial adapter. +Raw votes never enter these adapters; narrative inputs contain signed aggregate +counts from the math snapshot. Comment ids are supplied explicitly. +""" +from decimal import Decimal +from datetime import datetime, timezone +import hashlib +import json +import os +from pathlib import Path +import sys + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) +# Load the frozen stdlib-only codec without importing unrelated math poller modules. +import importlib.util +_spec = importlib.util.spec_from_file_location('delphi_graph_codec', ROOT / 'polismath/delphi_storage/codec.py') +_codec = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(_codec) +encode_family, decode_family = _codec.encode_family, _codec.decode_family +item_from_python, to_python = _codec.item_from_python, _codec.to_python + +MODELS = {'graph_embed': 'sentence-transformers/all-MiniLM-L6-v2', + 'graph_cluster': 'delphi-umap-evoc/1', 'graph_topics': 'delphi-tfidf-keywords/1', + 'graph_narrative': 'local-narrative-fixture/1'} + + +def code_digest(): + paths = [Path(__file__), ROOT / 'umap_narrative/numerical_stages.py', + ROOT / 'umap_narrative/polismath_commentgraph/utils/converter.py', + ROOT / 'umap_narrative/polismath_commentgraph/schemas/dynamo_models.py', + ROOT / 'umap_narrative/narrative_data.py'] + paths.extend(sorted((ROOT / 'umap_narrative/report_experimental').rglob('*.xml'))) + digest = hashlib.sha256() + for path in paths: + digest.update(path.read_bytes()) + return digest.hexdigest() + + +def model_digest(path): + """Pin actual local model bytes, independent of machine-specific cache paths.""" + path = Path(path) + files = sorted(p for p in path.rglob('*') if p.is_file() and '.cache' not in p.parts) + if not files or not any(p.name.endswith(('.safetensors', '.bin')) for p in files): + raise ValueError('local embedding model weights required') + h = hashlib.sha256() + for p in files: + h.update(p.relative_to(path).as_posix().encode() + b'\0') + with p.open('rb') as f: + for chunk in iter(lambda: f.read(1024 * 1024), b''): + h.update(chunk) + return h.hexdigest() + + +def _decimal(value): + if isinstance(value, float): + return Decimal(str(value)) + if isinstance(value, dict): + return {k: _decimal(v) for k, v in value.items()} + if isinstance(value, list): + return [_decimal(v) for v in value] + return value + + +def family_files(families): + return {name: encode_family(name, [item_from_python(_decimal(row)) for row in rows]).decode() + for name, rows in families.items()} + + +def read_family(output, name): + family, rows = decode_family(output['family_files'][name].encode()) + if family != name: + raise ValueError('family identity mismatch') + return [{k: to_python(v) for k, v in row.items()} for row in rows] + + +def upstream(frame, role): + artifact = frame['input']['artifacts'][role] + if hashlib.sha256(artifact['payload'].encode()).hexdigest() != artifact['sha256']: + raise ValueError('upstream artifact digest mismatch') + value = json.loads(artifact['payload']) + hydrated = frame.get('result_families', {}).get(role) + if hydrated is not None: + value['family_files'] = hydrated + return value + + + +def narrative_sections(topics, context, ids, texts): + """Use existing topic/global filters, selection limits, XML and report prompts.""" + import asyncio + import xmltodict + import xml.etree.ElementTree as ET + from umap_narrative.narrative_data import NarrativeSelection + if context.get('schema') != 'delphi-narrative-context/1': + raise ValueError('invalid narrative context') + text_by_id = dict(zip(ids,texts)) + records = [{**row,'comment':text_by_id[row['comment_id']]} for row in context['comments'] if row['comment_id'] in text_by_id] + for topic in topics: + members=set(topic['comment_ids']) + for row in records: + if row['comment_id'] in members: + row[f"layer{topic['layer_id']}_cluster_id"]=topic['cluster_id'] + selector = NarrativeSelection() + entries=[(topic,dict(topic_cluster_id=topic['cluster_id'],topic_layer_id=topic['layer_id'], + topic_citations=topic['comment_ids']), 'topics') for topic in topics] + for name,filter_type,threshold in [('groups','comment_extremity',1.0), + ('group_informed_consensus','group_aware_consensus','dynamic'),('uncertainty','uncertainty_ratio',0.2)]: + entries.append((dict(topic_label=name,size=len(records),_global=name), + dict(filter_type=filter_type,filter_threshold=threshold),name)) + directory=ROOT/'umap_narrative/report_experimental' + system=(directory/'system.xml').read_text() + result=[] + for topic,args,template in entries: + selected=[row for row in records if selector.filter_topics(row,**args)] + if not selected: + continue + structured=asyncio.run(selector.get_comments_as_xml(dict(processed_comments=records),selector.filter_topics,args)) + document=xmltodict.parse((directory/'subtaskPrompts'/f'{template}.xml').read_text()) + document['polisAnalysisPrompt']['data']={'content':{'structured_comments':structured}} + topic=dict(topic,_prompt=xmltodict.unparse(document,pretty=True),_system=system, + _allowed_ids=[int(row.attrib['id']) for row in ET.fromstring(structured).findall('comment')]) + result.append(topic) + return result + +def execute(frame): + d = frame['input']['declared'] + stage = frame['stage'] + if MODELS.get(stage) != d['model']: + raise ValueError('stage model mismatch') + texts = d['snapshot']['data']['texts'] + config = d['config'] + ids = config['comment_ids'] + if not 5 <= len(texts) <= 2000 or any(not isinstance(t, str) or not t.strip() or len(t) > 4096 for t in texts): + raise ValueError('expected 5–2000 nonempty bounded texts') + if len(ids) != len(texts) or len(set(ids)) != len(ids) or any(type(i) is not int or i < 0 for i in ids): + raise ValueError('unique nonnegative comment ids required') + zid = str(frame['zid']) + families = {} + output = {'model': d['model'], 'statement_count': len(texts)} + if stage == 'graph_embed': + import numpy as np + from sentence_transformers import SentenceTransformer + model_path = os.environ['DELPHI_EMBED_MODEL_PATH'] + if model_digest(model_path) != config['model_sha256']: + raise ValueError('embedding model bytes differ from admission') + model = SentenceTransformer(model_path, device='cpu', local_files_only=True) + vectors = model.encode(texts, convert_to_numpy=True, batch_size=32, show_progress_bar=False) + if vectors.shape != (len(texts), 384) or not np.isfinite(vectors).all() or (np.linalg.norm(vectors, axis=1) == 0).any(): + raise ValueError('invalid MiniLM embedding output') + families['Delphi_CommentEmbeddings'] = [dict(conversation_id=zid, comment_id=i, + embedding=dict(vector=v.tolist(), dimensions=384, model=d['model'])) for i,v in zip(ids,vectors)] + elif stage == 'graph_cluster': + import numpy as np + source = read_family(upstream(frame, 'embeddings'), 'Delphi_CommentEmbeddings') + by_id = {int(row['comment_id']): row for row in source} + vectors = np.asarray([by_id[i]['embedding']['vector'] for i in ids], dtype=np.float32) + if str(ROOT / 'umap_narrative') not in sys.path: + sys.path.insert(0, str(ROOT / 'umap_narrative')) + from umap_narrative.numerical_stages import project_and_cluster, characterize_comment_clusters + from polismath_commentgraph.utils.converter import DataConverter + points, layers = project_and_cluster(vectors) + if not layers or any(len(layer) != len(texts) for layer in layers) or not np.isfinite(points).all(): + raise ValueError('invalid Delphi clustering output') + output.update(layers=[layer.tolist() for layer in layers], points=points.tolist()) + dump = lambda models: [model.model_dump(exclude_none=True) for model in models] + families['Delphi_CommentHierarchicalClusterAssignments'] = dump( + DataConverter.batch_convert_clusters(zid,layers,points,ids)) + families['Delphi_UMAPGraph'] = dump(DataConverter.batch_convert_umap_edges(zid,points,layers,comment_ids=ids)) + families['Delphi_UMAPConversationConfig'] = [DataConverter.create_conversation_meta( + zid,vectors,layers).model_dump(exclude_none=True)] + elif stage == 'graph_topics': + import numpy as np + if str(ROOT / 'umap_narrative') not in sys.path: + sys.path.insert(0, str(ROOT / 'umap_narrative')) + from umap_narrative.numerical_stages import project_and_cluster, characterize_comment_clusters + from polismath_commentgraph.utils.converter import DataConverter + clusters = upstream(frame, 'clusters') + layers, points = [np.asarray(layer) for layer in clusters['layers']], np.asarray(clusters['points']) + characteristics, names, features = {}, {}, [] + for layer_id, labels in enumerate(layers): + chars = {str(key):value for key,value in characterize_comment_clusters(labels,texts).items()} + characteristics[f'layer{layer_id}'] = chars + names[f'layer{layer_id}'] = {key:'Keywords: '+', '.join(value.get('top_words',[])[:3]) + for key,value in chars.items()} + features.extend(model.model_dump(exclude_none=True) for model in + DataConverter.batch_convert_cluster_characteristics(zid,chars,layer_id)) + topics = [model.model_dump(exclude_none=True) for model in + DataConverter.batch_convert_topics(zid,layers,points,texts,names,characteristics)] + for topic in topics: + indexes = [i for i,label in enumerate(layers[topic['layer_id']]) if label == topic['cluster_id']] + topic['sample_statements'] = [dict(id=ids[i],text=texts[i]) for i in indexes[:3]] + topic['comment_ids'] = [ids[i] for i in indexes] + families['Delphi_CommentClustersStructureKeywords'] = topics + families['Delphi_CommentClustersFeatures'] = features + output['topics'] = topics + else: + topics = upstream(frame, 'topics')['topics'] + if config.get('narrative_context'): + topics = narrative_sections(topics, config['narrative_context'], ids, texts) + report_id = config['report_id'] + job = frame['job_id'] + # A visible fixture exercises the complete queue/read/render contract only. + names, reports = [], [] + completed_at = datetime.now(timezone.utc).isoformat() + for topic in topics: + label = topic.get('cluster_id',0) + layer = topic.get('layer_id',0) + title = 'Fixed proof topic' + if not topic.get('_global'): + names.append(dict(conversation_id=zid,topic_key=f'{job}#{layer}#{label}',layer_id=layer,cluster_id=label, + topic_name=title,model_name=d['model'],job_id=job,created_at=completed_at)) + section = f"{job}_global_{topic['_global']}" if topic.get('_global') else f'{job}_{layer}_{label}' + report = {'title':title,'paragraphs':[{'title':'Local provider fixture', + 'sentences':[{'clauses':[{'text':"Fixed narrative stand-in for queue, Postgres storage and report rendering proof. No LLM provider was called.",'citations':[]}]}]}], + 'provider_fixture':True} + reports.append(dict(report_id=report_id,section=section,model=d['model'], + rid_section_model=f'{report_id}#{section}#{d["model"]}',timestamp=completed_at, + job_id=job,report_data=json.dumps(report,sort_keys=True),metadata={'provider_fixture':True})) + completed_at = datetime.now(timezone.utc).isoformat() + for row in names: + row['created_at'] = completed_at + for row in reports: + row['timestamp'] = completed_at + families['Delphi_CommentClustersLLMTopicNames'] = names + families['Delphi_NarrativeReports'] = reports + if config.get('narrative_context'): + families['Delphi_CommentExtremity'] = [dict(conversation_id=zid,comment_id=str(row['comment_id']), + extremity_value=row['comment_extremity'],calculation_method='pca_based', + calculation_timestamp=completed_at,component_values={}) for row in config['narrative_context']['comments']] + output.update(provider_fixture=True,topics=len(topics), + text='Fixed stand-in; no LLM provider was called.') + output['family_files'] = family_files(families) + return output diff --git a/delphi/scripts/delphi_narrative_snapshot.py b/delphi/scripts/delphi_narrative_snapshot.py new file mode 100644 index 0000000000..2c5ebb1a77 --- /dev/null +++ b/delphi/scripts/delphi_narrative_snapshot.py @@ -0,0 +1,134 @@ +"""Read one consistent, sign-aware narrative summary from local Postgres. + +Uses the existing Delphi GroupDataProcessor and semantic vote loader. No votes +are written; no Dynamo client is initialized. Raw votes never enter job frames. +""" +import hashlib +import json +from pathlib import Path +import re +import sys + +ROOT = Path(__file__).resolve().parents[1] +for path in [ROOT, ROOT/'umap_narrative']: + if str(path) not in sys.path: + sys.path.insert(0,str(path)) + + + +def unfold_group_assignments(math): + """Expand the math contract's base-cluster IDs, never mistake them for PIDs.""" + base = math.get('base-clusters') + groups = math.get('group-clusters') + if (not isinstance(base, dict) or not isinstance(base.get('id'), list) + or not isinstance(base.get('members'), list) + or len(base['id']) != len(base['members']) or not isinstance(groups, list)): + raise ValueError('invalid folded group mapping shape') + + def identity(value): + if type(value) is not int or value < 0: + raise ValueError('invalid folded group mapping identity') + return value + + mapping = {} + participants = set() + for bid, members in zip(base['id'], base['members']): + identity(bid) + if bid in mapping or not isinstance(members, list): + raise ValueError('duplicate or invalid base cluster') + mapping[bid] = members + for pid in members: + identity(pid) + if pid in participants: + raise ValueError('duplicate participant in base clusters') + participants.add(pid) + assignments, unfolded, used_bids, seen_groups = {}, [], set(), set() + for group in groups: + if not isinstance(group, dict) or not isinstance(group.get('members'), list): + raise ValueError('invalid folded group') + gid = identity(group.get('id')) + if gid in seen_groups: + raise ValueError('duplicate group identity') + seen_groups.add(gid) + members = [] + for bid in group['members']: + identity(bid) + if bid not in mapping or bid in used_bids: + raise ValueError('missing or multiply assigned base cluster') + used_bids.add(bid) + members.extend(mapping[bid]) + for pid in members: + assignments[str(pid)] = gid + unfolded.append((gid, members)) + if not assignments: + raise ValueError('empty folded group assignments') + if 'group_clusters' in math: + alias = math['group_clusters'] + if not isinstance(alias, list) or len(alias) != len(unfolded): + raise ValueError('unfolded group alias mismatch') + for group, (gid, members) in zip(alias, unfolded): + if not isinstance(group, dict) or not isinstance(group.get('members'), list): + raise ValueError('invalid unfolded group alias') + identity(group.get('id')) + for pid in group['members']: + identity(pid) + if group['id'] != gid or group['members'] != members: + raise ValueError('unfolded group alias mismatch') + return assignments + + +def build_narrative_context(connection, zid, math_env): + from psycopg2.extras import RealDictCursor + from polismath_commentgraph.utils.storage import PostgresClient + from polismath_commentgraph.utils.group_data import GroupDataProcessor + from polismath.utils.vote_convention import RowConventionSource, database_row_fetcher, using_convention_source + + class BoundClient(PostgresClient): + def __init__(self): + pass + + def query(self, query, params=None): + query = re.sub(r'(? 524288: + raise ValueError('stage artifact exceeds 512 KiB') + return dict(schema='polis-job-artifact-manifest/1',job_id=frame['job_id'],run_id=frame['run_id'],attempt_id=frame['attempt_id'],stage=stage, + input_sha256=frame['input_sha256'],outcome='succeeded',output=dict(role='result',schema=stage+'/1',payload=payload,sha256=hashlib.sha256(payload.encode()).hexdigest())) + if declared['model'] in {'sentence-transformers/all-MiniLM-L6-v2', 'delphi-umap-evoc/1', 'delphi-tfidf-keywords/1', 'local-narrative-fixture/1'}: + from delphi_graph_stages import execute, code_digest + if declared['config'].get('adapter_sha256') != code_digest(): + raise ValueError('numerical adapter provenance mismatch') + output = execute(frame) + if os.environ.get('DELPHI_OUTPUT_MANIFEST'): + files = output.pop('family_files') + directory = Path(os.environ['DELPHI_OUTPUT_MANIFEST']).parent + output['family_spool'] = {} + for family, wire in files.items(): + filename = family + '.jsonl' + (directory / filename).write_text(wire, encoding='utf-8') + output['family_spool'][family] = dict(file=filename, sha256=hashlib.sha256(wire.encode()).hexdigest()) + payload = json.dumps(output, sort_keys=True, separators=(',', ':'), allow_nan=False) + if len(payload.encode()) > 524288: + raise ValueError('stage artifact exceeds 512 KiB') + return dict(schema='polis-job-artifact-manifest/1',job_id=frame['job_id'],run_id=frame['run_id'],attempt_id=frame['attempt_id'],stage=stage, + input_sha256=frame['input_sha256'],outcome='succeeded',output=dict(role='result',schema=stage+'/1',payload=payload,sha256=hashlib.sha256(payload.encode()).hexdigest())) + for artifact in inp['artifacts'].values(): + if hashlib.sha256(artifact['payload'].encode()).hexdigest() != artifact['sha256']: + raise ValueError('artifact content mismatch') + if stage == 'graph_embed': + if declared['model'] != 'local-token-count/1': + raise ValueError('unsupported embedding model') + texts = declared['snapshot']['data']['texts'] + if not texts or len(texts) > 100 or any(not isinstance(t, str) or len(t)>4096 for t in texts): + raise ValueError('bounded text snapshot required') + vocabulary = sorted({word for text in texts for word in text.lower().split()}) + output = {'vocabulary': vocabulary, 'vectors': [[text.lower().split().count(w) for w in vocabulary] for text in texts]} + elif stage == 'graph_cluster': + if declared['model'] != 'local-nearest-centroid/1': + raise ValueError('unsupported cluster model') + vectors = json.loads(inp['artifacts']['embeddings']['payload'])['vectors'] + # Deterministic small reference Lloyd fit; independent of legacy resets. + centers = [list(map(float, v)) for v in (vectors[0], vectors[-1])] + labels = [] + for _ in range(20): + labels = [min(range(len(centers)), key=lambda k: sum((a-b)**2 for a,b in zip(v, centers[k]))) for v in vectors] + updated = [[sum(v[d] for v,l in zip(vectors, labels) if l==k)/labels.count(k) if k in labels else centers[k][d] for d in range(len(vectors[0]))] for k in range(len(centers))] + if updated == centers: + break + centers = updated + output = {'labels': labels, 'centers': centers} + elif stage == 'graph_narrative': + if declared['model'] != 'local-cluster-summary/1': + raise ValueError('unsupported narrative model; paid adapters need admission and budget integration') + clusters = json.loads(inp['artifacts']['clusters']['payload']) + output = {'text': f"{len(clusters['labels'])} statements in {len(set(clusters['labels']))} clusters.", 'labels': clusters['labels']} + else: + raise ValueError('unsupported stage') + payload = json.dumps(output, sort_keys=True, separators=(',', ':')) + return dict(schema='polis-job-artifact-manifest/1',job_id=frame['job_id'],run_id=frame['run_id'],attempt_id=frame['attempt_id'],stage=stage, + input_sha256=frame['input_sha256'],outcome='succeeded',output=dict(role='result',schema=stage+'/1',payload=payload,sha256=hashlib.sha256(payload.encode()).hexdigest())) + + +if __name__ == '__main__': + frame = json.loads(Path(os.environ['DELPHI_FRAME']).read_text()) + manifest = run(frame) + path = Path(os.environ['DELPHI_OUTPUT_MANIFEST']) + temporary = path.with_suffix('.tmp') + temporary.write_text(json.dumps(manifest, sort_keys=True, separators=(',', ':'))) + temporary.replace(path) diff --git a/delphi/tests/dynamo_removal/Dockerfile b/delphi/tests/dynamo_removal/Dockerfile new file mode 100644 index 0000000000..7465206146 --- /dev/null +++ b/delphi/tests/dynamo_removal/Dockerfile @@ -0,0 +1,15 @@ +# CI builds DELPHI_BASE from delphi/Dockerfile. Build boxes may reuse an existing +# dependency image; all candidate Python sources and the freshly built daemon +# are copied below, so old installed application sources cannot win imports. +ARG DELPHI_BASE=polis-dynamo-deps:local +FROM ${DELPHI_BASE} +COPY delphi/ /app/ +COPY --from=queue-binary polis-jobs /usr/local/bin/polis-jobs +RUN python -m pip install --no-cache-dir pytest==8.3.5 pytest-cov==6.0.0 +ENV PYTHONPATH=/app:/app/scripts PYTHONDONTWRITEBYTECODE=1 +ENV DELPHI_EMBED_MODEL_PATH=/opt/polis/embedding/all-MiniLM-L6-v2 +RUN if [ ! -f "$DELPHI_EMBED_MODEL_PATH/config.json" ]; then \ + python -c 'from sentence_transformers import SentenceTransformer; import os; SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2").save(os.environ["DELPHI_EMBED_MODEL_PATH"])'; \ + fi +ENTRYPOINT [] +CMD ["polis-jobs"] diff --git a/delphi/tests/dynamo_removal/Dockerfile.dockerignore b/delphi/tests/dynamo_removal/Dockerfile.dockerignore new file mode 100644 index 0000000000..3f51585a06 --- /dev/null +++ b/delphi/tests/dynamo_removal/Dockerfile.dockerignore @@ -0,0 +1,6 @@ +.git +**/node_modules +**/target +**/.venv +**/.env +**/__pycache__ diff --git a/delphi/tests/dynamo_removal/admit_demo.py b/delphi/tests/dynamo_removal/admit_demo.py new file mode 100644 index 0000000000..e5998f4fcd --- /dev/null +++ b/delphi/tests/dynamo_removal/admit_demo.py @@ -0,0 +1,26 @@ +from contextlib import closing +"""Read the complete public demo statement snapshot and admit actual Delphi jobs.""" +import hashlib,json,os +from pathlib import Path +import psycopg2 +from job_graph_client import GraphClient,numerical_spec +from delphi_graph_stages import model_digest +from delphi_narrative_snapshot import build_narrative_context + +with psycopg2.connect(os.environ['DATABASE_URL']) as conn: + conn.set_session(isolation_level='REPEATABLE READ', readonly=True) + narrative_context=build_narrative_context(conn,1424,os.environ['MATH_ENV']) + with conn.cursor() as cur: + cur.execute('SELECT tid,txt FROM comments WHERE zid=%s ORDER BY tid',(1424,)) + rows=cur.fetchall() + data={'texts':[text for _,text in rows]} + cur.execute('SELECT public.pd_graph_hash(%s::jsonb)',(json.dumps(data),)) + sha=cur.fetchone()[0] +with closing(psycopg2.connect(os.environ['QUEUE_DATABASE_URL'])) as conn: + plan=numerical_spec(data['texts'],[tid for tid,_ in rows],'rlocaldynamo1424',sha, + model_digest(os.environ['DELPHI_EMBED_MODEL_PATH']), + narrative_context=narrative_context) + result=GraphClient(conn,'demo1424').admit(1424,'delphi','full-public-fixture-v1',plan) + result['statements']=len(rows) + Path('/proof/demo-admitted.json').write_text(json.dumps(result,indent=2)) + print(json.dumps(result)) diff --git a/delphi/tests/dynamo_removal/audit_narrative.py b/delphi/tests/dynamo_removal/audit_narrative.py new file mode 100644 index 0000000000..1ff871d9b9 --- /dev/null +++ b/delphi/tests/dynamo_removal/audit_narrative.py @@ -0,0 +1,128 @@ +"""Read-only independent recount of the admitted aggregate vote context.""" +import argparse +from collections import Counter +import hashlib +import json +import os +from pathlib import Path +import re +import sys + +ROOT = Path(__file__).resolve().parents[2] +sys.path[:0] = [str(ROOT), str(ROOT / "scripts")] +from polismath.utils.vote_convention import SEMANTIC_AGREE, SEMANTIC_DISAGREE, SEMANTIC_PASS + +SEMANTIC_SIGNS = (SEMANTIC_DISAGREE, SEMANTIC_PASS, SEMANTIC_AGREE) +COUNT_SIGNS = (("agrees", SEMANTIC_AGREE), ("disagrees", SEMANTIC_DISAGREE), ("passes", SEMANTIC_PASS)) + + +def digest(value, **kwargs): + return hashlib.sha256(json.dumps(value, **kwargs).encode()).hexdigest() + + +def canonical_groups(math): + bases = dict(zip(math["base-clusters"]["id"], math["base-clusters"]["members"])) + assert len(bases) == len(math["base-clusters"]["id"]) + groups = {g["id"]: [pid for bid in g["members"] for pid in bases[bid]] + for g in math["group-clusters"]} + assert len(groups) == len(math["group-clusters"]) + assert groups == {g["id"]: g["members"] for g in math["group_clusters"]} + assignments = {str(pid): gid for gid, members in groups.items() for pid in members} + assert len(assignments) == sum(map(len, groups.values())) + return groups, assignments + + +def audit_context(context, math, votes, statement_ids): + groups, assignments = canonical_groups(math) + assert context["group_mapping"] == "base-clusters-unfold/1" + assert context["source_convention"] == "semantic:+1=agree" + assert context["group_assignments_sha256"] == digest(assignments, sort_keys=True, separators=(",", ":")) + assert context["math_sha256"] == digest(math, sort_keys=True, separators=(",", ":"), allow_nan=False) + counts = Counter((v["tid"], assignments.get(str(v["pid"])), v["vote"]) + for v in votes if v["vote"] is not None) + overall = Counter() + for (tid, gid, sign), count in counts.items(): + assert sign in SEMANTIC_SIGNS + overall[tid, sign] += count + rows = context["comments"] + assert len(rows) == len(statement_ids) == len({row["comment_id"] for row in rows}) + assert {row["comment_id"] for row in rows} == set(statement_ids) + scopes = 0 + for row in rows: + tid = row["comment_id"] + assert row["comment-id"] == tid + for name, sign in COUNT_SIGNS: + assert row["total-" + name] == row[name] == overall[tid, sign], (tid, name) + assert row["total-votes"] == row["votes"] == sum(overall[tid, s] for s in SEMANTIC_SIGNS) + scopes += 1 + present = 0 + for gid in groups: + total = sum(counts[tid, gid, s] for s in SEMANTIC_SIGNS) + present += total > 0 + for name, sign in COUNT_SIGNS: + assert row.get(f"group-{gid}-{name}", 0) == counts[tid, gid, sign], (tid, gid, name) + assert row.get(f"group-{gid}-votes", 0) == total + scopes += 1 + assert row["num_groups"] == present + return dict(comments=len(rows), scoped_counts=scopes, count_fields=scopes * 4, + group_sizes={str(g): len(m) for g, m in groups.items()}, + latest_votes=len(votes), unassigned_votes=sum(n for (t, g, s), n in counts.items() if g is None)) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--proof", type=Path, default=Path("/proof")) + parser.add_argument("--env", default="demo1424") + parser.add_argument("--zid", type=int, default=1424) + parser.add_argument("--scope", default="delphi") + parser.add_argument("--context-only", type=Path, help="diagnostic recount; does not certify published reports") + args = parser.parse_args() + import psycopg2 + from psycopg2.extras import RealDictCursor + from polismath.utils.vote_convention import RowConventionSource, database_row_fetcher, using_convention_source, load_semantic_votes + from polismath.delphi_storage.postgres import PostgresResultReader + from delphi_graph_stages import code_digest + with psycopg2.connect(os.environ["DATABASE_URL"]) as conn: + conn.set_session(isolation_level="REPEATABLE READ", readonly=True) + def query(sql, params=None): + with conn.cursor(cursor_factory=RealDictCursor) as cur: + cur.execute(re.sub(r"(? actual queued graph child. + +prepare seeds generated codec data and admits work. verify waits for the real +daemon and publishes via the public generation-checked RPC. This script never +claims or finalizes a queue job and never fabricates process exit proof. +""" +import argparse +from contextlib import closing +import json +import os +from pathlib import Path +import subprocess +import sys +import time +import uuid + +import psycopg2 + +DELPHI = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(DELPHI)) +sys.path.insert(0, str(DELPHI / "scripts")) +from job_graph_client import GraphClient +from polismath.delphi_storage import legacy_import as legacy +from polismath.delphi_storage.codec import decode_family, encode_family, item_from_python +from polismath.delphi_storage.golden_corpus import corpus +from polismath.delphi_storage.postgres import PostgresResultReader + +ZID, REPORT, ENV, SCOPE = 9001, "r9001generated", "proof-import", "generated-import" + + +def cli(*arguments): + completed = subprocess.run([sys.executable, str(DELPHI / "scripts/import_dynamo_export.py"), + *map(str, arguments)], capture_output=True, text=True) + if completed.returncode: + raise RuntimeError(completed.stderr.strip()) + return json.loads(completed.stdout) + + +def seed_database(): + with psycopg2.connect(os.environ["DATABASE_URL"]) as connection: + with connection.cursor() as cursor: + cursor.execute("INSERT INTO users(uid,hname,email) VALUES (%s,'Generated importer owner'," + "'importer@example.invalid') ON CONFLICT(uid) DO NOTHING", (ZID,)) + cursor.execute("INSERT INTO conversations(zid,owner,topic) VALUES (%s,%s,'Generated importer') " + "ON CONFLICT(zid) DO NOTHING", (ZID, ZID)) + cursor.execute("INSERT INTO reports(report_id,zid) SELECT %s,%s WHERE NOT EXISTS" + "(SELECT 1 FROM reports WHERE report_id=%s)", (REPORT, ZID, REPORT)) + legacy.validate_report_mapping(connection, ZID, [REPORT]) + + +def prepare(root, endpoint): + seed_database() + client = legacy.local_client(endpoint) + data = corpus() + # Deliberately incompatible result data proves durable quarantine, no votes. + data["Delphi_CommentEmbeddings"].append(item_from_python(dict( + conversation_id=str(ZID), comment_id=999, text="generated\0quarantine"))) + for family, rows in sorted(data.items()): + key = legacy.FAMILIES[family]["key"] + client.create_table(TableName=family, BillingMode="PAY_PER_REQUEST", + AttributeDefinitions=[dict(AttributeName=n, AttributeType=t) for n, t in key], + KeySchema=[dict(AttributeName=n, KeyType="HASH" if i == 0 else "RANGE") + for i, (n, _) in enumerate(key)]) + client.get_waiter("table_exists").wait(TableName=family) + for row in rows: + client.put_item(TableName=family, Item=row) + directory = root / ("export-" + uuid.uuid4().hex) + args = ["export-local", directory, "--endpoint", endpoint] + for family in sorted(data): + args.extend(["--family", family]) + exported = cli(*args) + assert set(exported) == set(legacy.FAMILIES) + source = legacy.read_export(directory) + for family in data: + assert source[family].encode() == encode_family(family, data[family]) + preview = cli("preview", directory, "--zid", ZID, "--report-id", REPORT) + arguments = ["enqueue", directory, "--zid", ZID, "--report-id", REPORT, + "--env", ENV, "--scope", SCOPE] + first, again = cli(*arguments), cli(*arguments) + assert first["outcome"] == "enqueued", first + assert again["outcome"] == "existing" and first["graph_id"] == again["graph_id"] + assert first["counts"] == preview["counts"] + receipt = dict(graph_id=first["graph_id"], root_job_id=first["root_job_id"], + export_dir=str(directory), source_sha256=first["source_sha256"], + counts=first["counts"], families=len(data), duplicate_outcome=again["outcome"]) + (root / "import-receipt.json").write_text(json.dumps(receipt, sort_keys=True, indent=2) + "\n") + print(json.dumps(dict(phase="prepared", **receipt), sort_keys=True)) + + +def verify(root, wait_seconds): + receipt = json.loads((root / "import-receipt.json").read_text()) + with closing(psycopg2.connect(os.environ["QUEUE_DATABASE_URL"])) as connection: + graph = GraphClient(connection, ENV) + deadline = time.monotonic() + wait_seconds + while True: + status = graph.status(receipt["graph_id"]) + assert len(status["nodes"]) == 1 + node = status["nodes"][0] + state = node["readiness"]["state"] + if state == "succeeded": + break + if state in {"dead", "cancelled"}: + raise AssertionError("import graph failed: " + state) + if time.monotonic() >= deadline: + raise AssertionError("real importer worker has not completed: " + state) + time.sleep(1) + artifact = node["artifact"] + payload = artifact["payload"] + assert artifact["content_sha"] == legacy.digest(payload.encode()) + output = json.loads(payload) + assert output["counts"] == receipt["counts"] + source = legacy.read_export(receipt["export_dir"]) + assert output == legacy.import_output(source, ZID, [REPORT]) + for family in legacy.CONTROL_FAMILIES: + assert output["legacy_control_files"][family] == source[family] + assert output["counts"]["Delphi_CommentEmbeddings"]["quarantined"] == 1 + assert decode_family(output["quarantine"]["Delphi_CommentEmbeddings"]["codec_wire"].encode())[1][0]["text"]["S"] == "generated\0quarantine" + served = graph.served(ZID, SCOPE) + if served is None: + publication = graph.publish(receipt["graph_id"], node["job_id"], 0) + assert publication["outcome"] == "published", publication + else: + assert served["bundle"]["root"] == artifact["artifact_id"] + reader = PostgresResultReader(connection, ENV) + bundle = reader.read_served_bundle(ZID, SCOPE) + assert set(bundle["families"]) == set(legacy.FAMILIES) - legacy.CONTROL_FAMILIES + for family, rows in bundle["families"].items(): + assert len(rows) == receipt["counts"][family]["imported"] + # Reencode native PG readback to prove tags, decimals, sets, and bytes. + assert encode_family(family, [item_from_python(row) for row in rows]).decode() == output["family_files"][family] + repeated = ["enqueue", receipt["export_dir"], "--zid", ZID, "--report-id", REPORT, + "--env", ENV, "--scope", SCOPE] + again = cli(*repeated) + assert again["outcome"] == "existing" and again["graph_id"] == receipt["graph_id"] + result = dict(phase="verified", graph_id=receipt["graph_id"], artifact_id=artifact["artifact_id"], + result_families=len(bundle["families"]), archived_control_families=len(output["legacy_control_files"]), + quarantined_rows=1, source_families=len(receipt["counts"]), + source_rows=sum(c["source"] for c in receipt["counts"].values()), + imported_rows=sum(c["imported"] for c in receipt["counts"].values()), + archived_rows=sum(c["archived"] for c in receipt["counts"].values()), + attempts=node["attempts"], generation=bundle["generation"], + duplicate_outcome=again["outcome"]) + (root / "import-verification.json").write_text(json.dumps(result, sort_keys=True, indent=2) + "\n") + print(json.dumps(result, sort_keys=True)) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("phase", choices=("prepare", "verify")) + parser.add_argument("--directory", type=Path, required=True) + parser.add_argument("--endpoint", default="http://127.0.0.1:8000") + parser.add_argument("--wait-seconds", type=int, default=0) + args = parser.parse_args() + args.directory.mkdir(parents=True, exist_ok=True) + if args.phase == "prepare": + prepare(args.directory, args.endpoint) + else: + verify(args.directory, args.wait_seconds) + + +if __name__ == "__main__": + main() diff --git a/delphi/tests/dynamo_removal/math_proof.py b/delphi/tests/dynamo_removal/math_proof.py new file mode 100644 index 0000000000..04cc9bb420 --- /dev/null +++ b/delphi/tests/dynamo_removal/math_proof.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +"""Verify a real math job's receipt and promote its staged local demo bundle.""" +import argparse +from contextlib import closing +import json +import os +import time + +import psycopg2 + +from polismath.database.postgres import PostgresClient, PostgresConfig, staged_newer +from polismath.poller.capacity_queue import QueueClient, QueueSettings, decode_frame_uri +from polismath.poller.rebuild_child import check_child_label + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--job-id", required=True) + parser.add_argument("--zid", type=int, required=True) + parser.add_argument("--staged-label", required=True) + parser.add_argument("--target-label", required=True) + parser.add_argument("--wait-seconds", type=int, default=0) + args = parser.parse_args() + # This proof never promotes production labels. + check_child_label(args.target_label, served_env="prod", target_label=args.staged_label, env={}) + queue = QueueClient(QueueSettings(os.environ["MATH_CAPACITY_QUEUE_DSN"], + os.environ["MATH_CAPACITY_QUEUE_ENV"])) + deadline = time.monotonic() + args.wait_seconds + while True: + status = queue.job_status(args.job_id) + if status["state"] == "succeeded": + break + if status["state"] in {"dead", "cancelled"} or time.monotonic() >= deadline: + raise AssertionError("real math job not successful: " + status["state"]) + time.sleep(1) + frame = json.loads(decode_frame_uri(status["input"]["uri"])) + assert frame["zid"] == args.zid + assert frame["config"]["staged_label"] == args.staged_label + assert frame["config"]["target_label"] == args.target_label + receipt = queue.receipt(args.job_id) + assert receipt.finalized, "successful job has no valid stored manifest receipt" + database_url = os.environ["DATABASE_URL"] + pg = PostgresClient(PostgresConfig(url=database_url, math_env=args.target_label, ssl_mode="disable")) + pg.initialize() + try: + with closing(psycopg2.connect(database_url)) as lock: + lock.autocommit = True + with lock.cursor() as cursor: + cursor.execute("SELECT pg_try_advisory_lock(hashtext(%s))", + ("polis-math-python:" + args.target_label,)) + assert cursor.fetchone()[0], "another writer owns target label" + fps = pg.math_fingerprints([args.zid], [args.staged_label, args.target_label]) + staged = fps.get((args.zid, args.staged_label)) + target = fps.get((args.zid, args.target_label)) + assert staged and staged.complete and receipt.binds(staged, args.staged_label) + if staged_newer(staged, target): + pg.promote_bundle(args.zid, from_env=args.staged_label, to_env=args.target_label, + expected_target=target, expected_staged=staged) + promoted = pg.math_fingerprints([args.zid], [args.target_label])[(args.zid, args.target_label)] + assert promoted.complete and promoted.lvt == staged.lvt + print(json.dumps(dict(outcome="verified-and-promoted", job_id=args.job_id, + attempts=status["attempt_count"], receipt_sha256=receipt.output_sha256, + staged_math_tick=staged.math_tick, target_math_tick=promoted.math_tick, + vote_hwm=promoted.lvt, target_label=args.target_label), sort_keys=True)) + finally: + pg.shutdown() + + +if __name__ == "__main__": + main() diff --git a/delphi/tests/dynamo_removal/seed.py b/delphi/tests/dynamo_removal/seed.py new file mode 100644 index 0000000000..8cdf0251e3 --- /dev/null +++ b/delphi/tests/dynamo_removal/seed.py @@ -0,0 +1,51 @@ +"""Local public-fixture importer; explicit source convention, all votes via module.""" +import argparse +import csv +import json +import os +from pathlib import Path +import psycopg2 +from psycopg2.extras import execute_values, RealDictCursor +from polismath.utils.vote_convention import ( + semantic_vote, storage_vote, database_row_fetcher, RowConventionSource, +) + + +def main(): + parser=argparse.ArgumentParser() + parser.add_argument('--source-agree',required=True,type=int,choices=(-1,1)) + args=parser.parse_args() + root=Path(__file__).resolve().parents[2]/'real_data' + data=next(root.glob('*-biodiversity')) + comments=list(csv.DictReader(next(data.glob('*comments.csv')).open())) + votes=list(csv.DictReader(next(data.glob('*votes.csv')).open())) + pids=sorted({int(v['voter-id']) for v in votes}|{int(c['author-id']) for c in comments}) + zid=1424 + with psycopg2.connect(os.environ['DATABASE_URL']) as conn: + def query(sql): + with conn.cursor(cursor_factory=RealDictCursor) as cursor: + cursor.execute(sql);return cursor.fetchall() + convention=RowConventionSource(database_row_fetcher(query)).current() + with conn.cursor() as cur: + cur.execute("INSERT INTO users(uid,hname,email) VALUES (%s,'Public fixture owner','fixture@example.invalid')",(zid,)) + cur.execute("INSERT INTO conversations(zid,owner,topic,description,is_active,topics_enabled) VALUES (%s,%s,'Biodiversity public fixture','Local queue and Postgres proof',true,true)",(zid,zid)) + cur.execute("INSERT INTO reports(rid,zid,report_id) VALUES (%s,%s,'rlocaldynamo1424')",(zid,zid)) + cur.execute("INSERT INTO zinvites(zid,zinvite) VALUES(%s,'local-dynamo-1424')",(zid,)) + # Each participant receives a unique test account. + execute_values(cur,'INSERT INTO users(uid,hname,email) VALUES %s',[(100000+p,'Fixture participant',f'fixture{p}@example.invalid') for p in pids]) + execute_values(cur,'INSERT INTO participants(zid,pid,uid,created,mod) VALUES %s',[(zid,p,100000+p,1700000000000,0) for p in pids]) + execute_values(cur,'INSERT INTO comments(zid,tid,pid,uid,txt,mod,is_meta,created,modified,active) VALUES %s',[ + (zid,int(x['comment-id']),int(x['author-id']),100000+int(x['author-id']),x['comment-body'],int(x['moderated']),False,int(x['timestamp'])*1000,int(x['timestamp'])*1000,True) for x in comments]) + converted=[(zid,int(x['voter-id']),int(x['comment-id']),storage_vote(semantic_vote(int(x['vote']),args.source_agree),convention.agree_value),int(x['timestamp'])*1000) for x in votes] + execute_values(cur,'INSERT INTO votes(zid,pid,tid,vote,created) VALUES %s',converted,page_size=1) + cur.execute('UPDATE participants p SET vote_count=v.n FROM (SELECT pid,count(*) n FROM votes WHERE zid=%s GROUP BY pid) v WHERE p.zid=%s AND p.pid=v.pid',(zid,zid)) + cur.execute('UPDATE conversations SET participant_count=%s WHERE zid=%s',(len(pids),zid)) + cur.execute('SELECT tid,vote,count(*) FROM votes WHERE zid=%s GROUP BY tid,vote',(zid,)) + actual={(tid,semantic_vote(v,convention.agree_value)):n for tid,v,n in cur.fetchall()} + expected={} + for x in votes: + k=(int(x['comment-id']),semantic_vote(int(x['vote']),args.source_agree));expected[k]=expected.get(k,0)+1 + assert actual==expected,'vote polarity/count self-check failed' + print(json.dumps(dict(comments=len(comments),participants=len(pids),votes=len(votes),vote_count_groups_checked=len(expected),source_agree=args.source_agree,storage_agree=convention.agree_value))) + +if __name__=='__main__':main() diff --git a/delphi/tests/dynamo_removal/wait_dynamo.py b/delphi/tests/dynamo_removal/wait_dynamo.py new file mode 100644 index 0000000000..12b7d475b0 --- /dev/null +++ b/delphi/tests/dynamo_removal/wait_dynamo.py @@ -0,0 +1,15 @@ +"""Wait for the owned local DynamoDB service, never a default AWS endpoint.""" +import time +from polismath.delphi_storage.legacy_import import local_client +client=local_client('http://127.0.0.1:8000') +last=None +for _ in range(60): + try: + client.list_tables() + print('Local DynamoDB ready') + break + except Exception as exc: + last=exc + time.sleep(1) +else: + raise RuntimeError('Owned DynamoDB did not become ready') from last diff --git a/delphi/tests/dynamo_removal/wait_publish.py b/delphi/tests/dynamo_removal/wait_publish.py new file mode 100644 index 0000000000..15767e4681 --- /dev/null +++ b/delphi/tests/dynamo_removal/wait_publish.py @@ -0,0 +1,54 @@ +from contextlib import closing +"""Observe actual jobs, assert independent retry, then publish one coherent bundle.""" +import json,os,time +from pathlib import Path +import psycopg2 +from job_graph_client import GraphClient +from polismath.delphi_storage.postgres import PostgresResultReader +root=Path('/proof'); admitted=json.loads((root/'demo-admitted.json').read_text()) +with closing(psycopg2.connect(os.environ['QUEUE_DATABASE_URL'])) as conn: + client=GraphClient(conn,'demo1424') + # Bound the proof without changing worker leases. + deadline=time.monotonic()+1800 + prior=None + while time.monotonic()0 + assert counts['Delphi_NarrativeReports']>0 + # Bind coverage to the computed topic structure, so omitting a name and its + # report together cannot make a partial narrative publication pass. + job=nodes['n']['job_id'] + structure=bundle['families']['Delphi_CommentClustersStructureKeywords'] + expected_topics={f"{job}#{int(t['layer_id'])}#{int(t['cluster_id'])}" for t in structure} + assert len(expected_topics)==len(structure)>0, 'duplicate computed topic' + names=bundle['families']['Delphi_CommentClustersLLMTopicNames'] + assert len(names)==len(expected_topics) and {t['topic_key'] for t in names}==expected_topics, 'complete topic names required' + expected_sections={key.replace('#','_') for key in expected_topics} + expected_sections.update(f'{job}_global_{name}' for name in ('groups','group_informed_consensus','uncertainty')) + reports=bundle['families']['Delphi_NarrativeReports'] + assert len(reports)==len(expected_sections), 'duplicate or missing report section' + assert {row['section'] for row in reports}==expected_sections, 'complete topic and global reports required' + assert all(row['job_id']==job for row in reports), 'reports must belong to published narrative job' + assert all(row['metadata']['provider_fixture'] is True and json.loads(row['report_data'])['provider_fixture'] is True for row in reports) + assert all(row['model']=='local-narrative-fixture/1' for row in reports) +with psycopg2.connect(os.environ['DATABASE_URL']) as conn: + with conn.cursor() as cur: + cur.execute('SELECT stage,attempt_count FROM polis_queue_jobs WHERE env=%s AND job_id=ANY(%s::uuid[])',('demo1424',[n['job_id'] for n in nodes.values()])) + attempts=dict(cur.fetchall()) + assert attempts['graph_embed']==1 and attempts['graph_cluster']==2,attempts + assert attempts['graph_topics']==1 and attempts['graph_narrative']==1,attempts +receipt={'statement_count':316,'attempts':attempts,'published_generation':bundle['generation'],'family_rows':counts,'provider':'fixed stand-in; no LLM calls'} +(root/'demo-result.json').write_text(json.dumps(receipt,indent=2));print(json.dumps(receipt),flush=True) diff --git a/delphi/tests/job_graph/adversarial.py b/delphi/tests/job_graph/adversarial.py new file mode 100644 index 0000000000..fca71c06fa --- /dev/null +++ b/delphi/tests/job_graph/adversarial.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +"""RPC-boundary and race controls, complement the real daemon scenario in prove.py.""" +import concurrent.futures +import copy +import importlib.util +import json +import os +from pathlib import Path +import subprocess as sp +import time +import uuid +from prove import * + + +def claim(cls='delphi',stage='graph_embed'): + return rpc('pd_graph_claim',"'proof'","1::smallint",lit(uuid.uuid4()),lit(uuid.uuid4()),'10',lit(cls),lit(stage)) + +def ident(c):return ["'proof'",lit(c['job_id']),lit(c['owner_id']),lit(c['attempt_id']),lit(c['lease_epoch'])] +def finish(c,manifest,seq): + raw=json.dumps(manifest,sort_keys=True,separators=(',',':')) + sha=hashlib.sha256(raw.encode()).hexdigest() + sql(f"INSERT INTO polis_queue_logs(env,attempt_id,seq,stream,line) VALUES('proof',{lit(c['attempt_id'])},{seq},'manifest',{lit(raw)})",True) + return rpc('pd_graph_finalize',*ident(c),"'unused'",lit(sha)),sha + +def main(): + # Existing output is unaffected by every rejection below. + served=rpc('pd_graph_served',"'proof'",'1',"'retry'") + only=spec();only['nodes']=only['nodes'][:1] + with concurrent.futures.ThreadPoolExecutor(2) as pool: + results=list(pool.map(lambda _:graph('concurrent',only),range(2))) + assert results[0]['graph_id']==results[1]['graph_id'] + record('P07_concurrent_idempotent_admission',responses=results) + g=results[0];n=nodes(g)['e'] + # The installed membership/edge triggers close owner-control bypasses too. + sql(f"UPDATE delphi_graphs SET sealed=false WHERE graph_id={lit(g['graph_id'])}",ok=False) + sql(f"UPDATE delphi_graph_nodes SET declared='{{}}' WHERE job_id={lit(n['job_id'])}",ok=False) + with concurrent.futures.ThreadPoolExecutor(2) as pool: + cs=list(pool.map(lambda _:claim(),range(2))) + assert sorted(c['outcome'] for c in cs)==['none','owned'];c=next(x for x in cs if x['outcome']=='owned') + # No process was spawned for this RPC control: exit proof is honest. + rpc('pq_end_attempt',*ident(c),"'confirm_exit'","NULL",'true') + module_spec=importlib.util.spec_from_file_location('stage',ROOT/'delphi/scripts/job_graph_stage.py') + module=importlib.util.module_from_spec(module_spec);module_spec.loader.exec_module(module) + frame=dict(schema='polis-job-stage-frame/1',job_id=c['job_id'],run_id=c['run_id'],attempt_id=c['attempt_id'],stage=c['stage'],input=c['graph_input'],input_sha256=c['graph_input_sha'],input_json=c['graph_input_wire']) + manifest=module.run(frame) + for i,(field,value) in enumerate([('input_sha256','0'*64),('run_id',str(uuid.uuid4())),('attempt_id',str(uuid.uuid4()))]): + bad=copy.deepcopy(manifest);bad[field]=value + assert finish(c,bad,i)[0]['outcome']=='invalid_output' + bad=copy.deepcopy(manifest);bad['output']['sha256']='0'*64 + assert finish(c,bad,3)[0]['outcome']=='invalid_output' + good,sha=finish(c,manifest,4);assert good['outcome']=='succeeded' + assert rpc('pd_graph_finalize',*ident(c),"'lost-reply'",lit(sha))['outcome']=='already_succeeded' + badid=ident(c);badid[2]=lit(uuid.uuid4()) + assert rpc('pd_graph_finalize',*badid,"'stale'",lit(sha))['outcome']=='fenced' + assert sql(f"SELECT count(*) FROM delphi_artifacts WHERE job_id={lit(c['job_id'])}")=='1' + sql(f"UPDATE delphi_artifacts SET run_id={lit(uuid.uuid4())} WHERE job_id={lit(c['job_id'])}",ok=False) + record('P08_P09_wrong_bindings_lost_finalize_ack_stale_owner_single_artifact') + rpc('pd_graph_reconcile',"'proof'") + # An uncommitted admitted graph is invisible to claims; after seal+commit + # only its root may run and both dependency edges are already visible. + s=spec();query=f"BEGIN;SET ROLE polis_queue_executor;SELECT pd_graph_admit('proof',1,'seal-race','first',{js(s)},NULL);SELECT pg_sleep(2);COMMIT;" + proc=sp.Popen(C+['exec','-T','postgres','psql','-XqAt','-v','ON_ERROR_STOP=1','-U','postgres','-d',DB],stdin=sp.PIPE,stdout=sp.PIPE,stderr=sp.PIPE,text=True) + proc.stdin.write(query);proc.stdin.close();time.sleep(.5) + assert claim()['outcome']=='none' + out=proc.stdout.read();err=proc.stderr.read();assert proc.wait()==0,err + race=json.loads(next(l for l in out.splitlines() if l.startswith('{')));rn=nodes(race) + c=claim();assert c['job_id']==rn['e']['job_id'];assert claim(stage='graph_cluster,graph_narrative')['outcome']=='none' + # A closed edge cannot be removed, nor a late dependency appended. + sql(f"DELETE FROM delphi_graph_edges WHERE consumer={lit(rn['c']['job_id'])}",ok=False) + sql(f"UPDATE delphi_jobs SET parent_job_id={lit(rn['n']['job_id'])} WHERE job_id={lit(rn['e']['job_id'])}",ok=False) + rpc('pq_fail',*ident(c),'true',"'negative-control'",'true') + record('P07_seal_claim_race_and_late_mutation_refusal') + # Durable provider intent, lost submission ACK, no duplicate execution. + provider=graph('provider-unknown',only);c=claim();assert c['job_id']==nodes(provider)['e']['job_id'] + request=str(uuid.uuid4()) + rpc('pd_provider_intent',*ident(c),lit(request),"'local-provider-fixture'","decode(repeat('a',64),'hex')") + rpc('pd_provider_update',*ident(c),lit(request),"'submission_unknown'",'NULL') + parked=rpc('pq_fail',*ident(c),'false',"'lost_provider_ack'",'true');assert parked['state']=='parked' + assert claim()['outcome']=='none' + assert sql(f"SELECT count(*) FROM delphi_provider_requests WHERE job_id={lit(c['job_id'])}")=='1' + record('P10_unknown_provider_intent_stays_parked_no_reclaim',job=c['job_id']) + # Expired ownership with an unconfirmed child cannot gain another owner. + stale=graph('stale-lease',only);c=claim();assert c['job_id']==nodes(stale)['e']['job_id'] + sql(f"UPDATE polis_queue_jobs SET locked_until=clock_timestamp()-interval '1 second' WHERE job_id={lit(c['job_id'])}") + rpc('pq_reap',"'proof'",'NULL','100',"'delphi'") + assert claim()['outcome']=='none' + assert rpc('pd_graph_finalize',*ident(c),"'stale'",lit('0'*64))['outcome']=='fenced' + record('P09_expired_lease_requires_exit_proof') + assert rpc('pd_graph_served',"'proof'",'1',"'retry'")==served + record('P12_old_served_bundle_survives_all_controls') + +if __name__=='__main__':main() diff --git a/delphi/tests/job_graph/child_fixture.py b/delphi/tests/job_graph/child_fixture.py new file mode 100644 index 0000000000..114c12eac0 --- /dev/null +++ b/delphi/tests/job_graph/child_fixture.py @@ -0,0 +1,30 @@ +"""Failure injection around the actual reference adapter; never shipped as a worker.""" +import importlib.util +import json +import os +from pathlib import Path +import sys +import time + +frame = json.loads(Path(os.environ['DELPHI_FRAME']).read_text()) +root = Path(os.environ['GRAPH_PROOF_ROOT']) +counter = root/'starts'/frame['job_id'] +counter.parent.mkdir(exist_ok=True) +count = int(counter.read_text())+1 if counter.exists() else 1 +counter.write_text(str(count)) +control = json.loads((root/'control.json').read_text()) +(root/'frames').mkdir(exist_ok=True) +(root/'frames'/f"{frame['job_id']}-{count}.json").write_text(json.dumps(frame,sort_keys=True)) +if frame['stage']==control.get('hold_stage'): + (root/'held-child.json').write_text(json.dumps(dict(pid=os.getpid(),job_id=frame['job_id']))) + while True: + time.sleep(.1) +if frame['stage']=='graph_cluster' and control.get('fail_clusters',0)>=count: + Path(os.environ['DELPHI_OUTPUT_MANIFEST']).with_name('uncommitted-staging.json').write_text('{"unpublished":true}') + sys.exit(1) +source = Path(os.environ['GRAPH_STAGE_SOURCE']) +spec = importlib.util.spec_from_file_location('graph_stage',source) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) +manifest = module.run(frame) +Path(os.environ['DELPHI_OUTPUT_MANIFEST']).write_text(json.dumps(manifest,sort_keys=True,separators=(',',':'))) diff --git a/delphi/tests/job_graph/compose.yml b/delphi/tests/job_graph/compose.yml new file mode 100644 index 0000000000..fe26fd80db --- /dev/null +++ b/delphi/tests/job_graph/compose.yml @@ -0,0 +1,14 @@ +services: + postgres: + image: postgres:17-alpine + environment: + POSTGRES_HOST_AUTH_METHOD: trust + POSTGRES_USER: postgres + POSTGRES_DB: postgres + ports: + - "127.0.0.1:${POLIS_RECOVERY_PG_PORT:?set an owned port}:5432" + healthcheck: + test: [CMD-SHELL, "pg_isready -U postgres"] + interval: 1s + timeout: 3s + retries: 30 diff --git a/delphi/tests/job_graph/installs.py b/delphi/tests/job_graph/installs.py new file mode 100644 index 0000000000..245d93c479 --- /dev/null +++ b/delphi/tests/job_graph/installs.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +"""Apply the core SQL on fresh local fixtures; no deployment runner or fake receipts.""" +import hashlib +import sys +from prove import ROOT, C, DB, OUT, record, sql, sp +MIG = ROOT / 'server/postgres/migrations' + +def query(db, text, ok=True): + p = sp.run(C + ['exec', '-T', 'postgres', 'psql', '-XqAt', '-v', + 'ON_ERROR_STOP=1', '-U', 'postgres', '-d', db], + input=text, text=True, capture_output=True) + if (p.returncode == 0) != ok: + raise AssertionError((db, p.stdout[-1000:], p.stderr[-2000:])) + return p.stdout.strip() if ok else p.stderr + +def baseline(db): + query('postgres', 'CREATE DATABASE ' + db) + paths = sorted(p for p in MIG.glob('*.sql') if p.name[:6].isdigit() + and int(p.name[:6]) <= 24) + assert [int(p.name[:6]) for p in paths] == [n for n in range(25) if n != 20] + for p in paths: + query(db, p.read_text()) + assert query(db, 'SELECT contract_version FROM polis_queue_install') == 'polis-queue/3' + +forward = (MIG / '000027_create_sealed_job_graphs.sql').read_text() +down = (MIG / 'down/000027_drop_sealed_job_graphs.sql').read_text() +if sys.argv[1:] == ['--bootstrap']: + baseline(DB) + query(DB, forward) + print('Applied unchanged 000000–000024 prerequisites (no 20), then trimmed M27') +else: + for line in (MIG / 'down/000027-files.sha256').read_text().splitlines(): + digest, name = line.split() + assert hashlib.sha256((MIG / name).read_bytes()).hexdigest() == digest + record('forward_and_down_seals_match') + db = DB + '_empty_down' + baseline(db) + query(db, forward) + # The down migration itself compares the complete /3 catalog to the baseline. + query(db, down) + assert query(db, 'SELECT contract_version FROM polis_queue_install') == 'polis-queue/3' + query(db, forward) + assert query(db, 'SELECT contract_version FROM polis_queue_install') == 'polis-queue/5' + assert 'nonempty graph contract' in sql(down, ok=False) + record('empty_forward_down_reapply_exact_catalog_nonempty_refusal') + # Never adopt the earlier unreleased M27 under the new seal. + assert 'queue /3 catalog drift' in query(DB, forward, ok=False) + record('reapply_requires_explicit_migration_management') diff --git a/delphi/tests/job_graph/process_failure.py b/delphi/tests/job_graph/process_failure.py new file mode 100644 index 0000000000..2b3c8954d6 --- /dev/null +++ b/delphi/tests/job_graph/process_failure.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +"""Kernel process kill while staging, followed by a retry on the same served scope.""" +import shutil +from prove import * + +def main(): + # Reuse the first proof's E through the actual served-bundle reader. + before=rpc('pd_graph_served',"'proof'",'1',"'retry'") + original=next(a for a in before['bundle']['artifacts'] if a['schema_version']=='graph_embed/1') + declared_sha=sql(f"SELECT pd_graph_hash(declared) FROM delphi_graph_nodes WHERE job_id={lit(original['job_id'])}") + s=spec();s['nodes']=s['nodes'][1:] + s['nodes'][0]['inputs']=[dict(artifact_id=original['artifact_id'],sha256=original['content_sha'],contract_sha256=declared_sha,role='embeddings')] + rpc('pd_graph_reconcile',"'proof'") + g=graph('retry',s,'kernel-rebuild');ns=nodes(g) + (OUT/'control.json').write_text(json.dumps(dict(hold_stage='graph_cluster'))) + worker=start('held-writer') + until=time.monotonic()+30 + while not (OUT/'held-child.json').exists() and time.monotonic()&2 + exit 2 +fi +trap '"${compose[@]}" down -v > "$proof_output/cleanup.log" 2>&1' EXIT +(cd queue-rs + cargo fmt --all --check + cargo build --locked --bins + cargo test --locked + cargo clippy --locked --all-targets -- -D warnings + cargo clippy --locked --all-targets --features jobs-integration -- -D warnings +) 2>&1 | tee "$proof_output/rust.log" +"${compose[@]}" up -d --wait +python3 delphi/tests/job_graph/installs.py --bootstrap 2>&1 | tee "$proof_output/install.log" +python3 delphi/tests/job_graph/prove.py 2>&1 | tee "$proof_output/proof.log" +GRAPH_PROOF_ROOT="$proof_output/process" python3 delphi/tests/job_graph/process_failure.py 2>&1 | tee "$proof_output/process.log" +GRAPH_PROOF_ROOT="$proof_output/adversarial" python3 delphi/tests/job_graph/adversarial.py 2>&1 | tee "$proof_output/adversarial.log" +GRAPH_PROOF_ROOT="$proof_output/installs" python3 delphi/tests/job_graph/installs.py 2>&1 | tee "$proof_output/installs.log" +# Same legacy compatibility command as queue-rs-ci, isolated in this owned server. +"${compose[@]}" exec -T postgres psql -U postgres -c 'CREATE DATABASE queue_acceptance' +export POLIS_JOBS_TEST_DATABASE_URL="postgresql://postgres@127.0.0.1:$POLIS_RECOVERY_PG_PORT/queue_acceptance" +export POLIS_JOBS_TEST_PYTHON=python3 +(cd queue-rs && cargo test --locked --features jobs-integration --test jobs_integration -- --test-threads 4) 2>&1 | tee "$proof_output/legacy.log" + +# Existing Node adapter gets an independent fresh /5 database: graph fixtures +# deliberately use fixed identifiers and must not perturb its serial sequences. +GRAPH_PROOF_DB=graph_node python3 delphi/tests/job_graph/installs.py --bootstrap 2>&1 | tee "$proof_output/node-install.log" +export DATABASE_URL="postgresql://postgres@127.0.0.1:$POLIS_RECOVERY_PG_PORT/graph_node" +export DATABASE_SSL=false +export NODE_ENV=test +(cd server && npx jest --config jest.job-graphs.config.ts --ci --runInBand) 2>&1 | tee "$proof_output/node.log" diff --git a/delphi/tests/job_graph/test_numerical_stages.py b/delphi/tests/job_graph/test_numerical_stages.py new file mode 100644 index 0000000000..7641f020ba --- /dev/null +++ b/delphi/tests/job_graph/test_numerical_stages.py @@ -0,0 +1,155 @@ +"""Fast adapter boundary checks; full numeric execution belongs to mm5 proof.""" +import hashlib +import importlib.util +import json +from pathlib import Path +import sys +import tempfile +import unittest + +SCRIPTS = Path(__file__).resolve().parents[2] / 'scripts' +sys.path.insert(0,str(SCRIPTS)) +import delphi_graph_stages as stages +import job_graph_stage +from delphi_narrative_snapshot import unfold_group_assignments + + +class NumericalBoundary(unittest.TestCase): + def test_folded_groups_expand_base_ids_to_actual_participants(self): + math = {'base-clusters': {'id': [7, 42], 'members': [[90, 0], [351]]}, + 'group-clusters': [{'id': 8, 'members': [42]}, {'id': 3, 'members': [7]}], + 'group_clusters': [{'id': 8, 'members': [351]}, {'id': 3, 'members': [90, 0]}]} + original = json.dumps(math) + self.assertEqual(unfold_group_assignments(math), {'351': 8, '90': 3, '0': 3}) + self.assertEqual(json.dumps(math), original) + self.assertNotIn('42', unfold_group_assignments(math)) + + def test_malformed_folded_groups_refuse_instead_of_using_ids_as_participants(self): + valid = {'base-clusters': {'id': [7, 42], 'members': [[90, 0], [351]]}, + 'group-clusters': [{'id': 8, 'members': [42]}, {'id': 3, 'members': [7]}], + 'group_clusters': [{'id': 8, 'members': [351]}, {'id': 3, 'members': [90, 0]}]} + mutations = [ + lambda m: m.pop('base-clusters'), + lambda m: m['base-clusters']['id'].__setitem__(1, 7), + lambda m: m['base-clusters']['members'][1].append(90), + lambda m: m['group-clusters'][0]['members'].__setitem__(0, 999), + lambda m: m['group-clusters'][1]['members'].append(42), + lambda m: m['group-clusters'][1].__setitem__('id', 8), + lambda m: m['group_clusters'][0]['members'].__setitem__(0, 42), + lambda m: m['group_clusters'][0]['members'].__setitem__(0, True), + lambda m: m['base-clusters']['id'].__setitem__(0, 7.0), + ] + for mutate in mutations: + with self.subTest(mutation=mutations.index(mutate)): + math = json.loads(json.dumps(valid)); mutate(math) + with self.assertRaises(ValueError): + unfold_group_assignments(math) + + def frame(self): + topics = {'topics':[dict(cluster_id=0,layer_id=0,topic_label='Trees',size=5)]} + payload = json.dumps(topics) + declared = dict(model=stages.MODELS['graph_narrative'],mode='full', + code=hashlib.sha256((SCRIPTS/'job_graph_stage.py').read_bytes()).hexdigest(), + runtime='python-'+sys.version.split()[0],seed=42, + snapshot=dict(data=dict(texts=['synthetic text']*5)), + config=dict(comment_ids=list(range(5)),report_id='synthetic-report',adapter_sha256=stages.code_digest())) + inp = dict(schema='polis-job-input/1',declared=declared, + artifacts=dict(topics=dict(payload=payload,sha256=hashlib.sha256(payload.encode()).hexdigest()))) + frame = dict(schema='polis-job-stage-frame/1',stage='graph_narrative',zid=9001, + job_id='00000000-0000-4000-8000-000000009001',run_id='run',attempt_id='attempt',input=inp) + return self.digest(frame) + + def digest(self, frame): + wire=json.dumps(frame['input']) + frame.update(input_json=wire,input_sha256=hashlib.sha256(wire.encode()).hexdigest()) + return frame + + def test_fixture_is_visible_and_codec_roundtrips(self): + result=job_graph_stage.run(self.frame()) + output=json.loads(result['output']['payload']) + self.assertTrue(output['provider_fixture']) + rows=stages.read_family(output,'Delphi_NarrativeReports') + self.assertTrue(json.loads(rows[0]['report_data'])['provider_fixture']) + self.assertEqual(rows[0]['model'],'local-narrative-fixture/1') + self.assertEqual(result['output']['sha256'],hashlib.sha256(result['output']['payload'].encode()).hexdigest()) + + def test_signed_wire_is_authoritative_over_reserialized_duplicate(self): + frame=self.frame() + frame['input']['declared']['config']['summary_metric']=0.12345678901234568 + self.digest(frame) + # Model serde_json's redundant f64 reserialization without changing wire. + frame['input']=json.loads(frame['input_json']) + frame['input']['declared']['config']['summary_metric']=0.12345678901234567 + output=json.loads(job_graph_stage.run(frame)['output']['payload']) + self.assertTrue(output['provider_fixture']) + + def test_changed_signed_wire_refused(self): + frame=self.frame(); frame['input_json'] += ' ' + with self.assertRaisesRegex(ValueError,'resolved input digest'): + job_graph_stage.run(frame) + + def test_upstream_digest_refused(self): + frame=self.frame(); frame['input']['artifacts']['topics']['payload']='{}' + with self.assertRaisesRegex(ValueError,'upstream artifact digest'): + job_graph_stage.run(self.digest(frame)) + + def test_code_change_refused(self): + frame=self.frame(); frame['input']['declared']['config']['adapter_sha256']='0'*64 + with self.assertRaisesRegex(ValueError,'provenance'): + job_graph_stage.run(self.digest(frame)) + + def test_duplicate_comment_ids_refused(self): + frame=self.frame(); frame['input']['declared']['config']['comment_ids']=[0]*5 + with self.assertRaisesRegex(ValueError,'comment ids'): + job_graph_stage.run(self.digest(frame)) + + def test_too_many_texts_refused_without_truncation(self): + frame=self.frame(); frame['input']['declared']['snapshot']['data']['texts']=['text']*2001 + with self.assertRaisesRegex(ValueError,'bounded texts'): + job_graph_stage.run(self.digest(frame)) + + def test_real_summary_selects_topic_and_all_eligible_global_sections(self): + # Aggregate semantic counts, never raw storage votes or DB inserts. + ids=list(range(1,6)); texts=['generated statement '+str(i) for i in ids] + records=[dict(comment_id=i,**{'comment-id':i,'total-votes':10,'total-agrees':6, + 'total-disagrees':1,'total-passes':3},votes=10,agrees=6,disagrees=1,passes=3, + comment_extremity=1.5,group_aware_consensus=0.9,num_groups=2) for i in ids] + context=dict(schema='delphi-narrative-context/1',comments=records) + topic=dict(layer_id=0,cluster_id=1,comment_ids=ids,topic_label='Generated',size=5) + selected=stages.narrative_sections([topic],context,ids,texts) + self.assertEqual(len(selected),4) + self.assertEqual({p.get('_global') for p in selected}, + {None,'groups','group_informed_consensus','uncertainty'}) + self.assertTrue(all(p['_allowed_ids']==ids for p in selected)) + self.assertTrue(all('generated statement 1' in p['_prompt'] for p in selected)) + + def test_citation_ids_match_actual_xml_after_legacy_comment_limit(self): + import xml.etree.ElementTree as ET + import xmltodict + ids=list(range(1,121));texts=['generated statement '+str(i) for i in ids] + records=[dict(comment_id=i,**{'comment-id':i,'total-votes':10,'total-agrees':6, + 'total-disagrees':1,'total-passes':3},votes=10,agrees=6,disagrees=1,passes=3, + comment_extremity=1.5,group_aware_consensus=0.9,num_groups=2) for i in ids] + sections=stages.narrative_sections([],dict(schema='delphi-narrative-context/1',comments=records),ids,texts) + self.assertEqual(len(sections),3) + for section in sections: + structured=xmltodict.parse(section['_prompt'])['polisAnalysisPrompt']['data']['content']['structured_comments'] + actual=[int(row.attrib['id']) for row in ET.fromstring(structured).findall('comment')] + self.assertEqual(section['_allowed_ids'],actual) + self.assertLess(len(actual),len(ids)) + + def test_model_digest_changes_with_weights(self): + with tempfile.TemporaryDirectory() as directory: + path=Path(directory)/'model.safetensors'; path.write_bytes(b'generated model bytes') + old=stages.model_digest(directory) + path.write_bytes(b'changed generated bytes') + self.assertNotEqual(old,stages.model_digest(directory)) + + def test_float_codec_conversion_is_explicit(self): + output=dict(family_files=stages.family_files({'Delphi_UMAPGraph':[ + dict(conversation_id='9001',edge_id='0_0',position={'x':0.25,'y':0.5})]})) + from decimal import Decimal + self.assertEqual(stages.read_family(output,'Delphi_UMAPGraph')[0]['position']['x'],Decimal('0.25')) + +if __name__=='__main__': + unittest.main() diff --git a/delphi/tests/poller/test_capacity_queue_postgres.py b/delphi/tests/poller/test_capacity_queue_postgres.py index d47475c555..5f25c832ec 100644 --- a/delphi/tests/poller/test_capacity_queue_postgres.py +++ b/delphi/tests/poller/test_capacity_queue_postgres.py @@ -235,6 +235,32 @@ def release(self, scope): class TestTheContract: + def test_operator_cli_sizes_and_admits_once(self, queue_db, db, env, labels, + monkeypatch, capsys): + from scripts import enqueue_math_rebuild as cli + + small, large = labels + (zid,) = fresh_zids(1) + seed_conversation(db, zid, participants=3, comments=3) + monkeypatch.setenv("DATABASE_URL", queue_db[0]) + monkeypatch.setenv("MATH_CAPACITY_QUEUE_DSN", queue_db[1]) + monkeypatch.setenv("MATH_CAPACITY_QUEUE_ENV", env) + args = ["--zid", str(zid), "--staged-label", large, + "--target-label", small, "--source-commit", COMMIT] + assert cli.main(args + ["--dry-run"]) == 0 + dry = json.loads(capsys.readouterr().out) + assert dry["outcome"] == "dry_run" and jobs(db, env) == 0 + assert dry["sizes"] == {"votes": 6, "voters": 3, "comments": 3} + assert cli.main(args) == 0 + first = json.loads(capsys.readouterr().out) + assert first["outcome"] == "enqueued" + assert first["config"] == dry["config"] + assert cli.main(args) == 0 + second = json.loads(capsys.readouterr().out) + assert second["outcome"] == "existing" + assert second["job_id"] == first["job_id"] and jobs(db, env) == 1 + assert job_row(db, env, first["job_id"])[1:3] == ("math_rebuild", "large") + def test_enqueue_as_the_executor_login_and_the_row_it_makes(self, queue_db, db, env, labels): small, large = labels (zid,) = fresh_zids(1) diff --git a/delphi/tests/poller/test_enqueue_math_rebuild.py b/delphi/tests/poller/test_enqueue_math_rebuild.py new file mode 100644 index 0000000000..94fbbf0aed --- /dev/null +++ b/delphi/tests/poller/test_enqueue_math_rebuild.py @@ -0,0 +1,72 @@ +"""Operator admissions share the real queue contract, even for a small conversation.""" +from unittest.mock import Mock +import json + +import pytest + +from scripts import enqueue_math_rebuild as cli +from polismath.poller.admission import MemoryModel + + +COMMIT = "a" * 40 + + +@pytest.mark.parametrize("zid,staged,target,commit", [ + (0, "stage", "python", COMMIT), + (7, "python", "target", COMMIT), + (7, "prod", "target", COMMIT), + (7, "stage", "stage", COMMIT), + (7, "stage", "python", "unknown"), + (7, "stage/invalid", "python", COMMIT), +]) +def test_refuses_invalid_admission_before_database(monkeypatch, zid, staged, target, commit): + connection = Mock(side_effect=AssertionError("must not connect")) + monkeypatch.setattr(cli.psycopg2, "connect", connection) + assert cli.main(["--zid", str(zid), "--staged-label", staged, + "--target-label", target, "--source-commit", commit]) == 2 + connection.assert_not_called() + + +def test_small_conversation_uses_existing_admission_without_threshold_override(): + queue = Mock() + queue.enqueue_math_rebuild.return_value = ("enqueued", "generated-job") + model = MemoryModel() + result = cli.admit(7, staged_label="staged", target_label="python", + source_commit=COMMIT, model=model, snapshot=((36, 6, 6), 123), + queue=queue) + assert result["outcome"] == "enqueued" + assert result["config"]["need_bytes"] == model.above_base_bytes(36, 6, 6) + assert result["config"]["input_through_ms"] == 123 + queue.enqueue_math_rebuild.assert_called_once_with( + 7, config=result["config"], staged_label="staged", target_label="python") + + +def test_dry_run_is_read_only(): + queue = Mock() + result = cli.admit(7, staged_label="staged", target_label="python", + source_commit=COMMIT, model=MemoryModel(), + snapshot=((0, 0, 0), None), queue=queue, dry_run=True) + assert result["outcome"] == "dry_run" and result["job_id"] is None + queue.enqueue_math_rebuild.assert_not_called() + + +def test_database_errors_do_not_disclose_connection_details(monkeypatch, capsys): + monkeypatch.setenv("DATABASE_URL", "generated-db-url") + monkeypatch.setattr(cli, "read_snapshot", Mock( + side_effect=cli.psycopg2.OperationalError("sensitive-connection-detail"))) + assert cli.main(["--zid", "7", "--staged-label", "stage", "--target-label", "python", + "--source-commit", COMMIT, "--dry-run"]) == 2 + assert "sensitive-connection-detail" not in capsys.readouterr().err + + +def test_active_scope_conflict_reports_failure_and_preserves_existing_job(monkeypatch, capsys): + monkeypatch.setenv("DATABASE_URL", "generated-db-url") + monkeypatch.setattr(cli, "read_snapshot", Mock(return_value=((36, 6, 6), 123))) + queue = Mock() + queue.enqueue_math_rebuild.return_value = ("conflict", "existing-job") + monkeypatch.setattr(cli, "QueueClient", Mock(return_value=queue)) + assert cli.main(["--zid", "7", "--staged-label", "stage", "--target-label", "python", + "--source-commit", COMMIT]) == 1 + result = json.loads(capsys.readouterr().out) + assert result["outcome"] == "conflict" and result["job_id"] == "existing-job" + queue.enqueue_math_rebuild.assert_called_once() diff --git a/delphi/tests/test_after_install_hook.py b/delphi/tests/test_after_install_hook.py index c9f86a350c..65181cdc61 100644 --- a/delphi/tests/test_after_install_hook.py +++ b/delphi/tests/test_after_install_hook.py @@ -221,8 +221,34 @@ def test_the_stop_hook_leaves_every_service_running(tmp_path, service_type): assert "AfterInstall" in proc.stdout -def test_unknown_role_refuses_before_migration_or_replacement(tmp_path): - proc, log = _deploy(tmp_path, "unknown") - assert proc.returncode != 0 - assert not _calls(log, "docker") - assert not _calls(log, "docker-compose") +@pytest.mark.parametrize("service_type,worker_class", [ + ("server", None), ("delphi", None), ("math", None), + ("delphi-large", "large"), ("delphi-worker", "delphi"), +]) +def test_migration_succeeds_before_any_service_replacement(tmp_path, service_type, worker_class): + proc, log = _deploy(tmp_path, service_type, worker_class=worker_class) + assert proc.returncode == 0, proc.stdout + proc.stderr + build = "docker build -t polis-migrate:deploy -f queue-rs/polis-migrate/Dockerfile ." + apply = "docker run --rm --network host --env-file .env polis-migrate:deploy deploy" + assert log.count(build) == log.count(apply) == 1 + assert log.index(build) < log.index(apply) + replacements = [i for i, call in enumerate(log) if call.startswith(( + "docker-compose down", "docker rm ", "docker system prune", + "docker-compose up", "systemctl restart"))] + assert replacements or service_type == "math" + assert not replacements or log.index(apply) < min(replacements) + + +@pytest.mark.parametrize("failure", ["build", "run"]) +@pytest.mark.parametrize("service_type,worker_class", [ + ("server", None), ("delphi", None), ("math", None), + ("delphi-large", "large"), ("delphi-worker", "delphi"), +]) +def test_migration_failure_preserves_every_running_service(tmp_path, service_type, worker_class, failure): + proc, log = _deploy(tmp_path, service_type, worker_class=worker_class, + migration_failure=failure) + assert proc.returncode == 17, proc.stdout + proc.stderr + assert _calls(log, "docker-compose") == [] + assert _calls(log, "systemctl") == [] + docker = _calls(log, "docker") + assert [call.split()[0] for call in docker] == (["build"] if failure == "build" else ["build", "run"]) diff --git a/delphi/tests/test_delphi_legacy_import.py b/delphi/tests/test_delphi_legacy_import.py new file mode 100644 index 0000000000..71ad5bbcad --- /dev/null +++ b/delphi/tests/test_delphi_legacy_import.py @@ -0,0 +1,248 @@ +"""All twenty generated codec families cross the local export/import boundary.""" +import copy +import json +import os +import stat +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import MagicMock, Mock + +import pytest + +from polismath.delphi_storage import legacy_import as legacy +from polismath.delphi_storage.codec import decode_family, encode_family, item_from_python +from polismath.delphi_storage.golden_corpus import GOLDEN, corpus + + +ZID = 9001 +REPORT = "r9001generated" + + +def files(): + return {family: encode_family(family, rows).decode() for family, rows in corpus().items()} + + +def test_every_family_accounted_for_and_control_rows_never_activated(): + source = files() + result = legacy.import_output(source, ZID, [REPORT]) + assert set(result["counts"]) == set(legacy.FAMILIES) + for family, wire in source.items(): + counts = result["counts"][family] + assert counts["source"] == sum(counts[k] for k in ("imported", "archived", "quarantined")) + if family in legacy.CONTROL_FAMILIES: + assert result["legacy_control_files"][family] == wire + assert family not in result["family_files"] and counts["imported"] == 0 + else: + assert result["family_files"][family] == wire + + +def test_nul_quarantine_preserves_exact_row_and_other_row_imports(): + family = "Delphi_CommentEmbeddings" + rows = [item_from_python(dict(conversation_id=str(ZID), comment_id=1, text="valid")), + item_from_python(dict(conversation_id=str(ZID), comment_id=2, text="bad\0value"))] + result = legacy.import_output({family: encode_family(family, rows).decode()}, ZID, []) + assert result["counts"][family] == dict(source=2, imported=1, archived=0, quarantined=1) + assert result["quarantine"][family]["reason"] == "postgres-jsonb-nul" + assert decode_family(result["family_files"][family].encode())[1] == rows[:1] + assert decode_family(result["quarantine"][family]["codec_wire"].encode())[1] == rows[1:] + # Canonical wire in a JSON string has escaped backslashes, never literal NUL. + assert "\0" not in json.dumps(result) + + +def test_binary_nul_is_not_quarantined(): + assert not legacy.has_nul({"B": b"\0"}) + assert legacy.has_nul({"M": {"bad\0key": {"S": "value"}}}) + + +@pytest.mark.parametrize("zid,reports", [(9002, [REPORT]), (ZID, []), (ZID, ["wrong-report"])]) +def test_cross_conversation_or_unbound_report_refused(zid, reports): + with pytest.raises(ValueError, match="mismatch|binding"): + legacy.import_output(files(), zid, reports) + + +def test_manifest_reader_accepts_frozen_golden_bytes(): + assert legacy.read_export(GOLDEN) == files() + + +def test_local_export_consumes_all_pages_and_roundtrips(tmp_path): + family = "Delphi_CommentEmbeddings" + rows = corpus()[family] + client = Mock() + token = {"conversation_id": {"S": str(ZID)}, "comment_id": {"N": "1"}} + client.scan.side_effect = [{"Items": rows[:1], "LastEvaluatedKey": token}, {"Items": rows[1:]}] + legacy.export_local(client, tmp_path, [family]) + assert client.scan.call_args_list[1].kwargs["ExclusiveStartKey"] == token + assert legacy.read_export(tmp_path)[family] == encode_family(family, rows).decode() + + +@pytest.mark.parametrize("endpoint", ["https://dynamodb.us-east-1.amazonaws.com", "http://example.com", + "http://127.0.0.1@evil.invalid", "http://127.0.0.1/?target=prod"]) +def test_export_refuses_nonlocal_endpoints_before_sdk(endpoint): + with pytest.raises(ValueError, match="loopback"): + legacy.local_client(endpoint) + + +def test_changed_export_refused(tmp_path): + family = "Delphi_CommentEmbeddings" + client = Mock() + client.scan.return_value = {"Items": corpus()[family]} + legacy.export_local(client, tmp_path, [family]) + path = tmp_path / (family + ".jsonl") + path.write_bytes(path.read_bytes().replace(b"generated-embedding-model", b"changed-embedding-model")) + with pytest.raises(ValueError, match="digest"): + legacy.read_export(tmp_path) + + +def frame(): + spec = legacy.build_spec(files(), ZID, [REPORT]) + return dict(stage="graph_narrative", zid=ZID, + input=dict(declared=spec["nodes"][0]["declared"], artifacts={})) + + +def test_import_worker_rechecks_source_and_code_provenance(): + actual = frame() + assert legacy.execute_import(actual) == legacy.import_output(files(), ZID, [REPORT]) + for name in ("source_sha256", "importer_sha256", "codec_sha256"): + changed = copy.deepcopy(actual) + changed["input"]["declared"]["config"][name] = "0" * 64 + with pytest.raises(ValueError, match="digest|provenance"): + legacy.execute_import(changed) + + +def test_normal_graph_stage_manifest_binds_import_payload(): + from scripts import job_graph_stage + actual = frame() + actual.update(schema="polis-job-stage-frame/1", job_id="generated-job", + run_id="generated-run", attempt_id="generated-attempt") + actual["input"]["schema"] = "polis-job-input/1" + wire = json.dumps(actual["input"]) + actual.update(input_json=wire, input_sha256=legacy.digest(wire.encode())) + manifest = job_graph_stage.run(actual) + payload = manifest["output"]["payload"] + assert manifest["output"]["sha256"] == legacy.digest(payload.encode()) + assert json.loads(payload)["counts"] == legacy.import_output(files(), ZID, [REPORT])["counts"] + + +def test_identical_export_produces_identical_request_and_different_export_does_not(): + first = legacy.build_spec(files(), ZID, [REPORT]) + assert first == legacy.build_spec(files(), ZID, [REPORT]) + changed = files() + changed["Delphi_CommentEmbeddings"] = changed["Delphi_CommentEmbeddings"].replace( + "generated-embedding-model", "changed-embedding-model") + assert first != legacy.build_spec(changed, ZID, [REPORT]) + + +def test_report_mapping_uses_actual_database_relationship(): + connection = MagicMock() + cursor = connection.cursor.return_value.__enter__.return_value + cursor.fetchall.return_value = [(REPORT,)] + legacy.validate_report_mapping(connection, ZID, [REPORT]) + cursor.fetchall.return_value = [] + with pytest.raises(ValueError, match="attached"): + legacy.validate_report_mapping(connection, ZID, [REPORT]) + + +def test_oversized_artifact_refused_without_truncation(): + family = "Delphi_CommentEmbeddings" + source = {family: encode_family(family, [item_from_python(dict( + conversation_id=str(ZID), comment_id=1, text="x" * legacy.MAX_BYTES))]).decode()} + with pytest.raises(ValueError, match="bounded inline"): + legacy.import_output(source, ZID, []) + + +@pytest.mark.parametrize("target", ["SHA256SUMS", "Delphi_CommentEmbeddings.jsonl"]) +@pytest.mark.parametrize("kind", ["symlink", "fifo"]) +def test_export_reader_refuses_nonregular_inputs_without_blocking(tmp_path, target, kind): + family = "Delphi_CommentEmbeddings" + raw = encode_family(family, [item_from_python(dict(conversation_id=str(ZID), comment_id=1))]) + (tmp_path / (family + ".jsonl")).write_bytes(raw) + (tmp_path / "SHA256SUMS").write_text(f"{legacy.digest(raw)} {family}.jsonl\n") + path = tmp_path / target + original = path.read_bytes() + path.unlink() + if kind == "symlink": + outside = tmp_path / "original" + outside.write_bytes(original) + path.symlink_to(outside) + else: + os.mkfifo(path) + # A subprocess timeout is the regression assertion: an ordinary FIFO open + # would block forever before any regular-file validation could run. + program = """from polismath.delphi_storage.legacy_import import read_export +import sys +try: + read_export(sys.argv[1]) +except (OSError, ValueError): + sys.exit(0) +sys.exit(1) +""" + result = subprocess.run([sys.executable, "-c", program, str(tmp_path)], + cwd=Path(legacy.__file__).resolve().parents[2], capture_output=True, timeout=5) + assert result.returncode == 0, result.stderr.decode() + + +@pytest.mark.parametrize("target", ["SHA256SUMS", "Delphi_CommentEmbeddings.jsonl"]) +def test_export_reader_refuses_oversize_regular_files(tmp_path, target): + family = "Delphi_CommentEmbeddings" + raw = encode_family(family, []) + (tmp_path / (family + ".jsonl")).write_bytes(raw) + (tmp_path / "SHA256SUMS").write_text(f"{legacy.digest(raw)} {family}.jsonl\n") + limit = legacy.MAX_MANIFEST_BYTES if target == "SHA256SUMS" else legacy.MAX_BYTES + with (tmp_path / target).open("wb") as source: + source.truncate(limit + 1) + with pytest.raises(ValueError, match="byte limit"): + legacy.read_export(tmp_path) + + +def test_export_reader_enforces_actual_bytes_if_file_grows_after_stat(tmp_path, monkeypatch): + source = tmp_path / "growing" + source.write_bytes(b"x" * 33) + monkeypatch.setattr(legacy.os, "fstat", lambda fd: SimpleNamespace(st_mode=stat.S_IFREG, st_size=0)) + with pytest.raises(ValueError, match="byte limit"): + legacy.read_regular_file(source, 32) + + +@pytest.mark.parametrize("paged", [False, True]) +def test_export_budget_stops_before_fetching_more_or_writing_oversize_family(tmp_path, paged): + family = "Delphi_CommentEmbeddings" + rows = [item_from_python(dict(conversation_id=str(ZID), comment_id=i, text="x" * 240_000)) for i in range(2)] + token = {"conversation_id": {"S": str(ZID)}, "comment_id": {"N": "1"}} + pages = ([{"Items": rows[:1], "LastEvaluatedKey": token}, {"Items": rows[1:], "LastEvaluatedKey": token}] + if paged else [{"Items": rows, "LastEvaluatedKey": token}]) + client = Mock() + client.scan.side_effect = pages + with pytest.raises(ValueError, match="bounded inline"): + legacy.export_local(client, tmp_path, [family]) + assert client.scan.call_count == len(pages) + assert not (tmp_path / (family + ".jsonl")).exists() + assert not (tmp_path / "SHA256SUMS").exists() + + +def test_export_budget_is_shared_across_families(tmp_path): + families = ["Delphi_CommentEmbeddings", "Delphi_CommentExtremity"] + client = Mock() + client.scan.side_effect = [ + {"Items": [item_from_python(dict(conversation_id=str(ZID), comment_id=1, text="x" * 240_000))]}, + {"Items": [item_from_python(dict(conversation_id=str(ZID), comment_id="1", text="x" * 240_000))]}, + ] + with pytest.raises(ValueError, match="bounded inline"): + legacy.export_local(client, tmp_path, families) + assert client.scan.call_count == 2 + assert not (tmp_path / "SHA256SUMS").exists() + + +def test_export_reader_shares_byte_limit_across_families(tmp_path): + source = { + "Delphi_CommentEmbeddings": [item_from_python(dict(conversation_id=str(ZID), comment_id=1, text="x" * 240_000))], + "Delphi_CommentExtremity": [item_from_python(dict(conversation_id=str(ZID), comment_id="1", text="x" * 240_000))], + } + manifest = [] + for family, rows in source.items(): + raw = encode_family(family, rows) + (tmp_path / (family + ".jsonl")).write_bytes(raw) + manifest.append(f"{legacy.digest(raw)} {family}.jsonl\n") + (tmp_path / "SHA256SUMS").write_text("".join(manifest)) + with pytest.raises(ValueError, match="byte limit"): + legacy.read_export(tmp_path) diff --git a/delphi/tests/test_delphi_postgres_results.py b/delphi/tests/test_delphi_postgres_results.py new file mode 100644 index 0000000000..745ffd8ba9 --- /dev/null +++ b/delphi/tests/test_delphi_postgres_results.py @@ -0,0 +1,46 @@ +"""Codec boundary tests; no database mocks stand in for the separate SQL proof.""" +import json +import unittest +from decimal import Decimal + +from polismath.delphi_storage.codec import CodecError, decode_family +from polismath.delphi_storage.postgres import family_files, decode_rows + + +class PostgresResultCodecTest(unittest.TestCase): + def test_exact_numeric_binary_and_set_roundtrip(self): + family = "Delphi_CommentEmbeddings" + rows = [{"conversation_id": "1", "comment_id": Decimal("7"), + "embedding": [Decimal("0.1234567890123456789012345678")], + "binary": b"\x00\xff", "labels": {"tree", "water"}, + "document": '{"preserved":"as string"}'}] + wire = family_files({family: rows})[family] + self.assertEqual(decode_rows(family, [json.loads(x) for x in wire.splitlines()[1:]]), rows) + self.assertEqual(decode_family(wire.encode())[0], family) + + def test_empty_family_is_explicit(self): + family = "Delphi_CommentEmbeddings" + self.assertEqual(decode_family(family_files({family: []})[family].encode())[1], []) + self.assertEqual(decode_rows(family, []), []) + + def test_queue_tables_are_not_results(self): + for family in ["Delphi_JobQueue", "Delphi_JobActiveGuard", "unknown"]: + with self.subTest(family=family), self.assertRaises(CodecError): + family_files({family: []}) + + def test_duplicate_key_refused(self): + row = {"conversation_id": "1", "comment_id": 1} + with self.assertRaises(CodecError): + family_files({"Delphi_CommentEmbeddings": [row, row]}) + + def test_float_refused_instead_of_losing_precision(self): + with self.assertRaises(CodecError): + family_files({"Delphi_CommentEmbeddings": [{"conversation_id": "1", "comment_id": 1, "value": 0.25}]}) + + def test_read_invalid_tag_refused(self): + with self.assertRaises(CodecError): + decode_rows("Delphi_CommentEmbeddings", [{"conversation_id": {"S": "1"}, "comment_id": {"N": "01"}}]) + + +if __name__ == "__main__": + unittest.main() diff --git a/delphi/tests/test_delphi_result_resource.py b/delphi/tests/test_delphi_result_resource.py new file mode 100644 index 0000000000..86a781b605 --- /dev/null +++ b/delphi/tests/test_delphi_result_resource.py @@ -0,0 +1,125 @@ +import os +import unittest +from unittest.mock import patch, Mock +from decimal import Decimal +from polismath.delphi_storage.resource import PostgresResource, result_resource +from polismath.delphi_storage.codec import item_from_python, encode_family + + +class Cursor: + def __init__(self,rows):self.rows=rows;self.archives=[];self.calls=[] + def __enter__(self):return self + def __exit__(self,*args):pass + def execute(self,sql,args=()):self.calls.append((sql,args)) + def fetchall(self):return self.archives if 'legacy_controls' in self.calls[-1][0] else self.rows + + +class Connection: + def __init__(self,rows):self.cur=Cursor(rows) + def cursor(self):return self.cur + + +class ResultResourceTest(unittest.TestCase): + def setUp(self): + self.env=patch.dict(os.environ,DELPHI_RESULT_BACKEND='postgres',DELPHI_RESULT_ENV='test-results') + self.env.start();self.addCleanup(self.env.stop) + self.rows=[(1,'scope',1,item_from_python(dict(conversation_id='1',comment_id=i,value=Decimal('0.25')))) for i in range(3)] + self.connection=Connection(self.rows) + self.table=PostgresResource(self.connection).Table('Delphi_CommentEmbeddings') + + def test_query_pushes_bound_key_and_preserves_decimal(self): + reply=self.table.query(KeyConditionExpression='conversation_id = :id',ExpressionAttributeValues={':id':'1'}) + self.assertEqual(reply['Count'],3) + self.assertEqual(reply['Items'][0]['value'],Decimal('0.25')) + sql,binds=self.connection.cur.calls[0] + self.assertIn('item->%s = %s::jsonb',sql) + self.assertIn('conversation_id',binds) + + def test_pagination_and_generation_change(self): + first=self.table.query(Limit=1) + self.assertEqual(self.table.query(Limit=1,ExclusiveStartKey=first['LastEvaluatedKey'])['Items'][0]['comment_id'],1) + self.connection.cur.rows=[(zid,'scope',2,row) for zid,_,_,row in self.rows] + with self.assertRaisesRegex(ValueError,'generation changed'): + self.table.query(ExclusiveStartKey=first['LastEvaluatedKey']) + + def test_ambiguous_scopes_refused(self): + self.connection.cur.rows.append((1,'other',1,item_from_python(dict(conversation_id='1',comment_id=0,value=Decimal('1'))))) + with self.assertRaisesRegex(ValueError,'ambiguous'): + self.table.query() + + def test_cross_conversation_cursor_tracks_each_generation(self): + self.connection.cur.rows += [(2,'scope',1,item_from_python(dict(conversation_id='2',comment_id=0)))] + first=self.table.scan(Limit=1) + self.connection.cur.rows=[(zid,scope,2 if zid==1 else generation,item) for zid,scope,generation,item in self.connection.cur.rows] + with self.assertRaisesRegex(ValueError,'generation changed'): + self.table.scan(ExclusiveStartKey=first['LastEvaluatedKey']) + + def test_legacy_metadata_retains_nul_and_exact_numbers_without_activation(self): + connection=Connection([({'job_id':'current','status':'COMPLETED'},)]) + archived=[dict(job_id='legacy',conversation_id='1',status='PROCESSING',logs='a\0b',job_config={'large':Decimal('9007199254740993')},binary=b'bytes'),dict(job_id='current',status='PROCESSING')] + wire=encode_family('Delphi_JobQueue',[item_from_python(item) for item in archived]).decode() + connection.cur.archives=[(1,'scope',4,wire)] + reply=PostgresResource(connection).Table('Delphi_JobQueue').scan() + legacy=next(item for item in reply['Items'] if item['job_id']=='legacy') + self.assertTrue(legacy['archived']);self.assertEqual(legacy['logs'],'a\0b') + self.assertEqual(legacy['job_config']['large'],Decimal('9007199254740993')) + self.assertEqual(legacy['binary'],b'bytes') + self.assertEqual(next(item for item in reply['Items'] if item['job_id']=='current')['status'],'COMPLETED') + + def test_mutation_refused(self): + with self.assertRaisesRegex(RuntimeError,'immutable'): + self.table.put_item(Item={}) + + def test_postgres_selection_never_calls_boto(self): + # Import is lazy: this test works with no boto3 package installed. + with patch.dict('sys.modules',{'boto3':None}): + self.assertIsInstance(result_resource(),PostgresResource) + + def test_index_sort_uses_created_time(self): + rows=[({'job_id':'z-older','created_at':'2025-01-01T00:00:00Z'},), + ({'job_id':'a-newer','created_at':'2026-01-01T00:00:00Z'},)] + table=PostgresResource(Connection(rows)).Table('Delphi_JobQueue') + self.assertEqual(table.query(IndexName='ConversationIndex',ScanIndexForward=False,Limit=1)['Items'][0]['job_id'],'a-newer') + + def test_job_get_reads_bound_attempt_logs(self): + connection=Connection([({'job_id':'generated-job'},)]) + fetch=connection.cur.fetchall + def rows(): + sql=connection.cur.calls[-1][0] + if 'pq_job_status' in sql:return [({'attempt_id':'generated-attempt'},)] + if 'pq_attempt_logs' in sql:return [('2026-01-01T00:00:00Z','INFO','actual child output')] + return fetch() + connection.cur.fetchall=rows + item=PostgresResource(connection).Table('Delphi_JobQueue').get_item(Key={'job_id':'generated-job'})['Item'] + self.assertEqual(item['log_attempt_id'],'generated-attempt') + self.assertIn('actual child output',item['logs']) + self.assertEqual(connection.cur.calls[-1][1],['test-results','generated-attempt']) + self.assertIn('NULL,1000',connection.cur.calls[-1][0]) + + def test_job_metadata_uses_same_graph_scope_as_archives(self): + connection=Connection([]) + with patch.dict(os.environ,DELPHI_RESULT_SCOPE='delphi'): + PostgresResource(connection).Table('Delphi_JobQueue').scan() + self.assertEqual(len(connection.cur.calls),2) + for sql,binds in connection.cur.calls: + self.assertIn('scope_key=%s',sql) + self.assertEqual(binds[-1],'delphi') + + def test_default_and_explicit_dynamo_forward_unchanged(self): + for backend in (None, 'dynamodb'): + with self.subTest(backend=backend), patch.dict(os.environ, {}, clear=True): + if backend is not None: + os.environ['DELPHI_RESULT_BACKEND'] = backend + boto = Mock() + with patch.dict('sys.modules', {'boto3': boto}): + result = result_resource(endpoint_url='http://localhost:8000') + self.assertIs(result, boto.resource.return_value) + boto.resource.assert_called_once_with('dynamodb', endpoint_url='http://localhost:8000') + + def test_unknown_backend_refused(self): + with patch.dict(os.environ,DELPHI_RESULT_BACKEND='postgress'): + with self.assertRaisesRegex(ValueError,'invalid'): + result_resource() + + +if __name__=='__main__':unittest.main() diff --git a/delphi/tests/test_narrative_audit.py b/delphi/tests/test_narrative_audit.py new file mode 100644 index 0000000000..4f11ef4405 --- /dev/null +++ b/delphi/tests/test_narrative_audit.py @@ -0,0 +1,47 @@ +"""Independent proof auditor catches group-ID confusion and factual drift.""" +import importlib.util +from pathlib import Path +from polismath.utils.vote_convention import SEMANTIC_AGREE, SEMANTIC_DISAGREE + +import pytest + +spec = importlib.util.spec_from_file_location("audit_narrative", Path(__file__).parent / "dynamo_removal/audit_narrative.py") +audit = importlib.util.module_from_spec(spec) +spec.loader.exec_module(audit) + + +def fixture(): + math = {"base-clusters": {"id": [500, 600], "members": [[0], [77]]}, + "group-clusters": [{"id": 5, "members": [500]}, {"id": 9, "members": [600]}], + "group_clusters": [{"id": 5, "members": [0]}, {"id": 9, "members": [77]}]} + context = dict(group_mapping="base-clusters-unfold/1", source_convention="semantic:+1=agree", + group_assignments_sha256=audit.digest({"0": 5, "77": 9}, sort_keys=True, separators=(",", ":")), + math_sha256=audit.digest(math, sort_keys=True, separators=(",", ":"), allow_nan=False), + comments=[{"comment_id": 0, "comment-id": 0, "num_groups": 2, + "votes": 2, "agrees": 1, "disagrees": 1, "passes": 0, + "total-votes": 2, "total-agrees": 1, "total-disagrees": 1, "total-passes": 0, + "group-5-votes": 1, "group-5-agrees": 1, + "group-9-votes": 1, "group-9-disagrees": 1}]) + votes = [dict(tid=0, pid=0, vote=SEMANTIC_AGREE), dict(tid=0, pid=77, vote=SEMANTIC_DISAGREE)] + return context, math, votes + + +def test_canonical_membership_uses_ids_not_array_positions(): + context, math, votes = fixture() + result = audit.audit_context(context, math, votes, [0]) + assert result["group_sizes"] == {"5": 1, "9": 1} + assert result["scoped_counts"] == 3 + + +def test_recount_rejects_reversed_group_direction(): + context, math, votes = fixture() + context["comments"][0]["group-5-agrees"] = 0 + with pytest.raises(AssertionError): + audit.audit_context(context, math, votes, [0]) + + +def test_recount_rejects_different_unfolded_alias(): + context, math, votes = fixture() + math["group_clusters"][0]["members"] = [77] + with pytest.raises(AssertionError): + audit.audit_context(context, math, votes, [0]) diff --git a/delphi/umap_narrative/502_calculate_priorities.py b/delphi/umap_narrative/502_calculate_priorities.py index eb50dbf0d7..3bbad098bd 100755 --- a/delphi/umap_narrative/502_calculate_priorities.py +++ b/delphi/umap_narrative/502_calculate_priorities.py @@ -10,6 +10,7 @@ import argparse import boto3 +from polismath.delphi_storage.resource import result_resource import json import logging import os @@ -51,7 +52,7 @@ def __init__(self, conversation_id: int, endpoint_url: str = None): boto3_kwargs['endpoint_url'] = endpoint_url # Initialize DynamoDB connection using the prepared arguments - self.dynamodb = boto3.resource('dynamodb', **boto3_kwargs) + self.dynamodb = result_resource('dynamodb', **boto3_kwargs) # Get table references self.comment_routing_table = self.dynamodb.Table('Delphi_CommentRouting') diff --git a/delphi/umap_narrative/701_static_datamapplot_for_layer.py b/delphi/umap_narrative/701_static_datamapplot_for_layer.py index 0ffb353cd6..67c2768154 100755 --- a/delphi/umap_narrative/701_static_datamapplot_for_layer.py +++ b/delphi/umap_narrative/701_static_datamapplot_for_layer.py @@ -14,6 +14,7 @@ import numpy as np import json import boto3 +from polismath.delphi_storage.resource import result_resource from boto3.dynamodb.conditions import Key import logging import sys @@ -37,7 +38,7 @@ class DynamoDBStorage: def __init__(self, endpoint_url=None): self.endpoint_url = endpoint_url or os.environ.get("DYNAMODB_ENDPOINT", "http://dynamodb-local:8000") self.region = os.environ.get("AWS_REGION", "us-east-1") - self.dynamodb = boto3.resource('dynamodb', endpoint_url=self.endpoint_url, region_name=self.region) + self.dynamodb = result_resource('dynamodb', endpoint_url=self.endpoint_url, region_name=self.region) # Define table names using the new Delphi_ naming scheme self.table_names = { diff --git a/delphi/umap_narrative/702_consensus_divisive_datamapplot.py b/delphi/umap_narrative/702_consensus_divisive_datamapplot.py index fefba91348..49a4768132 100755 --- a/delphi/umap_narrative/702_consensus_divisive_datamapplot.py +++ b/delphi/umap_narrative/702_consensus_divisive_datamapplot.py @@ -15,6 +15,7 @@ import matplotlib.pyplot as plt import json import boto3 +from polismath.delphi_storage.resource import result_resource import logging import traceback from decimal import Decimal @@ -68,7 +69,7 @@ def __init__(self, endpoint_url=None): else: self.endpoint_url = None self.region = DYNAMODB_CONFIG['region'] - self.dynamodb = boto3.resource('dynamodb', + self.dynamodb = result_resource('dynamodb', endpoint_url=self.endpoint_url, region_name=self.region, aws_access_key_id=DYNAMODB_CONFIG['access_key'], @@ -108,7 +109,7 @@ def load_data_from_dynamodb(zid, layer_num=0): # Set up DynamoDB client endpoint_url = os.environ.get('DYNAMODB_ENDPOINT') - dynamodb = boto3.resource('dynamodb', + dynamodb = result_resource('dynamodb', endpoint_url=endpoint_url, region_name=os.environ.get('AWS_REGION', 'us-east-1'), aws_access_key_id=os.environ.get('AWS_ACCESS_KEY_ID', 'fakeMyKeyId'), diff --git a/delphi/umap_narrative/801_narrative_report_batch.py b/delphi/umap_narrative/801_narrative_report_batch.py index 8adca5e5db..312bcb4cdb 100755 --- a/delphi/umap_narrative/801_narrative_report_batch.py +++ b/delphi/umap_narrative/801_narrative_report_batch.py @@ -27,6 +27,7 @@ import logging import argparse import boto3 +from polismath.delphi_storage.resource import result_resource import asyncio import numpy as np import pandas as pd @@ -66,7 +67,7 @@ def __init__(self, table_name="Delphi_NarrativeReports", dynamodb_resource=None) self.dynamodb = dynamodb_resource else: endpoint_url = os.environ.get('DYNAMODB_ENDPOINT') or None - self.dynamodb = boto3.resource( + self.dynamodb = result_resource( 'dynamodb', endpoint_url=endpoint_url, region_name=os.environ.get('AWS_REGION', 'us-east-1') @@ -145,61 +146,10 @@ def get_report(self, report_id, section, model): logger.error(f"Error getting report: {str(e)}") return None -class PolisConverter: - """Convert between CSV and XML formats for Polis data.""" - - @staticmethod - def convert_to_xml(comment_data): - """ - Convert comment data to XML format. - - Args: - comment_data: List of dictionaries with comment data - - Returns: - String with XML representation of the comment data - """ - # Create root element - root = ET.Element("polis-comments") - - # Process each comment - for record in comment_data: - # Extract base comment data - comment = ET.SubElement(root, "comment", { - "id": str(record.get("comment-id", "")), - "votes": str(record.get("total-votes", 0)), - "agrees": str(record.get("total-agrees", 0)), - "disagrees": str(record.get("total-disagrees", 0)), - "passes": str(record.get("total-passes", 0)), - }) - - # Add comment text - text = ET.SubElement(comment, "text") - text.text = record.get("comment", "") - - # Process group data - group_keys = [] - for key in record.keys(): - if key.startswith("group-") and key.count("-") >= 2: - group_id = key.split("-")[1] - if group_id not in group_keys: - group_keys.append(group_id) - - # Add data for each group - for group_id in group_keys: - group = ET.SubElement(comment, f"group-{group_id}", { - "votes": str(record.get(f"group-{group_id}-votes", 0)), - "agrees": str(record.get(f"group-{group_id}-agrees", 0)), - "disagrees": str(record.get(f"group-{group_id}-disagrees", 0)), - "passes": str(record.get(f"group-{group_id}-passes", 0)), - }) - - # Convert to string with pretty formatting - rough_string = ET.tostring(root, 'utf-8') - reparsed = parseString(rough_string) - return reparsed.toprettyxml(indent=" ") +from umap_narrative.narrative_data import PolisConverter, NarrativeSelection + -class BatchReportGenerator: +class BatchReportGenerator(NarrativeSelection): """Generate batch reports for Polis conversations.""" def __init__(self, conversation_id, model=None, no_cache=False, max_batch_size=20, job_id=None, layers=None, include_moderation=False, exclude_comment_selections=True): @@ -229,7 +179,7 @@ def __init__(self, conversation_id, model=None, no_cache=False, max_batch_size=2 logger.info(f"exclude_comment_selections: {exclude_comment_selections}") endpoint_url = os.environ.get('DYNAMODB_ENDPOINT') or None - self.dynamodb = boto3.resource( + self.dynamodb = result_resource( 'dynamodb', endpoint_url=endpoint_url, region_name=os.environ.get('AWS_REGION', 'us-east-1') @@ -572,328 +522,6 @@ async def get_topics(self): logger.error(f"A critical error occurred in get_topics: {str(e)}", exc_info=True) return [] - def filter_topics(self, comment, topic_cluster_id=None, topic_layer_id=None, topic_citations=None, sample_comments=None, filter_type=None, filter_threshold=None): - """Filter for comments that are part of a specific topic or meet global section criteria.""" - # Get comment ID - comment_id = comment.get('comment_id') - if not comment_id: - return False - - # Handle global section filtering - if filter_type is not None: - return self._apply_global_filter(comment, filter_type, filter_threshold) - - # Handle layer-specific topic filtering (existing logic) - if topic_cluster_id is not None and topic_layer_id is not None: - # Get the cluster ID for the specified layer - layer_cluster_key = f'layer{topic_layer_id}_cluster_id' - comment_cluster_id = comment.get(layer_cluster_key) - if comment_cluster_id is not None: - # Debug logging for cluster 0 - if str(topic_cluster_id) == "0" and comment_id in [1, 2, 3]: # Log first few comments - logger.info(f"DEBUG: Checking comment {comment_id} - layer{topic_layer_id}_cluster_id={comment_cluster_id}, topic_cluster_id={topic_cluster_id}") - logger.info(f"DEBUG: String comparison: '{str(comment_cluster_id)}' == '{str(topic_cluster_id)}' = {str(comment_cluster_id) == str(topic_cluster_id)}") - - # Simple string comparison is more reliable across different numeric types - if str(comment_cluster_id) == str(topic_cluster_id): - return True - - # Check if this comment ID is in our topic citations - if topic_citations and str(comment_id) in [str(c) for c in topic_citations]: - return True - - # If we have sample comments and not enough filtered comments, - # try to match based on text similarity - if sample_comments and len(sample_comments) > 0: - comment_text = comment.get('comment', '') - if not comment_text: - return False - - # Check if this comment text matches any sample comment - for sample in sample_comments: - # Skip non-string samples - if not isinstance(sample, str) or not sample: - continue - - # Simple substring match rather than complex word comparison - if sample.lower() in comment_text.lower() or comment_text.lower() in sample.lower(): - return True - - return False - - def _apply_global_filter(self, comment, filter_type, filter_threshold): - """ - Apply global section filtering based on Polis statistical metrics. - - Args: - comment: Comment data dictionary - filter_type: Type of filter ('comment_extremity', 'group_aware_consensus', 'uncertainty_ratio') - filter_threshold: Threshold value for filtering (or 'dynamic' for group_aware_consensus) - - Returns: - Boolean indicating whether comment passes the filter - """ - try: - if filter_type == "comment_extremity": - # Filter for comments that divide opinion groups (extremity > 1.0) - extremity = comment.get('comment_extremity', 0) - return extremity > filter_threshold - - elif filter_type == "group_aware_consensus": - # Filter for comments with broad cross-group agreement - # Uses dynamic thresholds based on number of groups - consensus = comment.get('group_aware_consensus', 0) - num_groups = comment.get('num_groups', 2) - - # Get dynamic threshold based on group count (matches Node.js logic) - if filter_threshold == "dynamic": - if num_groups == 2: - threshold = 0.7 - elif num_groups == 3: - threshold = 0.47 - elif num_groups == 4: - threshold = 0.32 - else: # 5+ groups - threshold = 0.24 - else: - threshold = filter_threshold - - return consensus > threshold - - elif filter_type == "uncertainty_ratio": - # Filter for comments with high uncertainty/unsure responses (>= 20% pass votes) - passes = comment.get('passes', 0) - votes = comment.get('votes', 0) - - if votes == 0: - return False - - uncertainty_ratio = passes / votes - return uncertainty_ratio >= filter_threshold - - else: - logger.warning(f"Unknown filter type: {filter_type}") - return False - - except Exception as e: - logger.error(f"Error applying global filter {filter_type}: {str(e)}") - return False - - def _get_dynamic_comment_limit(self, layer_id=None, total_layers=None, comment_count=None, filter_type=None): - """ - Calculate dynamic comment limit based on layer granularity and conversation size. - Implements the fractal approach where coarse layers get fewer, higher quality comments. - - Args: - layer_id: Current layer ID (None for global sections) - total_layers: Total number of available layers - comment_count: Total number of comments in conversation - filter_type: Type of filter (for global sections) - - Returns: - Integer comment limit for this section - """ - try: - # Base limits for different categories - base_limits = { - "global_sections": 50, # Fixed limit for global sections - "fine_layers": 100, # More comments for specific topics (layer 0) - "medium_layers": 75, # Balanced approach (middle layers) - "coarse_layers": 50 # Fewer, highest quality comments (top layer) - } - - # Determine category - if filter_type is not None: - # This is a global section - category = "global_sections" - elif layer_id is not None and total_layers is not None: - # This is a layer-specific topic - if layer_id == 0: - category = "fine_layers" # Most specific layer - elif layer_id == total_layers - 1: - category = "coarse_layers" # Most general layer - else: - category = "medium_layers" # Middle layers - else: - # Fallback to medium limit - category = "medium_layers" - - # Get base limit - limit = base_limits[category] - - # Scale down for very large conversations to manage token usage - if comment_count is not None: - if comment_count > 10000: - # Halve limits for huge conversations (>10k comments) - limit = int(limit * 0.5) - elif comment_count > 5000: - # Reduce by 25% for large conversations (5k-10k comments) - limit = int(limit * 0.75) - elif comment_count > 2000: - # Reduce by 10% for medium-large conversations (2k-5k comments) - limit = int(limit * 0.9) - - # Ensure minimum limit - limit = max(limit, 10) - - logger.debug(f"Dynamic comment limit: category={category}, base={base_limits[category]}, " - f"final={limit}, comment_count={comment_count}, layer_id={layer_id}") - - return limit - - except Exception as e: - logger.error(f"Error calculating dynamic comment limit: {str(e)}") - # Fallback to conservative limit - return 50 - - def _select_high_quality_comments(self, comments, limit, filter_type=None): - """ - Select the highest quality comments based on Polis statistical metrics. - - Args: - comments: List of comment dictionaries - limit: Maximum number of comments to select - filter_type: Type of filter being applied (affects sorting priority) - - Returns: - List of selected high-quality comments - """ - if len(comments) <= limit: - return comments - - try: - # Create sorting key based on filter type and available metrics - def get_sort_key(comment): - # Base score starts with vote count (engagement indicator) - votes = comment.get('votes', 0) - vote_score = int(votes) if isinstance(votes, (int, float)) else 0 - - # Add metric-specific scoring - if filter_type == "comment_extremity": - # For extremity filtering, prioritize highly divisive comments - extremity = comment.get('comment_extremity', 0) - metric_score = extremity * 1000 # Scale up for sorting - elif filter_type == "group_aware_consensus": - # For consensus filtering, prioritize high agreement comments - consensus = comment.get('group_aware_consensus', 0) - metric_score = consensus * 1000 # Scale up for sorting - elif filter_type == "uncertainty_ratio": - # For uncertainty filtering, prioritize comments with high pass rates - passes = comment.get('passes', 0) - total_votes = comment.get('votes', 1) - uncertainty = passes / max(total_votes, 1) - metric_score = uncertainty * 1000 # Scale up for sorting - else: - # For topic filtering, use a combination of votes and engagement - agrees = comment.get('agrees', 0) - disagrees = comment.get('disagrees', 0) - total_engagement = int(agrees) + int(disagrees) if isinstance(agrees, (int, float)) and isinstance(disagrees, (int, float)) else 0 - metric_score = total_engagement - - # Combine scores (metric score is primary, vote count is secondary) - return (metric_score, vote_score) - - # Sort comments by quality score (descending) - sorted_comments = sorted(comments, key=get_sort_key, reverse=True) - - # Select top comments up to limit - selected = sorted_comments[:limit] - - logger.info(f"Selected {len(selected)} high-quality comments from {len(comments)} " - f"(filter_type={filter_type}, limit={limit})") - - return selected - - except Exception as e: - logger.error(f"Error selecting high-quality comments: {str(e)}") - # Fallback to simple vote-based selection - try: - sorted_comments = sorted(comments, - key=lambda c: int(c.get('votes', 0)) if isinstance(c.get('votes'), (int, float)) else 0, - reverse=True) - return sorted_comments[:limit] - except Exception: - # Last resort: return first N comments - return comments[:limit] - - async def get_comments_as_xml(self, conversation_data: dict, filter_func=None, filter_args=None): - """Get comments as XML from pre-fetched data.""" - try: - # Use the data passed as an argument - data = conversation_data - - if not data: - logger.error("Received empty conversation data.") - return "" - - # Apply filter if provided - filtered_comments = data["processed_comments"] - - if filter_func: - if filter_args: - filtered_comments = [c for c in filtered_comments if filter_func(c, **filter_args)] - else: - filtered_comments = [c for c in filtered_comments if filter_func(c)] - - # Apply dynamic comment limiting with intelligent selection - if filter_func == self.filter_topics and len(filtered_comments) > 0: - # Get context for dynamic limit calculation - total_comment_count = len(data["processed_comments"]) - - # Extract layer and filter information from filter_args - layer_id = None - total_layers = None - filter_type = None - - if filter_args: - layer_id = filter_args.get('topic_layer_id') - filter_type = filter_args.get('filter_type') - - # Estimate total layers from conversation data (could be improved) - # For now, we'll determine this dynamically or use a reasonable default - if layer_id is not None: - # Try to determine total layers from available cluster data - # This is a heuristic - in practice you might want to pass this explicitly - total_layers = max(layer_id + 1, 3) # Assume at least 3 layers if we have layer data - - # Calculate dynamic limit - comment_limit = self._get_dynamic_comment_limit( - layer_id=layer_id, - total_layers=total_layers, - comment_count=total_comment_count, - filter_type=filter_type - ) - - # Apply intelligent comment selection if we exceed the limit - if len(filtered_comments) > comment_limit: - logger.info(f"Applying dynamic comment limit: {len(filtered_comments)} -> {comment_limit} " - f"(layer_id={layer_id}, filter_type={filter_type}, total_comments={total_comment_count})") - - # Use intelligent selection based on Polis metrics - filtered_comments = self._select_high_quality_comments( - filtered_comments, - comment_limit, - filter_type=filter_type - ) - else: - logger.info(f"No limiting needed: {len(filtered_comments)} comments <= limit of {comment_limit}") - else: - # For non-topic filtering, use a conservative limit to avoid token issues - max_comments = 100 - if len(filtered_comments) > max_comments: - logger.info(f"Applying conservative limit: {len(filtered_comments)} -> {max_comments}") - filtered_comments = self._select_high_quality_comments(filtered_comments, max_comments) - - # Convert to XML - xml = PolisConverter.convert_to_xml(filtered_comments) - - return xml - except Exception as e: - logger.error(f"Error in get_comments_as_xml: {str(e)}") - import traceback - logger.error(traceback.format_exc()) - return "" - async def prepare_batch_requests(self): """Prepare batch requests for all topics.""" logger.info("Fetching all conversation data ONCE...") diff --git a/delphi/umap_narrative/803_check_batch_status.py b/delphi/umap_narrative/803_check_batch_status.py index 5759bff9b9..94b66ee660 100755 --- a/delphi/umap_narrative/803_check_batch_status.py +++ b/delphi/umap_narrative/803_check_batch_status.py @@ -11,6 +11,7 @@ """ import os, sys, json, boto3, logging, argparse, asyncio +from polismath.delphi_storage.resource import result_resource from typing import Dict, Optional from datetime import datetime, timedelta, timezone from botocore.exceptions import ClientError @@ -46,7 +47,7 @@ def __init__(self): raw_endpoint = os.environ.get('DYNAMODB_ENDPOINT') endpoint_url = raw_endpoint if raw_endpoint and raw_endpoint.strip() else None - self.dynamodb = boto3.resource('dynamodb', endpoint_url=endpoint_url, region_name=os.environ.get('AWS_REGION', 'us-east-1')) + self.dynamodb = result_resource('dynamodb', endpoint_url=endpoint_url, region_name=os.environ.get('AWS_REGION', 'us-east-1')) self.job_table = self.dynamodb.Table('Delphi_JobQueue') self.report_table = self.dynamodb.Table('Delphi_NarrativeReports') # Provider token usage summed over the stored results, only when the diff --git a/delphi/umap_narrative/narrative_data.py b/delphi/umap_narrative/narrative_data.py new file mode 100644 index 0000000000..3e1129f96e --- /dev/null +++ b/delphi/umap_narrative/narrative_data.py @@ -0,0 +1,383 @@ +"""Pure narrative selection and XML formatting shared with legacy batch reports.""" +import logging +import xml.etree.ElementTree as ET +from xml.dom.minidom import parseString +logger = logging.getLogger(__name__) + +class PolisConverter: + """Convert between CSV and XML formats for Polis data.""" + + @staticmethod + def convert_to_xml(comment_data): + """ + Convert comment data to XML format. + + Args: + comment_data: List of dictionaries with comment data + + Returns: + String with XML representation of the comment data + """ + # Create root element + root = ET.Element("polis-comments") + + # Process each comment + for record in comment_data: + # Extract base comment data + comment = ET.SubElement(root, "comment", { + "id": str(record.get("comment-id", "")), + "votes": str(record.get("total-votes", 0)), + "agrees": str(record.get("total-agrees", 0)), + "disagrees": str(record.get("total-disagrees", 0)), + "passes": str(record.get("total-passes", 0)), + }) + + # Add comment text + text = ET.SubElement(comment, "text") + text.text = record.get("comment", "") + + # Process group data + group_keys = [] + for key in record.keys(): + if key.startswith("group-") and key.count("-") >= 2: + group_id = key.split("-")[1] + if group_id not in group_keys: + group_keys.append(group_id) + + # Add data for each group + for group_id in group_keys: + group = ET.SubElement(comment, f"group-{group_id}", { + "votes": str(record.get(f"group-{group_id}-votes", 0)), + "agrees": str(record.get(f"group-{group_id}-agrees", 0)), + "disagrees": str(record.get(f"group-{group_id}-disagrees", 0)), + "passes": str(record.get(f"group-{group_id}-passes", 0)), + }) + + # Convert to string with pretty formatting + rough_string = ET.tostring(root, 'utf-8') + reparsed = parseString(rough_string) + return reparsed.toprettyxml(indent=" ") + +class NarrativeSelection: + def filter_topics(self, comment, topic_cluster_id=None, topic_layer_id=None, topic_citations=None, sample_comments=None, filter_type=None, filter_threshold=None): + """Filter for comments that are part of a specific topic or meet global section criteria.""" + # Get comment ID + comment_id = comment.get('comment_id') + if comment_id is None: + return False + + # Handle global section filtering + if filter_type is not None: + return self._apply_global_filter(comment, filter_type, filter_threshold) + + # Handle layer-specific topic filtering (existing logic) + if topic_cluster_id is not None and topic_layer_id is not None: + # Get the cluster ID for the specified layer + layer_cluster_key = f'layer{topic_layer_id}_cluster_id' + comment_cluster_id = comment.get(layer_cluster_key) + if comment_cluster_id is not None: + # Debug logging for cluster 0 + if str(topic_cluster_id) == "0" and comment_id in [1, 2, 3]: # Log first few comments + logger.info(f"DEBUG: Checking comment {comment_id} - layer{topic_layer_id}_cluster_id={comment_cluster_id}, topic_cluster_id={topic_cluster_id}") + logger.info(f"DEBUG: String comparison: '{str(comment_cluster_id)}' == '{str(topic_cluster_id)}' = {str(comment_cluster_id) == str(topic_cluster_id)}") + + # Simple string comparison is more reliable across different numeric types + if str(comment_cluster_id) == str(topic_cluster_id): + return True + + # Check if this comment ID is in our topic citations + if topic_citations and str(comment_id) in [str(c) for c in topic_citations]: + return True + + # If we have sample comments and not enough filtered comments, + # try to match based on text similarity + if sample_comments and len(sample_comments) > 0: + comment_text = comment.get('comment', '') + if not comment_text: + return False + + # Check if this comment text matches any sample comment + for sample in sample_comments: + # Skip non-string samples + if not isinstance(sample, str) or not sample: + continue + + # Simple substring match rather than complex word comparison + if sample.lower() in comment_text.lower() or comment_text.lower() in sample.lower(): + return True + + return False + + def _apply_global_filter(self, comment, filter_type, filter_threshold): + """ + Apply global section filtering based on Polis statistical metrics. + + Args: + comment: Comment data dictionary + filter_type: Type of filter ('comment_extremity', 'group_aware_consensus', 'uncertainty_ratio') + filter_threshold: Threshold value for filtering (or 'dynamic' for group_aware_consensus) + + Returns: + Boolean indicating whether comment passes the filter + """ + try: + if filter_type == "comment_extremity": + # Filter for comments that divide opinion groups (extremity > 1.0) + extremity = comment.get('comment_extremity', 0) + return extremity > filter_threshold + + elif filter_type == "group_aware_consensus": + # Filter for comments with broad cross-group agreement + # Uses dynamic thresholds based on number of groups + consensus = comment.get('group_aware_consensus', 0) + num_groups = comment.get('num_groups', 2) + + # Get dynamic threshold based on group count (matches Node.js logic) + if filter_threshold == "dynamic": + if num_groups == 2: + threshold = 0.7 + elif num_groups == 3: + threshold = 0.47 + elif num_groups == 4: + threshold = 0.32 + else: # 5+ groups + threshold = 0.24 + else: + threshold = filter_threshold + + return consensus > threshold + + elif filter_type == "uncertainty_ratio": + # Filter for comments with high uncertainty/unsure responses (>= 20% pass votes) + passes = comment.get('passes', 0) + votes = comment.get('votes', 0) + + if votes == 0: + return False + + uncertainty_ratio = passes / votes + return uncertainty_ratio >= filter_threshold + + else: + logger.warning(f"Unknown filter type: {filter_type}") + return False + + except Exception as e: + logger.error(f"Error applying global filter {filter_type}: {str(e)}") + return False + + def _get_dynamic_comment_limit(self, layer_id=None, total_layers=None, comment_count=None, filter_type=None): + """ + Calculate dynamic comment limit based on layer granularity and conversation size. + Implements the fractal approach where coarse layers get fewer, higher quality comments. + + Args: + layer_id: Current layer ID (None for global sections) + total_layers: Total number of available layers + comment_count: Total number of comments in conversation + filter_type: Type of filter (for global sections) + + Returns: + Integer comment limit for this section + """ + try: + # Base limits for different categories + base_limits = { + "global_sections": 50, # Fixed limit for global sections + "fine_layers": 100, # More comments for specific topics (layer 0) + "medium_layers": 75, # Balanced approach (middle layers) + "coarse_layers": 50 # Fewer, highest quality comments (top layer) + } + + # Determine category + if filter_type is not None: + # This is a global section + category = "global_sections" + elif layer_id is not None and total_layers is not None: + # This is a layer-specific topic + if layer_id == 0: + category = "fine_layers" # Most specific layer + elif layer_id == total_layers - 1: + category = "coarse_layers" # Most general layer + else: + category = "medium_layers" # Middle layers + else: + # Fallback to medium limit + category = "medium_layers" + + # Get base limit + limit = base_limits[category] + + # Scale down for very large conversations to manage token usage + if comment_count is not None: + if comment_count > 10000: + # Halve limits for huge conversations (>10k comments) + limit = int(limit * 0.5) + elif comment_count > 5000: + # Reduce by 25% for large conversations (5k-10k comments) + limit = int(limit * 0.75) + elif comment_count > 2000: + # Reduce by 10% for medium-large conversations (2k-5k comments) + limit = int(limit * 0.9) + + # Ensure minimum limit + limit = max(limit, 10) + + logger.debug(f"Dynamic comment limit: category={category}, base={base_limits[category]}, " + f"final={limit}, comment_count={comment_count}, layer_id={layer_id}") + + return limit + + except Exception as e: + logger.error(f"Error calculating dynamic comment limit: {str(e)}") + # Fallback to conservative limit + return 50 + + def _select_high_quality_comments(self, comments, limit, filter_type=None): + """ + Select the highest quality comments based on Polis statistical metrics. + + Args: + comments: List of comment dictionaries + limit: Maximum number of comments to select + filter_type: Type of filter being applied (affects sorting priority) + + Returns: + List of selected high-quality comments + """ + if len(comments) <= limit: + return comments + + try: + # Create sorting key based on filter type and available metrics + def get_sort_key(comment): + # Base score starts with vote count (engagement indicator) + votes = comment.get('votes', 0) + vote_score = int(votes) if isinstance(votes, (int, float)) else 0 + + # Add metric-specific scoring + if filter_type == "comment_extremity": + # For extremity filtering, prioritize highly divisive comments + extremity = comment.get('comment_extremity', 0) + metric_score = extremity * 1000 # Scale up for sorting + elif filter_type == "group_aware_consensus": + # For consensus filtering, prioritize high agreement comments + consensus = comment.get('group_aware_consensus', 0) + metric_score = consensus * 1000 # Scale up for sorting + elif filter_type == "uncertainty_ratio": + # For uncertainty filtering, prioritize comments with high pass rates + passes = comment.get('passes', 0) + total_votes = comment.get('votes', 1) + uncertainty = passes / max(total_votes, 1) + metric_score = uncertainty * 1000 # Scale up for sorting + else: + # For topic filtering, use a combination of votes and engagement + agrees = comment.get('agrees', 0) + disagrees = comment.get('disagrees', 0) + total_engagement = int(agrees) + int(disagrees) if isinstance(agrees, (int, float)) and isinstance(disagrees, (int, float)) else 0 + metric_score = total_engagement + + # Combine scores (metric score is primary, vote count is secondary) + return (metric_score, vote_score) + + # Sort comments by quality score (descending) + sorted_comments = sorted(comments, key=get_sort_key, reverse=True) + + # Select top comments up to limit + selected = sorted_comments[:limit] + + logger.info(f"Selected {len(selected)} high-quality comments from {len(comments)} " + f"(filter_type={filter_type}, limit={limit})") + + return selected + + except Exception as e: + logger.error(f"Error selecting high-quality comments: {str(e)}") + # Fallback to simple vote-based selection + try: + sorted_comments = sorted(comments, + key=lambda c: int(c.get('votes', 0)) if isinstance(c.get('votes'), (int, float)) else 0, + reverse=True) + return sorted_comments[:limit] + except Exception: + # Last resort: return first N comments + return comments[:limit] + + async def get_comments_as_xml(self, conversation_data: dict, filter_func=None, filter_args=None): + """Get comments as XML from pre-fetched data.""" + try: + # Use the data passed as an argument + data = conversation_data + + if not data: + logger.error("Received empty conversation data.") + return "" + + # Apply filter if provided + filtered_comments = data["processed_comments"] + + if filter_func: + if filter_args: + filtered_comments = [c for c in filtered_comments if filter_func(c, **filter_args)] + else: + filtered_comments = [c for c in filtered_comments if filter_func(c)] + + # Apply dynamic comment limiting with intelligent selection + if filter_func == self.filter_topics and len(filtered_comments) > 0: + # Get context for dynamic limit calculation + total_comment_count = len(data["processed_comments"]) + + # Extract layer and filter information from filter_args + layer_id = None + total_layers = None + filter_type = None + + if filter_args: + layer_id = filter_args.get('topic_layer_id') + filter_type = filter_args.get('filter_type') + + # Estimate total layers from conversation data (could be improved) + # For now, we'll determine this dynamically or use a reasonable default + if layer_id is not None: + # Try to determine total layers from available cluster data + # This is a heuristic - in practice you might want to pass this explicitly + total_layers = max(layer_id + 1, 3) # Assume at least 3 layers if we have layer data + + # Calculate dynamic limit + comment_limit = self._get_dynamic_comment_limit( + layer_id=layer_id, + total_layers=total_layers, + comment_count=total_comment_count, + filter_type=filter_type + ) + + # Apply intelligent comment selection if we exceed the limit + if len(filtered_comments) > comment_limit: + logger.info(f"Applying dynamic comment limit: {len(filtered_comments)} -> {comment_limit} " + f"(layer_id={layer_id}, filter_type={filter_type}, total_comments={total_comment_count})") + + # Use intelligent selection based on Polis metrics + filtered_comments = self._select_high_quality_comments( + filtered_comments, + comment_limit, + filter_type=filter_type + ) + else: + logger.info(f"No limiting needed: {len(filtered_comments)} comments <= limit of {comment_limit}") + else: + # For non-topic filtering, use a conservative limit to avoid token issues + max_comments = 100 + if len(filtered_comments) > max_comments: + logger.info(f"Applying conservative limit: {len(filtered_comments)} -> {max_comments}") + filtered_comments = self._select_high_quality_comments(filtered_comments, max_comments) + + # Convert to XML + xml = PolisConverter.convert_to_xml(filtered_comments) + + return xml + except Exception as e: + logger.error(f"Error in get_comments_as_xml: {str(e)}") + import traceback + logger.error(traceback.format_exc()) + return "" + diff --git a/delphi/umap_narrative/numerical_stages.py b/delphi/umap_narrative/numerical_stages.py new file mode 100644 index 0000000000..44be4f0fa2 --- /dev/null +++ b/delphi/umap_narrative/numerical_stages.py @@ -0,0 +1,118 @@ +"""Pure numerical stages shared by legacy CLI and independently queued Delphi jobs. + +Extracted without changing parameters, EVOC fallback, or corpus TF-IDF semantics. +No storage clients or provider imports. +""" +import logging +import traceback +import numpy as np +import evoc +from umap import UMAP +from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer +logger = logging.getLogger(__name__) + + +def project_and_cluster(document_vectors): + # Generate 2D projection with UMAP + logger.info("Generating 2D projection with UMAP...") + document_map = UMAP(n_components=2, metric="cosine", random_state=42).fit_transform( + document_vectors + ) + + # Cluster with EVōC + logger.info("Clustering with EVōC...") + try: + clusterer = evoc.EVoC(min_samples=5) # Set min_samples to avoid empty clusters + cluster_labels = clusterer.fit_predict(document_vectors) + cluster_layers = clusterer.cluster_layers_ + + logger.info( + f"Found {len(np.unique(cluster_labels))} clusters at the finest level" + ) + for i, layer in enumerate(cluster_layers): + unique_clusters = np.unique(layer[layer >= 0]) + logger.info(f"Layer {i}: {len(unique_clusters)} clusters") + + except Exception as e: + logger.error(f"Error during EVōC clustering: {e}") + logger.error(traceback.format_exc()) + # Fallback to simple clustering + from sklearn.cluster import KMeans + + logger.info("Falling back to KMeans clustering...") + kmeans = KMeans(n_clusters=5, random_state=42) + cluster_labels = kmeans.fit_predict(document_vectors) + + # Create a simple layered clustering for demonstration + from sklearn.cluster import AgglomerativeClustering + + layer1 = AgglomerativeClustering(n_clusters=3).fit_predict(document_vectors) + layer2 = AgglomerativeClustering(n_clusters=2).fit_predict(document_vectors) + + cluster_layers = [cluster_labels, layer1, layer2] + logger.info( + f"Created {len(cluster_layers)} cluster layers with fallback clustering" + ) + + return document_map, cluster_layers + + +def characterize_comment_clusters(cluster_layer, comment_texts): + """ + Characterize comment clusters by common themes and keywords. + + Args: + cluster_layer: Cluster assignments for a specific layer + comment_texts: List of comment text strings + + Returns: + cluster_characteristics: Dictionary with cluster characterizations + """ + # Create a dictionary to store cluster characteristics + cluster_characteristics = {} + + # Get unique clusters + unique_clusters = np.unique(cluster_layer) + unique_clusters = unique_clusters[unique_clusters >= 0] # Remove noise points (-1) + + # Create TF-IDF vectorizer + vectorizer = CountVectorizer(max_features=1000, stop_words="english") + transformer = TfidfTransformer() + + # Fit and transform the entire corpus + X = vectorizer.fit_transform(comment_texts) + X_tfidf = transformer.fit_transform(X) + + # Get feature names + feature_names = vectorizer.get_feature_names_out() + + for cluster_id in unique_clusters: + # Get cluster members + cluster_members = np.where(cluster_layer == cluster_id)[0] + + if len(cluster_members) == 0: + continue + + # Get comment texts for this cluster + cluster_comments = [comment_texts[i] for i in cluster_members] + + # Find top words for this cluster by TF-IDF + cluster_tfidf = X_tfidf[cluster_members].toarray().mean(axis=0) + top_indices = np.argsort(cluster_tfidf)[-10:][::-1] # Top 10 words + top_words = [feature_names[i] for i in top_indices] + + # Get sample comments (shortest 3 for readability) + comment_lengths = [len(comment) for comment in cluster_comments] + shortest_indices = np.argsort(comment_lengths)[:3] # 3 shortest comments + sample_comments = [cluster_comments[i] for i in shortest_indices] + + # Add to cluster characteristics + cluster_characteristics[int(cluster_id)] = { + "size": len(cluster_members), + "top_words": top_words, + "top_tfidf_scores": [float(cluster_tfidf[i]) for i in top_indices], + "sample_comments": sample_comments, + } + + return cluster_characteristics + diff --git a/delphi/umap_narrative/polismath_commentgraph/utils/__init__.py b/delphi/umap_narrative/polismath_commentgraph/utils/__init__.py index 19de7a2273..c909bbb546 100644 --- a/delphi/umap_narrative/polismath_commentgraph/utils/__init__.py +++ b/delphi/umap_narrative/polismath_commentgraph/utils/__init__.py @@ -1,11 +1,13 @@ -""" -Utility functions for the Polis comment graph microservice. -""" +"""Utilities; numerical converters do not initialize storage dependencies.""" -from .storage import DynamoDBStorage -from .converter import DataConverter +__all__ = ['DynamoDBStorage', 'DataConverter'] -__all__ = [ - 'DynamoDBStorage', - 'DataConverter' -] \ No newline at end of file + +def __getattr__(name): + if name == 'DataConverter': + from .converter import DataConverter + return DataConverter + if name == 'DynamoDBStorage': + from .storage import DynamoDBStorage + return DynamoDBStorage + raise AttributeError(name) diff --git a/delphi/umap_narrative/polismath_commentgraph/utils/group_data.py b/delphi/umap_narrative/polismath_commentgraph/utils/group_data.py index 1343908f53..4865ef66aa 100644 --- a/delphi/umap_narrative/polismath_commentgraph/utils/group_data.py +++ b/delphi/umap_narrative/polismath_commentgraph/utils/group_data.py @@ -7,6 +7,7 @@ import json import logging import boto3 +from polismath.delphi_storage.resource import result_resource import os from typing import Dict, List, Any, Optional from collections import defaultdict @@ -48,7 +49,7 @@ def init_dynamodb(self): # Set up DynamoDB client WITHOUT explicit credentials. # Boto3 will use its default credential provider chain (env vars -> IAM role). - self.dynamodb = boto3.resource( + self.dynamodb = result_resource( 'dynamodb', endpoint_url=endpoint_url, region_name=region diff --git a/delphi/umap_narrative/polismath_commentgraph/utils/storage.py b/delphi/umap_narrative/polismath_commentgraph/utils/storage.py index b8dd49339c..e488de58b0 100644 --- a/delphi/umap_narrative/polismath_commentgraph/utils/storage.py +++ b/delphi/umap_narrative/polismath_commentgraph/utils/storage.py @@ -3,6 +3,7 @@ """ import boto3 +from polismath.delphi_storage.resource import result_resource import os import json import logging @@ -447,7 +448,7 @@ def __init__(self, region_name: str = None, endpoint_url: str = None): kwargs['aws_secret_access_key'] = aws_secret_access_key # Create the DynamoDB resource - self.dynamodb = boto3.resource('dynamodb', **kwargs) + self.dynamodb = result_resource('dynamodb', **kwargs) # Define table names self.table_names = { diff --git a/delphi/umap_narrative/reset_conversation.py b/delphi/umap_narrative/reset_conversation.py index 5a8f291fbd..4d7a53636c 100644 --- a/delphi/umap_narrative/reset_conversation.py +++ b/delphi/umap_narrative/reset_conversation.py @@ -36,6 +36,8 @@ def get_boto_resource(service_name: str): else: logger.info(f"AWS environment detected for {service_name}. Using IAM role credentials.") + if os.environ.get('DELPHI_RESULT_BACKEND') == 'postgres': + raise RuntimeError('Published Postgres results are immutable; submit a new graph run') return boto3.resource(service_name, **resource_args) diff --git a/delphi/umap_narrative/run_pipeline.py b/delphi/umap_narrative/run_pipeline.py index e7f7e5e060..497e3cb2e9 100755 --- a/delphi/umap_narrative/run_pipeline.py +++ b/delphi/umap_narrative/run_pipeline.py @@ -33,6 +33,7 @@ ) from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer from umap import UMAP +from umap_narrative.numerical_stages import characterize_comment_clusters # Configure logging logging.basicConfig( @@ -179,109 +180,12 @@ def process_comments(comments, conversation_id): embedding_model = SentenceTransformer(model_name) document_vectors = embedding_model.encode(comment_texts, show_progress_bar=True) - # Generate 2D projection with UMAP - logger.info("Generating 2D projection with UMAP...") - document_map = UMAP(n_components=2, metric="cosine", random_state=42).fit_transform( - document_vectors - ) - - # Cluster with EVōC - logger.info("Clustering with EVōC...") - try: - clusterer = evoc.EVoC(min_samples=5) # Set min_samples to avoid empty clusters - cluster_labels = clusterer.fit_predict(document_vectors) - cluster_layers = clusterer.cluster_layers_ - - logger.info( - f"Found {len(np.unique(cluster_labels))} clusters at the finest level" - ) - for i, layer in enumerate(cluster_layers): - unique_clusters = np.unique(layer[layer >= 0]) - logger.info(f"Layer {i}: {len(unique_clusters)} clusters") - - except Exception as e: - logger.error(f"Error during EVōC clustering: {e}") - logger.error(traceback.format_exc()) - # Fallback to simple clustering - from sklearn.cluster import KMeans - - logger.info("Falling back to KMeans clustering...") - kmeans = KMeans(n_clusters=5, random_state=42) - cluster_labels = kmeans.fit_predict(document_vectors) - - # Create a simple layered clustering for demonstration - from sklearn.cluster import AgglomerativeClustering - - layer1 = AgglomerativeClustering(n_clusters=3).fit_predict(document_vectors) - layer2 = AgglomerativeClustering(n_clusters=2).fit_predict(document_vectors) - - cluster_layers = [cluster_labels, layer1, layer2] - logger.info( - f"Created {len(cluster_layers)} cluster layers with fallback clustering" - ) + from umap_narrative.numerical_stages import project_and_cluster + document_map, cluster_layers = project_and_cluster(document_vectors) return document_map, document_vectors, cluster_layers, comment_texts, comment_ids -def characterize_comment_clusters(cluster_layer, comment_texts): - """ - Characterize comment clusters by common themes and keywords. - - Args: - cluster_layer: Cluster assignments for a specific layer - comment_texts: List of comment text strings - - Returns: - cluster_characteristics: Dictionary with cluster characterizations - """ - # Create a dictionary to store cluster characteristics - cluster_characteristics = {} - - # Get unique clusters - unique_clusters = np.unique(cluster_layer) - unique_clusters = unique_clusters[unique_clusters >= 0] # Remove noise points (-1) - - # Create TF-IDF vectorizer - vectorizer = CountVectorizer(max_features=1000, stop_words="english") - transformer = TfidfTransformer() - - # Fit and transform the entire corpus - X = vectorizer.fit_transform(comment_texts) - X_tfidf = transformer.fit_transform(X) - - # Get feature names - feature_names = vectorizer.get_feature_names_out() - - for cluster_id in unique_clusters: - # Get cluster members - cluster_members = np.where(cluster_layer == cluster_id)[0] - - if len(cluster_members) == 0: - continue - - # Get comment texts for this cluster - cluster_comments = [comment_texts[i] for i in cluster_members] - - # Find top words for this cluster by TF-IDF - cluster_tfidf = X_tfidf[cluster_members].toarray().mean(axis=0) - top_indices = np.argsort(cluster_tfidf)[-10:][::-1] # Top 10 words - top_words = [feature_names[i] for i in top_indices] - - # Get sample comments (shortest 3 for readability) - comment_lengths = [len(comment) for comment in cluster_comments] - shortest_indices = np.argsort(comment_lengths)[:3] # 3 shortest comments - sample_comments = [cluster_comments[i] for i in shortest_indices] - - # Add to cluster characteristics - cluster_characteristics[int(cluster_id)] = { - "size": len(cluster_members), - "top_words": top_words, - "top_tfidf_scores": [float(cluster_tfidf[i]) for i in top_indices], - "sample_comments": sample_comments, - } - - return cluster_characteristics - def create_comment_hover_info(cluster_layer, cluster_characteristics, comment_texts): """ diff --git a/docker-compose.test.yml b/docker-compose.test.yml index f489a829f1..f02113ac25 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -82,9 +82,12 @@ services: - ${AUTH_CERTS_PATH:-~/.simulacrum/certs}:/root/.simulacrum/certs:ro restart: unless-stopped depends_on: - - oidc-simulator - - postgres - - dynamodb-init + oidc-simulator: + condition: service_started + postgres: + condition: service_healthy + dynamodb-init: + condition: service_completed_successfully extra_hosts: - "host.docker.internal:host-gateway" @@ -112,7 +115,8 @@ services: networks: - polis-test depends_on: - - postgres + postgres: + condition: service_healthy restart: unless-stopped extra_hosts: - "host.docker.internal:host-gateway" @@ -160,6 +164,8 @@ services: build: context: ./server dockerfile: Dockerfile-db + additional_contexts: + queue-rs: ./queue-rs labels: polis_tag: test environment: @@ -167,7 +173,8 @@ services: - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} - POSTGRES_USER=${POSTGRES_USER} healthcheck: - test: ["CMD-SHELL", "pg_isready -U postgres"] + # The init server is socket-only; TCP refuses until migrations finish. + test: ["CMD-SHELL", "pg_isready -h 127.0.0.1 -U $$POSTGRES_USER -d $$POSTGRES_DB"] interval: 5s timeout: 5s retries: 5 @@ -175,7 +182,7 @@ services: networks: - polis-test ports: - - 5432:5432 + - "${POLIS_RECOVERY_PG_PORT:-5432}:5432" restart: unless-stopped # PostgreSQL configuration for development/testing command: > @@ -262,8 +269,10 @@ services: networks: - "polis-test" depends_on: - - postgres - - dynamodb-init + postgres: + condition: service_healthy + dynamodb-init: + condition: service_completed_successfully command: tail -f /dev/null restart: unless-stopped diff --git a/docker-compose.yml b/docker-compose.yml index d87a06b89f..30a6822fc7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -385,6 +385,8 @@ services: # Dockerfile-db (default): migrations-based initialization # Dockerfile-pdb: production dump restoration (requires prodclone.dump) dockerfile: Dockerfile-${DB_INIT_MODE:-db} + additional_contexts: + queue-rs: ./queue-rs labels: polis_tag: ${TAG:-dev} environment: diff --git a/docs/delphi-postgres-results.md b/docs/delphi-postgres-results.md new file mode 100644 index 0000000000..00d7030b18 --- /dev/null +++ b/docs/delphi-postgres-results.md @@ -0,0 +1,71 @@ +# Run-bound Delphi results (#1428) + +M28 stores all 18 result families from frozen `delphi-storage-codec/1`. Queue and +active-guard records remain control-plane records, never executable imported jobs. +No legacy table, vote value, or prior result is updated or deleted. + +`delphi_result_batches` binds one batch to an environment, graph job, graph run, +and attempt. `delphi_result_families` retains each canonical UTF-8 JSONL file and +its SHA-256. `delphi_result_rows` indexes tagged attributes by the family's real +composite key without converting decimal strings through floating point. JSONB +cannot represent NUL strings; importers must quarantine these with original bytes. + +A fenced worker or daemon calls: + +* `pd_result_put_family(env, job, owner, attempt, epoch, family, wire)`; identical + retries return the same digest, changed bytes at the same family key fail. +* `pd_result_seal(env, job, owner, attempt, epoch)`; returns the batch reference + `{schema, batch_id, sha256, families}`. A sealed batch refuses added families. +* The exact reference goes at artifact JSON `results`; graph finalization verifies + it and binds its artifact in the same transaction as queue success. Failed or + superseded attempts cannot affect any served generation. + +Small stage outputs may instead embed `family_files: {family: codecWire}`. The +artifact insert trigger validates, stores, seals and binds these within that same +transaction. The M27 artifact cap remains 512 KiB. The staged path permits up to +64 MiB per family and 256 MiB per batch. It does not require embedding full results +in the graph manifest or handing PostgreSQL credentials to a child process. + +Readers use `pd_result_artifact_family` for pinned dependencies and +`pd_result_served_bundle` for the coherent served generation. The view +`delphi_result_current_rows` provides environment, conversation, scope, generation, +family, tagged key, and tagged item for existing API filters. An entire family +comes from the closest artifact in the published dependency bundle; an explicitly +empty family shadows older ancestor rows. Existing graph publication CAS and +supersession checks determine visibility. Base tables are inaccessible to the +executor role, which receives only read APIs and fenced writers. Mutable human +annotations belong in separate generation-aware overrides, never these artifacts. + +`PostgresResultStore` and `PostgresResultReader` in +`polismath.delphi_storage.postgres` retain caller transaction ownership. They use +the existing frozen codec for Python resource values, validation, and reading. + +Validation commands (run on mm5): + +``` +cd delphi +python -m unittest tests.test_delphi_postgres_results -v +``` + +This exact command is added to the characterization workflow. The supplementary +real PostgreSQL campaign is `python delphi/tests/job_graph/results.py` with the +existing job-graph proof environment and an idle dedicated local queue. It covers +stale ownership, idempotence, changed retry refusal, atomic manifest rejection, +environment isolation, immutable tables, access denial and publication CAS. + +## Default and activation boundary + +M28/M29 are selected additive migrations in the release runner. Merging this +code does not switch readers or launch workers. An absent `DELPHI_RESULT_BACKEND` +uses DynamoDB in both Node and Python; `dynamodb` is the explicit default. +Only `postgres` selects the Postgres reader, with `DELPHI_RESULT_ENV` required +and `DELPHI_RESULT_SCOPE` optional. PostgreSQL errors never fall back to DynamoDB. +The server already receives these settings through its environment file; Python +reader processes need them in their own environment. No production flag is set +by this change. Verify published/imported coverage before Colin activates it. + +`delphi/scripts/import_dynamo_export.py` is an explicitly invoked command. No +startup hook, migration or service automatically runs it. See +`delphi/docs/LEGACY_DYNAMO_IMPORT.md` for its bounded input contract. The proof's +fixed narrative/name outputs demonstrate transport only. Provider execution, +text quality, old Dynamo table retirement and reader activation are separate. diff --git a/docs/job-graphs.md b/docs/job-graphs.md new file mode 100644 index 0000000000..0322b70662 --- /dev/null +++ b/docs/job-graphs.md @@ -0,0 +1,97 @@ +# Per-step jobs that keep completed work + +Each computation has its own run, recorded inputs and immutable saved result. +Dependants wait for their producers. A failed step retries using the same saved +inputs; completed steps do not rerun. Readers keep seeing the last complete +bundle until a complete replacement is published atomically. + +The reference example is embeddings → clusters → narrative. Its local adapters +use token counts, a small deterministic clustering fit and a cluster summary. +These exercise the real daemon and separate child processes. They are not the +production MiniLM, UMAP or provider pipeline; those adapters and report readers +are a separate integration. + +## Stored contract + +M27 extends M19/M23/M24 without changing those files. The queue contract is +`polis-queue/5`; `/4` remains reserved. Each step has one run and job; each retry +has a new attempt. Inputs record the data snapshot and digest, code digest, +model, runtime, seed, effective configuration and computation mode. This +adapter supports `full` only and refuses unsupported incremental requests. +Reusing embeddings while fully recomputing clusters is supported. + +Admission seals the complete graph in one transaction. Edges name producers +before their results exist, then bind once to immutable artifacts. Claims +include exact serialized resolved inputs and their digest; both the Rust daemon +and Python child verify them. Postgres binds results to the job, run and exited +attempt. Finalization checks ownership, lease and process-exit proof and is +idempotent after a lost reply. Failed attempt files are never served results. + +Publication uses a generation compare-and-swap to select a completed narrative +and its transitive artifacts. Old bundles are untouched by failures or worker +replacement. Completion of the embedding root alone does not release the graph +scope; durable reconciliation waits for all members and their exit proofs. +Class-depth demand excludes dependency-blocked work. + +`delphi/scripts/job_graph_client.py` provides internal admission, status, +publication and result reading. Callers must authorize the namespace before +using it; this is not a public HTTP API. Configure graph workers explicitly with +`POLIS_JOBS_STAGES=graph_embed,graph_cluster,graph_narrative`. Legacy and graph +stages require separate workers. Defaults continue to select legacy stages. +Existing workers without `/5` support must be replaced before applying M27. + +## Deliberately outside this core + +Scoped durable breakers and half-open probes, provider reconciliation and its +evidence table, and explicit superseding dead-branch redrive are deferred. +The sixth admission argument is retained for wire compatibility but must be +NULL. Retry exhaustion leaves the job dead and its dependants visibly blocked; +it never wipes completed results. The existing queue retry/dead-letter logic +and fail-closed provider-uncertainty checks remain. No paid-provider activation, +spend controls, new placement/scaling system or retention deletion is added. + +Limits are 32 nodes, four inputs per node, 100 texts, 1 MiB admission and +512 KiB per artifact. Each job has at most ten attempts. Memory declarations +are checked against class bounds (512 MiB Delphi, 2 GiB large); this is not OS +memory enforcement. Real adapters and larger result storage need their own +contracts. No artifact purge is enabled. + +## Apply and verify + +Release B restores the migration runner and selects M27 for fresh databases and +existing deployments. The deploy reconciles the supported legacy catalog before +applying pending files; the API checks their receipts before starting. +The earlier unreleased M27 +had different bytes: never adopt its old installation under this new checksum. +Rebuild disposable draft databases; released histories require a forward +migration, never a rewritten receipt. + +The empty down migration compares the restored catalog with its original `/3` +state. It refuses once graph data exists. A nonempty installation requires +compatible `/5` workers and readers; deleting results is not rollback. + +Hosted CI and mm5 use exactly the same entry: + +```sh +COMPOSE_PROJECT_NAME=polis-graph-test-local \ +POLIS_RECOVERY_PG_PORT=55467 RECOVERY_PG_PORT=55467 \ +bash delphi/tests/job_graph/run.sh +``` + +Run on a build box with Docker, the pinned Rust toolchain, Python 3 and Node 24; +first run `npm ci --no-audit --no-fund` in `server`. Choose an unused project and +port. The entry refuses existing project containers, starts only its own local +Postgres and removes its project on exit. It runs the graph scenarios, SQL +boundaries, empty down/reapply, Rust and existing Node/legacy queue tests. +Receipts go to `graph-proof/` or `GRAPH_PROOF_ROOT`. Fixtures contain generated +texts and zero vote rows. No provider calls are needed. + +## Release B integration + +The selected migration release now includes M27. After Release A has succeeded +on every host, the deploy hook runs `polis-migrate deploy`: reconcile a supported +pre-runner catalog through M22, then apply M19/M23/M24/M27 and check readiness, +before replacing services. A modern M24 history upgrades by applying M27 only; +existing receipt bytes are preserved. An unledgered or old-draft M27 catalog is +not silently adopted or re-sealed. The API refuses pending or changed receipts. +DynamoDB readers/importers and M28/M29 are outside this release. diff --git a/docs/migration-legacy-contract.md b/docs/migration-legacy-contract.md new file mode 100644 index 0000000000..6aec852d47 --- /dev/null +++ b/docs/migration-legacy-contract.md @@ -0,0 +1,175 @@ +# Supported legacy schema contract + +This release accepts the documented legacy schema without changing its data or +rewriting old migrations. The live schema is authoritative for the named legacy +variant; the existing bootstrap is a separately supported installation variant. +The immutable numbered files are execution history, not a claim that every +installation ran every file. Adoption predicates describe the supported states. + +The concrete admission changes are bounded: `worker_tasks.task_type` accepts +`text` or `varchar(99)`; renamed `pwreset_tokens.token` accepts `varchar(100)` +or `varchar(250)`; absent `conversations.branding_type` and +`math_ticks.caching_tick` are valid, but existing columns must have their known +integer/bigint types. Contributor agreement tables are outside this release's +adoption scope. M2 requires enforced owner/xid uniqueness and preserves an +optional enforced, nonpartial owner/uid unique index in either key order. +Other lengths, arbitrary text alternatives, invalid indexes and partial unique +owner/uid indexes remain refused. + +Six math payloads accept `json` alongside bootstrap `jsonb`: `math_bidtopid`, +`math_cache`, `math_exportstatus`, `math_main`, `math_profile`, `math_ptptstats`. +`math_report_correlationmatrix.data` remains `jsonb`; no observed legacy JSON +contract was established for it. This narrows an earlier broad helper. + +The file contract below records the rest of the reviewed live differences. +It does not add new global core-default, routine-body or ACL adoption checks. +These documentary contracts must not be mistaken for an exhaustive schema +certifier. Existing modern M3/M8–18 predicates continue to enforce the properties +introduced by those migrations; unsupported/partial schemas stop for review. + +## Named live attributes + +| Attribute | Legacy contract | Bootstrap alternative | +| --- | --- | --- | +| comments.uid | integer NOT NULL, default 0 | no default | +| comments.velocity | nullable real, default 1 | NOT NULL real | +| conversations.auth_needed_to_vote | default false | no default | +| conversations.auth_needed_to_write | default true | no default | +| conversations.auth_opt_fb | default true | no default | +| conversations.auth_opt_tw | default true | no default | +| conversations.auth_opt_allow_3rdparty | default true | no default | +| contexts/context_id, conversations/zid, courses/course_id, participant_metadata_answers/pmaid, participant_metadata_questions/pmqid, users/uid sequences | bigint sequence, maximum 9223372036854775807, integer owning column | integer sequence, maximum 2147483647 | + +Sequence configuration is not current sequence state. Never reset, narrow or +advance a live sequence to match bootstrap. Reviewed implementations of +`get_times_for_most_recent_visible_comments()`, `pid_auto()`, `tid_auto()`, +`random_string(integer)` and `random_polis_site_id()` are executable equivalents +of the public historical sources; formatting fingerprints differ. This release +never replaces those routines. Their exact historical source bindings are in +the deployment's private reconciliation record. + +Optional probe provisioning grants SELECT on comments, conversations, math_main, +math_ticks, participants and votes, plus public schema USAGE, to the configured +read-only probe role. It is separate from application and migration authority; +no role identity or grant is imposed on other deployments by this contract. +M19's actual migration session needs role creation/SET authority, public schema +USAGE/CREATE grant authority, conversations SELECT plus topic UPDATE and zid +REFERENCES grant authority. M23/M24 continue as the queue owner. Verify the +actual session; a superuser-only test is insufficient evidence for that session. + +## Other live attributes and scope + +| Attribute | Type | Nullability | +| --- | --- | --- | +| comments.curation | smallint | NOT NULL | +| participants_extended.encrypted_ip_address | character varying(9999) | nullable | +| participants_extended.encrypted_x_forwarded_for | character varying(9999) | nullable | +| suzinvites.modified | bigint | nullable | +| suzinvites.uid | integer | nullable | +| users.pwhash | character varying(128) | nullable | + +Existing `suzinvites.uid` references users.uid; no field/constraint is added, +dropped or backfilled during reconciliation. Legacy callers and generated +schema descriptions may observe these fields; fresh bootstrap does not acquire +them merely because the legacy contract documents them. Whole-tree generated +schema modernization is a separate change. + +The following complete research-triage list records the stable scope. These are +research dispositions, not claims that bootstrap or generated ORM declarations +were rewritten in this patch. The explicit adoption changes and separate legacy +file contract described above are implemented here; references below to canonical +files/ORM or generated-reader changes remain broader modernization obligations. +“Before +stable” means a file/adoption contract, never authorization for production DDL. +Ancillary objects are retained. No extension removal, name rewrite, course-invite +unique constraint addition, contributor rename or data cleanup is selected. + +| Difference | Scope | Contract disposition | +| --- | --- | --- | +| contributor | SET ASIDE | Defer ancillary file spelling repair for stable; preserve live correct table and working endpoint, scoped non-gating exclusion and graceful503 for absent schemas. No production operation. | +| branding | SYNC BEFORE STABLE | Canonical files omit branding_type to match production; reconcile required generated mappings, preserve exact downstream variants without DROP. | +| caching | SYNC BEFORE STABLE | Canonical files/ORM omit math_ticks.caching_tick to match production; preserve math_main active cursor and existing downstream variants. | +| course | SET ASIDE | Record production absence of invite uniqueness; set aside for narrow stable because M0 adoption does not test it. Later complete canonical files omit the invariant; no production constraint changes. | +| xids | SYNC BEFORE STABLE | Canonical files/adoption preserve both production XID unique invariants; revise prior M2 guard, no drop-owner/uid action. | +| worker_tasks.task_type text versus varchar(99) | SYNC BEFORE STABLE | Actual adoption refusal: align exact production type and preserve reviewed downstream variants. | +| pwreset_tokens.token varchar(100) versus varchar(250) | SYNC BEFORE STABLE | Actual adoption refusal: align exact production type and preserve reviewed downstream variants. | +| math_bidtopid.data json versus jsonb | SYNC BEFORE STABLE | Named JSON exception already admits this type. Record production JSON and supported JSONB variant; no conversion. | +| math_cache.data json versus jsonb | SYNC BEFORE STABLE | Named JSON exception already admits this type. Record production JSON and supported JSONB variant; no conversion. | +| math_exportstatus.data json versus jsonb | SYNC BEFORE STABLE | Named JSON exception already admits this type. Record production JSON and supported JSONB variant; no conversion. | +| math_main.data json versus jsonb | SYNC BEFORE STABLE | Named JSON exception already admits this type. Record production JSON and supported JSONB variant; no conversion. | +| math_profile.data json versus jsonb | SYNC BEFORE STABLE | Named JSON exception already admits this type. Record production JSON and supported JSONB variant; no conversion. | +| math_ptptstats.data json versus jsonb | SYNC BEFORE STABLE | Named JSON exception already admits this type. Record production JSON and supported JSONB variant; no conversion. | +| comments.uid default zero versus absent | SYNC BEFORE STABLE | Record exact live default/nullability in the file contract and reconcile affected generated readers. Existing M0 does not reject it. Defer behavior changes, backfill, or production DDL. | +| comments.velocity nullable versus NOT NULL | SYNC BEFORE STABLE | Record exact live default/nullability in the file contract and reconcile affected generated readers. Existing M0 does not reject it. Defer behavior changes, backfill, or production DDL. | +| conversations.auth_needed_to_vote default literal versus absent | SYNC BEFORE STABLE | Record exact live default/nullability in the file contract and reconcile affected generated readers. Existing M0 does not reject it. Defer behavior changes, backfill, or production DDL. | +| conversations.auth_needed_to_write default literal versus absent | SYNC BEFORE STABLE | Record exact live default/nullability in the file contract and reconcile affected generated readers. Existing M0 does not reject it. Defer behavior changes, backfill, or production DDL. | +| conversations.auth_opt_fb default literal versus absent | SYNC BEFORE STABLE | Record exact live default/nullability in the file contract and reconcile affected generated readers. Existing M0 does not reject it. Defer behavior changes, backfill, or production DDL. | +| conversations.auth_opt_tw default literal versus absent | SYNC BEFORE STABLE | Record exact live default/nullability in the file contract and reconcile affected generated readers. Existing M0 does not reject it. Defer behavior changes, backfill, or production DDL. | +| conversations.auth_opt_allow_3rdparty default literal versus absent | SYNC BEFORE STABLE | Record exact live default/nullability in the file contract and reconcile affected generated readers. Existing M0 does not reject it. Defer behavior changes, backfill, or production DDL. | +| contexts_context_id_seq | SYNC BEFORE STABLE | Record exact bigint sequence with integer owning column; preserve current state and consumers, no setval/ALTER. Existing adoption does not check its bounds. | +| conversations_zid_seq | SYNC BEFORE STABLE | Record exact bigint sequence with integer owning column; preserve current state and consumers, no setval/ALTER. Existing adoption does not check its bounds. | +| courses_course_id_seq | SYNC BEFORE STABLE | Record exact bigint sequence with integer owning column; preserve current state and consumers, no setval/ALTER. Existing adoption does not check its bounds. | +| participant_metadata_answers_pmaid_seq | SYNC BEFORE STABLE | Record exact bigint sequence with integer owning column; preserve current state and consumers, no setval/ALTER. Existing adoption does not check its bounds. | +| participant_metadata_questions_pmqid_seq | SYNC BEFORE STABLE | Record exact bigint sequence with integer owning column; preserve current state and consumers, no setval/ALTER. Existing adoption does not check its bounds. | +| users_uid_seq | SYNC BEFORE STABLE | Record exact bigint sequence with integer owning column; preserve current state and consumers, no setval/ALTER. Existing adoption does not check its bounds. | +| random_polis_site_id(integer) | SET ASIDE | Retain unused integer overload as ancillary legacy state; no current direct caller established. | +| get_times_for_most_recent_visible_comments() | SYNC BEFORE STABLE | Record reviewed production routine body/fingerprint for live trigger/caller contract. Executable equivalence established; no CREATE OR REPLACE and no existing body-hash adoption refusal. | +| pid_auto() | SYNC BEFORE STABLE | Record reviewed production routine body/fingerprint for live trigger/caller contract. Executable equivalence established; no CREATE OR REPLACE and no existing body-hash adoption refusal. | +| tid_auto() | SYNC BEFORE STABLE | Record reviewed production routine body/fingerprint for live trigger/caller contract. Executable equivalence established; no CREATE OR REPLACE and no existing body-hash adoption refusal. | +| random_string(integer) | SYNC BEFORE STABLE | Record reviewed production routine body/fingerprint for live trigger/caller contract. Executable equivalence established; no CREATE OR REPLACE and no existing body-hash adoption refusal. | +| random_polis_site_id() | SYNC BEFORE STABLE | Record reviewed production routine body/fingerprint for live trigger/caller contract. Executable equivalence established; no CREATE OR REPLACE and no existing body-hash adoption refusal. | +| comments | SYNC BEFORE STABLE | Bind existing optional probe provisioning contract; exact six SELECT/public-USAGE grants already have a source. No role/grant changes. | +| conversations | SYNC BEFORE STABLE | Bind existing optional probe provisioning contract; exact six SELECT/public-USAGE grants already have a source. No role/grant changes. | +| math_main | SYNC BEFORE STABLE | Bind existing optional probe provisioning contract; exact six SELECT/public-USAGE grants already have a source. No role/grant changes. | +| math_ticks | SYNC BEFORE STABLE | Bind existing optional probe provisioning contract; exact six SELECT/public-USAGE grants already have a source. No role/grant changes. | +| participants | SYNC BEFORE STABLE | Bind existing optional probe provisioning contract; exact six SELECT/public-USAGE grants already have a source. No role/grant changes. | +| votes | SYNC BEFORE STABLE | Bind existing optional probe provisioning contract; exact six SELECT/public-USAGE grants already have a source. No role/grant changes. | +| public | SYNC BEFORE STABLE | Bind existing optional probe provisioning contract; exact six SELECT/public-USAGE grants already have a source. No role/grant changes. | +| portable ownership and selected-release privileges | SYNC BEFORE STABLE | Use role parameters/capabilities, not hosted dbUser ownership in public bootstrap. Prove actual migration-session M19 grants/role authority and M23/M24 SET ROLE; reject partial conflicting installs. No role reassignment. | +| comments.curation | SYNC BEFORE STABLE | Record exact live/script-visible field in supported legacy file contract; reconcile generated declarations without changing live values. delphi/scripts/generate_cold_start_clojure.py:293–296 explicitly copies curation; client test hits are unrelated prose/UI. Fresh installs currently lack the script field. | +| conversations.dataset_explanation | SET ASIDE | No literal field reference at any scanned current pin; conversation-wide SELECT * and dynamic JSON remain indirect possibilities. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| conversations.is_curated | SET ASIDE | No literal field reference at the scanned current pins; generic conversation readers can expose it. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| conversations.need_suzinvite | SET ASIDE | No literal field reference at the scanned current pins; do not confuse auth handlers for suzinvites with this dormant flag. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| participants_extended.country_iso_code | SET ASIDE | No literal field reference in the scanned current pins. M7 removes country_code_iso (different spelling), NOT this field. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| participants_extended.encrypted_ip_address | SYNC BEFORE STABLE | Record exact live/script-visible field in supported legacy file contract; reconcile generated declarations without changing live values. server/src/participant.ts:291–300 and db/sql.ts:96–103 conditionally write/declare this field only for applicationName PolisWebServer. | +| participants_extended.encrypted_x_forwarded_for | SYNC BEFORE STABLE | Record exact live/script-visible field in supported legacy file contract; reconcile generated declarations without changing live values. server/src/participant.ts:291–300 and db/sql.ts:96–103 conditionally write/declare this field only for applicationName PolisWebServer. | +| suzinvites.modified | SYNC BEFORE STABLE | Record exact live/script-visible field in supported legacy file contract; reconcile generated declarations without changing live values. server/src/invites/suzinvites.ts:17,68 uses SELECT *; INSERTs at 40 and 167 omit modified. Whole-row consumers can observe it; bare modified inventory includes unrelated tables. | +| suzinvites.uid | SYNC BEFORE STABLE | Record exact live/script-visible field in supported legacy file contract; reconcile generated declarations without changing live values. server/src/invites/suzinvites.ts:17,68 SELECT *; normal INSERTs omit uid. Bare uid matches are broad; table-specific paths are the relevant indirect contract. | +| users.pwhash | SYNC BEFORE STABLE | Record exact live/script-visible field in supported legacy file contract; reconcile generated declarations without changing live values. bin/anonymize_users.sh: updates pwhash; generated Rust users row/catalog retains pwhash; M0 comments it out. No current password-login consumer established. | +| users.test | SET ASIDE | Bare test appears throughout test tooling and is not a users.test consumer; generic users SELECT * can expose it. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| conversation_invite_codes | SET ASIDE | No current literal SQL consumer established; whole-tree locations/zero-results and historical SQL pickaxe are retained. Generic queue/moderators hits are not proof of public.conversation_invite_codes usage. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| conversation_subscriptions | SET ASIDE | No current literal SQL consumer established; whole-tree locations/zero-results and historical SQL pickaxe are retained. Generic queue/moderators hits are not proof of public.conversation_subscriptions usage. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| error_reports | SET ASIDE | No current literal SQL consumer established; whole-tree locations/zero-results and historical SQL pickaxe are retained. Generic queue/moderators hits are not proof of public.error_reports usage. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| math_results_dev01 | SET ASIDE | No current literal SQL consumer established; whole-tree locations/zero-results and historical SQL pickaxe are retained. Generic queue/moderators hits are not proof of public.math_results_dev01 usage. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| minvites | SET ASIDE | No current literal SQL consumer established; whole-tree locations/zero-results and historical SQL pickaxe are retained. Generic queue/moderators hits are not proof of public.minvites usage. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| moderators | SET ASIDE | No current literal SQL consumer established; whole-tree locations/zero-results and historical SQL pickaxe are retained. Generic queue/moderators hits are not proof of public.moderators usage. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| nyt_users | SET ASIDE | No current literal SQL consumer established; whole-tree locations/zero-results and historical SQL pickaxe are retained. Generic queue/moderators hits are not proof of public.nyt_users usage. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| polismath_mod_claims | SET ASIDE | No current literal SQL consumer established; whole-tree locations/zero-results and historical SQL pickaxe are retained. Generic queue/moderators hits are not proof of public.polismath_mod_claims usage. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| queue | SET ASIDE | No current literal SQL consumer established; whole-tree locations/zero-results and historical SQL pickaxe are retained. Generic queue/moderators hits are not proof of public.queue usage. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| slack_participants_waiting_for_comments | SET ASIDE | No current literal SQL consumer established; whole-tree locations/zero-results and historical SQL pickaxe are retained. Generic queue/moderators hits are not proof of public.slack_participants_waiting_for_comments usage. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| slack_state_heap | SET ASIDE | No current literal SQL consumer established; whole-tree locations/zero-results and historical SQL pickaxe are retained. Generic queue/moderators hits are not proof of public.slack_state_heap usage. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| slack_state_stack | SET ASIDE | No current literal SQL consumer established; whole-tree locations/zero-results and historical SQL pickaxe are retained. Generic queue/moderators hits are not proof of public.slack_state_stack usage. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| slack_team_tokens | SET ASIDE | No current literal SQL consumer established; whole-tree locations/zero-results and historical SQL pickaxe are retained. Generic queue/moderators hits are not proof of public.slack_team_tokens usage. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| animals_id_auto | SET ASIDE | No current literal routine call established; votes_lastest_unique appears only as a bootstrap comment. All SQL snapshot bodies are available historically; current production body fingerprints are compared below. Dynamic/external SQL and trigger dependencies need the aggregate census. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| oid_auto | SET ASIDE | No current literal routine call established; votes_lastest_unique appears only as a bootstrap comment. All SQL snapshot bodies are available historically; current production body fingerprints are compared below. Dynamic/external SQL and trigger dependencies need the aggregate census. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| oid_auto_unlock | SET ASIDE | No current literal routine call established; votes_lastest_unique appears only as a bootstrap comment. All SQL snapshot bodies are available historically; current production body fingerprints are compared below. Dynamic/external SQL and trigger dependencies need the aggregate census. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| ptpt_id_auto | SET ASIDE | No current literal routine call established; votes_lastest_unique appears only as a bootstrap comment. All SQL snapshot bodies are available historically; current production body fingerprints are compared below. Dynamic/external SQL and trigger dependencies need the aggregate census. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| ptpt_id_auto_unlock | SET ASIDE | No current literal routine call established; votes_lastest_unique appears only as a bootstrap comment. All SQL snapshot bodies are available historically; current production body fingerprints are compared below. Dynamic/external SQL and trigger dependencies need the aggregate census. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| to_zinvite.integer | SET ASIDE | No current literal routine call established; votes_lastest_unique appears only as a bootstrap comment. All SQL snapshot bodies are available historically; current production body fingerprints are compared below. Dynamic/external SQL and trigger dependencies need the aggregate census. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| votes_foo.integer | SET ASIDE | No current literal routine call established; votes_lastest_unique appears only as a bootstrap comment. All SQL snapshot bodies are available historically; current production body fingerprints are compared below. Dynamic/external SQL and trigger dependencies need the aggregate census. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| votes_lastest_unique.integer | SET ASIDE | No current literal routine call established; votes_lastest_unique appears only as a bootstrap comment. All SQL snapshot bodies are available historically; current production body fingerprints are compared below. Dynamic/external SQL and trigger dependencies need the aggregate census. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| foobar | SET ASIDE | The four literal foobar hits are unrelated Clojure visualization/Rust child fixtures; no sequence user established. Historical dump contains it. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| animal_grp | SET ASIDE | No current literal consumer in tracked trees; historical animals_id_auto body references NEW.grp/enum_range. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| auth_tokens.auth_tokens_token_idx | SET ASIDE | Index planner use is implicit through auth_tokens queries; table-name inventory lists indirect consumers. No application can be declared independent solely because it does not name the index. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| participants.participants_uid_index | SET ASIDE | Index planner use is implicit through participants queries; table-name inventory lists indirect consumers. No application can be declared independent solely because it does not name the index. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| pwreset_tokens.pwreset_tokens_token_idx | SET ASIDE | Index planner use is implicit through pwreset_tokens queries; table-name inventory lists indirect consumers. No application can be declared independent solely because it does not name the index. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| votes.votes_zid_idx | SET ASIDE | Index planner use is implicit through votes queries; table-name inventory lists indirect consumers. No application can be declared independent solely because it does not name the index. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| postgis | SET ASIDE | server/Dockerfile-pdb:1 retains postgis image; bin/remove_postgis.sh is a destructive cleanup helper, NOT permission to run it. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| tablefunc | SET ASIDE | No literal current tree consumer; historical dump declares CREATE EXTENSION tablefunc. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| pg_stat_statements | SET ASIDE | server/src/ops/database.ts:4 explicitly says pg_stat_statements is NOT used (ruling R5: extension creation is a database change); the two unit-test hits do not establish a runtime consumer. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| unattributed_extension_members | SET ASIDE | Exact extension ownership remains unproven: 794 routine and 109 parent-object count mappings ready. Retain all candidates. No selected migration alters these objects; classification and any cleanup stay separate. | +| constraint_index_identity | SET ASIDE | Catalog generator and runner use semantic definitions. Exact alias-name consumers require a separate identifier search before any optional rename. Retain observed state; no dependency on M19/23/24 established. Ancillary cleanup does not gate this release. | +| social_settings.social_settings_uid_key | SET ASIDE | No current live uniqueness consumer established; generated declarations and historical usage alone do not gate queue migrations. Preserve existing unique constraint. | +| suzinvites.suzinvites_uid_fkey | SYNC BEFORE STABLE | Record existing live uniqueness/FK in file contract; no constraint creation or deletion. | +| conversations.conversations_zid_index | SET ASIDE | Redundant index variant does not block selected migrations. Retain current indexes, document optional shape; no rebuild. | +| users.users_uid_idx | SET ASIDE | Redundant index variant does not block selected migrations. Retain current indexes, document optional shape; no rebuild. | diff --git a/docs/migration-reconciliation.md b/docs/migration-reconciliation.md new file mode 100644 index 0000000000..9f99721148 --- /dev/null +++ b/docs/migration-reconciliation.md @@ -0,0 +1,70 @@ +# Reconcile an existing deployment + +Keep one record per database with the release being installed. This is a review +record for the normal `reconcile` → `apply` → `check` path; it is not a list of +host-specific skipped migrations. Never copy credentials, participant rows or +private identifiers into the public repository. + +1. Record the release commit and exact migration source hashes, database major + version, current application version, backup/restore evidence and installer + privileges. Confirm the previous application's stop hook: the first move to + deferred-stop hooks can still execute the old hook before migration refusal. +2. Collect a read-only catalog inventory through the deployment's existing DB + client. Preserve types, defaults, nullability, keys, index validity, routines, + triggers, grants and sequence configuration. Encode bigint values as strings + when transporting them through JavaScript JSON. Do not read sequence values + or vote/application rows as a shortcut to schema reconciliation. +3. Classify every logical migration: present, pending, partial, conflicting, + superseded by a known later contract, or outside the forward chain. Cite its + observed postconditions. Names, owners and physical column positions can + differ without changing the structure; compare key column names and actual + definitions. Do not count internal FK trigger OIDs as missing migrations. +4. Retain historical receipts separately. Filename aliases, duplicate legacy + timestamps and a current matching schema are different kinds of evidence. + Unknown legacy filenames, competing ledgers, nonempty undeclared vote data + and unknown routine implementations need reviewed resolution. Do not invent + execution dates or treat a latest filename as a complete prefix. +5. Prepare and test each needed forward repair on generated databases representing + the actual structural variant. Preserve existing rows, unrelated schemas, + grants and third-party dependants. Do not truncate or drop data to satisfy a + catalog predicate; do not change stored vote signs or overwrite a custom + routine merely to match a fresh installation. Older retirement SQL is not + authorization to delete retained deployment data. +6. Once all selected adoption postconditions are supported and reviewed, run + `polis-migrate reconcile --through NNNNNN` with the factual bound. A failure + must leave all prior history untouched. Then `apply` the ordered pending + migrations and run `check`. No history rows may be inserted manually. +7. Record the actual per-file outcomes and postconditions. M22 is resumable: + indexes can commit before their history row; inspect validity after a lost + connection and reuse valid results. Every other committed earlier file stays + applied if a later file fails. Do not run down scripts as automatic recovery. +8. Verify application health after service replacement. Detached Compose startup + alone is not a serving-health receipt. Keep migration readiness, process + readiness and application health as separate observations. + +Record fields: + +| Field | Evidence required | +| --- | --- | +| Release | Commit, SQL source hashes, release holds and pending PR composition | +| Observation | Catalog query version/hash, collection date, scope and transport | +| Per file | Logical identity, known aliases, state, exact postconditions and differences | +| History | Actual receipts and their provenance, or explicitly unknown | +| Preservation | Rows, custom objects, grants, active-writer compatibility and backup | +| Repair | Exact reviewed patch, generated regression evidence, failure recovery | +| Adoption | Explicit bound, atomic success or unchanged-history failure | +| Pending work | Ordered apply outcomes, concurrent-index progress where applicable | +| Completion | Runner check plus actual application health evidence | + +Fresh databases use `apply`, not reconciliation. Existing /1, /2 or /3 queues +must pass their recorded catalog contracts; install-table presence is not proof. +A partial legacy install needs compatible forward completion, not a fabricated +adoption. M21's release hold remains until its privilege, sequence and publisher +requirements are resolved. Convention declaration and any semantic sign operation +remain explicit operations; neither a catalog nor a migration merge supplies the +operator's declaration. + +This record does not by itself remediate unsupported historical variants, +retire destructive files, compose competing unreleased migration ledgers, or +make manual down scripts history-aware. Those release changes and their tests +must be complete before claiming an upgrade works across supported deployments. diff --git a/docs/migration-release-map.md b/docs/migration-release-map.md new file mode 100644 index 0000000000..28b207a658 --- /dev/null +++ b/docs/migration-release-map.md @@ -0,0 +1,77 @@ +# Migration source to release map + +These 46 source variants map to releases by exact SQL bytes. Source membership +does not prove execution on any database. The only actual source version label +established by the historical research is `1.0`; subsequent semantic versions +were not consistently assigned. Strict/OIDC numbers below are the previously +reviewed retrospective proposals, **not published tags or a new version ruling**. +The current M19/M23/M24/M27 bytes first ship at promotion PR #2994 (`5ded2e0a9`); +their semantic version is **UNASSIGNED**. A release owner must assign the real +release version before publishing one; this patch invents none. + +For an installation's execution record, use its receipts or record execution +as unknown. Never infer an apply date from this source table. The historical +version research is tracked separately from schema adoption. + +| Audited source variant | Role | Filename first present | Exact audited bytes first present | Disposition | +| --- | --- | --- | --- | --- | +| `000000_initial.sql` (#2997 reference; `2652134140cd`) | numbered_forward | declared 1.0 source (anchor) | `53817ae39`; strict 3.24.2 / OIDC 2.24.2 | SHIPPED_SOURCE_BYTES | +| `000001_update_pwreset_table.sql` (#2997 reference; `cb21278194c4`) | numbered_forward | declared 1.0 source (anchor) | declared 1.0 source (anchor) | SHIPPED_SOURCE_BYTES | +| `000002_add_xid_constraint.sql` (#2997 reference; `a27a8e63c79c`) | numbered_forward | declared 1.0 source (anchor) | declared 1.0 source (anchor) | SHIPPED_SOURCE_BYTES | +| `000003_add_origin_permanent_cookie_columns.sql` (#2997 reference; `0d7f27facfec`) | numbered_forward | declared 1.0 source (anchor) | declared 1.0 source (anchor) | SHIPPED_SOURCE_BYTES | +| `000004_drop_waitinglist_table.sql` (#2997 reference; `f2fc4184a965`) | numbered_forward | declared 1.0 source (anchor) | declared 1.0 source (anchor) | SHIPPED_SOURCE_BYTES | +| `000005_drop_slack_stripe_canvas.sql` (#2997 reference; `392e5b8aadb7`) | numbered_forward | declared 1.0 source (anchor) | declared 1.0 source (anchor) | SHIPPED_SOURCE_BYTES | +| `000006_update_votes_rule.sql` (#2997 reference; `8fb05b7b1b6a`) | numbered_forward | declared 1.0 source (anchor) | declared 1.0 source (anchor) | SHIPPED_SOURCE_BYTES | +| `000007_drop_geolocation_fields.sql` (#2997 reference; `f68b69b86112`) | numbered_forward | `8040579ef`; strict 1.1.0 / OIDC 1.1.0 | `8040579ef`; strict 1.1.0 / OIDC 1.1.0 | SHIPPED_SOURCE_BYTES | +| `000008_add_comment_priority.sql` (#2997 reference; `c867b53be3cc`) | numbered_forward | `8040579ef`; strict 1.1.0 / OIDC 1.1.0 | `8040579ef`; strict 1.1.0 / OIDC 1.1.0 | SHIPPED_SOURCE_BYTES | +| `000009_add_uuid_to_zinvites.sql` (#2997 reference; `43f36fe0b857`) | numbered_forward | `4ba8b0938`; strict 1.6.0 / OIDC 1.6.0 | `4ba8b0938`; strict 1.6.0 / OIDC 1.6.0 | SHIPPED_SOURCE_BYTES | +| `000010_create_oidc_user_mappings.sql` (#2997 reference; `450a3f69883a`) | numbered_forward | `3a7da1468`; strict 3.0.0 / OIDC 2.0.0 | `3a7da1468`; strict 3.0.0 / OIDC 2.0.0 | SHIPPED_SOURCE_BYTES | +| `000011_alter_suzinvites_xid_to_text.sql` (#2997 reference; `00a1eb2d9604`) | numbered_forward | `3a7da1468`; strict 3.0.0 / OIDC 2.0.0 | `3a7da1468`; strict 3.0.0 / OIDC 2.0.0 | SHIPPED_SOURCE_BYTES | +| `000012_create_topic_agenda_selections.sql` (#2997 reference; `cc513693124f`) | numbered_forward | `199569498`; strict 3.4.0 / OIDC 2.4.0 | `199569498`; strict 3.4.0 / OIDC 2.4.0 | SHIPPED_SOURCE_BYTES | +| `000013_create_treevite.sql` (#2997 reference; `4b8334f73246`) | numbered_forward | `b986fca0f`; strict 3.6.0 / OIDC 2.6.0 | `b986fca0f`; strict 3.6.0 / OIDC 2.6.0 | SHIPPED_SOURCE_BYTES | +| `000014_alter_reports_modlevel.sql` (#2997 reference; `c2af6d57af28`) | numbered_forward | `ee4405a44`; strict 3.7.2 / OIDC 2.7.2 | `ee4405a44`; strict 3.7.2 / OIDC 2.7.2 | SHIPPED_SOURCE_BYTES | +| `000015_add_xid_requirements.sql` (#2997 reference; `186c904addd0`) | numbered_forward | `a6d7215f7`; strict 3.19.0 / OIDC 2.19.0 | `06d6fa1af`; strict 3.23.0 / OIDC 2.23.0 | SHIPPED_SOURCE_BYTES | +| `000016_add_orig_id.sql` (#2997 reference; `6cdc0588c000`) | numbered_forward | `b13b316e9`; strict 3.21.0 / OIDC 2.21.0 | `b13b316e9`; strict 3.21.0 / OIDC 2.21.0 | SHIPPED_SOURCE_BYTES | +| `000017_create_byod_job_table.sql` (#2997 reference; `f27c03a1229f`) | numbered_forward | `b13b316e9`; strict 3.21.0 / OIDC 2.21.0 | `b13b316e9`; strict 3.21.0 / OIDC 2.21.0 | SHIPPED_SOURCE_BYTES | +| `000018_add_topics_enabled.sql` (#2997 reference; `a1e1c0572064`) | numbered_forward | `06d6fa1af`; strict 3.23.0 / OIDC 2.23.0 | `06d6fa1af`; strict 3.23.0 / OIDC 2.23.0 | SHIPPED_SOURCE_BYTES | +| `000019_create_polis_queue.sql` (#2997 reference; `fedfbcf9fc59`) | numbered_forward | `335418338`; strict 3.28.0 / OIDC 2.28.0 | `5ded2e0a9`; strict UNASSIGNED / OIDC UNASSIGNED | SHIPPED_SOURCE_BYTES | +| `000020_create_math_source_journal.sql` (PR#2739; `24c1dff07637`) | numbered_forward | NOT SHIPPED at checked endpoints | NOT SHIPPED at checked endpoints | PENDING_VARIANT_NOT_SHIPPED | +| `000021_create_polis_coordinator.sql` (#2997 reference; `d50f169ad7af`) | numbered_forward | `335418338`; strict 3.28.0 / OIDC 2.28.0 | `335418338`; strict 3.28.0 / OIDC 2.28.0 | SHIPPED_SOURCE_BYTES | +| `000022_add_poll_timestamp_indexes.sql` (#2997 reference; `14efc95b1478`) | numbered_forward | `2547b6ee5`; strict 3.30.0 / OIDC 2.30.0 | `2547b6ee5`; strict 3.30.0 / OIDC 2.30.0 | SHIPPED_SOURCE_BYTES | +| `000023_create_delphi_foundation.sql` (#2997 reference; `97437ea57d90`) | numbered_forward | `5ded2e0a9`; strict UNASSIGNED / OIDC UNASSIGNED | `5ded2e0a9`; strict UNASSIGNED / OIDC UNASSIGNED | SHIPPED_SOURCE_BYTES | +| `000024_create_polis_queue_large_class.sql` (#2997 reference; `68261afb81f2`) | numbered_forward | `5ded2e0a9`; strict UNASSIGNED / OIDC UNASSIGNED | `5ded2e0a9`; strict UNASSIGNED / OIDC UNASSIGNED | SHIPPED_SOURCE_BYTES | +| `000025_vote_convention.sql` (PR#2944; `cfff57e4f416`) | numbered_forward | NOT SHIPPED at checked endpoints | NOT SHIPPED at checked endpoints | PENDING_VARIANT_NOT_SHIPPED | +| `000026_create_polis_queue_retention.sql` (PR#2978; `ee29e37e94f5`) | numbered_forward | NOT SHIPPED at checked endpoints | NOT SHIPPED at checked endpoints | PENDING_VARIANT_NOT_SHIPPED | +| `000019_create_delphi_storage.sql` (PR#2600; `5808d85fe674`) | numbered_forward | NOT SHIPPED at checked endpoints | NOT SHIPPED at checked endpoints | PENDING_VARIANT_NOT_SHIPPED | +| `000012_create_topic_agenda_selections.sql` (PR#2110; `cc513693124f`) | numbered_forward | `199569498`; strict 3.4.0 / OIDC 2.4.0 | `199569498`; strict 3.4.0 / OIDC 2.4.0 | PENDING_VARIANT_BYTES_ALREADY_SHIPPED | +| `000000_initial.sql` (PR#2556; `183d22c0238a`) | numbered_forward | declared 1.0 source (anchor) | NOT SHIPPED at checked endpoints | PENDING_VARIANT_NOT_SHIPPED | +| `000026_create_polis_queue_retention.sql` (PR#2983; `0d1e357a72a9`) | numbered_forward | NOT SHIPPED at checked endpoints | NOT SHIPPED at checked endpoints | PENDING_VARIANT_NOT_SHIPPED | +| `000024_vote_sign_unflip.sql` (PR#2942; `3e67e0ab857d`) | held | NOT SHIPPED at checked endpoints | NOT SHIPPED at checked endpoints | PENDING_VARIANT_NOT_SHIPPED | +| `000023_vote_convention.sql` (PR#2942; `351628cfc1bc`) | fixture | NOT SHIPPED at checked endpoints | NOT SHIPPED at checked endpoints | PENDING_VARIANT_NOT_SHIPPED | +| `000025_drop_vote_convention.sql` (PR#2944; `9d747f6d0fe3`) | down | NOT SHIPPED at checked endpoints | NOT SHIPPED at checked endpoints | PENDING_VARIANT_NOT_SHIPPED | +| `000026_drop_polis_queue_retention.sql` (PR#2978; `747085bf1324`) | down | NOT SHIPPED at checked endpoints | NOT SHIPPED at checked endpoints | PENDING_VARIANT_NOT_SHIPPED | +| `000026_drop_polis_queue_retention.sql` (PR#2983; `063272749473`) | down | NOT SHIPPED at checked endpoints | NOT SHIPPED at checked endpoints | PENDING_VARIANT_NOT_SHIPPED | +| `db_000002.sql` (#2997 reference; `1f214c31e557`) | archive | declared 1.0 source (anchor) | declared 1.0 source (anchor) | SHIPPED_SOURCE_BYTES | +| `db_000004.sql` (#2997 reference; `3afacfa4c4e6`) | archive | declared 1.0 source (anchor) | declared 1.0 source (anchor) | SHIPPED_SOURCE_BYTES | +| `db_000006.sql` (#2997 reference; `4c6f61fa0a5b`) | archive | declared 1.0 source (anchor) | declared 1.0 source (anchor) | SHIPPED_SOURCE_BYTES | +| `db_000008.sql` (#2997 reference; `e5d4990d1dd7`) | archive | declared 1.0 source (anchor) | declared 1.0 source (anchor) | SHIPPED_SOURCE_BYTES | +| `db_000010.sql` (#2997 reference; `498344c75e9c`) | archive | declared 1.0 source (anchor) | declared 1.0 source (anchor) | SHIPPED_SOURCE_BYTES | +| `000019_drop_polis_queue.sql` (#2997 reference; `483de532876f`) | down | `335418338`; strict 3.28.0 / OIDC 2.28.0 | `335418338`; strict 3.28.0 / OIDC 2.28.0 | SHIPPED_SOURCE_BYTES | +| `000021_drop_polis_coordinator.sql` (#2997 reference; `f8547afa1e87`) | down | `335418338`; strict 3.28.0 / OIDC 2.28.0 | `335418338`; strict 3.28.0 / OIDC 2.28.0 | SHIPPED_SOURCE_BYTES | +| `000022_drop_poll_timestamp_indexes.sql` (#2997 reference; `fcf1f4695e48`) | down | `2547b6ee5`; strict 3.30.0 / OIDC 2.30.0 | `2547b6ee5`; strict 3.30.0 / OIDC 2.30.0 | SHIPPED_SOURCE_BYTES | +| `000023_drop_delphi_foundation.sql` (#2997 reference; `aa0a3d67766a`) | down | `5ded2e0a9`; strict UNASSIGNED / OIDC UNASSIGNED | `5ded2e0a9`; strict UNASSIGNED / OIDC UNASSIGNED | SHIPPED_SOURCE_BYTES | +| `000024_drop_polis_queue_large_class.sql` (#2997 reference; `08735f922a17`) | down | `5ded2e0a9`; strict UNASSIGNED / OIDC UNASSIGNED | `5ded2e0a9`; strict UNASSIGNED / OIDC UNASSIGNED | SHIPPED_SOURCE_BYTES | + + +The M20/M25/M26 PR-only variants, unflip fixture, archives and down scripts +are not in this release manifest. M21 is held. M4/M5/M7 are retained historical +sources and can only receive observed ADOPTED receipts in ordinary apply. + +## Release B addition + +M27 (`000027_create_sealed_job_graphs.sql`, SHA-256 +`fbbf948e4316010dd96344ae91542006c38299e52e0a0eb281371c039ab37775`) +is selected after M19/M23/M24. It adds the approved per-step graph core. +Its strict release version remains UNASSIGNED until the release decision; +this source selection is not evidence of a production application. +M28/M29 and the DynamoDB readers/importer are not part of this release. diff --git a/docs/migration-upgrade-notes.md b/docs/migration-upgrade-notes.md new file mode 100644 index 0000000000..37be4635b5 --- /dev/null +++ b/docs/migration-upgrade-notes.md @@ -0,0 +1,100 @@ +# Migration runner upgrade notes + +The first selected legacy upgrade records the supported existing schema, then +applies only M19, M23, M24 and M27. They create the job-system tables and routines. +They do not enable any worker or application feature flag. Production is never +changed merely to match old bootstrap files. + +## Choose the installation path + +| Starting state | Required path | Expected outcome | +| --- | --- | --- | +| Empty PostgreSQL 17+ | `apply`, then `check` | 21 executed files, three retirement ADOPTED receipts, 24 ready | +| Supported legacy schema through M18 plus valid M22, no queue/history | first-deploy report, `reconcile --through 000022`, `apply`, `check` | 20 ADOPTED; exactly M19/M23/M24/M27 APPLIED | +| Supported legacy schema through M18, missing M22 | `reconcile --through 000018`, `apply`, `check` | 19 ADOPTED; M19, concurrent M22, M23/M24/M27 APPLIED | +| Existing queue /1, /2 or /3 | exact installed catalog/receipts review, reconcile through actual installed version, then apply/check | compatible installed queue retained; only pending selected files execute | +| Modern valid runner history | `apply`, `check` | existing APPLIED/ADOPTED receipts retained; rerun executes zero | +| Legacy ledger with known filenames | bounded reconcile after catalog review | timestamps preserved, no fictional execution receipts | +| Retained M4/M5/M7 targets | stop for separately reviewed completion/retention plan | no automatic destructive migration or fake adoption | +| Partial/conflicting schema, unknown ledger/file, later queue variant | stop for explicit reviewed reconciliation | no blind replay, manual history insertion or source rewriting | + +The historical bootstrap and named legacy schema are distinct supported variants. +See [their exact contract](migration-legacy-contract.md). A successful limited +adoption contract is not an exhaustive certification of every custom object, +extension, historical routine implementation, data invariant or application path. + +## Deprecated removal files + +M4 historically removed `waitinglist`. M5 removed Slack OAuth/users/invites/bot +events, Stripe accounts/subscriptions, free-upgrade coupons, LTI users/context +memberships/OAuth credentials, Canvas callback/conversation tables, and the +`conversations.is_slack`, `conversations.lti_users_only`, `users.plan` columns. +M7 removed `geolocation_cache` and participants_extended's `country_code_iso`, +`encrypted_maxmind_response_city`, `ip_address`, `latitude`, `location`, +`longitude` and `x_forwarded_for` fields. + +All three files remain immutable historical evidence. Ordinary upgrades never +execute them. Absence of the exact named objects allows an ADOPTED receipt; +retained targets block without deletion. Other Slack tables, the differently +named `participants_extended.country_iso_code`, encrypted network fields, and +`facebook_users.location`/`twitter_users.location` are not removal targets. +A composite type's `location` attribute is not a participants_extended column. + +## Report-only first-deploy check + +`server/postgres/migrations/report/first-deploy.sql` is the first-deploy report. +Its SQL predicates are generated directly from the reviewed adoption sources, +with helper expressions inlined. It uses one bounded REPEATABLE READ READ ONLY +transaction followed by ROLLBACK, creates no temporary functions/tables, reads +no application rows, and returns only migration names, booleans and outcomes. +`python3 server/bin/build-migration-report.py --check` verifies its exact source +binding; regeneration is a reviewable file change. + +The operator sends the entire SQL file through **one connection** of the existing +server's database client, with existing connection/TLS settings. Do not deploy +the new runner or restart the app just to report. For a local or controlled +operator session, `psql -X -v ON_ERROR_STOP=1 -f ` is equivalent, using +the normal secret connection environment without a credential argument. On any +query error, ROLLBACK or close that same client before returning it to a pool. + +Expected initial-schema results: 20 `WOULD_ADOPT`, four `WOULD_APPLY`, and four +`OUTSIDE_RELEASE` entries. The report conservatively flags any existing queue +role for migration-session review rather than guessing inherited ADMIN/SET +authority. Existing ledgers/queues report review-required; they need the exact +normal reconciliation catalog verifier and a separate read-only examination. +A mismatch in any required predicate blocks the aggregate adoption forecast. +A runtime read-only role can report schema but cannot predict privileged DDL +success; check the intended migration session separately. + +The report proves no future lock acquisition, available capacity, successful +DDL or application health. Its snapshot expires; reconcile and apply recheck +their actual contracts when executed. M22 presence in the initial report is +required; missing M22 uses the explicit shorter-bound path above. + +## Privileges, interruption and deployment + +Use the intended migration session, including M19 role provisioning and grant +authority. Owning only the database or having normal app SELECT/INSERT access +is insufficient. M23/M24/M27 SET ROLE to the queue owner. Never grant broad rights +to a runtime login just to satisfy startup; startup needs metadata SELECT. + +Each ordinary file and its history receipt commit together. If M23 fails after +M19 commits, M19 remains applied while all M23 changes and its receipt roll back. +Do not describe that as a rollback of the whole release. M22 is the documented +concurrent-index exception; interrupted valid indexes may persist and be reused. +Check invalid/conflicting index recovery in [migrations.md](migrations.md). +Two runners serialize using the database advisory lock; they do not double-apply. +Lost commit connections require history inspection, not an assumed rollback. + +On the first deployment, an old successful revision's stop hook may have already +stopped the old application before the new migration hook runs. New deferred-stop +hooks protect old containers only after that transition. A later API startup +failure may occur after old containers have been removed. This patch supplies +no automatic rollback, previous-image fallback or CodeDeploy health guarantee. +Verify actual HTTP/application health after replacement and preserve a reviewed +recovery procedure. A detached Compose launch is insufficient evidence. + +Keep a per-deployment [reconciliation record](migration-reconciliation.md), +including exact source hashes, schema report, actual session authority, observed +receipts, supported variants, backup evidence and health result. Do not publish +credentials, application records or real conversation/report identifiers. diff --git a/docs/migrations.md b/docs/migrations.md index 307609c77d..4cc8f7c590 100644 --- a/docs/migrations.md +++ b/docs/migrations.md @@ -1,40 +1,201 @@ +# Database migrations -# Database Migrations +Deploy the schema before replacing application services. Every deployment uses +`polis-migrate deploy`; the API refuses startup if a required file is pending, +history is missing, or a recorded checksum disagrees with the release. +PostgreSQL 17 or newer is required. Keep a tested backup/restore procedure. -When we need to update the Polis database, we use SQL migration files. +Build once from the repository root (or use the release's migration image): -During initial provisioning of your Docker containers, all the migrations will be applied in order, and you won't need to think about this. -But if we update the database schema after your initial provisioning of your server via Docker, you'll need to manually apply each new SQL migration. +```sh +cargo build --locked --release --manifest-path queue-rs/Cargo.toml -p polis-migrate +export PATH="$PWD/queue-rs/target/release:$PATH" +# Set DATABASE_URL through your usual secret environment; never put it in argv. +polis-migrate apply +polis-migrate check +``` + +The image requires no local Rust installation: + +```sh +docker build -t polis-migrate:local -f queue-rs/polis-migrate/Dockerfile . +docker run --rm --network host --env-file .env polis-migrate:local apply +``` + +Use the database's actual network when it is inside Compose instead of the host +network. `POLIS_MIGRATIONS_DIR` or `--dir` selects the release's SQL directory. +TLS validates the certificate and hostname. The runner and API images include +a checksum-pinned public RDS CA bundle (official global truststore, retrieved +2026-10-09; SHA-256 `fe45bbebf92ad3e27a583bbb2ddd1553c521ed4d49af5514dc0a40372ea5395c`). `POLIS_MIGRATE_CA_FILE` supplies a +private PEM CA bundle (mount it into the container). Standalone Node startup +uses `NODE_EXTRA_CA_CERTS` for a private/RDS CA; standalone Rust uses +`POLIS_MIGRATE_CA_FILE`. For a disposable local +network only, a URL with `sslmode=disable` plus +`POLIS_MIGRATE_ALLOW_PLAINTEXT=local` permits Docker service DNS; loopback and +local sockets can use `sslmode=disable` without that variable. Never use this +local setting for a remote production database. -- Please note: **Backups are your responsibility.** These instructions assume - the data is disposable, and do not attempt to make backups. - - Pull requests are welcome if you'd like to see more guidance on this. - - Please submit an issue if you'd like to work on enabling backups through Docker Compose. -- Your database data is stored on a docker volume, which means that it will - persist even when you destroy all your docker containers. Be mindful of this. - - You can remove ALL volumes defined within a `docker-compose` file via: `docker compose --profile postgres down --volumes` - - You can remove ONE volume via `docker volume ls` and `docker volume rm ` -- SQL migrations can be found in [`server/postgres/migrations/`][] of this - repo. -- The path to the SQL file will be relative to its location in the docker - container filesystem, not your host system. +New Docker volumes run the same binary from `server/Dockerfile-db`. Existing +volumes do not rerun initialization: invoke `apply` before starting the new app. +The CodeDeploy hook invokes `deploy` (catalog-checked first adoption, apply, check) +before replacing containers. See the mandatory two-release transition in +[the upgrade guide](upgrading.md). Direct `apply` still refuses unadopted databases. +`server/bin/run-migrations.sh` delegates to the binary; the old Clojure entry +point delegates to that shell command. Neither replays every file anymore. -For example, if we add the migration file -`server/postgres/migrations/000001_update_pwreset_table.sql`, you'd run on your -host system: +## One-time adoption of an existing database + +An existing database without history is deliberately refused by `apply`. +Never replay `000000_initial.sql` on it. Use the known migration records and +inspect its schema to choose an explicit upper bound, then reconcile: ```sh -docker compose --profile postgres exec postgres psql --username postgres --dbname polis-dev --file=/docker-entrypoint-initdb.d/000001_update_pwreset_table.sql +# Example: the legacy schema through 000022, without the dormant queue. +polis-migrate reconcile --through 000022 +polis-migrate apply +polis-migrate check ``` -You can also run a local .sql file on a postgres container instance with this syntax: +The bound is not evidence of application. Every selected file must pass its +catalog postconditions in `server/postgres/migrations/adoption/`: enduring +initial tables/columns, renamed columns, unique and foreign-key constraints, +new columns/types/defaults, removed objects, the vote-update rule, and valid +index definitions. The exact [legacy variants](migration-legacy-contract.md) include six named +math `json` payloads and bounded type/column alternatives; unrelated types are refused. The missing queue migration 000019 is a recognized +hole **only when no queue/foundation tables or functions exist and no old +history row claims it**. The example adopts 000000–000018 and 000022 as ADOPTED; +`apply` then installs only 000019, 000023, 000024 and 000027. No vote data is inspected +or changed by reconciliation. + +Adoption is atomic: any failed postcondition leaves the old history untouched. +It records observation time and the checksum of the source being reconciled, +not a fictional historical execution date. If the 2021 `migrations(name, +completed_at)` table exists, its timestamps (including duplicates) are retained +in each row's `legacy_completed_at` array after all catalog checks pass. Unknown +or renamed legacy filenames stop for review; they are never silently replayed. +Already installed queues use their existing /1, /2 or /3 catalog verifiers, +including definitions/privileges and recorded installed catalogs. A matching +install row alone is insufficient. Select the actual upper version (for example +`--through 000024` for a previously initialized /3 Docker database). A partial +queue, later bound without a contract, unknown historical filename, or the +unreleased `schema_migrations` ledger stops for review. Never mark those by hand. + +## History and failure behavior + +`public.migrations` is the sole applied-history table. It records filename, +SHA-256 of the unmodified SQL source, APPLIED or ADOPTED, observation time/actor, +and any preserved legacy timestamps. Runtime roles need SELECT only, granted +on this metadata table; the deployment role needs the privileges required by +the actual migrations (including role administration for the queue). + +A database advisory lock serializes runners on the same connection across +per-file commits. A second runner waits up to five minutes, then reads the +committed history. Each ordinary file and its APPLIED row commit in one transaction. The two +000022 index builds are the explicit autocommit exception described below. +An optional historical outer BEGIN/COMMIT pair is removed; any other top-level +transaction control is rejected. Dollar-quoted function bodies stay intact. +SQL errors abort the file. Earlier successfully committed files stay applied. +Lock waits are bounded (five seconds for DDL), with five-minute statement and +transaction ceilings and a thirty-second idle-transaction limit. Existing SQL +may set tighter timeouts. No automatic destructive down migration runs. + +If the connection is lost at commit, the result can be unknown. Reconnect and +run `check`/`apply`: committed history is authoritative. Do not infer rollback +from a transport error. Restore changed historical files instead of editing +history to bypass checksum failures. + +For the exact released 000022 source, `apply` preflights both index identities, +then builds each missing watermark index with `CREATE INDEX CONCURRENTLY` in +autocommit, while retaining the same migration advisory lock. This is the normal +path on fresh and large populated databases. Valid existing indexes are checked +and preserved without rebuilding. The original SQL then rechecks both exact +index definitions in the transaction that records APPLIED. Its raw SQL checksum +is unchanged; changing that source requires review of this execution contract. + +An interruption can leave a valid first index, or an invalid index from a failed +concurrent build, without a migration history row. Rerunning `apply` reuses valid +indexes and builds only missing ones. It refuses invalid or conflicting objects +before building either index. Inspect an invalid index first; if it is the exact +interrupted watermark index, explicitly drop that index with `DROP INDEX +CONCURRENTLY public.` in autocommit, then rerun `apply`. The +runner never drops a preexisting index automatically. A differently defined +same-name object requires a reviewed resolution, not that drop instruction. +If history recording fails after both builds, both valid indexes remain; retry +records their checked state without rebuilding. Concurrent builds have the +same bounded statement/lock timeouts; a timeout is not a success receipt. + +## Release contents and new migrations + +`server/postgres/migrations/release.txt` explicitly selects every required file. +The Rust runner and Node startup gate use the same manifest; Docker initialization +uses that runner. A new top-level numbered SQL file must be selected or held, +otherwise startup/apply refuses it. Missing files, duplicate numbers or entries, +symlinks, and release/hold overlap refuse. Adding a file cannot silently extend +the release. Numbered source checksums remain unchanged. + +This release contains M0–M19 plus M22/M23/M24/M27. M4/M5/M7 are deprecated +observation-only entries. They never execute through `apply`: their named +removed objects must already be absent, then an ADOPTED receipt is recorded. +Any retained target stops before apply mutates schema/history; arrange a separate +reviewed upgrade for that deployment. Existing valid APPLIED receipts are kept. +Fresh bootstrap already omits their targets and gets three ADOPTED receipts; +it executes 21 files. No historical execution date is invented. + +M20 (draft journal), M21 (held coordinator), M25 (vote convention) and M26 +(retention) are excluded from the forward path. M21 stays in `held.txt`; the +other files are not shipped by this branch. An added unclassified copy refuses. +The selected legacy upgrade adopts M0–M18/M22 and applies **M19 → M23 → M24 → M27**. +A deployment missing M22 instead builds its indexes concurrently between M19 +and M23. Archives, down scripts and unflip files never enter this manifest. + +Read [the release map](migration-release-map.md) for historical source/release +provenance and [upgrade notes](migration-upgrade-notes.md) for each supported +deployment shape. Semantic versions that were never assigned remain explicitly +unassigned; source shipment does not establish database execution. + +Merging a required migration means it runs at the next deployment. Review its +compatibility with the still-running previous application, locking, privileges, +data effects and reversal/restore plan at merge time. Update these public +upgrade records in that change. The manual sitting and client stage/unstage +steps are retired; no separate staging ceremony is required. + +## CI and local test stacks + +The migration-runner CI fresh-image step and the mm2/mm5 proof use exactly: ```sh -docker exec -i polis-dev-postgres-1 psql -U postgres -d polis-dev < server/postgres/migrations/000006_update_votes_rule.sql +COMPOSE_PROJECT_NAME=polis-migrate-test-my-owned-run \ +POLIS_RECOVERY_PG_PORT=55851 \ +bash queue-rs/polis-migrate/tests/fresh-image.sh ``` -where `polis-dev-postgres-1` is the name of the running container (see the output of `docker ps`), `postgres` is the db username and `polis-dev` is the database. +Choose a unique project and unused port on each shared machine. This entrypoint +builds `server/Dockerfile-db`, initializes a fresh database, checks exact receipt +names/checksums/statuses against `release.txt` and the M4/M5/M7 retirement policy, +then restarts and checks that the complete history is unchanged. Both the ready +count and expected receipt set derive from the selection. It removes only its +project volumes/containers and project-specific image tag, including on failure. +It does not replay production rows or prove a production deployment. + +After building `docker-compose.test.yml`, run `bash ci/test-migrations.sh` +before starting application services. It waits for Postgres initialization, +then runs `apply` and `check` using the binary and SQL packaged in that image. +It works on fresh and existing test volumes; it never creates history by hand +or bypasses the API startup check. Cypress, server integration and Delphi CI +all use this entrypoint. Server integration also runs the Node startup check +before loading its in-process test app. + +Set a unique `COMPOSE_PROJECT_NAME` and `POLIS_RECOVERY_PG_PORT` for a shared +local machine. `POLIS_TEST_ENV_FILE` chooses a test env file (default `test.env`); +optional Compose arguments such as `-f local-ports.yml` support isolated test +stacks. Use the same options when starting and removing your stack. + -You'd do this for each new file, in numeric order. +## Deployment reconciliation records - [`server/postgres/migrations/`]: /server/postgres/migrations +Use [the reconciliation record](migration-reconciliation.md) for each deployment. +A catalog match is an observation, not proof that a historical file ran. A failed +predicate must lead to a reviewed forward repair or a documented compatible +variant, never to manually inserting a migration row. Unknown historical variants remain review cases; this document does not authorize +a deployment that fails reconciliation. Run the [read-only first-deploy report](migration-upgrade-notes.md) +before the first transition; it does not replace application-health verification. diff --git a/docs/queue-substrate.md b/docs/queue-substrate.md index 4e3a9ea276..1d8c0c8aa6 100644 --- a/docs/queue-substrate.md +++ b/docs/queue-substrate.md @@ -44,19 +44,11 @@ executor role holds **no** direct read or write on any queue table. ## Applying the migration -A **fresh** container applies it automatically: the postgres image copies -`server/postgres/migrations/*.sql` into `/docker-entrypoint-initdb.d`, so -`make start` on a new volume comes up with the schema present and the flag off. - -An **existing** database needs the file applied by hand, through the checked -wrapper, which feeds it to psql exactly as [docs/migrations.md](migrations.md) -describes. Apply this file alone; never replay the migrations directory as an -upgrade mechanism. - -```sh -server/postgres/bin/apply-migration.sh --free-bytes 000019 -- \ - docker exec -i polis-dev-postgres-1 psql -U postgres -d polis-dev -``` +Both fresh databases and upgrades use [`polis-migrate apply`](migrations.md). +Fresh Docker volumes run it during initialization; deployments run it before +service replacement. Existing pre-runner databases reconcile once first. +Successful files are recorded and skipped on later deploys. Queue activation +remains controlled by the existing flags; applying schema does not enable it. ### What the apply locks, and the window it needs @@ -71,16 +63,16 @@ open transaction that already wrote a `conversations` row; when the timeout fires the transaction aborts and nothing is applied. The changed queue tables and the new objects are `ACCESS EXCLUSIVE` for the same span. So "additive and empty" is not "cannot block users": **apply in an idle or controlled writer -window** (producers paused, no open writer on `conversations`), and apply -000019 and 000023 as two separate steps, each with its own window. +window** (producers paused, no open writer on `conversations`), and account for both +000019 and 000023 in the deployment window; the runner gives each its own transaction. ### The wrapper: preflight and budgets -`server/postgres/bin/apply-migration.sh` is the one way to apply 000019 or -000023 to an existing database. It takes the migration number and, after `--`, -the psql command to run (the SQL goes on its stdin, so `docker exec -i … psql` -and a plain `psql` both work). It refuses, sending nothing, unless every -preflight check passes, and prints each one: +The former `server/postgres/bin/apply-migration.sh` is retained for historical +rehearsal/reversal tests. It is **not** the deployment entry point and does not +update the migration history. Its former first-install checks are recorded below +for reference. The runner now owns transactional application, bounded waits, +checksums and history; use the [migration guide](migrations.md). | check | what it requires | |---|---| @@ -391,18 +383,10 @@ Two of those tables deserve plain words: Schema ruling S1 (2026-10-05) approved it as direction: one datastore and typed contracts, flag off, DynamoDB running every job family until each is moved one -at a time. **Applying it to production is a separate, explicit step by the -owner**, 000019 first (it has never been applied there), then 000023, each -through the wrapper and in its own idle or controlled writer window, because -each holds `ShareRowExclusiveLock` on `conversations` until it commits -([what the apply locks](#what-the-apply-locks-and-the-window-it-needs); the -wrapper's preflight and budgets are -[described above](#the-wrapper-preflight-and-budgets)): - -```sh -server/postgres/bin/apply-migration.sh --free-bytes 000023 -- \ - docker exec -i polis-dev-postgres-1 psql -U postgres -d polis-dev -``` +at a time. **Required migrations now apply during deployment**, in numeric order through +`polis-migrate apply`; review and authorize their schema effects at merge time. +Both 000019 and 000023 still hold `ShareRowExclusiveLock` on conversations, bounded +by the runner's lock and transaction limits. See [migration operations](migrations.md). The applier must be able to `SET ROLE polis_queue_owner`; the file creates no role. It refuses, changing nothing, when 000019 is absent, when any diff --git a/docs/upgrading.md b/docs/upgrading.md index 787e645389..837cdb0ac5 100644 --- a/docs/upgrading.md +++ b/docs/upgrading.md @@ -1,16 +1,45 @@ # Upgrade Guide -## Safe deployment hooks (release A) +## Migration runner transition -This release changes deployment hooks only; it does not introduce the migration -runner or an API migration startup check. Install it successfully on every box -before enabling the runner in release B. CodeDeploy uses the PREVIOUS successful -revision's ApplicationStop, so this first transition can still interrupt service. -New hooks defer replacement until AfterInstall and validate the server's HTTP and -database routes. A failure in ValidateService fails the deployment; it does not -automatically restore containers or undo schema changes. Delphi/worker validation -checks process/container liveness, not real-time math correctness. +Use [the migration runner](migrations.md) before replacing services. Existing +databases need one catalog-checked `reconcile` before their first runner deploy; +new databases initialize through the runner. Missing or changed history prevents +API startup. Migrations 000019/23/24/27 now apply during deployment; the coordinator +000021 remains explicitly held in this release. This installs schema only, not +queue producers or workers. PostgreSQL 17+ is required. +Read the [per-deployment upgrade paths and read-only report](migration-upgrade-notes.md) +before the first transition. One explicit release manifest selects the files; +M20/M21/M25/M26 are outside this forward release. Historical M4/M5/M7 never +execute automatically: absence of their removal targets permits ADOPTED receipts, +while retained targets stop for review without deletion. The +[named legacy contract](migration-legacy-contract.md) preserves supported live +variants, and the [release map](migration-release-map.md) distinguishes shipped +source from actual execution and unassigned semantic versions. + +Install **release A** (safe hooks, no runner or startup check) successfully on +EVERY deployment-group instance before this release B. CodeDeploy executes +ApplicationStop from the previous successful revision. A's first transition may +still interrupt service; B's migration refusal preserves A's running services. +An instance that missed A must not receive B. Rolling back to pre-A hooks removes +this protection; restore A before retrying B. + +AfterInstall builds the runner, invokes `polis-migrate deploy`, and only after +success replaces the role's services. Under one database advisory lock, deploy +reconciles the catalog through 000022 if history is absent/legacy on an existing +database, applies the selected pending migrations, and checks readiness. A fresh +database goes directly to apply; modern history is checked rather than adopted +again. Any mismatch stops. No error is used as a signal to retry reconciliation. +This release's production-shaped transition adopts 20 files and applies only +M19/M23/M24/M27. Existing queue catalogs newer than the adoption bound still require +their separately reviewed explicit reconciliation before this deploy. + +ValidateService checks the server's HTTP and database routes, plus container +liveness; Delphi/worker checks prove container/process liveness only. Validation +failure fails deployment, not an automatic container rollback or schema undo. +Successful earlier migrations may remain after a later failure. Preserve backup +and migration-specific recovery procedures. No automatic down migration runs. ## Configuration Changes (Q1 2023) diff --git a/example.env b/example.env index 48b3ce6508..cf2fb315e3 100644 --- a/example.env +++ b/example.env @@ -348,3 +348,10 @@ ENCRYPTION_PASSWORD_00001= # (Deprecated) Basic Auth settings for certain requests between math and api services. WEBSERVER_PASS=ws-pass WEBSERVER_USERNAME=ws-user + +###### DELPHI RESULT READERS ###### +# Unset means DynamoDB. Only an explicit postgres value selects the new reader. +# PostgreSQL needs M28/M29 and verified published/imported coverage first. +# DELPHI_RESULT_BACKEND=dynamodb +# DELPHI_RESULT_ENV= +# DELPHI_RESULT_SCOPE= diff --git a/queue-rs/Cargo.lock b/queue-rs/Cargo.lock index 8dc073d4f3..606e3f04ff 100644 --- a/queue-rs/Cargo.lock +++ b/queue-rs/Cargo.lock @@ -568,6 +568,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "polis-migrate" +version = "0.1.0" +dependencies = [ + "anyhow", + "native-tls", + "postgres", + "postgres-native-tls", + "sha2 0.10.9", +] + [[package]] name = "polis-queue-adapter" version = "0.1.0" diff --git a/queue-rs/Cargo.toml b/queue-rs/Cargo.toml index bfc1730620..6920223fc5 100644 --- a/queue-rs/Cargo.toml +++ b/queue-rs/Cargo.toml @@ -1,7 +1,7 @@ [workspace] # The root package (the polis-queue/1 adapter and the polis-jobs daemon) and the # polis-api route server share one lockfile, one toolchain and one CI job. -members = [".", "polis-api"] +members = [".", "polis-api", "polis-migrate"] resolver = "3" [package] diff --git a/queue-rs/polis-api/conformance/nginx-routing.sh b/queue-rs/polis-api/conformance/nginx-routing.sh index 48a8219f1f..d3f12eed61 100755 --- a/queue-rs/polis-api/conformance/nginx-routing.sh +++ b/queue-rs/polis-api/conformance/nginx-routing.sh @@ -37,17 +37,48 @@ stub() { # container alias port status body stub "$p-node" server 5000 200 node stub "$p-alpha" client-participation-alpha 4321 200 alpha get() { docker exec "$p-proxy" wget -qO- "http://127.0.0.1$1" 2>/dev/null || echo "(no answer)"; } -# A raw HTTP/1.0 exchange through the proxy; prints the X-Upstream that answered. -upstream_of() { # method path [extra header lines] [body] - printf '%s %s HTTP/1.0\r\nHost: localhost\r\n%s\r\n%s' "$1" "$2" "${3:-}" "${4:-}" \ - | docker exec -i "$p-proxy" nc -w 5 127.0.0.1 80 \ - | tr -d '\r' | sed -n 's/^X-Upstream: //p' +# BusyBox nc can exit on stdin EOF before the HTTP response arrives (nginx +# logs 499). Use a bounded TCP client on an ephemeral loopback-only proxy port. +expect_upstream() { # expected upstream method path [extra header lines] [body] + local want=$1 response got + shift + response=$(python3 - "$p-proxy" "$@" <<'PYTHON' +import socket +import subprocess +import sys + +proxy, method, path, *rest = sys.argv[1:] +headers, body = (rest + ["", ""])[:2] +request = f"{method} {path} HTTP/1.0\r\nHost: localhost\r\n{headers}\r\n{body}".encode() +address = subprocess.check_output(["docker", "port", proxy, "80/tcp"], text=True).strip() +host, port = address.rsplit(":", 1) +assert host == "127.0.0.1", f"test proxy must bind loopback only: {address}" +with socket.create_connection((host, int(port)), timeout=5) as client: + client.sendall(request) + while True: + chunk = client.recv(65536) + if not chunk: + break + sys.stdout.buffer.write(chunk) + sys.stdout.buffer.flush() + +PYTHON + ) || { printf '%s\n' "$response" >&2; fail "raw HTTP transport failed: $1 $2"; } + + got=$(printf '%s\n' "$response" | tr -d '\r' | sed -n 's/^X-Upstream: //p') + if [[ $got != "$want" ]]; then + printf 'Raw response for %s %s (expected X-Upstream: %s):\n%s\n' "$1" "$2" "$want" "$response" >&2 + docker logs "$p-proxy" >&2 2>&1 || true + docker logs "$p-node" >&2 2>&1 || true + fail "$1 $2 answered X-Upstream='$got', expected '$want'" + fi } + start() { # routes [docker run args...] local routes=${1:-} shift || true docker rm -f "$p-proxy" >/dev/null 2>&1 || true - docker run -d --name "$p-proxy" --network "$net" -e RUST_API_ROUTES="$routes" "$@" "$image" >/dev/null + docker run -d --name "$p-proxy" --network "$net" -p 127.0.0.1::80 -e RUST_API_ROUTES="$routes" "$@" "$image" >/dev/null for _ in $(seq 20); do docker exec "$p-proxy" wget -qO- http://127.0.0.1/ >/dev/null 2>&1 && return 0 sleep 0.5 @@ -111,12 +142,12 @@ expect '/api/v3/math/pca2?conversation_id=x' rust "route named twice" [[ $(included | grep -c 'location = /api/v3/math/pca2') == 1 ]] || fail "route named twice: rendered more than once" echo "ok 7 route named twice: rendered once, nginx serves, pca2 -> polis-api" -[[ $(upstream_of GET /api/v3/math/pca2) == polis-api ]] || fail "plain GET should reach polis-api" -[[ $(upstream_of HEAD /api/v3/math/pca2) == polis-api ]] || fail "HEAD should reach polis-api" -[[ $(upstream_of OPTIONS /api/v3/math/pca2) == server ]] || fail "OPTIONS should go to node" -[[ $(upstream_of POST /api/v3/math/pca2 $'Content-Length: 2\r\n' '{}') == server ]] || fail "POST should go to node" -[[ $(upstream_of GET /api/v3/math/pca2 $'Content-Type: application/json\r\nContent-Encoding: gzip\r\nContent-Length: 2\r\n' '{}') == server ]] \ - || fail "GET with a body should go to node" +expect_upstream polis-api GET /api/v3/math/pca2 +expect_upstream polis-api HEAD /api/v3/math/pca2 +expect_upstream server OPTIONS /api/v3/math/pca2 +expect_upstream server POST /api/v3/math/pca2 $'Content-Length: 2\r\n' '{}' +expect_upstream server GET /api/v3/math/pca2 $'Content-Type: application/json\r\nContent-Encoding: gzip\r\nContent-Length: 2\r\n' '{}' + echo "ok 8 only GET/HEAD without a body reach polis-api; OPTIONS, POST, GET with a body -> node" for bad in "RUST_API_UPSTREAM=bad host;" "RUST_API_RESOLVER=bad resolver" "RUST_API_RESOLVER=1.2.3.4; }"; do diff --git a/queue-rs/polis-migrate/Cargo.toml b/queue-rs/polis-migrate/Cargo.toml new file mode 100644 index 0000000000..6403b1bb90 --- /dev/null +++ b/queue-rs/polis-migrate/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "polis-migrate" +version = "0.1.0" +edition = "2024" +rust-version = "1.88" +publish = false + +[dependencies] +anyhow = "1" +native-tls = "=0.2.14" +postgres = "0.19" +postgres-native-tls = "=0.5.3" +sha2 = "0.10" + +[lints.clippy] +unwrap_used = "deny" +expect_used = "deny" diff --git a/queue-rs/polis-migrate/Dockerfile b/queue-rs/polis-migrate/Dockerfile new file mode 100644 index 0000000000..c55dfb59fe --- /dev/null +++ b/queue-rs/polis-migrate/Dockerfile @@ -0,0 +1,20 @@ +# Build from repository root. One binary for deploys and manual/local upgrades. +FROM docker.io/library/rust:1.98.1-slim-bookworm AS build +RUN apt-get update && apt-get install -y --no-install-recommends pkg-config libssl-dev curl ca-certificates \ + && rm -rf /var/lib/apt/lists/* +# Public RDS truststore retrieved over HTTPS 2026-10-09; checksum pins its bytes. +RUN curl --fail --location --proto '=https' --tlsv1.2 https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem -o /rds-ca.pem \ + && echo 'fe45bbebf92ad3e27a583bbb2ddd1553c521ed4d49af5514dc0a40372ea5395c /rds-ca.pem' | sha256sum --check +WORKDIR /src +COPY queue-rs/ ./ +RUN cargo build --release --locked -p polis-migrate +FROM docker.io/library/debian:bookworm-slim +RUN apt-get update && apt-get install -y --no-install-recommends libssl3 ca-certificates \ + && rm -rf /var/lib/apt/lists/* +COPY --from=build /src/target/release/polis-migrate /usr/local/bin/polis-migrate +COPY server/postgres/migrations /migrations +COPY --from=build /rds-ca.pem /etc/polis/rds-ca.pem +ENV POLIS_MIGRATIONS_DIR=/migrations POLIS_MIGRATE_CA_FILE=/etc/polis/rds-ca.pem +USER 65534:65534 +ENTRYPOINT ["polis-migrate"] +CMD ["apply"] diff --git a/queue-rs/polis-migrate/Dockerfile.dockerignore b/queue-rs/polis-migrate/Dockerfile.dockerignore new file mode 100644 index 0000000000..2850efb53f --- /dev/null +++ b/queue-rs/polis-migrate/Dockerfile.dockerignore @@ -0,0 +1,8 @@ +** +!queue-rs/ +!queue-rs/** +queue-rs/target/ +!server/ +!server/postgres/ +!server/postgres/migrations/ +!server/postgres/migrations/** diff --git a/queue-rs/polis-migrate/src/indexes.rs b/queue-rs/polis-migrate/src/indexes.rs new file mode 100644 index 0000000000..f2127f0c9e --- /dev/null +++ b/queue-rs/polis-migrate/src/indexes.rs @@ -0,0 +1,66 @@ +//! Resumable, autocommit execution for the immutable watermark index migration. +//! The caller holds the database-wide migration lock across every statement. +use anyhow::{Context, Result, ensure}; +use postgres::Client; + +const INDEXES: [(&str, &str, &str); 2] = [ + ("votes_created_idx", "votes", "created"), + ("comments_modified_idx", "comments", "modified"), +]; + +fn state(client: &mut Client, name: &str, table: &str, column: &str) -> Result> { + let rows = client.query( + "SELECT x.indisvalid AND x.indisready AND x.indislive AS usable, + pg_get_indexdef(c.oid), c.relkind + FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace + LEFT JOIN pg_index x ON x.indexrelid=c.oid + WHERE n.nspname='public' AND c.relname=$1", + &[&name], + )?; + let Some(row) = rows.first() else { + return Ok(None); + }; + let definition: Option = row.get(1); + let expected = format!("CREATE INDEX {name} ON public.{table} USING btree ({column})"); + ensure!( + definition.as_deref() == Some(expected.as_str()), + "000022: conflicting object public.{name}; preserve it and resolve its definition before retrying" + ); + let usable: Option = row.get(0); + Ok(Some(usable == Some(true))) +} + +pub fn prepare(client: &mut Client) -> Result<()> { + // Preflight BOTH identities before changing either one. A conflicting or + // invalid preexisting index is never dropped or replaced automatically. + for (name, table, column) in INDEXES { + if let Some(usable) = state(client, name, table, column)? { + ensure!( + usable, + "000022: public.{name} is invalid/not ready; inspect the interrupted build, then explicitly DROP INDEX CONCURRENTLY public.{name} before retrying apply; history has not recorded 000022" + ); + } + } + for (name, table, column) in INDEXES { + // Recheck after the preceding build. Uncoordinated external DDL is not + // serialized by the runner's advisory lock; any collision fails closed. + if state(client, name, table, column)? == Some(true) { + continue; + } + ensure!( + state(client, name, table, column)?.is_none(), + "000022: concurrent catalog change for public.{name}; inspect before retrying" + ); + println!("BUILDING CONCURRENTLY public.{name}"); + client.batch_execute(&format!( + "CREATE INDEX CONCURRENTLY {name} ON public.{table} USING btree ({column})" + )).with_context(|| format!( + "000022: concurrent build interrupted for public.{name}; inspect validity before retrying; valid earlier indexes are retained and no 000022 history row was committed" + ))?; + ensure!( + state(client, name, table, column)? == Some(true), + "000022: index public.{name} did not reach the expected valid state" + ); + } + Ok(()) +} diff --git a/queue-rs/polis-migrate/src/lib.rs b/queue-rs/polis-migrate/src/lib.rs new file mode 100644 index 0000000000..f40e4b6b54 --- /dev/null +++ b/queue-rs/polis-migrate/src/lib.rs @@ -0,0 +1,617 @@ +//! One migration history, one connection and one database advisory lock. +//! SQL files remain immutable; only their optional outer BEGIN/COMMIT is removed. +mod indexes; +mod sql; +use anyhow::{Context, Result, bail, ensure}; +use postgres::{ + Client, GenericClient, + config::{Host, SslMode}, +}; +use sha2::{Digest, Sha256}; +use std::{collections::BTreeMap, fs, path::Path, time::Duration}; + +pub struct Migration { + pub name: String, + pub checksum: String, + pub body: String, + pub retired: bool, +} +// Fixed database-wide key, shared by apply/reconcile. A different DB has its own lock. +pub const LOCK: i64 = 0x506f6c69734d6967; +const HISTORY: &str = "CREATE TABLE public.migrations ( + name text PRIMARY KEY, checksum text NOT NULL CHECK(length(checksum)=64), + status text NOT NULL CHECK(status IN ('APPLIED','ADOPTED')), + recorded_at timestamptz NOT NULL DEFAULT clock_timestamp(), + recorded_by text NOT NULL DEFAULT session_user, + legacy_completed_at bigint[] NOT NULL DEFAULT '{}'); + REVOKE ALL ON public.migrations FROM PUBLIC; + GRANT SELECT ON public.migrations TO PUBLIC;"; + +pub fn load(dir: &Path) -> Result> { + let mut files = BTreeMap::new(); + let mut versions = std::collections::BTreeSet::new(); + let selected = manifest(dir, "release.txt")?; + let held = manifest(dir, "held.txt")?; + ensure!( + selected.is_disjoint(&held), + "release and held manifests overlap" + ); + for entry in fs::read_dir(dir)? { + let entry = entry?; + let name = entry + .file_name() + .into_string() + .map_err(|_| anyhow::anyhow!("non-UTF8 migration name"))?; + if !name.ends_with(".sql") { + continue; + } + ensure!( + entry.file_type()?.is_file(), + "migration must be a regular file: {name}" + ); + ensure!( + name.len() > 11 + && name.as_bytes()[..6].iter().all(u8::is_ascii_digit) + && name.as_bytes()[6] == b'_' + && name.as_bytes()[7..name.len() - 4] + .iter() + .all(|b| b.is_ascii_alphanumeric() || *b == b'_'), + "invalid migration name: {name}" + ); + ensure!( + versions.insert(name[..6].to_owned()), + "duplicate migration number: {name}" + ); + let source = fs::read_to_string(entry.path())?; + files.insert( + name.clone(), + Migration { + name: name.clone(), + checksum: format!("{:x}", Sha256::digest(source.as_bytes())), + body: sql::body(&source)?, + retired: matches!(&name[..6], "000004" | "000005" | "000007"), + }, + ); + } + for name in selected.iter().chain(&held) { + ensure!( + files.contains_key(name), + "manifest migration is missing: {name}" + ); + } + for name in files.keys() { + ensure!( + selected.contains(name) || held.contains(name), + "migration absent from release/held manifest: {name}" + ); + } + let migrations: Vec<_> = files + .into_values() + .filter(|m| selected.contains(&m.name)) + .collect(); + ensure!( + migrations + .first() + .is_some_and(|m| m.name == "000000_initial.sql"), + "missing initial migration" + ); + Ok(migrations) +} + +fn manifest(dir: &Path, file: &str) -> Result> { + let text = fs::read_to_string(dir.join(file)).with_context(|| format!("read {file}"))?; + let mut names = std::collections::BTreeSet::new(); + for name in text + .lines() + .filter(|s| !s.is_empty() && !s.starts_with('#')) + { + ensure!( + name.len() > 11 + && name.as_bytes()[..6].iter().all(u8::is_ascii_digit) + && name.as_bytes()[6] == b'_' + && name.ends_with(".sql") + && name.as_bytes()[7..name.len() - 4] + .iter() + .all(|b| b.is_ascii_alphanumeric() || *b == b'_'), + "invalid migration in {file}: {name}" + ); + ensure!( + names.insert(name.to_owned()), + "duplicate migration in {file}: {name}" + ); + } + Ok(names) +} + +// Retired historical DDL is never executable, even on a fresh installation. +// Check catalog absence directly; no temporary functions or user-data reads. +fn retired_absent(client: &mut impl GenericClient, m: &Migration) -> Result { + let (tables, columns): (&[&str], &[(&str, &str)]) = match &m.name[..6] { + "000004" => (&["waitinglist"], &[]), + "000005" => ( + &[ + "slack_oauth_access_tokens", + "slack_users", + "slack_user_invites", + "slack_bot_events", + "stripe_accounts", + "stripe_subscriptions", + "coupons_for_free_upgrades", + "lti_users", + "lti_context_memberships", + "canvas_assignment_callback_info", + "canvas_assignment_conversation_info", + "lti_oauthv1_credentials", + ], + &[ + ("conversations", "is_slack"), + ("conversations", "lti_users_only"), + ("users", "plan"), + ], + ), + "000007" => ( + &["geolocation_cache"], + &[ + ("participants_extended", "country_code_iso"), + ("participants_extended", "encrypted_maxmind_response_city"), + ("participants_extended", "ip_address"), + ("participants_extended", "latitude"), + ("participants_extended", "location"), + ("participants_extended", "longitude"), + ("participants_extended", "x_forwarded_for"), + ], + ), + _ => bail!("not a retired migration: {}", m.name), + }; + for table in tables { + if exists(client, &format!("public.{table}"))? { + return Ok(false); + } + } + for (table, column) in columns { + let present: bool = client.query_one("SELECT EXISTS(SELECT 1 FROM pg_attribute WHERE attrelid=to_regclass('public.' || $1) AND attname=$2 AND attnum>0 AND NOT attisdropped)", &[table, column])?.get(0); + if present { + return Ok(false); + } + } + Ok(true) +} + +pub fn connect(dsn: &str) -> Result { + let mut cfg: postgres::Config = dsn + .parse() + .map_err(|_| anyhow::anyhow!("invalid DATABASE_URL"))?; + cfg.connect_timeout(Duration::from_secs(10)) + .application_name("polis-migrate"); + ensure!( + !cfg.get_hosts().is_empty() && cfg.get_hostaddrs().is_empty(), + "DATABASE_URL must name a host and cannot use hostaddr" + ); + if cfg.get_ssl_mode() == SslMode::Disable { + // Docker-local tests may use service DNS; remote plaintext is never implicit. + ensure!( + std::env::var("POLIS_MIGRATE_ALLOW_PLAINTEXT").as_deref() == Ok("local") + || cfg.get_hosts().iter().all(|h| match h { + Host::Tcp(h) => h.parse::().is_ok_and(|a| a.is_loopback()), + #[cfg(unix)] + Host::Unix(p) => p.is_absolute(), + }), + "plaintext requires loopback/socket or explicit POLIS_MIGRATE_ALLOW_PLAINTEXT=local" + ); + cfg.connect(postgres::NoTls) + .context("connect to migration database") + } else { + // require encryption and validate hostname + certificate even with sslmode=require. + cfg.ssl_mode(SslMode::Require); + let mut tls = native_tls::TlsConnector::builder(); + if let Ok(path) = std::env::var("POLIS_MIGRATE_CA_FILE") { + let bytes = fs::read(path).context("read migration CA file")?; + // RDS and local CA bundles can contain several certificates. + for pem in String::from_utf8(bytes)?.split_inclusive("-----END CERTIFICATE-----") { + if pem.contains("-----BEGIN CERTIFICATE-----") { + tls.add_root_certificate(native_tls::Certificate::from_pem(pem.as_bytes())?); + } + } + } + cfg.connect(postgres_native_tls::MakeTlsConnector::new(tls.build()?)) + .context("connect to migration database (verified TLS)") + } +} + +fn setup(client: &mut Client) -> Result<()> { + let version: i32 = client + .query_one("SELECT current_setting('server_version_num')::integer", &[])? + .get(0); + ensure!(version >= 170000, "PostgreSQL 17 or newer is required"); + client.batch_execute("SET standard_conforming_strings=on; SET search_path=pg_catalog,public; SET statement_timeout='5min'; SET lock_timeout='5s'; SET idle_in_transaction_session_timeout='30s'; SET transaction_timeout='5min';")?; + Ok(()) +} +fn exists(client: &mut impl GenericClient, name: &str) -> Result { + Ok(client + .query_one("SELECT to_regclass($1) IS NOT NULL", &[&name])? + .get(0)) +} +fn history( + client: &mut impl GenericClient, + migrations: &[Migration], +) -> Result> { + ensure!( + exists(client, "public.migrations")?, + "migration history missing; run polis-migrate apply for a fresh database, or reconcile for an existing database (docs/migrations.md)" + ); + let rows = client + .query( + "SELECT name, checksum, status FROM public.migrations ORDER BY name", + &[], + ) + .context("unreconciled legacy history; run polis-migrate reconcile")?; + let mut result = BTreeMap::new(); + for row in rows { + let name: String = row.get(0); + let checksum: String = row.get(1); + let status: String = row.get(2); + ensure!( + matches!(status.as_str(), "APPLIED" | "ADOPTED"), + "unverified history: {name}" + ); + let m = migrations.iter().find(|m| m.name == name).ok_or_else(|| { + anyhow::anyhow!("history names migration absent from this release: {name}") + })?; + ensure!( + checksum == m.checksum, + "migration checksum mismatch: {name}; restore the released source; do not replay it" + ); + ensure!( + result.insert(name.clone(), checksum).is_none(), + "duplicate history: {name}" + ); + } + Ok(result) +} +pub fn check(client: &mut Client, migrations: &[Migration]) -> Result<()> { + setup(client)?; + let mut tx = client.build_transaction().read_only(true).start()?; + let applied = history(&mut tx, migrations)?; + let pending: Vec<_> = migrations + .iter() + .filter(|m| !applied.contains_key(&m.name)) + .map(|m| m.name.as_str()) + .collect(); + ensure!( + pending.is_empty(), + "pending migrations: {}; run polis-migrate apply before starting the server", + pending.join(", ") + ); + tx.commit()?; + println!("migration check: {} ready", migrations.len()); + Ok(()) +} +fn lock(client: &mut Client) -> Result<()> { + setup(client)?; + // A blocking SELECT pg_advisory_lock retains a snapshot while waiting. + // CREATE INDEX CONCURRENTLY in the holder can wait for that same snapshot, + // deadlocking two runners. Each try returns before the bounded client wait, + // so no server statement/snapshot is held by the waiting runner. + println!("waiting for migration lock"); + let deadline = std::time::Instant::now() + Duration::from_secs(300); + loop { + let acquired: bool = client + .query_one("SELECT pg_try_advisory_lock($1)", &[&LOCK])? + .get(0); + if acquired { + break; + } + ensure!( + std::time::Instant::now() < deadline, + "timed out waiting for migration lock; another runner is still active" + ); + std::thread::sleep(Duration::from_millis(100)); + } + println!("migration lock acquired"); + Ok(()) +} +fn record( + client: &mut impl GenericClient, + m: &Migration, + status: &str, + legacy: &[i64], +) -> Result<()> { + client.execute("INSERT INTO public.migrations(name,checksum,status,legacy_completed_at) VALUES($1,$2,$3,$4)", &[&m.name,&m.checksum,&status,&legacy])?; + Ok(()) +} +pub fn apply(client: &mut Client, migrations: &[Migration]) -> Result { + lock(client)?; + let count = apply_locked(client, migrations)?; + client.query_one("SELECT pg_advisory_unlock($1)", &[&LOCK])?; + Ok(count) +} + +fn apply_locked(client: &mut Client, migrations: &[Migration]) -> Result { + // Validate history and every pending retirement before CREATE HISTORY or + // unrelated forward DDL. A refusal leaves the database unchanged. + let prior = if exists(client, "public.migrations")? { + history(client, migrations)? + } else { + BTreeMap::new() + }; + for m in migrations + .iter() + .filter(|m| m.retired && !prior.contains_key(&m.name)) + { + ensure!( + retired_absent(client, m)?, + "retired migration {} still has removed objects; no changes made; explicit operator review required", + m.name + ); + } + // A no-history populated DB is never treated as a fresh install. Check ALL + // public relations, not only conversations, before allowing 000000. + if !exists(client, "public.migrations")? { + let populated: bool = client.query_one("SELECT EXISTS(SELECT 1 FROM pg_class WHERE relnamespace='public'::regnamespace AND relkind IN ('r','p','v','m','S','f')) OR EXISTS(SELECT 1 FROM pg_proc WHERE pronamespace='public'::regnamespace) OR EXISTS(SELECT 1 FROM pg_type WHERE typnamespace='public'::regnamespace AND typtype IN ('e','d'))", &[])?.get(0); + ensure!( + !populated, + "existing database has no history; run reconcile before apply; 000000 will not be replayed" + ); + client.batch_execute(HISTORY)?; + } + let applied = history(client, migrations)?; + let mut count = 0; + for m in migrations.iter().filter(|m| !applied.contains_key(&m.name)) { + if m.name.starts_with("000000_") { + let existing: bool = client.query_one("SELECT EXISTS(SELECT 1 FROM pg_class WHERE relnamespace='public'::regnamespace AND relkind IN ('r','p','v','m','S','f') AND relname <> 'migrations') OR EXISTS(SELECT 1 FROM pg_proc WHERE pronamespace='public'::regnamespace) OR EXISTS(SELECT 1 FROM pg_type WHERE typnamespace='public'::regnamespace AND typtype IN ('e','d'))", &[])?.get(0); + ensure!( + !existing, + "refusing initial migration on an existing schema; reconcile first" + ); + } + if m.name == "000022_add_poll_timestamp_indexes.sql" { + // Source-bound special execution; never silently reinterpret a new + // version of this SQL. The original body below still checks both + // definitions before its history row commits. + ensure!( + m.checksum == "14efc95b14787b52d70680ea06cfef020cd2224e82495258fd8eb324501b7b3e", + "000022 source changed; review the concurrent execution contract" + ); + indexes::prepare(client)?; + } + let mut tx = client.transaction()?; + tx.batch_execute("SET LOCAL standard_conforming_strings=on; SET LOCAL search_path=public,pg_catalog; SET LOCAL lock_timeout='5s'; SET LOCAL statement_timeout='5min'; SET LOCAL transaction_timeout='5min';")?; + let status = if m.retired { + ensure!( + retired_absent(&mut tx, m)?, + "retired migration {} has retained objects; refusing adoption", + m.name + ); + "ADOPTED" + } else { + tx.batch_execute(&m.body).with_context(|| { + format!("migration {} failed (transaction not committed)", m.name) + })?; + // Restore the login's context after queue scripts SET LOCAL ROLE. + tx.batch_execute("RESET ROLE; SET LOCAL search_path=pg_catalog,public")?; + "APPLIED" + }; + record(&mut tx, m, status, &[])?; + tx.commit().with_context(|| { + format!( + "commit outcome unknown for {}; reconnect and check history before retrying", + m.name + ) + })?; + println!("{status} {}", m.name); + if status == "APPLIED" { + count += 1; + } + } + println!("applied {count} migration(s)"); + Ok(count) +} + +// Reuse the queue's existing catalog verifiers for older Docker installations. +// Only temporary catalog-reading functions are installed here, never queue DDL. +fn queue_version(tx: &mut impl GenericClient, migrations: &[Migration]) -> Result { + let present: bool = tx.query_one("SELECT EXISTS(SELECT 1 FROM pg_class WHERE relnamespace='public'::regnamespace AND (starts_with(relname,'polis_queue_') OR starts_with(relname,'delphi_'))) OR EXISTS(SELECT 1 FROM pg_proc WHERE pronamespace='public'::regnamespace AND (starts_with(proname,'pq_') OR starts_with(proname,'pd_')))",&[])?.get(0); + if !present { + return Ok(0); + } + ensure!( + exists(tx, "public.polis_queue_install")?, + "partial queue installation; no adoption committed" + ); + let mut definitions = 0; + for m in migrations + .iter() + .filter(|m| m.name.starts_with("000019_") || m.name.starts_with("000024_")) + { + for statement in sql::statements(&m.body)? { + if [ + "pq_catalog", + "pq_assert_catalog", + "pq_assert_signatures", + "pq_assert_functions", + "pd_state", + "pq3_state", + ] + .iter() + .any(|name| { + statement.starts_with(&format!("CREATE OR REPLACE FUNCTION pg_temp.{name}(")) + }) { + tx.batch_execute(&statement)?; + definitions += 1; + } + } + } + ensure!( + definitions == 7, + "queue adoption verifier definitions changed; review the catalog contract" + ); + if exists(tx, "public.polis_queue_large_class_install")? { + ensure!( + exists(tx, "public.delphi_foundation_install")?, + "partial queue /3 installation" + ); + let ok: bool=tx.query_one("SELECT (SELECT count(*)=1 FROM public.polis_queue_large_class_install) AND EXISTS(SELECT 1 FROM public.polis_queue_large_class_install WHERE singleton AND installed=pg_temp.pq3_state()) AND (SELECT count(*)=1 FROM public.delphi_foundation_install) AND (SELECT count(*)=1 AND bool_and(contract_version='polis-queue/3') FROM public.polis_queue_install)",&[])?.get(0); + ensure!(ok, "queue /3 catalog postconditions fail"); + Ok(24) + } else if exists(tx, "public.delphi_foundation_install")? { + let ok: bool=tx.query_one("SELECT (SELECT count(*)=1 FROM public.delphi_foundation_install) AND EXISTS(SELECT 1 FROM public.delphi_foundation_install WHERE singleton AND installed=pg_temp.pd_state()) AND (SELECT count(*)=1 AND bool_and(contract_version='polis-queue/2') FROM public.polis_queue_install)",&[])?.get(0); + ensure!(ok, "queue /2 catalog postconditions fail"); + Ok(23) + } else { + tx.batch_execute("SELECT pg_temp.pq_assert_catalog(false); SELECT pg_temp.pq_assert_signatures(false); SELECT pg_temp.pq_assert_functions(false)").context("queue /1 catalog postconditions fail")?; + ensure!( + tx.query_one("SELECT count(*)=1 FROM public.polis_queue_install", &[])? + .get::<_, bool>(0), + "queue /1 install record missing" + ); + Ok(19) + } +} + +/// Adoption is explicitly bounded by the operator, but every file is checked. +/// The missing 19 queue is a known hole in the pre-runner production baseline. +/// Existing queues use their catalog verifiers, never install-row presence alone. +pub fn reconcile( + client: &mut Client, + migrations: &[Migration], + dir: &Path, + through: &str, +) -> Result { + ensure!( + through.len() == 6 && through.bytes().all(|b| b.is_ascii_digit()), + "--through requires six digits" + ); + ensure!( + migrations.iter().any(|m| &m.name[..6] == through), + "unknown --through version" + ); + lock(client)?; + let count = reconcile_locked(client, migrations, dir, through)?; + client.query_one("SELECT pg_advisory_unlock($1)", &[&LOCK])?; + Ok(count) +} + +fn reconcile_locked( + client: &mut Client, + migrations: &[Migration], + dir: &Path, + through: &str, +) -> Result { + let mut tx = client.transaction()?; + let modern = tx.query_one("SELECT EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema='public' AND table_name='migrations' AND column_name='checksum')", &[])?.get::<_,bool>(0); + if modern { + history(&mut tx, migrations)?; + bail!("history is already reconciled; use apply/check"); + } + ensure!( + !exists(&mut tx, "public.schema_migrations")?, + "another schema_migrations ledger exists; resolve its unverified rows before reconciliation" + ); + let mut legacy: BTreeMap> = BTreeMap::new(); + if exists(&mut tx, "public.migrations")? { + for row in tx.query( + "SELECT name,completed_at FROM public.migrations ORDER BY name,completed_at", + &[], + )? { + let name: String = row.get(0); + ensure!( + migrations + .iter() + .any(|m| m.name == name && &m.name[..6] <= through), + "legacy history contains unknown/renamed/later migration {name}; resolve against its catalog, never replay blindly" + ); + legacy.entry(name).or_default().push(row.get(1)); + } + } + // Helpers are transaction-local and contain only catalog reads. + tx.batch_execute(&fs::read_to_string(dir.join("adoption/helpers.sql"))?)?; + let queue = queue_version(&mut tx, migrations)?; + ensure!( + queue <= through.parse::()?, + "queue is newer than --through; select its actual installed version" + ); + let mut adopted = Vec::new(); + for m in migrations.iter().filter(|m| &m.name[..6] <= through) { + let number = m.name[..6].parse::()?; + if matches!(number, 19 | 23 | 24) { + if queue >= number { + adopted.push(m); + } else { + ensure!( + !legacy.contains_key(&m.name), + "legacy queue history row was not verified: {}", + m.name + ); + } + continue; + } + let path = dir.join("adoption").join(&m.name); + let query = fs::read_to_string(path).with_context(|| { + format!( + "no adoption contract for {}; stop and review this existing installation", + m.name + ) + })?; + let ok: bool = tx + .query_one(&query, &[]) + .with_context(|| format!("catalog check for {}", m.name))? + .get(0); + ensure!( + ok, + "catalog postconditions fail for {}; no adoption rows committed; inspect schema before retrying", + m.name + ); + adopted.push(m); + } + for name in legacy.keys() { + ensure!( + adopted.iter().any(|m| m.name == *name), + "legacy row not verified: {name}" + ); + } + if exists(&mut tx, "public.migrations")? { + tx.batch_execute("DROP TABLE public.migrations")?; + } + tx.batch_execute(HISTORY)?; + for m in &adopted { + record( + &mut tx, + m, + "ADOPTED", + legacy.get(&m.name).map(Vec::as_slice).unwrap_or(&[]), + )?; + } + tx.commit()?; + for m in &adopted { + println!("ADOPTED {}", m.name); + } + println!( + "adopted {} migration(s); no migration SQL replayed", + adopted.len() + ); + Ok(adopted.len()) +} + +/// Release deployment: classify history while holding the same lock as apply. +/// The 000022 bound is this release's reviewed legacy contract, not a guess +/// from a failed apply. Later/partial/unknown schemas must still pass reconcile. +pub fn deploy(client: &mut Client, migrations: &[Migration], dir: &Path) -> Result<()> { + lock(client)?; + let modern: bool = client.query_one("SELECT EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema='public' AND table_name='migrations' AND column_name='checksum')", &[])?.get(0); + let populated: bool = client.query_one("SELECT EXISTS(SELECT 1 FROM pg_class WHERE relnamespace='public'::regnamespace AND relkind IN ('r','p','v','m','S','f')) OR EXISTS(SELECT 1 FROM pg_proc WHERE pronamespace='public'::regnamespace) OR EXISTS(SELECT 1 FROM pg_type WHERE typnamespace='public'::regnamespace AND typtype IN ('e','d'))", &[])?.get(0); + ensure!( + !exists(client, "public.schema_migrations")?, + "another schema_migrations ledger exists; resolve it before deployment" + ); + if !modern && populated { + ensure!( + migrations.iter().any(|m| m.name.starts_with("000022_")), + "deploy requires the reviewed 000022 adoption bound" + ); + reconcile_locked(client, migrations, dir, "000022")?; + } + apply_locked(client, migrations)?; + check(client, migrations)?; + client.query_one("SELECT pg_advisory_unlock($1)", &[&LOCK])?; + Ok(()) +} diff --git a/queue-rs/polis-migrate/src/main.rs b/queue-rs/polis-migrate/src/main.rs new file mode 100644 index 0000000000..eccd9ed0b6 --- /dev/null +++ b/queue-rs/polis-migrate/src/main.rs @@ -0,0 +1,82 @@ +use anyhow::{Result, bail}; +use std::{env, path::PathBuf}; +fn run() -> Result<()> { + let mut args = env::args().skip(1); + let command = args.next().unwrap_or_default(); + if matches!(command.as_str(), "--help" | "-h") { + println!( + "polis-migrate deploy|apply|check|reconcile [--dir PATH] [--through NNNNNN]\nDATABASE_URL is read from the environment; reconcile requires --through. See docs/migrations.md." + ); + return Ok(()); + } + let mut dir = PathBuf::from( + env::var("POLIS_MIGRATIONS_DIR").unwrap_or_else(|_| "server/postgres/migrations".into()), + ); + let mut through = None; + while let Some(arg) = args.next() { + match arg.as_str() { + "--dir" => { + dir = args + .next() + .ok_or_else(|| anyhow::anyhow!("--dir needs a path"))? + .into() + } + "--through" => { + through = Some( + args.next() + .ok_or_else(|| anyhow::anyhow!("--through needs a version"))?, + ) + } + _ => bail!("unknown argument: {arg}"), + } + } + if !matches!(command.as_str(), "deploy" | "apply" | "check" | "reconcile") { + bail!("expected deploy, apply, check or reconcile (see --help)"); + } + if (command == "reconcile") != through.is_some() { + bail!("only reconcile requires --through NNNNNN"); + } + let migrations = polis_migrate::load(&dir)?; + let dsn = env::var("DATABASE_URL").map_err(|_| anyhow::anyhow!("DATABASE_URL is required"))?; + let mut client = polis_migrate::connect(&dsn)?; + match command.as_str() { + "deploy" => polis_migrate::deploy(&mut client, &migrations, &dir)?, + "apply" => { + polis_migrate::apply(&mut client, &migrations)?; + } + "check" => polis_migrate::check(&mut client, &migrations)?, + "reconcile" => { + polis_migrate::reconcile( + &mut client, + &migrations, + &dir, + through.as_deref().unwrap_or(""), + )?; + } + _ => unreachable!(), + } + Ok(()) +} +fn main() { + if let Err(error) = run() { + // Do not print connection strings, SQL statements or database error DETAIL + // (which may quote row values). Context + SQLSTATE are sufficient here. + for cause in error.chain() { + if let Some(pg) = cause.downcast_ref::() { + if let Some(db) = pg.as_db_error() { + eprintln!( + "database refused migration: SQLSTATE {} (inspect the named migration)", + db.code().code() + ); + } else { + eprintln!( + "database connection failed; check connectivity and TLS configuration" + ); + } + break; + } + eprintln!("{cause}"); + } + std::process::exit(1); + } +} diff --git a/queue-rs/polis-migrate/src/sql.rs b/queue-rs/polis-migrate/src/sql.rs new file mode 100644 index 0000000000..efcd023884 --- /dev/null +++ b/queue-rs/polis-migrate/src/sql.rs @@ -0,0 +1,282 @@ +//! Split only top-level SQL. Dollar bodies, quoted strings and comments are opaque. +//! This is deliberately not a general SQL rewriter: accept one optional outer +//! BEGIN/COMMIT pair and refuse every other transaction-control statement. +use anyhow::{Result, bail, ensure}; + +fn ident_start(c: char) -> bool { + c.is_ascii_alphabetic() || c == '_' || !c.is_ascii() +} + +fn ident_continue(c: char) -> bool { + ident_start(c) || c.is_ascii_digit() || c == '$' +} + +// PostgreSQL continues single-quoted strings across whitespace containing a +// newline, retaining the first segment's E-string escape mode. Line comments +// count as whitespace here; block comments terminate this lexical construct. +fn continued_string(bytes: &[u8], mut next: usize) -> Option { + let mut newline = false; + while next < bytes.len() { + match bytes[next] { + b'\n' | b'\r' => { + newline = true; + next += 1; + } + b' ' | b'\t' | b'\x0c' | b'\x0b' => next += 1, + b'-' if bytes[next..].starts_with(b"--") => { + while next < bytes.len() && !matches!(bytes[next], b'\n' | b'\r') { + next += 1; + } + } + b'\'' if newline => return Some(next + 1), + _ => return None, + } + } + None +} + +// Keep executable bytes alongside comment-free text used only for inspection. +// Removing comments from the executed source can change string-literal parsing. +fn lex(sql: &str) -> Result> { + let b = sql.as_bytes(); + let (mut i, mut part, mut raw, mut out) = (0, String::new(), String::new(), Vec::new()); + while i < b.len() { + let start = i; + if b[i..].starts_with(b"--") { + while i < b.len() && b[i] != b'\n' { + i += 1; + } + part.push(' '); + raw.push_str(&sql[start..i]); + continue; + } + if b[i..].starts_with(b"/*") { + let mut depth = 1; + i += 2; + while i < b.len() && depth > 0 { + if b[i..].starts_with(b"/*") { + depth += 1; + i += 2; + } else if b[i..].starts_with(b"*/") { + depth -= 1; + i += 2; + } else { + i += 1; + } + } + ensure!(depth == 0, "unterminated SQL comment"); + part.push(' '); + raw.push_str(&sql[start..i]); + continue; + } + if b[i] == b'\'' || b[i] == b'"' { + let quote = b[i]; + let escaped = quote == b'\'' + && i > 0 + && matches!(b[i - 1], b'e' | b'E') + && (i == 1 || !sql[..i - 1].chars().next_back().is_some_and(ident_continue)); + i += 1; + let mut closed = false; + while i < b.len() { + if escaped && b[i] == b'\\' { + i += 2; + } else if b[i] == quote { + i += 1; + if i < b.len() && b[i] == quote { + i += 1; + } else if quote == b'\'' + && let Some(next) = continued_string(b, i) + { + i = next; + } else { + closed = true; + break; + } + } else { + i += 1; + } + } + ensure!(closed && i <= b.len(), "unterminated SQL quote"); + } else if b[i] == b'$' { + let mut j = i + 1; + for c in sql[j..].chars() { + if ident_start(c) || (j > i + 1 && c.is_ascii_digit()) { + j += c.len_utf8(); + } else { + break; + } + } + if j < b.len() && b[j] == b'$' { + let tag = &sql[i..=j]; + let rest = &sql[j + 1..]; + let end = rest + .find(tag) + .ok_or_else(|| anyhow::anyhow!("unterminated dollar body"))?; + i = j + 1 + end + tag.len(); + } else { + i += 1; + } + } else if sql[i..].chars().next().is_some_and(ident_start) { + // '$' is legal inside an unquoted identifier. Consume the whole + // identifier before looking for a dollar-quoted body; PostgreSQL + // requires a delimiter between an identifier and such a body. + for c in sql[i..].chars() { + if !ident_continue(c) { + break; + } + i += c.len_utf8(); + } + } else if b[i] == b';' { + if !part.trim().is_empty() { + out.push((part.trim().to_owned(), raw.trim_start().to_owned())); + } + part.clear(); + raw.clear(); + i += 1; + continue; + } else { + // Work at UTF-8 character boundaries even in unquoted identifiers. + i += sql[i..].chars().next().map(char::len_utf8).unwrap_or(1); + } + part.push_str(&sql[start..i]); + raw.push_str(&sql[start..i]); + } + if !part.trim().is_empty() { + out.push((part.trim().to_owned(), raw.trim_start().to_owned())); + } + Ok(out) +} + +pub fn statements(sql: &str) -> Result> { + Ok(lex(sql)? + .into_iter() + .map(|(inspection, _)| inspection) + .collect()) +} + +pub fn body(sql: &str) -> Result { + let mut parts = lex(sql)?; + ensure!(!parts.is_empty(), "empty migration"); + if parts[0].0.eq_ignore_ascii_case("BEGIN") { + ensure!( + parts + .last() + .is_some_and(|s| s.0.eq_ignore_ascii_case("COMMIT")), + "BEGIN without final COMMIT" + ); + parts.remove(0); + parts.pop(); + } + for (part, _) in &parts { + let first = part + .split_whitespace() + .next() + .unwrap_or("") + .to_ascii_uppercase(); + if matches!( + first.as_str(), + "BEGIN" + | "START" + | "COMMIT" + | "END" + | "ROLLBACK" + | "ABORT" + | "SAVEPOINT" + | "RELEASE" + | "PREPARE" + ) || first.starts_with('\\') + { + bail!("migration contains transaction control or a psql command: {first}"); + } + } + Ok(parts + .into_iter() + .map(|(_, raw)| raw) + .collect::>() + .join(";\n") + + ";") +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn dollar_and_comment_bodies() -> Result<()> { + let s = "-- BEGIN;\nBEGIN; DO $q$ BEGIN RAISE NOTICE 'COMMIT;'; END $q$; /* a /* b */ c */ COMMIT;"; + assert_eq!(body(s)?, "DO $q$ BEGIN RAISE NOTICE 'COMMIT;'; END $q$;"); + Ok(()) + } + #[test] + fn nested_control_refused() { + for s in [ + "BEGIN; SELECT 1; COMMIT; COMMIT;", + "SELECT 1; ROLLBACK;", + "START TRANSACTION; SELECT 1;", + "BEGIN; SELECT 1;", + "SELECT 1; \\i file", + ] { + assert!(body(s).is_err(), "{s}"); + } + } + #[test] + fn quotes_and_unicode() -> Result<()> { + assert_eq!( + statements("SELECT E'a\\';b', 'c'';d', \"é;\"; SELECT '𝄞';")?.len(), + 2 + ); + Ok(()) + } + #[test] + fn incomplete_refused() { + for s in ["SELECT 'x", "DO $$ x", "/* unclosed", "-- empty"] { + assert!(body(s).is_err()); + } + } + #[test] + fn dollar_identifiers_do_not_quote_following_statements() -> Result<()> { + let source = + "CREATE TABLE a$tag$ (x int); COMMIT; CREATE TABLE b$tag$ (y int); SELECT 1/0;"; + assert_eq!(statements(source)?.len(), 4); + assert!(body(source).is_err()); + assert!(body("CREATE TABLE é$tag$ (x int); COMMIT; CREATE TABLE z$tag$ (y int);").is_err()); + assert_eq!( + statements("SELECT ordinary$identifier, other$$identifier;")?.len(), + 1 + ); + Ok(()) + } + #[test] + fn unicode_dollar_tags_preserve_function_bodies() -> Result<()> { + let source = "DO $тег_1$ BEGIN PERFORM 1; END $тег_1$; SELECT 2;"; + assert_eq!(statements(source)?.len(), 2); + assert!(body(source)?.contains("BEGIN PERFORM 1; END")); + Ok(()) + } + #[test] + fn executable_comments_are_preserved() -> Result<()> { + let source = "SELECT 'a' /* line\n /* nested */ end */ 'b';"; + assert_eq!(body(source)?, source); + assert!(body("BEGIN; SELECT 1; COMMIT/* trailing */;")?.contains("SELECT 1")); + assert!(body("SELECT 1; COMMIT/* trailing */;").is_err()); + Ok(()) + } + #[test] + fn escaped_string_continuations_keep_their_mode() -> Result<()> { + let source = "SELECT E'a' -- explanation\n'b\\'c'; SELECT 2;"; + assert_eq!(statements(source)?.len(), 2); + assert!(body(source)?.contains("'b\\'c'")); + Ok(()) + } + #[test] + fn typed_literals_after_identifiers_use_ordinary_string_mode() -> Result<()> { + // E is a prefix only at a token boundary. Dollar and non-ASCII characters + // are PostgreSQL identifier continuations too; use the same rule above. + for name in ["typed$E", "typedéE", "typedeE"] { + let source = format!("SELECT {name}'a\\'; SELECT 2;"); + assert_eq!(statements(&source)?.len(), 2, "{name}"); + assert!(body(&source).is_ok(), "{name}"); + } + assert_eq!(statements("SELECT E'a\\\\'; SELECT 2;")?.len(), 2); + Ok(()) + } +} diff --git a/queue-rs/polis-migrate/tests/adoption-cases.json b/queue-rs/polis-migrate/tests/adoption-cases.json new file mode 100644 index 0000000000..9faa38ef83 --- /dev/null +++ b/queue-rs/polis-migrate/tests/adoption-cases.json @@ -0,0 +1,1112 @@ +[ + { + "name": "canonical_3", + "migration": 3, + "mutation": "", + "expected": "t" + }, + { + "name": "default_added_participants_extended_permanent_cookie", + "migration": 3, + "mutation": "ALTER TABLE participants_extended ALTER COLUMN permanent_cookie SET DEFAULT 'constructed';", + "expected": "f" + }, + { + "name": "nullability_participants_extended_permanent_cookie", + "migration": 3, + "mutation": "ALTER TABLE participants_extended ALTER COLUMN permanent_cookie SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_added_participants_extended_origin", + "migration": 3, + "mutation": "ALTER TABLE participants_extended ALTER COLUMN origin SET DEFAULT 'constructed';", + "expected": "f" + }, + { + "name": "nullability_participants_extended_origin", + "migration": 3, + "mutation": "ALTER TABLE participants_extended ALTER COLUMN origin SET NOT NULL;", + "expected": "f" + }, + { + "name": "canonical_8", + "migration": 8, + "mutation": "", + "expected": "t" + }, + { + "name": "default_removed_conversations_importance_enabled", + "migration": 8, + "mutation": "ALTER TABLE conversations ALTER COLUMN importance_enabled DROP DEFAULT;", + "expected": "f" + }, + { + "name": "default_removed_votes_high_priority", + "migration": 8, + "mutation": "ALTER TABLE votes ALTER COLUMN high_priority DROP DEFAULT;", + "expected": "f" + }, + { + "name": "canonical_9", + "migration": 9, + "mutation": "", + "expected": "t" + }, + { + "name": "default_added_zinvites_uuid", + "migration": 9, + "mutation": "ALTER TABLE zinvites ALTER COLUMN uuid SET DEFAULT '00000000-0000-0000-0000-000000000001';", + "expected": "f" + }, + { + "name": "nullability_zinvites_uuid", + "migration": 9, + "mutation": "ALTER TABLE zinvites ALTER COLUMN uuid SET NOT NULL;", + "expected": "f" + }, + { + "name": "canonical_10", + "migration": 10, + "mutation": "", + "expected": "t" + }, + { + "name": "default_added_oidc_user_mappings_oidc_sub", + "migration": 10, + "mutation": "ALTER TABLE oidc_user_mappings ALTER COLUMN oidc_sub SET DEFAULT 'constructed';", + "expected": "f" + }, + { + "name": "default_added_oidc_user_mappings_uid", + "migration": 10, + "mutation": "ALTER TABLE oidc_user_mappings ALTER COLUMN uid SET DEFAULT 7;", + "expected": "f" + }, + { + "name": "default_removed_oidc_user_mappings_created", + "migration": 10, + "mutation": "ALTER TABLE oidc_user_mappings ALTER COLUMN created DROP DEFAULT;", + "expected": "f" + }, + { + "name": "nullability_oidc_user_mappings_created", + "migration": 10, + "mutation": "ALTER TABLE oidc_user_mappings ALTER COLUMN created SET NOT NULL;", + "expected": "f" + }, + { + "name": "canonical_11", + "migration": 11, + "mutation": "", + "expected": "t" + }, + { + "name": "canonical_12", + "migration": 12, + "mutation": "", + "expected": "t" + }, + { + "name": "default_added_topic_agenda_selections_zid", + "migration": 12, + "mutation": "ALTER TABLE topic_agenda_selections ALTER COLUMN zid SET DEFAULT 7;", + "expected": "f" + }, + { + "name": "default_added_topic_agenda_selections_pid", + "migration": 12, + "mutation": "ALTER TABLE topic_agenda_selections ALTER COLUMN pid SET DEFAULT 7;", + "expected": "f" + }, + { + "name": "default_removed_topic_agenda_selections_archetypal_selections", + "migration": 12, + "mutation": "ALTER TABLE topic_agenda_selections ALTER COLUMN archetypal_selections DROP DEFAULT;", + "expected": "f" + }, + { + "name": "default_added_topic_agenda_selections_delphi_job_id", + "migration": 12, + "mutation": "ALTER TABLE topic_agenda_selections ALTER COLUMN delphi_job_id SET DEFAULT 'constructed';", + "expected": "f" + }, + { + "name": "nullability_topic_agenda_selections_delphi_job_id", + "migration": 12, + "mutation": "ALTER TABLE topic_agenda_selections ALTER COLUMN delphi_job_id SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_removed_topic_agenda_selections_total_selections", + "migration": 12, + "mutation": "ALTER TABLE topic_agenda_selections ALTER COLUMN total_selections DROP DEFAULT;", + "expected": "f" + }, + { + "name": "default_removed_topic_agenda_selections_created_at", + "migration": 12, + "mutation": "ALTER TABLE topic_agenda_selections ALTER COLUMN created_at DROP DEFAULT;", + "expected": "f" + }, + { + "name": "nullability_topic_agenda_selections_created_at", + "migration": 12, + "mutation": "ALTER TABLE topic_agenda_selections ALTER COLUMN created_at SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_removed_topic_agenda_selections_updated_at", + "migration": 12, + "mutation": "ALTER TABLE topic_agenda_selections ALTER COLUMN updated_at DROP DEFAULT;", + "expected": "f" + }, + { + "name": "nullability_topic_agenda_selections_updated_at", + "migration": 12, + "mutation": "ALTER TABLE topic_agenda_selections ALTER COLUMN updated_at SET NOT NULL;", + "expected": "f" + }, + { + "name": "canonical_13", + "migration": 13, + "mutation": "", + "expected": "t" + }, + { + "name": "default_removed_conversations_treevite_enabled", + "migration": 13, + "mutation": "ALTER TABLE conversations ALTER COLUMN treevite_enabled DROP DEFAULT;", + "expected": "f" + }, + { + "name": "nullability_conversations_treevite_enabled", + "migration": 13, + "mutation": "ALTER TABLE conversations ALTER COLUMN treevite_enabled SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_removed_treevite_waves_id", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves ALTER COLUMN id DROP DEFAULT;", + "expected": "f" + }, + { + "name": "default_added_treevite_waves_zid", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves ALTER COLUMN zid SET DEFAULT 7;", + "expected": "f" + }, + { + "name": "default_added_treevite_waves_wave", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves ALTER COLUMN wave SET DEFAULT 7;", + "expected": "f" + }, + { + "name": "default_added_treevite_waves_parent_wave", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves ALTER COLUMN parent_wave SET DEFAULT 7;", + "expected": "f" + }, + { + "name": "nullability_treevite_waves_parent_wave", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves ALTER COLUMN parent_wave SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_added_treevite_waves_size", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves ALTER COLUMN size SET DEFAULT 7;", + "expected": "f" + }, + { + "name": "nullability_treevite_waves_size", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves ALTER COLUMN size SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_added_treevite_waves_invites_per_user", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves ALTER COLUMN invites_per_user SET DEFAULT 7;", + "expected": "f" + }, + { + "name": "default_removed_treevite_waves_owner_invites", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves ALTER COLUMN owner_invites DROP DEFAULT;", + "expected": "f" + }, + { + "name": "default_removed_treevite_waves_created_at", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves ALTER COLUMN created_at DROP DEFAULT;", + "expected": "f" + }, + { + "name": "nullability_treevite_waves_created_at", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves ALTER COLUMN created_at SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_removed_treevite_waves_updated_at", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves ALTER COLUMN updated_at DROP DEFAULT;", + "expected": "f" + }, + { + "name": "nullability_treevite_waves_updated_at", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves ALTER COLUMN updated_at SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_removed_treevite_invites_id", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER COLUMN id DROP DEFAULT;", + "expected": "f" + }, + { + "name": "default_added_treevite_invites_zid", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER COLUMN zid SET DEFAULT 7;", + "expected": "f" + }, + { + "name": "default_added_treevite_invites_wave_id", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER COLUMN wave_id SET DEFAULT 7;", + "expected": "f" + }, + { + "name": "default_added_treevite_invites_parent_invite_id", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER COLUMN parent_invite_id SET DEFAULT 7;", + "expected": "f" + }, + { + "name": "nullability_treevite_invites_parent_invite_id", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER COLUMN parent_invite_id SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_removed_treevite_invites_status", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER COLUMN status DROP DEFAULT;", + "expected": "f" + }, + { + "name": "default_added_treevite_invites_invite_owner_pid", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER COLUMN invite_owner_pid SET DEFAULT 7;", + "expected": "f" + }, + { + "name": "nullability_treevite_invites_invite_owner_pid", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER COLUMN invite_owner_pid SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_added_treevite_invites_invite_used_by_pid", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER COLUMN invite_used_by_pid SET DEFAULT 7;", + "expected": "f" + }, + { + "name": "nullability_treevite_invites_invite_used_by_pid", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER COLUMN invite_used_by_pid SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_added_treevite_invites_invite_used_at", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER COLUMN invite_used_at SET DEFAULT '2000-01-01';", + "expected": "f" + }, + { + "name": "nullability_treevite_invites_invite_used_at", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER COLUMN invite_used_at SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_removed_treevite_invites_created_at", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER COLUMN created_at DROP DEFAULT;", + "expected": "f" + }, + { + "name": "nullability_treevite_invites_created_at", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER COLUMN created_at SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_removed_treevite_invites_updated_at", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER COLUMN updated_at DROP DEFAULT;", + "expected": "f" + }, + { + "name": "nullability_treevite_invites_updated_at", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER COLUMN updated_at SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_removed_treevite_login_codes_id", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes ALTER COLUMN id DROP DEFAULT;", + "expected": "f" + }, + { + "name": "default_added_treevite_login_codes_zid", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes ALTER COLUMN zid SET DEFAULT 7;", + "expected": "f" + }, + { + "name": "default_added_treevite_login_codes_pid", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes ALTER COLUMN pid SET DEFAULT 7;", + "expected": "f" + }, + { + "name": "default_added_treevite_login_codes_login_code_hash", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes ALTER COLUMN login_code_hash SET DEFAULT 'constructed';", + "expected": "f" + }, + { + "name": "default_removed_treevite_login_codes_fp_kid", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes ALTER COLUMN fp_kid DROP DEFAULT;", + "expected": "f" + }, + { + "name": "default_removed_treevite_login_codes_revoked", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes ALTER COLUMN revoked DROP DEFAULT;", + "expected": "f" + }, + { + "name": "default_added_treevite_login_codes_expires_at", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes ALTER COLUMN expires_at SET DEFAULT '2000-01-01';", + "expected": "f" + }, + { + "name": "nullability_treevite_login_codes_expires_at", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes ALTER COLUMN expires_at SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_added_treevite_login_codes_last_used_at", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes ALTER COLUMN last_used_at SET DEFAULT '2000-01-01';", + "expected": "f" + }, + { + "name": "nullability_treevite_login_codes_last_used_at", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes ALTER COLUMN last_used_at SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_removed_treevite_login_codes_created_at", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes ALTER COLUMN created_at DROP DEFAULT;", + "expected": "f" + }, + { + "name": "nullability_treevite_login_codes_created_at", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes ALTER COLUMN created_at SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_removed_treevite_login_codes_updated_at", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes ALTER COLUMN updated_at DROP DEFAULT;", + "expected": "f" + }, + { + "name": "nullability_treevite_login_codes_updated_at", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes ALTER COLUMN updated_at SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_added_treevite_invites_invite_code", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER COLUMN invite_code SET DEFAULT 'constructed';", + "expected": "f" + }, + { + "name": "default_added_treevite_login_codes_login_code_fingerprint", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes ALTER COLUMN login_code_fingerprint SET DEFAULT 'constructed';", + "expected": "f" + }, + { + "name": "default_added_treevite_login_codes_login_code_lookup", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes ALTER COLUMN login_code_lookup SET DEFAULT 'constructed';", + "expected": "f" + }, + { + "name": "nullability_treevite_login_codes_login_code_lookup", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes ALTER COLUMN login_code_lookup SET NOT NULL;", + "expected": "f" + }, + { + "name": "canonical_14", + "migration": 14, + "mutation": "", + "expected": "t" + }, + { + "name": "default_removed_reports_mod_level", + "migration": 14, + "mutation": "ALTER TABLE reports ALTER COLUMN mod_level DROP DEFAULT;", + "expected": "f" + }, + { + "name": "canonical_15", + "migration": 15, + "mutation": "", + "expected": "t" + }, + { + "name": "default_removed_conversations_xid_required", + "migration": 15, + "mutation": "ALTER TABLE conversations ALTER COLUMN xid_required DROP DEFAULT;", + "expected": "f" + }, + { + "name": "default_added_xid_whitelist_zid", + "migration": 15, + "mutation": "ALTER TABLE xid_whitelist ALTER COLUMN zid SET DEFAULT 7;", + "expected": "f" + }, + { + "name": "nullability_xid_whitelist_zid", + "migration": 15, + "mutation": "ALTER TABLE xid_whitelist ALTER COLUMN zid SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_added_xids_zid", + "migration": 15, + "mutation": "ALTER TABLE xids ALTER COLUMN zid SET DEFAULT 7;", + "expected": "f" + }, + { + "name": "nullability_xids_zid", + "migration": 15, + "mutation": "ALTER TABLE xids ALTER COLUMN zid SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_added_xids_pid", + "migration": 15, + "mutation": "ALTER TABLE xids ALTER COLUMN pid SET DEFAULT 7;", + "expected": "f" + }, + { + "name": "nullability_xids_pid", + "migration": 15, + "mutation": "ALTER TABLE xids ALTER COLUMN pid SET NOT NULL;", + "expected": "f" + }, + { + "name": "canonical_16", + "migration": 16, + "mutation": "", + "expected": "t" + }, + { + "name": "default_added_comments_original_id", + "migration": 16, + "mutation": "ALTER TABLE comments ALTER COLUMN original_id SET DEFAULT '00000000-0000-0000-0000-000000000001';", + "expected": "f" + }, + { + "name": "nullability_comments_original_id", + "migration": 16, + "mutation": "ALTER TABLE comments ALTER COLUMN original_id SET NOT NULL;", + "expected": "f" + }, + { + "name": "canonical_17", + "migration": 17, + "mutation": "", + "expected": "t" + }, + { + "name": "default_removed_byod_import_jobs_id", + "migration": 17, + "mutation": "ALTER TABLE byod_import_jobs ALTER COLUMN id DROP DEFAULT;", + "expected": "f" + }, + { + "name": "default_added_byod_import_jobs_zid", + "migration": 17, + "mutation": "ALTER TABLE byod_import_jobs ALTER COLUMN zid SET DEFAULT 7;", + "expected": "f" + }, + { + "name": "default_added_byod_import_jobs_s3_key", + "migration": 17, + "mutation": "ALTER TABLE byod_import_jobs ALTER COLUMN s3_key SET DEFAULT 'constructed';", + "expected": "f" + }, + { + "name": "default_removed_byod_import_jobs_status", + "migration": 17, + "mutation": "ALTER TABLE byod_import_jobs ALTER COLUMN status DROP DEFAULT;", + "expected": "f" + }, + { + "name": "nullability_byod_import_jobs_status", + "migration": 17, + "mutation": "ALTER TABLE byod_import_jobs ALTER COLUMN status SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_removed_byod_import_jobs_stage", + "migration": 17, + "mutation": "ALTER TABLE byod_import_jobs ALTER COLUMN stage DROP DEFAULT;", + "expected": "f" + }, + { + "name": "nullability_byod_import_jobs_stage", + "migration": 17, + "mutation": "ALTER TABLE byod_import_jobs ALTER COLUMN stage SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_added_byod_import_jobs_error_message", + "migration": 17, + "mutation": "ALTER TABLE byod_import_jobs ALTER COLUMN error_message SET DEFAULT 'constructed';", + "expected": "f" + }, + { + "name": "nullability_byod_import_jobs_error_message", + "migration": 17, + "mutation": "ALTER TABLE byod_import_jobs ALTER COLUMN error_message SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_removed_byod_import_jobs_created_at", + "migration": 17, + "mutation": "ALTER TABLE byod_import_jobs ALTER COLUMN created_at DROP DEFAULT;", + "expected": "f" + }, + { + "name": "nullability_byod_import_jobs_created_at", + "migration": 17, + "mutation": "ALTER TABLE byod_import_jobs ALTER COLUMN created_at SET NOT NULL;", + "expected": "f" + }, + { + "name": "default_removed_byod_import_jobs_updated_at", + "migration": 17, + "mutation": "ALTER TABLE byod_import_jobs ALTER COLUMN updated_at DROP DEFAULT;", + "expected": "f" + }, + { + "name": "nullability_byod_import_jobs_updated_at", + "migration": 17, + "mutation": "ALTER TABLE byod_import_jobs ALTER COLUMN updated_at SET NOT NULL;", + "expected": "f" + }, + { + "name": "canonical_18", + "migration": 18, + "mutation": "", + "expected": "t" + }, + { + "name": "default_removed_conversations_topics_enabled", + "migration": 18, + "mutation": "ALTER TABLE conversations ALTER COLUMN topics_enabled DROP DEFAULT;", + "expected": "f" + }, + { + "name": "missing_oidc_user_mappings_pkey", + "migration": 10, + "mutation": "ALTER TABLE oidc_user_mappings DROP CONSTRAINT oidc_user_mappings_pkey CASCADE;", + "expected": "f" + }, + { + "name": "missing_oidc_user_mappings_uid_fkey", + "migration": 10, + "mutation": "ALTER TABLE oidc_user_mappings DROP CONSTRAINT oidc_user_mappings_uid_fkey CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_oidc_user_mappings_uid_fkey", + "migration": 10, + "mutation": "ALTER TABLE oidc_user_mappings DROP CONSTRAINT oidc_user_mappings_uid_fkey CASCADE; ALTER TABLE oidc_user_mappings ADD CONSTRAINT oidc_user_mappings_uid_fkey FOREIGN KEY (uid) REFERENCES users(uid) ON DELETE CASCADE NOT VALID;", + "expected": "f" + }, + { + "name": "deferred_oidc_user_mappings_uid_fkey", + "migration": 10, + "mutation": "ALTER TABLE oidc_user_mappings ALTER CONSTRAINT oidc_user_mappings_uid_fkey DEFERRABLE INITIALLY DEFERRED;", + "expected": "f" + }, + { + "name": "missing_oidc_user_mappings_uid_key", + "migration": 10, + "mutation": "ALTER TABLE oidc_user_mappings DROP CONSTRAINT oidc_user_mappings_uid_key CASCADE;", + "expected": "f" + }, + { + "name": "disabled_triggers_oidc_user_mappings", + "migration": 10, + "mutation": "ALTER TABLE oidc_user_mappings DISABLE TRIGGER ALL;", + "expected": "f" + }, + { + "name": "missing_fk_conversation", + "migration": 12, + "mutation": "ALTER TABLE topic_agenda_selections DROP CONSTRAINT fk_conversation CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_fk_conversation", + "migration": 12, + "mutation": "ALTER TABLE topic_agenda_selections DROP CONSTRAINT fk_conversation CASCADE; ALTER TABLE topic_agenda_selections ADD CONSTRAINT fk_conversation FOREIGN KEY (zid) REFERENCES conversations(zid) ON DELETE CASCADE NOT VALID;", + "expected": "f" + }, + { + "name": "deferred_fk_conversation", + "migration": 12, + "mutation": "ALTER TABLE topic_agenda_selections ALTER CONSTRAINT fk_conversation DEFERRABLE INITIALLY DEFERRED;", + "expected": "f" + }, + { + "name": "missing_fk_participant", + "migration": 12, + "mutation": "ALTER TABLE topic_agenda_selections DROP CONSTRAINT fk_participant CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_fk_participant", + "migration": 12, + "mutation": "ALTER TABLE topic_agenda_selections DROP CONSTRAINT fk_participant CASCADE; ALTER TABLE topic_agenda_selections ADD CONSTRAINT fk_participant FOREIGN KEY (zid, pid) REFERENCES participants(zid, pid) ON DELETE CASCADE NOT VALID;", + "expected": "f" + }, + { + "name": "deferred_fk_participant", + "migration": 12, + "mutation": "ALTER TABLE topic_agenda_selections ALTER CONSTRAINT fk_participant DEFERRABLE INITIALLY DEFERRED;", + "expected": "f" + }, + { + "name": "missing_topic_agenda_selections_pkey", + "migration": 12, + "mutation": "ALTER TABLE topic_agenda_selections DROP CONSTRAINT topic_agenda_selections_pkey CASCADE;", + "expected": "f" + }, + { + "name": "disabled_triggers_topic_agenda_selections", + "migration": 12, + "mutation": "ALTER TABLE topic_agenda_selections DISABLE TRIGGER ALL;", + "expected": "f" + }, + { + "name": "missing_treevite_waves_invites_per_user_check", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves DROP CONSTRAINT treevite_waves_invites_per_user_check CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_treevite_waves_invites_per_user_check", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves DROP CONSTRAINT treevite_waves_invites_per_user_check CASCADE; ALTER TABLE treevite_waves ADD CONSTRAINT treevite_waves_invites_per_user_check CHECK ((invites_per_user >= 0)) NOT VALID;", + "expected": "f" + }, + { + "name": "missing_treevite_waves_not_both_zero", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves DROP CONSTRAINT treevite_waves_not_both_zero CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_treevite_waves_not_both_zero", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves DROP CONSTRAINT treevite_waves_not_both_zero CASCADE; ALTER TABLE treevite_waves ADD CONSTRAINT treevite_waves_not_both_zero CHECK (((invites_per_user > 0) OR (owner_invites > 0))) NOT VALID;", + "expected": "f" + }, + { + "name": "missing_treevite_waves_owner_invites_check", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves DROP CONSTRAINT treevite_waves_owner_invites_check CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_treevite_waves_owner_invites_check", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves DROP CONSTRAINT treevite_waves_owner_invites_check CASCADE; ALTER TABLE treevite_waves ADD CONSTRAINT treevite_waves_owner_invites_check CHECK ((owner_invites >= 0)) NOT VALID;", + "expected": "f" + }, + { + "name": "missing_treevite_waves_parent_wave_check", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves DROP CONSTRAINT treevite_waves_parent_wave_check CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_treevite_waves_parent_wave_check", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves DROP CONSTRAINT treevite_waves_parent_wave_check CASCADE; ALTER TABLE treevite_waves ADD CONSTRAINT treevite_waves_parent_wave_check CHECK (((parent_wave IS NULL) OR (parent_wave >= 0))) NOT VALID;", + "expected": "f" + }, + { + "name": "missing_treevite_waves_pkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves DROP CONSTRAINT treevite_waves_pkey CASCADE;", + "expected": "f" + }, + { + "name": "missing_treevite_waves_size_check", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves DROP CONSTRAINT treevite_waves_size_check CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_treevite_waves_size_check", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves DROP CONSTRAINT treevite_waves_size_check CASCADE; ALTER TABLE treevite_waves ADD CONSTRAINT treevite_waves_size_check CHECK (((size IS NULL) OR (size >= 0))) NOT VALID;", + "expected": "f" + }, + { + "name": "missing_treevite_waves_wave_check", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves DROP CONSTRAINT treevite_waves_wave_check CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_treevite_waves_wave_check", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves DROP CONSTRAINT treevite_waves_wave_check CASCADE; ALTER TABLE treevite_waves ADD CONSTRAINT treevite_waves_wave_check CHECK ((wave >= 1)) NOT VALID;", + "expected": "f" + }, + { + "name": "missing_treevite_waves_zid_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves DROP CONSTRAINT treevite_waves_zid_fkey CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_treevite_waves_zid_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves DROP CONSTRAINT treevite_waves_zid_fkey CASCADE; ALTER TABLE treevite_waves ADD CONSTRAINT treevite_waves_zid_fkey FOREIGN KEY (zid) REFERENCES conversations(zid) ON DELETE CASCADE NOT VALID;", + "expected": "f" + }, + { + "name": "deferred_treevite_waves_zid_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves ALTER CONSTRAINT treevite_waves_zid_fkey DEFERRABLE INITIALLY DEFERRED;", + "expected": "f" + }, + { + "name": "missing_treevite_waves_zid_wave_key", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves DROP CONSTRAINT treevite_waves_zid_wave_key CASCADE;", + "expected": "f" + }, + { + "name": "disabled_triggers_treevite_waves", + "migration": 13, + "mutation": "ALTER TABLE treevite_waves DISABLE TRIGGER ALL;", + "expected": "f" + }, + { + "name": "missing_treevite_invites_code_unique", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites DROP CONSTRAINT treevite_invites_code_unique CASCADE;", + "expected": "f" + }, + { + "name": "missing_treevite_invites_owner_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites DROP CONSTRAINT treevite_invites_owner_fkey CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_treevite_invites_owner_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites DROP CONSTRAINT treevite_invites_owner_fkey CASCADE; ALTER TABLE treevite_invites ADD CONSTRAINT treevite_invites_owner_fkey FOREIGN KEY (zid, invite_owner_pid) REFERENCES participants(zid, pid) NOT VALID;", + "expected": "f" + }, + { + "name": "deferred_treevite_invites_owner_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER CONSTRAINT treevite_invites_owner_fkey DEFERRABLE INITIALLY DEFERRED;", + "expected": "f" + }, + { + "name": "missing_treevite_invites_parent_invite_id_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites DROP CONSTRAINT treevite_invites_parent_invite_id_fkey CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_treevite_invites_parent_invite_id_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites DROP CONSTRAINT treevite_invites_parent_invite_id_fkey CASCADE; ALTER TABLE treevite_invites ADD CONSTRAINT treevite_invites_parent_invite_id_fkey FOREIGN KEY (parent_invite_id) REFERENCES treevite_invites(id) ON DELETE SET NULL NOT VALID;", + "expected": "f" + }, + { + "name": "deferred_treevite_invites_parent_invite_id_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER CONSTRAINT treevite_invites_parent_invite_id_fkey DEFERRABLE INITIALLY DEFERRED;", + "expected": "f" + }, + { + "name": "missing_treevite_invites_pkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites DROP CONSTRAINT treevite_invites_pkey CASCADE;", + "expected": "f" + }, + { + "name": "missing_treevite_invites_status_check", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites DROP CONSTRAINT treevite_invites_status_check CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_treevite_invites_status_check", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites DROP CONSTRAINT treevite_invites_status_check CASCADE; ALTER TABLE treevite_invites ADD CONSTRAINT treevite_invites_status_check CHECK ((status = ANY (ARRAY[0, 1, 2, 3]))) NOT VALID;", + "expected": "f" + }, + { + "name": "missing_treevite_invites_used_by_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites DROP CONSTRAINT treevite_invites_used_by_fkey CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_treevite_invites_used_by_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites DROP CONSTRAINT treevite_invites_used_by_fkey CASCADE; ALTER TABLE treevite_invites ADD CONSTRAINT treevite_invites_used_by_fkey FOREIGN KEY (zid, invite_used_by_pid) REFERENCES participants(zid, pid) NOT VALID;", + "expected": "f" + }, + { + "name": "deferred_treevite_invites_used_by_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER CONSTRAINT treevite_invites_used_by_fkey DEFERRABLE INITIALLY DEFERRED;", + "expected": "f" + }, + { + "name": "missing_treevite_invites_wave_id_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites DROP CONSTRAINT treevite_invites_wave_id_fkey CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_treevite_invites_wave_id_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites DROP CONSTRAINT treevite_invites_wave_id_fkey CASCADE; ALTER TABLE treevite_invites ADD CONSTRAINT treevite_invites_wave_id_fkey FOREIGN KEY (wave_id) REFERENCES treevite_waves(id) ON DELETE CASCADE NOT VALID;", + "expected": "f" + }, + { + "name": "deferred_treevite_invites_wave_id_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER CONSTRAINT treevite_invites_wave_id_fkey DEFERRABLE INITIALLY DEFERRED;", + "expected": "f" + }, + { + "name": "missing_treevite_invites_zid_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites DROP CONSTRAINT treevite_invites_zid_fkey CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_treevite_invites_zid_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites DROP CONSTRAINT treevite_invites_zid_fkey CASCADE; ALTER TABLE treevite_invites ADD CONSTRAINT treevite_invites_zid_fkey FOREIGN KEY (zid) REFERENCES conversations(zid) ON DELETE CASCADE NOT VALID;", + "expected": "f" + }, + { + "name": "deferred_treevite_invites_zid_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites ALTER CONSTRAINT treevite_invites_zid_fkey DEFERRABLE INITIALLY DEFERRED;", + "expected": "f" + }, + { + "name": "disabled_triggers_treevite_invites", + "migration": 13, + "mutation": "ALTER TABLE treevite_invites DISABLE TRIGGER ALL;", + "expected": "f" + }, + { + "name": "missing_treevite_login_codes_fp_unique", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes DROP CONSTRAINT treevite_login_codes_fp_unique CASCADE;", + "expected": "f" + }, + { + "name": "missing_treevite_login_codes_lookup_unique", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes DROP CONSTRAINT treevite_login_codes_lookup_unique CASCADE;", + "expected": "f" + }, + { + "name": "missing_treevite_login_codes_participant_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes DROP CONSTRAINT treevite_login_codes_participant_fkey CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_treevite_login_codes_participant_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes DROP CONSTRAINT treevite_login_codes_participant_fkey CASCADE; ALTER TABLE treevite_login_codes ADD CONSTRAINT treevite_login_codes_participant_fkey FOREIGN KEY (zid, pid) REFERENCES participants(zid, pid) ON DELETE CASCADE NOT VALID;", + "expected": "f" + }, + { + "name": "deferred_treevite_login_codes_participant_fkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes ALTER CONSTRAINT treevite_login_codes_participant_fkey DEFERRABLE INITIALLY DEFERRED;", + "expected": "f" + }, + { + "name": "missing_treevite_login_codes_pid_unique", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes DROP CONSTRAINT treevite_login_codes_pid_unique CASCADE;", + "expected": "f" + }, + { + "name": "missing_treevite_login_codes_pkey", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes DROP CONSTRAINT treevite_login_codes_pkey CASCADE;", + "expected": "f" + }, + { + "name": "disabled_triggers_treevite_login_codes", + "migration": 13, + "mutation": "ALTER TABLE treevite_login_codes DISABLE TRIGGER ALL;", + "expected": "f" + }, + { + "name": "missing_xid_whitelist_zid_fkey", + "migration": 15, + "mutation": "ALTER TABLE xid_whitelist DROP CONSTRAINT xid_whitelist_zid_fkey CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_xid_whitelist_zid_fkey", + "migration": 15, + "mutation": "ALTER TABLE xid_whitelist DROP CONSTRAINT xid_whitelist_zid_fkey CASCADE; ALTER TABLE xid_whitelist ADD CONSTRAINT xid_whitelist_zid_fkey FOREIGN KEY (zid) REFERENCES conversations(zid) ON DELETE CASCADE NOT VALID;", + "expected": "f" + }, + { + "name": "deferred_xid_whitelist_zid_fkey", + "migration": 15, + "mutation": "ALTER TABLE xid_whitelist ALTER CONSTRAINT xid_whitelist_zid_fkey DEFERRABLE INITIALLY DEFERRED;", + "expected": "f" + }, + { + "name": "disabled_triggers_xid_whitelist", + "migration": 15, + "mutation": "ALTER TABLE xid_whitelist DISABLE TRIGGER ALL;", + "expected": "f" + }, + { + "name": "missing_xids_zid_fkey", + "migration": 15, + "mutation": "ALTER TABLE xids DROP CONSTRAINT xids_zid_fkey CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_xids_zid_fkey", + "migration": 15, + "mutation": "ALTER TABLE xids DROP CONSTRAINT xids_zid_fkey CASCADE; ALTER TABLE xids ADD CONSTRAINT xids_zid_fkey FOREIGN KEY (zid) REFERENCES conversations(zid) ON DELETE CASCADE NOT VALID;", + "expected": "f" + }, + { + "name": "deferred_xids_zid_fkey", + "migration": 15, + "mutation": "ALTER TABLE xids ALTER CONSTRAINT xids_zid_fkey DEFERRABLE INITIALLY DEFERRED;", + "expected": "f" + }, + { + "name": "missing_xids_pid_fkey", + "migration": 15, + "mutation": "ALTER TABLE xids DROP CONSTRAINT xids_pid_fkey CASCADE;", + "expected": "f" + }, + { + "name": "unvalidated_xids_pid_fkey", + "migration": 15, + "mutation": "ALTER TABLE xids DROP CONSTRAINT xids_pid_fkey CASCADE; ALTER TABLE xids ADD CONSTRAINT xids_pid_fkey FOREIGN KEY (zid, pid) REFERENCES participants(zid, pid) ON DELETE SET NULL NOT VALID;", + "expected": "f" + }, + { + "name": "deferred_xids_pid_fkey", + "migration": 15, + "mutation": "ALTER TABLE xids ALTER CONSTRAINT xids_pid_fkey DEFERRABLE INITIALLY DEFERRED;", + "expected": "f" + }, + { + "name": "disabled_triggers_xids", + "migration": 15, + "mutation": "ALTER TABLE xids DISABLE TRIGGER ALL;", + "expected": "f" + }, + { + "name": "missing_byod_import_jobs_pkey", + "migration": 17, + "mutation": "ALTER TABLE byod_import_jobs DROP CONSTRAINT byod_import_jobs_pkey CASCADE;", + "expected": "f" + }, + { + "name": "deferred_unique", + "migration": 10, + "mutation": "ALTER TABLE oidc_user_mappings DROP CONSTRAINT oidc_user_mappings_uid_key; ALTER TABLE oidc_user_mappings ADD CONSTRAINT generated_uid_unique UNIQUE(uid) DEFERRABLE INITIALLY DEFERRED;", + "expected": "f" + }, + { + "name": "m3_wrong_type", + "migration": 3, + "mutation": "ALTER TABLE participants_extended ALTER COLUMN origin TYPE text;", + "expected": "f" + }, + { + "name": "m11_preserved_default", + "migration": 11, + "mutation": "ALTER TABLE suzinvites ALTER COLUMN xid SET DEFAULT 'constructed';", + "expected": "t" + }, + { + "name": "renamed_equivalent_unique", + "migration": 10, + "mutation": "ALTER TABLE oidc_user_mappings RENAME CONSTRAINT oidc_user_mappings_uid_key TO generated_equivalent_key;", + "expected": "t" + }, + { + "name": "m3_generated_column", + "migration": 3, + "mutation": "ALTER TABLE participants_extended DROP COLUMN origin; ALTER TABLE participants_extended ADD COLUMN origin varchar(9999) GENERATED ALWAYS AS ('constructed'::varchar(9999)) STORED;", + "expected": "f" + }, + { + "name": "m17_identity_column", + "migration": 17, + "mutation": "ALTER TABLE byod_import_jobs ALTER COLUMN id DROP DEFAULT; ALTER TABLE byod_import_jobs ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY;", + "expected": "f" + } +] diff --git a/queue-rs/polis-migrate/tests/adoption.py b/queue-rs/polis-migrate/tests/adoption.py new file mode 100644 index 0000000000..07a08fa04c --- /dev/null +++ b/queue-rs/polis-migrate/tests/adoption.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +"""Persistent adoption-contract regression tests; generated DBs and no vote rows. + +Run after starting tests/compose.yml with unique COMPOSE_PROJECT_NAME and ports. +Uses the built polis-migrate in this checkout; no packet or original source needed. +""" +import json, pathlib, unittest +from prove import MIG, runner, sql + +class Adoption(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.db='adoption_contract_template' + sql('postgres',f'CREATE DATABASE {cls.db}') + for path in sorted(MIG.glob('*.sql')): + if int(path.name[:6]) <= 18 or path.name.startswith('000022_'): + sql(cls.db,path.read_text()) + sql(cls.db,"INSERT INTO users(hname) VALUES('constructed adoption sentinel')") + + def test_01_catalog_controls(self): + cases=json.loads(pathlib.Path(__file__).with_name('adoption-cases.json').read_text()) + helpers=(MIG/'adoption/helpers.sql').read_text() + for case in cases: + with self.subTest(case=case['name']): + predicate=next((MIG/'adoption').glob(f"{case['migration']:06d}_*.sql")).read_text() + out=sql(self.db,'BEGIN;\n'+case['mutation']+'\n'+helpers+'\n'+predicate+'\nROLLBACK;') + # psql also prints transaction/function command tags. + values=[line for line in out.splitlines() if line in ('t','f')] + self.assertEqual(values,[case['expected']]) + + def test_02_conflicts_leave_no_history(self): + cases=[(3,"ALTER TABLE participants_extended ALTER COLUMN origin SET DEFAULT 'constructed';"), + (13,'ALTER TABLE treevite_invites DISABLE TRIGGER ALL;'), + (17,'ALTER TABLE byod_import_jobs ALTER COLUMN id DROP DEFAULT;')] + for n,mutation in cases: + with self.subTest(migration=n): + db=f'adoption_contract_conflict_{n}' + sql('postgres',f'CREATE DATABASE {db} TEMPLATE {self.db}') + sql(db,mutation) + p=runner(db,'reconcile','--through','000022',ok=False) + self.assertIn(f'{n:06d}_',p.stderr) + self.assertEqual(sql(db,"SELECT to_regclass('public.migrations') IS NULL"),'t') + + def test_03_canonical_adoption(self): + db='adoption_contract_success' + sql('postgres',f'CREATE DATABASE {db} TEMPLATE {self.db}') + runner(db,'reconcile','--through','000022') + self.assertEqual(sql(db,"SELECT count(*) FROM migrations WHERE status='ADOPTED'"),'20') + self.assertEqual(sql(db,'SELECT hname FROM users'),'constructed adoption sentinel') + +if __name__=='__main__': unittest.main(verbosity=2) diff --git a/queue-rs/polis-migrate/tests/compose.yml b/queue-rs/polis-migrate/tests/compose.yml new file mode 100644 index 0000000000..fe26fd80db --- /dev/null +++ b/queue-rs/polis-migrate/tests/compose.yml @@ -0,0 +1,14 @@ +services: + postgres: + image: postgres:17-alpine + environment: + POSTGRES_HOST_AUTH_METHOD: trust + POSTGRES_USER: postgres + POSTGRES_DB: postgres + ports: + - "127.0.0.1:${POLIS_RECOVERY_PG_PORT:?set an owned port}:5432" + healthcheck: + test: [CMD-SHELL, "pg_isready -U postgres"] + interval: 1s + timeout: 3s + retries: 30 diff --git a/queue-rs/polis-migrate/tests/fresh-image.sh b/queue-rs/polis-migrate/tests/fresh-image.sh new file mode 100644 index 0000000000..80f4c41d97 --- /dev/null +++ b/queue-rs/polis-migrate/tests/fresh-image.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# CI and work boxes run this same build/bootstrap/restart proof. +set -euo pipefail +cd "$(dirname "$0")/../../.." +case "${COMPOSE_PROJECT_NAME:-}" in + polis-migrate-test-?*) ;; + *) echo 'Set an owned COMPOSE_PROJECT_NAME starting polis-migrate-test-' >&2; exit 2 ;; +esac +: "${POLIS_RECOVERY_PG_PORT:?set an owned port}" +export RECOVERY_PG_PORT="$POLIS_RECOVERY_PG_PORT" +export POLIS_MIGRATE_TEST_IMAGE="${COMPOSE_PROJECT_NAME}:fresh-image" +compose=(docker compose -f queue-rs/polis-migrate/tests/compose.yml -f queue-rs/polis-migrate/tests/image.yml) +cleanup() { + local status=$? + trap - EXIT + "${compose[@]}" down -v || status=1 + if docker image inspect "$POLIS_MIGRATE_TEST_IMAGE" >/dev/null 2>&1; then + docker image rm "$POLIS_MIGRATE_TEST_IMAGE" || status=1 + fi + exit "$status" +} +trap cleanup EXIT +python3 queue-rs/polis-migrate/tests/image-proof-test.py +"${compose[@]}" down -v +docker build --build-context queue-rs=queue-rs -t "$POLIS_MIGRATE_TEST_IMAGE" -f server/Dockerfile-db server +"${compose[@]}" up -d --wait --wait-timeout 120 +python3 queue-rs/polis-migrate/tests/image-proof.py diff --git a/queue-rs/polis-migrate/tests/image-proof-test.py b/queue-rs/polis-migrate/tests/image-proof-test.py new file mode 100644 index 0000000000..be4181fe9c --- /dev/null +++ b/queue-rs/polis-migrate/tests/image-proof-test.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""Regression controls for fresh-image release evolution and corrupt receipts.""" +import importlib.util +import pathlib +import tempfile +import unittest + +spec = importlib.util.spec_from_file_location( + "image_proof", pathlib.Path(__file__).with_name("image-proof.py")) +proof = importlib.util.module_from_spec(spec) +spec.loader.exec_module(proof) + + +class ImageProofTests(unittest.TestCase): + def setUp(self): + self.expected = proof.expected_receipts(proof.MIGRATIONS) + self.rows = [{"name": name, **receipt} for name, receipt in self.expected.items()] + + def test_current_release_includes_adoption(self): + proof.assert_receipts(self.rows, self.expected) + self.assertEqual({row["name"][:6] for row in self.rows if row["status"] == "ADOPTED"}, + proof.RETIRED) + self.assertTrue(any(row["status"] == "APPLIED" for row in self.rows)) + + def test_same_count_wrong_status_refuses(self): + next(row for row in self.rows if row["status"] == "ADOPTED")["status"] = "APPLIED" + with self.assertRaises(AssertionError): + proof.assert_receipts(self.rows, self.expected) + + def test_same_count_wrong_name_refuses(self): + self.rows[0]["name"] = "999999_unselected.sql" + with self.assertRaises(AssertionError): + proof.assert_receipts(self.rows, self.expected) + + def test_same_count_wrong_checksum_refuses(self): + self.rows[0]["checksum"] = "0" * 64 + with self.assertRaises(AssertionError): + proof.assert_receipts(self.rows, self.expected) + + def test_missing_receipt_refuses(self): + with self.assertRaises(AssertionError): + proof.assert_receipts(self.rows[1:], self.expected) + + def test_extra_receipt_refuses(self): + with self.assertRaises(AssertionError): + proof.assert_receipts(self.rows + [{**self.rows[0], "name": "999999_extra.sql"}], + self.expected) + + def test_duplicate_receipt_refuses(self): + with self.assertRaises(AssertionError): + proof.assert_receipts(self.rows + [self.rows[0]], self.expected) + + def test_release_grows_and_shrinks_without_count_edits(self): + with tempfile.TemporaryDirectory() as tmp: + directory = pathlib.Path(tmp) + names = ["000000_initial.sql", "000004_drop_waitinglist_table.sql", + "000030_future.sql"] + for name in names: + (directory / name).write_text("-- constructed SQL source\n") + for selected in (names[:2], names, names[:1]): + with self.subTest(selected=selected): + (directory / "release.txt").write_text( + "# constructed release selection\n\n" + "\n".join(selected) + "\n") + expected = proof.expected_receipts(directory) + self.assertEqual(set(expected), set(selected)) + self.assertEqual(len(expected), len(selected)) + proof.assert_receipts([{"name": name, **receipt} + for name, receipt in expected.items()], expected) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/queue-rs/polis-migrate/tests/image-proof.py b/queue-rs/polis-migrate/tests/image-proof.py new file mode 100644 index 0000000000..76f6453aef --- /dev/null +++ b/queue-rs/polis-migrate/tests/image-proof.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""Check the selected fresh-image receipts and preserve them across restart.""" +import collections +import hashlib +import json +import os +import pathlib +import re +import subprocess + + +# Observation-only retirement policy in polis_migrate::load/retired_absent. +# These are migration identities, not a count of the current release. +RETIRED = {"000004", "000005", "000007"} +ROOT = pathlib.Path(__file__).resolve().parents[3] +MIGRATIONS = ROOT / "server/postgres/migrations" + + +def expected_receipts(directory): + names = [line for line in (directory / "release.txt").read_text().splitlines() + if line and not line.startswith("#")] + assert names and len(names) == len(set(names)), "empty/duplicate release selection" + assert all(re.fullmatch(r"[0-9]{6}_[A-Za-z0-9_]+\.sql", name) for name in names) + return { + name: {"status": "ADOPTED" if name[:6] in RETIRED else "APPLIED", + "checksum": hashlib.sha256((directory / name).read_bytes()).hexdigest()} + for name in names + } + + +def assert_receipts(rows, expected): + actual = {row["name"]: {"status": row["status"], "checksum": row["checksum"]} + for row in rows} + assert len(rows) == len(actual), "duplicate history names" + assert actual == expected, f"fresh history differs from release: {actual!r} != {expected!r}" + + +def main(): + assert os.environ.get("COMPOSE_PROJECT_NAME", "").startswith("polis-migrate-test-") + directory = pathlib.Path(__file__).resolve().parent + compose = ["docker", "compose", "-f", str(directory / "compose.yml"), + "-f", str(directory / "image.yml")] + + def run(*args): + return subprocess.check_output(compose + list(args), text=True).strip() + + def query(sql): + return run("exec", "-T", "postgres", "psql", "-X", "-v", "ON_ERROR_STOP=1", + "-U", "postgres", "-d", "postgres", "-Atc", sql) + + def history(): + return [json.loads(line) for line in + query("SELECT row_to_json(m) FROM migrations m ORDER BY name").splitlines()] + + expected = expected_receipts(MIGRATIONS) + before = history() + assert_receipts(before, expected) + assert query("SELECT to_regclass('public.polis_coordinator_install') IS NULL") == "t" + run("restart", "postgres") + run("up", "-d", "--wait", "--wait-timeout", "120") + assert history() == before, "restart changed migration receipts" + assert run("exec", "-T", "-e", + "DATABASE_URL=host=/var/run/postgresql user=postgres dbname=postgres sslmode=disable", + "-e", "POLIS_MIGRATIONS_DIR=/migrations", "postgres", "polis-migrate", "check" + ) == f"migration check: {len(expected)} ready" + counts = dict(sorted(collections.Counter(row["status"] for row in before).items())) + print(f"image proof: 4 checks PASS; {len(expected)} selected receipts; {counts}") + + +if __name__ == "__main__": + main() diff --git a/queue-rs/polis-migrate/tests/image.yml b/queue-rs/polis-migrate/tests/image.yml new file mode 100644 index 0000000000..dcaa565723 --- /dev/null +++ b/queue-rs/polis-migrate/tests/image.yml @@ -0,0 +1,7 @@ +services: + postgres: + image: ${POLIS_MIGRATE_TEST_IMAGE:?run fresh-image.sh with an owned project} + healthcheck: + # The initdb temporary server accepts socket connections before the runner + # has finished. TCP readiness waits for the final server after initialization. + test: [CMD-SHELL, "pg_isready -h 127.0.0.1 -U postgres"] diff --git a/queue-rs/polis-migrate/tests/indexes.py b/queue-rs/polis-migrate/tests/indexes.py new file mode 100644 index 0000000000..657ffd6660 --- /dev/null +++ b/queue-rs/polis-migrate/tests/indexes.py @@ -0,0 +1,84 @@ +"""Real generated Postgres runner controls. No vote rows are inserted.""" +import importlib.util,json,os,pathlib,shutil,subprocess,tempfile,time,unittest +ROOT=pathlib.Path(__file__).resolve().parents[3] +spec=importlib.util.spec_from_file_location('proof',ROOT/'queue-rs/polis-migrate/tests/prove.py');p=importlib.util.module_from_spec(spec);spec.loader.exec_module(p) +ORIGINAL=os.environ.get('ORIGINAL_RUNNER') +class IndexProof(unittest.TestCase): + def setUp(self): + self.tmp=tempfile.TemporaryDirectory();self.addCleanup(self.tmp.cleanup);self.d=pathlib.Path(self.tmp.name) + (self.d/'held.txt').write_text('') + self.m=p.MIG/'000022_add_poll_timestamp_indexes.sql';shutil.copy(self.m,self.d/self.m.name) + def db(self,name,extra=''): + db='idx_'+name;p.sql('postgres',f'CREATE DATABASE {db}') + (self.d/'000000_initial.sql').write_text('CREATE TABLE public.votes(created bigint); CREATE TABLE public.comments(modified bigint, other integer);'+extra) + return db + def runnew(self,db,*args,**kw): + (self.d/'release.txt').write_text(''.join(f.name+'\n' for f in sorted(self.d.glob('*.sql')))) + return p.runner(db,*args,dir=self.d,**kw) + def test_01_original_refuses_large_candidate_passes(self): + db=self.db('large','INSERT INTO public.comments SELECT x,0 FROM generate_series(1,100001) x;') + if ORIGINAL: + before=subprocess.run([ORIGINAL,'apply','--dir',str(self.d)],env=p.env(db),text=True,capture_output=True,timeout=60) + self.assertNotEqual(before.returncode,0);self.assertIn('000022',before.stderr) + self.assertEqual(p.sql(db,"SELECT count(*) FROM migrations WHERE name LIKE '000022%'"),'0') + self.assertIn('BUILDING CONCURRENTLY',self.runnew(db,'apply').stdout) + self.runnew(db,'check');self.assertEqual(p.sql(db,'SELECT count(*) FROM comments'),'100001') + self.assertEqual(p.sql(db,'SELECT count(*) FROM votes'),'0') + def test_02_partial_success_resumes_without_rebuild(self): + db=self.db('partial','CREATE INDEX votes_created_idx ON public.votes(created);') + # Bootstrap only first to observe its OID before the pending index runs. + m=self.d/self.m.name;m.rename(self.d/'index.pending') + self.runnew(db,'apply');oid=p.sql(db,"SELECT 'public.votes_created_idx'::regclass::oid") + (self.d/'index.pending').rename(m) + self.runnew(db,'apply');self.assertEqual(oid,p.sql(db,"SELECT 'public.votes_created_idx'::regclass::oid")) + self.assertIn('applied 0 migration(s)',self.runnew(db,'apply').stdout) + def test_03_collision_preflight_preserves_objects(self): + db=self.db('collision','CREATE INDEX comments_modified_idx ON public.comments(other);') + r=self.runnew(db,'apply',ok=False);self.assertIn('conflicting object',r.stderr) + self.assertEqual(p.sql(db,"SELECT to_regclass('public.votes_created_idx') IS NULL"),'t') + self.assertEqual(p.sql(db,"SELECT count(*) FROM migrations WHERE name LIKE '000022%'"),'0') + def test_04_history_failure_resumes_committed_indexes(self): + db=self.db('history',"CREATE FUNCTION public.refuse_index_record() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.name LIKE '000022%' THEN RAISE EXCEPTION 'generated history fault'; END IF; RETURN NEW; END $$; CREATE TRIGGER refuse_index_record BEFORE INSERT ON migrations FOR EACH ROW EXECUTE FUNCTION public.refuse_index_record();") + self.runnew(db,'apply',ok=False) + before=p.sql(db,"SELECT indexrelid FROM pg_index WHERE indrelid IN ('votes'::regclass,'comments'::regclass) ORDER BY indexrelid") + self.assertEqual(len(before.splitlines()),2) + self.assertEqual(p.sql(db,"SELECT count(*) FROM migrations WHERE name LIKE '000022%'"),'0') + p.sql(db,'DROP TRIGGER refuse_index_record ON migrations');self.runnew(db,'apply') + self.assertEqual(before,p.sql(db,"SELECT indexrelid FROM pg_index WHERE indrelid IN ('votes'::regclass,'comments'::regclass) ORDER BY indexrelid")) + def holder(self,db): + q=subprocess.Popen(p.COMPOSE+['exec','-T','postgres','psql','-X','-U','postgres','-d',db,'-Atq'],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True) + q.stdin.write("BEGIN; INSERT INTO comments VALUES(1,1); SELECT 'held';\n");q.stdin.flush();self.assertEqual(q.stdout.readline().strip(),'held');return q + def release(self,q): + q.stdin.write('COMMIT;\n\\q\n');q.stdin.flush();q.communicate(timeout=15) + def wait_index(self,db): + deadline=time.monotonic()+15 + while time.monotonic()18 and not s.startswith('000022_')] +BIN=ROOT/'queue-rs/target/debug/polis-migrate' +COMPOSE=['docker','compose','-f',str(pathlib.Path(__file__).with_name('compose.yml'))] +assert os.environ.get('COMPOSE_PROJECT_NAME','').startswith('polis-migrate-test-') +PORT=os.environ['POLIS_RECOVERY_PG_PORT'] +assert os.environ['RECOVERY_PG_PORT']==PORT + +def sql(db,query): + p=subprocess.run(COMPOSE+['exec','-T','postgres','psql','-X','-v','ON_ERROR_STOP=1','-U','postgres','-d',db,'-At'],input=query,text=True,capture_output=True) + if p.returncode: raise AssertionError(p.stderr) + return p.stdout.strip() +def env(db): return dict(os.environ,DATABASE_URL=f'postgresql://postgres@127.0.0.1:{PORT}/{db}?sslmode=disable') +def runner(db,*args,dir=MIG,ok=True): + p=subprocess.run([str(BIN),*args,'--dir',str(dir)],env=env(db),text=True,capture_output=True,timeout=350) + if (p.returncode==0)!=ok: raise AssertionError(f'{args} exit={p.returncode}\n{p.stdout}\n{p.stderr}') + return p + +def gate(db,ok,dir=MIG): + # The actual module imported first by server/index.ts; pg uses generated loopback DB. + p=subprocess.run(['node',str(ROOT/('server/src/db/migrations.cjs' if ok else 'server/dist/index.js'))],env=dict(env(db),POLIS_MIGRATIONS_DIR=str(dir)),text=True,capture_output=True,timeout=30) + if (p.returncode==0)!=ok: raise AssertionError(f'Node gate {p.returncode}: {p.stdout} {p.stderr}') + if not ok: + assert 'Server startup refused:' in p.stderr and 'MODULE_NOT_FOUND' not in p.stderr, p.stderr + return p + +class Proof(unittest.TestCase): + def new(self,name): + db='migrate_'+name + sql('postgres',f'CREATE DATABASE {db}') + return db + def legacy(self,name): + db=self.new(name) + for p in sorted(MIG.glob('*.sql')): + if int(p.name[:6])<=18 or p.name.startswith('000022_'): sql(db,p.read_text()) + sql(db,"INSERT INTO users(hname) VALUES('generated migration sentinel');") + return db + def test_18_deploy_legacy_and_unchanged_rerun(self): + db=self.legacy('deploy_legacy') + first=runner(db,'deploy') + self.assertIn('adopted 20 migration(s)',first.stdout) + self.assertEqual([x.split()[1][:6] for x in first.stdout.splitlines() if x.startswith('APPLIED ')],PENDING) + before=sql(db,'SELECT row_to_json(m) FROM migrations m ORDER BY name') + self.assertIn('applied 0 migration(s)',runner(db,'deploy').stdout) + self.assertEqual(before,sql(db,'SELECT row_to_json(m) FROM migrations m ORDER BY name')) + + def test_19_deploy_does_not_adopt_corrupt_modern_history(self): + db=self.new('deploy_corrupt');runner(db,'deploy') + sql(db,"UPDATE migrations SET checksum=repeat('0',64) WHERE name LIKE '000023_%'") + before=sql(db,'SELECT row_to_json(m) FROM migrations m ORDER BY name') + self.assertIn('checksum mismatch',runner(db,'deploy',ok=False).stderr) + self.assertEqual(before,sql(db,'SELECT row_to_json(m) FROM migrations m ORDER BY name')) + + def test_20_deploy_concurrent_first_adoption(self): + db=self.legacy('deploy_concurrent') + commands=[subprocess.Popen([str(BIN),'deploy','--dir',str(MIG)],env=env(db),stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True) for _ in range(2)] + outputs=[] + for p in commands: + out,err=p.communicate(timeout=350) + self.assertEqual(p.returncode,0,out+err);outputs.append(out) + self.assertEqual(sum('adopted 20 migration(s)' in x for x in outputs),1) + self.assertEqual(sum(f'applied {len(PENDING)} migration(s)' in x for x in outputs),1) + runner(db,'check') + + def test_21_deploy_legacy_ledger_preserves_timestamps(self): + db=self.legacy('deploy_old_ledger') + sql(db,"CREATE TABLE migrations(name text,completed_at bigint); INSERT INTO migrations VALUES('000000_initial.sql',1000),('000000_initial.sql',2000)") + runner(db,'deploy') + self.assertEqual(sql(db,"SELECT legacy_completed_at FROM migrations WHERE name='000000_initial.sql'"),'{1000,2000}') + + def test_22_deploy_rejects_unverified_ledger(self): + db=self.legacy('deploy_other_ledger') + sql(db,'CREATE TABLE schema_migrations(name text)') + self.assertIn('another schema_migrations ledger',runner(db,'deploy',ok=False).stderr) + self.assertEqual(sql(db,"SELECT to_regclass('migrations') IS NULL"),'t') + + def test_23_modern_v3_upgrades_to_v5_preserving_receipts(self): + db=self.new('modern_v3') + with tempfile.TemporaryDirectory() as tmp: + d=pathlib.Path(tmp)/'m';shutil.copytree(MIG,d) + f=d/'release.txt';f.write_text('\n'.join(x for x in f.read_text().splitlines() if not x.startswith(('000027_', '000028_', '000029_')))+'\n') + for number in ('000027', '000028', '000029'): + next(d.glob(number+'_*.sql')).unlink() + runner(db,'deploy',dir=d) + before=sql(db,'SELECT row_to_json(m) FROM migrations m ORDER BY name') + gate(db,False) + output=runner(db,'deploy').stdout + self.assertIn('applied 3 migration(s)',output) + self.assertEqual(before,sql(db,"SELECT row_to_json(m) FROM migrations m WHERE name < '000027' ORDER BY name")) + self.assertEqual(sql(db,'SELECT contract_version FROM polis_queue_install'),'polis-queue/5') + gate(db,True) + + def test_25_release_b_upgrades_results_preserving_receipts(self): + db=self.new('release_b') + with tempfile.TemporaryDirectory() as tmp: + d=pathlib.Path(tmp)/'m';shutil.copytree(MIG,d) + f=d/'release.txt';f.write_text('\n'.join(x for x in f.read_text().splitlines() if not x.startswith(('000028_', '000029_')))+'\n') + for number in ('000028', '000029'): + next(d.glob(number+'_*.sql')).unlink() + runner(db,'deploy',dir=d) + before=sql(db,'SELECT row_to_json(m) FROM migrations m ORDER BY name') + gate(db,False) + output=runner(db,'deploy').stdout + self.assertEqual([x.split()[1][:6] for x in output.splitlines() if x.startswith('APPLIED ')], ['000028','000029']) + self.assertEqual(before,sql(db,"SELECT row_to_json(m) FROM migrations m WHERE name < '000028' ORDER BY name")) + gate(db,True) + self.assertIn('applied 0 migration(s)',runner(db,'deploy').stdout) + + def test_24_unledgered_graph_is_not_readopted(self): + db=self.new('unledgered_v5');runner(db,'deploy') + sql(db,'DROP TABLE migrations') + self.assertIn('queue /3 catalog postconditions fail',runner(db,'deploy',ok=False).stderr) + self.assertEqual(sql(db,"SELECT to_regclass('migrations') IS NULL"),'t') + + def test_15_treevite_checks_required_for_adoption(self): + db=self.legacy('treevite_contract') + constraints=sql(db,"SELECT conrelid::regclass::text || '|' || conname || '|' || pg_get_constraintdef(oid) FROM pg_constraint WHERE conrelid IN ('treevite_waves'::regclass,'treevite_invites'::regclass) AND contype='c' ORDER BY conname").splitlines() + self.assertEqual(len(constraints),7) + for row in constraints: + table,name,definition=row.split('|',2) + with self.subTest(constraint=name): + sql(db,f'ALTER TABLE {table} DROP CONSTRAINT {name}') + refused=runner(db,'reconcile','--through','000022',ok=False) + self.assertIn('000013_create_treevite.sql',refused.stderr) + self.assertEqual(sql(db,"SELECT to_regclass('public.migrations') IS NULL"),'t') + sql(db,f'ALTER TABLE {table} ADD CONSTRAINT {name} {definition}') + runner(db,'reconcile','--through','000022') + self.assertEqual(sql(db,"SELECT status FROM migrations WHERE name='000013_create_treevite.sql'"),'ADOPTED') + + def test_16_json_equivalence_is_only_for_legacy_math(self): + db=self.legacy('json_contract') + helpers=(MIG/'adoption/helpers.sql').read_text() + sql(db,'CREATE TABLE public.unrelated_json(data json)') + result=sql(db,helpers+"SELECT pg_temp.col('unrelated_json','data','jsonb')") + self.assertEqual(result.splitlines()[-1],'f') + for table in ['math_main','math_profile','math_ptptstats','math_cache', + 'math_bidtopid','math_exportstatus']: + sql(db,f'ALTER TABLE {table} ALTER COLUMN data TYPE json USING data::json') + sql(db,'ALTER TABLE math_report_correlationmatrix ALTER COLUMN data TYPE json USING data::json') + self.assertIn('000000',runner(db,'reconcile','--through','000022',ok=False).stderr) + self.assertEqual(sql(db,"SELECT to_regclass('public.migrations') IS NULL"),'t') + sql(db,'ALTER TABLE math_report_correlationmatrix ALTER COLUMN data TYPE jsonb USING data::jsonb') + runner(db,'reconcile','--through','000022') + + def test_01_fresh(self): + db=self.new('fresh') + self.assertIn(f'applied {len(APPLIED)} migration(s)',runner(db,'apply').stdout) + self.assertEqual(sql(db,"SELECT count(*) FROM migrations WHERE status='APPLIED'"),str(len(APPLIED))) + runner(db,'check');gate(db,True) + self.assertEqual(sql(db,"SELECT to_regclass('public.polis_coordinator_install') IS NULL"),'t') + def test_02_reconcile_then_pending(self): + db=self.legacy('legacy') + runner(db,'apply',ok=False); runner(db,'check',ok=False); gate(db,False) + p=runner(db,'reconcile','--through','000022') + self.assertIn('adopted 20 migration(s)',p.stdout) + self.assertEqual(sql(db,"SELECT count(*) FROM migrations WHERE status='ADOPTED'"),'20') + gate(db,False) + p=runner(db,'apply') + self.assertEqual([s.split()[1][:6] for s in p.stdout.splitlines() if s.startswith('APPLIED ')],PENDING) + runner(db,'check');gate(db,True) + self.assertEqual(sql(db,"SELECT hname FROM users"),'generated migration sentinel') + def test_03_noop(self): + db=self.new('noop');runner(db,'apply') + before=sql(db,'SELECT row_to_json(m) FROM migrations m ORDER BY name') + self.assertIn('applied 0 migration(s)',runner(db,'apply').stdout) + self.assertEqual(before,sql(db,'SELECT row_to_json(m) FROM migrations m ORDER BY name')) + def test_04_failure_atomic_and_startup_refusal(self): + db=self.legacy('failure');runner(db,'reconcile','--through','000022') + # Force the real 000023 file to fail after 000019 commits, then prove it + # leaves no half-created foundation or APPLIED row. The preexisting + # collision is generated; remove it explicitly before the recovery run. + sql(db,'CREATE TABLE public.delphi_jobs (collision integer)') + p=runner(db,'apply',ok=False) + self.assertIn('000023',p.stderr) + self.assertEqual(sql(db,"SELECT count(*) FROM migrations WHERE name LIKE '000023%'"),'0') + self.assertEqual(sql(db,"SELECT to_regclass('public.delphi_job_inputs') IS NULL"),'t') + self.assertEqual(sql(db,"SELECT NOT EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema='public' AND table_name='polis_queue_install' AND column_name='contract_version')"),'t') + runner(db,'check',ok=False);gate(db,False) + sql(db,'DROP TABLE public.delphi_jobs') + runner(db,'apply');gate(db,True) + def test_05_two_runners(self): + db=self.new('race') + # A third, owned session holds the migration lock until both real runners + # are observed waiting. Release it; one applies, the other observes history. + holder=subprocess.Popen(COMPOSE+['exec','-T','postgres','psql','-X','-U','postgres','-d',db,'-At'],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True) + holder.stdin.write('SELECT pg_advisory_lock(5795970445610936679);\n');holder.stdin.flush() + holder.stdout.readline() + args=[str(BIN),'apply','--dir',str(MIG)] + a=subprocess.Popen(args,env=env(db),stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True) + b=subprocess.Popen(args,env=env(db),stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True) + deadline=time.monotonic()+20 + try: + while time.monotonic()) -> Result { + if claim.stage.starts_with("graph_") { + return Ok( + json!({"schema":"polis-job-stage-frame/1", "env":claim.env, "zid":adm.zid, + "job_id":claim.job_id,"run_id":claim.run_id,"attempt_id":claim.attempt_id, + "lease_epoch":claim.lease_epoch,"stage":claim.stage,"input":adm.inputs, + "input_sha256":adm.config["input_sha256"],"input_json":adm.config["input_wire"]}), + ); + } let config = if claim.stage == "math_rebuild" { MathConfig::from_admission(adm)?.to_json() } else { @@ -331,6 +339,9 @@ pub fn command_args( app.join("umap_narrative/803_check_batch_status.py"), vec![format!("--job-id={}", claim.job_id)], ), + ("graph_embed" | "graph_cluster" | "graph_topics" | "graph_narrative", "run") => { + (app.join("scripts/job_graph_stage.py"), vec![]) + } ("math_rebuild", "run") => (app.join("scripts/math_poller.py"), vec!["--job".to_owned()]), (stage, phase) => bail!("no command for stage {stage} phase {phase}"), }) @@ -341,6 +352,57 @@ pub struct Spawned { pub pgid: i32, } +const GRAPH_ENVIRONMENT: &[&str] = &[ + "PATH", + "HOME", + "TMPDIR", + "TMP", + "TEMP", + "LANG", + "LC_ALL", + "TZ", + "PYTHONPATH", + "PYTHONUNBUFFERED", + "PYTHONDONTWRITEBYTECODE", + "OMP_NUM_THREADS", + "MKL_NUM_THREADS", + "OPENBLAS_NUM_THREADS", + "NUMBA_NUM_THREADS", + "NUMEXPR_NUM_THREADS", + "VECLIB_MAXIMUM_THREADS", + "TOKENIZERS_PARALLELISM", + "HF_HOME", + "HF_HUB_OFFLINE", + "TRANSFORMERS_OFFLINE", + "XDG_CACHE_HOME", + "NUMBA_CACHE_DIR", + "DELPHI_EMBED_MODEL_PATH", + // Existing process-boundary proof fixture paths; neither carries credentials. + "GRAPH_PROOF_ROOT", + "GRAPH_STAGE_SOURCE", +]; + +fn child_environment( + command: &mut Command, + stage: &str, + inherited: impl IntoIterator, +) { + if stage.starts_with("graph_") { + command.env_clear(); + for (key, value) in inherited { + if key + .to_str() + .is_some_and(|key| GRAPH_ENVIRONMENT.contains(&key)) + { + command.env(key, value); + } + } + } + command + .env_remove("QUEUE_DATABASE_URL") + .env_remove("POLIS_JOBS_PASSWORD_FILE"); +} + /// Spawn in a new session (pgid = pid). Queue credentials never reach the child. #[allow(clippy::too_many_arguments)] pub fn spawn( @@ -356,12 +418,11 @@ pub fn spawn( batch_id: Option<&str>, ) -> Result { let mut command = Command::new(python); + child_environment(&mut command, &claim.stage, std::env::vars_os()); command .arg(script) .args(args) .current_dir(app) - .env_remove("QUEUE_DATABASE_URL") - .env_remove("POLIS_JOBS_PASSWORD_FILE") .env("DELPHI_JOB_ID", &claim.job_id) .env("DELPHI_RUN_ID", &claim.run_id) .env("DELPHI_ATTEMPT_ID", &claim.attempt_id) @@ -501,6 +562,77 @@ pub fn kill_and_reap( mod tests { use super::*; + #[test] + fn graph_children_receive_only_model_and_runtime_environment() -> Result<()> { + let inherited: Vec<(std::ffi::OsString, std::ffi::OsString)> = [ + "DATABASE_URL", + "DELPHI_RESULT_DATABASE_URL", + "MATH_CAPACITY_QUEUE_DSN", + "QUEUE_DATABASE_URL", + "PGHOST", + "PGPORT", + "PGDATABASE", + "PGUSER", + "PGPASSWORD", + "PGPASSFILE", + "PGSERVICE", + "PGSERVICEFILE", + "POSTGRES_PASSWORD", + "DB_PASSWORD", + "POLIS_JOBS_PASSWORD_FILE", + "UNRECOGNIZED_DATABASE_DSN", + "DELPHI_EMBED_MODEL_PATH", + "OMP_NUM_THREADS", + "PATH", + ] + .into_iter() + .map(|key| (key.into(), format!("generated-{key}").into())) + .collect(); + let mut command = Command::new("/usr/bin/env"); + command.envs(inherited.clone()); + child_environment(&mut command, "graph_embed", inherited); + let output = command.output()?; + assert!(output.status.success()); + let lines = String::from_utf8(output.stdout)?; + let keys: Vec<&str> = lines + .lines() + .filter_map(|line| line.split_once('=').map(|(key, _)| key)) + .collect(); + assert_eq!(keys.len(), 3); + assert!( + keys.iter().all(|key| { + ["DELPHI_EMBED_MODEL_PATH", "OMP_NUM_THREADS", "PATH"].contains(key) + }) + ); + Ok(()) + } + #[test] + fn math_environment_keeps_its_database_access() { + let mut command = Command::new("python"); + command + .env("DATABASE_URL", "generated-main") + .env("MATH_CAPACITY_QUEUE_DSN", "generated-capacity") + .env("QUEUE_DATABASE_URL", "generated-queue") + .env("POLIS_JOBS_PASSWORD_FILE", "generated-file"); + child_environment(&mut command, "math_rebuild", std::iter::empty()); + let actual: std::collections::BTreeMap<_, _> = command.get_envs().collect(); + assert_eq!( + actual.get(std::ffi::OsStr::new("DATABASE_URL")), + Some(&Some(std::ffi::OsStr::new("generated-main"))) + ); + assert_eq!( + actual.get(std::ffi::OsStr::new("MATH_CAPACITY_QUEUE_DSN")), + Some(&Some(std::ffi::OsStr::new("generated-capacity"))) + ); + assert_eq!( + actual.get(std::ffi::OsStr::new("QUEUE_DATABASE_URL")), + Some(&None) + ); + assert_eq!( + actual.get(std::ffi::OsStr::new("POLIS_JOBS_PASSWORD_FILE")), + Some(&None) + ); + } #[test] fn base64url_round_trip() { for input in [ diff --git a/queue-rs/src/jobs/claim.rs b/queue-rs/src/jobs/claim.rs index a57007472b..6f379eff4a 100644 --- a/queue-rs/src/jobs/claim.rs +++ b/queue-rs/src/jobs/claim.rs @@ -62,7 +62,14 @@ fn claim_one( json!(cfg.lease_seconds), json!(cfg.worker_class.name()), ]; - let reply = match rpc.call("pq_claim", &args)? { + let graph = cfg.stages.iter().any(|s| s.starts_with("graph_")); + let mut args = args.to_vec(); + if graph { + args.push(json!( + cfg.stages.iter().cloned().collect::>().join(",") + )); + } + let reply = match rpc.call(if graph { "pd_graph_claim" } else { "pq_claim" }, &args)? { Completion::Committed(r) => r, Completion::Unknown(r) => { // Never reissue an uncertain claim: renew the exact token. @@ -329,6 +336,10 @@ pub fn run(cfg: Config) -> i32 { } } } + if cfg.stages.iter().any(|s| s.starts_with("graph_")) && ctx.contract() != "polis-queue/5" { + line("graph stages require polis-queue/5"); + return EXIT_CONTRACT; + } recover(&ctx, &mut rpc); let mut listener: Option = rpc.listener().ok(); let mut jobs: Vec> = vec![]; @@ -387,6 +398,11 @@ pub fn run(cfg: Config) -> i32 { if Instant::now() >= next_reap { next_reap = Instant::now() + cfg.reap_interval; retry_pending(&ctx, &mut rpc); + if ctx.contract() == "polis-queue/5" + && let Err(e) = rpc.committed("pd_graph_reconcile", &[json!(cfg.env)]) + { + line(&format!("graph release reconciliation failed: {e}")); + } if let Err(e) = reaper.tick(&mut rpc, cfg.worker_class.name(), &owner, &ctx.counters) { db_failures += 1; line(&format!("reap failed: {e}")); diff --git a/queue-rs/src/jobs/config.rs b/queue-rs/src/jobs/config.rs index c8ac054829..e18f18e0a2 100644 --- a/queue-rs/src/jobs/config.rs +++ b/queue-rs/src/jobs/config.rs @@ -6,6 +6,12 @@ use std::{collections::BTreeSet, path::PathBuf, time::Duration}; /// `polis_queue_jobs_stage_check`). pub const KNOWN_STAGES: [&str; 2] = ["delphi_full_pipeline", "delphi_narrative"]; /// The stages of class `large`, which `polis-queue/3` admits (000024). +pub const GRAPH_STAGES: [&str; 4] = [ + "graph_embed", + "graph_cluster", + "graph_topics", + "graph_narrative", +]; pub const LARGE_STAGES: [&str; 1] = ["math_rebuild"]; /// The worker class the daemon claims as (`POLIS_JOBS_WORKER_CLASS`). Each @@ -47,8 +53,8 @@ impl WorkerClass { /// worker of this class may start on. pub fn contracts(self) -> &'static [&'static str] { match self { - Self::Delphi => &["polis-queue/2", "polis-queue/3"], - Self::Large => &["polis-queue/3"], + Self::Delphi => &["polis-queue/2", "polis-queue/3", "polis-queue/5"], + Self::Large => &["polis-queue/3", "polis-queue/5"], } } @@ -201,9 +207,11 @@ pub fn load Option>(get: F) -> Result .map(str::to_owned) .collect(); if stages.is_empty() - || stages - .iter() - .any(|s| !worker_class.stages().contains(&s.as_str())) + || stages.iter().any(|s| { + !worker_class.stages().contains(&s.as_str()) + && !(GRAPH_STAGES.contains(&s.as_str()) + && (worker_class == WorkerClass::Delphi || s == "graph_cluster")) + }) { return err(format!( "POLIS_JOBS_STAGES must be a non-empty subset of {} (the stages of class {})", @@ -211,6 +219,11 @@ pub fn load Option>(get: F) -> Result worker_class.name() )); } + if stages.iter().any(|s| s.starts_with("graph_")) + && stages.iter().any(|s| !s.starts_with("graph_")) + { + return err("graph and legacy stages require separate workers"); + } let lease = number(&get, "POLIS_JOBS_LEASE_SECONDS", 120, 10, 900)?; let heartbeat = number(&get, "POLIS_JOBS_HEARTBEAT_SECONDS", 30, 1, 900)?; if heartbeat * 3 >= lease { diff --git a/queue-rs/src/jobs/graph.rs b/queue-rs/src/jobs/graph.rs new file mode 100644 index 0000000000..3976f63584 --- /dev/null +++ b/queue-rs/src/jobs/graph.rs @@ -0,0 +1,326 @@ +//! Versioned sealed-graph boundary. SQL is final authority for content binding. +use super::{ + child::Admission, + manifest::{Invalid, Manifest, Outcome}, + sha256_hex, +}; +use anyhow::{Result, ensure}; +use serde_json::{Value, json}; +pub fn admission(reply: &Value) -> Result { + ensure!( + reply["schema_version"] == "polis-queue/5" + && reply["graph_input"]["schema"] == "polis-job-input/1", + "graph_input_schema" + ); + let sha = reply["graph_input_sha"].as_str().unwrap_or_default(); + ensure!( + sha.len() == 64 && sha.bytes().all(|c| c.is_ascii_hexdigit()), + "graph_input_digest" + ); + let wire = reply["graph_input_wire"] + .as_str() + .ok_or_else(|| anyhow::anyhow!("graph_input_wire"))?; + ensure!( + sha256_hex(wire.as_bytes()) == sha + && serde_json::from_str::(wire)? == reply["graph_input"], + "graph_input_digest" + ); + Ok(Admission { + zid: reply["zid"] + .as_i64() + .ok_or_else(|| anyhow::anyhow!("graph_zid"))?, + report_id: None, + config: json!({"input_sha256":sha,"input_wire":wire}), + inputs: reply["graph_input"].clone(), + }) +} +pub fn validate( + bytes: Option<&[u8]>, + job: &str, + attempt: &str, + stage: &str, +) -> Result { + let b = bytes.ok_or(Invalid::Missing)?; + if b.len() > super::manifest::MAX_BYTES { + return Err(Invalid::TooLarge); + } + let text = std::str::from_utf8(b).map_err(|_| Invalid::NotUtf8)?; + let m: Value = serde_json::from_str(text).map_err(|_| Invalid::NotJson)?; + let keys = [ + "schema", + "job_id", + "run_id", + "attempt_id", + "stage", + "input_sha256", + "outcome", + "output", + ]; + if m.as_object() + .is_none_or(|o| o.len() != keys.len() || keys.iter().any(|k| !o.contains_key(*k))) + || m["schema"] != "polis-job-artifact-manifest/1" + || m["job_id"] != job + || m["attempt_id"] != attempt + || m["stage"] != stage + || m["outcome"] != "succeeded" + || m["output"]["role"] != "result" + || m["output"]["schema"] != format!("{stage}/1") + || m["output"]["payload"] + .as_str() + .is_none_or(|p| p.len() > 524288 || m["output"]["sha256"] != sha256_hex(p.as_bytes())) + { + return Err(Invalid::Field("graph_manifest")); + } + Ok(Manifest { + text: text.into(), + sha256: sha256_hex(b), + outcome: Outcome::Succeeded, + phase: "run".into(), + recheck_after: None, + batch_id: None, + batch_ids: vec![], + tokens_in: None, + tokens_out: None, + }) +} + +/// Hydrate only immutable, digest-bound result bytes; queue credentials stay in daemon. +pub fn hydrate(rpc: &mut super::rpc::Rpc, frame: &mut Value) -> Result<()> { + if frame["stage"] != "graph_cluster" + || frame["input"]["declared"]["model"] != "delphi-umap-evoc/1" + { + return Ok(()); + } + let artifact = &frame["input"]["artifacts"]["embeddings"]; + let payload: Value = serde_json::from_str(artifact["payload"].as_str().unwrap_or_default())?; + if payload.get("family_files").is_some() { + return Ok(()); + } + let family = "Delphi_CommentEmbeddings"; + let reply = rpc.committed( + "pd_result_artifact_wire", + &[ + frame["env"].clone(), + artifact["artifact_id"].clone(), + json!(family), + ], + )?; + let wire = reply["wire"] + .as_str() + .ok_or_else(|| anyhow::anyhow!("result_wire_missing"))?; + ensure!( + wire.len() <= 67_108_864 + && reply["sha256"] == sha256_hex(wire.as_bytes()) + && reply["batch_id"] == payload["results"]["batch_id"] + && reply["batch_sha256"] == payload["results"]["sha256"], + "result_wire_binding" + ); + frame["result_families"] = json!({"embeddings": {family: wire}}); + Ok(()) +} + +/// Read only a bounded regular spool file; a FIFO must never block the daemon. +fn read_family_spool( + directory: &std::path::Path, + family: &str, + value: &Value, + total: &mut u64, +) -> Result { + ensure!( + !family.is_empty() + && family + .bytes() + .all(|c| c.is_ascii_alphanumeric() || c == b'_'), + "result_family_name" + ); + let filename = format!("{family}.jsonl"); + ensure!(value["file"] == filename, "result_spool_path"); + let bytes = super::manifest::read_regular_file(&directory.join(filename), 67_108_864) + .map_err(|error| anyhow::anyhow!("result_spool_file:{error}"))?; + let wire = String::from_utf8(bytes)?; + let next_total = total + .checked_add(wire.len() as u64) + .ok_or_else(|| anyhow::anyhow!("result_spool_bound"))?; + ensure!(next_total <= 268_435_456, "result_spool_bound"); + ensure!( + value["sha256"] == sha256_hex(wire.as_bytes()), + "result_spool_digest" + ); + *total = next_total; + Ok(wire) +} + +/// Persist detached family bytes after process exit and before artifact finalize. +/// The daemon creates the final manifest linking the attempt's sealed batch. +pub fn stage_results( + rpc: &mut super::rpc::Rpc, + claim: &super::child::Claim, + directory: &std::path::Path, + manifest: Manifest, +) -> Result { + let mut document: Value = serde_json::from_str(&manifest.text)?; + let mut payload: Value = + serde_json::from_str(document["output"]["payload"].as_str().unwrap_or_default())?; + let Some(spool) = payload.get("family_spool") else { + return Ok(manifest); + }; + let files = spool + .as_object() + .ok_or_else(|| anyhow::anyhow!("result_spool_shape"))?; + ensure!(!files.is_empty() && files.len() <= 18, "result_spool_count"); + let mut total = 0_u64; + for (family, value) in files { + let wire = read_family_spool(directory, family, value, &mut total)?; + let mut args = super::task::identity(claim); + args.extend([json!(family), json!(wire)]); + let reply = rpc.committed("pd_result_put_family", &args)?; + ensure!(reply["sha256"] == value["sha256"], "result_stage_receipt"); + } + let results = rpc.committed("pd_result_seal", &super::task::identity(claim))?; + ensure!( + results["schema"] == "delphi-result-batch/1" && results["batch_id"] == claim.attempt_id, + "result_batch_receipt" + ); + payload + .as_object_mut() + .ok_or_else(|| anyhow::anyhow!("result_payload"))? + .remove("family_spool"); + payload["results"] = results; + let wire = serde_json::to_string(&payload)?; + document["output"]["sha256"] = json!(sha256_hex(wire.as_bytes())); + document["output"]["payload"] = json!(wire); + let bytes = serde_json::to_vec(&document)?; + let result = validate(Some(&bytes), &claim.job_id, &claim.attempt_id, &claim.stage) + .map_err(|_| anyhow::anyhow!("result_manifest_invalid"))?; + let temporary = directory.join("output-manifest.sealed.tmp"); + std::fs::write(&temporary, &bytes)?; + std::fs::rename(temporary, directory.join("output-manifest.json"))?; + Ok(result) +} + +#[cfg(test)] +mod tests { + use super::*; + struct SpoolDirectory(std::path::PathBuf); + impl SpoolDirectory { + fn new() -> Result { + let path = + std::env::temp_dir().join(format!("polis-graph-spool-{}", uuid::Uuid::new_v4())); + std::fs::create_dir(&path)?; + Ok(Self(path)) + } + } + impl Drop for SpoolDirectory { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } + } + fn spool_description(bytes: &[u8]) -> Value { + json!({"file":"Family.jsonl","sha256":sha256_hex(bytes)}) + } + #[test] + fn spool_regular_file_preserves_exact_bytes_and_counts() -> Result<()> { + let dir = SpoolDirectory::new()?; + let wire = b"{\"number\":1}\n"; + std::fs::write(dir.0.join("Family.jsonl"), wire)?; + let mut total = 4; + assert_eq!( + read_family_spool(&dir.0, "Family", &spool_description(wire), &mut total)?.as_bytes(), + wire + ); + assert_eq!(total, 4 + wire.len() as u64); + Ok(()) + } + #[test] + fn spool_symlink_is_refused() -> Result<()> { + let dir = SpoolDirectory::new()?; + std::fs::write(dir.0.join("target"), b"data")?; + std::os::unix::fs::symlink(dir.0.join("target"), dir.0.join("Family.jsonl"))?; + assert!(read_family_spool(&dir.0, "Family", &spool_description(b"data"), &mut 0).is_err()); + Ok(()) + } + #[test] + fn spool_fifo_is_refused_without_waiting_for_writer() -> Result<()> { + let dir = SpoolDirectory::new()?; + use std::os::unix::ffi::OsStrExt; + let name = std::ffi::CString::new(dir.0.join("Family.jsonl").as_os_str().as_bytes())?; + // SAFETY: name is a live, NUL-terminated path; mkfifo retains no pointer. + assert_eq!(unsafe { libc::mkfifo(name.as_ptr(), 0o600) }, 0); + let path = dir.0.clone(); + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let _ = tx + .send(read_family_spool(&path, "Family", &spool_description(b""), &mut 0).is_err()); + }); + assert!(rx.recv_timeout(std::time::Duration::from_secs(2))?); + Ok(()) + } + #[test] + fn spool_digest_mismatch_is_refused() -> Result<()> { + let dir = SpoolDirectory::new()?; + std::fs::write(dir.0.join("Family.jsonl"), b"changed")?; + assert!( + read_family_spool(&dir.0, "Family", &spool_description(b"original"), &mut 0).is_err() + ); + Ok(()) + } + #[test] + fn spool_sparse_oversize_is_refused_without_reading() -> Result<()> { + let dir = SpoolDirectory::new()?; + std::fs::File::create(dir.0.join("Family.jsonl"))?.set_len(67_108_865)?; + let error = read_family_spool(&dir.0, "Family", &spool_description(b""), &mut 0).err(); + assert_eq!( + error.map(|e| e.to_string()), + Some("result_spool_file:manifest TooLarge".into()) + ); + Ok(()) + } + #[test] + fn spool_path_and_total_bound_are_refused() -> Result<()> { + let dir = SpoolDirectory::new()?; + std::fs::write(dir.0.join("Family.jsonl"), b"data")?; + assert!( + read_family_spool(&dir.0, "../Family", &spool_description(b"data"), &mut 0).is_err() + ); + assert!( + read_family_spool(&dir.0, "Family", &json!({"file":"../Family.jsonl"}), &mut 0) + .is_err() + ); + assert!( + read_family_spool( + &dir.0, + "Family", + &spool_description(b"data"), + &mut 268_435_456 + ) + .is_err() + ); + let mut total = u64::MAX; + assert!( + read_family_spool(&dir.0, "Family", &spool_description(b"data"), &mut total).is_err() + ); + Ok(()) + } + #[test] + fn resolved_bytes_are_checked_before_dispatch() { + let wire = r#"{"schema":"polis-job-input/1","declared":{},"artifacts":{}}"#; + let mut r = json!({"schema_version":"polis-queue/5","graph_input":serde_json::from_str::(wire).unwrap_or_else(|e| panic!("{e}")), + "graph_input_wire":wire,"graph_input_sha":sha256_hex(wire.as_bytes()),"zid":1}); + assert!(admission(&r).is_ok()); + r["graph_input"]["declared"] = json!({"tampered":true}); + assert!(admission(&r).is_err()); + r["schema_version"] = json!("polis-queue/3"); + assert!(admission(&r).is_err()); + } + #[test] + fn content_mismatch_and_legacy_receipts_refuse() { + let mut m = json!({"schema":"polis-job-artifact-manifest/1","job_id":"j","run_id":"r","attempt_id":"a", + "stage":"graph_embed","input_sha256":"0".repeat(64),"outcome":"succeeded", + "output":{"role":"result","schema":"graph_embed/1","payload":"{}","sha256":sha256_hex(b"{}")}}); + assert!(validate(Some(m.to_string().as_bytes()), "j", "a", "graph_embed").is_ok()); + m["output"]["payload"] = json!("changed"); + assert!(validate(Some(m.to_string().as_bytes()), "j", "a", "graph_embed").is_err()); + m["schema"] = json!("polis-jobs.output-manifest/1"); + assert!(validate(Some(m.to_string().as_bytes()), "j", "a", "graph_embed").is_err()); + } +} diff --git a/queue-rs/src/jobs/manifest.rs b/queue-rs/src/jobs/manifest.rs index e7480b0ba6..56fc3a20e7 100644 --- a/queue-rs/src/jobs/manifest.rs +++ b/queue-rs/src/jobs/manifest.rs @@ -49,6 +49,43 @@ impl std::fmt::Display for Invalid { } } +/// Open a bounded regular child output without following links or waiting on FIFOs. +pub(super) fn read_regular_file( + path: &std::path::Path, + maximum: usize, +) -> Result, Invalid> { + use std::io::Read; + use std::os::unix::fs::OpenOptionsExt; + let file = std::fs::OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK) + .open(path) + .map_err(|error| { + if error.kind() == std::io::ErrorKind::NotFound { + Invalid::Missing + } else { + Invalid::Field("file_open") + } + })?; + let metadata = file + .metadata() + .map_err(|_| Invalid::Field("file_metadata"))?; + if !metadata.is_file() { + return Err(Invalid::Field("file_type")); + } + if metadata.len() > maximum as u64 { + return Err(Invalid::TooLarge); + } + let mut bytes = Vec::new(); + file.take(maximum as u64 + 1) + .read_to_end(&mut bytes) + .map_err(|_| Invalid::Field("file_read"))?; + if bytes.len() > maximum { + return Err(Invalid::TooLarge); + } + Ok(bytes) +} + fn is_timestamp(s: &str) -> bool { // RFC 3339 date-time with an explicit offset, e.g. 2026-10-03T12:00:00Z. let b = s.as_bytes(); @@ -319,6 +356,23 @@ mod tests { validate(Some(&bytes), J, A, "delphi_full_pipeline") } + #[test] + fn manifest_file_reader_enforces_the_limit_before_validation() -> anyhow::Result<()> { + let path = std::env::temp_dir().join(format!("polis-manifest-{}", uuid::Uuid::new_v4())); + assert_eq!(read_regular_file(&path, MAX_BYTES), Err(Invalid::Missing)); + let file = std::fs::File::create(&path)?; + file.set_len(MAX_BYTES as u64)?; + assert_eq!( + read_regular_file(&path, MAX_BYTES) + .map_err(|e| anyhow::anyhow!("{e}"))? + .len(), + MAX_BYTES + ); + file.set_len(MAX_BYTES as u64 + 1)?; + assert_eq!(read_regular_file(&path, MAX_BYTES), Err(Invalid::TooLarge)); + std::fs::remove_file(path)?; + Ok(()) + } #[test] fn valid_manifest_keeps_exact_bytes_and_hash() { let bytes = b"{\"schema\": \"polis-jobs.output-manifest/1\" }"; diff --git a/queue-rs/src/jobs/mod.rs b/queue-rs/src/jobs/mod.rs index dc6567ad03..cecda34f6a 100644 --- a/queue-rs/src/jobs/mod.rs +++ b/queue-rs/src/jobs/mod.rs @@ -8,6 +8,7 @@ pub mod child; pub mod claim; pub mod config; +pub mod graph; pub mod journal; pub mod lease; pub mod logs; diff --git a/queue-rs/src/jobs/rpc.rs b/queue-rs/src/jobs/rpc.rs index 54fba284bf..df2c3de82d 100644 --- a/queue-rs/src/jobs/rpc.rs +++ b/queue-rs/src/jobs/rpc.rs @@ -23,7 +23,7 @@ const TABLES: [&str; 5] = [ /// The installed contracts the daemon knows: `polis-queue/2` (000023) and /// `polis-queue/3` (000024, the large class). Which one a worker may start /// on is the class's say (`WorkerClass::contracts`). -pub const CONTRACTS: [&str; 2] = ["polis-queue/2", "polis-queue/3"]; +pub const CONTRACTS: [&str; 3] = ["polis-queue/2", "polis-queue/3", "polis-queue/5"]; pub struct Rpc { connector: Arc, diff --git a/queue-rs/src/jobs/task.rs b/queue-rs/src/jobs/task.rs index d34ea63cae..3ad8af0fa2 100644 --- a/queue-rs/src/jobs/task.rs +++ b/queue-rs/src/jobs/task.rs @@ -338,10 +338,16 @@ pub fn run(ctx: Arc, claim: Claim, reply: Value) { let cfg = ctx.cfg.clone(); let mut rpc = Rpc::new(ctx.connector.clone(), &claim.env); let started = Instant::now(); - let admission = match child::decode_admission( - reply["input"]["uri"].as_str().unwrap_or_default(), - reply["input"]["sha256"].as_str().unwrap_or_default(), - ) { + let graph = claim.stage.starts_with("graph_"); + let admission_result = if graph { + super::graph::admission(&reply) + } else { + child::decode_admission( + reply["input"]["uri"].as_str().unwrap_or_default(), + reply["input"]["sha256"].as_str().unwrap_or_default(), + ) + }; + let admission = match admission_result { Ok(a) => a, Err(_) => return refuse_without_child(&ctx, &mut rpc, &claim, true, "frame_invalid"), }; @@ -399,7 +405,7 @@ pub fn run(ctx: Arc, claim: Claim, reply: Value) { let manifest_path = dir.join("output-manifest.json"); let frame_path = dir.join("frame.json"); let batch = recheck.as_ref().map(|(_, b)| b.as_str()); - let frame = match child::frame(&claim, &admission, &phase, batch) { + let mut frame = match child::frame(&claim, &admission, &phase, batch) { Ok(f) => f, Err(_) => { refuse_without_child(&ctx, &mut rpc, &claim, true, "math_config_invalid"); @@ -407,6 +413,11 @@ pub fn run(ctx: Arc, claim: Claim, reply: Value) { return; } }; + if graph && super::graph::hydrate(&mut rpc, &mut frame).is_err() { + refuse_without_child(&ctx, &mut rpc, &claim, false, "result_hydration_failed"); + let _ = ctx.journal.remove(&claim.attempt_id); + return; + } let prepared = std::fs::create_dir_all(&dir).and_then(|_| std::fs::write(&frame_path, frame.to_string())); let command = child::command_args(&cfg.app_path, &claim, &admission, &phase); @@ -611,13 +622,20 @@ pub fn run(ctx: Arc, claim: Claim, reply: Value) { None => Exit::Code(255), }, }; - let manifest_bytes = std::fs::read(&manifest_path).ok(); - let manifest: Result = manifest::validate( - manifest_bytes.as_deref(), - &claim.job_id, - &claim.attempt_id, - &claim.stage, - ); + let manifest_bytes = manifest::read_regular_file(&manifest_path, manifest::MAX_BYTES); + let validate = if graph { + super::graph::validate + } else { + manifest::validate + }; + let mut manifest: Result = manifest_bytes + .and_then(|bytes| validate(Some(&bytes), &claim.job_id, &claim.attempt_id, &claim.stage)); + if graph && outcome::decide(&exit, &manifest) == Action::Finalize { + manifest = manifest.and_then(|value| { + super::graph::stage_results(&mut rpc, &claim, &dir, value) + .map_err(|_| manifest::Invalid::Field("result_staging_failed")) + }); + } let mut action = outcome::decide(&exit, &manifest); let id = identity(&claim); // Provider bookkeeping while still the owner (or as the late submitter). @@ -735,7 +753,11 @@ pub fn run(ctx: Arc, claim: Claim, reply: Value) { (true, Ok(_)) => { let fin = terminal( &mut rpc, - "pq_finalize", + if graph { + "pd_graph_finalize" + } else { + "pq_finalize" + }, &with(id.clone(), &[json!(manifest_uri), json!(m.sha256)]), ); match fin { diff --git a/queue-rs/src/lib.rs b/queue-rs/src/lib.rs index 81752f60fd..2857fc504d 100644 --- a/queue-rs/src/lib.rs +++ b/queue-rs/src/lib.rs @@ -65,6 +65,14 @@ fn signature(name: &str) -> Result<&'static [&'static str]> { /// and the four-argument reaper can never fall back to a `/1` form. pub fn signature_v2(name: &str) -> Result<&'static [&'static str]> { Ok(match name { + "pd_graph_claim" => &[ + "text", "smallint", "uuid", "uuid", "integer", "text", "text", + ], + "pd_graph_finalize" => &["text", "uuid", "uuid", "uuid", "bigint", "text", "text"], + "pd_graph_reconcile" => &["text"], + "pd_result_put_family" => &["text", "uuid", "uuid", "uuid", "bigint", "text", "text"], + "pd_result_seal" => &["text", "uuid", "uuid", "uuid", "bigint"], + "pd_result_artifact_wire" => &["text", "uuid", "text"], "pq_claim" => &["text", "smallint", "uuid", "uuid", "integer", "text"], "pq_class_depth" => &["text", "text"], "pq_heartbeat" => &["text", "uuid", "uuid", "uuid", "bigint", "integer"], diff --git a/queue-rs/tests/jobs_integration.rs b/queue-rs/tests/jobs_integration.rs index 2597cb0d34..d992268a5d 100644 --- a/queue-rs/tests/jobs_integration.rs +++ b/queue-rs/tests/jobs_integration.rs @@ -94,6 +94,9 @@ fn ensure_templates() { .filter(|p| { p.extension().is_some_and(|e| e == "sql") && p.file_name().unwrap().to_string_lossy().as_bytes()[0].is_ascii_digit() + && p.file_name().unwrap().to_string_lossy()[..6] + .parse::() + .is_ok_and(|n| n <= 24) }) .collect(); chain.sort(); diff --git a/scripts/after_install.sh b/scripts/after_install.sh index 13db753c0f..ced1bfe0b5 100644 --- a/scripts/after_install.sh +++ b/scripts/after_install.sh @@ -87,6 +87,12 @@ case "$SERVICE_FROM_FILE" in *) echo "Error: Unknown service type: [$SERVICE_FROM_FILE]"; exit 1 ;; esac +# Apply the release schema before stopping/replacing any healthy service. +# All roles use the same database lock; a second box waits, then applies nothing. +# Build the client with this checkout. No manual image staging or local Rust needed. +sudo docker build -t polis-migrate:deploy -f queue-rs/polis-migrate/Dockerfile . +sudo docker run --rm --network host --env-file .env polis-migrate:deploy deploy + # Validate configuration before replacing any running container. Compose replaces # only this Compose project below; never remove unrelated containers or prune here. sudo /usr/local/bin/docker-compose config --quiet diff --git a/scripts/prove-dynamo-report.sh b/scripts/prove-dynamo-report.sh new file mode 100755 index 0000000000..54ca189d6b --- /dev/null +++ b/scripts/prove-dynamo-report.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# Local mm5 report smoke against an already published generated Postgres run. +set -euo pipefail +cd "$(dirname "$0")/.." +repo_root=$PWD +: "${COMPOSE_PROJECT_NAME:?owned local project required}" +: "${DYNAMO_PROOF_OUTPUT:?absolute evidence directory required}" +: "${DYNAMO_PROOF_REPORT_ID:?generated local report id required}" +: "${DYNAMO_PROOF_PG_DATABASE:?local generated database required}" +[[ "$COMPOSE_PROJECT_NAME" == polis-graph-test-* ]] +[[ "$DYNAMO_PROOF_REPORT_ID" == rlocal* ]] +[[ "$DYNAMO_PROOF_PG_DATABASE" == dynamo1424* ]] +[[ "$DYNAMO_PROOF_OUTPUT" = /* ]] +proof_network="${COMPOSE_PROJECT_NAME}_default" +proof_pg="${COMPOSE_PROJECT_NAME}-postgres-1" +proof_prefix="${DYNAMO_PROOF_CONTAINER_PREFIX:-astra-dynamo1424-${COMPOSE_PROJECT_NAME#polis-graph-test-}}" +[[ "$proof_prefix" == astra-dynamo1424-* ]] +proof_server="$proof_prefix-server" +proof_web="$proof_prefix-web" +proof_port="${DYNAMO_PROOF_PORT:-58244}" +proof_image="${DYNAMO_PROOF_SERVER_IMAGE:-$proof_prefix:server}" +mkdir -p "$DYNAMO_PROOF_OUTPUT" +# Each invocation owns these exact names. Refuse to replace existing processes. +if docker container inspect "$proof_server" >/dev/null 2>&1 || docker container inspect "$proof_web" >/dev/null 2>&1; then + echo "proof container name already exists; choose DYNAMO_PROOF_CONTAINER_PREFIX" >&2 + exit 2 +fi +cleanup() { + docker logs "$proof_server" > "$DYNAMO_PROOF_OUTPUT/server.log" 2>&1 || true + docker logs "$proof_web" > "$DYNAMO_PROOF_OUTPUT/nginx.log" 2>&1 || true + if [[ "${DYNAMO_PROOF_KEEP_RUNNING:-0}" != 1 ]]; then + docker rm -f "$proof_server" "$proof_web" >/dev/null 2>&1 || true + fi +} +trap cleanup EXIT +# Optional dependency caches are read-only inputs. Hosted CI can start empty. +if [[ -z "${DYNAMO_PROOF_SERVER_IMAGE:-}" ]]; then + docker build --target dev -t "$proof_image" server > "$DYNAMO_PROOF_OUTPUT/server-image-build.log" 2>&1 +fi +if [[ ! -d client-report/node_modules ]]; then + (cd client-report && npm ci --no-audit --no-fund) > "$DYNAMO_PROOF_OUTPUT/report-dependencies.log" 2>&1 +fi +if [[ -z "${DYNAMO_PROOF_PLAYWRIGHT:-}" ]]; then + proof_qa="$DYNAMO_PROOF_OUTPUT/qa" + mkdir -p "$proof_qa" + npm install --prefix "$proof_qa" --no-save --package-lock=false --no-audit --no-fund playwright@1.64.0 > "$DYNAMO_PROOF_OUTPUT/browser-dependencies.log" 2>&1 + export DYNAMO_PROOF_PLAYWRIGHT="$proof_qa/node_modules/playwright" + export PLAYWRIGHT_BROWSERS_PATH="$proof_qa/browsers" + browser_install=(install chromium) + if [[ "${CI:-}" == true ]]; then browser_install=(install --with-deps chromium); fi + node "$DYNAMO_PROOF_PLAYWRIGHT/cli.js" "${browser_install[@]}" >> "$DYNAMO_PROOF_OUTPUT/browser-dependencies.log" 2>&1 +fi +node --test ci/dynamo-removal/report-quality.test.cjs > "$DYNAMO_PROOF_OUTPUT/report-quality-unit.log" 2>&1 +# Same public OIDC settings as docker-compose.test.yml; anonymous report reads +# still use the real AuthProvider. These must exist at Webpack build time. +(cd client-report && AUTH_AUDIENCE=users AUTH_CLIENT_ID=dev-client-id \ + AUTH_ISSUER=https://localhost:3000/ AUTH_NAMESPACE=https://pol.is/ npm run build:prod) > "$DYNAMO_PROOF_OUTPUT/report-build.log" 2>&1 +sed "s/astra-dynamo1424-server/$proof_server/" ci/dynamo-removal/report-nginx.conf > "$DYNAMO_PROOF_OUTPUT/nginx.conf" +docker run -d --name "$proof_server" --network "$proof_network" \ + --label polis.proof=dynamo1424 --env-file test.env \ + -e NODE_ENV=development -e "DATABASE_URL=postgresql://postgres@$proof_pg:5432/$DYNAMO_PROOF_PG_DATABASE" \ + -e DATABASE_SSL=false -e "MATH_ENV=${DYNAMO_PROOF_ENV:-demo1424}" \ + -e DELPHI_RESULT_BACKEND=postgres -e "DELPHI_RESULT_ENV=${DYNAMO_PROOF_ENV:-demo1424}" \ + -e "DELPHI_RESULT_SCOPE=${DYNAMO_PROOF_SCOPE:-delphi}" -e POLIS_QUEUE_SUBSTRATE_ENABLED=true \ + -e DYNAMODB_ENDPOINT=http://127.0.0.1:9 -e AWS_EC2_METADATA_DISABLED=true -e DD_TRACE_ENABLED=false \ + -e "SERVICE_URL=http://localhost:$proof_port" -v "$repo_root/server:/candidate:ro" \ + --entrypoint sh "$proof_image" -c 'cp -a /candidate/. /app/; cd /app; npm run build && node dist/index.js' \ + > "$DYNAMO_PROOF_OUTPUT/server-container.txt" +for _ in $(seq 1 90); do + if docker logs "$proof_server" 2>&1 | grep 'Server started on port' >/dev/null; then break; fi + if [[ "$(docker inspect "$proof_server" --format '{{.State.Running}}')" != true ]]; then + docker logs "$proof_server" >&2;exit 1 + fi + sleep 1 +done +docker logs "$proof_server" 2>&1 | grep 'Server started on port' >/dev/null +docker run -d --name "$proof_web" --network "$proof_network" --label polis.proof=dynamo1424 \ + -p "127.0.0.1:$proof_port:8080" \ + -v "$DYNAMO_PROOF_OUTPUT/nginx.conf:/etc/nginx/nginx.conf:ro" \ + -v "$repo_root/client-report/dist:/report:ro" nginx:1.21.5-alpine > "$DYNAMO_PROOF_OUTPUT/web-container.txt" +export DYNAMO_PROOF_REPORT_URL="http://127.0.0.1:$proof_port" +for _ in $(seq 1 30); do curl -fsS "$DYNAMO_PROOF_REPORT_URL/" >/dev/null && break; sleep 1; done +node ci/dynamo-removal/report-proof.cjs > "$DYNAMO_PROOF_OUTPUT/browser.log" 2>&1 +cat "$DYNAMO_PROOF_OUTPUT/browser.log" diff --git a/scripts/test-dynamo-removal.sh b/scripts/test-dynamo-removal.sh new file mode 100755 index 0000000000..c6081520cf --- /dev/null +++ b/scripts/test-dynamo-removal.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash +# The hosted CI and mm5 entry point. Generated/public fixture data only. +set -euo pipefail +cd "$(dirname "$0")/.." +repo=$PWD +: "${COMPOSE_PROJECT_NAME:?set a unique polis-graph-test-* project}" +: "${POLIS_RECOVERY_PG_PORT:?set an unused local port}" +: "${RECOVERY_PG_PORT:?set the same local port}" +[[ "$COMPOSE_PROJECT_NAME" == polis-graph-test-* ]] +[[ "$POLIS_RECOVERY_PG_PORT" == "$RECOVERY_PG_PORT" ]] +proof=${DYNAMO_PROOF_ROOT:-$repo/.dynamo-proof/$COMPOSE_PROJECT_NAME} +mkdir -p "$proof"; proof=$(cd "$proof" && pwd) +if [[ -e "$proof/demo-admitted.json" || -e "$proof/cluster-failed-once" ]]; then echo "Proof directory already contains a run; choose a new directory" >&2; exit 2; fi +pg="${COMPOSE_PROJECT_NAME}-postgres-1" +image="${COMPOSE_PROJECT_NAME}:worker" +base=${DELPHI_PROOF_BASE:-polis-dynamo-deps:local} +compose=(docker compose -f delphi/tests/job_graph/compose.yml) +if docker inspect "$pg" >/dev/null 2>&1; then echo 'Proof project already exists; choose a new project' >&2; exit 2; fi +owned=() +cleanup() { + for container in "${owned[@]-}"; do [[ -n "$container" ]] || continue; docker logs "$container" > "$proof/$container.log" 2>&1 || true; done + if [[ "${DYNAMO_PROOF_KEEP_RUNNING:-0}" != 1 ]]; then + for container in "${owned[@]-}"; do [[ -n "$container" ]] || continue; docker rm -f "$container" >/dev/null 2>&1 || true; done + "${compose[@]}" down -v > "$proof/cleanup.log" 2>&1 || true + fi +} +trap cleanup EXIT +# Only dependency layers may be reused. Candidate source + daemon are rebuilt. +if [[ -z "${DELPHI_PROOF_BASE:-}" ]]; then + docker build --build-context queue-rs=queue-rs -t "$base" delphi > "$proof/dependencies-build.log" 2>&1 +fi +mkdir -p "$proof/binary" "$proof/target-linux" +(cd queue-rs && cargo fmt --check && cargo test --locked --lib && cargo build --locked -p polis-migrate) > "$proof/rust-unit.log" 2>&1 +# Keep Linux process-group fencing intact: the actual daemon and Python child +# execute together in the dedicated worker container, never through docker exec. +docker run --rm --name "${COMPOSE_PROJECT_NAME}-build" \ + -v "$repo/queue-rs:/source:ro" -v "$proof/target-linux:/target" \ + -v "${DYNAMO_PROOF_CARGO_REGISTRY:-${COMPOSE_PROJECT_NAME}-cargo}:/usr/local/cargo/registry" -w /source \ + -e CARGO_TARGET_DIR=/target \ + "${DYNAMO_PROOF_RUST_IMAGE:-rust:1.98.1-slim-bookworm}" \ + bash -c 'unset RUSTUP_TOOLCHAIN; export RUSTUP_TOOLCHAIN=$(basename /usr/local/rustup/toolchains/*); apt-get update -qq && apt-get install -y -qq pkg-config libssl-dev && cargo build --locked --bin polis-jobs' > "$proof/linux-build.log" 2>&1 +cp "$proof/target-linux/debug/polis-jobs" "$proof/binary/" +docker build -f delphi/tests/dynamo_removal/Dockerfile --build-arg "DELPHI_BASE=$base" \ + --build-context "queue-binary=$proof/binary" -t "$image" . > "$proof/worker-build.log" 2>&1 +"${compose[@]}" up -d --wait +# Use the release runner and real receipts, including M28/M29. The ordinary +# API startup check below stays enabled and verifies this exact source chain. +docker exec "$pg" psql -X -U postgres -v ON_ERROR_STOP=1 -c 'CREATE DATABASE dynamo1424' +DATABASE_URL="postgresql://postgres@127.0.0.1:$POLIS_RECOVERY_PG_PORT/dynamo1424?sslmode=disable" \ + POLIS_MIGRATIONS_DIR="$repo/server/postgres/migrations" \ + "$repo/queue-rs/target/debug/polis-migrate" apply > "$proof/install.log" 2>&1 +DATABASE_URL="postgresql://postgres@127.0.0.1:$POLIS_RECOVERY_PG_PORT/dynamo1424?sslmode=disable" \ + POLIS_MIGRATIONS_DIR="$repo/server/postgres/migrations" \ + "$repo/queue-rs/target/debug/polis-migrate" check >> "$proof/install.log" 2>&1 +python3 - "$proof/sql-source-sha256.json" <<'PYHASH' +import hashlib,json,pathlib,sys +paths=sorted(pathlib.Path('server/postgres/migrations').glob('*.sql')) +pathlib.Path(sys.argv[1]).write_text(json.dumps({str(p):hashlib.sha256(p.read_bytes()).hexdigest() for p in paths},indent=2)+'\n') +PYHASH +docker exec "$pg" psql -U postgres -v ON_ERROR_STOP=1 -c 'CREATE ROLE dynamo1424_worker LOGIN; GRANT polis_queue_executor TO dynamo1424_worker' +common=(--network "container:$pg" -e DATABASE_URL=postgresql://postgres@127.0.0.1/dynamo1424 \ + -e 'QUEUE_DATABASE_URL=postgresql://dynamo1424_worker@127.0.0.1/dynamo1424?sslmode=disable' \ + -e 'MATH_CAPACITY_QUEUE_DSN=postgresql://dynamo1424_worker@127.0.0.1/dynamo1424?sslmode=disable' \ + -e MATH_CAPACITY_QUEUE_ENV=demo1424 -e DATABASE_SSL_MODE=disable -v "$proof:/proof") +docker run --rm "${common[@]}" "$image" python tests/dynamo_removal/seed.py --source-agree=+1 | tee "$proof/seed.log" +docker run --rm --network none "$image" python -m pytest --noconftest -o addopts= -q \ + tests/poller/test_enqueue_math_rebuild.py tests/test_delphi_legacy_import.py tests/test_delphi_storage_codec.py \ + tests/test_delphi_postgres_results.py tests/test_delphi_result_resource.py tests/job_graph/test_numerical_stages.py \ + tests/test_narrative_audit.py > "$proof/python-unit.log" 2>&1 +# The same server tests and SQL boundary campaign run in hosted CI and on mm5. +server_image=${DYNAMO_PROOF_SERVER_IMAGE:-${COMPOSE_PROJECT_NAME}:server} +if [[ -z "${DYNAMO_PROOF_SERVER_IMAGE:-}" ]]; then + docker build --target dev -t "$server_image" server > "$proof/server-image-build.log" 2>&1 +fi +export DYNAMO_PROOF_SERVER_IMAGE="$server_image" +docker run --rm --network none -v "$repo/server:/candidate:ro" -v "$repo/delphi:/delphi:ro" \ + -e DATABASE_URL=postgresql://postgres@127.0.0.1:1/generated --entrypoint sh "$server_image" \ + -c 'cp -a /candidate/. /app/; cd /app; npm run build && npx jest --config characterization/delphi/jest.codec.config.json --runInBand' > "$proof/server-unit.log" 2>&1 +GRAPH_PROOF_ROOT="$proof/results-sql" GRAPH_PROOF_DB=dynamo1424 uv run --no-project --with PyYAML==6.0.2 python delphi/tests/job_graph/results.py > "$proof/results-sql.log" 2>&1 +worker_common=("${common[@]}" --memory 4g --cpus 3 -e POLIS_JOBS_ENABLED=1 -e QUEUE_ENV=demo1424 \ + -e POLIS_JOBS_TRANSPORT=loopback -e POLIS_JOBS_POLL_SECONDS=1 -e POLIS_JOBS_LEASE_SECONDS=60 \ + -e POLIS_JOBS_HEARTBEAT_SECONDS=5 -e POLIS_JOBS_JOURNAL_DIR=/worker/journal -e POLIS_JOBS_WORK_DIR=/worker/jobs \ + -e DELPHI_APP_PATH=/app) +math="${COMPOSE_PROJECT_NAME}-math"; owned+=("$math") +docker run --rm "${common[@]}" "$image" python scripts/enqueue_math_rebuild.py --zid 1424 \ + --staged-label demo1424-stage --target-label demo1424 --source-commit ce1038340d608e568681d33aa3fb31f59b366dee > "$proof/math-admission.json" +docker run -d --name "$math" "${worker_common[@]}" -v "$proof/math-worker:/worker" \ + -e POLIS_JOBS_WORKER_CLASS=large -e POLIS_JOBS_STAGES=math_rebuild -e POLIS_JOBS_PYTHON=python \ + -e MATH_ENV=demo1424-stage -e MATH_POLLER_MEMORY_LIMIT_MB=4096 \ + -e MATH_POLLER_SOURCE_COMMIT=ce1038340d608e568681d33aa3fb31f59b366dee "$image" polis-jobs +job=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["job_id"])' "$proof/math-admission.json") +docker run --rm "${common[@]}" "$image" python tests/dynamo_removal/math_proof.py --job-id "$job" \ + --zid 1424 --staged-label demo1424-stage --target-label demo1424 --wait-seconds 300 | tee "$proof/math-verification.log" +# Fixed narrative/name stand-ins exercise the same queue and result transport. +numerical=("${common[@]}" -e MATH_ENV=demo1424-stage) +docker run --rm "${numerical[@]}" "$image" python tests/dynamo_removal/admit_demo.py | tee "$proof/demo-admission.log" +delphi="${COMPOSE_PROJECT_NAME}-delphi"; owned+=("$delphi") +docker run -d --name "$delphi" "${worker_common[@]}" -v "$proof/delphi-worker:/worker" \ + -e POLIS_JOBS_WORKER_CLASS=delphi -e POLIS_JOBS_STAGES=graph_embed,graph_cluster,graph_topics,graph_narrative \ + -e POLIS_JOBS_PYTHON=/app/tests/dynamo_removal/fault_python.py \ + "$image" polis-jobs +docker run --rm "${numerical[@]}" "$image" python tests/dynamo_removal/wait_publish.py | tee "$proof/demo-verification.log" +docker run --rm "${numerical[@]}" "$image" python tests/dynamo_removal/audit_narrative.py | tee "$proof/narrative-audit.log" +# Actual DynamoDB export import; stop Dynamo before the queued import completes. +dynamo="${COMPOSE_PROJECT_NAME}-dynamo"; owned+=("$dynamo") +docker run -d --name "$dynamo" --network "container:$pg" amazon/dynamodb-local:3.3.1 -jar DynamoDBLocal.jar -sharedDb -inMemory +docker run --rm "${common[@]}" "$image" python tests/dynamo_removal/wait_dynamo.py +docker run --rm "${common[@]}" "$image" python tests/dynamo_removal/import_proof.py prepare --directory /proof/import --endpoint http://127.0.0.1:8000 +docker stop "$dynamo" +import_worker="${COMPOSE_PROJECT_NAME}-import"; owned+=("$import_worker") +docker run -d --name "$import_worker" "${worker_common[@]}" -v "$proof/import-worker:/worker" \ + -e QUEUE_ENV=proof-import -e POLIS_JOBS_WORKER_CLASS=delphi -e POLIS_JOBS_STAGES=graph_narrative -e POLIS_JOBS_PYTHON=python "$image" polis-jobs +docker run --rm "${common[@]}" "$image" python tests/dynamo_removal/import_proof.py verify --directory /proof/import --wait-seconds 300 +test "$(docker inspect "$dynamo" --format '{{.State.Running}}')" = false +# Build and render the actual report against the standard API. +export DYNAMO_PROOF_PG_DATABASE=dynamo1424 DYNAMO_PROOF_REPORT_ID=rlocaldynamo1424 +export DYNAMO_PROOF_ENV=demo1424 DYNAMO_PROOF_SCOPE=delphi DYNAMO_PROOF_OUTPUT="$proof/report" +bash scripts/prove-dynamo-report.sh +test "$(docker inspect "$dynamo" --format '{{.State.Running}}')" = false +docker inspect "$dynamo" --format '{{json .State}}' > "$proof/dynamo-stopped.json" +printf 'PASS math queue, full Delphi graph, retry/reuse, PostgreSQL report, stopped Dynamo import\n' diff --git a/server/Dockerfile b/server/Dockerfile index 0ef7bd96d4..b23a8774c1 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -18,7 +18,14 @@ ARG NODE_ENV WORKDIR /app # Install Python and necessary tools (including pip) -RUN apk add --no-cache python3 py3-pip +RUN apk add --no-cache python3 py3-pip ca-certificates + +# Verified startup migration checks use the same public RDS CA pin as the +# runner image. Deployments with another private CA can override this env. +RUN mkdir -p /etc/polis \ + && wget -O /etc/polis/rds-ca.pem https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem \ + && echo 'fe45bbebf92ad3e27a583bbb2ddd1553c521ed4d49af5514dc0a40372ea5395c /etc/polis/rds-ca.pem' | sha256sum -c - +ENV NODE_EXTRA_CA_CERTS=/etc/polis/rds-ca.pem # This package is needed at runtime RUN apk add libpq-dev diff --git a/server/Dockerfile-db b/server/Dockerfile-db index 3ba7e8a75c..3d1e57bda5 100644 --- a/server/Dockerfile-db +++ b/server/Dockerfile-db @@ -1,11 +1,12 @@ -# Example Usage: -# docker build -t polis-db:local -f Dockerfile-db . -# docker network create polis-net -# docker run --rm --name polis-db --network polis-net -p 5432:5432 \ -# -e POSTGRES_DB=polis-dev -e POSTGRES_PASSWORD=postgres polis-db:local - +# Build with --build-context queue-rs=queue-rs (Compose supplies it). +# New volumes use the same runner/history as every later deploy. +FROM docker.io/library/rust:1.98.1-alpine AS migrate-build +RUN apk add --no-cache musl-dev openssl-dev openssl-libs-static pkgconfig +WORKDIR /src +COPY --from=queue-rs . . +RUN cargo build --release --locked -p polis-migrate FROM postgres:17-alpine -# Used when no existing database on postgres volume, including first initialization. -# See: docs/deployment.md#database-migrations -# See: https://github.com/docker-library/docs/blob/master/postgres/README.md#initialization-scripts -COPY ./postgres/migrations/*.sql /docker-entrypoint-initdb.d/ +RUN apk add --no-cache libssl3 ca-certificates +COPY --from=migrate-build /src/target/release/polis-migrate /usr/local/bin/polis-migrate +COPY ./postgres/migrations /migrations +COPY ./postgres/init-migrations.sh /docker-entrypoint-initdb.d/00-migrations.sh diff --git a/server/README.md b/server/README.md index f800d84f87..78445edd8c 100644 --- a/server/README.md +++ b/server/README.md @@ -52,36 +52,19 @@ root folder of the polis project. To run everything but the API server in this f `docker compose -f docker-compose.yml -f docker-compose.dev.yml --profile postgres up math postgres file-server ses-local`. In this case the polis-dev database should be accessible at the default DATABASE_URL seen in server/example.env. -3\. Connect to the new database then run the migrations in its shell. You can skip this step if you built the -database with docker compose. - -```psql -\connect polis -\i postgres/migrations/000000_initial.sql -\i postgres/migrations/000001_update_pwreset_table.sql -\i postgres/migrations/000002_add_xid_constraint.sql -\i postgres/migrations/000003_add_origin_permanent_cookie_columns.sql -\i postgres/migrations/000004_drop_waitinglist_table.sql -\i postgres/migrations/000005_drop_slack_stripe_canvas.sql -\i postgres/migrations/000006_update_votes_rule.sql -\i postgres/migrations/000007_drop_geolocation_fields.sql -\i postgres/migrations/000008_add_comment_priority.sql -\i postgres/migrations/000009_add_uuid_to_zinvites.sql -``` - -You can also separately run `psql -d polis -f postgres/migrations/000000_initial.sql` and -`psql -d polis -f postgres/migrations/000001_update_pwreset_table.sql` etc. from the shell. - -Alternatively, you can use the provided migration script to run all migrations in sequence: +3\. Set `DATABASE_URL` in the environment and run the migration runner from the +repository root. A fresh Compose database does this during initialization. ```sh -# Using DATABASE_URL from environment -./bin/run-migrations.sh - -# Or providing the URL as argument -./bin/run-migrations.sh "postgres://username:password@localhost:5432/polis" +cargo build --locked --release --manifest-path queue-rs/Cargo.toml -p polis-migrate +queue-rs/target/release/polis-migrate apply +queue-rs/target/release/polis-migrate check ``` +Existing databases need catalog-checked reconciliation once. See +[database migrations](../docs/migrations.md) for the commands and release holds. +Do not replay the initial SQL or pass credentials as command-line arguments. + 4\. Update database connection settings in `.env`. Replace the username, password, and database_name in the DATABASE_URL ```sh diff --git a/server/__tests__/integration/conversation-stats.test.ts b/server/__tests__/integration/conversation-stats.test.ts index b6253b431a..52ee7399ad 100644 --- a/server/__tests__/integration/conversation-stats.test.ts +++ b/server/__tests__/integration/conversation-stats.test.ts @@ -9,6 +9,7 @@ import { submitVote, } from "../setup/api-test-helpers"; import { getPooledTestUser } from "../setup/test-user-helpers"; +import pg from "../../src/db/pg-query"; interface ConversationStats { voteTimes: number[]; @@ -85,8 +86,14 @@ describe("Conversation Stats API", () => { }); test("GET /api/v3/conversationStats - should accept until parameter", async () => { - // Get current time in milliseconds - const currentTimeMs = Date.now(); + // Use the same clock as the stored timestamps. A Docker VM can be ahead + // of the host, so Date.now() immediately after the vote may exclude it. + // The route uses a strict < cutoff; advance past the current DB millisecond. + const rows = (await pg.queryP_readOnly( + "SELECT now_as_millis() AS now", + [] + )) as Array<{ now: string }>; + const currentTimeMs = Number(rows[0].now) + 1; const response: Response = await agent.get( `/api/v3/conversationStats?conversation_id=${conversationId}&until=${currentTimeMs}` diff --git a/server/__tests__/integration/delphi-job-table.test.ts b/server/__tests__/integration/delphi-job-table.test.ts index 327ea6cd33..767f137af1 100644 --- a/server/__tests__/integration/delphi-job-table.test.ts +++ b/server/__tests__/integration/delphi-job-table.test.ts @@ -1,6 +1,8 @@ /** * The Delphi job table: migration 000023, contract polis-queue/2, and the * large worker class on top of it: migration 000024, contract polis-queue/3. + * Migration 000027 adds graph stages under the install contract polis-queue/5; + * M29 adds graph_topics; existing RPC wire versions and worker classes remain unchanged. * * The migrations reach the test database the way every migration does: the * postgres image applies server/postgres/migrations/*.sql once at initdb. So @@ -66,13 +68,13 @@ async function one(text: string, values?: unknown[]): Promise { return Object.values(result.rows[0] ?? {})[0] as T; } -describe("the Delphi job table (000023) under the large class (000024)", () => { - it("records contract polis-queue/3 in the queue's install row", async () => { +describe("the Delphi job table (000023) with large class (000024) and graph stages (000027)", () => { + it("records contract polis-queue/5 in the queue's install row", async () => { expect( await one( "SELECT contract_version FROM public.polis_queue_install", ), - ).toBe("polis-queue/3"); + ).toBe("polis-queue/5"); }); it.each(JOB_TABLES)( @@ -101,14 +103,14 @@ describe("the Delphi job table (000023) under the large class (000024)", () => { }, ); - it("admits exactly the daemon's two stages and the rebuild beside noop, and holds no job of any", async () => { + it("admits exactly noop, both daemon stages, rebuild and the three graph stages, and holds no job of any", async () => { expect( await one( "SELECT pg_get_constraintdef(oid) FROM pg_constraint " + "WHERE conrelid = 'public.polis_queue_jobs'::regclass AND conname = 'polis_queue_jobs_stage_check'", ), ).toBe( - "CHECK ((stage = ANY (ARRAY['noop'::text, 'delphi_full_pipeline'::text, 'delphi_narrative'::text, 'math_rebuild'::text])))", + "CHECK ((stage = ANY (ARRAY['noop'::text, 'delphi_full_pipeline'::text, 'delphi_narrative'::text, 'math_rebuild'::text, 'graph_embed'::text, 'graph_cluster'::text, 'graph_topics'::text, 'graph_narrative'::text])))", ); expect( await one( @@ -117,7 +119,7 @@ describe("the Delphi job table (000023) under the large class (000024)", () => { ).toBe(0); }); - it("admits exactly the classes noop, delphi and large, and binds the rebuild stage to the large class", async () => { + it("admits exactly noop, delphi and large, with large required for rebuild and optional for graph clustering", async () => { expect( await one( "SELECT pg_get_constraintdef(oid) FROM pg_constraint " + @@ -132,7 +134,7 @@ describe("the Delphi job table (000023) under the large class (000024)", () => { "WHERE conrelid = 'public.polis_queue_jobs'::regclass AND conname = 'pq_stage_large'", ), ).toBe( - "CHECK (((stage = 'math_rebuild'::text) = (worker_class = 'large'::text)))", + "CHECK ((((stage = 'math_rebuild'::text) AND (worker_class = 'large'::text)) OR ((stage = 'graph_cluster'::text) AND (worker_class = ANY (ARRAY['delphi'::text, 'large'::text]))) OR ((stage <> ALL (ARRAY['math_rebuild'::text, 'graph_cluster'::text])) AND (worker_class <> 'large'::text))))", ); }); diff --git a/server/__tests__/integration/queue-substrate.test.ts b/server/__tests__/integration/queue-substrate.test.ts index 4960a56ac2..02eda17d44 100644 --- a/server/__tests__/integration/queue-substrate.test.ts +++ b/server/__tests__/integration/queue-substrate.test.ts @@ -827,6 +827,11 @@ describeProvisioned( it("replays for a superuser against the real conversations schema, or refuses once 000023 is in place", async () => { requireProvisioning(); if (foundationInstalled) { + // A newer additive contract must remain untouched by an old replay. + const installedBeforeReplay = await sql( + mainPool, + "SELECT contract_version FROM public.polis_queue_install" + ); await expect(runMigration(mainPool)).rejects.toThrow( /queue catalog drift/ ); @@ -835,7 +840,7 @@ describeProvisioned( mainPool, "SELECT contract_version FROM public.polis_queue_install" ) - ).toBe(largeClassInstalled ? "polis-queue/3" : "polis-queue/2"); + ).toBe(installedBeforeReplay); return; } await runMigration(mainPool); diff --git a/server/__tests__/unit/binBoundaries.test.ts b/server/__tests__/unit/binBoundaries.test.ts index 84fda1ae60..051908c71e 100644 --- a/server/__tests__/unit/binBoundaries.test.ts +++ b/server/__tests__/unit/binBoundaries.test.ts @@ -13,9 +13,10 @@ beforeEach(() => { afterEach(() => fs.rmSync(scratch, { recursive: true, force: true })); const publicEnvUrl = "postgresql://public:public-fixture@public.invalid/public"; -const publicArgUrl = "postgresql://public:other-fixture@other.invalid/public"; - -function migrationFixture(files: Record = {}) { +// The shell entrypoint delegates policy and SQL execution to polis-migrate. +// Its boundary keeps the DSN in the environment and preserves runner failures; +// real SQL/history/startup refusal is covered in polis-migrate/tests/prove.py. +function migrationFixture() { const script = path.join(scratch, "server/bin/run-migrations.sh"); const migrations = path.join(scratch, "server/postgres/migrations"); const fakeBin = path.join(scratch, "fake-bin"); @@ -23,129 +24,84 @@ function migrationFixture(files: Record = {}) { fs.mkdirSync(path.dirname(script), { recursive: true }); fs.mkdirSync(migrations, { recursive: true }); fs.mkdirSync(fakeBin); - // Byte-identical script relocated so it can see only public SQL fixtures. fs.copyFileSync(path.join(serverRoot, "bin/run-migrations.sh"), script); - for (const [name, contents] of Object.entries(files)) { - const file = path.join(migrations, name); - fs.mkdirSync(path.dirname(file), { recursive: true }); - fs.writeFileSync(file, contents); - } - const fakePsql = path.join(fakeBin, "psql"); - fs.writeFileSync( - fakePsql, - `#!${process.execPath} + const fakeRunner = path.join(fakeBin, "polis-migrate"); + fs.writeFileSync(fakeRunner, `#!${process.execPath} const fs = require('fs'); -const path = require('path'); -const [dsn, flag, file, extra] = process.argv.slice(2); -const allowed = ${JSON.stringify([publicEnvUrl, publicArgUrl])}; -if (!allowed.includes(dsn) || flag !== '-f' || extra || !file || - path.dirname(file) !== process.env.PUBLIC_MIGRATIONS) { - throw new Error('not an admitted public invocation'); -} -const contents = fs.readFileSync(file, 'utf8'); -if (!contents.startsWith('-- public fixture')) throw new Error('not public fixture SQL'); fs.appendFileSync(process.env.PUBLIC_TRACE, JSON.stringify({ - dsn, file: path.basename(file), password: process.env.PGPASSWORD, contents + args: process.argv.slice(2), dsn: process.env.DATABASE_URL }) + '\\n'); -process.exit(path.basename(file) === process.env.PUBLIC_FAIL ? 17 : 0); -` - ); - fs.chmodSync(fakePsql, 0o700); +process.exit(Number(process.env.PUBLIC_EXIT)); +`); + fs.chmodSync(fakeRunner, 0o700); return { - run(args: string[] = [], envUrl?: string, fail = "") { - const result = spawnSync("/bin/bash", [script, ...args], { + migrations, + fakeRunner, + run(options: { args?: string[]; code?: number; binary?: string; envUrl?: string } = {}) { + const result = spawnSync("/bin/bash", [script, ...(options.args || [])], { cwd: scratch, encoding: "utf8", timeout: 5000, env: { PATH: `${fakeBin}:/usr/bin:/bin`, - PUBLIC_MIGRATIONS: migrations, PUBLIC_TRACE: trace, - PUBLIC_FAIL: fail, - ...(envUrl === undefined ? {} : { DATABASE_URL: envUrl }), + PUBLIC_EXIT: String(options.code || 0), + DATABASE_URL: options.envUrl ?? publicEnvUrl, + ...(options.binary ? { POLIS_MIGRATE_BIN: options.binary } : {}), }, }); if (result.error) throw result.error; const calls = fs.existsSync(trace) - ? fs - .readFileSync(trace, "utf8") - .trim() - .split("\n") - .filter(Boolean) - .map((line) => JSON.parse(line)) + ? fs.readFileSync(trace, "utf8").trim().split("\n").map((line) => JSON.parse(line)) : []; return { ...result, calls }; }, }; } -const publicSql = "-- public fixture\nSELECT 'public value';\n"; - -test("migration runner requires a URL before invoking any psql process", () => { - const result = migrationFixture({ "001.sql": publicSql }).run(); - expect(result.status).toBe(1); - expect(result.stdout).toContain("DATABASE_URL is not set"); - expect(result.stdout).not.toContain("All migrations completed successfully"); - expect(result.calls).toEqual([]); +test("migration wrapper delegates once with the checkout's absolute source directory", () => { + const fixture = migrationFixture(); + const result = fixture.run(); + expect(result.status).toBe(0); + expect(result.calls).toEqual([{ args: ["apply", "--dir", fixture.migrations], dsn: publicEnvUrl }]); }); -test("migration runner sorts top-level SQL and ignores nested SQL and other extensions", () => { - const result = migrationFixture({ - "010-last.sql": publicSql, - "002-first.sql": publicSql, - "nested/001-ignored.sql": publicSql, - "003-ignored.txt": "not SQL", - }).run([], publicEnvUrl); +test("migration wrapper keeps the DSN out of argv and output", () => { + const result = migrationFixture().run(); expect(result.status).toBe(0); - expect(result.calls.map((call) => call.file)).toEqual([ - "002-first.sql", - "010-last.sql", - ]); - for (const call of result.calls) { - expect(call.dsn).toBe(publicEnvUrl); - expect(call.password).toBe("public-fixture"); - expect(call.contents).toBe(publicSql); - } - expect(result.stdout).toContain("All migrations completed successfully!"); + expect(result.calls).toHaveLength(1); + expect(result.calls[0].args.join(" ")).not.toContain(publicEnvUrl); + expect(result.stdout + result.stderr).not.toContain(publicEnvUrl); }); -test("explicit migration URL overrides the environment and binds matching password", () => { - const result = migrationFixture({ "001.sql": publicSql }).run( - [publicArgUrl], - publicEnvUrl - ); +test("migration wrapper supports an explicitly installed binary path containing spaces", () => { + const fixture = migrationFixture(); + const binary = path.join(scratch, "installed runner"); + fs.copyFileSync(fixture.fakeRunner, binary); + const result = fixture.run({ binary }); expect(result.status).toBe(0); expect(result.calls).toHaveLength(1); - expect(result.calls[0].dsn).toBe(publicArgUrl); - expect(result.calls[0].password).toBe("other-fixture"); }); -test("empty explicit URL preserves the environment URL", () => { - const result = migrationFixture({ "001.sql": publicSql }).run( - [""], - publicEnvUrl - ); - expect(result.status).toBe(0); - expect(result.calls[0].dsn).toBe(publicEnvUrl); +test("migration wrapper preserves argument boundaries for runner validation", () => { + const fixture = migrationFixture(); + const result = fixture.run({ args: ["--unknown-option", "value with spaces"], code: 2 }); + expect(result.status).toBe(2); + expect(result.calls[0].args).toEqual(["apply", "--dir", fixture.migrations, "--unknown-option", "value with spaces"]); }); -test("psql failure stops later migrations and cannot print completion", () => { - const result = migrationFixture({ - "001.sql": publicSql, - "002.sql": publicSql, - "003.sql": publicSql, - }).run([], publicEnvUrl, "002.sql"); +test("migration wrapper propagates runner failure without a success message or retry", () => { + const result = migrationFixture().run({ code: 17 }); expect(result.status).toBe(17); - expect(result.calls.map((call) => call.file)).toEqual(["001.sql", "002.sql"]); - expect(result.stdout).not.toContain("Applying migration: 003.sql"); - expect(result.stdout).not.toContain("All migrations completed successfully"); + expect(result.calls).toHaveLength(1); + expect(result.stdout).toBe(""); }); -test("empty migration directory completes without inventing a database call", () => { - const result = migrationFixture().run([], publicEnvUrl); - expect(result.status).toBe(0); +test("missing runner refuses without falling back to untracked SQL", () => { + const result = migrationFixture().run({ binary: path.join(scratch, "absent") }); + expect(result.status).not.toBe(0); // Bash 3/macOS: 1; Bash 5/Linux: 127. expect(result.calls).toEqual([]); - expect(result.stdout).toContain("All migrations completed successfully!"); + expect(result.stdout).toBe(""); }); function resetBoundary(databaseUrl: unknown) { diff --git a/server/__tests__/unit/delphi-job-guard.test.ts b/server/__tests__/unit/delphi-job-guard.test.ts index 2b99284546..50f0595910 100644 --- a/server/__tests__/unit/delphi-job-guard.test.ts +++ b/server/__tests__/unit/delphi-job-guard.test.ts @@ -1,3 +1,4 @@ +jest.mock("../../src/db/pg-query", () => ({__esModule:true,default:{queryP:jest.fn()}})); /** * P-003 S3 — failure and resolution paths of the Delphi active-work guard. * diff --git a/server/__tests__/unit/delphi-postgres-liveness.test.ts b/server/__tests__/unit/delphi-postgres-liveness.test.ts new file mode 100644 index 0000000000..1f0d56d91c --- /dev/null +++ b/server/__tests__/unit/delphi-postgres-liveness.test.ts @@ -0,0 +1,61 @@ +jest.mock("../../src/db/pg-query", () => ({__esModule:true,default:{queryP:jest.fn()}})); +jest.mock("../../src/utils/logger", () => ({__esModule:true,default:{warn:jest.fn(),info:jest.fn()}})); +jest.mock("@aws-sdk/client-dynamodb", () => ({DynamoDB:jest.fn(() => {throw new Error("AWS must not be constructed");})})); +jest.mock("@aws-sdk/lib-dynamodb", () => ({DynamoDBDocument:{from:jest.fn(() => {throw new Error("AWS must not be constructed");})}})); +import pg from "../../src/db/pg-query"; +import {assessConversationLiveness} from "../../src/routes/delphi/jobGuard"; + +beforeEach(() => { + jest.clearAllMocks(); + process.env.DELPHI_RESULT_BACKEND="postgres"; + process.env.DELPHI_RESULT_ENV="generated"; + process.env.DELPHI_RESULT_SCOPE="published"; +}); +afterAll(() => { + delete process.env.DELPHI_RESULT_BACKEND; + delete process.env.DELPHI_RESULT_ENV; + delete process.env.DELPHI_RESULT_SCOPE; +}); + +test("Postgres liveness uses bound namespace and one authoritative snapshot without AWS",async () => { + (pg.queryP as jest.Mock).mockResolvedValue([ + {job_id:"finished",status:"COMPLETED",work_live:false}, + {job_id:"unconfirmed-process",status:"FAILED",work_live:true}, + {job_id:"provider-pending",status:"COMPLETED",work_live:true} + ]); + const store={sweepConversation:jest.fn()}; + const result=await assessConversationLiveness("42' OR TRUE",store as any); + expect(result.complete).toBe(true); + expect([...result.liveByJobId]).toEqual([["finished",false],["unconfirmed-process",true],["provider-pending",true]]); + expect(result.rowsByJobId.get("finished").status).toBe("COMPLETED"); + expect(store.sweepConversation).not.toHaveBeenCalled(); + expect(pg.queryP).toHaveBeenCalledTimes(1); + const [sql,values]=(pg.queryP as jest.Mock).mock.calls[0]; + expect(values).toEqual(["generated","42' OR TRUE","published"]); + expect(sql).not.toContain("42' OR TRUE"); + expect(sql).toContain("process_exit_confirmed_at IS NULL"); + expect(sql).toContain("submission_unknown"); +}); + +test("Postgres failure stays uncertain without falling back to Dynamo",async () => { + (pg.queryP as jest.Mock).mockRejectedValue(new Error("database unavailable")); + const store={sweepConversation:jest.fn()}; + const result=await assessConversationLiveness("42",store as any); + expect(result.complete).toBe(false); + expect(result.liveByJobId.size).toBe(0); + expect(store.sweepConversation).not.toHaveBeenCalled(); +}); + +test("missing result environment refuses an unscoped query",async () => { + delete process.env.DELPHI_RESULT_ENV; + await expect(assessConversationLiveness("42")).rejects.toThrow("DELPHI_RESULT_ENV"); + expect(pg.queryP).not.toHaveBeenCalled(); +}); + +test("legacy backend preserves the supplied admission store",async () => { + process.env.DELPHI_RESULT_BACKEND="dynamodb"; + const store={sweepConversation:jest.fn().mockResolvedValue({kind:"none"})}; + expect((await assessConversationLiveness("42",store as any)).complete).toBe(true); + expect(store.sweepConversation).toHaveBeenCalledWith("42"); + expect(pg.queryP).not.toHaveBeenCalled(); +}); diff --git a/server/__tests__/unit/delphi-postgres-read-routes.test.ts b/server/__tests__/unit/delphi-postgres-read-routes.test.ts new file mode 100644 index 0000000000..9aad88977f --- /dev/null +++ b/server/__tests__/unit/delphi-postgres-read-routes.test.ts @@ -0,0 +1,36 @@ +jest.mock("../../src/db/pg-query",()=>({__esModule:true,default:{queryP:jest.fn()}})); +jest.mock("../../src/utils/logger",()=>({__esModule:true,default:{info:jest.fn(),warn:jest.fn(),error:jest.fn(),debug:jest.fn()}})); +jest.mock("../../src/config",()=>({__esModule:true,default:jest.requireActual("../../src/config").default})); +jest.mock("../../src/utils/parameter",()=>({getZidFromReport:jest.fn().mockResolvedValue(1)})); +jest.mock("../../src/routes/delphi/jobGuard",()=>({assessConversationLiveness:jest.fn()})); +jest.mock("@aws-sdk/client-s3",()=>({S3Client:jest.fn(()=>{throw new Error("no S3 allowed")}),ListObjectsV2Command:jest.fn()})); +import pg from "../../src/db/pg-query"; +import {S3Client} from "@aws-sdk/client-s3"; +import {DynamoDBClient} from "@aws-sdk/client-dynamodb"; +import {assessConversationLiveness} from "../../src/routes/delphi/jobGuard"; +import {handle_GET_delphi_visualizations} from "../../src/routes/delphi/visualizations"; +import {getCurrentDelphiJobId} from "../../src/routes/delphi/topicAgenda"; +import {encodeFamily} from "../../src/utils/delphiStorageCodec"; +beforeEach(()=>{ + jest.clearAllMocks();jest.spyOn(DynamoDBClient.prototype,"send").mockRejectedValue(new Error("no DynamoDB allowed") as never);process.env.DELPHI_RESULT_BACKEND="postgres";process.env.DELPHI_RESULT_ENV="test";process.env.DELPHI_RESULT_SCOPE="delphi"; +}); +afterAll(()=>{delete process.env.DELPHI_RESULT_BACKEND;delete process.env.DELPHI_RESULT_ENV;delete process.env.DELPHI_RESULT_SCOPE;}); +test("agenda attributes selections to the published root, not a newer completed math job",async()=>{ + (pg.queryP as jest.Mock).mockResolvedValue([{job_id:"served-root"}]); + expect(await getCurrentDelphiJobId("1")).toBe("served-root"); + expect((pg.queryP as jest.Mock).mock.calls[0][0]).toContain("delphi_result_publications"); + expect((pg.queryP as jest.Mock).mock.calls[0][1]).toEqual(["test",1,"delphi"]); + expect(DynamoDBClient.prototype.send).not.toHaveBeenCalled(); +}); +test("visualization metadata is local and archived processing controls are never live work",async()=>{ + const codec_wire=encodeFamily("Delphi_JobQueue",[{job_id:{S:"historical"},conversation_id:{S:"1"},created_at:{S:"2026-01-01T00:00:00Z"},status:{S:"PROCESSING"}}]).toString(); + (pg.queryP as jest.Mock).mockResolvedValueOnce([{job_id:"current",conversation_id:"1",created_at:"2026-01-02T00:00:00Z",status:"COMPLETED"}]) + .mockResolvedValueOnce([{zid:1,scope_key:"delphi",generation:"1",codec_wire}]); + (assessConversationLiveness as jest.Mock).mockResolvedValue({complete:true,liveByJobId:new Map([["current",false]]),rowsByJobId:new Map()}); + const res:any={status:jest.fn().mockReturnThis(),json:jest.fn().mockReturnThis()}; + await handle_GET_delphi_visualizations({query:{report_id:"rlocaltest"}} as any,res); + expect(res.json.mock.calls[0][0]).toMatchObject({status:"success",visualizations:[],jobs:expect.arrayContaining([ + expect.objectContaining({jobId:"historical",status:"PROCESSING",archived:true,workLive:false}),expect.objectContaining({jobId:"current",workLive:false}) + ])}); + expect(S3Client).not.toHaveBeenCalled();expect(DynamoDBClient.prototype.send).not.toHaveBeenCalled(); +}); diff --git a/server/__tests__/unit/delphiResultSnapshot.test.ts b/server/__tests__/unit/delphiResultSnapshot.test.ts new file mode 100644 index 0000000000..9a36207f3f --- /dev/null +++ b/server/__tests__/unit/delphiResultSnapshot.test.ts @@ -0,0 +1,55 @@ +import { EventEmitter } from "events"; +import pg from "../../src/db/pg-query"; +import { delphiResultSnapshot, resultQuery } from "../../src/utils/delphiResultSnapshot"; +jest.mock("../../src/db/pg-query", () => ({__esModule:true, default:{connectResultSnapshot:jest.fn(),queryP:jest.fn()}})); +jest.mock("../../src/utils/logger", () => ({__esModule:true,default:{error:jest.fn()}})); +const tick = () => new Promise(resolve => setImmediate(resolve)); +beforeEach(() => { jest.clearAllMocks(); process.env.DELPHI_RESULT_BACKEND="postgres"; }); +afterAll(() => { delete process.env.DELPHI_RESULT_BACKEND; }); +test("parallel family reads share a transaction and finish/close release once", async () => { + const client = Object.assign(new EventEmitter(), {query:jest.fn().mockResolvedValue({rows:[{value:1}]}),release:jest.fn()}); + (pg.connectResultSnapshot as jest.Mock).mockResolvedValue(client); + const res = new EventEmitter(); + await new Promise((resolve,reject) => delphiResultSnapshot({} as any,res as any, (() => { + Promise.all([resultQuery("SELECT topics"),resultQuery("SELECT assignments")]).then(() => resolve(),reject); + }) as any)); + expect(pg.connectResultSnapshot).toHaveBeenCalledTimes(1); + expect(client.query.mock.calls[0][0]).toBe("BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY"); + res.emit("finish"); res.emit("close"); await tick(); + expect(client.query.mock.calls.filter(([sql]) => sql === "ROLLBACK")).toHaveLength(1); + expect(client.release).toHaveBeenCalledTimes(1); +}); +test("requests without results acquire no client, and failed setup releases it", async () => { + const empty = new EventEmitter(); + delphiResultSnapshot({} as any,empty as any, (()=>{}) as any); + empty.emit("close"); expect(pg.connectResultSnapshot).not.toHaveBeenCalled(); + const failure = new Error("begin failed"); + const client = Object.assign(new EventEmitter(), {query:jest.fn().mockRejectedValue(failure),release:jest.fn()}); + (pg.connectResultSnapshot as jest.Mock).mockResolvedValue(client); + const res = new EventEmitter(); + await new Promise((resolve,reject) => delphiResultSnapshot({} as any,res as any, (() => { + resultQuery("SELECT topics").then(()=>reject(new Error("expected failure")),error=>{ + expect(error).toBe(failure); resolve(); + }); + }) as any)); + res.emit("close"); await tick(); + expect(client.release).toHaveBeenCalledTimes(1); + expect(client.release).toHaveBeenCalledWith(failure); +}); + +test("connection errors cannot crash the process or reuse a failed snapshot", async () => { + const client = Object.assign(new EventEmitter(), {query:jest.fn().mockResolvedValue({rows:[]}),release:jest.fn()}); + (pg.connectResultSnapshot as jest.Mock).mockResolvedValue(client); + const res = new EventEmitter(); + const failure = new Error("idle transaction terminated"); + await new Promise((resolve,reject) => delphiResultSnapshot({} as any,res as any, (() => { + (async () => { + await resultQuery("SELECT topics"); + client.emit("error",failure); + await expect(resultQuery("SELECT assignments")).rejects.toBe(failure); + resolve(); + })().catch(reject); + }) as any)); + res.emit("finish"); await tick(); + expect(client.release).toHaveBeenCalledWith(failure); +}); diff --git a/server/__tests__/unit/delphiResults.test.ts b/server/__tests__/unit/delphiResults.test.ts new file mode 100644 index 0000000000..fd60535362 --- /dev/null +++ b/server/__tests__/unit/delphiResults.test.ts @@ -0,0 +1,103 @@ +jest.mock("../../src/db/pg-query", () => ({__esModule:true,default:{queryP:jest.fn()}})); +import pg from "../../src/db/pg-query"; +import {sendPostgresResult,sqlPredicate,resultClient,decodeAttribute} from "../../src/utils/delphiResults"; +import {encodeFamily} from "../../src/utils/delphiStorageCodec"; +class QueryCommand { constructor(public input:any) {} } +class GetCommand { constructor(public input:any) {} } +const family="Delphi_CommentEmbeddings"; +const row=(id:number,generation="1",scope_key="scope")=>({zid:1,scope_key,generation,item:{conversation_id:{S:"1"},comment_id:{N:String(id)}}}); + +beforeEach(()=>{jest.clearAllMocks();process.env.DELPHI_RESULT_BACKEND="postgres";process.env.DELPHI_RESULT_ENV="generated";delete process.env.DELPHI_RESULT_SCOPE;}); +afterAll(()=>{delete process.env.DELPHI_RESULT_BACKEND;delete process.env.DELPHI_RESULT_ENV;}); + +test("key expressions use parameters for untrusted values",()=>{ + const values:any[]=[]; + const text=sqlPredicate("#c = :c AND begins_with(topic_key, :prefix)",{ExpressionAttributeNames:{"#c":"conversation_id"},ExpressionAttributeValues:{":c":"1' OR TRUE",":prefix":"%_"}},values); + expect(text).not.toContain("1' OR TRUE");expect(text).toContain("starts_with");expect(values).toContain('%_'); +}); +test("pagination carries generation and refuses mixed snapshots",async()=>{ + (pg.queryP as jest.Mock).mockResolvedValue([row(0),row(1)]); + const first=await sendPostgresResult(new QueryCommand({TableName:family,Limit:1})); + expect(first.Items[0].comment_id).toBe(0); + const second=await sendPostgresResult(new QueryCommand({TableName:family,Limit:1,ExclusiveStartKey:first.LastEvaluatedKey})); + expect(second.Items[0].comment_id).toBe(1); + (pg.queryP as jest.Mock).mockResolvedValue([row(0,"2"),row(1,"2")]); + await expect(sendPostgresResult(new QueryCommand({TableName:family,ExclusiveStartKey:first.LastEvaluatedKey}))).rejects.toThrow("generation changed"); +}); +test("SQL is scoped to environment and configured publication",async()=>{ + process.env.DELPHI_RESULT_SCOPE="selected";(pg.queryP as jest.Mock).mockResolvedValue([row(0)]); + await sendPostgresResult(new QueryCommand({TableName:family,KeyConditionExpression:"conversation_id = :id",ExpressionAttributeValues:{":id":"1"}})); + expect((pg.queryP as jest.Mock).mock.calls[0][1]).toEqual(['generated',family,'selected','conversation_id',JSON.stringify({S:'1'})]); +}); +test("conflicting published scopes fail visibly",async()=>{ + (pg.queryP as jest.Mock).mockResolvedValue([row(0),{...row(0,"1","other"),item:{...row(0).item,text:{S:"different"}}}]); + await expect(sendPostgresResult(new QueryCommand({TableName:family}))).rejects.toThrow("Ambiguous"); +}); +test("Postgres error never creates an AWS client",async()=>{ + const factory=jest.fn();(pg.queryP as jest.Mock).mockRejectedValue(new Error("database unavailable")); + const client=resultClient(factory); + await expect(client.send(new QueryCommand({TableName:family}))).rejects.toThrow("database unavailable"); + expect(factory).not.toHaveBeenCalled(); +}); +test("unsafe integral numeric values fail instead of rounding",()=>{ + expect(()=>decodeAttribute({N:"9007199254740993"})).toThrow("precision"); +}); + +test("ConversationIndex selects newest job by creation time, not UUID",async()=>{ + (pg.queryP as jest.Mock).mockResolvedValueOnce([ + {job_id:"z-older",conversation_id:"1",created_at:"2025-01-01T00:00:00Z",status:"COMPLETED"}, + {job_id:"a-newest",conversation_id:"1",created_at:"2026-01-01T00:00:00Z",status:"COMPLETED"} + ]).mockResolvedValueOnce([]); + const reply=await sendPostgresResult(new QueryCommand({TableName:"Delphi_JobQueue",IndexName:"ConversationIndex",ScanIndexForward:false,Limit:1})); + expect(reply.Items[0].job_id).toBe("a-newest"); +}); + +test("archived metadata preserves NUL, exposes exact unsafe decimal and never replaces a current queue row",async()=>{ + const codec_wire=encodeFamily("Delphi_JobQueue",[ + {job_id:{S:"legacy"},conversation_id:{S:"1"},status:{S:"PROCESSING"},logs:{S:"a\0b"},job_config:{M:{large:{N:"9007199254740993"}}}}, + {job_id:{S:"current"},status:{S:"PROCESSING"}} + ]).toString(); + (pg.queryP as jest.Mock).mockResolvedValueOnce([{job_id:"current",status:"COMPLETED"}]) + .mockResolvedValueOnce([{zid:1,scope_key:"scope",generation:"4",codec_wire}]); + const reply=await sendPostgresResult(new QueryCommand({TableName:"Delphi_JobQueue"})); + const legacy=reply.Items.find((item:any)=>item.job_id==="legacy"); + expect(legacy).toMatchObject({archived:true,logs:"a\0b",status:"PROCESSING",job_config:{large:"9007199254740993"}}); + expect(legacy.unreadable_metadata_reason).toContain("precision"); + expect(legacy.legacy_control_item.job_config.M.large.N).toBe("9007199254740993"); + expect(reply.Items.find((item:any)=>item.job_id==="current").status).toBe("COMPLETED"); +}); + +test("modern job metadata uses the same graph scope as liveness and archived controls",async()=>{ + process.env.DELPHI_RESULT_SCOPE="delphi"; + (pg.queryP as jest.Mock).mockResolvedValue([]); + await sendPostgresResult(new QueryCommand({TableName:"Delphi_JobQueue"})); + expect((pg.queryP as jest.Mock).mock.calls).toHaveLength(2); + for (const [sql,values] of (pg.queryP as jest.Mock).mock.calls) { + expect(sql).toContain("scope_key=$2");expect(values).toEqual(["generated","delphi"]); + } +}); + +test("job get binds current attempt logs while job listings do not fetch log payloads",async()=>{ + const job_id="11111111-1111-1111-1111-111111111111",attempt="22222222-2222-2222-2222-222222222222"; + (pg.queryP as jest.Mock).mockResolvedValueOnce([{job_id,conversation_id:"1"}]).mockResolvedValueOnce([]) + .mockResolvedValueOnce([{value:{attempt_id:attempt}}]).mockResolvedValueOnce([{timestamp:"2026-01-01T00:00:00Z",level:"INFO",message:"actual child output"}]); + const reply=await sendPostgresResult(new GetCommand({TableName:"Delphi_JobQueue",Key:{job_id}})); + expect(reply.Item.logs.entries[0].message).toBe("actual child output");expect(reply.Item.log_attempt_id).toBe(attempt); + const calls=(pg.queryP as jest.Mock).mock.calls; + expect(calls[2][1]).toEqual(["generated",job_id]);expect(calls[3][1]).toEqual(["generated",attempt]); + expect(calls[3][0]).toContain("pq_attempt_logs($1::text,$2::uuid,NULL,1000)"); + expect(calls[3][0]).toContain("stream IN ('stdout','stderr')"); +}); + +// A merge must preserve the existing reader without any activation setting. +test.each([undefined, "dynamodb"])("backend %s forwards unchanged to DynamoDB", async backend => { + if (backend === undefined) delete process.env.DELPHI_RESULT_BACKEND; + else process.env.DELPHI_RESULT_BACKEND = backend; + const command = new QueryCommand({TableName:family}); + const response = {Items:[{legacy:true}]}; + const send = jest.fn().mockResolvedValue(response); + const factory = jest.fn(() => ({send})); + expect(await resultClient(factory).send(command)).toBe(response); + expect(send).toHaveBeenCalledWith(command); + expect(pg.queryP).not.toHaveBeenCalled(); +}); diff --git a/server/__tests__/unit/delphiStoragePagination.test.ts b/server/__tests__/unit/delphiStoragePagination.test.ts new file mode 100644 index 0000000000..3b5bc6e2a6 --- /dev/null +++ b/server/__tests__/unit/delphiStoragePagination.test.ts @@ -0,0 +1,49 @@ +jest.mock("../../src/db/pg-query", () => ({__esModule:true,default:{queryP:jest.fn()}})); +jest.mock("../../src/config", () => ({__esModule:true,default:jest.requireActual("../../src/config").default})); +jest.mock("../../src/utils/logger", () => ({__esModule:true,default:{debug:jest.fn(),error:jest.fn(),warn:jest.fn()}})); +jest.mock("../../src/utils/dynamoClient", () => ({makeDynamoClient:jest.fn(() => {throw new Error("DynamoDB must not be constructed");})})); +import pg from "../../src/db/pg-query"; +import {makeDynamoClient} from "../../src/utils/dynamoClient"; +import DynamoStorageService from "../../src/utils/storage"; + +const row=(key:string,report_data:string,generation="1") => ({ + zid:1,scope_key:"report",generation,item:{ + rid_section_model:{S:key},timestamp:{S:"2026-01-01T00:00:00Z"},report_data:{S:report_data} + } +}); +beforeEach(() => { + jest.clearAllMocks(); + process.env.DELPHI_RESULT_BACKEND="postgres"; + process.env.DELPHI_RESULT_ENV="generated"; + process.env.DELPHI_RESULT_SCOPE="report"; +}); +afterAll(() => { + delete process.env.DELPHI_RESULT_BACKEND; + delete process.env.DELPHI_RESULT_ENV; + delete process.env.DELPHI_RESULT_SCOPE; +}); + +test("report reader follows an empty filtered page and retains later row order and JSON string bytes",async () => { + const bytes=['{"text":"first Ω","nested":[1,2]}','{"text":"second\\nline"}']; + const rows=Array.from({length:1000},(_,i)=>row(`generated-other#${String(i).padStart(4,"0")}`,"unrelated")); + rows.push(row("rlocalpage#section-a",bytes[0]),row("rlocalpage#section-b",bytes[1])); + (pg.queryP as jest.Mock).mockResolvedValue(rows); + const result=await new DynamoStorageService("report_narrative_store").getAllByReportID("rlocalpage#"); + expect(result.success).toBe(true); + expect(result.data?.map(item=>item.rid_section_model)).toEqual(["rlocalpage#section-a","rlocalpage#section-b"]); + expect(result.data?.map(item=>item.report_data)).toEqual(bytes); + expect(pg.queryP).toHaveBeenCalledTimes(2); + expect(makeDynamoClient).not.toHaveBeenCalled(); +}); + +test("a changed generation on a later page refuses partial report success",async () => { + const rows=Array.from({length:1001},(_,i)=>row(`rlocalpage#${String(i).padStart(4,"0")}`,"stored")); + (pg.queryP as jest.Mock).mockResolvedValueOnce(rows) + .mockResolvedValueOnce(rows.map(item=>({...item,generation:"2"}))); + const result=await new DynamoStorageService("report_narrative_store").getAllByReportID("rlocalpage#"); + expect(result.success).toBe(false); + expect(result.data).toBeUndefined(); + expect(result.error?.message).toContain("generation changed"); + expect(pg.queryP).toHaveBeenCalledTimes(2); + expect(makeDynamoClient).not.toHaveBeenCalled(); +}); diff --git a/server/app.ts b/server/app.ts index 23121b6f39..60f2ab2a39 100644 --- a/server/app.ts +++ b/server/app.ts @@ -14,6 +14,7 @@ import morgan from "morgan"; import timeout from "connect-timeout"; import server from "./src/server"; +import { delphiResultSnapshot } from "./src/utils/delphiResultSnapshot"; import Config from "./src/config"; import { makeFileFetcher } from "./src/utils/file-fetcher"; import logger from "./src/utils/logger"; @@ -325,6 +326,7 @@ export const appReady = helpersInitialized.then( //////////////////////////////////////////// app.use(middleware_responseTime_start); + app.use(delphiResultSnapshot); app.use(redirectIfNotHttps); app.use(express.bodyParser({ limit: "50mb" })); diff --git a/server/bin/build-migration-report.py b/server/bin/build-migration-report.py new file mode 100644 index 0000000000..d53e728e3f --- /dev/null +++ b/server/bin/build-migration-report.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python3 +"""Build the first-deploy catalog report by inlining the adoption SQL helpers. + +No database connection. --check compares bytes without changing any files. +The generated script executes in a read-only transaction, with no temp objects. +""" +import argparse +import hashlib +from pathlib import Path +import re + +MIG = Path(__file__).resolve().parents[1] / "postgres/migrations" + + +def tokens(sql): + # All helper/adoption sources use ordinary quoted strings, comments and $$. + return re.findall(r"--[^\n]*|/\*[\s\S]*?\*/|'(?:''|[^'])*'|[A-Za-z_][A-Za-z_0-9]*|\s+|.", sql) + + +def substitute(sql, arguments): + return ''.join('(' + arguments[t] + ')' if t in arguments else t for t in tokens(sql)) + + +def helpers(source): + result = {} + pattern = r'CREATE FUNCTION pg_temp\.(\w+)\((.*?)\)\s*RETURNS boolean LANGUAGE sql AS \$\$(.*?)\$\$;' + for name, signature, body in re.findall(pattern, source, re.S): + params = [] + for part in signature.split(','): + bits = part.strip().split() + params.append((bits[0], bits[3] if len(bits) == 4 and bits[2] == 'DEFAULT' else None)) + result[name] = (params, body.strip().removesuffix(';')) + assert len(result) == 6, 'review changed helper source before regenerating' + return result + + +def inline(query, definitions): + ts = tokens(query) + out = [] + i = 0 + while i < len(ts): + if ts[i] != 'pg_temp': + out.append(ts[i]); i += 1; continue + assert ts[i+1] == '.' and ts[i+2] in definitions and ts[i+3] == '(' + name = ts[i+2] + i += 4 + depth = 1 + args = [''] + while depth: + token = ts[i]; i += 1 + if token == '(': + depth += 1 + elif token == ')': + depth -= 1 + if not depth: + break + if token == ',' and depth == 1: + args.append('') + else: + args[-1] += token + params, body = definitions[name] + assert len(args) <= len(params) + values = {} + for index, (param, default) in enumerate(params): + value = args[index].strip() if index < len(args) else default + assert value is not None + values[param] = value + out.append('(' + substitute(body, values) + ')') + return ''.join(out) + + +def build(): + source = (MIG/'adoption/helpers.sql').read_text() + definitions = helpers(source) + files = sorted((MIG/'adoption').glob('0*.sql')) + def manifest(path): + lines = [s for s in path.read_text().splitlines() if s and not s.startswith('#')] + assert len(lines) == len(set(lines)), 'duplicate manifest entry' + assert all(re.fullmatch(r'\d{6}_[A-Za-z0-9_]+\.sql', s) for s in lines) + return set(lines) + selected = manifest(MIG/'release.txt') + held = manifest(MIG/'held.txt') + pending = {'000019_create_polis_queue.sql', '000023_create_delphi_foundation.sql', + '000024_create_polis_queue_large_class.sql', '000027_create_sealed_job_graphs.sql', '000028_create_delphi_results.sql', + '000029_extend_delphi_graph_stages.sql'} + assert selected == {p.name for p in files} | pending, 'report scope differs from release selection' + assert held == {'000021_create_polis_coordinator.sql'}, 'review changed hold policy' + numbered = sorted(MIG.glob('*.sql')) + assert all(re.fullmatch(r'\d{6}_[A-Za-z0-9_]+\.sql', p.name) for p in numbered), 'invalid numbered source name' + assert len({p.name[:6] for p in numbered}) == len(numbered), 'duplicate migration number' + assert {p.name for p in numbered} == selected | held, 'unclassified/missing numbered SQL' + assert all(p.is_file() and not p.is_symlink() for p in numbered), 'nonregular numbered source' + bindings = [MIG/'adoption/helpers.sql', MIG/'release.txt', MIG/'held.txt'] + files + numbered + header = ['-- GENERATED by server/bin/build-migration-report.py; do not edit.', + '-- First deployment only: no existing ledger or queue. Other states refuse.', + '-- This is a catalog forecast, not DDL success or deployment-health proof.'] + for path in bindings: + header.append('-- sha256 ' + hashlib.sha256(path.read_bytes()).hexdigest() + ' ' + str(path.relative_to(MIG))) + # Each original predicate remains a scalar subquery. SQL helpers are inlined, + # never installed, and role/catalog identity fields never leave the database. + rows = [] + for path in files: + rows.append("SELECT '"+path.name+"'::text AS migration, ("+inline(path.read_text().strip().removesuffix(';'), definitions)+") AS catalog_match") + checks = '\nUNION ALL\n'.join(rows) + return '\n'.join(header)+'''\nBEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY; +SET LOCAL search_path=pg_catalog,public; +SET LOCAL statement_timeout='30s'; +SET LOCAL lock_timeout='2s'; +WITH checks AS ( +'''+checks+''' +), scope AS ( + SELECT to_regclass('public.migrations') IS NULL + AND to_regclass('public.schema_migrations') IS NULL + AND NOT EXISTS (SELECT 1 FROM pg_class WHERE relnamespace='public'::regnamespace + AND (starts_with(relname,'polis_queue_') OR starts_with(relname,'delphi_'))) + AND NOT EXISTS (SELECT 1 FROM pg_proc WHERE pronamespace='public'::regnamespace + AND (starts_with(proname,'pq_') OR starts_with(proname,'pd_'))) AS initial_state +), authority AS ( + SELECT + has_database_privilege(current_user,current_database(),'TEMP') + AND has_schema_privilege(current_user,'public','USAGE') + AND has_schema_privilege(current_user,'public','CREATE') AS reconcile_authority, + EXISTS (SELECT 1 FROM pg_roles WHERE rolname=current_user AND (rolsuper OR rolcreaterole)) AS create_roles, + NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname IN ('polis_queue_owner','polis_queue_executor')) AS queue_roles_absent, + has_schema_privilege(current_user,'public','USAGE WITH GRANT OPTION') + AND has_schema_privilege(current_user,'public','CREATE WITH GRANT OPTION') AS schema_grants, + EXISTS (SELECT 1 FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace + WHERE n.nspname='public' AND c.relname='conversations' + AND has_table_privilege(current_user,c.oid,'SELECT WITH GRANT OPTION') + AND has_column_privilege(current_user,c.oid,'topic','UPDATE WITH GRANT OPTION') + AND has_column_privilege(current_user,c.oid,'zid','REFERENCES WITH GRANT OPTION')) AS table_grants +), readiness AS ( + SELECT initial_state AND (SELECT bool_and(catalog_match) FROM checks) + AND current_setting('server_version_num')::integer >= 170000 AS can_reconcile, + create_roles AND queue_roles_absent AND schema_grants AND table_grants AS can_provision, + initial_state, reconcile_authority, create_roles, queue_roles_absent, schema_grants, table_grants + FROM scope CROSS JOIN authority +), output AS ( + SELECT migration, catalog_match, + CASE WHEN NOT initial_state THEN 'REVIEW_EXISTING_LEDGER_OR_QUEUE' + WHEN NOT catalog_match THEN 'CATALOG_MISMATCH' + WHEN NOT can_reconcile THEN 'BLOCKED_BY_OTHER_CATALOG_CHECK' + WHEN NOT reconcile_authority THEN 'REVIEW_MIGRATION_SESSION_AUTHORITY' + ELSE 'WOULD_ADOPT' END AS outcome + FROM checks CROSS JOIN readiness + UNION ALL + SELECT migration, NULL::boolean, + CASE WHEN NOT can_reconcile THEN 'BLOCKED_RECONCILIATION' + WHEN NOT reconcile_authority OR NOT can_provision THEN 'REVIEW_MIGRATION_SESSION_AUTHORITY' + ELSE 'WOULD_APPLY' END + FROM (VALUES ('000019_create_polis_queue.sql'),('000023_create_delphi_foundation.sql'), + ('000024_create_polis_queue_large_class.sql'), + ('000027_create_sealed_job_graphs.sql'), + ('000028_create_delphi_results.sql'), + ('000029_extend_delphi_graph_stages.sql')) p(migration) CROSS JOIN readiness + UNION ALL + SELECT migration, NULL::boolean, 'OUTSIDE_RELEASE' + FROM (VALUES ('000020'),('000021'),('000025'),('000026')) p(migration) +) +SELECT migration, catalog_match, outcome FROM output ORDER BY migration; +ROLLBACK; +''' + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--check', action='store_true') + args = parser.parse_args() + output = MIG/'report/first-deploy.sql' + text = build() + if args.check: + assert output.read_text() == text, 'report is stale; regenerate and review' + print('PASS: report matches all source predicates and release metadata') + else: + output.parent.mkdir(exist_ok=True) + output.write_text(text) + print(output) + + +if __name__ == '__main__': + main() diff --git a/server/bin/run-migrations.sh b/server/bin/run-migrations.sh index 911ebe6680..34de794ca6 100755 --- a/server/bin/run-migrations.sh +++ b/server/bin/run-migrations.sh @@ -1,28 +1,5 @@ -#!/bin/bash -set -e - -# Use DATABASE_URL from environment or from argument if provided -if [ -n "$1" ]; then - DATABASE_URL="$1" -fi - -# Check if DATABASE_URL is set -if [ -z "$DATABASE_URL" ]; then - echo "Error: DATABASE_URL is not set. Please provide it as an environment variable or argument." - echo "Usage: $0 [DATABASE_URL]" - exit 1 -fi - -# Directory containing migration files -MIGRATIONS_DIR="$(dirname "$(dirname "$0")")/postgres/migrations" - -echo "Running migrations from $MIGRATIONS_DIR" - -# Get all migration files sorted by name (only from top-level directory) -for migration in $(find "$MIGRATIONS_DIR" -maxdepth 1 -name "*.sql" | sort); do - echo "Applying migration: $(basename "$migration")" - PGPASSWORD=$(echo "$DATABASE_URL" | sed -E 's/.*:([^:]+)@.*/\1/') \ - psql "$DATABASE_URL" -f "$migration" -done - -echo "All migrations completed successfully!" +#!/usr/bin/env bash +set -euo pipefail +# DATABASE_URL stays in the environment, never an argv or a traced shell command. +root=$(cd "$(dirname "$0")/../.." && pwd) +exec "${POLIS_MIGRATE_BIN:-polis-migrate}" apply --dir "$root/server/postgres/migrations" "$@" diff --git a/server/characterization/delphi/jest.codec.config.json b/server/characterization/delphi/jest.codec.config.json index f94ac9f2db..2be309511d 100644 --- a/server/characterization/delphi/jest.codec.config.json +++ b/server/characterization/delphi/jest.codec.config.json @@ -1,6 +1,21 @@ { "rootDir": "../../", - "transform": { "^.+\\.ts$": ["ts-jest", { "tsconfig": "./tsconfig.json" }] }, + "transform": { + "^.+\\.ts$": [ + "ts-jest", + { + "tsconfig": "./tsconfig.json" + } + ] + }, "testEnvironment": "node", - "testMatch": ["**/__tests__/unit/delphiStorageCodec.test.ts"] + "testMatch": [ + "**/__tests__/unit/delphiStorageCodec.test.ts", + "**/__tests__/unit/delphiStoragePagination.test.ts", + "**/__tests__/unit/delphiResults.test.ts", + "**/__tests__/unit/delphiResultSnapshot.test.ts", + "**/__tests__/unit/delphi-postgres-liveness.test.ts", + "**/__tests__/unit/delphi-job-guard.test.ts", + "**/__tests__/unit/delphi-postgres-read-routes.test.ts" + ] } diff --git a/server/characterization/routing/runtime.cjs b/server/characterization/routing/runtime.cjs index b03cee4fde..ad32228e5b 100644 --- a/server/characterization/routing/runtime.cjs +++ b/server/characterization/routing/runtime.cjs @@ -4,7 +4,7 @@ const vm = require('node:vm'); const ts = require('typescript'); const underscore = require('underscore'); const server = path.resolve(__dirname, '../..'); -const realFiles = new Set(['nextComment.ts','comment.ts','conversation.ts','db/sql.ts','utils/pca.ts','utils/commentClusters.ts','votes/convention.ts','utils/zinvite.ts']); +const realFiles = new Set(['nextComment.ts','comment.ts','conversation.ts','db/sql.ts','utils/pca.ts','utils/commentClusters.ts','votes/convention.ts','utils/zinvite.ts','utils/delphiResults.ts','utils/delphiResultSnapshot.ts','utils/delphiStorageCodec.ts']); exports.runtime = function runtime(client, fixture, mutate = null) { let now = 1700000000000; @@ -93,7 +93,7 @@ exports.runtime = function runtime(client, fixture, mutate = null) { if(spec==='@aws-sdk/client-dynamodb') return {DynamoDBClient:class {}}; if(spec==='@aws-sdk/lib-dynamodb') return {DynamoDBDocumentClient:{from:()=>dynamo},QueryCommand:class {constructor(input){this.input=input;}}}; if(spec==='@google-cloud/translate') return {v2:{Translate:class {async translate(txt,lang){observation.translations++;return [`${lang}: ${txt}`];}}}}; - if(!['lru-cache','sql','zlib','crypto'].includes(spec)) throw Error(`unapproved external import ${spec}`); + if(!['lru-cache','sql','zlib','crypto','async_hooks'].includes(spec)) throw Error(`unapproved external import ${spec}`); return require(spec); }; const math=Object.create(Math); math.random=()=>{const n=0.625;observation.draws.push(n);return n;}; diff --git a/server/index.ts b/server/index.ts index dfba6599a7..35c5e36ee8 100644 --- a/server/index.ts +++ b/server/index.ts @@ -1,30 +1,25 @@ -/** - * Server entry point - * This file is responsible for starting the server after the app is configured - */ -import app from "./app"; -import Config from "./src/config"; -import { startNotificationLoop } from "./src/routes/notify"; -import logger from "./src/utils/logger"; +/** Check the release's migrations before importing the app or starting work. */ +import "dotenv/config"; +import { checkMigrations } from "./src/db/migrations.cjs"; -if (Config.nodeEnv === "production") { - // eslint-disable-next-line @typescript-eslint/no-unused-vars, @typescript-eslint/no-var-requires - const tracer = require("dd-trace").init(); -} - -/** - * Start the server on the configured port or a provided port - * @param {number} [port=Config.serverPort] - The port to listen on - * @returns {Object} The server instance - */ -function startServer(port = Config.serverPort) { - const server = app.listen(port); - logger.info(`Server started on port ${port}`); +async function startServer(port?: number) { + await checkMigrations(); + const { default: Config } = await import("./src/config"); + if (Config.nodeEnv === "production") { + // eslint-disable-next-line @typescript-eslint/no-var-requires + require("dd-trace").init(); + } + const { default: app } = await import("./app"); + const { startNotificationLoop } = await import("./src/routes/notify"); + const { default: logger } = await import("./src/utils/logger"); + startNotificationLoop(); + const server = app.listen(port ?? Config.serverPort); + logger.info(`Server started on port ${port ?? Config.serverPort}`); return server; } -startNotificationLoop(); -startServer(); - +startServer().catch((error) => { + process.stderr.write(`Server startup refused: ${error.message}\n`); + process.exit(1); +}); export { startServer }; -export default app; diff --git a/server/jest.job-graphs.config.ts b/server/jest.job-graphs.config.ts new file mode 100644 index 0000000000..bd568d0350 --- /dev/null +++ b/server/jest.job-graphs.config.ts @@ -0,0 +1,12 @@ +// Database-only compatibility checks in the job-graph CI entry. These tests own +// their database/role fixtures; they do not need the full HTTP/OIDC test stack. +import base from './jest.config'; +export default { + ...base, + testMatch: ['**/__tests__/integration/queue-substrate.test.ts'], + setupFilesAfterEnv: [], + globalSetup: undefined, + globalTeardown: undefined, + collectCoverage: false, + reporters: ['default'], +}; diff --git a/server/postgres/bin/apply-migration.sh b/server/postgres/bin/apply-migration.sh index e6ec532005..9c3c2e05f8 100755 --- a/server/postgres/bin/apply-migration.sh +++ b/server/postgres/bin/apply-migration.sh @@ -1,5 +1,8 @@ #!/usr/bin/env bash # +# HISTORICAL REHEARSAL HELPER ONLY. Deployments must use polis-migrate apply +# (docs/migrations.md). This script does not record applied migration history. +# # apply-migration.sh: the checked apply wrapper for the queue migrations # 000019 (polis-queue/1), 000023 (polis-queue/2, the Delphi job table) and # 000024 (polis-queue/3, the large worker class). diff --git a/server/postgres/init-migrations.sh b/server/postgres/init-migrations.sh new file mode 100644 index 0000000000..b70b0545ea --- /dev/null +++ b/server/postgres/init-migrations.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -eu +# Official postgres entrypoint sources executable/nonexecutable shell hooks only +# during fresh initialization; its temporary server listens on this local socket. +export DATABASE_URL="host=/var/run/postgresql user=$POSTGRES_USER dbname=$POSTGRES_DB sslmode=disable" +export POLIS_MIGRATIONS_DIR=/migrations +polis-migrate apply diff --git a/server/postgres/migrations/000027_create_sealed_job_graphs.sql b/server/postgres/migrations/000027_create_sealed_job_graphs.sql new file mode 100644 index 0000000000..1118176e42 --- /dev/null +++ b/server/postgres/migrations/000027_create_sealed_job_graphs.sql @@ -0,0 +1,528 @@ +-- Per-step runs, immutable results and inputs, dependency waiting, atomic publication. +-- Contract /5; /4 belongs to held retention. No provider remediation or scoped breakers. +-- Forward-only compatibility with the original M19/M23/M24 checksums. +BEGIN; +SET LOCAL lock_timeout='5s'; +SET LOCAL ROLE polis_queue_owner; +SET LOCAL search_path=pg_catalog,pg_temp; +CREATE OR REPLACE FUNCTION pg_temp.pq_catalog(p_table oid) RETURNS jsonb +LANGUAGE sql SET search_path=pg_catalog,pg_temp AS $catalog$ +SELECT jsonb_build_object( + 'columns',(SELECT jsonb_agg(jsonb_build_array(a.attname,format_type(a.atttypid,a.atttypmod),a.attnotnull,a.attidentity,a.attgenerated,co.collname,pg_get_expr(d.adbin,d.adrelid),NULLIF(a.attacl::text,'{}')) ORDER BY a.attnum) + FROM pg_attribute a LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum LEFT JOIN pg_collation co ON co.oid=a.attcollation + WHERE a.attrelid=c.oid AND a.attnum>0 AND NOT a.attisdropped), + 'constraints',(SELECT jsonb_agg(jsonb_build_array(conname,pg_get_constraintdef(oid),convalidated,connoinherit) ORDER BY conname) FROM pg_constraint WHERE conrelid=c.oid), + 'indexes',(SELECT jsonb_agg(jsonb_build_array(ic.relname,pg_get_indexdef(i.indexrelid),i.indisvalid,i.indisready) ORDER BY ic.relname) FROM pg_index i JOIN pg_class ic ON ic.oid=i.indexrelid WHERE i.indrelid=c.oid), + 'triggers',(SELECT jsonb_agg(jsonb_build_array(t.tgname,pg_get_triggerdef(t.oid),t.tgenabled) ORDER BY t.tgname) FROM pg_trigger t WHERE t.tgrelid=c.oid AND NOT t.tgisinternal), + 'owner',pg_get_userbyid(c.relowner),'kind',c.relkind,'rls',c.relrowsecurity,'force_rls',c.relforcerowsecurity,'options',c.reloptions, + 'acl',(SELECT jsonb_agg(jsonb_build_array(CASE WHEN x.grantee=0 THEN 'PUBLIC' ELSE pg_get_userbyid(x.grantee) END,x.privilege_type,x.is_grantable) ORDER BY x.grantee=0,pg_get_userbyid(x.grantee),x.privilege_type,x.is_grantable) FROM aclexplode(COALESCE(c.relacl,acldefault('r',c.relowner))) x) + ) FROM pg_class c WHERE c.oid=p_table +$catalog$; +CREATE OR REPLACE FUNCTION pg_temp.pd_state() RETURNS jsonb LANGUAGE sql SET search_path=pg_catalog,pg_temp AS $s$ +SELECT jsonb_build_object('tables',(SELECT jsonb_object_agg(c.relname,pg_temp.pq_catalog(c.oid)) FROM pg_class c + WHERE c.relnamespace='public'::regnamespace AND c.relkind='r' AND (starts_with(c.relname,'polis_queue_') OR starts_with(c.relname,'delphi_')) AND c.relname<>'delphi_foundation_install'), + 'functions',(SELECT jsonb_object_agg(p.oid::regprocedure::text,jsonb_build_array(pg_get_functiondef(p.oid),p.proacl::text,pg_get_userbyid(p.proowner))) + FROM pg_proc p WHERE p.pronamespace='public'::regnamespace AND (starts_with(p.proname,'pq_') OR starts_with(p.proname,'pd_')))) +$s$; +-- The /3 state: the same, less this file's own install table. +CREATE OR REPLACE FUNCTION pg_temp.pq3_state() RETURNS jsonb LANGUAGE sql SET search_path=pg_catalog,pg_temp AS $s$ +SELECT jsonb_build_object('tables',(SELECT jsonb_object_agg(c.relname,pg_temp.pq_catalog(c.oid)) FROM pg_class c + WHERE c.relnamespace='public'::regnamespace AND c.relkind='r' AND (starts_with(c.relname,'polis_queue_') OR starts_with(c.relname,'delphi_')) + AND c.relname NOT IN ('delphi_foundation_install','polis_queue_large_class_install')), + 'functions',(SELECT jsonb_object_agg(p.oid::regprocedure::text,jsonb_build_array(pg_get_functiondef(p.oid),p.proacl::text,pg_get_userbyid(p.proowner))) + FROM pg_proc p WHERE p.pronamespace='public'::regnamespace AND (starts_with(p.proname,'pq_') OR starts_with(p.proname,'pd_')))) +$s$; + +DO $$ BEGIN + IF NOT EXISTS(SELECT 1 FROM public.polis_queue_large_class_install WHERE installed=pg_temp.pq3_state()) + THEN RAISE EXCEPTION 'queue /3 catalog drift'; END IF; +END $$; +DO $$ BEGIN PERFORM set_config('graph.baseline',pg_temp.pq3_state()::text,true); END $$; +CREATE TABLE public.delphi_graph_install(singleton boolean PRIMARY KEY CHECK(singleton), baseline jsonb NOT NULL); +INSERT INTO public.delphi_graph_install VALUES(true,current_setting('graph.baseline')::jsonb); +ALTER TABLE public.polis_queue_install DROP CONSTRAINT polis_queue_install_contract_version_check; +UPDATE public.polis_queue_install SET contract_version='polis-queue/5'; +ALTER TABLE public.polis_queue_install ADD CHECK(contract_version='polis-queue/5'); +ALTER TABLE public.polis_queue_install ALTER COLUMN contract_version SET DEFAULT 'polis-queue/5'; +ALTER TABLE public.polis_queue_runs DROP CONSTRAINT polis_queue_runs_contract_version_check; +ALTER TABLE public.polis_queue_runs ADD CHECK(contract_version IN ('polis-queue/1','polis-queue/2','polis-queue/3','polis-queue/5')); +ALTER TABLE public.polis_queue_jobs DROP CONSTRAINT polis_queue_jobs_stage_check; +ALTER TABLE public.polis_queue_jobs ADD CHECK(stage IN ('noop','delphi_full_pipeline','delphi_narrative','math_rebuild','graph_embed','graph_cluster','graph_narrative')); +ALTER TABLE public.polis_queue_jobs DROP CONSTRAINT pq_stage_large; +ALTER TABLE public.polis_queue_jobs ADD CONSTRAINT pq_stage_large CHECK( + (stage='math_rebuild' AND worker_class='large') OR + (stage='graph_cluster' AND worker_class IN ('delphi','large')) OR + (stage NOT IN ('math_rebuild','graph_cluster') AND worker_class<>'large')); + +CREATE TABLE public.delphi_graphs( + env text NOT NULL, graph_id uuid NOT NULL, zid integer NOT NULL REFERENCES public.conversations(zid), + scope_key text NOT NULL CHECK(length(scope_key) BETWEEN 1 AND 128), request_key text NOT NULL, + request jsonb NOT NULL, root_job_id uuid NOT NULL, sealed boolean NOT NULL DEFAULT false, + supersedes uuid, created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY(env,graph_id), UNIQUE(env,scope_key,request_key), UNIQUE(env,zid,graph_id), + FOREIGN KEY(env,supersedes) REFERENCES public.delphi_graphs(env,graph_id) +); +CREATE TABLE public.delphi_graph_nodes( + env text NOT NULL, zid integer NOT NULL, graph_id uuid NOT NULL, job_id uuid NOT NULL, + run_id uuid NOT NULL, node_key text NOT NULL CHECK(node_key ~ '^[a-z][a-z0-9_]{0,31}$'), + declared jsonb NOT NULL, resolved jsonb, resolved_sha text, + PRIMARY KEY(env,job_id), UNIQUE(env,run_id), UNIQUE(env,job_id,run_id), UNIQUE(env,graph_id,node_key), + FOREIGN KEY(env,zid,graph_id) REFERENCES public.delphi_graphs(env,zid,graph_id), + FOREIGN KEY(env,job_id) REFERENCES public.delphi_jobs(env,job_id), + FOREIGN KEY(env,run_id) REFERENCES public.polis_queue_runs(env,run_id), + CHECK((resolved IS NULL)=(resolved_sha IS NULL)) +); +CREATE TABLE public.delphi_artifacts( + env text NOT NULL, artifact_id uuid NOT NULL DEFAULT gen_random_uuid(), job_id uuid NOT NULL, + run_id uuid NOT NULL, attempt_id uuid NOT NULL, output_role text NOT NULL CHECK(output_role='result'), + schema_version text NOT NULL, payload text NOT NULL CHECK(octet_length(payload)<=524288), + content_sha text NOT NULL CHECK(content_sha ~ '^[0-9a-f]{64}$'), + byte_count integer NOT NULL CHECK(byte_count>=0), created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY(env,artifact_id), UNIQUE(env,job_id,output_role), + FOREIGN KEY(env,job_id) REFERENCES public.delphi_graph_nodes(env,job_id), + FOREIGN KEY(env,job_id,run_id) REFERENCES public.delphi_graph_nodes(env,job_id,run_id), + FOREIGN KEY(env,job_id,attempt_id) REFERENCES public.polis_queue_attempts(env,job_id,attempt_id), + CHECK(content_sha=encode(sha256(convert_to(payload,'UTF8')),'hex')), + CHECK(byte_count=octet_length(payload)) +); +CREATE TABLE public.delphi_graph_edges( + env text NOT NULL, consumer uuid NOT NULL, producer uuid NOT NULL, input_role text NOT NULL, + expected_sha text CHECK(expected_sha ~ '^[0-9a-f]{64}$'), artifact_id uuid, + PRIMARY KEY(env,consumer,input_role), CHECK(consumer<>producer), + FOREIGN KEY(env,consumer) REFERENCES public.delphi_graph_nodes(env,job_id), + FOREIGN KEY(env,producer) REFERENCES public.delphi_graph_nodes(env,job_id), + FOREIGN KEY(env,artifact_id) REFERENCES public.delphi_artifacts(env,artifact_id) +); +CREATE INDEX delphi_graph_edges_producer ON public.delphi_graph_edges(env,producer); +CREATE TABLE public.delphi_graph_served( + env text NOT NULL, zid integer NOT NULL REFERENCES public.conversations(zid), scope_key text NOT NULL, + generation bigint NOT NULL CHECK(generation>0), artifact_id uuid NOT NULL, + PRIMARY KEY(env,zid,scope_key), FOREIGN KEY(env,artifact_id) REFERENCES public.delphi_artifacts(env,artifact_id) +); +CREATE FUNCTION public.pd_graph_hash(j jsonb) RETURNS text LANGUAGE sql IMMUTABLE + SET search_path=pg_catalog,pg_temp AS $$ SELECT encode(sha256(convert_to(j::text,'UTF8')),'hex') $$; +CREATE FUNCTION public.pd_graph_immutable() RETURNS trigger LANGUAGE plpgsql SET search_path=pg_catalog,pg_temp AS $$ +BEGIN RAISE EXCEPTION 'immutable graph result'; END $$; +CREATE TRIGGER graph_artifact_immutable BEFORE UPDATE OR DELETE ON public.delphi_artifacts FOR EACH ROW EXECUTE FUNCTION public.pd_graph_immutable(); +CREATE FUNCTION public.pd_graph_node_guard() RETURNS trigger LANGUAGE plpgsql SET search_path=pg_catalog,pg_temp AS $$ +BEGIN + IF TG_OP='DELETE' THEN RAISE EXCEPTION 'immutable graph node'; END IF; + IF TG_OP='UPDATE' AND (to_jsonb(NEW)-'resolved'-'resolved_sha' IS DISTINCT FROM to_jsonb(OLD)-'resolved'-'resolved_sha' + OR OLD.resolved IS NOT NULL) THEN RAISE EXCEPTION 'immutable graph input'; END IF; + IF TG_OP='INSERT' AND EXISTS(SELECT 1 FROM public.delphi_graphs WHERE env=NEW.env AND graph_id=NEW.graph_id AND sealed) + THEN RAISE EXCEPTION 'sealed graph'; END IF; + RETURN NEW; +END $$; +CREATE TRIGGER graph_node_guard BEFORE INSERT OR UPDATE OR DELETE ON public.delphi_graph_nodes FOR EACH ROW EXECUTE FUNCTION public.pd_graph_node_guard(); +CREATE FUNCTION public.pd_graph_edge_guard() RETURNS trigger LANGUAGE plpgsql SET search_path=pg_catalog,pg_temp AS $$ +#variable_conflict use_column +DECLARE c public.delphi_graph_nodes; p public.delphi_graph_nodes; +BEGIN + IF TG_OP='DELETE' THEN RAISE EXCEPTION 'immutable graph edge'; END IF; + SELECT * INTO STRICT c FROM public.delphi_graph_nodes WHERE env=NEW.env AND job_id=NEW.consumer; + SELECT * INTO STRICT p FROM public.delphi_graph_nodes WHERE env=NEW.env AND job_id=NEW.producer; + IF c.zid<>p.zid THEN RAISE EXCEPTION 'dependency namespace'; END IF; + IF TG_OP='UPDATE' THEN + IF to_jsonb(NEW)-'artifact_id' IS DISTINCT FROM to_jsonb(OLD)-'artifact_id' OR OLD.artifact_id IS NOT NULL + THEN RAISE EXCEPTION 'immutable dependency'; END IF; + ELSE + IF EXISTS(SELECT 1 FROM public.delphi_graphs WHERE env=c.env AND graph_id=c.graph_id AND sealed) + THEN RAISE EXCEPTION 'sealed graph'; END IF; + IF EXISTS(WITH RECURSIVE up(id) AS (SELECT NEW.producer UNION SELECT e.producer FROM public.delphi_graph_edges e JOIN up ON e.consumer=up.id WHERE e.env=NEW.env) SELECT 1 FROM up WHERE id=NEW.consumer) + THEN RAISE EXCEPTION 'dependency cycle'; END IF; + END IF; + IF NEW.artifact_id IS NOT NULL AND NOT EXISTS(SELECT 1 FROM public.delphi_artifacts a + WHERE a.env=NEW.env AND a.artifact_id=NEW.artifact_id AND a.job_id=NEW.producer + AND (NEW.expected_sha IS NULL OR a.content_sha=NEW.expected_sha)) THEN RAISE EXCEPTION 'artifact binding'; END IF; + RETURN NEW; +END $$; +CREATE TRIGGER graph_edge_guard BEFORE INSERT OR UPDATE OR DELETE ON public.delphi_graph_edges FOR EACH ROW EXECUTE FUNCTION public.pd_graph_edge_guard(); +CREATE FUNCTION public.pd_graph_parent_guard() RETURNS trigger LANGUAGE plpgsql SET search_path=pg_catalog,pg_temp AS $$ +BEGIN + IF TG_OP='UPDATE' AND (NEW.parent_job_id IS DISTINCT FROM OLD.parent_job_id OR NEW.run_id IS DISTINCT FROM OLD.run_id OR NEW.env<>OLD.env OR NEW.zid<>OLD.zid) + AND EXISTS(SELECT 1 FROM public.delphi_graph_nodes WHERE env=OLD.env AND job_id=OLD.job_id) + THEN RAISE EXCEPTION 'immutable graph membership'; END IF; + IF TG_OP='INSERT' AND NEW.parent_job_id IS NOT NULL AND EXISTS( + SELECT 1 FROM public.delphi_graph_nodes n JOIN public.delphi_graphs g USING(env,graph_id) WHERE n.env=NEW.env AND n.job_id=NEW.parent_job_id AND g.sealed) + THEN RAISE EXCEPTION 'late child in sealed graph'; END IF; + RETURN NEW; +END $$; +CREATE TRIGGER graph_parent_guard BEFORE INSERT OR UPDATE ON public.delphi_jobs FOR EACH ROW EXECUTE FUNCTION public.pd_graph_parent_guard(); + +CREATE OR REPLACE FUNCTION public.pd_queue_binding() RETURNS trigger +LANGUAGE plpgsql SECURITY DEFINER SET search_path=pg_catalog,pg_temp AS $$ +#variable_conflict use_column +DECLARE q public.polis_queue_jobs; j public.delphi_jobs; r public.polis_queue_runs; +BEGIN + SELECT * INTO q FROM public.polis_queue_jobs WHERE env=NEW.env AND job_id=NEW.job_id; + IF EXISTS(SELECT 1 FROM public.delphi_graph_nodes gn WHERE gn.env=q.env AND gn.run_id=q.run_id) AND (SELECT count(*) FROM public.polis_queue_jobs jq WHERE jq.env=q.env AND jq.run_id=q.run_id)<>1 THEN RAISE EXCEPTION 'one computational job per run'; END IF; + IF q.stage LIKE 'graph_%' THEN + IF NOT EXISTS(SELECT 1 FROM public.delphi_graph_nodes n JOIN public.polis_queue_runs r USING(env,run_id) JOIN public.delphi_jobs d ON d.env=n.env AND d.job_id=n.job_id WHERE n.env=q.env AND n.job_id=q.job_id AND n.run_id=q.run_id AND r.contract_version='polis-queue/5' AND d.run_id=q.run_id AND d.zid=r.zid AND d.kind=substring(q.stage from 7)) THEN RAISE EXCEPTION 'invalid graph execution binding'; END IF; + RETURN NULL; + END IF; + IF q.stage='noop' THEN RETURN NULL; END IF; + SELECT * INTO j FROM public.delphi_jobs WHERE env=q.env AND job_id=q.job_id; + SELECT * INTO r FROM public.polis_queue_runs WHERE env=q.env AND run_id=q.run_id; + IF j.job_id IS NULL OR j.zid<>r.zid OR j.origin<>'queued' OR j.run_id IS DISTINCT FROM q.run_id + OR r.contract_version<>(CASE WHEN q.stage='math_rebuild' THEN 'polis-queue/3' ELSE 'polis-queue/2' END) + OR (q.stage='delphi_full_pipeline' AND j.kind<>'full_pipeline') + OR (q.stage='delphi_narrative' AND j.kind<>'narrative') + OR (q.stage='math_rebuild' AND (j.kind<>'math_rebuild' OR q.worker_class<>'large' OR j.report_id IS NOT NULL)) + THEN RAISE EXCEPTION 'invalid logical execution binding'; END IF; + RETURN NULL; +END $$; +CREATE OR REPLACE FUNCTION public.pq_result(p_outcome text, j public.polis_queue_jobs, p_published boolean DEFAULT false) +RETURNS jsonb LANGUAGE sql VOLATILE SET search_path=pg_catalog,pg_temp SET TimeZone='UTC' AS $$ + SELECT jsonb_build_object('schema_version',CASE WHEN j.stage IS NULL OR j.stage='noop' THEN 'polis-queue/1' WHEN j.stage LIKE 'graph_%' THEN 'polis-queue/5' WHEN j.stage='math_rebuild' THEN 'polis-queue/3' ELSE 'polis-queue/2' END,'outcome',p_outcome, + 'env',j.env,'job_id',j.job_id,'run_id',j.run_id, + 'attempt_id',COALESCE(j.attempt_id,j.terminal_attempt_id),'owner_id',j.owner_id, + 'lease_epoch',j.lease_epoch::text,'version',j.version::text,'mgmt_version',j.mgmt_version::text,'locked_until',j.locked_until, + 'state',j.state,'output_sha256',j.output_sha256,'published',COALESCE(p_published,false), + 'stage',j.stage,'stage_instance',j.stage_instance,'attempt_count',j.attempt_count,'max_attempts',j.max_attempts,'parked_attempt_count',j.parked_attempt_count, + 'eligible_at',j.eligible_at,'first_parked_at',j.first_parked_at,'last_error_code',j.last_error_code, + 'input',(SELECT jsonb_build_object('uri',r.input_uri,'sha256',r.input_sha256,'config_sha256',r.config_sha256,'code_image_digest',r.code_image_digest) FROM public.polis_queue_runs r WHERE r.env=j.env AND r.run_id=j.run_id)) +$$; +CREATE OR REPLACE FUNCTION public.pq_claim(p_env text,p_priority smallint,p_owner uuid,p_attempt uuid,p_lease_seconds integer,p_worker_class text) +RETURNS jsonb LANGUAGE plpgsql SECURITY DEFINER SET search_path=pg_catalog,pg_temp AS $$ +#variable_conflict use_column +DECLARE j public.polis_queue_jobs; +BEGIN + IF p_owner IS NULL OR p_attempt IS NULL OR p_lease_seconds IS NULL OR p_lease_seconds NOT BETWEEN 10 AND 900 + OR p_worker_class IS NULL OR p_worker_class NOT IN ('delphi','large') THEN RAISE EXCEPTION 'invalid claim'; END IF; + WITH candidate AS ( + SELECT q.env,q.job_id FROM public.polis_queue_jobs q JOIN public.delphi_jobs d ON d.env=q.env AND d.job_id=q.job_id + WHERE q.env=p_env AND q.priority=p_priority AND q.worker_class=p_worker_class + AND q.stage NOT LIKE 'graph_%' + AND q.state IN ('queued','retry_wait') AND q.eligible_at<=statement_timestamp() + AND q.attempt_count-q.parked_attempt_count'succeeded' OR p.output_manifest_digest IS DISTINCT FROM i.producer_output_digest)) + -- An expired lease is not evidence that the previous process stopped. + AND NOT EXISTS(SELECT 1 FROM public.polis_queue_attempts a WHERE a.env=q.env AND a.job_id=q.job_id AND a.process_exit_confirmed_at IS NULL) + AND NOT EXISTS(SELECT 1 FROM public.delphi_provider_requests pr WHERE pr.env=q.env AND pr.job_id=q.job_id AND pr.state IN ('intent','submission_unknown')) + ORDER BY q.eligible_at,q.created_at,q.job_id FOR UPDATE OF q SKIP LOCKED LIMIT 1 + ) UPDATE public.polis_queue_jobs q SET state='running',owner_id=p_owner,attempt_id=p_attempt, + locked_until=clock_timestamp()+make_interval(secs=>p_lease_seconds),lease_epoch=q.lease_epoch+1, + version=q.version+1,mgmt_version=q.mgmt_version+1,attempt_count=q.attempt_count+1,updated_at=clock_timestamp() + FROM candidate c WHERE q.env=c.env AND q.job_id=c.job_id RETURNING q.* INTO j; + IF NOT FOUND THEN RETURN jsonb_build_object('schema_version',CASE p_worker_class WHEN 'large' THEN 'polis-queue/3' ELSE 'polis-queue/2' END,'outcome','none'); END IF; + INSERT INTO public.polis_queue_attempts(env,attempt_id,job_id,owner_id,lease_epoch,outcome) + VALUES(j.env,j.attempt_id,j.job_id,j.owner_id,j.lease_epoch,'running'); + RETURN public.pq_result('owned',j); +END $$; +CREATE FUNCTION public.pd_graph_seal_guard() RETURNS trigger LANGUAGE plpgsql SET search_path=pg_catalog,pg_temp AS $$ +BEGIN + IF TG_OP='DELETE' THEN RAISE EXCEPTION 'immutable graph'; END IF; + IF OLD.sealed OR NOT NEW.sealed OR to_jsonb(OLD)-'sealed' IS DISTINCT FROM to_jsonb(NEW)-'sealed' + THEN RAISE EXCEPTION 'immutable sealed graph'; END IF; + RETURN NEW; +END $$; +CREATE TRIGGER graph_seal_guard BEFORE UPDATE OR DELETE ON public.delphi_graphs FOR EACH ROW EXECUTE FUNCTION public.pd_graph_seal_guard(); + +-- The admission JSON is a closed, bounded local-artifact contract. Every node +-- names its exact snapshot, code/model/runtime/config/seed and full-fit mode. +CREATE FUNCTION public.pd_graph_admit(p_env text,p_zid integer,p_scope text,p_key text,p_spec jsonb,p_supersedes uuid DEFAULT NULL) +RETURNS jsonb LANGUAGE plpgsql SECURITY DEFINER SET search_path=pg_catalog,pg_temp AS $$ +#variable_conflict use_column +DECLARE g public.delphi_graphs; gid uuid=gen_random_uuid(); root uuid; + n jsonb; e jsonb; jid uuid; rid uuid; prod uuid; aid uuid; decl jsonb; reply jsonb; + v_stage text; cls text; +BEGIN + IF p_supersedes IS NOT NULL THEN RAISE EXCEPTION 'superseding dead branches is not supported'; END IF; + IF p_env IS NULL OR p_env !~ '^[a-z0-9_-]{1,64}$' OR p_scope IS NULL OR length(p_scope) NOT BETWEEN 1 AND 128 + OR p_key IS NULL OR length(p_key) NOT BETWEEN 1 AND 128 OR p_spec->>'schema' IS DISTINCT FROM 'polis-job-graph/1' + OR jsonb_typeof(p_spec->'nodes') IS DISTINCT FROM 'array' OR jsonb_array_length(p_spec->'nodes') NOT BETWEEN 1 AND 32 + OR p_spec-'schema'-'nodes'<>'{}'::jsonb OR octet_length(p_spec::text)>1048576 THEN RAISE EXCEPTION 'invalid graph'; END IF; + PERFORM 1 FROM public.conversations WHERE zid=p_zid FOR KEY SHARE; + IF NOT FOUND THEN RAISE EXCEPTION 'unknown conversation'; END IF; + PERFORM pg_advisory_xact_lock(hashtextextended(jsonb_build_array('pd:scope',p_env,p_scope)::text,0)); + SELECT * INTO g FROM public.delphi_graphs WHERE env=p_env AND scope_key=p_scope AND request_key=p_key; + IF FOUND THEN + IF g.zid<>p_zid OR g.request IS DISTINCT FROM p_spec OR g.supersedes IS DISTINCT FROM p_supersedes THEN RAISE EXCEPTION 'graph request conflict'; END IF; + RETURN jsonb_build_object('outcome','existing','graph_id',g.graph_id,'root_job_id',g.root_job_id); + END IF; + IF EXISTS(SELECT 1 FROM public.delphi_job_guards WHERE env=p_env AND scope_key=p_scope) THEN RAISE EXCEPTION 'scope busy'; END IF; + root=gen_random_uuid(); + INSERT INTO public.delphi_graphs(env,graph_id,zid,scope_key,request_key,request,root_job_id,supersedes) + VALUES(p_env,gid,p_zid,p_scope,p_key,p_spec,root,p_supersedes); + FOR n IN SELECT value FROM jsonb_array_elements(p_spec->'nodes') LOOP + v_stage=n->>'stage'; cls=n->>'class'; decl=n->'declared'; + IF n-'key'-'stage'-'class'-'declared'-'inputs'-'max_attempts'<>'{}'::jsonb + OR n->>'key' IS NULL OR v_stage IS NULL OR v_stage NOT IN ('graph_embed','graph_cluster','graph_narrative') + OR cls IS NULL OR NOT ((v_stage='graph_cluster' AND cls IN ('delphi','large')) OR (v_stage<>'graph_cluster' AND cls='delphi')) + OR jsonb_typeof(decl) IS DISTINCT FROM 'object' + OR decl-'snapshot'-'code'-'model'-'runtime'-'seed'-'config'-'mode'-'memory_bytes'-'work_units'<>'{}'::jsonb + OR decl->>'mode' IS DISTINCT FROM 'full' + OR decl->>'code' IS NULL OR decl->>'code' !~ '^[0-9a-f]{40,64}$' + OR decl->>'model' IS DISTINCT FROM (CASE v_stage WHEN 'graph_embed' THEN 'local-token-count/1' WHEN 'graph_cluster' THEN 'local-nearest-centroid/1' WHEN 'graph_narrative' THEN 'local-cluster-summary/1' END) OR COALESCE(decl->>'runtime','')='' + OR jsonb_typeof(decl->'seed') IS DISTINCT FROM 'number' OR jsonb_typeof(decl->'config') IS DISTINCT FROM 'object' + OR jsonb_typeof(decl->'snapshot') IS DISTINCT FROM 'object' + OR jsonb_typeof(decl->'snapshot'->'data'->'texts') IS DISTINCT FROM 'array' + OR jsonb_array_length(decl->'snapshot'->'data'->'texts') NOT BETWEEN 1 AND 100 + OR (decl->'snapshot'->'data')-'texts'<>'{}'::jsonb + OR public.pd_graph_hash(decl->'snapshot'->'data') IS DISTINCT FROM decl->'snapshot'->>'sha256' + OR (decl->'snapshot')-'data'-'sha256'<>'{}'::jsonb + OR COALESCE((decl->>'memory_bytes')::bigint,0) NOT BETWEEN 1 AND (CASE cls WHEN 'delphi' THEN 536870912 ELSE 2147483648 END) + OR COALESCE((decl->>'work_units')::integer,0) NOT BETWEEN 1 AND 10000 + OR jsonb_typeof(n->'inputs') IS DISTINCT FROM 'array' OR jsonb_array_length(n->'inputs')>4 + OR COALESCE((n->>'max_attempts')::integer,0) NOT BETWEEN 1 AND 10 + THEN RAISE EXCEPTION 'invalid stage contract (incremental not supported)'; END IF; + IF v_stage='graph_embed' AND jsonb_array_length(n->'inputs')<>0 OR v_stage<>'graph_embed' AND jsonb_array_length(n->'inputs')<>1 + THEN RAISE EXCEPTION 'stage input arity'; END IF; + jid=CASE WHEN NOT EXISTS(SELECT 1 FROM public.delphi_graph_nodes WHERE env=p_env AND graph_id=gid) THEN root ELSE gen_random_uuid() END; + rid=gen_random_uuid(); + reply=public.pq_enqueue(p_env,p_zid,'graph:'||p_scope||':'||(n->>'key'),'graph-admission',p_key, + public.pd_graph_hash(n),rid,jid,'graph://'||gid::text||'/'||(n->>'key'),public.pd_graph_hash(decl),public.pd_graph_hash(decl->'config'), + decl->>'code',1::smallint,(n->>'max_attempts')::integer); + IF reply->>'outcome'<>'enqueued' THEN RAISE EXCEPTION 'graph product conflict'; END IF; + UPDATE public.polis_queue_runs SET contract_version='polis-queue/5' WHERE env=p_env AND run_id=rid; + INSERT INTO public.delphi_jobs(job_id,env,zid,kind,parent_job_id,run_id,origin,replayable,reuse_eligible,status,config_effective,code_version,model_versions) + VALUES(jid,p_env,p_zid,substring(v_stage from 7),CASE WHEN jid<>root THEN root END,rid,'queued',true,true,'queued',decl->'config',decl->>'code',jsonb_build_object('exact',decl->>'model')); + INSERT INTO public.delphi_graph_nodes VALUES(p_env,p_zid,gid,jid,rid,n->>'key',decl,NULL,NULL); + UPDATE public.polis_queue_jobs SET stage=n->>'stage',worker_class=cls WHERE env=p_env AND job_id=jid; + END LOOP; + FOR n IN SELECT value FROM jsonb_array_elements(p_spec->'nodes') LOOP + SELECT job_id INTO STRICT jid FROM public.delphi_graph_nodes WHERE env=p_env AND graph_id=gid AND node_key=n->>'key'; + FOR e IN SELECT value FROM jsonb_array_elements(n->'inputs') LOOP + aid=NULL; + IF e-'node'-'artifact_id'-'sha256'-'contract_sha256'-'role'<>'{}'::jsonb + OR e->>'role' IS DISTINCT FROM (CASE n->>'stage' WHEN 'graph_cluster' THEN 'embeddings' WHEN 'graph_narrative' THEN 'clusters' END) + OR (e ? 'node')=(e ? 'artifact_id') THEN RAISE EXCEPTION 'invalid input role or reference'; END IF; + IF e ? 'node' THEN + SELECT job_id INTO STRICT prod FROM public.delphi_graph_nodes WHERE env=p_env AND graph_id=gid AND node_key=e->>'node'; + ELSE + aid=(e->>'artifact_id')::uuid; + SELECT a.job_id INTO STRICT prod FROM public.delphi_artifacts a JOIN public.delphi_graph_nodes pn USING(env,job_id) + WHERE a.env=p_env AND a.artifact_id=aid AND pn.zid=p_zid AND a.content_sha=e->>'sha256' AND public.pd_graph_hash(pn.declared)=e->>'contract_sha256'; + END IF; + IF NOT EXISTS(SELECT 1 FROM public.polis_queue_jobs qp WHERE qp.env=p_env AND qp.job_id=prod AND qp.stage=CASE n->>'stage' WHEN 'graph_cluster' THEN 'graph_embed' WHEN 'graph_narrative' THEN 'graph_cluster' END) + THEN RAISE EXCEPTION 'input stage mismatch'; END IF; + IF (SELECT gn.declared->'snapshot'->>'sha256' FROM public.delphi_graph_nodes gn WHERE gn.env=p_env AND gn.job_id=prod) IS DISTINCT FROM n->'declared'->'snapshot'->>'sha256' THEN RAISE EXCEPTION 'upstream snapshot mismatch'; END IF; + INSERT INTO public.delphi_graph_edges VALUES(p_env,jid,prod,e->>'role',e->>'sha256',aid); + END LOOP; + END LOOP; + UPDATE public.delphi_graphs SET sealed=true WHERE env=p_env AND graph_id=gid; + INSERT INTO public.delphi_job_guards VALUES(p_env,p_scope,p_zid,root,public.pd_graph_hash(p_spec)); + RETURN jsonb_build_object('outcome','enqueued','graph_id',gid,'root_job_id',root); +END $$; + +CREATE FUNCTION public.pd_graph_readiness(p_env text,p_job uuid) RETURNS jsonb +LANGUAGE sql STABLE SECURITY DEFINER SET search_path=pg_catalog,pg_temp AS $$ + SELECT jsonb_build_object('schema','polis-job-readiness/1','job_id',p_job,'state',q.state, + 'blockers',COALESCE((SELECT jsonb_agg(jsonb_build_object('producer',e.producer,'reason',CASE + WHEN p.state='dead' THEN 'dependency_dead' WHEN p.state='cancelled' THEN 'dependency_cancelled' + WHEN p.state<>'succeeded' THEN 'waiting_for_input' + WHEN a.artifact_id IS NULL THEN 'unresolved_output' + ELSE 'digest_mismatch' END) ORDER BY e.input_role) + FROM public.delphi_graph_edges e JOIN public.polis_queue_jobs p ON p.env=e.env AND p.job_id=e.producer + LEFT JOIN public.delphi_artifacts a ON a.env=e.env AND a.job_id=e.producer + WHERE e.env=p_env AND e.consumer=p_job AND (p.state<>'succeeded' OR a.artifact_id IS NULL OR (e.expected_sha IS NOT NULL AND e.expected_sha<>a.content_sha))),'[]'::jsonb)) + FROM public.polis_queue_jobs q JOIN public.delphi_graph_nodes n USING(env,job_id) WHERE q.env=p_env AND q.job_id=p_job +$$; +CREATE FUNCTION public.pd_graph_claim(p_env text,p_priority smallint,p_owner uuid,p_attempt uuid,p_lease integer,p_class text,p_stages text) +RETURNS jsonb LANGUAGE plpgsql SECURITY DEFINER SET search_path=pg_catalog,pg_temp AS $$ +#variable_conflict use_column +DECLARE j public.polis_queue_jobs; n public.delphi_graph_nodes; inputs jsonb; v_resolved jsonb; +BEGIN + IF p_owner IS NULL OR p_attempt IS NULL OR p_lease IS NULL OR p_lease NOT BETWEEN 10 AND 900 + OR p_class IS NULL OR p_class NOT IN ('delphi','large') THEN RAISE EXCEPTION 'invalid graph claim'; END IF; + SELECT q.* INTO j FROM public.polis_queue_jobs q JOIN public.delphi_graph_nodes n USING(env,job_id) + JOIN public.delphi_graphs g USING(env,graph_id) + WHERE q.env=p_env AND q.priority=p_priority AND q.worker_class=p_class AND q.stage=ANY(string_to_array(p_stages,',')) + AND g.sealed AND q.state IN ('queued','retry_wait') AND q.eligible_at<=statement_timestamp() + AND q.attempt_count-q.parked_attempt_count'blockers'='[]'::jsonb + AND NOT EXISTS(SELECT 1 FROM public.polis_queue_attempts a WHERE a.env=q.env AND a.job_id=q.job_id AND a.process_exit_confirmed_at IS NULL) + AND NOT EXISTS(SELECT 1 FROM public.delphi_provider_requests pr WHERE pr.env=q.env AND pr.job_id=q.job_id AND pr.state IN ('intent','submission_unknown','submitted')) + ORDER BY q.eligible_at,q.created_at,q.job_id FOR UPDATE OF q SKIP LOCKED LIMIT 1; + IF NOT FOUND THEN RETURN jsonb_build_object('schema_version','polis-queue/5','outcome','none'); END IF; + UPDATE public.delphi_graph_edges e SET artifact_id=a.artifact_id FROM public.delphi_artifacts a + WHERE e.env=p_env AND e.consumer=j.job_id AND e.artifact_id IS NULL AND a.env=e.env AND a.job_id=e.producer + AND (e.expected_sha IS NULL OR e.expected_sha=a.content_sha); + SELECT * INTO STRICT n FROM public.delphi_graph_nodes WHERE env=p_env AND job_id=j.job_id; + IF n.resolved IS NULL THEN + SELECT COALESCE(jsonb_object_agg(e.input_role,jsonb_build_object('artifact_id',a.artifact_id,'producer',a.job_id,'run_id',a.run_id, + 'attempt_id',a.attempt_id,'sha256',a.content_sha,'schema',a.schema_version,'payload',a.payload)),'{}'::jsonb) + INTO inputs FROM public.delphi_graph_edges e JOIN public.delphi_artifacts a USING(env,artifact_id) WHERE e.env=p_env AND e.consumer=j.job_id; + v_resolved=jsonb_build_object('schema','polis-job-input/1','declared',n.declared,'artifacts',inputs); + UPDATE public.delphi_graph_nodes SET resolved=v_resolved,resolved_sha=public.pd_graph_hash(v_resolved) WHERE env=p_env AND job_id=j.job_id; + n.resolved=v_resolved; n.resolved_sha=public.pd_graph_hash(v_resolved); + END IF; + UPDATE public.polis_queue_jobs SET state='running',owner_id=p_owner,attempt_id=p_attempt, + locked_until=clock_timestamp()+make_interval(secs=>p_lease),lease_epoch=lease_epoch+1, + version=version+1,mgmt_version=mgmt_version+1,attempt_count=attempt_count+1,updated_at=clock_timestamp() + WHERE env=p_env AND job_id=j.job_id RETURNING * INTO j; + INSERT INTO public.polis_queue_attempts(env,attempt_id,job_id,owner_id,lease_epoch,outcome) + VALUES(p_env,p_attempt,j.job_id,p_owner,j.lease_epoch,'running'); + RETURN public.pq_result('owned',j)||jsonb_build_object('graph_input',n.resolved,'graph_input_wire',n.resolved::text,'graph_input_sha',n.resolved_sha,'zid',n.zid); +END $$; + +CREATE FUNCTION public.pd_graph_finalize(p_env text,p_job uuid,p_owner uuid,p_attempt uuid,p_epoch bigint,p_uri text,p_sha text) +RETURNS jsonb LANGUAGE plpgsql SECURITY DEFINER SET search_path=pg_catalog,pg_temp AS $$ +#variable_conflict use_column +DECLARE j public.polis_queue_jobs; a public.polis_queue_attempts; n public.delphi_graph_nodes; m jsonb; raw text; outdoc jsonb; +BEGIN + j=public.pd_lock(p_env,p_job); + SELECT * INTO a FROM public.polis_queue_attempts WHERE env=p_env AND attempt_id=p_attempt AND job_id=p_job; + IF j.state='succeeded' AND j.terminal_attempt_id=p_attempt AND a.owner_id=p_owner AND a.lease_epoch=p_epoch THEN + RETURN public.pq_result(CASE WHEN a.output_sha256=p_sha THEN 'already_succeeded' ELSE 'invalid_output' END,j); + END IF; + IF j.stage NOT LIKE 'graph_%' OR NOT public.pq_owns(j,p_owner,p_attempt,p_epoch) THEN RETURN public.pq_result('fenced',j); END IF; + IF a.process_exit_confirmed_at IS NULL THEN RAISE EXCEPTION 'process exit proof required'; END IF; + SELECT * INTO STRICT n FROM public.delphi_graph_nodes WHERE env=p_env AND job_id=p_job; + SELECT line INTO raw FROM public.polis_queue_logs WHERE env=p_env AND attempt_id=p_attempt AND stream='manifest' + AND encode(sha256(convert_to(line,'UTF8')),'hex')=p_sha ORDER BY seq DESC LIMIT 1; + IF raw IS NULL THEN RETURN public.pq_result('invalid_output',j); END IF; + m=raw::jsonb; outdoc=m->'output'; + IF m->>'schema' IS DISTINCT FROM 'polis-job-artifact-manifest/1' + OR m->>'job_id' IS DISTINCT FROM p_job::text OR m->>'run_id' IS DISTINCT FROM j.run_id::text + OR m->>'attempt_id' IS DISTINCT FROM p_attempt::text OR m->>'stage' IS DISTINCT FROM j.stage + OR m->>'input_sha256' IS DISTINCT FROM n.resolved_sha OR m->>'outcome' IS DISTINCT FROM 'succeeded' + OR m-'schema'-'job_id'-'run_id'-'attempt_id'-'stage'-'input_sha256'-'outcome'-'output'<>'{}'::jsonb + OR outdoc-'role'-'schema'-'payload'-'sha256'<>'{}'::jsonb + OR outdoc->>'role' IS DISTINCT FROM 'result' OR outdoc->>'schema' IS DISTINCT FROM j.stage||'/1' + OR jsonb_typeof(outdoc->'payload') IS DISTINCT FROM 'string' OR octet_length(outdoc->>'payload')>524288 + OR outdoc->>'sha256' IS DISTINCT FROM encode(sha256(convert_to(outdoc->>'payload','UTF8')),'hex') + THEN RETURN public.pq_result('invalid_output',j); END IF; + IF EXISTS(SELECT 1 FROM public.delphi_provider_requests WHERE env=p_env AND job_id=p_job AND state IN ('intent','submission_unknown','submitted')) + THEN RAISE EXCEPTION 'provider request unresolved'; END IF; + INSERT INTO public.delphi_artifacts(env,job_id,run_id,attempt_id,output_role,schema_version,payload,content_sha,byte_count) + VALUES(p_env,p_job,j.run_id,p_attempt,'result',outdoc->>'schema',outdoc->>'payload',outdoc->>'sha256',octet_length(outdoc->>'payload')); + UPDATE public.polis_queue_attempts SET outcome='succeeded',ended_at=clock_timestamp(),output_sha256=p_sha WHERE env=p_env AND attempt_id=p_attempt; + UPDATE public.polis_queue_jobs SET state='succeeded',terminal_attempt_id=p_attempt,owner_id=NULL,attempt_id=NULL,locked_until=NULL, + output_sha256=p_sha,version=version+1,mgmt_version=mgmt_version+1,updated_at=clock_timestamp() WHERE env=p_env AND job_id=p_job RETURNING * INTO j; + UPDATE public.delphi_jobs SET output_manifest_digest=decode(p_sha,'hex') WHERE env=p_env AND job_id=p_job; + UPDATE public.polis_queue_runs SET state='succeeded',expected_output_uri='pg-artifact://'||p_job::text,expected_output_sha256=p_sha,output_sha256=p_sha WHERE env=p_env AND run_id=j.run_id; + RETURN public.pq_result('succeeded',j,false); +END $$; +CREATE FUNCTION public.pd_graph_reconcile(p_env text) RETURNS jsonb +LANGUAGE plpgsql SECURITY DEFINER SET search_path=pg_catalog,pg_temp AS $$ +#variable_conflict use_column +DECLARE g record; released integer=0; +BEGIN + FOR g IN SELECT d.scope_key FROM public.delphi_job_guards d JOIN public.delphi_graphs g ON g.env=d.env AND g.root_job_id=d.root_job_id + WHERE d.env=p_env AND NOT EXISTS(SELECT 1 FROM public.delphi_graph_nodes gn JOIN public.polis_queue_jobs jq USING(env,job_id) WHERE gn.env=g.env AND gn.graph_id=g.graph_id AND jq.state NOT IN ('succeeded','dead','cancelled')) AND NOT EXISTS(SELECT 1 FROM public.delphi_graph_nodes gn JOIN public.polis_queue_attempts a USING(env,job_id) WHERE gn.env=g.env AND gn.graph_id=g.graph_id AND a.process_exit_confirmed_at IS NULL) AND NOT EXISTS(SELECT 1 FROM public.delphi_graph_nodes gn JOIN public.delphi_provider_requests pr USING(env,job_id) WHERE gn.env=g.env AND gn.graph_id=g.graph_id AND pr.state IN ('intent','submission_unknown','submitted')) ORDER BY d.scope_key LIMIT 100 LOOP + IF public.pd_release_scope(p_env,g.scope_key) THEN released=released+1; END IF; + END LOOP; + RETURN jsonb_build_object('released',released); +END $$; +CREATE FUNCTION public.pd_graph_view(p_env text,p_graph uuid) RETURNS jsonb +LANGUAGE sql STABLE SECURITY DEFINER SET search_path=pg_catalog,pg_temp AS $$ + SELECT jsonb_build_object('schema','polis-job-graph-status/1','graph_id',g.graph_id,'supersedes',g.supersedes,'sealed',g.sealed, + 'guard_held',EXISTS(SELECT 1 FROM public.delphi_job_guards WHERE env=g.env AND root_job_id=g.root_job_id), + 'nodes',(SELECT jsonb_agg(jsonb_build_object('key',n.node_key,'job_id',n.job_id,'run_id',n.run_id,'declared_sha',public.pd_graph_hash(n.declared), + 'readiness',public.pd_graph_readiness(n.env,n.job_id),'attempts',q.attempt_count,'mgmt_version',q.mgmt_version::text, + 'input_sha256',n.resolved_sha,'provider_requests',(SELECT COALESCE(jsonb_agg(jsonb_build_object('request_id',pr.request_id,'state',pr.state,'batch_id',pr.provider_batch_id)),'[]'::jsonb) FROM public.delphi_provider_requests pr WHERE pr.env=n.env AND pr.job_id=n.job_id),'artifact',(SELECT to_jsonb(a) FROM public.delphi_artifacts a WHERE a.env=n.env AND a.job_id=n.job_id)) ORDER BY n.node_key) + FROM public.delphi_graph_nodes n JOIN public.polis_queue_jobs q USING(env,job_id) WHERE n.env=g.env AND n.graph_id=g.graph_id)) + FROM public.delphi_graphs g WHERE g.env=p_env AND g.graph_id=p_graph +$$; +CREATE FUNCTION public.pd_graph_bundle(p_env text,p_artifact uuid) RETURNS jsonb +LANGUAGE sql STABLE SECURITY DEFINER SET search_path=pg_catalog,pg_temp AS $$ + WITH RECURSIVE upstream(id) AS ( + SELECT p_artifact UNION SELECT e.artifact_id FROM upstream u JOIN public.delphi_artifacts a ON a.env=p_env AND a.artifact_id=u.id + JOIN public.delphi_graph_edges e ON e.env=a.env AND e.consumer=a.job_id WHERE e.artifact_id IS NOT NULL) + SELECT jsonb_build_object('schema','polis-job-bundle/1','root',p_artifact, + 'artifacts',jsonb_agg(to_jsonb(a) ORDER BY a.artifact_id)) FROM upstream u JOIN public.delphi_artifacts a ON a.env=p_env AND a.artifact_id=u.id +$$; +CREATE FUNCTION public.pd_graph_publish(p_env text,p_graph uuid,p_job uuid,p_expected bigint) RETURNS jsonb +LANGUAGE plpgsql SECURITY DEFINER SET search_path=pg_catalog,pg_temp AS $$ +#variable_conflict use_column +DECLARE g public.delphi_graphs; aid uuid; current_generation bigint; +BEGIN + SELECT * INTO STRICT g FROM public.delphi_graphs WHERE env=p_env AND graph_id=p_graph AND sealed; + PERFORM pg_advisory_xact_lock(hashtextextended(jsonb_build_array('pd:scope',p_env,g.scope_key)::text,0)); + SELECT a.artifact_id INTO STRICT aid FROM public.delphi_graph_nodes n JOIN public.delphi_artifacts a USING(env,job_id) + JOIN public.polis_queue_jobs q USING(env,job_id) WHERE n.env=p_env AND n.graph_id=p_graph AND n.job_id=p_job AND q.stage='graph_narrative' AND q.state='succeeded'; + IF EXISTS(SELECT 1 FROM public.delphi_graph_nodes n JOIN public.polis_queue_jobs q USING(env,job_id) WHERE n.env=p_env AND n.graph_id=p_graph AND q.state<>'succeeded') + OR EXISTS(SELECT 1 FROM public.delphi_graphs x WHERE x.env=p_env AND x.zid=g.zid AND x.scope_key=g.scope_key AND x.created_at>g.created_at) THEN RAISE EXCEPTION 'incomplete or superseded bundle'; END IF; + SELECT generation INTO current_generation FROM public.delphi_graph_served WHERE env=p_env AND zid=g.zid AND scope_key=g.scope_key FOR UPDATE; + IF COALESCE(current_generation,0) IS DISTINCT FROM p_expected THEN RETURN jsonb_build_object('outcome','conflict'); END IF; + INSERT INTO public.delphi_graph_served VALUES(p_env,g.zid,g.scope_key,p_expected+1,aid) + ON CONFLICT(env,zid,scope_key) DO UPDATE SET generation=EXCLUDED.generation,artifact_id=EXCLUDED.artifact_id; + RETURN jsonb_build_object('outcome','published','generation',p_expected+1,'bundle',public.pd_graph_bundle(p_env,aid)); +END $$; +CREATE FUNCTION public.pd_graph_served(p_env text,p_zid integer,p_scope text) RETURNS jsonb +LANGUAGE sql STABLE SECURITY DEFINER SET search_path=pg_catalog,pg_temp AS $$ + SELECT jsonb_build_object('generation',generation,'bundle',public.pd_graph_bundle(env,artifact_id)) + FROM public.delphi_graph_served WHERE env=p_env AND zid=p_zid AND scope_key=p_scope +$$; +CREATE FUNCTION public.pd_graph_depth(p_env text,p_class text) RETURNS jsonb +LANGUAGE sql STABLE SECURITY DEFINER SET search_path=pg_catalog,pg_temp AS $$ + SELECT jsonb_build_object('schema','polis-job-depth/1', + 'runnable',count(*) FILTER(WHERE state IN ('queued','retry_wait') AND eligible_at<=statement_timestamp() AND public.pd_graph_readiness(q.env,q.job_id)->'blockers'='[]'::jsonb AND q.attempt_count-q.parked_attempt_count'blockers'<>'[]'::jsonb), + 'running',count(*) FILTER(WHERE state='running'),'dead',count(*) FILTER(WHERE state='dead')) + FROM public.polis_queue_jobs q WHERE env=p_env AND worker_class=p_class AND stage LIKE 'graph_%' +$$; +CREATE OR REPLACE FUNCTION public.pd_finalize(p_env text,p_job uuid,p_owner uuid,p_attempt uuid,p_epoch bigint,p_uri text,p_sha text) +RETURNS jsonb LANGUAGE plpgsql SET search_path=pg_catalog,pg_temp SET TimeZone='UTC' AS $$ +DECLARE j public.polis_queue_jobs; a public.polis_queue_attempts; r public.polis_queue_runs; raw text; m jsonb; +BEGIN + j=public.pd_lock(p_env,p_job); + IF j.stage LIKE 'graph_%' THEN RAISE EXCEPTION 'graph manifest required'; END IF; + SELECT * INTO a FROM public.polis_queue_attempts WHERE env=p_env AND job_id=p_job AND attempt_id=p_attempt; + SELECT * INTO r FROM public.polis_queue_runs WHERE env=p_env AND run_id=j.run_id; + IF j.state='succeeded' AND j.terminal_attempt_id=p_attempt AND a.owner_id=p_owner AND a.lease_epoch=p_epoch THEN + RETURN public.pq_result(CASE WHEN a.output_sha256=p_sha AND r.expected_output_uri=p_uri THEN 'already_succeeded' ELSE 'invalid_output' END,j); + END IF; + IF NOT public.pq_owns(j,p_owner,p_attempt,p_epoch) THEN RETURN public.pq_result('fenced',j); END IF; + IF a.process_exit_confirmed_at IS NULL THEN RAISE EXCEPTION 'process exit proof required'; END IF; + IF p_uri IS NULL OR p_uri !~ '^file://.+' OR length(p_uri)>2048 OR p_sha IS NULL OR p_sha !~ '^[0-9a-f]{64}$' + THEN RETURN public.pq_result('invalid_output',j); END IF; + SELECT line INTO raw FROM public.polis_queue_logs WHERE env=p_env AND attempt_id=p_attempt AND stream='manifest' + AND encode(sha256(convert_to(line,'UTF8')),'hex')=p_sha ORDER BY seq DESC LIMIT 1; + IF raw IS NULL THEN RETURN public.pq_result('invalid_output',j); END IF; + BEGIN m=raw::jsonb; EXCEPTION WHEN invalid_text_representation THEN RETURN public.pq_result('invalid_output',j); END; + IF m->>'schema' IS DISTINCT FROM 'polis-jobs.output-manifest/1' + OR m->>'job_id' IS DISTINCT FROM p_job::text OR m->>'attempt_id' IS DISTINCT FROM p_attempt::text + OR m->>'stage' IS DISTINCT FROM j.stage OR m->>'outcome' IS DISTINCT FROM 'succeeded' + OR COALESCE(m->>'phase','') NOT IN ('submit','recheck','run') + OR jsonb_typeof(m->'outputs') IS DISTINCT FROM 'array' + OR jsonb_typeof(m->'inputs') IS DISTINCT FROM 'object' + OR jsonb_typeof(m->'models') IS DISTINCT FROM 'object' + OR jsonb_typeof(m->'cost') IS DISTINCT FROM 'object' + OR (m ? 'artifacts' AND m->'artifacts'<>'[]'::jsonb) + THEN RETURN public.pq_result('invalid_output',j); END IF; + IF EXISTS(SELECT 1 FROM jsonb_array_elements(m->'outputs') o WHERE o->>'store' IS DISTINCT FROM 'dynamodb' + OR COALESCE(o->>'table','')='' OR NOT (o ? 'keys' OR o ? 'key_prefix') OR COALESCE(o->>'rows','') !~ '^[0-9]+$') + THEN RETURN public.pq_result('invalid_output',j); END IF; + IF EXISTS(SELECT 1 FROM public.delphi_provider_requests WHERE env=p_env AND job_id=p_job AND state IN ('intent','submission_unknown','submitted')) + THEN RAISE EXCEPTION 'provider request unresolved'; END IF; + UPDATE public.polis_queue_attempts SET outcome='succeeded',ended_at=clock_timestamp(),output_sha256=p_sha WHERE env=p_env AND attempt_id=p_attempt; + UPDATE public.polis_queue_jobs SET state='succeeded',terminal_attempt_id=p_attempt,owner_id=NULL,attempt_id=NULL,locked_until=NULL, + output_sha256=p_sha,version=version+1,mgmt_version=mgmt_version+1,updated_at=clock_timestamp() WHERE env=p_env AND job_id=p_job RETURNING * INTO j; + UPDATE public.delphi_jobs SET output_manifest_digest=decode(p_sha,'hex') WHERE env=p_env AND job_id=p_job; + UPDATE public.polis_queue_runs SET state='succeeded',expected_output_uri=p_uri,expected_output_sha256=p_sha,output_sha256=p_sha WHERE env=p_env AND run_id=j.run_id; + RETURN public.pq_result('succeeded',j,false); +END $$; +-- Explicit operator remediation, never granted to an ordinary executor. +-- Preserve the existing class-depth wire; graph blocked work is not scale-out demand. +CREATE OR REPLACE FUNCTION public.pq_class_depth(p_env text,p_worker_class text) +RETURNS jsonb LANGUAGE plpgsql STABLE SECURITY DEFINER SET search_path=pg_catalog,pg_temp SET TimeZone='UTC' AS $$ +DECLARE c record; +BEGIN + IF p_env IS NULL OR p_env='' OR p_worker_class IS NULL OR p_worker_class NOT IN ('delphi','large') + THEN RAISE EXCEPTION 'invalid class depth read'; END IF; + SELECT count(*) FILTER (WHERE q.state IN ('queued','retry_wait') AND (q.stage NOT LIKE 'graph_%' OR (q.eligible_at<=statement_timestamp() AND q.attempt_count-q.parked_attempt_count'blockers'='[]'::jsonb AND NOT EXISTS(SELECT 1 FROM public.polis_queue_attempts a WHERE a.env=q.env AND a.job_id=q.job_id AND a.process_exit_confirmed_at IS NULL) AND NOT EXISTS(SELECT 1 FROM public.delphi_provider_requests pr WHERE pr.env=q.env AND pr.job_id=q.job_id AND pr.state IN ('intent','submission_unknown','submitted'))))) AS queued, + count(*) FILTER (WHERE q.state='running') AS leased, + count(*) FILTER (WHERE q.state='parked') AS parked, + count(*) FILTER (WHERE q.state='dead') AS dead, + min(q.created_at) FILTER (WHERE q.state NOT IN ('succeeded','dead','cancelled')) AS oldest + INTO c FROM public.polis_queue_jobs q WHERE q.env=p_env AND q.worker_class=p_worker_class; + RETURN jsonb_build_object('schema_version','polis-queue/3','outcome','class_depth','env',p_env,'worker_class',p_worker_class, + 'queued',c.queued,'leased',c.leased,'parked',c.parked,'dead',c.dead,'oldest_unresolved_created_at',c.oldest); +END $$; +-- No table access or internal helper privilege crosses the executor boundary. +DO $$ DECLARE r record; BEGIN + FOR r IN SELECT oid::regclass AS name FROM pg_class WHERE relnamespace='public'::regnamespace AND (relname LIKE 'delphi_graph_%' OR relname='delphi_graphs' OR relname='delphi_artifacts') AND relkind='r' LOOP + EXECUTE format('REVOKE ALL ON %s FROM PUBLIC,polis_queue_executor',r.name); + END LOOP; + FOR r IN SELECT oid::regprocedure AS name FROM pg_proc WHERE pronamespace='public'::regnamespace AND proname LIKE 'pd_graph_%' LOOP + EXECUTE format('REVOKE ALL ON FUNCTION %s FROM PUBLIC,polis_queue_executor',r.name); + END LOOP; +END $$; +GRANT EXECUTE ON FUNCTION public.pd_graph_admit(text,integer,text,text,jsonb,uuid),public.pd_graph_claim(text,smallint,uuid,uuid,integer,text,text), + public.pd_graph_finalize(text,uuid,uuid,uuid,bigint,text,text),public.pd_graph_reconcile(text), + public.pd_graph_view(text,uuid),public.pd_graph_readiness(text,uuid),public.pd_graph_depth(text,text), + public.pd_graph_publish(text,uuid,uuid,bigint),public.pd_graph_served(text,integer,text) TO polis_queue_executor; +COMMIT; diff --git a/server/postgres/migrations/000028_create_delphi_results.sql b/server/postgres/migrations/000028_create_delphi_results.sql new file mode 100644 index 0000000000..a25a384c18 --- /dev/null +++ b/server/postgres/migrations/000028_create_delphi_results.sql @@ -0,0 +1,255 @@ +-- Draft #1428, step 7. Run-bound, immutable Delphi results on graph contract /5. +-- Does not rewrite historical migrations or mutate any vote or result row. +BEGIN; +SET LOCAL lock_timeout='5s'; +GRANT SELECT(report_id,zid) ON public.reports TO polis_queue_owner; +SET LOCAL ROLE polis_queue_owner; +SET LOCAL search_path=pg_catalog,pg_temp; +CREATE TABLE public.delphi_result_catalog(family text PRIMARY KEY, key_spec jsonb NOT NULL); +INSERT INTO public.delphi_result_catalog VALUES +('Delphi_PCAConversationConfig','[["zid", "S"]]'::jsonb), +('Delphi_PCAResults','[["zid", "S"], ["math_tick", "N"]]'::jsonb), +('Delphi_KMeansClusters','[["zid_tick", "S"], ["group_id", "N"]]'::jsonb), +('Delphi_CommentRouting','[["zid_tick", "S"], ["comment_id", "S"]]'::jsonb), +('Delphi_RepresentativeComments','[["zid_tick_gid", "S"], ["comment_id", "S"]]'::jsonb), +('Delphi_PCAParticipantProjections','[["zid_tick", "S"], ["participant_id", "S"]]'::jsonb), +('Delphi_UMAPConversationConfig','[["conversation_id", "S"]]'::jsonb), +('Delphi_CommentEmbeddings','[["conversation_id", "S"], ["comment_id", "N"]]'::jsonb), +('Delphi_CommentHierarchicalClusterAssignments','[["conversation_id", "S"], ["comment_id", "N"]]'::jsonb), +('Delphi_CommentClustersStructureKeywords','[["conversation_id", "S"], ["cluster_key", "S"]]'::jsonb), +('Delphi_UMAPGraph','[["conversation_id", "S"], ["edge_id", "S"]]'::jsonb), +('Delphi_CommentClustersFeatures','[["conversation_id", "S"], ["cluster_key", "S"]]'::jsonb), +('Delphi_CommentClustersLLMTopicNames','[["conversation_id", "S"], ["topic_key", "S"]]'::jsonb), +('Delphi_NarrativeReports','[["rid_section_model", "S"], ["timestamp", "S"]]'::jsonb), +('Delphi_CommentExtremity','[["conversation_id", "S"], ["comment_id", "S"]]'::jsonb), +('Delphi_TopicAgendaSelections','[["conversation_id", "S"], ["participant_id", "S"]]'::jsonb), +('Delphi_CollectiveStatement','[["zid_topic_jobid", "S"]]'::jsonb), +('report_narrative_store','[["rid_section_model", "S"], ["timestamp", "S"]]'::jsonb); +CREATE TABLE public.delphi_result_batches( + env text NOT NULL, batch_id uuid NOT NULL, job_id uuid NOT NULL, run_id uuid NOT NULL, + sealed_sha text, artifact_id uuid, created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY(env,batch_id), UNIQUE(env,artifact_id), + FOREIGN KEY(env,job_id,run_id) REFERENCES public.delphi_graph_nodes(env,job_id,run_id), + FOREIGN KEY(env,job_id,batch_id) REFERENCES public.polis_queue_attempts(env,job_id,attempt_id), + FOREIGN KEY(env,artifact_id) REFERENCES public.delphi_artifacts(env,artifact_id) +); +CREATE TABLE public.delphi_result_families( + env text NOT NULL,batch_id uuid NOT NULL,family text NOT NULL REFERENCES public.delphi_result_catalog, + codec_wire text NOT NULL, content_sha text NOT NULL, row_count integer NOT NULL CHECK(row_count>=0), + PRIMARY KEY(env,batch_id,family), FOREIGN KEY(env,batch_id) REFERENCES public.delphi_result_batches, + CHECK(content_sha=encode(sha256(convert_to(codec_wire,'UTF8')),'hex')) +); +CREATE TABLE public.delphi_result_rows( + env text NOT NULL,batch_id uuid NOT NULL,family text NOT NULL,item_key jsonb NOT NULL, + ordinal integer NOT NULL CHECK(ordinal>0),item jsonb NOT NULL CHECK(jsonb_typeof(item)='object'), + PRIMARY KEY(env,batch_id,family,item_key), UNIQUE(env,batch_id,family,ordinal), + FOREIGN KEY(env,batch_id,family) REFERENCES public.delphi_result_families +); +CREATE TRIGGER result_family_immutable BEFORE UPDATE OR DELETE ON public.delphi_result_families + FOR EACH ROW EXECUTE FUNCTION public.pd_graph_immutable(); +CREATE TRIGGER result_row_immutable BEFORE UPDATE OR DELETE ON public.delphi_result_rows + FOR EACH ROW EXECUTE FUNCTION public.pd_graph_immutable(); +-- Validate AttributeValue tags before making a result readable. No numeric value +-- passes through a float; the immutable original bytes remain the audit record. +CREATE FUNCTION public.pd_result_valid_value(v jsonb) RETURNS boolean +LANGUAGE plpgsql IMMUTABLE SET search_path=pg_catalog,pg_temp AS $$ +DECLARE tag text;body jsonb;x jsonb;t text;n numeric; +BEGIN + IF jsonb_typeof(v)<>'object' OR (SELECT count(*) FROM jsonb_object_keys(v))<>1 THEN RETURN false; END IF; + SELECT key,value INTO tag,body FROM jsonb_each(v); + IF tag='S' THEN RETURN jsonb_typeof(body)='string'; + ELSIF tag='BOOL' THEN RETURN jsonb_typeof(body)='boolean'; + ELSIF tag='NULL' THEN RETURN body='true'::jsonb; + ELSIF tag='N' THEN + IF jsonb_typeof(body)<>'string' THEN RETURN false; END IF; + t=body#>>'{}'; + IF length(t)>256 OR t !~ '^-?(0|[1-9][0-9]*)([.][0-9]*[1-9])?$' OR t='-0' THEN RETURN false; END IF; + n=t::numeric; + RETURN (n=0 OR (abs(n)>=1e-130::numeric AND abs(n)<1e126::numeric)) + AND length(trim(both '0' from replace(replace(t,'-',''),'.','')))<=38; + ELSIF tag='B' THEN + IF jsonb_typeof(body)<>'string' THEN RETURN false; END IF; + t=body#>>'{}';RETURN replace(encode(decode(t,'base64'),'base64'),chr(10),'')=t; + ELSIF tag='M' THEN + IF jsonb_typeof(body)<>'object' THEN RETURN false; END IF; + FOR x IN SELECT value FROM jsonb_each(body) LOOP IF NOT public.pd_result_valid_value(x) THEN RETURN false; END IF; END LOOP; + ELSIF tag IN ('L','SS','NS','BS') THEN + IF jsonb_typeof(body)<>'array' THEN RETURN false; END IF; + IF tag<>'L' AND (jsonb_array_length(body)=0 OR (SELECT count(DISTINCT value) FROM jsonb_array_elements(body))<>jsonb_array_length(body)) THEN RETURN false; END IF; + FOR x IN SELECT value FROM jsonb_array_elements(body) LOOP + IF NOT public.pd_result_valid_value(CASE tag WHEN 'L' THEN x ELSE jsonb_build_object(left(tag,1),x) END) THEN RETURN false; END IF; + END LOOP; + ELSE RETURN false; END IF; + RETURN true; +EXCEPTION WHEN OTHERS THEN RETURN false; +END $$; +-- Internal insertion helper; caller has already acquired the queue row lock and fence. +CREATE FUNCTION public.pd_result_insert_family(p_env text,p_job uuid,p_attempt uuid,p_family text,p_wire text) +RETURNS jsonb LANGUAGE plpgsql SET search_path=pg_catalog,pg_temp AS $$ +#variable_conflict use_column +DECLARE n public.delphi_graph_nodes;b public.delphi_result_batches;spec jsonb;head jsonb; + lines text[];rowdoc jsonb;k jsonb;part jsonb;i integer;sha text;old public.delphi_result_families; +BEGIN + SELECT * INTO STRICT n FROM public.delphi_graph_nodes WHERE env=p_env AND job_id=p_job; + SELECT key_spec INTO STRICT spec FROM public.delphi_result_catalog WHERE family=p_family; + IF p_wire IS NULL OR octet_length(p_wire)>67108864 OR right(p_wire,1)<>chr(10) THEN RAISE EXCEPTION 'invalid result family bytes'; END IF; + sha=encode(sha256(convert_to(p_wire,'UTF8')),'hex'); + INSERT INTO public.delphi_result_batches(env,batch_id,job_id,run_id) VALUES(p_env,p_attempt,p_job,n.run_id) ON CONFLICT DO NOTHING; + SELECT * INTO STRICT b FROM public.delphi_result_batches WHERE env=p_env AND batch_id=p_attempt FOR UPDATE; + IF b.job_id<>p_job OR b.run_id<>n.run_id THEN RAISE EXCEPTION 'result run binding'; END IF; + SELECT * INTO old FROM public.delphi_result_families WHERE env=p_env AND batch_id=p_attempt AND family=p_family; + IF FOUND THEN + IF old.codec_wire<>p_wire THEN RAISE EXCEPTION 'immutable result family conflict'; END IF; + RETURN jsonb_build_object('sha256',old.content_sha,'row_count',old.row_count); + END IF; + IF b.sealed_sha IS NOT NULL THEN RAISE EXCEPTION 'result batch sealed'; END IF; + IF COALESCE((SELECT sum(octet_length(codec_wire)) FROM public.delphi_result_families WHERE env=p_env AND batch_id=p_attempt),0)+octet_length(p_wire)>268435456 THEN RAISE EXCEPTION 'result batch byte limit'; END IF; + lines=string_to_array(left(p_wire,length(p_wire)-1),chr(10));head=lines[1]::jsonb; + IF head IS DISTINCT FROM jsonb_build_object('codec','delphi-storage-codec/1','family',p_family,'key',(SELECT jsonb_agg(x->0) FROM jsonb_array_elements(spec) x)) + THEN RAISE EXCEPTION 'invalid result codec header'; END IF; + INSERT INTO public.delphi_result_families VALUES(p_env,p_attempt,p_family,p_wire,sha,cardinality(lines)-1); + FOR i IN 2..cardinality(lines) LOOP + rowdoc=lines[i]::jsonb;k='[]'::jsonb; + IF jsonb_typeof(rowdoc)<>'object' OR EXISTS(SELECT 1 FROM jsonb_each(rowdoc) a WHERE a.key='' OR NOT public.pd_result_valid_value(a.value)) THEN RAISE EXCEPTION 'invalid result row'; END IF; + FOR part IN SELECT value FROM jsonb_array_elements(spec) LOOP + IF rowdoc->(part->>0) IS NULL OR jsonb_typeof(rowdoc->(part->>0))<>'object' + OR NOT (rowdoc->(part->>0) ? (part->>1)) OR (SELECT count(*) FROM jsonb_object_keys(rowdoc->(part->>0)))<>1 + OR jsonb_typeof(rowdoc->(part->>0)->(part->>1))<>'string' THEN RAISE EXCEPTION 'invalid result key'; END IF; + k=k||jsonb_build_array(rowdoc->(part->>0)); + END LOOP; + IF (rowdoc ? 'conversation_id' AND rowdoc->'conversation_id'->>'S' IS DISTINCT FROM n.zid::text) + OR (rowdoc ? 'zid' AND COALESCE(rowdoc->'zid'->>'S',rowdoc->'zid'->>'N') IS DISTINCT FROM n.zid::text) + OR (rowdoc ? 'zid_tick' AND split_part(rowdoc->'zid_tick'->>'S',':',1)<>n.zid::text) + OR (rowdoc ? 'zid_tick_gid' AND split_part(rowdoc->'zid_tick_gid'->>'S',':',1)<>n.zid::text) + OR (rowdoc ? 'zid_topic_jobid' AND split_part(rowdoc->'zid_topic_jobid'->>'S','#',1)<>n.zid::text) + THEN RAISE EXCEPTION 'result conversation mismatch'; END IF; + IF rowdoc ? 'rid_section_model' AND NOT EXISTS( + SELECT 1 FROM public.reports r WHERE r.zid=n.zid AND r.report_id=split_part(rowdoc->'rid_section_model'->>'S','#',1) + AND (NOT rowdoc ? 'report_id' OR rowdoc->'report_id'->>'S'=r.report_id)) + THEN RAISE EXCEPTION 'result report conversation mismatch'; END IF; + INSERT INTO public.delphi_result_rows VALUES(p_env,p_attempt,p_family,k,i-1,rowdoc); + END LOOP; + RETURN jsonb_build_object('sha256',sha,'row_count',cardinality(lines)-1); +END $$; +CREATE FUNCTION public.pd_result_seal_internal(p_env text,p_attempt uuid) RETURNS jsonb +LANGUAGE plpgsql SET search_path=pg_catalog,pg_temp AS $$ +DECLARE digest text;names jsonb; +BEGIN + SELECT encode(sha256(convert_to(COALESCE(string_agg(family||':'||content_sha||':'||row_count::text,chr(10) ORDER BY family COLLATE "C"),''),'UTF8')),'hex'), + COALESCE(jsonb_agg(family ORDER BY family COLLATE "C"),'[]'::jsonb) INTO digest,names + FROM public.delphi_result_families WHERE env=p_env AND batch_id=p_attempt; + UPDATE public.delphi_result_batches SET sealed_sha=digest WHERE env=p_env AND batch_id=p_attempt AND sealed_sha IS NULL; + IF NOT EXISTS(SELECT 1 FROM public.delphi_result_batches WHERE env=p_env AND batch_id=p_attempt AND sealed_sha=digest) THEN RAISE EXCEPTION 'result seal mismatch'; END IF; + RETURN jsonb_build_object('schema','delphi-result-batch/1','batch_id',p_attempt,'sha256',digest,'families',names); +END $$; +CREATE FUNCTION public.pd_result_put_family(p_env text,p_job uuid,p_owner uuid,p_attempt uuid,p_epoch bigint,p_family text,p_wire text) +RETURNS jsonb LANGUAGE plpgsql SECURITY DEFINER SET search_path=pg_catalog,pg_temp AS $$ +DECLARE j public.polis_queue_jobs; +BEGIN + j=public.pd_lock(p_env,p_job); + IF NOT public.pq_owns(j,p_owner,p_attempt,p_epoch) THEN RAISE EXCEPTION 'result writer fenced'; END IF; + RETURN public.pd_result_insert_family(p_env,p_job,p_attempt,p_family,p_wire); +END $$; +CREATE FUNCTION public.pd_result_seal(p_env text,p_job uuid,p_owner uuid,p_attempt uuid,p_epoch bigint) +RETURNS jsonb LANGUAGE plpgsql SECURITY DEFINER SET search_path=pg_catalog,pg_temp AS $$ +DECLARE j public.polis_queue_jobs; +BEGIN + j=public.pd_lock(p_env,p_job); + IF NOT public.pq_owns(j,p_owner,p_attempt,p_epoch) THEN RAISE EXCEPTION 'result writer fenced'; END IF; + RETURN public.pd_result_seal_internal(p_env,p_attempt); +END $$; +CREATE FUNCTION public.pd_result_bind_artifact() RETURNS trigger LANGUAGE plpgsql SET search_path=pg_catalog,pg_temp AS $$ +DECLARE doc jsonb;ref jsonb;pair record;b public.delphi_result_batches;expected jsonb; +BEGIN + doc=NEW.payload::jsonb; + IF doc ? 'family_files' THEN + IF doc ? 'results' OR jsonb_typeof(doc->'family_files')<>'object' OR doc->'family_files'='{}'::jsonb THEN RAISE EXCEPTION 'invalid inline result families'; END IF; + FOR pair IN SELECT key,value FROM jsonb_each(doc->'family_files') LOOP + IF jsonb_typeof(pair.value)<>'string' THEN RAISE EXCEPTION 'invalid inline codec bytes'; END IF; + PERFORM public.pd_result_insert_family(NEW.env,NEW.job_id,NEW.attempt_id,pair.key,pair.value#>>'{}'); + END LOOP; + ref=public.pd_result_seal_internal(NEW.env,NEW.attempt_id); + ELSIF doc ? 'results' THEN ref=doc->'results'; + ELSE RETURN NEW; END IF; -- Reference graph artifacts remain compatible. + SELECT * INTO STRICT b FROM public.delphi_result_batches WHERE env=NEW.env AND batch_id=NEW.attempt_id FOR UPDATE; + expected=public.pd_result_seal_internal(NEW.env,NEW.attempt_id); + IF ref IS DISTINCT FROM expected OR b.job_id<>NEW.job_id OR b.run_id<>NEW.run_id OR b.artifact_id IS NOT NULL THEN RAISE EXCEPTION 'artifact result binding'; END IF; + UPDATE public.delphi_result_batches SET artifact_id=NEW.artifact_id WHERE env=NEW.env AND batch_id=NEW.attempt_id; + RETURN NEW; +END $$; +-- AFTER INSERT permits the batch's FK to the newly fenced artifact. +CREATE TRIGGER result_artifact_bind AFTER INSERT ON public.delphi_artifacts FOR EACH ROW EXECUTE FUNCTION public.pd_result_bind_artifact(); +CREATE FUNCTION public.pd_result_bundle_rows(p_env text,p_root uuid) +RETURNS TABLE(family text,item_key jsonb,item jsonb,ordinal integer) LANGUAGE sql STABLE SECURITY DEFINER SET search_path=pg_catalog,pg_temp AS $$ + WITH RECURSIVE upstream(id,depth,path) AS ( + SELECT p_root,0,ARRAY[p_root] UNION ALL + SELECT e.artifact_id,u.depth+1,u.path||e.artifact_id FROM upstream u + JOIN public.delphi_artifacts a ON a.env=p_env AND a.artifact_id=u.id + JOIN public.delphi_graph_edges e ON e.env=a.env AND e.consumer=a.job_id + WHERE e.artifact_id IS NOT NULL AND NOT e.artifact_id=ANY(u.path) + ), chosen AS ( + SELECT DISTINCT ON(f.family) f.family,b.batch_id FROM upstream u + JOIN public.delphi_result_batches b ON b.env=p_env AND b.artifact_id=u.id + JOIN public.delphi_result_families f ON f.env=b.env AND f.batch_id=b.batch_id + ORDER BY f.family,u.depth,u.id + ) SELECT r.family,r.item_key,r.item,r.ordinal FROM chosen c JOIN public.delphi_result_rows r + ON r.env=p_env AND r.batch_id=c.batch_id AND r.family=c.family +$$; +CREATE FUNCTION public.pd_result_artifact_wire(p_env text,p_artifact uuid,p_family text) RETURNS jsonb +LANGUAGE sql STABLE SECURITY DEFINER SET search_path=pg_catalog,pg_temp AS $$ + SELECT jsonb_build_object('wire',f.codec_wire,'sha256',f.content_sha,'batch_sha256',b.sealed_sha,'batch_id',b.batch_id) + FROM public.delphi_result_batches b JOIN public.delphi_result_families f USING(env,batch_id) + WHERE b.env=p_env AND b.artifact_id=p_artifact AND f.family=p_family +$$; +CREATE FUNCTION public.pd_result_artifact_family(p_env text,p_artifact uuid,p_family text) RETURNS jsonb +LANGUAGE sql STABLE SECURITY DEFINER SET search_path=pg_catalog,pg_temp AS $$ + SELECT COALESCE(jsonb_agg(item ORDER BY ordinal),'[]'::jsonb) FROM public.pd_result_bundle_rows(p_env,p_artifact) WHERE family=p_family +$$; +CREATE VIEW public.delphi_result_current_rows WITH(security_barrier=true) AS + SELECT s.env,s.zid,s.scope_key,s.generation,r.family,r.item_key,r.item + FROM public.delphi_graph_served s CROSS JOIN LATERAL public.pd_result_bundle_rows(s.env,s.artifact_id) r; +CREATE FUNCTION public.pd_result_served(p_env text,p_zid integer,p_scope text,p_family text) RETURNS jsonb +LANGUAGE sql STABLE SECURITY DEFINER SET search_path=pg_catalog,pg_temp AS $$ + SELECT public.pd_result_artifact_family(p_env,s.artifact_id,p_family) FROM public.delphi_graph_served s + WHERE s.env=p_env AND s.zid=p_zid AND s.scope_key=p_scope +$$; +CREATE FUNCTION public.pd_result_served_bundle(p_env text,p_zid integer,p_scope text) RETURNS jsonb +LANGUAGE sql STABLE SECURITY DEFINER SET search_path=pg_catalog,pg_temp AS $$ + SELECT jsonb_build_object('generation',s.generation,'families',COALESCE((SELECT jsonb_object_agg(family,items) FROM + (SELECT family,jsonb_agg(item ORDER BY ordinal) items FROM public.pd_result_bundle_rows(p_env,s.artifact_id) GROUP BY family) f),'{}'::jsonb)) + FROM public.delphi_graph_served s WHERE s.env=p_env AND s.zid=p_zid AND s.scope_key=p_scope +$$; +-- Archive wire stays text: PostgreSQL JSONB cannot represent NUL in a legacy +-- attribute. Decode the canonical inner file in the language codec, not SQL. +CREATE VIEW public.delphi_result_legacy_controls WITH(security_barrier=true) AS + SELECT s.env,s.zid,s.scope_key,s.generation,c.key AS family,c.value AS codec_wire + FROM public.delphi_graph_served s JOIN public.delphi_artifacts a USING(env,artifact_id) + CROSS JOIN LATERAL jsonb_each_text(COALESCE(a.payload::jsonb->'legacy_control_files','{}'::jsonb)) c + WHERE c.key IN ('Delphi_JobQueue','Delphi_JobActiveGuard'); +CREATE VIEW public.delphi_result_publications WITH(security_barrier=true) AS + SELECT s.env,s.zid,s.scope_key,s.generation,s.artifact_id,a.job_id::text + FROM public.delphi_graph_served s JOIN public.delphi_artifacts a USING(env,artifact_id); +REVOKE ALL ON public.delphi_result_legacy_controls,public.delphi_result_publications FROM PUBLIC,polis_queue_executor; +GRANT SELECT ON public.delphi_result_legacy_controls,public.delphi_result_publications TO polis_queue_executor; +CREATE VIEW public.delphi_result_jobs WITH(security_barrier=true) AS + SELECT env,job_id::text,zid::text AS conversation_id,report_id, + CASE status WHEN 'succeeded' THEN 'COMPLETED' WHEN 'dead' THEN 'FAILED' WHEN 'running' THEN 'PROCESSING' ELSE upper(status) END AS status, + kind AS job_type,config_effective AS job_config, + to_char(created_at AT TIME ZONE 'UTC','YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS created_at, + to_char(completed_at AT TIME ZONE 'UTC','YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS completed_at,error, + (SELECT g.scope_key FROM public.delphi_graph_nodes n JOIN public.delphi_graphs g USING(env,graph_id) + WHERE n.env=delphi_jobs.env AND n.job_id=delphi_jobs.job_id) AS scope_key + FROM public.delphi_jobs; +REVOKE ALL ON public.delphi_result_jobs FROM PUBLIC,polis_queue_executor; +GRANT SELECT ON public.delphi_result_jobs TO polis_queue_executor; +REVOKE ALL ON public.delphi_result_catalog,public.delphi_result_batches,public.delphi_result_families,public.delphi_result_rows,public.delphi_result_current_rows FROM PUBLIC,polis_queue_executor; +DO $$ DECLARE f record; BEGIN + FOR f IN SELECT oid::regprocedure name FROM pg_proc WHERE pronamespace='public'::regnamespace AND starts_with(proname,'pd_result_') LOOP + EXECUTE format('REVOKE ALL ON FUNCTION %s FROM PUBLIC,polis_queue_executor',f.name); END LOOP; +END $$; +GRANT SELECT ON public.delphi_result_current_rows TO polis_queue_executor; +GRANT EXECUTE ON FUNCTION public.pd_result_put_family(text,uuid,uuid,uuid,bigint,text,text), + public.pd_result_seal(text,uuid,uuid,uuid,bigint),public.pd_result_artifact_family(text,uuid,text), + public.pd_result_bundle_rows(text,uuid),public.pd_result_artifact_wire(text,uuid,text), + public.pd_result_served(text,integer,text,text),public.pd_result_served_bundle(text,integer,text) TO polis_queue_executor; +COMMIT; diff --git a/server/postgres/migrations/000029_extend_delphi_graph_stages.sql b/server/postgres/migrations/000029_extend_delphi_graph_stages.sql new file mode 100644 index 0000000000..ee474570b2 --- /dev/null +++ b/server/postgres/migrations/000029_extend_delphi_graph_stages.sql @@ -0,0 +1,96 @@ +-- #1427: actual bounded Delphi numerical graph on #1432 core M27. +-- Superseding dead branches, scoped breakers and provider remediation remain deferred. +BEGIN; +SET LOCAL lock_timeout='5s'; +SET LOCAL ROLE polis_queue_owner; +SET LOCAL search_path=pg_catalog,pg_temp; +ALTER TABLE public.polis_queue_jobs DROP CONSTRAINT polis_queue_jobs_stage_check; +ALTER TABLE public.polis_queue_jobs ADD CHECK(stage IN ('noop','delphi_full_pipeline','delphi_narrative','math_rebuild','graph_embed','graph_cluster','graph_topics','graph_narrative')); +ALTER TABLE public.delphi_jobs DROP CONSTRAINT delphi_jobs_kind_check; +ALTER TABLE public.delphi_jobs ADD CHECK(kind IN ('full_pipeline','embed','snapshot','umap','cluster','keywords','topics','topic_name','narrative','collective_statement','visualize','math_rebuild','legacy_import','legacy_queue_record')); +CREATE OR REPLACE FUNCTION public.pd_graph_admit(p_env text,p_zid integer,p_scope text,p_key text,p_spec jsonb,p_supersedes uuid DEFAULT NULL) +RETURNS jsonb LANGUAGE plpgsql SECURITY DEFINER SET search_path=pg_catalog,pg_temp AS $$ +#variable_conflict use_column +DECLARE g public.delphi_graphs; gid uuid=gen_random_uuid(); root uuid; + n jsonb; e jsonb; jid uuid; rid uuid; prod uuid; aid uuid; decl jsonb; reply jsonb; + v_stage text; cls text; +BEGIN + IF p_supersedes IS NOT NULL THEN RAISE EXCEPTION 'superseding dead branches is not supported'; END IF; + IF p_env IS NULL OR p_env !~ '^[a-z0-9_-]{1,64}$' OR p_scope IS NULL OR length(p_scope) NOT BETWEEN 1 AND 128 + OR p_key IS NULL OR length(p_key) NOT BETWEEN 1 AND 128 OR p_spec->>'schema' IS DISTINCT FROM 'polis-job-graph/1' + OR jsonb_typeof(p_spec->'nodes') IS DISTINCT FROM 'array' OR jsonb_array_length(p_spec->'nodes') NOT BETWEEN 1 AND 32 + OR p_spec-'schema'-'nodes'<>'{}'::jsonb OR octet_length(p_spec::text)>1048576 THEN RAISE EXCEPTION 'invalid graph'; END IF; + PERFORM 1 FROM public.conversations WHERE zid=p_zid FOR KEY SHARE; + IF NOT FOUND THEN RAISE EXCEPTION 'unknown conversation'; END IF; + PERFORM pg_advisory_xact_lock(hashtextextended(jsonb_build_array('pd:scope',p_env,p_scope)::text,0)); + SELECT * INTO g FROM public.delphi_graphs WHERE env=p_env AND scope_key=p_scope AND request_key=p_key; + IF FOUND THEN + IF g.zid<>p_zid OR g.request IS DISTINCT FROM p_spec OR g.supersedes IS DISTINCT FROM p_supersedes THEN RAISE EXCEPTION 'graph request conflict'; END IF; + RETURN jsonb_build_object('outcome','existing','graph_id',g.graph_id,'root_job_id',g.root_job_id); + END IF; + IF EXISTS(SELECT 1 FROM public.delphi_job_guards WHERE env=p_env AND scope_key=p_scope) THEN RAISE EXCEPTION 'scope busy'; END IF; + root=gen_random_uuid(); + INSERT INTO public.delphi_graphs(env,graph_id,zid,scope_key,request_key,request,root_job_id,supersedes) + VALUES(p_env,gid,p_zid,p_scope,p_key,p_spec,root,p_supersedes); + FOR n IN SELECT value FROM jsonb_array_elements(p_spec->'nodes') LOOP + v_stage=n->>'stage'; cls=n->>'class'; decl=n->'declared'; + IF n-'key'-'stage'-'class'-'declared'-'inputs'-'max_attempts'<>'{}'::jsonb + OR n->>'key' IS NULL OR v_stage IS NULL OR v_stage NOT IN ('graph_embed','graph_cluster','graph_topics','graph_narrative') + OR cls IS NULL OR NOT ((v_stage='graph_cluster' AND cls IN ('delphi','large')) OR (v_stage<>'graph_cluster' AND cls='delphi')) + OR jsonb_typeof(decl) IS DISTINCT FROM 'object' + OR decl-'snapshot'-'code'-'model'-'runtime'-'seed'-'config'-'mode'-'memory_bytes'-'work_units'<>'{}'::jsonb + OR decl->>'mode' IS DISTINCT FROM 'full' + OR decl->>'code' IS NULL OR decl->>'code' !~ '^[0-9a-f]{40,64}$' + OR NOT COALESCE((CASE v_stage WHEN 'graph_embed' THEN decl->>'model' IN ('local-token-count/1','sentence-transformers/all-MiniLM-L6-v2') WHEN 'graph_cluster' THEN decl->>'model' IN ('local-nearest-centroid/1','delphi-umap-evoc/1') WHEN 'graph_topics' THEN decl->>'model' = 'delphi-tfidf-keywords/1' WHEN 'graph_narrative' THEN decl->>'model' IN ('local-cluster-summary/1','local-narrative-fixture/1','legacy-dynamo-export/1') ELSE false END),false) OR COALESCE(decl->>'runtime','')='' + OR jsonb_typeof(decl->'seed') IS DISTINCT FROM 'number' OR jsonb_typeof(decl->'config') IS DISTINCT FROM 'object' + OR jsonb_typeof(decl->'snapshot') IS DISTINCT FROM 'object' + OR jsonb_typeof(decl->'snapshot'->'data'->'texts') IS DISTINCT FROM 'array' + OR jsonb_array_length(decl->'snapshot'->'data'->'texts') NOT BETWEEN 1 AND 2000 + OR (decl->'snapshot'->'data')-'texts'<>'{}'::jsonb + OR public.pd_graph_hash(decl->'snapshot'->'data') IS DISTINCT FROM decl->'snapshot'->>'sha256' + OR (decl->'snapshot')-'data'-'sha256'<>'{}'::jsonb + OR COALESCE((decl->>'memory_bytes')::bigint,0) NOT BETWEEN 1 AND (CASE cls WHEN 'delphi' THEN 4294967296 ELSE 2147483648 END) + OR COALESCE((decl->>'work_units')::integer,0) NOT BETWEEN 1 AND 10000 + OR jsonb_typeof(n->'inputs') IS DISTINCT FROM 'array' OR jsonb_array_length(n->'inputs')>4 + OR COALESCE((n->>'max_attempts')::integer,0) NOT BETWEEN 1 AND 10 + THEN RAISE EXCEPTION 'invalid stage contract (incremental not supported)'; END IF; + IF (v_stage='graph_embed' OR decl->>'model'='legacy-dynamo-export/1') AND jsonb_array_length(n->'inputs')<>0 OR (v_stage<>'graph_embed' AND decl->>'model'<>'legacy-dynamo-export/1') AND jsonb_array_length(n->'inputs')<>1 + THEN RAISE EXCEPTION 'stage input arity'; END IF; + jid=CASE WHEN NOT EXISTS(SELECT 1 FROM public.delphi_graph_nodes WHERE env=p_env AND graph_id=gid) THEN root ELSE gen_random_uuid() END; + rid=gen_random_uuid(); + reply=public.pq_enqueue(p_env,p_zid,'graph:'||p_scope||':'||(n->>'key'),'graph-admission',p_key, + public.pd_graph_hash(n),rid,jid,'graph://'||gid::text||'/'||(n->>'key'),public.pd_graph_hash(decl),public.pd_graph_hash(decl->'config'), + decl->>'code',1::smallint,(n->>'max_attempts')::integer); + IF reply->>'outcome'<>'enqueued' THEN RAISE EXCEPTION 'graph product conflict'; END IF; + UPDATE public.polis_queue_runs SET contract_version='polis-queue/5' WHERE env=p_env AND run_id=rid; + INSERT INTO public.delphi_jobs(job_id,env,zid,kind,parent_job_id,run_id,origin,replayable,reuse_eligible,status,config_effective,code_version,model_versions) + VALUES(jid,p_env,p_zid,substring(v_stage from 7),CASE WHEN jid<>root THEN root END,rid,'queued',true,true,'queued',decl->'config',decl->>'code',jsonb_build_object('exact',decl->>'model')); + INSERT INTO public.delphi_graph_nodes VALUES(p_env,p_zid,gid,jid,rid,n->>'key',decl,NULL,NULL); + UPDATE public.polis_queue_jobs SET stage=n->>'stage',worker_class=cls WHERE env=p_env AND job_id=jid; + END LOOP; + FOR n IN SELECT value FROM jsonb_array_elements(p_spec->'nodes') LOOP + SELECT job_id INTO STRICT jid FROM public.delphi_graph_nodes WHERE env=p_env AND graph_id=gid AND node_key=n->>'key'; + FOR e IN SELECT value FROM jsonb_array_elements(n->'inputs') LOOP + aid=NULL; + IF e-'node'-'artifact_id'-'sha256'-'contract_sha256'-'role'<>'{}'::jsonb + OR e->>'role' IS DISTINCT FROM (CASE n->>'stage' WHEN 'graph_cluster' THEN 'embeddings' WHEN 'graph_topics' THEN 'clusters' WHEN 'graph_narrative' THEN CASE WHEN n->'declared'->>'model'='local-cluster-summary/1' THEN 'clusters' ELSE 'topics' END END) + OR (e ? 'node')=(e ? 'artifact_id') THEN RAISE EXCEPTION 'invalid input role or reference'; END IF; + IF e ? 'node' THEN + SELECT job_id INTO STRICT prod FROM public.delphi_graph_nodes WHERE env=p_env AND graph_id=gid AND node_key=e->>'node'; + ELSE + aid=(e->>'artifact_id')::uuid; + SELECT a.job_id INTO STRICT prod FROM public.delphi_artifacts a JOIN public.delphi_graph_nodes pn USING(env,job_id) + WHERE a.env=p_env AND a.artifact_id=aid AND pn.zid=p_zid AND a.content_sha=e->>'sha256' AND public.pd_graph_hash(pn.declared)=e->>'contract_sha256'; + END IF; + IF NOT EXISTS(SELECT 1 FROM public.polis_queue_jobs qp WHERE qp.env=p_env AND qp.job_id=prod AND qp.stage=CASE n->>'stage' WHEN 'graph_cluster' THEN 'graph_embed' WHEN 'graph_topics' THEN 'graph_cluster' WHEN 'graph_narrative' THEN CASE WHEN n->'declared'->>'model'='local-cluster-summary/1' THEN 'graph_cluster' ELSE 'graph_topics' END END) + THEN RAISE EXCEPTION 'input stage mismatch'; END IF; + IF (SELECT gn.declared->'snapshot'->>'sha256' FROM public.delphi_graph_nodes gn WHERE gn.env=p_env AND gn.job_id=prod) IS DISTINCT FROM n->'declared'->'snapshot'->>'sha256' THEN RAISE EXCEPTION 'upstream snapshot mismatch'; END IF; + INSERT INTO public.delphi_graph_edges VALUES(p_env,jid,prod,e->>'role',e->>'sha256',aid); + END LOOP; + END LOOP; + UPDATE public.delphi_graphs SET sealed=true WHERE env=p_env AND graph_id=gid; + INSERT INTO public.delphi_job_guards VALUES(p_env,p_scope,p_zid,root,public.pd_graph_hash(p_spec)); + RETURN jsonb_build_object('outcome','enqueued','graph_id',gid,'root_job_id',root); +END $$; + +COMMIT; diff --git a/server/postgres/migrations/adoption/000000_initial.sql b/server/postgres/migrations/adoption/000000_initial.sql new file mode 100644 index 0000000000..3de3898aa1 --- /dev/null +++ b/server/postgres/migrations/adoption/000000_initial.sql @@ -0,0 +1,357 @@ +-- Named legacy variants are documented in docs/migration-legacy-contract.md. +-- Contributor agreement tables are outside this release adoption boundary. +-- Catalog postconditions only; this file never replays migration DDL. +SELECT NOT EXISTS (SELECT 1 FROM (VALUES + ('users','uid','integer'), + ('users','hname','character varying(746)'), + ('users','created','bigint'), + ('users','username','character varying(128)'), + ('users','email','character varying(256)'), + ('users','is_owner','boolean'), + ('users','zinvite','character varying(300)'), + ('users','oinvite','character varying(300)'), + ('users','tut','smallint'), + ('users','site_id','character varying(256)'), + ('users','site_owner','boolean'), + ('site_domain_whitelist','site_id','character varying(256)'), + ('site_domain_whitelist','domain_whitelist','character varying(999)'), + ('site_domain_whitelist','domain_whitelist_override_key','character varying(999)'), + ('site_domain_whitelist','modified','bigint'), + ('site_domain_whitelist','created','bigint'), + ('metrics','uid','integer'), + ('metrics','type','integer'), + ('metrics','dur','integer'), + ('metrics','hashedpc','integer'), + ('metrics','created','bigint'), + ('auth_tokens','token','character varying(32)'), + ('auth_tokens','uid','integer'), + ('auth_tokens','created','bigint'), + ('jianiuevyew','uid','integer'), + ('jianiuevyew','pwhash','character varying(128)'), + ('apikeysndvweifu','uid','integer'), + ('apikeysndvweifu','apikey','character varying(32)'), + ('apikeysndvweifu','created','bigint'), + ('courses','course_id','integer'), + ('courses','topic','character varying(1000)'), + ('courses','description','character varying(1000)'), + ('courses','owner','integer'), + ('courses','course_invite','character varying(32)'), + ('courses','created','bigint'), + ('conversations','zid','integer'), + ('conversations','topic','character varying(1000)'), + ('conversations','description','character varying(50000)'), + ('conversations','link_url','character varying(9999)'), + ('conversations','parent_url','character varying(9999)'), + ('conversations','upvotes','integer'), + ('conversations','participant_count','integer'), + ('conversations','is_anon','boolean'), + ('conversations','is_active','boolean'), + ('conversations','is_draft','boolean'), + ('conversations','is_public','boolean'), + ('conversations','is_data_open','boolean'), + ('conversations','profanity_filter','boolean'), + ('conversations','spam_filter','boolean'), + ('conversations','strict_moderation','boolean'), + ('conversations','prioritize_seed','boolean'), + ('conversations','vis_type','integer'), + ('conversations','write_type','integer'), + ('conversations','help_type','integer'), + ('conversations','write_hint_type','integer'), + ('conversations','style_btn','character varying(500)'), + ('conversations','socialbtn_type','integer'), + ('conversations','subscribe_type','integer'), + ('conversations','bgcolor','character varying(20)'), + ('conversations','help_bgcolor','character varying(20)'), + ('conversations','help_color','character varying(20)'), + ('conversations','email_domain','character varying(200)'), + ('conversations','use_xid_whitelist','boolean'), + ('conversations','owner','integer'), + ('conversations','org_id','integer'), + ('conversations','context','character varying(1000)'), + ('conversations','course_id','integer'), + ('conversations','owner_sees_participation_stats','boolean'), + ('conversations','auth_needed_to_vote','boolean'), + ('conversations','auth_needed_to_write','boolean'), + ('conversations','auth_opt_fb','boolean'), + ('conversations','auth_opt_tw','boolean'), + ('conversations','auth_opt_allow_3rdparty','boolean'), + ('conversations','modified','bigint'), + ('conversations','created','bigint'), + ('participant_metadata_questions','pmqid','integer'), + ('participant_metadata_questions','zid','integer'), + ('participant_metadata_questions','key','character varying(999)'), + ('participant_metadata_questions','alive','boolean'), + ('participant_metadata_questions','created','bigint'), + ('participant_metadata_answers','pmaid','integer'), + ('participant_metadata_answers','pmqid','integer'), + ('participant_metadata_answers','zid','integer'), + ('participant_metadata_answers','value','character varying(999)'), + ('participant_metadata_answers','alive','boolean'), + ('participant_metadata_answers','created','bigint'), + ('contexts','context_id','integer'), + ('contexts','name','character varying(300)'), + ('contexts','creator','integer'), + ('contexts','is_public','boolean'), + ('contexts','created','bigint'), + ('inviters','inviter_uid','integer'), + ('inviters','invited_email','character varying(999)'), + ('inviters','created','bigint'), + ('upvotes','uid','integer'), + ('upvotes','zid','integer'), + ('oinvites','oinvite','character varying(300)'), + ('oinvites','note','character varying(999)'), + ('oinvites','created','bigint'), + ('einvites','einvite','character varying(100)'), + ('einvites','email','character varying(999)'), + ('einvites','created','bigint'), + ('email_validations','email','character varying(999)'), + ('email_validations','created','bigint'), + ('zinvites','zid','integer'), + ('zinvites','zinvite','character varying(300)'), + ('zinvites','created','bigint'), + ('beta','name','character varying(999)'), + ('beta','email','character varying(200)'), + ('beta','organization','character varying(200)'), + ('beta','created','bigint'), + ('participants','pid','integer'), + ('participants','uid','integer'), + ('participants','zid','integer'), + ('participants','vote_count','integer'), + ('participants','last_interaction','bigint'), + ('participants','subscribed','integer'), + ('participants','last_notified','bigint'), + ('participants','nsli','smallint'), + ('participants','mod','integer'), + ('participants','created','bigint'), + ('participants_extended','uid','integer'), + ('participants_extended','zid','integer'), + ('participants_extended','referrer','character varying(9999)'), + ('participants_extended','parent_url','character varying(9999)'), + ('participants_extended','created','bigint'), + ('participants_extended','modified','bigint'), + ('participants_extended','subscribe_email','character varying(256)'), + ('participants_extended','show_translation_activated','boolean'), + ('participant_locations','zid','integer'), + ('participant_locations','uid','integer'), + ('participant_locations','pid','integer'), + ('participant_locations','lat','double precision'), + ('participant_locations','lng','double precision'), + ('participant_locations','created','bigint'), + ('participant_locations','source','integer'), + ('xids','uid','integer'), + ('xids','owner','integer'), + ('xids','xid','text'), + ('xids','x_profile_image_url','character varying(3000)'), + ('xids','x_name','character varying(746)'), + ('xids','x_email','character varying(256)'), + ('xids','created','bigint'), + ('xids','modified','bigint'), + ('xid_whitelist','owner','integer'), + ('xid_whitelist','xid','text'), + ('xid_whitelist','created','bigint'), + ('notification_tasks','zid','integer'), + ('notification_tasks','modified','bigint'), + ('participant_metadata_choices','zid','integer'), + ('participant_metadata_choices','pid','integer'), + ('participant_metadata_choices','pmqid','integer'), + ('participant_metadata_choices','pmaid','integer'), + ('participant_metadata_choices','alive','boolean'), + ('participant_metadata_choices','created','bigint'), + ('twitter_users','uid','integer'), + ('twitter_users','twitter_user_id','bigint'), + ('twitter_users','screen_name','character varying(999)'), + ('twitter_users','name','character varying(9999)'), + ('twitter_users','followers_count','integer'), + ('twitter_users','friends_count','integer'), + ('twitter_users','verified','boolean'), + ('twitter_users','profile_image_url_https','character varying(9999)'), + ('twitter_users','location','character varying(9999)'), + ('twitter_users','response','json'), + ('twitter_users','modified','bigint'), + ('twitter_users','created','bigint'), + ('facebook_users','uid','integer'), + ('facebook_users','fb_user_id','text'), + ('facebook_users','fb_name','character varying(9999)'), + ('facebook_users','fb_link','character varying(9999)'), + ('facebook_users','fb_public_profile','text'), + ('facebook_users','fb_login_status','text'), + ('facebook_users','fb_auth_response','text'), + ('facebook_users','fb_access_token','text'), + ('facebook_users','fb_granted_scopes','text'), + ('facebook_users','fb_location_id','character varying(100)'), + ('facebook_users','location','character varying(9999)'), + ('facebook_users','response','text'), + ('facebook_users','fb_friends_response','text'), + ('facebook_users','created','bigint'), + ('facebook_users','modified','bigint'), + ('social_settings','uid','integer'), + ('social_settings','polis_pic','character varying(3000)'), + ('facebook_friends','uid','integer'), + ('facebook_friends','friend','integer'), + ('suzinvites','owner','integer'), + ('suzinvites','zid','integer'), + ('suzinvites','created','bigint'), + ('suzinvites','suzinvite','character varying(32)'), + ('comments','tid','integer'), + ('comments','zid','integer'), + ('comments','pid','integer'), + ('comments','uid','integer'), + ('comments','created','bigint'), + ('comments','modified','bigint'), + ('comments','txt','character varying(1000)'), + ('comments','velocity','real'), + ('comments','mod','integer'), + ('comments','lang','character varying(10)'), + ('comments','lang_confidence','real'), + ('comments','active','boolean'), + ('comments','is_meta','boolean'), + ('comments','tweet_id','bigint'), + ('comments','quote_src_url','character varying(1000)'), + ('comments','anon','boolean'), + ('comments','is_seed','boolean'), + ('comment_translations','zid','integer'), + ('comment_translations','tid','integer'), + ('comment_translations','src','integer'), + ('comment_translations','txt','character varying(9999)'), + ('comment_translations','lang','character varying(10)'), + ('comment_translations','created','bigint'), + ('comment_translations','modified','bigint'), + ('conversation_translations','zid','integer'), + ('conversation_translations','src','integer'), + ('conversation_translations','topic','character varying(9999)'), + ('conversation_translations','description','character varying(9999)'), + ('conversation_translations','lang','character varying(10)'), + ('conversation_translations','created','bigint'), + ('conversation_translations','modified','bigint'), + ('reports','rid','bigint'), + ('reports','report_id','character varying(300)'), + ('reports','zid','integer'), + ('reports','created','bigint'), + ('reports','modified','bigint'), + ('reports','report_name','character varying(999)'), + ('reports','label_x_neg','character varying(999)'), + ('reports','label_x_pos','character varying(999)'), + ('reports','label_y_neg','character varying(999)'), + ('reports','label_y_pos','character varying(999)'), + ('reports','label_group_0','character varying(999)'), + ('reports','label_group_1','character varying(999)'), + ('reports','label_group_2','character varying(999)'), + ('reports','label_group_3','character varying(999)'), + ('reports','label_group_4','character varying(999)'), + ('reports','label_group_5','character varying(999)'), + ('reports','label_group_6','character varying(999)'), + ('reports','label_group_7','character varying(999)'), + ('reports','label_group_8','character varying(999)'), + ('reports','label_group_9','character varying(999)'), + ('report_comment_selections','zid','integer'), + ('report_comment_selections','rid','bigint'), + ('report_comment_selections','tid','integer'), + ('report_comment_selections','selection','smallint'), + ('report_comment_selections','modified','bigint'), + ('worker_tasks','created','bigint'), + ('worker_tasks','math_env','character varying(999)'), + ('worker_tasks','attempts','smallint'), + ('worker_tasks','task_data','jsonb'), + ('worker_tasks','task_type','text'), + ('worker_tasks','task_bucket','bigint'), + ('worker_tasks','finished_time','bigint'), + ('math_ticks','zid','integer'), + ('math_ticks','math_tick','bigint'), + ('math_ticks','math_env','character varying(999)'), + ('math_ticks','modified','bigint'), + ('math_main','zid','integer'), + ('math_main','math_env','character varying(999)'), + ('math_main','data','jsonb'), + ('math_main','last_vote_timestamp','bigint'), + ('math_main','caching_tick','bigint'), + ('math_main','math_tick','bigint'), + ('math_main','modified','bigint'), + ('math_profile','zid','integer'), + ('math_profile','math_env','character varying(999)'), + ('math_profile','data','jsonb'), + ('math_profile','modified','bigint'), + ('math_ptptstats','zid','integer'), + ('math_ptptstats','math_env','character varying(999)'), + ('math_ptptstats','math_tick','bigint'), + ('math_ptptstats','data','jsonb'), + ('math_ptptstats','modified','bigint'), + ('math_cache','zid','integer'), + ('math_cache','math_env','character varying(999)'), + ('math_cache','data','jsonb'), + ('math_cache','modified','bigint'), + ('math_bidtopid','zid','integer'), + ('math_bidtopid','math_env','character varying(999)'), + ('math_bidtopid','math_tick','bigint'), + ('math_bidtopid','data','jsonb'), + ('math_bidtopid','modified','bigint'), + ('math_exportstatus','zid','integer'), + ('math_exportstatus','math_env','character varying(999)'), + ('math_exportstatus','filename','character varying(9999)'), + ('math_exportstatus','data','jsonb'), + ('math_exportstatus','modified','bigint'), + ('math_report_correlationmatrix','rid','bigint'), + ('math_report_correlationmatrix','math_env','character varying(999)'), + ('math_report_correlationmatrix','data','jsonb'), + ('math_report_correlationmatrix','math_tick','bigint'), + ('math_report_correlationmatrix','modified','bigint'), + ('votes','zid','integer'), + ('votes','pid','integer'), + ('votes','tid','integer'), + ('votes','vote','smallint'), + ('votes','weight_x_32767','smallint'), + ('votes','created','bigint'), + ('votes_latest_unique','zid','integer'), + ('votes_latest_unique','pid','integer'), + ('votes_latest_unique','tid','integer'), + ('votes_latest_unique','vote','smallint'), + ('votes_latest_unique','weight_x_32767','smallint'), + ('votes_latest_unique','modified','bigint'), + ('crowd_mod','zid','integer'), + ('crowd_mod','pid','integer'), + ('crowd_mod','tid','integer'), + ('crowd_mod','created','bigint'), + ('crowd_mod','as_important','boolean'), + ('crowd_mod','as_factual','boolean'), + ('crowd_mod','as_feeling','boolean'), + ('crowd_mod','as_notmyfeeling','boolean'), + ('crowd_mod','as_notgoodidea','boolean'), + ('crowd_mod','as_notfact','boolean'), + ('crowd_mod','as_unsure','boolean'), + ('crowd_mod','as_spam','boolean'), + ('crowd_mod','as_abusive','boolean'), + ('crowd_mod','as_offtopic','boolean'), + ('event_ptpt_no_more_comments','zid','integer'), + ('event_ptpt_no_more_comments','pid','integer'), + ('event_ptpt_no_more_comments','votes_placed','smallint'), + ('event_ptpt_no_more_comments','created','bigint'), + ('stars','zid','integer'), + ('stars','pid','integer'), + ('stars','tid','integer'), + ('stars','starred','integer'), + ('stars','created','bigint'), + ('trashes','zid','integer'), + ('trashes','pid','integer'), + ('trashes','tid','integer'), + ('trashes','trashed','integer'), + ('trashes','created','bigint'), + ('permanentcookiezidjoins','zid','integer'), + ('permanentcookiezidjoins','cookie','character varying(32)'), + ('permanentcookiezidjoins','created','bigint'), + ('page_ids','site_id','character varying(100)'), + ('page_ids','page_id','character varying(100)'), + ('page_ids','zid','integer'), + ('demographic_data','uid','integer'), + ('demographic_data','fb_gender','integer'), + ('demographic_data','ms_birth_year_estimate_fb','integer'), + ('demographic_data','ms_gender_estimate_fb','integer'), + ('demographic_data','fb_timestamp','bigint'), + ('demographic_data','ms_fb_timestamp','bigint'), + ('demographic_data','ms_response','character varying(9999)'), + ('demographic_data','gender_guess','integer'), + ('demographic_data','birth_year_guess','integer')) AS expected(t,c,typ) WHERE NOT pg_temp.col(t,c,typ)) + AND NOT EXISTS (SELECT 1 FROM unnest(ARRAY['users','site_domain_whitelist','metrics','auth_tokens','jianiuevyew','apikeysndvweifu','courses','conversations','participant_metadata_questions','participant_metadata_answers','contexts','inviters','upvotes','oinvites','einvites','email_validations','zinvites','beta','participants','participants_extended','participant_locations','xids','xid_whitelist','notification_tasks','participant_metadata_choices','twitter_users','facebook_users','social_settings','facebook_friends','suzinvites','comments','comment_translations','conversation_translations','reports','report_comment_selections','worker_tasks','math_ticks','math_main','math_profile','math_ptptstats','math_cache','math_bidtopid','math_exportstatus','math_report_correlationmatrix','votes','votes_latest_unique','crowd_mod','event_ptpt_no_more_comments','stars','trashes','permanentcookiezidjoins','page_ids','demographic_data']) t WHERE to_regclass('public.'||t) IS NULL) + AND (pg_temp.col('pwreset_tokens','token','character varying(100)') OR pg_temp.col('password_reset_tokens','pwresettoken','character varying(100)')) + AND (pg_temp.absent_column('conversations','branding_type') OR pg_temp.col('conversations','branding_type','integer')) + AND (pg_temp.absent_column('math_ticks','caching_tick') OR pg_temp.col('math_ticks','caching_tick','bigint')) + AND to_regprocedure('public.now_as_millis()') IS NOT NULL + AND pg_temp.con('votes_latest_unique','UNIQUE (zid, pid, tid)'); diff --git a/server/postgres/migrations/adoption/000001_update_pwreset_table.sql b/server/postgres/migrations/adoption/000001_update_pwreset_table.sql new file mode 100644 index 0000000000..7752542f42 --- /dev/null +++ b/server/postgres/migrations/adoption/000001_update_pwreset_table.sql @@ -0,0 +1,4 @@ +-- Catalog postconditions only; this file never replays migration DDL. +SELECT to_regclass('public.password_reset_tokens') IS NULL + AND pg_temp.col('pwreset_tokens','token','character varying(100)') + AND pg_temp.absent_column('pwreset_tokens','pwresettoken'); diff --git a/server/postgres/migrations/adoption/000002_add_xid_constraint.sql b/server/postgres/migrations/adoption/000002_add_xid_constraint.sql new file mode 100644 index 0000000000..41e11d827e --- /dev/null +++ b/server/postgres/migrations/adoption/000002_add_xid_constraint.sql @@ -0,0 +1,14 @@ +-- Both the public bootstrap variant (owner,xid only) and the authoritative +-- legacy variant (also owner,uid) are supported. Never drop either invariant. +SELECT pg_temp.con_exact('xids','UNIQUE (owner, xid)') + -- If the optional legacy unique index exists, it must enforce the complete key. + AND NOT EXISTS ( + SELECT 1 FROM pg_index i + JOIN pg_attribute owner ON owner.attrelid=i.indrelid AND owner.attname='owner' + JOIN pg_attribute uid ON uid.attrelid=i.indrelid AND uid.attname='uid' + WHERE i.indrelid=to_regclass('public.xids') AND i.indisunique + AND i.indnkeyatts=2 + AND ARRAY[i.indkey[0],i.indkey[1]] @> ARRAY[owner.attnum,uid.attnum] + AND (NOT i.indisvalid OR NOT i.indisready OR NOT i.indislive + OR NOT i.indimmediate OR i.indpred IS NOT NULL) + ); diff --git a/server/postgres/migrations/adoption/000003_add_origin_permanent_cookie_columns.sql b/server/postgres/migrations/adoption/000003_add_origin_permanent_cookie_columns.sql new file mode 100644 index 0000000000..4f0a5cee75 --- /dev/null +++ b/server/postgres/migrations/adoption/000003_add_origin_permanent_cookie_columns.sql @@ -0,0 +1,3 @@ +-- Catalog postconditions only; this file never replays migration DDL. +SELECT pg_temp.col_exact('participants_extended','permanent_cookie','character varying(32)',false) + AND pg_temp.col_exact('participants_extended','origin','character varying(9999)',false); diff --git a/server/postgres/migrations/adoption/000004_drop_waitinglist_table.sql b/server/postgres/migrations/adoption/000004_drop_waitinglist_table.sql new file mode 100644 index 0000000000..61fac26e19 --- /dev/null +++ b/server/postgres/migrations/adoption/000004_drop_waitinglist_table.sql @@ -0,0 +1,2 @@ +-- Catalog postconditions only; this file never replays migration DDL. +SELECT to_regclass('public.waitinglist') IS NULL; diff --git a/server/postgres/migrations/adoption/000005_drop_slack_stripe_canvas.sql b/server/postgres/migrations/adoption/000005_drop_slack_stripe_canvas.sql new file mode 100644 index 0000000000..f0db14fc7d --- /dev/null +++ b/server/postgres/migrations/adoption/000005_drop_slack_stripe_canvas.sql @@ -0,0 +1,16 @@ +-- Catalog postconditions only; this file never replays migration DDL. +SELECT to_regclass('public.slack_oauth_access_tokens') IS NULL + AND to_regclass('public.slack_users') IS NULL + AND to_regclass('public.slack_user_invites') IS NULL + AND to_regclass('public.slack_bot_events') IS NULL + AND to_regclass('public.stripe_accounts') IS NULL + AND to_regclass('public.stripe_subscriptions') IS NULL + AND to_regclass('public.coupons_for_free_upgrades') IS NULL + AND to_regclass('public.lti_users') IS NULL + AND to_regclass('public.lti_context_memberships') IS NULL + AND to_regclass('public.canvas_assignment_callback_info') IS NULL + AND to_regclass('public.canvas_assignment_conversation_info') IS NULL + AND to_regclass('public.lti_oauthv1_credentials') IS NULL + AND pg_temp.absent_column('conversations','is_slack') + AND pg_temp.absent_column('conversations','lti_users_only') + AND pg_temp.absent_column('users','plan'); diff --git a/server/postgres/migrations/adoption/000006_update_votes_rule.sql b/server/postgres/migrations/adoption/000006_update_votes_rule.sql new file mode 100644 index 0000000000..b79f8b8243 --- /dev/null +++ b/server/postgres/migrations/adoption/000006_update_votes_rule.sql @@ -0,0 +1,2 @@ +-- Catalog postconditions only; this file never replays migration DDL. +SELECT EXISTS (SELECT 1 FROM pg_rewrite WHERE ev_class=to_regclass('public.votes') AND rulename='on_vote_insert_update_unique_table' AND ev_enabled='O' AND regexp_replace(pg_get_ruledef(oid), '\s+', ' ', 'g') = 'CREATE RULE on_vote_insert_update_unique_table AS ON INSERT TO public.votes DO INSERT INTO votes_latest_unique (zid, pid, tid, vote, weight_x_32767, modified) VALUES (new.zid, new.pid, new.tid, new.vote, new.weight_x_32767, new.created) ON CONFLICT(zid, pid, tid) DO UPDATE SET vote = excluded.vote, modified = excluded.modified;'); diff --git a/server/postgres/migrations/adoption/000007_drop_geolocation_fields.sql b/server/postgres/migrations/adoption/000007_drop_geolocation_fields.sql new file mode 100644 index 0000000000..c8f44df378 --- /dev/null +++ b/server/postgres/migrations/adoption/000007_drop_geolocation_fields.sql @@ -0,0 +1,9 @@ +-- Catalog postconditions only; this file never replays migration DDL. +SELECT to_regclass('public.geolocation_cache') IS NULL + AND pg_temp.absent_column('participants_extended','country_code_iso') + AND pg_temp.absent_column('participants_extended','encrypted_maxmind_response_city') + AND pg_temp.absent_column('participants_extended','ip_address') + AND pg_temp.absent_column('participants_extended','latitude') + AND pg_temp.absent_column('participants_extended','location') + AND pg_temp.absent_column('participants_extended','longitude') + AND pg_temp.absent_column('participants_extended','x_forwarded_for'); diff --git a/server/postgres/migrations/adoption/000008_add_comment_priority.sql b/server/postgres/migrations/adoption/000008_add_comment_priority.sql new file mode 100644 index 0000000000..11072158e0 --- /dev/null +++ b/server/postgres/migrations/adoption/000008_add_comment_priority.sql @@ -0,0 +1,3 @@ +-- Catalog postconditions only; this file never replays migration DDL. +SELECT pg_temp.col_exact('conversations','importance_enabled','boolean',true,'false') + AND pg_temp.col_exact('votes','high_priority','boolean',true,'false'); diff --git a/server/postgres/migrations/adoption/000009_add_uuid_to_zinvites.sql b/server/postgres/migrations/adoption/000009_add_uuid_to_zinvites.sql new file mode 100644 index 0000000000..adf006212d --- /dev/null +++ b/server/postgres/migrations/adoption/000009_add_uuid_to_zinvites.sql @@ -0,0 +1,2 @@ +-- Catalog postconditions only; this file never replays migration DDL. +SELECT pg_temp.col_exact('zinvites','uuid','uuid',false); diff --git a/server/postgres/migrations/adoption/000010_create_oidc_user_mappings.sql b/server/postgres/migrations/adoption/000010_create_oidc_user_mappings.sql new file mode 100644 index 0000000000..6ca36bcdb7 --- /dev/null +++ b/server/postgres/migrations/adoption/000010_create_oidc_user_mappings.sql @@ -0,0 +1,8 @@ +-- Catalog postconditions only; this file never replays migration DDL. +SELECT pg_temp.col_exact('oidc_user_mappings','oidc_sub','character varying(255)',true) + AND pg_temp.col_exact('oidc_user_mappings','uid','integer',true) + AND pg_temp.col_exact('oidc_user_mappings','created','bigint',false,'now_as_millis()') + AND pg_temp.con_exact('oidc_user_mappings','PRIMARY KEY (oidc_sub)') + AND pg_temp.con_exact('oidc_user_mappings','UNIQUE (uid)') + AND pg_temp.con_exact('oidc_user_mappings','FOREIGN KEY (uid) REFERENCES users(uid) ON DELETE CASCADE') + AND pg_temp.idx('idx_oidc_mappings_uid','CREATE INDEX idx_oidc_mappings_uid ON public.oidc_user_mappings USING btree (uid)'); diff --git a/server/postgres/migrations/adoption/000011_alter_suzinvites_xid_to_text.sql b/server/postgres/migrations/adoption/000011_alter_suzinvites_xid_to_text.sql new file mode 100644 index 0000000000..5235d98d42 --- /dev/null +++ b/server/postgres/migrations/adoption/000011_alter_suzinvites_xid_to_text.sql @@ -0,0 +1,2 @@ +-- Catalog postconditions only; this file never replays migration DDL. +SELECT pg_temp.col('suzinvites','xid','text',true); diff --git a/server/postgres/migrations/adoption/000012_create_topic_agenda_selections.sql b/server/postgres/migrations/adoption/000012_create_topic_agenda_selections.sql new file mode 100644 index 0000000000..62832a1743 --- /dev/null +++ b/server/postgres/migrations/adoption/000012_create_topic_agenda_selections.sql @@ -0,0 +1,15 @@ +-- Catalog postconditions only; this file never replays migration DDL. +SELECT pg_temp.col_exact('topic_agenda_selections','zid','integer',true) + AND pg_temp.col_exact('topic_agenda_selections','pid','integer',true) + AND pg_temp.col_exact('topic_agenda_selections','archetypal_selections','jsonb',true,'''[]''::jsonb') + AND pg_temp.col_exact('topic_agenda_selections','delphi_job_id','text',false) + AND pg_temp.col_exact('topic_agenda_selections','total_selections','integer',true,'0') + AND pg_temp.col_exact('topic_agenda_selections','created_at','timestamp with time zone',false,'CURRENT_TIMESTAMP') + AND pg_temp.col_exact('topic_agenda_selections','updated_at','timestamp with time zone',false,'CURRENT_TIMESTAMP') + AND pg_temp.con_exact('topic_agenda_selections','PRIMARY KEY (zid, pid)') + AND pg_temp.con_exact('topic_agenda_selections','FOREIGN KEY (zid) REFERENCES conversations(zid) ON DELETE CASCADE') + AND pg_temp.con_exact('topic_agenda_selections','FOREIGN KEY (zid, pid) REFERENCES participants(zid, pid) ON DELETE CASCADE') + AND pg_temp.idx('idx_topic_agenda_selections_zid','CREATE INDEX idx_topic_agenda_selections_zid ON public.topic_agenda_selections USING btree (zid)') + AND pg_temp.idx('idx_topic_agenda_selections_pid','CREATE INDEX idx_topic_agenda_selections_pid ON public.topic_agenda_selections USING btree (pid)') + AND pg_temp.idx('idx_topic_agenda_selections_delphi_job_id','CREATE INDEX idx_topic_agenda_selections_delphi_job_id ON public.topic_agenda_selections USING btree (delphi_job_id)') + AND pg_temp.idx('idx_topic_agenda_selections_created_at','CREATE INDEX idx_topic_agenda_selections_created_at ON public.topic_agenda_selections USING btree (created_at)'); diff --git a/server/postgres/migrations/adoption/000013_create_treevite.sql b/server/postgres/migrations/adoption/000013_create_treevite.sql new file mode 100644 index 0000000000..613b2b764f --- /dev/null +++ b/server/postgres/migrations/adoption/000013_create_treevite.sql @@ -0,0 +1,70 @@ +-- Catalog postconditions only; this file never replays migration DDL. +SELECT pg_temp.col_exact('conversations','treevite_enabled','boolean',false,'false') + AND pg_temp.col_exact('treevite_waves','id','bigint',true,'nextval(''treevite_waves_id_seq''::regclass)') + AND pg_temp.col_exact('treevite_waves','zid','integer',true) + AND pg_temp.col_exact('treevite_waves','wave','integer',true) + AND pg_temp.col_exact('treevite_waves','parent_wave','integer',false) + AND pg_temp.col_exact('treevite_waves','size','integer',false) + AND pg_temp.col_exact('treevite_waves','invites_per_user','integer',true) + AND pg_temp.col_exact('treevite_waves','owner_invites','integer',true,'0') + AND pg_temp.col_exact('treevite_waves','created_at','timestamp with time zone',false,'CURRENT_TIMESTAMP') + AND pg_temp.col_exact('treevite_waves','updated_at','timestamp with time zone',false,'CURRENT_TIMESTAMP') + AND pg_temp.con_exact('treevite_waves','PRIMARY KEY (id)') + AND pg_temp.col_exact('treevite_invites','id','bigint',true,'nextval(''treevite_invites_id_seq''::regclass)') + AND pg_temp.col_exact('treevite_invites','zid','integer',true) + AND pg_temp.col_exact('treevite_invites','wave_id','bigint',true) + AND pg_temp.col_exact('treevite_invites','parent_invite_id','bigint',false) + AND pg_temp.col_exact('treevite_invites','status','smallint',true,'0') + AND pg_temp.col_exact('treevite_invites','invite_owner_pid','integer',false) + AND pg_temp.col_exact('treevite_invites','invite_used_by_pid','integer',false) + AND pg_temp.col_exact('treevite_invites','invite_used_at','timestamp with time zone',false) + AND pg_temp.col_exact('treevite_invites','created_at','timestamp with time zone',false,'CURRENT_TIMESTAMP') + AND pg_temp.col_exact('treevite_invites','updated_at','timestamp with time zone',false,'CURRENT_TIMESTAMP') + AND pg_temp.con_exact('treevite_invites','PRIMARY KEY (id)') + AND pg_temp.col_exact('treevite_login_codes','id','bigint',true,'nextval(''treevite_login_codes_id_seq''::regclass)') + AND pg_temp.col_exact('treevite_login_codes','zid','integer',true) + AND pg_temp.col_exact('treevite_login_codes','pid','integer',true) + AND pg_temp.col_exact('treevite_login_codes','login_code_hash','text',true) + AND pg_temp.col_exact('treevite_login_codes','fp_kid','smallint',true,'1') + AND pg_temp.col_exact('treevite_login_codes','revoked','boolean',true,'false') + AND pg_temp.col_exact('treevite_login_codes','expires_at','timestamp with time zone',false) + AND pg_temp.col_exact('treevite_login_codes','last_used_at','timestamp with time zone',false) + AND pg_temp.col_exact('treevite_login_codes','created_at','timestamp with time zone',false,'CURRENT_TIMESTAMP') + AND pg_temp.col_exact('treevite_login_codes','updated_at','timestamp with time zone',false,'CURRENT_TIMESTAMP') + AND pg_temp.con_exact('treevite_login_codes','PRIMARY KEY (id)') + AND pg_temp.con_exact('treevite_waves','UNIQUE (zid, wave)') + AND pg_temp.con_exact('treevite_waves','FOREIGN KEY (zid) REFERENCES conversations(zid) ON DELETE CASCADE') + AND pg_temp.con_exact('treevite_invites','UNIQUE (zid, invite_code)') + AND pg_temp.con_exact('treevite_invites','FOREIGN KEY (zid) REFERENCES conversations(zid) ON DELETE CASCADE') + AND pg_temp.con_exact('treevite_invites','FOREIGN KEY (wave_id) REFERENCES treevite_waves(id) ON DELETE CASCADE') + AND pg_temp.con_exact('treevite_invites','FOREIGN KEY (parent_invite_id) REFERENCES treevite_invites(id) ON DELETE SET NULL') + AND pg_temp.con_exact('treevite_invites','FOREIGN KEY (zid, invite_owner_pid) REFERENCES participants(zid, pid)') + AND pg_temp.con_exact('treevite_invites','FOREIGN KEY (zid, invite_used_by_pid) REFERENCES participants(zid, pid)') + AND pg_temp.con_exact('treevite_login_codes','FOREIGN KEY (zid, pid) REFERENCES participants(zid, pid) ON DELETE CASCADE') + AND pg_temp.con_exact('treevite_login_codes','UNIQUE (zid, pid)') + AND pg_temp.con_exact('treevite_login_codes','UNIQUE (zid, login_code_fingerprint)') + AND pg_temp.con_exact('treevite_login_codes','UNIQUE (zid, login_code_lookup)') + AND pg_temp.idx('idx_treevite_waves_zid','CREATE INDEX idx_treevite_waves_zid ON public.treevite_waves USING btree (zid)') + AND pg_temp.idx('idx_treevite_waves_wave','CREATE INDEX idx_treevite_waves_wave ON public.treevite_waves USING btree (wave)') + AND pg_temp.idx('idx_treevite_waves_parent','CREATE INDEX idx_treevite_waves_parent ON public.treevite_waves USING btree (zid, parent_wave)') + AND pg_temp.idx('idx_treevite_invites_zid','CREATE INDEX idx_treevite_invites_zid ON public.treevite_invites USING btree (zid)') + AND pg_temp.idx('idx_treevite_invites_zid_status','CREATE INDEX idx_treevite_invites_zid_status ON public.treevite_invites USING btree (zid, status)') + AND pg_temp.idx('idx_treevite_invites_wave_id','CREATE INDEX idx_treevite_invites_wave_id ON public.treevite_invites USING btree (wave_id)') + AND pg_temp.idx('idx_treevite_invites_parent','CREATE INDEX idx_treevite_invites_parent ON public.treevite_invites USING btree (parent_invite_id)') + AND pg_temp.idx('idx_treevite_invites_owner_pid','CREATE INDEX idx_treevite_invites_owner_pid ON public.treevite_invites USING btree (invite_owner_pid)') + AND pg_temp.idx('idx_treevite_invites_used_by_pid','CREATE INDEX idx_treevite_invites_used_by_pid ON public.treevite_invites USING btree (invite_used_by_pid)') + AND pg_temp.idx('idx_treevite_invites_code','CREATE INDEX idx_treevite_invites_code ON public.treevite_invites USING btree (invite_code)') + AND pg_temp.idx('idx_treevite_login_codes_zid','CREATE INDEX idx_treevite_login_codes_zid ON public.treevite_login_codes USING btree (zid)') + AND pg_temp.idx('idx_treevite_login_codes_pid','CREATE INDEX idx_treevite_login_codes_pid ON public.treevite_login_codes USING btree (pid)') + AND pg_temp.idx('idx_treevite_login_codes_fp','CREATE INDEX idx_treevite_login_codes_fp ON public.treevite_login_codes USING btree (login_code_fingerprint)') + AND pg_temp.idx('idx_treevite_login_codes_lookup','CREATE INDEX idx_treevite_login_codes_lookup ON public.treevite_login_codes USING btree (zid, login_code_lookup)') + AND pg_temp.col_exact('treevite_invites','invite_code','character varying(64)',true) + AND pg_temp.col_exact('treevite_login_codes','login_code_fingerprint','character varying(128)',true) + AND pg_temp.col_exact('treevite_login_codes','login_code_lookup','character varying(128)',false) + AND pg_temp.con_exact('treevite_waves','CHECK (((invites_per_user > 0) OR (owner_invites > 0)))') + AND pg_temp.con_exact('treevite_waves','CHECK (((parent_wave IS NULL) OR (parent_wave >= 0)))') + AND pg_temp.con_exact('treevite_waves','CHECK (((size IS NULL) OR (size >= 0)))') + AND pg_temp.con_exact('treevite_waves','CHECK ((invites_per_user >= 0))') + AND pg_temp.con_exact('treevite_waves','CHECK ((owner_invites >= 0))') + AND pg_temp.con_exact('treevite_waves','CHECK ((wave >= 1))') + AND pg_temp.con_exact('treevite_invites','CHECK ((status = ANY (ARRAY[0, 1, 2, 3])))'); diff --git a/server/postgres/migrations/adoption/000014_alter_reports_modlevel.sql b/server/postgres/migrations/adoption/000014_alter_reports_modlevel.sql new file mode 100644 index 0000000000..762f142a17 --- /dev/null +++ b/server/postgres/migrations/adoption/000014_alter_reports_modlevel.sql @@ -0,0 +1,2 @@ +-- Catalog postconditions only; this file never replays migration DDL. +SELECT pg_temp.col_exact('reports','mod_level','smallint',true,'''-2''::integer'); diff --git a/server/postgres/migrations/adoption/000015_add_xid_requirements.sql b/server/postgres/migrations/adoption/000015_add_xid_requirements.sql new file mode 100644 index 0000000000..584bf1cca0 --- /dev/null +++ b/server/postgres/migrations/adoption/000015_add_xid_requirements.sql @@ -0,0 +1,15 @@ +-- Catalog postconditions only; this file never replays migration DDL. +SELECT pg_temp.col_exact('conversations','xid_required','boolean',true,'false') + AND pg_temp.col_exact('xid_whitelist','zid','integer',false) + AND pg_temp.col_exact('xids','zid','integer',false) + AND pg_temp.col_exact('xids','pid','integer',false) + AND pg_temp.con_exact('xid_whitelist','FOREIGN KEY (zid) REFERENCES conversations(zid) ON DELETE CASCADE') + AND pg_temp.con_exact('xids','FOREIGN KEY (zid) REFERENCES conversations(zid) ON DELETE CASCADE') + AND pg_temp.con_exact('xids','FOREIGN KEY (zid, pid) REFERENCES participants(zid, pid) ON DELETE SET NULL') + AND pg_temp.idx('idx_xid_whitelist_zid','CREATE INDEX idx_xid_whitelist_zid ON public.xid_whitelist USING btree (zid)') + AND pg_temp.idx('idx_xid_whitelist_xid','CREATE INDEX idx_xid_whitelist_xid ON public.xid_whitelist USING btree (xid)') + AND pg_temp.idx('idx_xids_zid','CREATE INDEX idx_xids_zid ON public.xids USING btree (zid)') + AND pg_temp.idx('idx_xids_xid','CREATE INDEX idx_xids_xid ON public.xids USING btree (xid)') + AND pg_temp.idx('idx_xids_pid','CREATE INDEX idx_xids_pid ON public.xids USING btree (pid)') + AND pg_temp.idx('idx_xids_zid_xid','CREATE INDEX idx_xids_zid_xid ON public.xids USING btree (zid, xid)') + AND pg_temp.idx('idx_xids_uid_zid','CREATE INDEX idx_xids_uid_zid ON public.xids USING btree (uid, zid)'); diff --git a/server/postgres/migrations/adoption/000016_add_orig_id.sql b/server/postgres/migrations/adoption/000016_add_orig_id.sql new file mode 100644 index 0000000000..c864197137 --- /dev/null +++ b/server/postgres/migrations/adoption/000016_add_orig_id.sql @@ -0,0 +1,2 @@ +-- Catalog postconditions only; this file never replays migration DDL. +SELECT pg_temp.col_exact('comments','original_id','uuid',false); diff --git a/server/postgres/migrations/adoption/000017_create_byod_job_table.sql b/server/postgres/migrations/adoption/000017_create_byod_job_table.sql new file mode 100644 index 0000000000..43ed6a91db --- /dev/null +++ b/server/postgres/migrations/adoption/000017_create_byod_job_table.sql @@ -0,0 +1,12 @@ +-- Catalog postconditions only; this file never replays migration DDL. +SELECT (SELECT array_agg(enumlabel::text ORDER BY enumsortorder) FROM pg_enum WHERE enumtypid=to_regtype('public.job_status')) = ARRAY['pending','processing','completed','failed'] + AND pg_temp.col_exact('byod_import_jobs','id','integer',true,'nextval(''byod_import_jobs_id_seq''::regclass)') + AND pg_temp.col_exact('byod_import_jobs','zid','integer',true) + AND pg_temp.col_exact('byod_import_jobs','s3_key','text',true) + AND pg_temp.col_exact('byod_import_jobs','status','job_status',false,'''pending''::job_status') + AND pg_temp.col_exact('byod_import_jobs','stage','text',false,'''init''::text') + AND pg_temp.col_exact('byod_import_jobs','error_message','text',false) + AND pg_temp.col_exact('byod_import_jobs','created_at','timestamp with time zone',false,'now()') + AND pg_temp.col_exact('byod_import_jobs','updated_at','timestamp with time zone',false,'now()') + AND pg_temp.con_exact('byod_import_jobs','PRIMARY KEY (id)') + AND pg_temp.idx('idx_byod_jobs_zid','CREATE INDEX idx_byod_jobs_zid ON public.byod_import_jobs USING btree (zid)'); diff --git a/server/postgres/migrations/adoption/000018_add_topics_enabled.sql b/server/postgres/migrations/adoption/000018_add_topics_enabled.sql new file mode 100644 index 0000000000..5a9299078b --- /dev/null +++ b/server/postgres/migrations/adoption/000018_add_topics_enabled.sql @@ -0,0 +1,2 @@ +-- Catalog postconditions only; this file never replays migration DDL. +SELECT pg_temp.col_exact('conversations','topics_enabled','boolean',true,'false'); diff --git a/server/postgres/migrations/adoption/000022_add_poll_timestamp_indexes.sql b/server/postgres/migrations/adoption/000022_add_poll_timestamp_indexes.sql new file mode 100644 index 0000000000..8ba522e217 --- /dev/null +++ b/server/postgres/migrations/adoption/000022_add_poll_timestamp_indexes.sql @@ -0,0 +1,3 @@ +-- Catalog postconditions only; this file never replays migration DDL. +SELECT pg_temp.idx('votes_created_idx','CREATE INDEX votes_created_idx ON public.votes USING btree (created)') + AND pg_temp.idx('comments_modified_idx','CREATE INDEX comments_modified_idx ON public.comments USING btree (modified)'); diff --git a/server/postgres/migrations/adoption/helpers.sql b/server/postgres/migrations/adoption/helpers.sql new file mode 100644 index 0000000000..de607fd80b --- /dev/null +++ b/server/postgres/migrations/adoption/helpers.sql @@ -0,0 +1,59 @@ +-- Reconciliation only. These pg_temp helpers vanish when the connection closes. +-- A nullability/default argument checks only when specified by that migration. +CREATE FUNCTION pg_temp.col(t text,c text,typ text,nn boolean DEFAULT NULL,def text DEFAULT NULL) +RETURNS boolean LANGUAGE sql AS $$ + SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||t) AND a.attname=c AND NOT a.attisdropped + AND (format_type(a.atttypid,a.atttypmod)=typ + -- Exact public bootstrap alternatives to the authoritative legacy contract. + OR (t='worker_tasks' AND c='task_type' AND typ='text' + AND format_type(a.atttypid,a.atttypmod)='character varying(99)') + OR (t IN ('pwreset_tokens','password_reset_tokens') + AND c IN ('token','pwresettoken') AND typ='character varying(100)' + AND format_type(a.atttypid,a.atttypmod)='character varying(250)') + -- Only historical math payloads have an established json equivalent. + -- Do not adopt arbitrary json columns in newer jsonb contracts. + OR (typ='jsonb' AND a.atttypid='json'::regtype AND c='data' + AND t IN ('math_main','math_profile','math_ptptstats','math_cache', + 'math_bidtopid','math_exportstatus'))) + AND (nn IS NULL OR a.attnotnull=nn) + AND (def IS NULL OR pg_get_expr(d.adbin,d.adrelid)=def)); +$$; +CREATE FUNCTION pg_temp.con(t text,definition text) RETURNS boolean LANGUAGE sql AS $$ + SELECT EXISTS(SELECT 1 FROM pg_constraint WHERE conrelid=to_regclass('public.'||t) + AND convalidated AND pg_get_constraintdef(oid)=definition); +$$; +CREATE FUNCTION pg_temp.idx(n text,definition text) RETURNS boolean LANGUAGE sql AS $$ + SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||n) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=definition); +$$; +CREATE FUNCTION pg_temp.absent_column(t text,c text) RETURNS boolean LANGUAGE sql AS $$ + SELECT to_regclass('public.'||t) IS NOT NULL AND NOT EXISTS(SELECT 1 FROM pg_attribute + WHERE attrelid=to_regclass('public.'||t) AND attname=c AND NOT attisdropped); +$$; + +-- Full contracts for newly created columns. NULL means no default, not a wildcard. +-- Keep col() unchanged for legacy/core and ALTER TYPE-only postconditions. +CREATE FUNCTION pg_temp.col_exact(t text,c text,typ text,nn boolean,def text DEFAULT NULL) +RETURNS boolean LANGUAGE sql AS $$ + SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||t) AND a.attname=c AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=typ AND a.attnotnull=nn + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM def); +$$; +-- Definition checks include actions/keys; catalog flags also require enforcement. +-- Preserve the legacy helper for contracts awaiting a separate policy decision. +CREATE FUNCTION pg_temp.con_exact(t text,definition text) RETURNS boolean LANGUAGE sql AS $$ + SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||t) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=definition + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O')); +$$; diff --git a/server/postgres/migrations/down/000027-files.sha256 b/server/postgres/migrations/down/000027-files.sha256 new file mode 100644 index 0000000000..9b80b8fc3c --- /dev/null +++ b/server/postgres/migrations/down/000027-files.sha256 @@ -0,0 +1,2 @@ +fbbf948e4316010dd96344ae91542006c38299e52e0a0eb281371c039ab37775 000027_create_sealed_job_graphs.sql +cb2c16f421329a8f049d3d66a0ddbb7c76abaa9f7d2cae049642a98677694170 down/000027_drop_sealed_job_graphs.sql diff --git a/server/postgres/migrations/down/000027_drop_sealed_job_graphs.sql b/server/postgres/migrations/down/000027_drop_sealed_job_graphs.sql new file mode 100644 index 0000000000..56de2ebb5f --- /dev/null +++ b/server/postgres/migrations/down/000027_drop_sealed_job_graphs.sql @@ -0,0 +1,62 @@ +-- Empty draft reversal only. Runner history is never edited here. +BEGIN; +SET LOCAL ROLE polis_queue_owner; +SET LOCAL search_path=pg_catalog,pg_temp; +SET LOCAL lock_timeout='5s'; +CREATE OR REPLACE FUNCTION pg_temp.pq_catalog(p_table oid) RETURNS jsonb +LANGUAGE sql SET search_path=pg_catalog,pg_temp AS $catalog$ +SELECT jsonb_build_object( + 'columns',(SELECT jsonb_agg(jsonb_build_array(a.attname,format_type(a.atttypid,a.atttypmod),a.attnotnull,a.attidentity,a.attgenerated,co.collname,pg_get_expr(d.adbin,d.adrelid),NULLIF(a.attacl::text,'{}')) ORDER BY a.attnum) + FROM pg_attribute a LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum LEFT JOIN pg_collation co ON co.oid=a.attcollation + WHERE a.attrelid=c.oid AND a.attnum>0 AND NOT a.attisdropped), + 'constraints',(SELECT jsonb_agg(jsonb_build_array(conname,pg_get_constraintdef(oid),convalidated,connoinherit) ORDER BY conname) FROM pg_constraint WHERE conrelid=c.oid), + 'indexes',(SELECT jsonb_agg(jsonb_build_array(ic.relname,pg_get_indexdef(i.indexrelid),i.indisvalid,i.indisready) ORDER BY ic.relname) FROM pg_index i JOIN pg_class ic ON ic.oid=i.indexrelid WHERE i.indrelid=c.oid), + 'triggers',(SELECT jsonb_agg(jsonb_build_array(t.tgname,pg_get_triggerdef(t.oid),t.tgenabled) ORDER BY t.tgname) FROM pg_trigger t WHERE t.tgrelid=c.oid AND NOT t.tgisinternal), + 'owner',pg_get_userbyid(c.relowner),'kind',c.relkind,'rls',c.relrowsecurity,'force_rls',c.relforcerowsecurity,'options',c.reloptions, + 'acl',(SELECT jsonb_agg(jsonb_build_array(CASE WHEN x.grantee=0 THEN 'PUBLIC' ELSE pg_get_userbyid(x.grantee) END,x.privilege_type,x.is_grantable) ORDER BY x.grantee=0,pg_get_userbyid(x.grantee),x.privilege_type,x.is_grantable) FROM aclexplode(COALESCE(c.relacl,acldefault('r',c.relowner))) x) + ) FROM pg_class c WHERE c.oid=p_table +$catalog$; +CREATE OR REPLACE FUNCTION pg_temp.pd_state() RETURNS jsonb LANGUAGE sql SET search_path=pg_catalog,pg_temp AS $s$ +SELECT jsonb_build_object('tables',(SELECT jsonb_object_agg(c.relname,pg_temp.pq_catalog(c.oid)) FROM pg_class c + WHERE c.relnamespace='public'::regnamespace AND c.relkind='r' AND (starts_with(c.relname,'polis_queue_') OR starts_with(c.relname,'delphi_')) AND c.relname<>'delphi_foundation_install'), + 'functions',(SELECT jsonb_object_agg(p.oid::regprocedure::text,jsonb_build_array(pg_get_functiondef(p.oid),p.proacl::text,pg_get_userbyid(p.proowner))) + FROM pg_proc p WHERE p.pronamespace='public'::regnamespace AND (starts_with(p.proname,'pq_') OR starts_with(p.proname,'pd_')))) +$s$; +-- The /3 state: the same, less this file's own install table. +CREATE OR REPLACE FUNCTION pg_temp.pq3_state() RETURNS jsonb LANGUAGE sql SET search_path=pg_catalog,pg_temp AS $s$ +SELECT jsonb_build_object('tables',(SELECT jsonb_object_agg(c.relname,pg_temp.pq_catalog(c.oid)) FROM pg_class c + WHERE c.relnamespace='public'::regnamespace AND c.relkind='r' AND (starts_with(c.relname,'polis_queue_') OR starts_with(c.relname,'delphi_')) + AND c.relname NOT IN ('delphi_foundation_install','polis_queue_large_class_install')), + 'functions',(SELECT jsonb_object_agg(p.oid::regprocedure::text,jsonb_build_array(pg_get_functiondef(p.oid),p.proacl::text,pg_get_userbyid(p.proowner))) + FROM pg_proc p WHERE p.pronamespace='public'::regnamespace AND (starts_with(p.proname,'pq_') OR starts_with(p.proname,'pd_')))) +$s$; + + +DO $$ DECLARE r record; baseline jsonb; BEGIN + LOCK TABLE public.delphi_graphs IN ACCESS EXCLUSIVE MODE; + IF EXISTS(SELECT 1 FROM public.delphi_graphs) + THEN RAISE EXCEPTION 'nonempty graph contract: retain schema and use compatible workers'; END IF; + SELECT i.baseline||'{}'::jsonb INTO STRICT baseline FROM public.delphi_graph_install i; + -- Drop only draft objects. Restore original function definitions and ACLs below. + DROP TRIGGER graph_parent_guard ON public.delphi_jobs; + DROP TABLE public.delphi_graph_served,public.delphi_graph_edges,public.delphi_artifacts, + public.delphi_graph_nodes,public.delphi_graphs,public.delphi_graph_install CASCADE; + FOR r IN SELECT oid::regprocedure AS name FROM pg_proc WHERE pronamespace='public'::regnamespace AND proname LIKE 'pd_graph_%' LOOP + EXECUTE format('DROP FUNCTION %s CASCADE',r.name); + END LOOP; + ALTER TABLE public.polis_queue_install DROP CONSTRAINT polis_queue_install_contract_version_check; + UPDATE public.polis_queue_install SET contract_version='polis-queue/3'; + ALTER TABLE public.polis_queue_install ADD CONSTRAINT polis_queue_install_contract_version_check CHECK(contract_version='polis-queue/3'); + ALTER TABLE public.polis_queue_install ALTER COLUMN contract_version SET DEFAULT 'polis-queue/3'; + ALTER TABLE public.polis_queue_runs DROP CONSTRAINT polis_queue_runs_contract_version_check; + ALTER TABLE public.polis_queue_runs ADD CONSTRAINT polis_queue_runs_contract_version_check CHECK(contract_version IN ('polis-queue/1','polis-queue/2','polis-queue/3')); + ALTER TABLE public.polis_queue_jobs DROP CONSTRAINT polis_queue_jobs_stage_check; + ALTER TABLE public.polis_queue_jobs ADD CONSTRAINT polis_queue_jobs_stage_check CHECK(stage IN ('noop','delphi_full_pipeline','delphi_narrative','math_rebuild')); + ALTER TABLE public.polis_queue_jobs DROP CONSTRAINT pq_stage_large; + ALTER TABLE public.polis_queue_jobs ADD CONSTRAINT pq_stage_large CHECK((stage='math_rebuild')=(worker_class='large')); + FOR r IN SELECT key,value FROM jsonb_each(baseline->'functions') LOOP + EXECUTE r.value->>0; + END LOOP; + IF baseline IS DISTINCT FROM pg_temp.pq3_state() THEN RAISE EXCEPTION 'graph down did not restore exact /3 catalog'; END IF; +END $$; +COMMIT; diff --git a/server/postgres/migrations/held.txt b/server/postgres/migrations/held.txt new file mode 100644 index 0000000000..b8cd53cfc9 --- /dev/null +++ b/server/postgres/migrations/held.txt @@ -0,0 +1,3 @@ +# Release-wide hold: the coordinator is dormant and not an API requirement. +# Moving this file into the required chain needs an explicit schema release. +000021_create_polis_coordinator.sql diff --git a/server/postgres/migrations/release.txt b/server/postgres/migrations/release.txt new file mode 100644 index 0000000000..f0e85e03eb --- /dev/null +++ b/server/postgres/migrations/release.txt @@ -0,0 +1,28 @@ +# Selected stable runner release. Unknown numbered files fail closed. +# M4/M5/M7 are retirement receipts only; their DDL never executes. +000000_initial.sql +000001_update_pwreset_table.sql +000002_add_xid_constraint.sql +000003_add_origin_permanent_cookie_columns.sql +000004_drop_waitinglist_table.sql +000005_drop_slack_stripe_canvas.sql +000006_update_votes_rule.sql +000007_drop_geolocation_fields.sql +000008_add_comment_priority.sql +000009_add_uuid_to_zinvites.sql +000010_create_oidc_user_mappings.sql +000011_alter_suzinvites_xid_to_text.sql +000012_create_topic_agenda_selections.sql +000013_create_treevite.sql +000014_alter_reports_modlevel.sql +000015_add_xid_requirements.sql +000016_add_orig_id.sql +000017_create_byod_job_table.sql +000018_add_topics_enabled.sql +000019_create_polis_queue.sql +000022_add_poll_timestamp_indexes.sql +000023_create_delphi_foundation.sql +000024_create_polis_queue_large_class.sql +000027_create_sealed_job_graphs.sql +000028_create_delphi_results.sql +000029_extend_delphi_graph_stages.sql diff --git a/server/postgres/migrations/report/first-deploy.sql b/server/postgres/migrations/report/first-deploy.sql new file mode 100644 index 0000000000..6c6c16e123 --- /dev/null +++ b/server/postgres/migrations/report/first-deploy.sql @@ -0,0 +1,1405 @@ +-- GENERATED by server/bin/build-migration-report.py; do not edit. +-- First deployment only: no existing ledger or queue. Other states refuse. +-- This is a catalog forecast, not DDL success or deployment-health proof. +-- sha256 8fa1066d9fc29c1df528b300199c619a2f9edecf1432926ad6a8146ebc650593 adoption/helpers.sql +-- sha256 abf8dcce59a2202bfbb77d6970ef9dc91bb0ad299fa076e3d8d72716761c1d1b release.txt +-- sha256 8a8b24fa47a25c613a8329aa79f413ac7461a93b12134045ae1cf2457e2ee3b7 held.txt +-- sha256 c6b5c71247d129964dd89ba560a4522caef1abbb9379ad1b3445d6ed96be805b adoption/000000_initial.sql +-- sha256 28ef33976beb9c7c9248ac800f29aab048f4410c2170ad12e3ade71d72e5ecb5 adoption/000001_update_pwreset_table.sql +-- sha256 c3393049eae0a663ecea42a4917a06b37521a8ff508d39b308f8fd9da5606bf9 adoption/000002_add_xid_constraint.sql +-- sha256 3b1c9a390cbf23882455f1f3d05119df4a306f9a04d18e8c0e2cff3374895b86 adoption/000003_add_origin_permanent_cookie_columns.sql +-- sha256 5fae81bb9ff86062125487df98c1742e7ce5c45466fbeed7f6f63cbf94b1933b adoption/000004_drop_waitinglist_table.sql +-- sha256 02c90a416148f29c0717c01bcc97e66d6e8083ab3a2fabc5dc6bc716e963447e adoption/000005_drop_slack_stripe_canvas.sql +-- sha256 42d43c346ea90d0504698df29f90dd2c7e57860338c633da11410aaf9aaf5148 adoption/000006_update_votes_rule.sql +-- sha256 5a4110278c60157f21e5cb920ece834f6d6f99e6d4fa34b9b55bf0c6e80587c0 adoption/000007_drop_geolocation_fields.sql +-- sha256 a10a1a1881044409d98efd6042b5b28933a4579bce32738be1c4bc268640f172 adoption/000008_add_comment_priority.sql +-- sha256 a5605b5a292f5b4d47849cb7f2b678211e3bcdc9722cf9e5eabe54f455a5aa7c adoption/000009_add_uuid_to_zinvites.sql +-- sha256 23d5179cd0b556a71fe8b1ce56d3ac9e614cdee59832ce7892a012f0193cfe66 adoption/000010_create_oidc_user_mappings.sql +-- sha256 c991ffd93cd9051302dd51ab8d5bd6025128fc9728330e29aa963306a9b80f3e adoption/000011_alter_suzinvites_xid_to_text.sql +-- sha256 c31bd6811efee0808da5abb32ba63c47deebe6c6e89baa1a80ddcacc9e250234 adoption/000012_create_topic_agenda_selections.sql +-- sha256 eed1d9e223322db2b234428d350b6548793e54b94315932658b693d8263a6001 adoption/000013_create_treevite.sql +-- sha256 6b99d93d2f90f03bee170305c4b1305f03636882a84cdd08e107a763f1fc0120 adoption/000014_alter_reports_modlevel.sql +-- sha256 30c8d693d4f0d523c68a375bf0160e71e574f0a155badc363e085c4688ff0625 adoption/000015_add_xid_requirements.sql +-- sha256 ffb57aff8be7bc81bf10899133fe2a243cb20b58eb6b73afb3b76a66d5632478 adoption/000016_add_orig_id.sql +-- sha256 b72fbd498c13aceb7973100331c58b55a468b7e4447645a3e5c93f79a1126fd6 adoption/000017_create_byod_job_table.sql +-- sha256 96bf4a25a5733ac3dce8a7f25a4f374b5ee9bed924e95e048f73a343d2ddf539 adoption/000018_add_topics_enabled.sql +-- sha256 9764fc94c4fe4ac19626381146a588da3f6b4d32cac53ab59ae87b401ea8af8a adoption/000022_add_poll_timestamp_indexes.sql +-- sha256 2652134140cd8796ee3ab64995d509206bab3ebe8a03bba0cf48038a32dc6cea 000000_initial.sql +-- sha256 cb21278194c4b7db510c12bf1a7761b3a905410a7b3769edeb0d1865b6ec047b 000001_update_pwreset_table.sql +-- sha256 a27a8e63c79cd055198fa201b1745c85e0cd0e0b75f228b7153571631e529c07 000002_add_xid_constraint.sql +-- sha256 0d7f27facfecff1327573d3e1a5f65a3277ea94a30bcd8692606f00a7e496d3d 000003_add_origin_permanent_cookie_columns.sql +-- sha256 f2fc4184a965ffad01913d0e71a806c6f7e2aecc5f59279c11b0cd3ed3a77528 000004_drop_waitinglist_table.sql +-- sha256 392e5b8aadb7a85767820ec97fcfb0853821d35089be3e8c3dfe70df24782839 000005_drop_slack_stripe_canvas.sql +-- sha256 8fb05b7b1b6a8b123cea6b2680054d8da2fe48585c51a2f9f108b8b174d85c60 000006_update_votes_rule.sql +-- sha256 f68b69b86112ad52e2cb79a9b631ea77a0285b24201b41c15d9c13257f7cd9cc 000007_drop_geolocation_fields.sql +-- sha256 c867b53be3cca6bce059b19ea9eb3950120fb5840a1476af409924ca55905261 000008_add_comment_priority.sql +-- sha256 43f36fe0b8578bc4b32761a2b9d7a5181a8159ff8b0c7bb0a1a73f802ec5f670 000009_add_uuid_to_zinvites.sql +-- sha256 450a3f69883aa56c3bc85994ab1baf5e208634d88435a156fe2a22738b1d9f9a 000010_create_oidc_user_mappings.sql +-- sha256 00a1eb2d9604804a56705eaac92bb6d37b6c696abdfd9bb079d4a1c494ff0ab6 000011_alter_suzinvites_xid_to_text.sql +-- sha256 cc513693124f031ce1d55e7e73b14b4e3c40a4070e65465adbc621e63b1008cf 000012_create_topic_agenda_selections.sql +-- sha256 4b8334f73246c69bc9ecadd4a2dc00c8cfb642351ba70ed4a9a58b882a4ac53f 000013_create_treevite.sql +-- sha256 c2af6d57af2866f1f259bb41b94dbf730eca499a4b18f092c4aeeebce5a95025 000014_alter_reports_modlevel.sql +-- sha256 186c904addd0c8a42c10057e6ae362fe2799cc076e877bef3ffa14e9852184e7 000015_add_xid_requirements.sql +-- sha256 6cdc0588c000fbbe89d578bc2f0aa60422b92b5320aba737d6d83f89d8d990db 000016_add_orig_id.sql +-- sha256 f27c03a1229f296fab21cee059b743c7137bf502e4178854e596a8efa4808b55 000017_create_byod_job_table.sql +-- sha256 a1e1c0572064d88877d87142bc4e40132b84672bbe84392e99f9f6f4c2ea107e 000018_add_topics_enabled.sql +-- sha256 fedfbcf9fc594c3e53193cacb76dfd35558d81a449d507fcacb3198f5e39beae 000019_create_polis_queue.sql +-- sha256 d50f169ad7afe12d14582a6a746c622d402ecafd8131aae246812263bf2d5e82 000021_create_polis_coordinator.sql +-- sha256 14efc95b14787b52d70680ea06cfef020cd2224e82495258fd8eb324501b7b3e 000022_add_poll_timestamp_indexes.sql +-- sha256 97437ea57d90c51cc664385ebb6e8ec3d8684ac0f8e7b0c60e84df1a010531f7 000023_create_delphi_foundation.sql +-- sha256 68261afb81f286bed45fdff6ab52e052392d1dd32abdb1e78579f72644116697 000024_create_polis_queue_large_class.sql +-- sha256 fbbf948e4316010dd96344ae91542006c38299e52e0a0eb281371c039ab37775 000027_create_sealed_job_graphs.sql +-- sha256 0f90d8c7c6f9a440b2d9901dcf7a00f17222d01e48f38fb1c8c7d22be05156c8 000028_create_delphi_results.sql +-- sha256 b4dd41790639184c40418effd290ecd86d15cb8d9f46a1a77b616f37a63d5906 000029_extend_delphi_graph_stages.sql +BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY; +SET LOCAL search_path=pg_catalog,public; +SET LOCAL statement_timeout='30s'; +SET LOCAL lock_timeout='2s'; +WITH checks AS ( +SELECT '000000_initial.sql'::text AS migration, (-- Named legacy variants are documented in docs/migration-legacy-contract.md. +-- Contributor agreement tables are outside this release adoption boundary. +-- Catalog postconditions only; this file never replays migration DDL. +SELECT NOT EXISTS (SELECT 1 FROM (VALUES + ('users','uid','integer'), + ('users','hname','character varying(746)'), + ('users','created','bigint'), + ('users','username','character varying(128)'), + ('users','email','character varying(256)'), + ('users','is_owner','boolean'), + ('users','zinvite','character varying(300)'), + ('users','oinvite','character varying(300)'), + ('users','tut','smallint'), + ('users','site_id','character varying(256)'), + ('users','site_owner','boolean'), + ('site_domain_whitelist','site_id','character varying(256)'), + ('site_domain_whitelist','domain_whitelist','character varying(999)'), + ('site_domain_whitelist','domain_whitelist_override_key','character varying(999)'), + ('site_domain_whitelist','modified','bigint'), + ('site_domain_whitelist','created','bigint'), + ('metrics','uid','integer'), + ('metrics','type','integer'), + ('metrics','dur','integer'), + ('metrics','hashedpc','integer'), + ('metrics','created','bigint'), + ('auth_tokens','token','character varying(32)'), + ('auth_tokens','uid','integer'), + ('auth_tokens','created','bigint'), + ('jianiuevyew','uid','integer'), + ('jianiuevyew','pwhash','character varying(128)'), + ('apikeysndvweifu','uid','integer'), + ('apikeysndvweifu','apikey','character varying(32)'), + ('apikeysndvweifu','created','bigint'), + ('courses','course_id','integer'), + ('courses','topic','character varying(1000)'), + ('courses','description','character varying(1000)'), + ('courses','owner','integer'), + ('courses','course_invite','character varying(32)'), + ('courses','created','bigint'), + ('conversations','zid','integer'), + ('conversations','topic','character varying(1000)'), + ('conversations','description','character varying(50000)'), + ('conversations','link_url','character varying(9999)'), + ('conversations','parent_url','character varying(9999)'), + ('conversations','upvotes','integer'), + ('conversations','participant_count','integer'), + ('conversations','is_anon','boolean'), + ('conversations','is_active','boolean'), + ('conversations','is_draft','boolean'), + ('conversations','is_public','boolean'), + ('conversations','is_data_open','boolean'), + ('conversations','profanity_filter','boolean'), + ('conversations','spam_filter','boolean'), + ('conversations','strict_moderation','boolean'), + ('conversations','prioritize_seed','boolean'), + ('conversations','vis_type','integer'), + ('conversations','write_type','integer'), + ('conversations','help_type','integer'), + ('conversations','write_hint_type','integer'), + ('conversations','style_btn','character varying(500)'), + ('conversations','socialbtn_type','integer'), + ('conversations','subscribe_type','integer'), + ('conversations','bgcolor','character varying(20)'), + ('conversations','help_bgcolor','character varying(20)'), + ('conversations','help_color','character varying(20)'), + ('conversations','email_domain','character varying(200)'), + ('conversations','use_xid_whitelist','boolean'), + ('conversations','owner','integer'), + ('conversations','org_id','integer'), + ('conversations','context','character varying(1000)'), + ('conversations','course_id','integer'), + ('conversations','owner_sees_participation_stats','boolean'), + ('conversations','auth_needed_to_vote','boolean'), + ('conversations','auth_needed_to_write','boolean'), + ('conversations','auth_opt_fb','boolean'), + ('conversations','auth_opt_tw','boolean'), + ('conversations','auth_opt_allow_3rdparty','boolean'), + ('conversations','modified','bigint'), + ('conversations','created','bigint'), + ('participant_metadata_questions','pmqid','integer'), + ('participant_metadata_questions','zid','integer'), + ('participant_metadata_questions','key','character varying(999)'), + ('participant_metadata_questions','alive','boolean'), + ('participant_metadata_questions','created','bigint'), + ('participant_metadata_answers','pmaid','integer'), + ('participant_metadata_answers','pmqid','integer'), + ('participant_metadata_answers','zid','integer'), + ('participant_metadata_answers','value','character varying(999)'), + ('participant_metadata_answers','alive','boolean'), + ('participant_metadata_answers','created','bigint'), + ('contexts','context_id','integer'), + ('contexts','name','character varying(300)'), + ('contexts','creator','integer'), + ('contexts','is_public','boolean'), + ('contexts','created','bigint'), + ('inviters','inviter_uid','integer'), + ('inviters','invited_email','character varying(999)'), + ('inviters','created','bigint'), + ('upvotes','uid','integer'), + ('upvotes','zid','integer'), + ('oinvites','oinvite','character varying(300)'), + ('oinvites','note','character varying(999)'), + ('oinvites','created','bigint'), + ('einvites','einvite','character varying(100)'), + ('einvites','email','character varying(999)'), + ('einvites','created','bigint'), + ('email_validations','email','character varying(999)'), + ('email_validations','created','bigint'), + ('zinvites','zid','integer'), + ('zinvites','zinvite','character varying(300)'), + ('zinvites','created','bigint'), + ('beta','name','character varying(999)'), + ('beta','email','character varying(200)'), + ('beta','organization','character varying(200)'), + ('beta','created','bigint'), + ('participants','pid','integer'), + ('participants','uid','integer'), + ('participants','zid','integer'), + ('participants','vote_count','integer'), + ('participants','last_interaction','bigint'), + ('participants','subscribed','integer'), + ('participants','last_notified','bigint'), + ('participants','nsli','smallint'), + ('participants','mod','integer'), + ('participants','created','bigint'), + ('participants_extended','uid','integer'), + ('participants_extended','zid','integer'), + ('participants_extended','referrer','character varying(9999)'), + ('participants_extended','parent_url','character varying(9999)'), + ('participants_extended','created','bigint'), + ('participants_extended','modified','bigint'), + ('participants_extended','subscribe_email','character varying(256)'), + ('participants_extended','show_translation_activated','boolean'), + ('participant_locations','zid','integer'), + ('participant_locations','uid','integer'), + ('participant_locations','pid','integer'), + ('participant_locations','lat','double precision'), + ('participant_locations','lng','double precision'), + ('participant_locations','created','bigint'), + ('participant_locations','source','integer'), + ('xids','uid','integer'), + ('xids','owner','integer'), + ('xids','xid','text'), + ('xids','x_profile_image_url','character varying(3000)'), + ('xids','x_name','character varying(746)'), + ('xids','x_email','character varying(256)'), + ('xids','created','bigint'), + ('xids','modified','bigint'), + ('xid_whitelist','owner','integer'), + ('xid_whitelist','xid','text'), + ('xid_whitelist','created','bigint'), + ('notification_tasks','zid','integer'), + ('notification_tasks','modified','bigint'), + ('participant_metadata_choices','zid','integer'), + ('participant_metadata_choices','pid','integer'), + ('participant_metadata_choices','pmqid','integer'), + ('participant_metadata_choices','pmaid','integer'), + ('participant_metadata_choices','alive','boolean'), + ('participant_metadata_choices','created','bigint'), + ('twitter_users','uid','integer'), + ('twitter_users','twitter_user_id','bigint'), + ('twitter_users','screen_name','character varying(999)'), + ('twitter_users','name','character varying(9999)'), + ('twitter_users','followers_count','integer'), + ('twitter_users','friends_count','integer'), + ('twitter_users','verified','boolean'), + ('twitter_users','profile_image_url_https','character varying(9999)'), + ('twitter_users','location','character varying(9999)'), + ('twitter_users','response','json'), + ('twitter_users','modified','bigint'), + ('twitter_users','created','bigint'), + ('facebook_users','uid','integer'), + ('facebook_users','fb_user_id','text'), + ('facebook_users','fb_name','character varying(9999)'), + ('facebook_users','fb_link','character varying(9999)'), + ('facebook_users','fb_public_profile','text'), + ('facebook_users','fb_login_status','text'), + ('facebook_users','fb_auth_response','text'), + ('facebook_users','fb_access_token','text'), + ('facebook_users','fb_granted_scopes','text'), + ('facebook_users','fb_location_id','character varying(100)'), + ('facebook_users','location','character varying(9999)'), + ('facebook_users','response','text'), + ('facebook_users','fb_friends_response','text'), + ('facebook_users','created','bigint'), + ('facebook_users','modified','bigint'), + ('social_settings','uid','integer'), + ('social_settings','polis_pic','character varying(3000)'), + ('facebook_friends','uid','integer'), + ('facebook_friends','friend','integer'), + ('suzinvites','owner','integer'), + ('suzinvites','zid','integer'), + ('suzinvites','created','bigint'), + ('suzinvites','suzinvite','character varying(32)'), + ('comments','tid','integer'), + ('comments','zid','integer'), + ('comments','pid','integer'), + ('comments','uid','integer'), + ('comments','created','bigint'), + ('comments','modified','bigint'), + ('comments','txt','character varying(1000)'), + ('comments','velocity','real'), + ('comments','mod','integer'), + ('comments','lang','character varying(10)'), + ('comments','lang_confidence','real'), + ('comments','active','boolean'), + ('comments','is_meta','boolean'), + ('comments','tweet_id','bigint'), + ('comments','quote_src_url','character varying(1000)'), + ('comments','anon','boolean'), + ('comments','is_seed','boolean'), + ('comment_translations','zid','integer'), + ('comment_translations','tid','integer'), + ('comment_translations','src','integer'), + ('comment_translations','txt','character varying(9999)'), + ('comment_translations','lang','character varying(10)'), + ('comment_translations','created','bigint'), + ('comment_translations','modified','bigint'), + ('conversation_translations','zid','integer'), + ('conversation_translations','src','integer'), + ('conversation_translations','topic','character varying(9999)'), + ('conversation_translations','description','character varying(9999)'), + ('conversation_translations','lang','character varying(10)'), + ('conversation_translations','created','bigint'), + ('conversation_translations','modified','bigint'), + ('reports','rid','bigint'), + ('reports','report_id','character varying(300)'), + ('reports','zid','integer'), + ('reports','created','bigint'), + ('reports','modified','bigint'), + ('reports','report_name','character varying(999)'), + ('reports','label_x_neg','character varying(999)'), + ('reports','label_x_pos','character varying(999)'), + ('reports','label_y_neg','character varying(999)'), + ('reports','label_y_pos','character varying(999)'), + ('reports','label_group_0','character varying(999)'), + ('reports','label_group_1','character varying(999)'), + ('reports','label_group_2','character varying(999)'), + ('reports','label_group_3','character varying(999)'), + ('reports','label_group_4','character varying(999)'), + ('reports','label_group_5','character varying(999)'), + ('reports','label_group_6','character varying(999)'), + ('reports','label_group_7','character varying(999)'), + ('reports','label_group_8','character varying(999)'), + ('reports','label_group_9','character varying(999)'), + ('report_comment_selections','zid','integer'), + ('report_comment_selections','rid','bigint'), + ('report_comment_selections','tid','integer'), + ('report_comment_selections','selection','smallint'), + ('report_comment_selections','modified','bigint'), + ('worker_tasks','created','bigint'), + ('worker_tasks','math_env','character varying(999)'), + ('worker_tasks','attempts','smallint'), + ('worker_tasks','task_data','jsonb'), + ('worker_tasks','task_type','text'), + ('worker_tasks','task_bucket','bigint'), + ('worker_tasks','finished_time','bigint'), + ('math_ticks','zid','integer'), + ('math_ticks','math_tick','bigint'), + ('math_ticks','math_env','character varying(999)'), + ('math_ticks','modified','bigint'), + ('math_main','zid','integer'), + ('math_main','math_env','character varying(999)'), + ('math_main','data','jsonb'), + ('math_main','last_vote_timestamp','bigint'), + ('math_main','caching_tick','bigint'), + ('math_main','math_tick','bigint'), + ('math_main','modified','bigint'), + ('math_profile','zid','integer'), + ('math_profile','math_env','character varying(999)'), + ('math_profile','data','jsonb'), + ('math_profile','modified','bigint'), + ('math_ptptstats','zid','integer'), + ('math_ptptstats','math_env','character varying(999)'), + ('math_ptptstats','math_tick','bigint'), + ('math_ptptstats','data','jsonb'), + ('math_ptptstats','modified','bigint'), + ('math_cache','zid','integer'), + ('math_cache','math_env','character varying(999)'), + ('math_cache','data','jsonb'), + ('math_cache','modified','bigint'), + ('math_bidtopid','zid','integer'), + ('math_bidtopid','math_env','character varying(999)'), + ('math_bidtopid','math_tick','bigint'), + ('math_bidtopid','data','jsonb'), + ('math_bidtopid','modified','bigint'), + ('math_exportstatus','zid','integer'), + ('math_exportstatus','math_env','character varying(999)'), + ('math_exportstatus','filename','character varying(9999)'), + ('math_exportstatus','data','jsonb'), + ('math_exportstatus','modified','bigint'), + ('math_report_correlationmatrix','rid','bigint'), + ('math_report_correlationmatrix','math_env','character varying(999)'), + ('math_report_correlationmatrix','data','jsonb'), + ('math_report_correlationmatrix','math_tick','bigint'), + ('math_report_correlationmatrix','modified','bigint'), + ('votes','zid','integer'), + ('votes','pid','integer'), + ('votes','tid','integer'), + ('votes','vote','smallint'), + ('votes','weight_x_32767','smallint'), + ('votes','created','bigint'), + ('votes_latest_unique','zid','integer'), + ('votes_latest_unique','pid','integer'), + ('votes_latest_unique','tid','integer'), + ('votes_latest_unique','vote','smallint'), + ('votes_latest_unique','weight_x_32767','smallint'), + ('votes_latest_unique','modified','bigint'), + ('crowd_mod','zid','integer'), + ('crowd_mod','pid','integer'), + ('crowd_mod','tid','integer'), + ('crowd_mod','created','bigint'), + ('crowd_mod','as_important','boolean'), + ('crowd_mod','as_factual','boolean'), + ('crowd_mod','as_feeling','boolean'), + ('crowd_mod','as_notmyfeeling','boolean'), + ('crowd_mod','as_notgoodidea','boolean'), + ('crowd_mod','as_notfact','boolean'), + ('crowd_mod','as_unsure','boolean'), + ('crowd_mod','as_spam','boolean'), + ('crowd_mod','as_abusive','boolean'), + ('crowd_mod','as_offtopic','boolean'), + ('event_ptpt_no_more_comments','zid','integer'), + ('event_ptpt_no_more_comments','pid','integer'), + ('event_ptpt_no_more_comments','votes_placed','smallint'), + ('event_ptpt_no_more_comments','created','bigint'), + ('stars','zid','integer'), + ('stars','pid','integer'), + ('stars','tid','integer'), + ('stars','starred','integer'), + ('stars','created','bigint'), + ('trashes','zid','integer'), + ('trashes','pid','integer'), + ('trashes','tid','integer'), + ('trashes','trashed','integer'), + ('trashes','created','bigint'), + ('permanentcookiezidjoins','zid','integer'), + ('permanentcookiezidjoins','cookie','character varying(32)'), + ('permanentcookiezidjoins','created','bigint'), + ('page_ids','site_id','character varying(100)'), + ('page_ids','page_id','character varying(100)'), + ('page_ids','zid','integer'), + ('demographic_data','uid','integer'), + ('demographic_data','fb_gender','integer'), + ('demographic_data','ms_birth_year_estimate_fb','integer'), + ('demographic_data','ms_gender_estimate_fb','integer'), + ('demographic_data','fb_timestamp','bigint'), + ('demographic_data','ms_fb_timestamp','bigint'), + ('demographic_data','ms_response','character varying(9999)'), + ('demographic_data','gender_guess','integer'), + ('demographic_data','birth_year_guess','integer')) AS expected(t,c,typ) WHERE NOT (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||(t)) AND a.attname=(c) AND NOT a.attisdropped + AND (format_type(a.atttypid,a.atttypmod)=(typ) + -- Exact public bootstrap alternatives to the authoritative legacy contract. + OR ((t)='worker_tasks' AND (c)='task_type' AND (typ)='text' + AND format_type(a.atttypid,a.atttypmod)='character varying(99)') + OR ((t) IN ('pwreset_tokens','password_reset_tokens') + AND (c) IN ('token','pwresettoken') AND (typ)='character varying(100)' + AND format_type(a.atttypid,a.atttypmod)='character varying(250)') + -- Only historical math payloads have an established json equivalent. + -- Do not adopt arbitrary json columns in newer jsonb contracts. + OR ((typ)='jsonb' AND a.atttypid='json'::regtype AND (c)='data' + AND (t) IN ('math_main','math_profile','math_ptptstats','math_cache', + 'math_bidtopid','math_exportstatus'))) + AND ((NULL) IS NULL OR a.attnotnull=(NULL)) + AND ((NULL) IS NULL OR pg_get_expr(d.adbin,d.adrelid)=(NULL))))) + AND NOT EXISTS (SELECT 1 FROM unnest(ARRAY['users','site_domain_whitelist','metrics','auth_tokens','jianiuevyew','apikeysndvweifu','courses','conversations','participant_metadata_questions','participant_metadata_answers','contexts','inviters','upvotes','oinvites','einvites','email_validations','zinvites','beta','participants','participants_extended','participant_locations','xids','xid_whitelist','notification_tasks','participant_metadata_choices','twitter_users','facebook_users','social_settings','facebook_friends','suzinvites','comments','comment_translations','conversation_translations','reports','report_comment_selections','worker_tasks','math_ticks','math_main','math_profile','math_ptptstats','math_cache','math_bidtopid','math_exportstatus','math_report_correlationmatrix','votes','votes_latest_unique','crowd_mod','event_ptpt_no_more_comments','stars','trashes','permanentcookiezidjoins','page_ids','demographic_data']) t WHERE to_regclass('public.'||t) IS NULL) + AND ((SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('pwreset_tokens')) AND a.attname=('token') AND NOT a.attisdropped + AND (format_type(a.atttypid,a.atttypmod)=('character varying(100)') + -- Exact public bootstrap alternatives to the authoritative legacy contract. + OR (('pwreset_tokens')='worker_tasks' AND ('token')='task_type' AND ('character varying(100)')='text' + AND format_type(a.atttypid,a.atttypmod)='character varying(99)') + OR (('pwreset_tokens') IN ('pwreset_tokens','password_reset_tokens') + AND ('token') IN ('token','pwresettoken') AND ('character varying(100)')='character varying(100)' + AND format_type(a.atttypid,a.atttypmod)='character varying(250)') + -- Only historical math payloads have an established json equivalent. + -- Do not adopt arbitrary json columns in newer jsonb contracts. + OR (('character varying(100)')='jsonb' AND a.atttypid='json'::regtype AND ('token')='data' + AND ('pwreset_tokens') IN ('math_main','math_profile','math_ptptstats','math_cache', + 'math_bidtopid','math_exportstatus'))) + AND ((NULL) IS NULL OR a.attnotnull=(NULL)) + AND ((NULL) IS NULL OR pg_get_expr(d.adbin,d.adrelid)=(NULL)))) OR (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('password_reset_tokens')) AND a.attname=('pwresettoken') AND NOT a.attisdropped + AND (format_type(a.atttypid,a.atttypmod)=('character varying(100)') + -- Exact public bootstrap alternatives to the authoritative legacy contract. + OR (('password_reset_tokens')='worker_tasks' AND ('pwresettoken')='task_type' AND ('character varying(100)')='text' + AND format_type(a.atttypid,a.atttypmod)='character varying(99)') + OR (('password_reset_tokens') IN ('pwreset_tokens','password_reset_tokens') + AND ('pwresettoken') IN ('token','pwresettoken') AND ('character varying(100)')='character varying(100)' + AND format_type(a.atttypid,a.atttypmod)='character varying(250)') + -- Only historical math payloads have an established json equivalent. + -- Do not adopt arbitrary json columns in newer jsonb contracts. + OR (('character varying(100)')='jsonb' AND a.atttypid='json'::regtype AND ('pwresettoken')='data' + AND ('password_reset_tokens') IN ('math_main','math_profile','math_ptptstats','math_cache', + 'math_bidtopid','math_exportstatus'))) + AND ((NULL) IS NULL OR a.attnotnull=(NULL)) + AND ((NULL) IS NULL OR pg_get_expr(d.adbin,d.adrelid)=(NULL))))) + AND ((SELECT to_regclass('public.'||('conversations')) IS NOT NULL AND NOT EXISTS(SELECT 1 FROM pg_attribute + WHERE attrelid=to_regclass('public.'||('conversations')) AND attname=('branding_type') AND NOT attisdropped)) OR (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('conversations')) AND a.attname=('branding_type') AND NOT a.attisdropped + AND (format_type(a.atttypid,a.atttypmod)=('integer') + -- Exact public bootstrap alternatives to the authoritative legacy contract. + OR (('conversations')='worker_tasks' AND ('branding_type')='task_type' AND ('integer')='text' + AND format_type(a.atttypid,a.atttypmod)='character varying(99)') + OR (('conversations') IN ('pwreset_tokens','password_reset_tokens') + AND ('branding_type') IN ('token','pwresettoken') AND ('integer')='character varying(100)' + AND format_type(a.atttypid,a.atttypmod)='character varying(250)') + -- Only historical math payloads have an established json equivalent. + -- Do not adopt arbitrary json columns in newer jsonb contracts. + OR (('integer')='jsonb' AND a.atttypid='json'::regtype AND ('branding_type')='data' + AND ('conversations') IN ('math_main','math_profile','math_ptptstats','math_cache', + 'math_bidtopid','math_exportstatus'))) + AND ((NULL) IS NULL OR a.attnotnull=(NULL)) + AND ((NULL) IS NULL OR pg_get_expr(d.adbin,d.adrelid)=(NULL))))) + AND ((SELECT to_regclass('public.'||('math_ticks')) IS NOT NULL AND NOT EXISTS(SELECT 1 FROM pg_attribute + WHERE attrelid=to_regclass('public.'||('math_ticks')) AND attname=('caching_tick') AND NOT attisdropped)) OR (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('math_ticks')) AND a.attname=('caching_tick') AND NOT a.attisdropped + AND (format_type(a.atttypid,a.atttypmod)=('bigint') + -- Exact public bootstrap alternatives to the authoritative legacy contract. + OR (('math_ticks')='worker_tasks' AND ('caching_tick')='task_type' AND ('bigint')='text' + AND format_type(a.atttypid,a.atttypmod)='character varying(99)') + OR (('math_ticks') IN ('pwreset_tokens','password_reset_tokens') + AND ('caching_tick') IN ('token','pwresettoken') AND ('bigint')='character varying(100)' + AND format_type(a.atttypid,a.atttypmod)='character varying(250)') + -- Only historical math payloads have an established json equivalent. + -- Do not adopt arbitrary json columns in newer jsonb contracts. + OR (('bigint')='jsonb' AND a.atttypid='json'::regtype AND ('caching_tick')='data' + AND ('math_ticks') IN ('math_main','math_profile','math_ptptstats','math_cache', + 'math_bidtopid','math_exportstatus'))) + AND ((NULL) IS NULL OR a.attnotnull=(NULL)) + AND ((NULL) IS NULL OR pg_get_expr(d.adbin,d.adrelid)=(NULL))))) + AND to_regprocedure('public.now_as_millis()') IS NOT NULL + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint WHERE conrelid=to_regclass('public.'||('votes_latest_unique')) + AND convalidated AND pg_get_constraintdef(oid)=('UNIQUE (zid, pid, tid)')))) AS catalog_match +UNION ALL +SELECT '000001_update_pwreset_table.sql'::text AS migration, (-- Catalog postconditions only; this file never replays migration DDL. +SELECT to_regclass('public.password_reset_tokens') IS NULL + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('pwreset_tokens')) AND a.attname=('token') AND NOT a.attisdropped + AND (format_type(a.atttypid,a.atttypmod)=('character varying(100)') + -- Exact public bootstrap alternatives to the authoritative legacy contract. + OR (('pwreset_tokens')='worker_tasks' AND ('token')='task_type' AND ('character varying(100)')='text' + AND format_type(a.atttypid,a.atttypmod)='character varying(99)') + OR (('pwreset_tokens') IN ('pwreset_tokens','password_reset_tokens') + AND ('token') IN ('token','pwresettoken') AND ('character varying(100)')='character varying(100)' + AND format_type(a.atttypid,a.atttypmod)='character varying(250)') + -- Only historical math payloads have an established json equivalent. + -- Do not adopt arbitrary json columns in newer jsonb contracts. + OR (('character varying(100)')='jsonb' AND a.atttypid='json'::regtype AND ('token')='data' + AND ('pwreset_tokens') IN ('math_main','math_profile','math_ptptstats','math_cache', + 'math_bidtopid','math_exportstatus'))) + AND ((NULL) IS NULL OR a.attnotnull=(NULL)) + AND ((NULL) IS NULL OR pg_get_expr(d.adbin,d.adrelid)=(NULL)))) + AND (SELECT to_regclass('public.'||('pwreset_tokens')) IS NOT NULL AND NOT EXISTS(SELECT 1 FROM pg_attribute + WHERE attrelid=to_regclass('public.'||('pwreset_tokens')) AND attname=('pwresettoken') AND NOT attisdropped))) AS catalog_match +UNION ALL +SELECT '000002_add_xid_constraint.sql'::text AS migration, (-- Both the public bootstrap variant (owner,xid only) and the authoritative +-- legacy variant (also owner,uid) are supported. Never drop either invariant. +SELECT (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('xids')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('UNIQUE (owner, xid)') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + -- If the optional legacy unique index exists, it must enforce the complete key. + AND NOT EXISTS ( + SELECT 1 FROM pg_index i + JOIN pg_attribute owner ON owner.attrelid=i.indrelid AND owner.attname='owner' + JOIN pg_attribute uid ON uid.attrelid=i.indrelid AND uid.attname='uid' + WHERE i.indrelid=to_regclass('public.xids') AND i.indisunique + AND i.indnkeyatts=2 + AND ARRAY[i.indkey[0],i.indkey[1]] @> ARRAY[owner.attnum,uid.attnum] + AND (NOT i.indisvalid OR NOT i.indisready OR NOT i.indislive + OR NOT i.indimmediate OR i.indpred IS NOT NULL) + )) AS catalog_match +UNION ALL +SELECT '000003_add_origin_permanent_cookie_columns.sql'::text AS migration, (-- Catalog postconditions only; this file never replays migration DDL. +SELECT (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('participants_extended')) AND a.attname=('permanent_cookie') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('character varying(32)') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('participants_extended')) AND a.attname=('origin') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('character varying(9999)') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL)))) AS catalog_match +UNION ALL +SELECT '000004_drop_waitinglist_table.sql'::text AS migration, (-- Catalog postconditions only; this file never replays migration DDL. +SELECT to_regclass('public.waitinglist') IS NULL) AS catalog_match +UNION ALL +SELECT '000005_drop_slack_stripe_canvas.sql'::text AS migration, (-- Catalog postconditions only; this file never replays migration DDL. +SELECT to_regclass('public.slack_oauth_access_tokens') IS NULL + AND to_regclass('public.slack_users') IS NULL + AND to_regclass('public.slack_user_invites') IS NULL + AND to_regclass('public.slack_bot_events') IS NULL + AND to_regclass('public.stripe_accounts') IS NULL + AND to_regclass('public.stripe_subscriptions') IS NULL + AND to_regclass('public.coupons_for_free_upgrades') IS NULL + AND to_regclass('public.lti_users') IS NULL + AND to_regclass('public.lti_context_memberships') IS NULL + AND to_regclass('public.canvas_assignment_callback_info') IS NULL + AND to_regclass('public.canvas_assignment_conversation_info') IS NULL + AND to_regclass('public.lti_oauthv1_credentials') IS NULL + AND (SELECT to_regclass('public.'||('conversations')) IS NOT NULL AND NOT EXISTS(SELECT 1 FROM pg_attribute + WHERE attrelid=to_regclass('public.'||('conversations')) AND attname=('is_slack') AND NOT attisdropped)) + AND (SELECT to_regclass('public.'||('conversations')) IS NOT NULL AND NOT EXISTS(SELECT 1 FROM pg_attribute + WHERE attrelid=to_regclass('public.'||('conversations')) AND attname=('lti_users_only') AND NOT attisdropped)) + AND (SELECT to_regclass('public.'||('users')) IS NOT NULL AND NOT EXISTS(SELECT 1 FROM pg_attribute + WHERE attrelid=to_regclass('public.'||('users')) AND attname=('plan') AND NOT attisdropped))) AS catalog_match +UNION ALL +SELECT '000006_update_votes_rule.sql'::text AS migration, (-- Catalog postconditions only; this file never replays migration DDL. +SELECT EXISTS (SELECT 1 FROM pg_rewrite WHERE ev_class=to_regclass('public.votes') AND rulename='on_vote_insert_update_unique_table' AND ev_enabled='O' AND regexp_replace(pg_get_ruledef(oid), '\s+', ' ', 'g') = 'CREATE RULE on_vote_insert_update_unique_table AS ON INSERT TO public.votes DO INSERT INTO votes_latest_unique (zid, pid, tid, vote, weight_x_32767, modified) VALUES (new.zid, new.pid, new.tid, new.vote, new.weight_x_32767, new.created) ON CONFLICT(zid, pid, tid) DO UPDATE SET vote = excluded.vote, modified = excluded.modified;')) AS catalog_match +UNION ALL +SELECT '000007_drop_geolocation_fields.sql'::text AS migration, (-- Catalog postconditions only; this file never replays migration DDL. +SELECT to_regclass('public.geolocation_cache') IS NULL + AND (SELECT to_regclass('public.'||('participants_extended')) IS NOT NULL AND NOT EXISTS(SELECT 1 FROM pg_attribute + WHERE attrelid=to_regclass('public.'||('participants_extended')) AND attname=('country_code_iso') AND NOT attisdropped)) + AND (SELECT to_regclass('public.'||('participants_extended')) IS NOT NULL AND NOT EXISTS(SELECT 1 FROM pg_attribute + WHERE attrelid=to_regclass('public.'||('participants_extended')) AND attname=('encrypted_maxmind_response_city') AND NOT attisdropped)) + AND (SELECT to_regclass('public.'||('participants_extended')) IS NOT NULL AND NOT EXISTS(SELECT 1 FROM pg_attribute + WHERE attrelid=to_regclass('public.'||('participants_extended')) AND attname=('ip_address') AND NOT attisdropped)) + AND (SELECT to_regclass('public.'||('participants_extended')) IS NOT NULL AND NOT EXISTS(SELECT 1 FROM pg_attribute + WHERE attrelid=to_regclass('public.'||('participants_extended')) AND attname=('latitude') AND NOT attisdropped)) + AND (SELECT to_regclass('public.'||('participants_extended')) IS NOT NULL AND NOT EXISTS(SELECT 1 FROM pg_attribute + WHERE attrelid=to_regclass('public.'||('participants_extended')) AND attname=('location') AND NOT attisdropped)) + AND (SELECT to_regclass('public.'||('participants_extended')) IS NOT NULL AND NOT EXISTS(SELECT 1 FROM pg_attribute + WHERE attrelid=to_regclass('public.'||('participants_extended')) AND attname=('longitude') AND NOT attisdropped)) + AND (SELECT to_regclass('public.'||('participants_extended')) IS NOT NULL AND NOT EXISTS(SELECT 1 FROM pg_attribute + WHERE attrelid=to_regclass('public.'||('participants_extended')) AND attname=('x_forwarded_for') AND NOT attisdropped))) AS catalog_match +UNION ALL +SELECT '000008_add_comment_priority.sql'::text AS migration, (-- Catalog postconditions only; this file never replays migration DDL. +SELECT (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('conversations')) AND a.attname=('importance_enabled') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('boolean') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('false'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('votes')) AND a.attname=('high_priority') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('boolean') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('false')))) AS catalog_match +UNION ALL +SELECT '000009_add_uuid_to_zinvites.sql'::text AS migration, (-- Catalog postconditions only; this file never replays migration DDL. +SELECT (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('zinvites')) AND a.attname=('uuid') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('uuid') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL)))) AS catalog_match +UNION ALL +SELECT '000010_create_oidc_user_mappings.sql'::text AS migration, (-- Catalog postconditions only; this file never replays migration DDL. +SELECT (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('oidc_user_mappings')) AND a.attname=('oidc_sub') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('character varying(255)') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('oidc_user_mappings')) AND a.attname=('uid') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('oidc_user_mappings')) AND a.attname=('created') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('bigint') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('now_as_millis()'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('oidc_user_mappings')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('PRIMARY KEY (oidc_sub)') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('oidc_user_mappings')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('UNIQUE (uid)') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('oidc_user_mappings')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('FOREIGN KEY (uid) REFERENCES users(uid) ON DELETE CASCADE') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_oidc_mappings_uid')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_oidc_mappings_uid ON public.oidc_user_mappings USING btree (uid)')))) AS catalog_match +UNION ALL +SELECT '000011_alter_suzinvites_xid_to_text.sql'::text AS migration, (-- Catalog postconditions only; this file never replays migration DDL. +SELECT (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('suzinvites')) AND a.attname=('xid') AND NOT a.attisdropped + AND (format_type(a.atttypid,a.atttypmod)=('text') + -- Exact public bootstrap alternatives to the authoritative legacy contract. + OR (('suzinvites')='worker_tasks' AND ('xid')='task_type' AND ('text')='text' + AND format_type(a.atttypid,a.atttypmod)='character varying(99)') + OR (('suzinvites') IN ('pwreset_tokens','password_reset_tokens') + AND ('xid') IN ('token','pwresettoken') AND ('text')='character varying(100)' + AND format_type(a.atttypid,a.atttypmod)='character varying(250)') + -- Only historical math payloads have an established json equivalent. + -- Do not adopt arbitrary json columns in newer jsonb contracts. + OR (('text')='jsonb' AND a.atttypid='json'::regtype AND ('xid')='data' + AND ('suzinvites') IN ('math_main','math_profile','math_ptptstats','math_cache', + 'math_bidtopid','math_exportstatus'))) + AND ((true) IS NULL OR a.attnotnull=(true)) + AND ((NULL) IS NULL OR pg_get_expr(d.adbin,d.adrelid)=(NULL))))) AS catalog_match +UNION ALL +SELECT '000012_create_topic_agenda_selections.sql'::text AS migration, (-- Catalog postconditions only; this file never replays migration DDL. +SELECT (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('topic_agenda_selections')) AND a.attname=('zid') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('topic_agenda_selections')) AND a.attname=('pid') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('topic_agenda_selections')) AND a.attname=('archetypal_selections') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('jsonb') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('''[]''::jsonb'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('topic_agenda_selections')) AND a.attname=('delphi_job_id') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('text') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('topic_agenda_selections')) AND a.attname=('total_selections') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('0'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('topic_agenda_selections')) AND a.attname=('created_at') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('timestamp with time zone') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('CURRENT_TIMESTAMP'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('topic_agenda_selections')) AND a.attname=('updated_at') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('timestamp with time zone') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('CURRENT_TIMESTAMP'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('topic_agenda_selections')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('PRIMARY KEY (zid, pid)') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('topic_agenda_selections')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('FOREIGN KEY (zid) REFERENCES conversations(zid) ON DELETE CASCADE') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('topic_agenda_selections')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('FOREIGN KEY (zid, pid) REFERENCES participants(zid, pid) ON DELETE CASCADE') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_topic_agenda_selections_zid')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_topic_agenda_selections_zid ON public.topic_agenda_selections USING btree (zid)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_topic_agenda_selections_pid')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_topic_agenda_selections_pid ON public.topic_agenda_selections USING btree (pid)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_topic_agenda_selections_delphi_job_id')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_topic_agenda_selections_delphi_job_id ON public.topic_agenda_selections USING btree (delphi_job_id)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_topic_agenda_selections_created_at')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_topic_agenda_selections_created_at ON public.topic_agenda_selections USING btree (created_at)')))) AS catalog_match +UNION ALL +SELECT '000013_create_treevite.sql'::text AS migration, (-- Catalog postconditions only; this file never replays migration DDL. +SELECT (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('conversations')) AND a.attname=('treevite_enabled') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('boolean') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('false'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_waves')) AND a.attname=('id') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('bigint') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('nextval(''treevite_waves_id_seq''::regclass)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_waves')) AND a.attname=('zid') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_waves')) AND a.attname=('wave') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_waves')) AND a.attname=('parent_wave') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_waves')) AND a.attname=('size') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_waves')) AND a.attname=('invites_per_user') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_waves')) AND a.attname=('owner_invites') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('0'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_waves')) AND a.attname=('created_at') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('timestamp with time zone') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('CURRENT_TIMESTAMP'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_waves')) AND a.attname=('updated_at') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('timestamp with time zone') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('CURRENT_TIMESTAMP'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_waves')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('PRIMARY KEY (id)') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_invites')) AND a.attname=('id') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('bigint') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('nextval(''treevite_invites_id_seq''::regclass)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_invites')) AND a.attname=('zid') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_invites')) AND a.attname=('wave_id') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('bigint') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_invites')) AND a.attname=('parent_invite_id') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('bigint') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_invites')) AND a.attname=('status') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('smallint') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('0'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_invites')) AND a.attname=('invite_owner_pid') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_invites')) AND a.attname=('invite_used_by_pid') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_invites')) AND a.attname=('invite_used_at') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('timestamp with time zone') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_invites')) AND a.attname=('created_at') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('timestamp with time zone') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('CURRENT_TIMESTAMP'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_invites')) AND a.attname=('updated_at') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('timestamp with time zone') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('CURRENT_TIMESTAMP'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_invites')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('PRIMARY KEY (id)') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_login_codes')) AND a.attname=('id') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('bigint') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('nextval(''treevite_login_codes_id_seq''::regclass)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_login_codes')) AND a.attname=('zid') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_login_codes')) AND a.attname=('pid') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_login_codes')) AND a.attname=('login_code_hash') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('text') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_login_codes')) AND a.attname=('fp_kid') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('smallint') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('1'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_login_codes')) AND a.attname=('revoked') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('boolean') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('false'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_login_codes')) AND a.attname=('expires_at') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('timestamp with time zone') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_login_codes')) AND a.attname=('last_used_at') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('timestamp with time zone') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_login_codes')) AND a.attname=('created_at') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('timestamp with time zone') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('CURRENT_TIMESTAMP'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_login_codes')) AND a.attname=('updated_at') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('timestamp with time zone') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('CURRENT_TIMESTAMP'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_login_codes')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('PRIMARY KEY (id)') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_waves')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('UNIQUE (zid, wave)') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_waves')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('FOREIGN KEY (zid) REFERENCES conversations(zid) ON DELETE CASCADE') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_invites')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('UNIQUE (zid, invite_code)') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_invites')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('FOREIGN KEY (zid) REFERENCES conversations(zid) ON DELETE CASCADE') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_invites')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('FOREIGN KEY (wave_id) REFERENCES treevite_waves(id) ON DELETE CASCADE') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_invites')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('FOREIGN KEY (parent_invite_id) REFERENCES treevite_invites(id) ON DELETE SET NULL') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_invites')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('FOREIGN KEY (zid, invite_owner_pid) REFERENCES participants(zid, pid)') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_invites')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('FOREIGN KEY (zid, invite_used_by_pid) REFERENCES participants(zid, pid)') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_login_codes')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('FOREIGN KEY (zid, pid) REFERENCES participants(zid, pid) ON DELETE CASCADE') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_login_codes')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('UNIQUE (zid, pid)') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_login_codes')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('UNIQUE (zid, login_code_fingerprint)') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_login_codes')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('UNIQUE (zid, login_code_lookup)') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_treevite_waves_zid')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_treevite_waves_zid ON public.treevite_waves USING btree (zid)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_treevite_waves_wave')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_treevite_waves_wave ON public.treevite_waves USING btree (wave)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_treevite_waves_parent')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_treevite_waves_parent ON public.treevite_waves USING btree (zid, parent_wave)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_treevite_invites_zid')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_treevite_invites_zid ON public.treevite_invites USING btree (zid)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_treevite_invites_zid_status')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_treevite_invites_zid_status ON public.treevite_invites USING btree (zid, status)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_treevite_invites_wave_id')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_treevite_invites_wave_id ON public.treevite_invites USING btree (wave_id)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_treevite_invites_parent')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_treevite_invites_parent ON public.treevite_invites USING btree (parent_invite_id)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_treevite_invites_owner_pid')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_treevite_invites_owner_pid ON public.treevite_invites USING btree (invite_owner_pid)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_treevite_invites_used_by_pid')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_treevite_invites_used_by_pid ON public.treevite_invites USING btree (invite_used_by_pid)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_treevite_invites_code')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_treevite_invites_code ON public.treevite_invites USING btree (invite_code)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_treevite_login_codes_zid')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_treevite_login_codes_zid ON public.treevite_login_codes USING btree (zid)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_treevite_login_codes_pid')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_treevite_login_codes_pid ON public.treevite_login_codes USING btree (pid)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_treevite_login_codes_fp')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_treevite_login_codes_fp ON public.treevite_login_codes USING btree (login_code_fingerprint)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_treevite_login_codes_lookup')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_treevite_login_codes_lookup ON public.treevite_login_codes USING btree (zid, login_code_lookup)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_invites')) AND a.attname=('invite_code') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('character varying(64)') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_login_codes')) AND a.attname=('login_code_fingerprint') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('character varying(128)') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('treevite_login_codes')) AND a.attname=('login_code_lookup') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('character varying(128)') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_waves')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('CHECK (((invites_per_user > 0) OR (owner_invites > 0)))') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_waves')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('CHECK (((parent_wave IS NULL) OR (parent_wave >= 0)))') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_waves')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('CHECK (((size IS NULL) OR (size >= 0)))') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_waves')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('CHECK ((invites_per_user >= 0))') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_waves')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('CHECK ((owner_invites >= 0))') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_waves')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('CHECK ((wave >= 1))') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('treevite_invites')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('CHECK ((status = ANY (ARRAY[0, 1, 2, 3])))') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O')))) AS catalog_match +UNION ALL +SELECT '000014_alter_reports_modlevel.sql'::text AS migration, (-- Catalog postconditions only; this file never replays migration DDL. +SELECT (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('reports')) AND a.attname=('mod_level') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('smallint') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('''-2''::integer')))) AS catalog_match +UNION ALL +SELECT '000015_add_xid_requirements.sql'::text AS migration, (-- Catalog postconditions only; this file never replays migration DDL. +SELECT (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('conversations')) AND a.attname=('xid_required') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('boolean') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('false'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('xid_whitelist')) AND a.attname=('zid') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('xids')) AND a.attname=('zid') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('xids')) AND a.attname=('pid') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('xid_whitelist')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('FOREIGN KEY (zid) REFERENCES conversations(zid) ON DELETE CASCADE') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('xids')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('FOREIGN KEY (zid) REFERENCES conversations(zid) ON DELETE CASCADE') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('xids')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('FOREIGN KEY (zid, pid) REFERENCES participants(zid, pid) ON DELETE SET NULL') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_xid_whitelist_zid')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_xid_whitelist_zid ON public.xid_whitelist USING btree (zid)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_xid_whitelist_xid')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_xid_whitelist_xid ON public.xid_whitelist USING btree (xid)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_xids_zid')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_xids_zid ON public.xids USING btree (zid)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_xids_xid')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_xids_xid ON public.xids USING btree (xid)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_xids_pid')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_xids_pid ON public.xids USING btree (pid)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_xids_zid_xid')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_xids_zid_xid ON public.xids USING btree (zid, xid)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_xids_uid_zid')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_xids_uid_zid ON public.xids USING btree (uid, zid)')))) AS catalog_match +UNION ALL +SELECT '000016_add_orig_id.sql'::text AS migration, (-- Catalog postconditions only; this file never replays migration DDL. +SELECT (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('comments')) AND a.attname=('original_id') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('uuid') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL)))) AS catalog_match +UNION ALL +SELECT '000017_create_byod_job_table.sql'::text AS migration, (-- Catalog postconditions only; this file never replays migration DDL. +SELECT (SELECT array_agg(enumlabel::text ORDER BY enumsortorder) FROM pg_enum WHERE enumtypid=to_regtype('public.job_status')) = ARRAY['pending','processing','completed','failed'] + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('byod_import_jobs')) AND a.attname=('id') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('nextval(''byod_import_jobs_id_seq''::regclass)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('byod_import_jobs')) AND a.attname=('zid') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('integer') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('byod_import_jobs')) AND a.attname=('s3_key') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('text') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('byod_import_jobs')) AND a.attname=('status') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('job_status') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('''pending''::job_status'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('byod_import_jobs')) AND a.attname=('stage') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('text') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('''init''::text'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('byod_import_jobs')) AND a.attname=('error_message') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('text') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM (NULL))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('byod_import_jobs')) AND a.attname=('created_at') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('timestamp with time zone') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('now()'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('byod_import_jobs')) AND a.attname=('updated_at') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('timestamp with time zone') AND a.attnotnull=(false) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('now()'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_constraint c + WHERE c.conrelid=to_regclass('public.'||('byod_import_jobs')) + AND c.convalidated AND NOT c.condeferrable AND NOT c.condeferred + AND pg_get_constraintdef(c.oid)=('PRIMARY KEY (id)') + AND (c.contype NOT IN ('p','u') OR EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indexrelid=c.conindid + AND i.indisvalid AND i.indisready AND i.indislive AND i.indimmediate)) + AND NOT EXISTS (SELECT 1 FROM pg_trigger tr + WHERE tr.tgconstraint=c.oid AND tr.tgenabled <> 'O'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('idx_byod_jobs_zid')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX idx_byod_jobs_zid ON public.byod_import_jobs USING btree (zid)')))) AS catalog_match +UNION ALL +SELECT '000018_add_topics_enabled.sql'::text AS migration, (-- Catalog postconditions only; this file never replays migration DDL. +SELECT (SELECT EXISTS(SELECT 1 FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid=to_regclass('public.'||('conversations')) AND a.attname=('topics_enabled') AND NOT a.attisdropped + AND format_type(a.atttypid,a.atttypmod)=('boolean') AND a.attnotnull=(true) + AND a.attidentity='' AND a.attgenerated='' + AND pg_get_expr(d.adbin,d.adrelid) IS NOT DISTINCT FROM ('false')))) AS catalog_match +UNION ALL +SELECT '000022_add_poll_timestamp_indexes.sql'::text AS migration, (-- Catalog postconditions only; this file never replays migration DDL. +SELECT (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('votes_created_idx')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX votes_created_idx ON public.votes USING btree (created)'))) + AND (SELECT EXISTS(SELECT 1 FROM pg_index WHERE indexrelid=to_regclass('public.'||('comments_modified_idx')) + AND indisvalid AND indisready AND pg_get_indexdef(indexrelid)=('CREATE INDEX comments_modified_idx ON public.comments USING btree (modified)')))) AS catalog_match +), scope AS ( + SELECT to_regclass('public.migrations') IS NULL + AND to_regclass('public.schema_migrations') IS NULL + AND NOT EXISTS (SELECT 1 FROM pg_class WHERE relnamespace='public'::regnamespace + AND (starts_with(relname,'polis_queue_') OR starts_with(relname,'delphi_'))) + AND NOT EXISTS (SELECT 1 FROM pg_proc WHERE pronamespace='public'::regnamespace + AND (starts_with(proname,'pq_') OR starts_with(proname,'pd_'))) AS initial_state +), authority AS ( + SELECT + has_database_privilege(current_user,current_database(),'TEMP') + AND has_schema_privilege(current_user,'public','USAGE') + AND has_schema_privilege(current_user,'public','CREATE') AS reconcile_authority, + EXISTS (SELECT 1 FROM pg_roles WHERE rolname=current_user AND (rolsuper OR rolcreaterole)) AS create_roles, + NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname IN ('polis_queue_owner','polis_queue_executor')) AS queue_roles_absent, + has_schema_privilege(current_user,'public','USAGE WITH GRANT OPTION') + AND has_schema_privilege(current_user,'public','CREATE WITH GRANT OPTION') AS schema_grants, + EXISTS (SELECT 1 FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace + WHERE n.nspname='public' AND c.relname='conversations' + AND has_table_privilege(current_user,c.oid,'SELECT WITH GRANT OPTION') + AND has_column_privilege(current_user,c.oid,'topic','UPDATE WITH GRANT OPTION') + AND has_column_privilege(current_user,c.oid,'zid','REFERENCES WITH GRANT OPTION')) AS table_grants +), readiness AS ( + SELECT initial_state AND (SELECT bool_and(catalog_match) FROM checks) + AND current_setting('server_version_num')::integer >= 170000 AS can_reconcile, + create_roles AND queue_roles_absent AND schema_grants AND table_grants AS can_provision, + initial_state, reconcile_authority, create_roles, queue_roles_absent, schema_grants, table_grants + FROM scope CROSS JOIN authority +), output AS ( + SELECT migration, catalog_match, + CASE WHEN NOT initial_state THEN 'REVIEW_EXISTING_LEDGER_OR_QUEUE' + WHEN NOT catalog_match THEN 'CATALOG_MISMATCH' + WHEN NOT can_reconcile THEN 'BLOCKED_BY_OTHER_CATALOG_CHECK' + WHEN NOT reconcile_authority THEN 'REVIEW_MIGRATION_SESSION_AUTHORITY' + ELSE 'WOULD_ADOPT' END AS outcome + FROM checks CROSS JOIN readiness + UNION ALL + SELECT migration, NULL::boolean, + CASE WHEN NOT can_reconcile THEN 'BLOCKED_RECONCILIATION' + WHEN NOT reconcile_authority OR NOT can_provision THEN 'REVIEW_MIGRATION_SESSION_AUTHORITY' + ELSE 'WOULD_APPLY' END + FROM (VALUES ('000019_create_polis_queue.sql'),('000023_create_delphi_foundation.sql'), + ('000024_create_polis_queue_large_class.sql'), + ('000027_create_sealed_job_graphs.sql'), + ('000028_create_delphi_results.sql'), + ('000029_extend_delphi_graph_stages.sql')) p(migration) CROSS JOIN readiness + UNION ALL + SELECT migration, NULL::boolean, 'OUTSIDE_RELEASE' + FROM (VALUES ('000020'),('000021'),('000025'),('000026')) p(migration) +) +SELECT migration, catalog_match, outcome FROM output ORDER BY migration; +ROLLBACK; diff --git a/server/src/config.ts b/server/src/config.ts index 30167382fc..2b75faeed4 100644 --- a/server/src/config.ts +++ b/server/src/config.ts @@ -30,6 +30,11 @@ import("source-map-support").then((sourceMapSupport) => { }); export default { + // Delphi readers remain on DynamoDB unless explicitly selected. Keep these + // reads at the shared configuration boundary, including test-time selection. + get delphiResultBackend(): string | undefined { return process.env.DELPHI_RESULT_BACKEND; }, + get delphiResultEnv(): string | undefined { return process.env.DELPHI_RESULT_ENV; }, + get delphiResultScope(): string | undefined { return process.env.DELPHI_RESULT_SCOPE; }, domainOverride, isDevMode: devMode, reachableErrorHandler, diff --git a/server/src/db/migrations.cjs b/server/src/db/migrations.cjs new file mode 100644 index 0000000000..d7e6091776 --- /dev/null +++ b/server/src/db/migrations.cjs @@ -0,0 +1,62 @@ +/* eslint-disable no-restricted-properties -- Bootstrap must read its environment before config.ts and application modules load. */ +// Read-only startup gate. Uses the same raw SQL hashes and release-wide hold +// as polis-migrate; no application modules or background loops load before it. +const fs = require("node:fs"); +const path = require("node:path"); +const crypto = require("node:crypto"); +const { Client } = require("pg"); +const isTrue = require("boolean"); + +async function checkMigrations() { + const dir = process.env.POLIS_MIGRATIONS_DIR || path.resolve("postgres/migrations"); + function manifest(file) { + const names = new Set(); + for (const name of fs.readFileSync(path.join(dir, file), "utf8").split(/\r?\n/).filter((s) => s && !s.startsWith("#"))) { + if (name.trim() !== name || !/^\d{6}_[a-zA-Z0-9_]+\.sql$/.test(name) || names.has(name)) throw new Error(`Invalid or duplicate migration in ${file}: ${name}`); + names.add(name); + } + return names; + } + const selected = manifest("release.txt"); + const held = manifest("held.txt"); + if ([...selected].some((name) => held.has(name))) throw new Error("Release and held manifests overlap"); + const required = new Map(); + const versions = new Set(); + for (const name of fs.readdirSync(dir).sort()) { + if (!name.endsWith(".sql")) continue; + if (name.trim() !== name || !/^\d{6}_[a-zA-Z0-9_]+\.sql$/.test(name) || versions.has(name.slice(0, 6))) { + throw new Error(`Invalid or duplicate migration: ${name}`); + } + versions.add(name.slice(0, 6)); + if (!fs.lstatSync(path.join(dir, name)).isFile()) throw new Error(`Non-file migration: ${name}`); + if (held.has(name)) { held.delete(name); continue; } + if (!selected.delete(name)) throw new Error(`Migration absent from release/held manifest: ${name}`); + required.set(name, crypto.createHash("sha256").update(fs.readFileSync(path.join(dir, name))).digest("hex")); + } + if (held.size || selected.size || !required.has("000000_initial.sql")) throw new Error("Incomplete migration source directory"); + if (!process.env.DATABASE_URL) throw new Error("DATABASE_URL is required for migration check"); + const db = new Client({ connectionString: process.env.DATABASE_URL, + connectionTimeoutMillis: 10000, application_name: "polis-startup-migrations", + ssl: isTrue(process.env.DATABASE_SSL) ? { rejectUnauthorized: true } : undefined }); + try { + await db.connect(); + await db.query("BEGIN READ ONLY"); + await db.query("SET LOCAL statement_timeout='10s'; SET LOCAL lock_timeout='5s'"); + const { rows } = await db.query("SELECT name,checksum,status FROM public.migrations ORDER BY name"); + for (const row of rows) { + if (!required.has(row.name) || required.get(row.name) !== row.checksum || !["APPLIED", "ADOPTED"].includes(row.status)) { + throw new Error(`Migration history mismatch: ${row.name}`); + } + required.delete(row.name); + } + if (required.size) throw new Error(`Pending migrations: ${[...required.keys()].join(", ")}; run polis-migrate apply`); + await db.query("COMMIT"); + } catch (error) { + // Database DETAIL may contain data. Print only our own errors or SQLSTATE. + if (error.code) throw new Error(`Migration check failed (SQLSTATE ${error.code}); run polis-migrate check; reconcile legacy databases per docs/migrations.md`); + throw error; + } finally { await db.end(); } +} +module.exports = { checkMigrations }; +if (require.main === module) checkMigrations().then(() => process.stdout.write("migration check ready\n")) + .catch((error) => { process.stderr.write(`${error.message}\n`); process.exitCode = 1; }); diff --git a/server/src/db/pg-query.ts b/server/src/db/pg-query.ts index d982ef48a5..7252ae7916 100644 --- a/server/src/db/pg-query.ts +++ b/server/src/db/pg-query.ts @@ -257,6 +257,19 @@ function connectReadOnly() { return readPool.connect(); } +// Result requests hold a repeatable-read snapshot while other route work may +// borrow the primary pool. A separate bounded pool prevents pool starvation. +let resultSnapshotPool: Pool | undefined; +function connectResultSnapshot() { + if (!resultSnapshotPool) { + resultSnapshotPool = new Pool({ + ...pgConnection, max: 2, connectionTimeoutMillis: 30000, + } as unknown as PoolConfig); + resultSnapshotPool.on("error", error => logger.error("pg_result_snapshot_pool", error)); + } + return resultSnapshotPool.connect(); +} + // Session policy applied immediately after BEGIN, from // cost-reduction/04-plans/P-024-queue-substrate.md. These are declared initial // bounds for the queue substrate, not a general-purpose transaction profile; @@ -372,5 +385,6 @@ export default { stream_queryP_readOnly, connect, connectReadOnly, + connectResultSnapshot, withTransaction, }; diff --git a/server/src/nextComment.ts b/server/src/nextComment.ts index 50b1b5df4f..bbdc169bf8 100644 --- a/server/src/nextComment.ts +++ b/server/src/nextComment.ts @@ -1,5 +1,6 @@ import _ from "underscore"; import LruCache from "lru-cache"; +import { resultClient } from "./utils/delphiResults"; import { DynamoDBClient, DynamoDBClientConfig } from "@aws-sdk/client-dynamodb"; import { DynamoDBDocumentClient, QueryCommand } from "@aws-sdk/lib-dynamodb"; @@ -38,13 +39,12 @@ if (Config.dynamoDbEndpoint) { }; } -const dynamoClient = new DynamoDBClient(dynamoDBConfig); -const dynamoDocClient = DynamoDBDocumentClient.from(dynamoClient, { +const dynamoDocClient = resultClient(() => DynamoDBDocumentClient.from(new DynamoDBClient(dynamoDBConfig), { marshallOptions: { convertEmptyValues: true, removeUndefinedValues: true, }, -}); +})); const DELPHI_TOPIC_NAMES_TABLE = "Delphi_CommentClustersLLMTopicNames"; // This very much follows the outline of the random selection above, but factors out the probabilistic logic diff --git a/server/src/ops/delphiTopicNames.ts b/server/src/ops/delphiTopicNames.ts index 2d1f8840a2..2e6ce3b9d4 100644 --- a/server/src/ops/delphiTopicNames.ts +++ b/server/src/ops/delphiTopicNames.ts @@ -12,6 +12,7 @@ // instance role. The instance role already has dynamodb:Query on Delphi_*. import { DynamoDBDocumentClient, QueryCommand } from "@aws-sdk/lib-dynamodb"; +import { resultClient } from "../utils/delphiResults"; import { makeDynamoClient } from "../utils/dynamoClient"; export const TOPIC_NAMES_TABLE = "Delphi_CommentClustersLLMTopicNames"; @@ -92,7 +93,7 @@ export function makeDelphiTopicNameReader(): TopicNameReader { const out: TopicNames = new Map(); if (zids.length === 0) return out; try { - doc = doc || DynamoDBDocumentClient.from(makeDynamoClient()); + doc = doc || resultClient(() => DynamoDBDocumentClient.from(makeDynamoClient())); } catch { for (const zid of zids) out.set(zid, null); return out; diff --git a/server/src/routes/api/v3/feeds.ts b/server/src/routes/api/v3/feeds.ts index 2915e291bc..81d72e9dd9 100644 --- a/server/src/routes/api/v3/feeds.ts +++ b/server/src/routes/api/v3/feeds.ts @@ -1,3 +1,4 @@ +import { resultClient } from "../../../utils/delphiResults"; import { Request, Response } from "express"; import logger from "../../../utils/logger"; import { DynamoDBClient } from "@aws-sdk/client-dynamodb"; @@ -26,13 +27,12 @@ if (Config.dynamoDbEndpoint) { logger.info(`Using default AWS credential provider chain`); } } -const client = new DynamoDBClient(dynamoDBConfig); -const docClient = DynamoDBDocumentClient.from(client, { +const docClient = resultClient(() => DynamoDBDocumentClient.from(new DynamoDBClient(dynamoDBConfig), { marshallOptions: { convertEmptyValues: true, removeUndefinedValues: true, }, -}); +})); /** * Handler for feeds directory listing - shows available feeds for a report diff --git a/server/src/routes/collectiveStatement.ts b/server/src/routes/collectiveStatement.ts index d88f7e9056..5314584178 100644 --- a/server/src/routes/collectiveStatement.ts +++ b/server/src/routes/collectiveStatement.ts @@ -1,3 +1,4 @@ +import { resultClient } from "../utils/delphiResults"; import { Request, Response } from "express"; import logger from "../utils/logger"; import { DynamoDBClient } from "@aws-sdk/client-dynamodb"; @@ -32,13 +33,12 @@ if (Config.dynamoDbEndpoint) { }; } -const client = new DynamoDBClient(dynamoDBConfig); -const docClient = DynamoDBDocumentClient.from(client, { +const docClient = resultClient(() => DynamoDBDocumentClient.from(new DynamoDBClient(dynamoDBConfig), { marshallOptions: { convertEmptyValues: true, removeUndefinedValues: true, }, -}); +})); const anthropic = Config.anthropicApiKey ? new Anthropic({ diff --git a/server/src/routes/delphi.ts b/server/src/routes/delphi.ts index bc439779c8..89a53443e5 100644 --- a/server/src/routes/delphi.ts +++ b/server/src/routes/delphi.ts @@ -1,3 +1,4 @@ +import { resultClient } from "../utils/delphiResults"; import { Request, Response } from "express"; import logger from "../utils/logger"; import { DynamoDBClient } from "@aws-sdk/client-dynamodb"; @@ -28,13 +29,12 @@ if (Config.dynamoDbEndpoint) { }; } -const client = new DynamoDBClient(dynamoDBConfig); -const docClient = DynamoDBDocumentClient.from(client, { +const docClient = resultClient(() => DynamoDBDocumentClient.from(new DynamoDBClient(dynamoDBConfig), { marshallOptions: { convertEmptyValues: true, removeUndefinedValues: true, }, -}); +})); /** * Handler for Delphi API route that retrieves LLM topic names from DynamoDB diff --git a/server/src/routes/delphi/jobGuard.ts b/server/src/routes/delphi/jobGuard.ts index 632676d129..7d45674031 100644 --- a/server/src/routes/delphi/jobGuard.ts +++ b/server/src/routes/delphi/jobGuard.ts @@ -1,3 +1,4 @@ +import Config from "../../config"; /** * Server-side active-work deduplication for Delphi job submission (P-003 S3). * @@ -46,6 +47,8 @@ import { createHash, randomUUID } from "crypto"; import { DynamoDB, DynamoDBClientConfig } from "@aws-sdk/client-dynamodb"; import { DynamoDBDocument } from "@aws-sdk/lib-dynamodb"; import logger from "../../utils/logger"; +import pg from "../../db/pg-query"; +import { postgresResults } from "../../utils/delphiResults"; import { AwsCredentialsConfigurationError, buildDynamoClientConfig, @@ -968,6 +971,9 @@ export async function assessConversationLiveness( /** Rows as the first sweep read them, for callers needing more than liveness. */ rowsByJobId: Map; }> { + if (postgresResults()) { + return assessPostgresConversationLiveness(conversationId); + } const liveByJobId = new Map(); let rowsByJobId = new Map(); @@ -1047,6 +1053,58 @@ export async function assessConversationLiveness( return { complete: true, liveByJobId, rowsByJobId }; } +/** One primary-database snapshot covers graph peers, descendants, provider work + * and process-exit receipts. Archived metadata never becomes executable work. */ +async function assessPostgresConversationLiveness(conversationId: string) { + const liveByJobId = new Map(); + const rowsByJobId = new Map(); + const env = Config.delphiResultEnv; + if (!env) throw new Error("DELPHI_RESULT_ENV is required for Postgres results"); + const scope = Config.delphiResultScope || null; + try { + const rows = await pg.queryP(` + WITH RECURSIVE selected AS ( + SELECT j.*, n.graph_id FROM public.delphi_jobs j + LEFT JOIN public.delphi_graph_nodes n USING (env,job_id) + LEFT JOIN public.delphi_graphs g ON g.env=n.env AND g.graph_id=n.graph_id + WHERE j.env=$1 AND j.zid::text=$2 AND ($3::text IS NULL OR g.scope_key=$3) + ), related(root_id,job_id) AS ( + SELECT s.job_id,s.job_id FROM selected s + UNION + SELECT r.root_id,c.job_id FROM related r + JOIN public.delphi_jobs c ON c.env=$1 AND c.parent_job_id=r.job_id + ), work AS ( + SELECT root_id,job_id FROM related + UNION + SELECT s.job_id,n.job_id FROM selected s + JOIN public.delphi_graph_nodes n ON n.env=s.env AND n.graph_id=s.graph_id + ) + SELECT v.*, EXISTS ( + SELECT 1 FROM work w + JOIN public.delphi_jobs j ON j.env=$1 AND j.job_id=w.job_id + LEFT JOIN public.polis_queue_jobs q ON q.env=j.env AND q.job_id=j.job_id + WHERE w.root_id=s.job_id AND ( + COALESCE(q.state,j.status) NOT IN ('succeeded','dead','cancelled') + OR EXISTS (SELECT 1 FROM public.polis_queue_attempts a + WHERE a.env=j.env AND a.job_id=j.job_id AND a.process_exit_confirmed_at IS NULL) + OR EXISTS (SELECT 1 FROM public.delphi_provider_requests p + WHERE p.env=j.env AND p.job_id=j.job_id + AND p.state IN ('intent','submission_unknown','submitted')) + ) + ) AS work_live + FROM selected s JOIN public.delphi_result_jobs v ON v.env=s.env AND v.job_id=s.job_id::text + ORDER BY v.job_id`, [env, conversationId, scope]) as any[]; + for (const row of rows) { + liveByJobId.set(row.job_id, row.work_live !== false); + rowsByJobId.set(row.job_id, row); + } + return { complete: true, liveByJobId, rowsByJobId }; + } catch (error: any) { + logger.warn(`Postgres Delphi liveness unavailable: ${error?.message || error}`); + return { complete: false, liveByJobId, rowsByJobId }; + } +} + /** * A cheap fingerprint of everything about a row that would change the answer. * Two reads that produce the same anchor were not separated by a write. diff --git a/server/src/routes/delphi/reports.ts b/server/src/routes/delphi/reports.ts index f3fd0d95b4..119d4d7c20 100644 --- a/server/src/routes/delphi/reports.ts +++ b/server/src/routes/delphi/reports.ts @@ -1,3 +1,4 @@ +import { resultClient } from "../../utils/delphiResults"; import { Request, Response } from "express"; import logger from "../../utils/logger"; import { DynamoDBClient } from "@aws-sdk/client-dynamodb"; @@ -26,13 +27,12 @@ if (Config.dynamoDbEndpoint) { logger.info(`Using default AWS credential provider chain`); } } -const client = new DynamoDBClient(dynamoDBConfig); -const docClient = DynamoDBDocumentClient.from(client, { +const docClient = resultClient(() => DynamoDBDocumentClient.from(new DynamoDBClient(dynamoDBConfig), { marshallOptions: { convertEmptyValues: true, removeUndefinedValues: true, }, -}); +})); /** * Handler for Delphi API route that retrieves LLM-generated reports from DynamoDB diff --git a/server/src/routes/delphi/topicAgenda.ts b/server/src/routes/delphi/topicAgenda.ts index aa91b51cef..69b45e9878 100644 --- a/server/src/routes/delphi/topicAgenda.ts +++ b/server/src/routes/delphi/topicAgenda.ts @@ -1,3 +1,5 @@ +import { resultClient, postgresResults } from "../../utils/delphiResults"; +import { resultQuery } from "../../utils/delphiResultSnapshot"; import _ from "underscore"; import { DynamoDBClient } from "@aws-sdk/client-dynamodb"; import { @@ -30,13 +32,12 @@ if (Config.dynamoDbEndpoint) { }; } -const dynamoClient = new DynamoDBClient(dynamoDBConfig); -const docClient = DynamoDBDocumentClient.from(dynamoClient, { +const docClient = resultClient(() => DynamoDBDocumentClient.from(new DynamoDBClient(dynamoDBConfig), { marshallOptions: { convertEmptyValues: true, removeUndefinedValues: true, }, -}); +})); // Bound the work one participant save can trigger. Without a cap the query is // bounded only by the conversation's partition size; with COALESCE in place, @@ -47,8 +48,17 @@ const JOB_QUERY_PAGE_SIZE = 25; /** * Get the newest completed Delphi job ID for a conversation. */ -async function getCurrentDelphiJobId(zid: string): Promise { +export async function getCurrentDelphiJobId(zid: string): Promise { try { + if (postgresResults()) { + const env=Config.delphiResultEnv; + if (!env) throw new Error("DELPHI_RESULT_ENV is required for Postgres results"); + const scope=Config.delphiResultScope; + const rows=await resultQuery<{job_id:string}>(`SELECT job_id FROM public.delphi_result_publications + WHERE env=$1 AND zid=$2 ${scope ? "AND scope_key=$3" : ""}`,scope ? [env,Number(zid),scope] : [env,Number(zid)]); + if (rows.length>1) throw new Error("Ambiguous published job; configure DELPHI_RESULT_SCOPE"); + return rows[0]?.job_id || null; + } // Query the ConversationIndex GSI to find completed jobs for this conversation const queryParams: QueryCommandInput = { TableName: "Delphi_JobQueue", @@ -83,7 +93,7 @@ async function getCurrentDelphiJobId(zid: string): Promise { return null; } catch (error: any) { - logger.error("Error getting current Delphi job ID from DynamoDB", error); + logger.error("Error getting current published Delphi job ID", error); // Degrade instead of failing the request: a null here cannot erase an // existing attribution (the writes below COALESCE it), while throwing // would 500 the handler and drop the participant's selections entirely. diff --git a/server/src/routes/delphi/topicMod.ts b/server/src/routes/delphi/topicMod.ts index faa7cf0276..755e5b1784 100644 --- a/server/src/routes/delphi/topicMod.ts +++ b/server/src/routes/delphi/topicMod.ts @@ -1,3 +1,4 @@ +import { resultClient } from "../../utils/delphiResults"; import { Request, Response } from "express"; import logger from "../../utils/logger"; import { DynamoDBClient, DynamoDBClientConfig } from "@aws-sdk/client-dynamodb"; @@ -32,13 +33,12 @@ if (Config.dynamoDbEndpoint) { } } -const client = new DynamoDBClient(dynamoDBConfig); -const docClient = DynamoDBDocumentClient.from(client, { +const docClient = resultClient(() => DynamoDBDocumentClient.from(new DynamoDBClient(dynamoDBConfig), { marshallOptions: { convertEmptyValues: true, removeUndefinedValues: true, }, -}); +})); /** * Two of the tables this file reads have never existed in any environment. diff --git a/server/src/routes/delphi/topics.ts b/server/src/routes/delphi/topics.ts index 690b6608b9..2b5c9a9820 100644 --- a/server/src/routes/delphi/topics.ts +++ b/server/src/routes/delphi/topics.ts @@ -1,3 +1,4 @@ +import { resultClient } from "../../utils/delphiResults"; import { Request, Response } from "express"; import logger from "../../utils/logger"; import { DynamoDBClient, ListTablesCommand } from "@aws-sdk/client-dynamodb"; @@ -50,13 +51,13 @@ logger.info(`DynamoDB Config: `); // Create DynamoDB clients -const client = new DynamoDBClient(dynamoDBConfig); -const docClient = DynamoDBDocumentClient.from(client, { +const client = resultClient(() => new DynamoDBClient(dynamoDBConfig)); +const docClient = resultClient(() => DynamoDBDocumentClient.from(new DynamoDBClient(dynamoDBConfig), { marshallOptions: { convertEmptyValues: true, removeUndefinedValues: true, }, -}); +})); /** * Handler for Delphi API route that retrieves LLM topic names from DynamoDB diff --git a/server/src/routes/delphi/visualizations.ts b/server/src/routes/delphi/visualizations.ts index 512cc65cbf..b5d1d7c6c8 100644 --- a/server/src/routes/delphi/visualizations.ts +++ b/server/src/routes/delphi/visualizations.ts @@ -1,3 +1,4 @@ +import { resultClient, postgresResults } from "../../utils/delphiResults"; import { Request, Response } from "express"; import logger from "../../utils/logger"; import { getZidFromReport } from "../../utils/parameter"; @@ -27,13 +28,12 @@ if (Config.dynamoDbEndpoint) { }; } -const client = new DynamoDBClient(dynamoDBConfig); -const docClient = DynamoDBDocumentClient.from(client, { +const docClient = resultClient(() => DynamoDBDocumentClient.from(new DynamoDBClient(dynamoDBConfig), { marshallOptions: { convertEmptyValues: true, removeUndefinedValues: true, }, -}); +})); /** * Handler for Delphi API route that retrieves visualization information @@ -82,6 +82,14 @@ export async function handle_GET_delphi_visualizations( `Fetching visualizations for report_id: ${report_id}, conversation_id: ${conversation_id}` ); + // Queue graph results have no S3 static visualization artifact contract. + // Return actual job metadata without issuing a cloud listing. + if (postgresResults()) { + const metadata=await fetchJobMetadata(conversation_id); + return res.json({status:"success",report_id,visualizations:[], + jobs:Object.values(metadata).filter((job:any)=>!jobId || job.jobId===jobId)}); + } + // Configure S3 client const s3Config: any = { region: Config.AWS_REGION || "us-east-1", @@ -372,7 +380,8 @@ function processJobItems( // Additive: lets a reloaded client tell "finished" from "terminal row, // work still outstanding underneath" without a second request. A false // here only ever comes from the authoritative sweep. - workLive: livenessComplete ? liveByJobId.get(job_id) !== false : true, + ...(item.archived === true ? {archived:true} : {}), + workLive: item.archived === true ? false : livenessComplete ? liveByJobId.get(job_id) !== false : true, // Set when this server withdrew the job after losing a race: it names the // job that actually carries the work. The client follows it rather than // dropping the id it was acknowledged with. diff --git a/server/src/routes/topicStats.ts b/server/src/routes/topicStats.ts index cb06743276..23ce0e4a78 100644 --- a/server/src/routes/topicStats.ts +++ b/server/src/routes/topicStats.ts @@ -1,3 +1,4 @@ +import { resultClient } from "../utils/delphiResults"; import { Request, Response } from "express"; import logger from "../utils/logger"; import { DynamoDBClient } from "@aws-sdk/client-dynamodb"; @@ -23,13 +24,12 @@ if (Config.dynamoDbEndpoint) { }; } -const client = new DynamoDBClient(dynamoDBConfig); -const docClient = DynamoDBDocumentClient.from(client, { +const docClient = resultClient(() => DynamoDBDocumentClient.from(new DynamoDBClient(dynamoDBConfig), { marshallOptions: { convertEmptyValues: true, removeUndefinedValues: true, }, -}); +})); interface TopicMetrics { comment_count: number; diff --git a/server/src/utils/commentClusters.ts b/server/src/utils/commentClusters.ts index 0ead7b4337..da68b47d4c 100644 --- a/server/src/utils/commentClusters.ts +++ b/server/src/utils/commentClusters.ts @@ -1,3 +1,4 @@ +import { resultClient, postgresResults } from "./delphiResults"; import pg from "../db/pg-query"; import logger from "./logger"; import { DynamoDBClient } from "@aws-sdk/client-dynamodb"; @@ -89,13 +90,12 @@ function createDynamoDBClient(): DynamoDBDocumentClient { }; } - const client = new DynamoDBClient(dynamoDBConfig); - return DynamoDBDocumentClient.from(client, { + return resultClient(() => DynamoDBDocumentClient.from(new DynamoDBClient(dynamoDBConfig), { marshallOptions: { convertEmptyValues: true, removeUndefinedValues: true, }, - }); + })); } /** @@ -115,6 +115,8 @@ export async function getClusterAssignments( zid: number, useCache = false ): Promise> { + // A request snapshot must not reuse assignments from an older publication. + if (postgresResults()) useCache = false; // Check cache if enabled if (useCache) { const cached = clusterAssignmentsCache.get(zid); diff --git a/server/src/utils/delphiResultSnapshot.ts b/server/src/utils/delphiResultSnapshot.ts new file mode 100644 index 0000000000..a831b319f9 --- /dev/null +++ b/server/src/utils/delphiResultSnapshot.ts @@ -0,0 +1,56 @@ +import Config from "../config"; +/** One publication snapshot for every PostgreSQL result read in an HTTP request. */ +import { AsyncLocalStorage } from "async_hooks"; +import { RequestHandler } from "express"; +import { PoolClient } from "pg"; +import pg from "../db/pg-query"; +import logger from "./logger"; + +type Snapshot = { client?: Promise; closed: boolean; failure?: Error; onError?: (error: Error) => void }; +const snapshots = new AsyncLocalStorage(); + +export const delphiResultSnapshot: RequestHandler = (_req, res, next) => { + if (Config.delphiResultBackend !== "postgres") return next(); + const snapshot: Snapshot = { closed: false }; + const close = () => { + if (snapshot.closed) return; + snapshot.closed = true; + if (!snapshot.client) return; + void snapshot.client.then(async client => { + let failure: Error | undefined = snapshot.failure; + try { await client.query("ROLLBACK"); } + catch (error) { failure = error as Error; logger.error("delphi_result_snapshot_close", error); } + finally { + client.release(failure); + if (!failure && snapshot.onError) client.removeListener("error", snapshot.onError); + } + }, () => { /* Failed setup already released its client. */ }); + }; + res.once("finish", close); + res.once("close", close); + snapshots.run(snapshot, next); +}; + +/** Primary connection: replicas may lag a newly published result pointer. */ +export async function resultQuery(sql: string, params?: any[]): Promise { + const snapshot = snapshots.getStore(); + if (!snapshot) return pg.queryP(sql, params) as Promise; + if (snapshot.closed) throw new Error("Delphi result request has closed"); + if (snapshot.failure) throw snapshot.failure; + if (!snapshot.client) snapshot.client = pg.connectResultSnapshot().then(async client => { + snapshot.onError = error => { snapshot.failure = error; }; + client.on("error", snapshot.onError); + try { + await client.query("BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY"); + await client.query("SET LOCAL statement_timeout = '30s'; SET LOCAL idle_in_transaction_session_timeout = '30s'"); + return client; + } catch (error) { + client.release(error as Error); + // Discarded sockets may still emit an asynchronous error. + throw error; + } + }); + const client = await snapshot.client; + if (snapshot.failure) throw snapshot.failure; + return (await client.query(sql, params)).rows as T[]; +} diff --git a/server/src/utils/delphiResults.ts b/server/src/utils/delphiResults.ts new file mode 100644 index 0000000000..b4902f65df --- /dev/null +++ b/server/src/utils/delphiResults.ts @@ -0,0 +1,216 @@ +import Config from "../config"; +/** Published Delphi results. Backend choice is explicit and never falls back. */ +import { resultQuery } from "./delphiResultSnapshot"; +import { FAMILIES, decodeFamily, canonicalNumber } from "./delphiStorageCodec"; + +export function postgresResults(): boolean { + const value = Config.delphiResultBackend || "dynamodb"; + if (!["postgres", "dynamodb"].includes(value)) throw new Error("invalid DELPHI_RESULT_BACKEND"); + return value === "postgres"; +} +export function decodeAttribute(value: any): any { + if ("S" in value) return value.S; + if ("N" in value) { + const n = Number(value.N); + if (!Number.isFinite(n) || (Number.isInteger(n) && !Number.isSafeInteger(n))) { + throw new Error("result numeric value exceeds reader precision"); + } + return n; + } + if ("BOOL" in value) return value.BOOL; + if ("NULL" in value) return null; + if ("L" in value) return value.L.map(decodeAttribute); + if ("M" in value) return decodeItem(value.M); + if ("SS" in value) return new Set(value.SS); + if ("NS" in value) return new Set(value.NS.map((n: string) => decodeAttribute({N:n}))); + if ("B" in value) return Buffer.from(value.B, "base64"); + if ("BS" in value) return new Set(value.BS.map((b: string) => Buffer.from(b,"base64"))); + throw new Error("invalid stored AttributeValue"); +} +export function decodeItem(item: any): Record { + return Object.fromEntries(Object.entries(item).map(([k,v]) => [k, decodeAttribute(v)])); +} + +/** Historical controls are display-only. Preserve exact values even when JS + * cannot represent a legacy decimal; the raw tagged item remains available. */ +export function decodeArchivedItem(tagged: any): Record { + let reason: string | undefined; + const value = (av:any):any => { + if ("N" in av) { + const n=Number(av.N); + if (!Number.isFinite(n) || (Number.isInteger(n) && !Number.isSafeInteger(n)) || canonicalNumber(String(n)) !== av.N) { + reason="legacy numeric metadata exceeds JavaScript precision; exact decimal retained as text"; + return av.N; + } + } + if ("L" in av) return av.L.map(value); + if ("M" in av) return Object.fromEntries(Object.entries(av.M).map(([k,v])=>[k,value(v)])); + if ("NS" in av) return new Set(av.NS.map((n:string)=>value({N:n}))); + return decodeAttribute(av); + }; + const item=Object.fromEntries(Object.entries(tagged).map(([k,v])=>[k,value(v)])); + return {...item,archived:true,...(reason ? {unreadable_metadata_reason:reason,legacy_control_item:tagged} : {})}; +} + +/** Only the expression grammar used by the audited Delphi readers is accepted. */ +export function predicate(expression: string | undefined, input: any): (item: any) => boolean { + if (!expression) return () => true; + const resolve = (token: string, item: any) => token.startsWith(":") + ? input.ExpressionAttributeValues?.[token] + : item[input.ExpressionAttributeNames?.[token] || token]; + const clauses = expression.split(/\s+AND\s+/i).map(part => { + const equal = part.trim().match(/^([#\w]+)\s*=\s*([:#\w]+)$/); + if (equal) return (item: any) => resolve(equal[1],item) === resolve(equal[2],item); + const prefix = part.trim().match(/^begins_with\(\s*([#\w]+)\s*,\s*(:\w+)\s*\)$/); + if (prefix) return (item: any) => String(resolve(prefix[1],item) ?? "").startsWith(String(resolve(prefix[2],item))); + throw new Error(`unsupported result expression: ${part}`); + }); + return item => clauses.every(clause => clause(item)); +} + +/** Bound parameters only: fields and values never become SQL identifiers. */ +export function sqlPredicate(expression: string | undefined, input: any, values: any[]): string { + if (!expression) return "TRUE"; + const bind = (value: any) => { values.push(value); return `$${values.length}`; }; + const field = (name: string) => `item->${bind(input.ExpressionAttributeNames?.[name] || name)}::text`; + const operand = (token: string): string => { + if (!token.startsWith(":")) return field(token); + const value = input.ExpressionAttributeValues?.[token]; + const tagged = typeof value === "string" ? {S:value} : typeof value === "number" && Number.isFinite(value) + ? {N:String(value)} : typeof value === "boolean" ? {BOOL:value} : undefined; + if (!tagged) throw new Error("unsupported result expression value"); + return `${bind(JSON.stringify(tagged))}::jsonb`; + }; + return expression.split(/\s+AND\s+/i).map(part => { + const equal = part.trim().match(/^([#\w]+)\s*=\s*([:#\w]+)$/); + if (equal) return `${field(equal[1])} = ${operand(equal[2])}`; + const prefix = part.trim().match(/^begins_with\(\s*([#\w]+)\s*,\s*(:\w+)\s*\)$/); + if (prefix) { + const value = input.ExpressionAttributeValues?.[prefix[2]]; + if (typeof value !== "string") throw new Error("begins_with requires a string"); + return `starts_with(${field(prefix[1])}->>'S',${bind(value)})`; + } + throw new Error(`unsupported result expression: ${part}`); + }).join(" AND "); +} + +export async function sendPostgresResult(command: any): Promise { + const input = command.input; + const operation = command.constructor.name; + const env = Config.delphiResultEnv; + if (!env) throw new Error("DELPHI_RESULT_ENV is required for Postgres results"); + if (operation === "ListTablesCommand") return {TableNames: Object.keys(FAMILIES)}; + const family = input.TableName; + if (!(family in FAMILIES)) { + const error = new Error(`Unknown Delphi result family: ${family}`); + error.name = "ResourceNotFoundException"; + throw error; + } + if (operation === "DescribeTableCommand") { + await resultQuery("SELECT 1 FROM public.delphi_result_current_rows LIMIT 0"); + return {Table:{TableName:family,TableStatus:"ACTIVE"}}; + } + if (!["QueryCommand","ScanCommand","GetCommand"].includes(operation)) { + throw new Error("Published Delphi results are immutable; submit a new run"); + } + let items: any[]; + let generations: Record = {}; + if (family === "Delphi_JobQueue") { + const scope = Config.delphiResultScope; + const rows = await resultQuery(`SELECT * FROM public.delphi_result_jobs WHERE env=$1 + ${scope ? "AND scope_key=$2" : ""}`,scope ? [env,scope] : [env]) as any[]; + const archives = await resultQuery(`SELECT zid,scope_key,generation::text,codec_wire + FROM public.delphi_result_legacy_controls WHERE env=$1 AND family='Delphi_JobQueue' + ${scope ? "AND scope_key=$2" : ""} ORDER BY zid,scope_key`,scope ? [env,scope] : [env]) as any[]; + const activeIds = new Set(rows.map(row => row.job_id)); + const archived = new Map(); + for (const archive of archives) { + generations[`${archive.zid}:${archive.scope_key}`] = archive.generation; + const decoded = decodeFamily(Buffer.from(archive.codec_wire,"utf8")); + if (decoded.family !== family) throw new Error("legacy control family mismatch"); + for (const tagged of decoded.items) { + const item = decodeArchivedItem(tagged); + if (activeIds.has(item.job_id)) continue; + const prior = archived.get(item.job_id); + if (prior && JSON.stringify(prior) !== JSON.stringify(item)) throw new Error("Ambiguous archived job scopes; configure DELPHI_RESULT_SCOPE"); + archived.set(item.job_id,item); + } + } + items = [...archived.values(),...rows]; + } else { + const values: any[] = [env,family]; + let where = "env=$1 AND family=$2"; + const scope = Config.delphiResultScope; + if (scope) { values.push(scope); where += ` AND scope_key=$${values.length}`; } + where += " AND (" + sqlPredicate(input.KeyConditionExpression,input,values) + ")"; + for (const [name,value] of Object.entries(input.Key || {})) { + where += " AND (" + sqlPredicate("#key = :value", {ExpressionAttributeNames:{"#key":name},ExpressionAttributeValues:{":value":value}}, values) + ")"; + } + const rows = await resultQuery(`SELECT zid,scope_key,generation::text,item FROM public.delphi_result_current_rows + WHERE ${where} ORDER BY zid,scope_key,item_key::text`,values) as any[]; + const seen = new Map(); + items = []; + for (const row of rows) { + generations[`${row.zid}:${row.scope_key}`] = row.generation; + const item = decodeItem(row.item); + const key = JSON.stringify(FAMILIES[family].key.map(([name]) => item[name])); + if (seen.has(key)) { + if (JSON.stringify(seen.get(key)) !== JSON.stringify(row.item)) throw new Error("Ambiguous result scopes; configure DELPHI_RESULT_SCOPE"); + continue; + } + seen.set(key,row.item); items.push(item); + } + } + const cursorGenerations = input.ExclusiveStartKey?._polisPgGenerations; + if (input.ExclusiveStartKey && JSON.stringify(cursorGenerations) !== JSON.stringify(generations)) { + throw new Error("result generation changed; restart pagination"); + } + // Existing HTTP handlers perform authorization before reaching this adapter. + const matches = predicate(input.KeyConditionExpression,input); + items = items.filter(matches); + if (input.Key) items = items.filter(item => Object.entries(input.Key).every(([k,v]) => item[k] === v)); + const keys = FAMILIES[family].key.map(([k]) => k); + const key = (item: any) => Object.fromEntries(keys.map(k => [k,item[k]])); + const indexOrder: Record = {ConversationIndex:"created_at",StatusCreatedIndex:"created_at",ReportIdTimestampIndex:"timestamp","zid-created_at-index":"created_at"}; + if (input.IndexName && !indexOrder[input.IndexName]) throw new Error("unsupported result index"); + const order = input.IndexName ? indexOrder[input.IndexName] : keys[keys.length-1]; + if (input.IndexName) items = items.filter(item => item[order] !== undefined && item[order] !== null); + const compare = (a:any,b:any,fields:string[]) => { + for (const field of fields) { if(a[field]b[field])return 1; } + return 0; + }; + items.sort((a,b) => compare(a,b,[order,...keys])); + if (input.ScanIndexForward === false) items.reverse(); + if (input.ExclusiveStartKey) { + const offset = items.findIndex(item => Object.entries(input.ExclusiveStartKey).filter(([k]) => k !== "_polisPgGenerations").every(([k,v]) => item[k] === v)); + if (offset < 0) throw new Error("stale result cursor"); + items = items.slice(offset+1); + } + const limit = input.Limit ?? 1000; + if (!Number.isSafeInteger(limit) || limit < 1) throw new Error("invalid result limit"); + const page = items.slice(0,limit); + const filtered = page.filter(predicate(input.FilterExpression,input)); + if (operation === "GetCommand") { + const item=filtered[0]; + if (family === "Delphi_JobQueue" && item && item.archived !== true) { + const status=await resultQuery<{value:any}>("SELECT public.pq_job_status($1::text,$2::uuid) AS value",[env,item.job_id]); + const attempt=status[0]?.value?.attempt_id; + const entries=attempt ? await resultQuery(`SELECT + to_char(ts AT TIME ZONE 'UTC','YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS timestamp, + CASE stream WHEN 'stderr' THEN 'ERROR' ELSE 'INFO' END AS level,line AS message + FROM public.pq_attempt_logs($1::text,$2::uuid,NULL,1000) + WHERE stream IN ('stdout','stderr')`,[env,attempt]) : []; + return {Item:{...item,logs:{entries},log_attempt_id:attempt || null}}; + } + return {Item:item}; + } + return {Items:filtered, Count:filtered.length, ScannedCount:page.length, + ...(items.length > limit ? {LastEvaluatedKey:{...key(page[page.length-1]),_polisPgGenerations:generations}} : {})}; +} + +export function resultClient any}>(factory: () => T): T { + let legacy: T | undefined; + return {send: (command: any) => postgresResults() + ? sendPostgresResult(command) + : (legacy ||= factory()).send(command)} as T; +} diff --git a/server/src/utils/storage.ts b/server/src/utils/storage.ts index 297b0c81c6..9184d2134f 100644 --- a/server/src/utils/storage.ts +++ b/server/src/utils/storage.ts @@ -1,3 +1,4 @@ +import { resultClient } from "./delphiResults"; import { DeleteItemCommand, DescribeTableCommand, @@ -33,7 +34,7 @@ export default class DynamoStorageService { constructor(tableName: string, disableCache?: boolean) { // Shared credential precedence: local endpoint -> real configured keys -> // default AWS credential provider chain (the EC2 instance role in prod). - this.client = makeDynamoClient(); + this.client = resultClient(() => makeDynamoClient()); this.tableName = tableName; this.cacheDisabled = disableCache || false; } @@ -321,13 +322,20 @@ export default class DynamoStorageService { }, }; - const scanCommand = new ScanCommand(scanParams); - try { - const scanResponse = await this.client.send(scanCommand); - const items = scanResponse.Items; + const items: any[] = []; + let lastEvaluatedKey: Record | undefined; + do { + const scanResponse = await this.client.send(new ScanCommand({ + ...scanParams, + ExclusiveStartKey: lastEvaluatedKey, + })); + items.push(...(scanResponse.Items || [])); + lastEvaluatedKey = scanResponse.LastEvaluatedKey; + // Filtering can leave a page empty while later pages still match. + } while (lastEvaluatedKey); - if (!items || items.length === 0) { + if (items.length === 0) { logger.debug(`No items found with report ID prefix: ${reportIdPrefix}`); return { success: true, data: [] }; }