From e95865fab5c13d6423b276b1507e899deb131fce Mon Sep 17 00:00:00 2001 From: Colin Megill Date: Mon, 12 Oct 2026 00:59:55 -0700 Subject: [PATCH 1/3] CI: Delphi tests run through one entry point, in parallel ci/local/delphi-python.sh now holds every step of the Delphi pytest job (stack, migrations, copy-in, projection-gate wiring, pytest, coverage), and the workflow calls it and nothing else, so the same check runs on any Docker host. It removes published host ports so several runs can share one host. pytest runs in two passes: a parallel pass (pytest-xdist, one worker per CPU, --dist loadgroup, each dataset's tests on one worker so its conversation is computed once), then the tests marked `serial` sequentially. The serial marker is on the tests that collide when run concurrently: fixed ids or cluster-wide roles in the shared test database, each with the observed failure in a comment. Hosted CI splits the suite into three shard jobs (--shard I/3, by expected seconds per file or dataset group) and the `test` job combines them: it checks that every collected test ran in exactly one shard, adds up the counts and combines coverage into the same PR comment. A plain `pytest` still runs sequentially (-n0). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018NVGBuYCk4EZmiz4csUv9s --- .github/workflows/python-ci.yml | 195 ++++--------- .gitignore | 3 + ci/local/delphi-python.sh | 268 ++++++++++++++++++ ci/local/delphi-test-durations.json | 177 ++++++++++++ ci/local/delphi_python_results.py | 119 ++++++++ delphi/generate_coverage_md.py | 6 +- delphi/pyproject.toml | 13 +- delphi/tests/ci_partition.py | 171 +++++++++++ delphi/tests/conftest.py | 4 +- delphi/tests/poller/test_backfill_postgres.py | 4 +- .../poller/test_capacity_queue_postgres.py | 4 +- .../tests/test_certify_bundle_integration.py | 4 +- delphi/tests/test_engine_rebuild.py | 4 + delphi/tests/test_generator_vote_copy.py | 4 +- delphi/tests/test_prodclone_extract.py | 4 + 15 files changed, 825 insertions(+), 155 deletions(-) create mode 100755 ci/local/delphi-python.sh create mode 100644 ci/local/delphi-test-durations.json create mode 100644 ci/local/delphi_python_results.py create mode 100644 delphi/tests/ci_partition.py diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index 789357a366..ca8087cc76 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -19,6 +19,9 @@ on: - 'docker-compose*.yml' - '.github/workflows/python-ci.yml' - 'ci/test-migrations.sh' + - 'ci/local/delphi-python.sh' + - 'ci/local/delphi_python_results.py' + - 'ci/local/delphi-test-durations.json' - 'docker-compose.test.yml' - 'server/Dockerfile-db' - 'server/postgres/**' @@ -41,156 +44,80 @@ on: workflow_dispatch: jobs: - test: + # Every step of the check lives in ci/local/delphi-python.sh, which runs the + # same way on any Docker host. Hosted CI splits the suite into shards (each + # also parallel inside its job) and the `test` job puts the shards back + # together: it proves every test ran in exactly one shard, adds up the + # counts and combines coverage. + shard: + name: test shard ${{ matrix.shard }}/3 runs-on: ubuntu-latest - + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3] + steps: - name: Checkout code uses: actions/checkout@v4 - - name: 1. Free up disk space on runner + - name: Free up disk space on runner run: | - echo "Starting disk cleanup..." df -h sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell - echo "Disk space after cleanup:" df -h - - name: 2. Create .env file for CI - run: | - if [ -f "test.env" ]; then - cp test.env .env - echo "Using test.env for CI." - else - echo "::error::test.env file not found!" - exit 1 - fi - - # Add hostnames for container-to-container networking - # The 'delphi' service in 'docker-compose.test.yml' reads these - echo "POSTGRES_HOST=postgres" >> .env - echo "DYNAMODB_ENDPOINT=http://dynamodb:8000" >> .env - echo "AWS_S3_ENDPOINT=http://minio:9000" >> .env - # BuildKit is required by `RUN --mount=type=cache` in the npm Dockerfiles. - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: 3. Build Docker images - run: | - # Build all services in the test file (including delphi) - docker compose -f docker-compose.test.yml --env-file .env build + - name: Delphi pytest (shard ${{ matrix.shard }}/3) + env: + DELPHI_CHECK_OUT: ${{ github.workspace }}/delphi-results/shard-${{ matrix.shard }} + run: bash ci/local/delphi-python.sh --shard ${{ matrix.shard }}/3 - # Apply/check before starting application services on this volume. - bash ci/test-migrations.sh + - name: Upload shard results + if: always() + uses: actions/upload-artifact@v4 + with: + name: delphi-shard-${{ matrix.shard }} + path: delphi-results/shard-${{ matrix.shard }} + if-no-files-found: ignore - # The 'delphi' container will start and run 'tail -f /dev/null' - docker compose -f docker-compose.test.yml --env-file .env up -d - - echo "Waiting for services to be healthy..." - sleep 20 # Wait for containers to start - docker compose -f docker-compose.test.yml ps + test: + needs: shard + if: always() + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 - - name: 5. Check service health - run: | - # Wait for postgres to be ready - echo "Checking postgres..." - docker compose -f docker-compose.test.yml exec -T postgres \ - bash -c 'until pg_isready -U $POSTGRES_USER; do sleep 5; done' - echo "Postgres is ready." - - # The opt-in Postgres integration tests (require_polis_postgres) run here - # against the compose `postgres` service, whose image bakes the polis - # migrations through polis-migrate during initialization and the explicit CI step, - # rule are already applied. The pytest step exports POLIS_TEST_POSTGRES_URL; - # the cold-start generator under test is already baked into the delphi image - # (Dockerfile `COPY scripts/ ./scripts/`), built from this checkout. - - name: 6. Run Delphi Pytest + - name: Every shard passed run: | - echo "Copying test files into container..." - docker compose -f docker-compose.test.yml cp delphi/tests delphi:/app/tests - docker compose -f docker-compose.test.yml cp delphi/real_data delphi:/app/real_data - echo "Copying coverage script into container..." - docker compose -f docker-compose.test.yml cp delphi/generate_coverage_md.py delphi:/app/generate_coverage_md.py - echo "Copying script to be tested into container..." - docker compose -f docker-compose.test.yml cp delphi/polismath/run_math_pipeline.py delphi:/app/run_math_pipeline.py - docker compose -f docker-compose.test.yml cp delphi/umap_narrative delphi:/app/umap_narrative - # The same container-layout preparation and hook tests run on mm5. - bash scripts/test-deploy-hooks.sh - - # Wire a checkout-shaped root so delphi/tests/scripts collects AND RUNS the - # pure-Python inventory sweep + the DB-free gate unit tests (the conftest - # locator resolves POLIS_CHECKOUT_DIR). The scan inputs are copied in as REAL, - # traversable directories (NOT symlinks — os.walk would skip those). The set - # of inputs is DERIVED from the scanner's own declaration - # (`--print-scan-inputs`) so it can never diverge from what the sweep scans; - # a test asserts every declared input is present under /app/projgate. - echo "Wiring projection-gate scan inputs (derived from the scanner declaration)..." - docker compose -f docker-compose.test.yml exec -T delphi mkdir -p /app/projgate - # Iterate a CAPTURED list, not `... | while read`: a `docker compose exec` - # inside a while-read loop drains the piped stdin and truncates the list to - # its first item (observed: only server/src copied). - # Recordings tests also load the packer and its digest helper from ci/. - # battery_coverage.py is already included in the declared delphi/scripts. - # Representative payload tests import the box planner/gate and their - # helpers from this same checkout root; keep their relative paths intact. - # Light-shadow triage tests also import the compare/triage modules. - for rel in $(python3 delphi/scripts/projection_inventory.py --print-scan-inputs) \ - ci/p022_recordings_manifest.py ci/p022_battery_digest.py \ - ci/private_cert/images/gate.py ci/private_cert/images/probe.py \ - ci/private_cert/images/attribution.py ci/private_cert/images/diagnostic_projection.py ci/private_cert/images/selection_context.py \ - ci/private_cert/images/g12.py ci/private_cert/images/near_ties.py ci/private_cert/control.py \ - ci/private_cert/image_admission.py ci/probe_box/receipt.py \ - ci/probe_box/contracts.py ci/probe_box/light_shadow.py ci/probe_box/light_shadow_queries.py \ - ci/private_cert/images/light_shadow_compare.py ci/private_cert/images/light_shadow_triage.py \ - ci/private_cert/images/recipe.py scripts/run-import-dry-run.py; do - docker compose -f docker-compose.test.yml exec -T delphi mkdir -p "/app/projgate/$(dirname "$rel")" - docker compose -f docker-compose.test.yml cp "$rel" "delphi:/app/projgate/$rel" \ - || { echo "failed to copy scan input: $rel"; exit 1; } - done - - echo "Running tests and generating coverage report..." - docker compose \ - -f docker-compose.test.yml \ - --env-file .env \ - exec -T \ - -e AWS_DEFAULT_REGION=us-east-1 \ - -e AWS_REGION=us-east-1 \ - -e AWS_ACCESS_KEY_ID=dummy \ - -e AWS_SECRET_ACCESS_KEY=dummy \ - -e SKIP_GOLDEN=1 \ - -e POSTGRES_CONNECT_TIMEOUT=5 \ - -e POLIS_CHECKOUT_DIR=/app/projgate \ - delphi \ - bash -c " \ - set -e; \ - echo '--- Setting up DynamoDB Tables ---'; \ - python create_dynamodb_tables.py --region us-east-1; \ - echo '--- Running Pytest ---'; \ - export PYTHONPATH=\$PYTHONPATH:/app; \ - export POLIS_TEST_POSTGRES_URL=\"postgresql://\$DATABASE_USER:\$DATABASE_PASSWORD@\$DATABASE_HOST/\$DATABASE_NAME\"; \ - pytest --cov=polismath --cov=run_math_pipeline --cov=./umap_narrative --cov-report=xml:/app/coverage.xml /app/tests --ignore=/app/tests/test_pakistan_conversation.py - echo '--- Generating Coverage Comment Text ---'; \ - python /app/generate_coverage_md.py > /app/coverage-comment.md \ - " - - - name: 7. Copy coverage report from container - if: success() + echo "shards: ${{ needs.shard.result }}" + test "${{ needs.shard.result }}" = success + + - name: Download shard results + uses: actions/download-artifact@v4 + with: + pattern: delphi-shard-* + path: delphi-results + + - name: Combine shards (counts, coverage) + env: + DELPHI_CHECK_OUT: ${{ github.workspace }}/delphi-results/combined run: | - echo "Copying coverage-comment.md from delphi container..." - docker compose -f docker-compose.test.yml cp delphi:/app/coverage-comment.md . - echo "=== Coverage Report ===" - cat coverage-comment.md + bash ci/local/delphi-python.sh combine delphi-results/delphi-shard-1 delphi-results/delphi-shard-2 delphi-results/delphi-shard-3 + cp delphi-results/combined/coverage-comment.md coverage-comment.md - - name: 8. Upload Coverage Report - if: success() + - name: Upload Coverage Report uses: actions/upload-artifact@v4 with: name: coverage-report path: coverage-comment.md - - name: 9. Post Coverage Comment - if: success() && github.event_name == 'pull_request' + - name: Post Coverage Comment + if: github.event_name == 'pull_request' uses: actions/github-script@v6 with: github-token: ${{ secrets.GITHUB_TOKEN }} @@ -232,24 +159,8 @@ jobs: if (error.status === 403) { console.log('Note: Could not post coverage comment to PR.'); console.log('This is expected for PRs from forks due to GitHub token permissions.'); - console.log('Coverage report is available in the workflow logs (step 7) and as a downloadable artifact (step 8).'); + console.log('Coverage report is in the test job's log and in the coverage-report artifact.'); } else { console.log(`Unexpected error posting comment: ${error.message}`); } } - - - name: 10. Show service logs on failure - if: failure() - run: | - echo "=== Delphi service logs ===" - docker compose -f docker-compose.test.yml logs delphi || true - echo "=== Postgres service logs ===" - docker compose -f docker-compose.test.yml logs postgres || true - echo "=== DynamoDB service logs ===" - docker compose -f docker-compose.test.yml logs dynamodb || true - - - name: 11. Clean up services - if: always() - run: | - echo "Cleaning up services..." - docker compose -f docker-compose.test.yml down -v diff --git a/.gitignore b/.gitignore index 07a77464eb..fa4439bb7e 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,6 @@ __pycache__/ # Written into the CodeDeploy bundle by the deploy workflow; never committed. scripts/deploy-revision.txt + +# Results of ci/local/ check runs +.ci-out/ diff --git a/ci/local/delphi-python.sh b/ci/local/delphi-python.sh new file mode 100755 index 0000000000..97a2280ac5 --- /dev/null +++ b/ci/local/delphi-python.sh @@ -0,0 +1,268 @@ +#!/usr/bin/env bash +# Delphi Python Tests: the whole Delphi pytest suite, with coverage, inside the +# delphi test image, against the test stack's Postgres (migrations applied by +# ci/test-migrations.sh), DynamoDB and MinIO. The hosted workflow +# (.github/workflows/python-ci.yml) calls this script and nothing else, so a +# run on any Docker host is the same check. +# +# Usage: +# ci/local/delphi-python.sh the whole suite, then the coverage comment +# ci/local/delphi-python.sh --shard I/N only shard I of N (hosted CI runs N jobs) +# ci/local/delphi-python.sh combine DIR... merge shard result directories: prove every +# test ran in exactly one shard, add up the +# counts, combine coverage, write the comment +# +# How tests run: a parallel pass (pytest-xdist, one worker per CPU of the delphi +# container, --dist loadgroup so each dataset's conversation is computed by one +# worker), then a sequential pass of the tests marked `serial` (state shared +# across the whole Postgres cluster, or timing that concurrency would starve). +# Both passes write coverage into one data file. +# +# Knobs (all optional): +# DELPHI_PYTEST_WORKERS workers for the parallel pass (default: CPUs in the +# container); 0 runs everything in one sequential pass, +# exactly as before the parallel pass existed +# DELPHI_CHECK_PROJECT compose project name (default: delphi-python--) +# DELPHI_CHECK_OUT results directory (default: .ci-out/) +# DELPHI_CHECK_KEEP=1 leave the stack running afterwards +# DELPHI_PYTEST_ARGS extra pytest arguments for both passes +# +# Published host ports are removed (the tests talk to the services inside the +# compose network), so several runs can share one Docker host. +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$ROOT" +DURATIONS_FILE="ci/local/delphi-test-durations.json" + +log() { printf '[delphi-python] %s\n' "$*" >&2; } +die() { log "ERROR: $*"; exit 2; } +group() { if [ -n "${GITHUB_ACTIONS:-}" ]; then echo "::group::$*"; else printf '\n=== %s\n' "$*"; fi; } +endgroup() { if [ -n "${GITHUB_ACTIONS:-}" ]; then echo "::endgroup::"; fi; } +sha() { if command -v sha256sum >/dev/null 2>&1; then sha256sum | cut -c1-6; else shasum -a 256 | cut -c1-6; fi; } + +python_bin() { + local p + for p in python3.12 python3; do command -v "$p" >/dev/null 2>&1 && { echo "$p"; return; }; done + die "python3 is required" +} + +# ---------------------------------------------------------------- combine +if [ "${1:-}" = "combine" ]; then + shift + [ $# -ge 1 ] || die "combine needs at least one shard result directory" + OUT="${DELPHI_CHECK_OUT:-$ROOT/.ci-out/delphi-python-combined}" + mkdir -p "$OUT" + OUT="$(cd "$OUT" && pwd)" + PY="$(python_bin)" + group "check the shards and add up the counts" + "$PY" ci/local/delphi_python_results.py combine --out "$OUT" "$@" + endgroup + group "combine coverage" + # The coverage data holds paths relative to /app in the delphi image. Lay the + # same files out here (the image copies delphi/ to /app, and the workflow puts + # run_math_pipeline.py beside it), then combine and report from there. + cov_version="$(sed -n 's/^coverage==\([0-9.]*\).*/\1/p' delphi/requirements.lock | head -1)" + [ -n "$cov_version" ] || cov_version="$(awk '/^name = "coverage"$/{getline; gsub(/[^0-9.]/,"",$0); print; exit}' delphi/uv.lock)" + [ -n "$cov_version" ] || die "cannot find the pinned coverage version" + rm -rf "$OUT/venv" + if command -v uv >/dev/null 2>&1; then + uv venv -q -p "$PY" "$OUT/venv" && uv pip install -q -p "$OUT/venv/bin/python" "coverage==$cov_version" + else + "$PY" -m venv "$OUT/venv" && "$OUT/venv/bin/pip" install -q "coverage==$cov_version" + fi + rm -rf "$OUT/app"; mkdir -p "$OUT/app" + cp -R delphi/polismath delphi/umap_narrative "$OUT/app/" + cp delphi/polismath/run_math_pipeline.py "$OUT/app/run_math_pipeline.py" + i=0 + for d in "$@"; do + i=$((i + 1)) + [ -f "$d/coverage.data" ] || die "no coverage.data in $d" + cp "$d/coverage.data" "$OUT/app/.coverage.shard$i" + done + printf '[run]\nrelative_files = true\n' > "$OUT/app/.coveragerc" + (cd "$OUT/app" && "$OUT/venv/bin/coverage" combine -q --rcfile=.coveragerc . \ + && "$OUT/venv/bin/coverage" xml -q --rcfile=.coveragerc -o "$OUT/coverage.xml") + "$PY" delphi/generate_coverage_md.py "$OUT/coverage.xml" > "$OUT/coverage-comment.md" + endgroup + echo "=== Coverage Report ===" + cat "$OUT/coverage-comment.md" + exit 0 +fi + +# ---------------------------------------------------------------- run +SHARD="1/1" +while [ $# -gt 0 ]; do + case "$1" in + --shard) [ $# -ge 2 ] || die "--shard needs I/N"; SHARD="$2"; shift 2 ;; + *) die "unknown argument: $1" ;; + esac +done +case "$SHARD" in [0-9]*/[0-9]*) ;; *) die "--shard must look like 2/3" ;; esac +SHARD_I="${SHARD%/*}"; SHARD_N="${SHARD#*/}" +[ "$SHARD_I" -ge 1 ] && [ "$SHARD_I" -le "$SHARD_N" ] || die "--shard out of range: $SHARD" + +PROJECT="${DELPHI_CHECK_PROJECT:-delphi-python-${SHARD_I}of${SHARD_N}-$(printf '%s' "$ROOT" | sha)}" +OUT="${DELPHI_CHECK_OUT:-$ROOT/.ci-out/$PROJECT}" +rm -rf "$OUT"; mkdir -p "$OUT" +OUT="$(cd "$OUT" && pwd)" +log "project $PROJECT; shard $SHARD; results $OUT" + +# test.env plus the in-network hostnames the delphi service reads. +[ -f test.env ] || die "test.env not found" +{ cat test.env; echo "POSTGRES_HOST=postgres"; echo "DYNAMODB_ENDPOINT=http://dynamodb:8000"; echo "AWS_S3_ENDPOINT=http://minio:9000"; } > "$OUT/test.env" + +# No published host ports: nothing in this check reaches the stack from the host. +cat > "$OUT/no-host-ports.yml" <<'EOF' +services: + client-participation-alpha: { ports: !reset [] } + ses-local: { ports: !reset [] } + oidc-simulator: { ports: !reset [] } + postgres: { ports: !reset [] } + nginx-proxy: { ports: !reset [] } + minio: { ports: !reset [] } + dynamodb: { ports: !reset [] } +EOF +COMPOSE_ARGS=(-p "$PROJECT" -f docker-compose.test.yml -f "$OUT/no-host-ports.yml" --env-file "$OUT/test.env") +compose() { docker compose "${COMPOSE_ARGS[@]}" "$@"; } + +cleanup() { + local rc=$? + set +e + if [ "$rc" != 0 ]; then + group "service logs (failure)" + compose logs --no-color delphi postgres dynamodb 2>&1 | tail -n 400 + endgroup + fi + if [ -n "${DELPHI_CHECK_KEEP:-}" ]; then + log "DELPHI_CHECK_KEEP set: leaving $PROJECT running" + else + compose down -v --remove-orphans >/dev/null 2>&1 + fi + if [ "$rc" = 0 ]; then log "PASS"; else log "FAIL (exit $rc)"; fi + exit "$rc" +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +group "build images (every service in docker-compose.test.yml)" +compose build +endgroup + +group "apply and check migrations" +# test-migrations.sh puts its own -f docker-compose.test.yml first; it reads the +# env file from POLIS_TEST_ENV_FILE. +POLIS_TEST_ENV_FILE="$OUT/test.env" bash ci/test-migrations.sh -p "$PROJECT" -f "$OUT/no-host-ports.yml" +endgroup + +group "start the stack" +compose up -d +sleep 20 +compose ps +compose exec -T postgres bash -c 'until pg_isready -U $POSTGRES_USER; do sleep 5; done' +endgroup + +group "copy the tests and their inputs into the delphi container" +compose cp delphi/tests delphi:/app/tests +compose cp delphi/real_data delphi:/app/real_data +compose cp delphi/generate_coverage_md.py delphi:/app/generate_coverage_md.py +compose cp delphi/polismath/run_math_pipeline.py delphi:/app/run_math_pipeline.py +compose cp delphi/umap_narrative delphi:/app/umap_narrative +compose cp "$DURATIONS_FILE" delphi:/app/test-durations.json +# The same container-layout preparation and hook tests run on any Docker host. +bash scripts/test-deploy-hooks.sh "${COMPOSE_ARGS[@]}" + +# A checkout-shaped root so delphi/tests/scripts collects and runs the +# pure-Python inventory sweep and the DB-free gate unit tests (the conftest +# locator resolves POLIS_CHECKOUT_DIR). The scan inputs are copied in as real, +# traversable directories (not symlinks: os.walk would skip those). The set of +# inputs is derived from the scanner's own declaration (--print-scan-inputs) so +# it cannot diverge from what the sweep scans; a test asserts every declared +# input is present under /app/projgate. Recordings tests also load the packer +# and its digest helper from ci/; representative payload tests import the box +# planner/gate and their helpers from this same root; light-shadow triage tests +# import the compare/triage modules. +compose exec -T delphi mkdir -p /app/projgate +# Iterate a captured list: `docker compose exec` inside a while-read loop drains +# the piped stdin and truncates the list. +inputs="$(python3 delphi/scripts/projection_inventory.py --print-scan-inputs)" +for rel in $inputs \ + ci/p022_recordings_manifest.py ci/p022_battery_digest.py \ + ci/private_cert/images/gate.py ci/private_cert/images/probe.py \ + ci/private_cert/images/attribution.py ci/private_cert/images/diagnostic_projection.py ci/private_cert/images/selection_context.py \ + ci/private_cert/images/g12.py ci/private_cert/images/near_ties.py ci/private_cert/control.py \ + ci/private_cert/image_admission.py ci/probe_box/receipt.py \ + ci/probe_box/contracts.py ci/probe_box/light_shadow.py ci/probe_box/light_shadow_queries.py \ + ci/private_cert/images/light_shadow_compare.py ci/private_cert/images/light_shadow_triage.py \ + ci/private_cert/images/recipe.py scripts/run-import-dry-run.py; do + compose exec -T delphi mkdir -p "/app/projgate/$(dirname "$rel")" /app/out/coverage-comment.md || rc=1 + fi + exit $rc + ' +rc=$? +set -e +endgroup + +compose cp delphi:/app/out/. "$OUT/" || log "could not copy results out of the container" +compose cp delphi:/app/coverage.xml "$OUT/coverage.xml" 2>/dev/null || true +if [ -f "$OUT/manifest.json" ]; then + python3 ci/local/delphi_python_results.py summary "$OUT" || rc=1 +fi +if [ "$rc" = 0 ] && [ -f "$OUT/coverage-comment.md" ]; then + echo "=== Coverage Report ===" + cat "$OUT/coverage-comment.md" +fi +exit "$rc" diff --git a/ci/local/delphi-test-durations.json b/ci/local/delphi-test-durations.json new file mode 100644 index 0000000000..14fc2f12a6 --- /dev/null +++ b/ci/local/delphi-test-durations.json @@ -0,0 +1,177 @@ +{ + "units": { + "file:tests/job_graph/test_numerical_stages.py": 0.0, + "file:tests/poller/recovery/test_atomic_publish.py": 0.0, + "file:tests/poller/recovery/test_empty_publication.py": 0.0, + "file:tests/poller/recovery/test_r01_stage_failures.py": 0.0, + "file:tests/poller/recovery/test_r02_newer_revote_during_retry.py": 0.0, + "file:tests/poller/recovery/test_r03_park_then_silence.py": 0.0, + "file:tests/poller/recovery/test_r04_park_unpark_races.py": 0.0, + "file:tests/poller/recovery/test_r05_mid_batch_restart.py": 0.0, + "file:tests/poller/recovery/test_r06_lru_in_flight.py": 0.0, + "file:tests/poller/recovery/test_r07_overlapping_shards.py": 0.2, + "file:tests/poller/recovery/test_r08_timestamp_boundary.py": 0.0, + "file:tests/poller/recovery/test_r09_partial_tables_readers.py": 0.0, + "file:tests/poller/recovery/test_r10_poll_load_failure_fairness.py": 0.0, + "file:tests/poller/recovery/test_r11_namespace_config.py": 0.0, + "file:tests/poller/recovery/test_r12_publication_cursor.py": 0.0, + "file:tests/poller/test_admission.py": 2.8, + "file:tests/poller/test_backfill.py": 0.2, + "file:tests/poller/test_backfill_postgres.py": 40.5, + "file:tests/poller/test_cache_lock.py": 0.0, + "file:tests/poller/test_capacity.py": 0.0, + "file:tests/poller/test_capacity_queue.py": 0.1, + "file:tests/poller/test_capacity_queue_postgres.py": 12.5, + "file:tests/poller/test_child_process_fence.py": 0.8, + "file:tests/poller/test_coalescing.py": 0.0, + "file:tests/poller/test_collect_readiness.py": 65.0, + "file:tests/poller/test_empty_output.py": 0.0, + "file:tests/poller/test_enqueue_math_rebuild.py": 0.0, + "file:tests/poller/test_error_path.py": 0.0, + "file:tests/poller/test_integration_postgres.py": 0.1, + "file:tests/poller/test_load_or_init.py": 0.4, + "file:tests/poller/test_math_writer.py": 0.0, + "file:tests/poller/test_ops_fixture_lines.py": 0.0, + "file:tests/poller/test_park_rebuild_recovery.py": 0.0, + "file:tests/poller/test_poll_once_timeout.py": 0.0, + "file:tests/poller/test_postgres_client_pid_types.py": 0.0, + "file:tests/poller/test_postgres_vote_chokepoint.py": 0.0, + "file:tests/poller/test_promotion.py": 0.0, + "file:tests/poller/test_promotion_postgres.py": 26.9, + "file:tests/poller/test_readiness.py": 0.5, + "file:tests/poller/test_readiness_sources.py": 0.0, + "file:tests/poller/test_recovery.py": 0.0, + "file:tests/poller/test_revote_order.py": 0.0, + "file:tests/poller/test_serialization.py": 0.2, + "file:tests/poller/test_served_env_guard.py": 0.0, + "file:tests/poller/test_service.py": 0.0, + "file:tests/poller/test_single_writer_guards.py": 8.2, + "file:tests/poller/test_single_writer_lock.py": 68.0, + "file:tests/poller/test_watermark.py": 0.0, + "file:tests/projgate/projection_gate_test.py": 0.0, + "file:tests/projgate/projection_inventory_test.py": 1.7, + "file:tests/projgate/test_checkout_locator.py": 0.0, + "file:tests/replay_harness/test_attribution_observer.py": 0.6, + "file:tests/replay_harness/test_axis_continuity.py": 0.1, + "file:tests/replay_harness/test_battery_coverage.py": 0.1, + "file:tests/replay_harness/test_certify.py": 2.7, + "file:tests/replay_harness/test_certify_cache_keys_p019.py": 0.0, + "file:tests/replay_harness/test_certify_canonicalization.py": 0.1, + "file:tests/replay_harness/test_certify_cli.py": 0.0, + "file:tests/replay_harness/test_certify_strict_gate.py": 3.2, + "file:tests/replay_harness/test_ci_recordings_manifest.py": 0.3, + "file:tests/replay_harness/test_cli.py": 1.3, + "file:tests/replay_harness/test_clj_crosslang.py": 0.1, + "file:tests/replay_harness/test_clj_moderation_tail.py": 0.0, + "file:tests/replay_harness/test_decision_ties.py": 0.0, + "file:tests/replay_harness/test_driver.py": 1.0, + "file:tests/replay_harness/test_event_ingress.py": 0.0, + "file:tests/replay_harness/test_polarity_property.py": 8.7, + "file:tests/replay_harness/test_poller_equiv_compare.py": 0.6, + "file:tests/replay_harness/test_poller_equiv_envelope.py": 0.3, + "file:tests/replay_harness/test_poller_equiv_seed.py": 0.1, + "file:tests/replay_harness/test_real_data_local.py": 0.0, + "file:tests/replay_harness/test_real_data_mod_events.py": 0.0, + "file:tests/replay_harness/test_schedule.py": 0.1, + "file:tests/replay_harness/test_shard_bench.py": 0.1, + "file:tests/replay_harness/test_stage_oracle_e2e.py": 0.8, + "file:tests/replay_harness/test_stages.py": 0.2, + "file:tests/replay_harness/test_stepcompare.py": 0.1, + "file:tests/replay_harness/test_store.py": 0.0, + "file:tests/test_501_calculate_comment_extremity.py": 0.0, + "file:tests/test_after_install_hook.py": 0.8, + "file:tests/test_aws_credentials.py": 9.1, + "file:tests/test_base_cluster_lineage.py": 0.7, + "file:tests/test_batch_id.py": 4.2, + "file:tests/test_before_install_hook.py": 0.0, + "file:tests/test_benchmark_boundaries.py": 0.0, + "file:tests/test_benchmarks_importable.py": 0.0, + "file:tests/test_bool_flags.py": 40.8, + "file:tests/test_certify_bundle.py": 10.4, + "file:tests/test_certify_bundle_integration.py": 7.9, + "file:tests/test_certify_fixture_config.py": 0.2, + "file:tests/test_certify_probe_config.py": 1.6, + "file:tests/test_certify_served_math.py": 0.5, + "file:tests/test_clj_hash_order.py": 0.0, + "file:tests/test_clojure_comparer.py": 0.0, + "file:tests/test_clusters.py": 0.0, + "file:tests/test_compose_math_env.py": 2.0, + "file:tests/test_convergence_distance.py": 0.0, + "file:tests/test_conversation.py": 0.2, + "file:tests/test_conversation_smoke.py": 1.5, + "file:tests/test_coordinator_collection.py": 2.8, + "file:tests/test_corr.py": 0.0, + "file:tests/test_dataset_selection.py": 0.2, + "file:tests/test_datasets.py": 0.0, + "file:tests/test_degenerate_tick_parity.py": 0.2, + "file:tests/test_delphi_legacy_import.py": 1.2, + "file:tests/test_delphi_postgres_results.py": 0.0, + "file:tests/test_delphi_result_resource.py": 0.0, + "file:tests/test_delphi_result_resource_pagination.py": 0.3, + "file:tests/test_delphi_storage_codec.py": 0.0, + "file:tests/test_delphi_vote_convention_sites.py": 0.1, + "file:tests/test_delphi_writer.py": 0.2, + "file:tests/test_discrepancy_fixes.py": 0.2, + "file:tests/test_dsn_admission.py": 0.0, + "file:tests/test_dynamodb_consensus_roundtrip.py": 0.0, + "file:tests/test_dynamodb_float_serialization.py": 0.1, + "file:tests/test_edge_cases.py": 0.1, + "file:tests/test_empty_conversation_readers.py": 0.0, + "file:tests/test_engine_rebuild.py": 7.4, + "file:tests/test_entrypoint_boundaries.py": 0.0, + "file:tests/test_env_flags.py": 0.1, + "file:tests/test_generator_vote_copy.py": 0.0, + "file:tests/test_geometry_axis_pair.py": 0.1, + "file:tests/test_group_k_smoother.py": 0.3, + "file:tests/test_group_vote_counts.py": 0.0, + "file:tests/test_import_report.py": 0.7, + "file:tests/test_in_conv_greedy_carry.py": 0.2, + "file:tests/test_job_child_protocol.py": 14.9, + "file:tests/test_large_conv_tick_bench.py": 0.0, + "file:tests/test_legacy_blob_shape.py": 0.3, + "file:tests/test_legacy_kmeans.py": 0.1, + "file:tests/test_light_shadow_triage.py": 0.8, + "file:tests/test_math_pipeline_runs_e2e.py": 1.6, + "file:tests/test_math_writer_numpy_serialization.py": 0.1, + "file:tests/test_minio_access.py": 0.0, + "file:tests/test_mod_ptpt_leak_parity.py": 0.2, + "file:tests/test_mod_update_parity.py": 0.1, + "file:tests/test_most_distal_distance.py": 0.0, + "file:tests/test_narrative_audit.py": 0.0, + "file:tests/test_null_vote_fold.py": 0.4, + "file:tests/test_participant_info.py": 0.0, + "file:tests/test_pca_column_order.py": 0.5, + "file:tests/test_pca_edge_cases.py": 0.0, + "file:tests/test_pca_operation_parity.py": 0.0, + "file:tests/test_pca_unit.py": 0.0, + "file:tests/test_pca_warm_start.py": 0.1, + "file:tests/test_postgres_real_data.py": 0.1, + "file:tests/test_powerit_pca.py": 0.0, + "file:tests/test_priority_unmirror.py": 0.1, + "file:tests/test_probe_reader_snapshot.py": 0.1, + "file:tests/test_prodclone_extract.py": 0.2, + "file:tests/test_production_import.py": 0.6, + "file:tests/test_public_fixture_cli.py": 0.0, + "file:tests/test_queue_noop_executor.py": 0.0, + "file:tests/test_regression.py": 0.0, + "file:tests/test_repness_fallback_precision.py": 0.5, + "file:tests/test_repness_unit.py": 1.1, + "file:tests/test_report_math_env.py": 0.9, + "file:tests/test_representative_payloads.py": 6.2, + "file:tests/test_representative_selection.py": 0.3, + "file:tests/test_reset_conversation.py": 0.0, + "file:tests/test_run_delphi_exit_codes.py": 3.2, + "file:tests/test_run_math_pipeline_main.py": 0.0, + "file:tests/test_sanity_checks.py": 0.6, + "file:tests/test_scan_pacing.py": 0.4, + "file:tests/test_serialization_unfolding.py": 0.0, + "file:tests/test_umap_narrative_pipeline.py": 0.0, + "file:tests/test_validate_service_hook.py": 1.2, + "file:tests/test_vote_convention_source.py": 0.0, + "file:tests/topic_naming/test_job_routing.py": 0.0, + "file:tests/topic_naming/test_model_provider_batch.py": 0.4, + "file:tests/topic_naming/test_topic_naming.py": 0.0, + "group:dataset-biodiversity": 14.0, + "group:dataset-vw": 3.1 + } +} diff --git a/ci/local/delphi_python_results.py b/ci/local/delphi_python_results.py new file mode 100644 index 0000000000..e5335840fd --- /dev/null +++ b/ci/local/delphi_python_results.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +"""Result bookkeeping for ci/local/delphi-python.sh (standard library only). + + summary DIR print one run's counts (both passes added up) + combine --out OUT DIR... check that the shards' selections partition the + collected tests (each test in exactly one shard), + add up the counts, and write OUT/counts.json and + OUT/test-durations.json (seconds per shard unit, + the format of ci/local/delphi-test-durations.json) + +Counts are pytest's own summary numbers (passed, failed, skipped, xfailed, +xpassed, error), read from the controller of each pass. "deselected" is left +out: a test deselected by one pass or shard runs in another. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from collections import Counter, defaultdict +from pathlib import Path + +ORDER = ["passed", "failed", "error", "skipped", "xfailed", "xpassed"] + + +def run_counts(run_dir: Path) -> Counter: + total: Counter = Counter() + files = sorted(run_dir.glob("results-*.jsonl.counts.json")) + if not files: + raise SystemExit(f"{run_dir}: no pytest summary counts (did pytest start?)") + for f in files: + for key, n in json.loads(f.read_text()).items(): + if key != "deselected": + total[key] += n + return total + + +def fmt(c: Counter) -> str: + keys = ORDER + sorted(k for k in c if k not in ORDER) + return ", ".join(f"{c.get(k, 0)} {k}" for k in keys) + + +def durations(run_dir: Path, units: dict[str, str]) -> Counter: + out: Counter = Counter() + for f in sorted(run_dir.glob("results-*.jsonl")): + for line in f.read_text().splitlines(): + rec = json.loads(line) + unit = units.get(rec["nodeid"]) + if unit: + out[unit] += rec["duration"] + return out + + +def cmd_summary(args) -> int: + run_dir = Path(args.dir) + manifest = json.loads((run_dir / "manifest.json").read_text()) + c = run_counts(run_dir) + print(f"shard {manifest['shard']}: {len(manifest['selected'])} of " + f"{len(manifest['collected'])} collected tests selected; {fmt(c)}") + return 0 + + +def cmd_combine(args) -> int: + out = Path(args.out) + out.mkdir(parents=True, exist_ok=True) + collected = None + owner: dict[str, str] = {} + problems = [] + total: Counter = Counter() + unit_seconds: Counter = Counter() + for d in map(Path, args.dirs): + manifest = json.loads((d / "manifest.json").read_text()) + shard = manifest["shard"] + coll = sorted(manifest["collected"]) + if collected is None: + collected = coll + elif coll != collected: + problems.append(f"shard {shard} collected a different test set " + f"({len(coll)} vs {len(collected)})") + for nodeid in manifest["selected"]: + if nodeid in owner: + problems.append(f"{nodeid} selected by shards {owner[nodeid]} and {shard}") + owner[nodeid] = shard + c = run_counts(d) + print(f"shard {shard}: {len(manifest['selected'])} tests; {fmt(c)}") + total += c + unit_seconds += durations(d, manifest["units"]) + missing = sorted(set(collected or []) - set(owner)) + for nodeid in missing[:20]: + problems.append(f"{nodeid} was collected but no shard selected it") + if len(missing) > 20: + problems.append(f"... and {len(missing) - 20} more unselected tests") + print(f"all shards: {len(owner)} of {len(collected or [])} collected tests; {fmt(total)}") + (out / "counts.json").write_text(json.dumps(dict(total), sort_keys=True, indent=1) + "\n") + (out / "test-durations.json").write_text(json.dumps( + {"units": {u: round(s, 1) for u, s in sorted(unit_seconds.items())}}, + indent=1, sort_keys=True) + "\n") + if total.get("failed") or total.get("error"): + problems.append(f"{total.get('failed', 0)} failed and {total.get('error', 0)} errors across the shards") + for p in problems: + print(f"ERROR: {p}", file=sys.stderr) + return 1 if problems else 0 + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + sub = ap.add_subparsers(dest="cmd", required=True) + s = sub.add_parser("summary") + s.add_argument("dir") + c = sub.add_parser("combine") + c.add_argument("--out", required=True) + c.add_argument("dirs", nargs="+") + args = ap.parse_args() + return {"summary": cmd_summary, "combine": cmd_combine}[args.cmd](args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/delphi/generate_coverage_md.py b/delphi/generate_coverage_md.py index 3bc565cb57..bf0d3b656a 100644 --- a/delphi/generate_coverage_md.py +++ b/delphi/generate_coverage_md.py @@ -3,12 +3,12 @@ import xml.etree.ElementTree as ET import sys -def main(): +def main(path='/app/coverage.xml'): """ Parses a coverage.xml file and prints a Markdown table to stdout. """ try: - tree = ET.parse('/app/coverage.xml') + tree = ET.parse(path) root = tree.getroot() lines = [ @@ -77,4 +77,4 @@ def main(): sys.exit(1) if __name__ == "__main__": - main() \ No newline at end of file + main(*sys.argv[1:2]) \ No newline at end of file diff --git a/delphi/pyproject.toml b/delphi/pyproject.toml index 264abcabd8..4723ac35a7 100644 --- a/delphi/pyproject.toml +++ b/delphi/pyproject.toml @@ -134,11 +134,13 @@ include = [ # Pytest configuration [tool.pytest.ini_options] -# Force sequential execution (-n0) to leverage the session-scoped conversation cache. -# The cache shares computed conversations across test files, but each xdist worker -# has its own process with a separate cache. With only 2 datasets (biodiversity, vw), -# sequential execution with caching is ~6x faster than parallel execution where each -# worker recomputes the conversations independently. +# A plain `pytest` runs sequentially (-n0) so the session-scoped conversation cache +# is shared across test files: each xdist worker has its own process and cache, and +# with `--dist load` every worker would recompute the same conversations. +# ci/local/delphi-python.sh (the entry point CI calls) overrides this with +# `-n --dist loadgroup`: dataset-parametrized tests carry +# xdist_group(), so each conversation is still computed by one worker only. +# Tests marked `serial` run afterwards in a second, sequential pass. addopts = "-n0" filterwarnings = [ # Ignore python_multipart deprecation warning from ddtrace (third-party) @@ -150,6 +152,7 @@ markers = [ "local_dataset: mark test as using local (non-committed) datasets from real_data/.local/", "clojure_comparison: mark test as comparing with Clojure reference implementation (can be excluded with '-m \"not clojure_comparison\"')", "integration: mark test as an opt-in integration test needing a real Postgres (self-skips if docker/port unavailable)", + "serial: must not run concurrently with other tests (shared cluster-wide state or timing); the CI entry point runs these in a sequential pass after the parallel one", "recovery: P-022 §C poller recovery matrix (R01-R12); REQUIRES a real Postgres via POLIS_TEST_POSTGRES_URL and FAILS (never skips) without one — run via `make test-recovery`", ] diff --git a/delphi/tests/ci_partition.py b/delphi/tests/ci_partition.py new file mode 100644 index 0000000000..ef0d09d5a4 --- /dev/null +++ b/delphi/tests/ci_partition.py @@ -0,0 +1,171 @@ +"""Shards and result records for ci/local/delphi-python.sh. + +Registered from conftest.pytest_configure. It does nothing unless the entry +point sets one of these environment variables: + + DELPHI_TEST_SHARD="I/N" keep only shard I of N (1-based). Tests are + split by unit: an xdist_group name when the + test has one (so a dataset is computed in one + shard only), otherwise its module file. Units + are spread over shards by expected seconds + (ci/local/delphi-test-durations.json, a + per-test default for units it does not list), + longest first, so the shards finish together. + DELPHI_TEST_MANIFEST= write the collected and selected node ids + (before -m), so the combine step can prove every + test ran in exactly one shard. + DELPHI_TEST_RESULTS= append one JSON line per test report + (controller side under xdist) and write + pytest's summary counts to .counts.json. + +Selection never drops a test from the whole run: each unit is in exactly one +shard, and the combine step fails if the shard manifests do not add up to the +collected set. +""" + +from __future__ import annotations + +import json +import os +from pathlib import Path + +import pytest + +# Seconds assumed for a test in a unit the durations file does not list. +DEFAULT_TEST_SECONDS = 0.5 + + +def base_nodeid(nodeid: str) -> str: + """Strip the '@group' suffix xdist's loadgroup scheduler appends.""" + if "@" in nodeid.rsplit("::", 1)[-1]: + return nodeid.rsplit("@", 1)[0] + return nodeid + + +def dataset_of(item) -> str: + """The dataset a test is parametrized with ('' if none); the same rule as + conftest's reordering (composite '-' ids map to the dataset).""" + params = getattr(getattr(item, "callspec", None), "params", None) or {} + for name in ("dataset_name", "dataset_blob_id"): + value = params.get(name) + if isinstance(value, str): + for suffix in ("-incremental", "-cold_start"): + if value.endswith(suffix): + return value[: -len(suffix)] + return value + return "" + + +def unit_of(item) -> str: + mark = item.get_closest_marker("xdist_group") + if mark is not None: + name = mark.kwargs.get("name", mark.args[0] if mark.args else "") + if name: + return f"group:{name}" + return "file:" + base_nodeid(item.nodeid).split("::", 1)[0] + + +def _load_durations() -> dict[str, float]: + path = os.environ.get("DELPHI_TEST_DURATIONS") + if not path or not Path(path).is_file(): + return {} + data = json.loads(Path(path).read_text()) + return {str(k): float(v) for k, v in data.get("units", {}).items()} + + +def assign_shards(units: dict[str, int], durations: dict[str, float], count: int) -> dict[str, int]: + """Longest-expected unit first onto the least-loaded shard (1-based).""" + weight = {u: durations.get(u, n * DEFAULT_TEST_SECONDS) for u, n in units.items()} + loads = [0.0] * count + out = {} + for unit in sorted(units, key=lambda u: (-weight[u], u)): + shard = min(range(count), key=lambda s: (loads[s], s)) + loads[shard] += weight[unit] + out[unit] = shard + 1 + return out + + +def _parse_shard(text: str) -> tuple[int, int]: + try: + index, count = (int(p) for p in text.split("/")) + except ValueError: + raise pytest.UsageError(f"DELPHI_TEST_SHARD must look like 2/3, got {text!r}") + if not 1 <= index <= count: + raise pytest.UsageError(f"DELPHI_TEST_SHARD out of range: {text!r}") + return index, count + + +def _is_xdist_worker(config) -> bool: + return hasattr(config, "workerinput") + + +class CiPartition: + def __init__(self, config): + self.config = config + self.shard = os.environ.get("DELPHI_TEST_SHARD") or None + self.manifest = os.environ.get("DELPHI_TEST_MANIFEST") or None + self.results = os.environ.get("DELPHI_TEST_RESULTS") or None + # Opened once, before any test runs: some tests monkeypatch open(). + self._results_fh = None + if self.results and not _is_xdist_worker(config): + self._results_fh = open(self.results, "a", buffering=1) + + # tryfirst: see the full collection before -m/-k deselect anything. + @pytest.hookimpl(tryfirst=True) + def pytest_collection_modifyitems(self, session, config, items): + # Each dataset's tests share one session-scoped conversation cache: keep + # them on one xdist worker (--dist loadgroup) and in one shard, so the + # conversation is computed once rather than once per worker. + for item in items: + ds = dataset_of(item) + if ds and item.get_closest_marker("xdist_group") is None: + item.add_marker(pytest.mark.xdist_group(f"dataset-{ds}")) + collected = [base_nodeid(i.nodeid) for i in items] + units: dict[str, int] = {} + for item in items: + units[unit_of(item)] = units.get(unit_of(item), 0) + 1 + index, count = _parse_shard(self.shard) if self.shard else (1, 1) + if count > 1: + plan = assign_shards(units, _load_durations(), count) + keep, drop = [], [] + for item in items: + (keep if plan[unit_of(item)] == index else drop).append(item) + if drop: + config.hook.pytest_deselected(items=drop) + items[:] = keep + # One writer: the plain run, or the first xdist worker (every worker + # collects the same list; xdist refuses to run otherwise). + worker = getattr(config, "workerinput", {}).get("workerid") + if self.manifest and worker in (None, "gw0"): + Path(self.manifest).write_text(json.dumps({ + "shard": f"{index}/{count}", + "collected": collected, + "selected": [base_nodeid(i.nodeid) for i in items], + "units": {base_nodeid(i.nodeid): unit_of(i) for i in items}, + })) + + def pytest_runtest_logreport(self, report): + if self._results_fh is None: + return + node = getattr(report, "node", None) + self._results_fh.write(json.dumps({ + "nodeid": base_nodeid(report.nodeid), + "when": report.when, + "outcome": report.outcome, + "xfail": hasattr(report, "wasxfail"), + "duration": report.duration, + "worker": node.gateway.id if node is not None else None, + }) + "\n") + + # The same numbers as pytest's final summary line, from the controller. + def pytest_terminal_summary(self, terminalreporter): + if not self.results or _is_xdist_worker(self.config): + return + counts = {key: len(reports) for key, reports in terminalreporter.stats.items() + if key and key != "warnings"} + Path(self.results + ".counts.json").write_text(json.dumps(counts, sort_keys=True)) + + +def register(config): + if any(os.environ.get(k) for k in ("DELPHI_TEST_SHARD", "DELPHI_TEST_MANIFEST", "DELPHI_TEST_RESULTS")): + config.pluginmanager.register(CiPartition(config), "delphi_ci_partition") diff --git a/delphi/tests/conftest.py b/delphi/tests/conftest.py index 3632e438ec..5937286858 100644 --- a/delphi/tests/conftest.py +++ b/delphi/tests/conftest.py @@ -371,7 +371,9 @@ def _get_requested_datasets(config) -> set[str] | None: def pytest_configure(config): - """Register custom markers.""" + """Register custom markers, and the CI shard/result plugin when asked.""" + from tests.ci_partition import register as register_ci_partition + register_ci_partition(config) config.addinivalue_line( "markers", "use_discovered_datasets(use_blobs=False): dynamically parametrize with discovered " diff --git a/delphi/tests/poller/test_backfill_postgres.py b/delphi/tests/poller/test_backfill_postgres.py index e4a0c48951..2c0375856a 100644 --- a/delphi/tests/poller/test_backfill_postgres.py +++ b/delphi/tests/poller/test_backfill_postgres.py @@ -26,7 +26,9 @@ from tests.conftest import require_polis_postgres from tests.vote_fixtures import AGREE, DISAGREE, seed_vote -pytestmark = pytest.mark.integration +# serial: seeds fixed zids (13500...) in the shared test database and creates the +# cluster-wide role math_queue_test; concurrent workers collide on both. +pytestmark = [pytest.mark.integration, pytest.mark.serial] DAY_MS = 24 * 60 * 60 * 1000 VERIFY_SQL = (Path(__file__).resolve().parents[2] / "polismath" / "poller" diff --git a/delphi/tests/poller/test_capacity_queue_postgres.py b/delphi/tests/poller/test_capacity_queue_postgres.py index 5f25c832ec..5e286285ad 100644 --- a/delphi/tests/poller/test_capacity_queue_postgres.py +++ b/delphi/tests/poller/test_capacity_queue_postgres.py @@ -41,7 +41,9 @@ from tests.conftest import require_polis_postgres from tests.poller.test_backfill_postgres import _TABLES, coherent, q, seed_conversation -pytestmark = pytest.mark.integration +# serial: creates the cluster-wide login role math_queue_test (fixed name, shared +# with test_backfill_postgres.py); concurrent workers fail with "role already exists". +pytestmark = [pytest.mark.integration, pytest.mark.serial] MB = 1024 * 1024 HERE = Path(__file__).resolve().parent diff --git a/delphi/tests/test_certify_bundle_integration.py b/delphi/tests/test_certify_bundle_integration.py index 5e84a8d914..f2ab6458b8 100644 --- a/delphi/tests/test_certify_bundle_integration.py +++ b/delphi/tests/test_certify_bundle_integration.py @@ -40,7 +40,9 @@ #: The stored value of each vote the seeds write, named by meaning. STORED = {name: seed_vote(name) for name in ("agree", "disagree", "pass")} -pytestmark = pytest.mark.integration +# serial: seeds fixed conversation ids (zid 1990000001...) in the shared test +# database; a second worker seeding the same rows hits a duplicate key. +pytestmark = [pytest.mark.integration, pytest.mark.serial] # --------------------------------------------------------------------------- # Planted public-fixture identifiers — the redaction assertions look for these. diff --git a/delphi/tests/test_engine_rebuild.py b/delphi/tests/test_engine_rebuild.py index 7be529f3fd..ddfd5ca666 100644 --- a/delphi/tests/test_engine_rebuild.py +++ b/delphi/tests/test_engine_rebuild.py @@ -27,6 +27,10 @@ LABEL = "probe" ZIDS = (9101, 9102) +# serial: seeds these fixed zids in the shared test database; concurrent workers +# collide on duplicate keys. +pytestmark = pytest.mark.serial + def run(argv, environ=None): out = io.StringIO() diff --git a/delphi/tests/test_generator_vote_copy.py b/delphi/tests/test_generator_vote_copy.py index 3850bf307a..6bfe164f28 100644 --- a/delphi/tests/test_generator_vote_copy.py +++ b/delphi/tests/test_generator_vote_copy.py @@ -29,7 +29,9 @@ from tests.conftest import require_polis_postgres from tests.vote_fixtures import AGREE, DISAGREE, seed_vote -pytestmark = pytest.mark.integration +# serial: every test seeds and deletes fixed zids (SOURCE_ZID, FAKE_ZID) in the +# shared test database; two workers at once saw each other's rows (assert 7 == 12). +pytestmark = [pytest.mark.integration, pytest.mark.serial] def _load_generator(): diff --git a/delphi/tests/test_prodclone_extract.py b/delphi/tests/test_prodclone_extract.py index 988d727c09..bfc94bbbad 100644 --- a/delphi/tests/test_prodclone_extract.py +++ b/delphi/tests/test_prodclone_extract.py @@ -637,7 +637,11 @@ def _seed(cur, zid, uid_start, *, votes, comments, participants, banned_pids=()) ) +# serial: seeds zids 101001-101005 in the shared test database and leaves them +# there; when it ran before test_certify_bundle_integration.py, that survey picked +# zid 101005 as its zero-vote conversation instead of the one it planted. @pytest.mark.integration +@pytest.mark.serial def test_survey_and_extract_round_trip(tmp_path, monkeypatch): from tests.conftest import require_polis_postgres From f75bfb88d1edc8bfa01e64fc1f860ea28db377d5 Mon Sep 17 00:00:00 2001 From: Colin Megill Date: Mon, 12 Oct 2026 01:28:43 -0700 Subject: [PATCH 2/3] CI: shard by expected wall time; rebuild fixture outside the poller window The shard plan now weighs each unit by expected wall seconds: parallel-pass seconds shared by four workers, plus serial-pass seconds at full weight. The durations file holds [parallel, serial] seconds per unit, measured on a hosted run. The plan uses a fixed worker count, so every machine computes the same plan. test_engine_rebuild's generated conversations had vote timestamps at 2026-10-03, inside the 10-day start window of the stack's live math-python poller. The poller could compute them under its own label, and the assertion that only the rebuild label has rows then failed (assert 2 == 0). Whether it did depended on which tests ran first, so a shard hit it. The fixture now uses 2020 timestamps. The assertion is unchanged. Shard results no longer include the compose inputs (stack/). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018NVGBuYCk4EZmiz4csUv9s --- .github/workflows/python-ci.yml | 4 +- ci/local/delphi-python.sh | 11 +- ci/local/delphi-test-durations.json | 867 ++++++++++++++++++++++------ ci/local/delphi_python_results.py | 26 +- delphi/tests/ci_partition.py | 21 +- delphi/tests/test_engine_rebuild.py | 6 +- 6 files changed, 742 insertions(+), 193 deletions(-) diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index ca8087cc76..dcb2fb3323 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -81,7 +81,9 @@ jobs: uses: actions/upload-artifact@v4 with: name: delphi-shard-${{ matrix.shard }} - path: delphi-results/shard-${{ matrix.shard }} + path: | + delphi-results/shard-${{ matrix.shard }} + !delphi-results/shard-${{ matrix.shard }}/stack if-no-files-found: ignore test: diff --git a/ci/local/delphi-python.sh b/ci/local/delphi-python.sh index 97a2280ac5..7907a883d5 100755 --- a/ci/local/delphi-python.sh +++ b/ci/local/delphi-python.sh @@ -104,16 +104,17 @@ SHARD_I="${SHARD%/*}"; SHARD_N="${SHARD#*/}" PROJECT="${DELPHI_CHECK_PROJECT:-delphi-python-${SHARD_I}of${SHARD_N}-$(printf '%s' "$ROOT" | sha)}" OUT="${DELPHI_CHECK_OUT:-$ROOT/.ci-out/$PROJECT}" -rm -rf "$OUT"; mkdir -p "$OUT" +rm -rf "$OUT"; mkdir -p "$OUT/stack" OUT="$(cd "$OUT" && pwd)" +STACK="$OUT/stack" # compose inputs, kept apart from the results log "project $PROJECT; shard $SHARD; results $OUT" # test.env plus the in-network hostnames the delphi service reads. [ -f test.env ] || die "test.env not found" -{ cat test.env; echo "POSTGRES_HOST=postgres"; echo "DYNAMODB_ENDPOINT=http://dynamodb:8000"; echo "AWS_S3_ENDPOINT=http://minio:9000"; } > "$OUT/test.env" +{ cat test.env; echo "POSTGRES_HOST=postgres"; echo "DYNAMODB_ENDPOINT=http://dynamodb:8000"; echo "AWS_S3_ENDPOINT=http://minio:9000"; } > "$STACK/test.env" # No published host ports: nothing in this check reaches the stack from the host. -cat > "$OUT/no-host-ports.yml" <<'EOF' +cat > "$STACK/no-host-ports.yml" <<'EOF' services: client-participation-alpha: { ports: !reset [] } ses-local: { ports: !reset [] } @@ -123,7 +124,7 @@ services: minio: { ports: !reset [] } dynamodb: { ports: !reset [] } EOF -COMPOSE_ARGS=(-p "$PROJECT" -f docker-compose.test.yml -f "$OUT/no-host-ports.yml" --env-file "$OUT/test.env") +COMPOSE_ARGS=(-p "$PROJECT" -f docker-compose.test.yml -f "$STACK/no-host-ports.yml" --env-file "$STACK/test.env") compose() { docker compose "${COMPOSE_ARGS[@]}" "$@"; } cleanup() { @@ -153,7 +154,7 @@ endgroup group "apply and check migrations" # test-migrations.sh puts its own -f docker-compose.test.yml first; it reads the # env file from POLIS_TEST_ENV_FILE. -POLIS_TEST_ENV_FILE="$OUT/test.env" bash ci/test-migrations.sh -p "$PROJECT" -f "$OUT/no-host-ports.yml" +POLIS_TEST_ENV_FILE="$STACK/test.env" bash ci/test-migrations.sh -p "$PROJECT" -f "$STACK/no-host-ports.yml" endgroup group "start the stack" diff --git a/ci/local/delphi-test-durations.json b/ci/local/delphi-test-durations.json index 14fc2f12a6..88269f2f64 100644 --- a/ci/local/delphi-test-durations.json +++ b/ci/local/delphi-test-durations.json @@ -1,177 +1,696 @@ { "units": { - "file:tests/job_graph/test_numerical_stages.py": 0.0, - "file:tests/poller/recovery/test_atomic_publish.py": 0.0, - "file:tests/poller/recovery/test_empty_publication.py": 0.0, - "file:tests/poller/recovery/test_r01_stage_failures.py": 0.0, - "file:tests/poller/recovery/test_r02_newer_revote_during_retry.py": 0.0, - "file:tests/poller/recovery/test_r03_park_then_silence.py": 0.0, - "file:tests/poller/recovery/test_r04_park_unpark_races.py": 0.0, - "file:tests/poller/recovery/test_r05_mid_batch_restart.py": 0.0, - "file:tests/poller/recovery/test_r06_lru_in_flight.py": 0.0, - "file:tests/poller/recovery/test_r07_overlapping_shards.py": 0.2, - "file:tests/poller/recovery/test_r08_timestamp_boundary.py": 0.0, - "file:tests/poller/recovery/test_r09_partial_tables_readers.py": 0.0, - "file:tests/poller/recovery/test_r10_poll_load_failure_fairness.py": 0.0, - "file:tests/poller/recovery/test_r11_namespace_config.py": 0.0, - "file:tests/poller/recovery/test_r12_publication_cursor.py": 0.0, - "file:tests/poller/test_admission.py": 2.8, - "file:tests/poller/test_backfill.py": 0.2, - "file:tests/poller/test_backfill_postgres.py": 40.5, - "file:tests/poller/test_cache_lock.py": 0.0, - "file:tests/poller/test_capacity.py": 0.0, - "file:tests/poller/test_capacity_queue.py": 0.1, - "file:tests/poller/test_capacity_queue_postgres.py": 12.5, - "file:tests/poller/test_child_process_fence.py": 0.8, - "file:tests/poller/test_coalescing.py": 0.0, - "file:tests/poller/test_collect_readiness.py": 65.0, - "file:tests/poller/test_empty_output.py": 0.0, - "file:tests/poller/test_enqueue_math_rebuild.py": 0.0, - "file:tests/poller/test_error_path.py": 0.0, - "file:tests/poller/test_integration_postgres.py": 0.1, - "file:tests/poller/test_load_or_init.py": 0.4, - "file:tests/poller/test_math_writer.py": 0.0, - "file:tests/poller/test_ops_fixture_lines.py": 0.0, - "file:tests/poller/test_park_rebuild_recovery.py": 0.0, - "file:tests/poller/test_poll_once_timeout.py": 0.0, - "file:tests/poller/test_postgres_client_pid_types.py": 0.0, - "file:tests/poller/test_postgres_vote_chokepoint.py": 0.0, - "file:tests/poller/test_promotion.py": 0.0, - "file:tests/poller/test_promotion_postgres.py": 26.9, - "file:tests/poller/test_readiness.py": 0.5, - "file:tests/poller/test_readiness_sources.py": 0.0, - "file:tests/poller/test_recovery.py": 0.0, - "file:tests/poller/test_revote_order.py": 0.0, - "file:tests/poller/test_serialization.py": 0.2, - "file:tests/poller/test_served_env_guard.py": 0.0, - "file:tests/poller/test_service.py": 0.0, - "file:tests/poller/test_single_writer_guards.py": 8.2, - "file:tests/poller/test_single_writer_lock.py": 68.0, - "file:tests/poller/test_watermark.py": 0.0, - "file:tests/projgate/projection_gate_test.py": 0.0, - "file:tests/projgate/projection_inventory_test.py": 1.7, - "file:tests/projgate/test_checkout_locator.py": 0.0, - "file:tests/replay_harness/test_attribution_observer.py": 0.6, - "file:tests/replay_harness/test_axis_continuity.py": 0.1, - "file:tests/replay_harness/test_battery_coverage.py": 0.1, - "file:tests/replay_harness/test_certify.py": 2.7, - "file:tests/replay_harness/test_certify_cache_keys_p019.py": 0.0, - "file:tests/replay_harness/test_certify_canonicalization.py": 0.1, - "file:tests/replay_harness/test_certify_cli.py": 0.0, - "file:tests/replay_harness/test_certify_strict_gate.py": 3.2, - "file:tests/replay_harness/test_ci_recordings_manifest.py": 0.3, - "file:tests/replay_harness/test_cli.py": 1.3, - "file:tests/replay_harness/test_clj_crosslang.py": 0.1, - "file:tests/replay_harness/test_clj_moderation_tail.py": 0.0, - "file:tests/replay_harness/test_decision_ties.py": 0.0, - "file:tests/replay_harness/test_driver.py": 1.0, - "file:tests/replay_harness/test_event_ingress.py": 0.0, - "file:tests/replay_harness/test_polarity_property.py": 8.7, - "file:tests/replay_harness/test_poller_equiv_compare.py": 0.6, - "file:tests/replay_harness/test_poller_equiv_envelope.py": 0.3, - "file:tests/replay_harness/test_poller_equiv_seed.py": 0.1, - "file:tests/replay_harness/test_real_data_local.py": 0.0, - "file:tests/replay_harness/test_real_data_mod_events.py": 0.0, - "file:tests/replay_harness/test_schedule.py": 0.1, - "file:tests/replay_harness/test_shard_bench.py": 0.1, - "file:tests/replay_harness/test_stage_oracle_e2e.py": 0.8, - "file:tests/replay_harness/test_stages.py": 0.2, - "file:tests/replay_harness/test_stepcompare.py": 0.1, - "file:tests/replay_harness/test_store.py": 0.0, - "file:tests/test_501_calculate_comment_extremity.py": 0.0, - "file:tests/test_after_install_hook.py": 0.8, - "file:tests/test_aws_credentials.py": 9.1, - "file:tests/test_base_cluster_lineage.py": 0.7, - "file:tests/test_batch_id.py": 4.2, - "file:tests/test_before_install_hook.py": 0.0, - "file:tests/test_benchmark_boundaries.py": 0.0, - "file:tests/test_benchmarks_importable.py": 0.0, - "file:tests/test_bool_flags.py": 40.8, - "file:tests/test_certify_bundle.py": 10.4, - "file:tests/test_certify_bundle_integration.py": 7.9, - "file:tests/test_certify_fixture_config.py": 0.2, - "file:tests/test_certify_probe_config.py": 1.6, - "file:tests/test_certify_served_math.py": 0.5, - "file:tests/test_clj_hash_order.py": 0.0, - "file:tests/test_clojure_comparer.py": 0.0, - "file:tests/test_clusters.py": 0.0, - "file:tests/test_compose_math_env.py": 2.0, - "file:tests/test_convergence_distance.py": 0.0, - "file:tests/test_conversation.py": 0.2, - "file:tests/test_conversation_smoke.py": 1.5, - "file:tests/test_coordinator_collection.py": 2.8, - "file:tests/test_corr.py": 0.0, - "file:tests/test_dataset_selection.py": 0.2, - "file:tests/test_datasets.py": 0.0, - "file:tests/test_degenerate_tick_parity.py": 0.2, - "file:tests/test_delphi_legacy_import.py": 1.2, - "file:tests/test_delphi_postgres_results.py": 0.0, - "file:tests/test_delphi_result_resource.py": 0.0, - "file:tests/test_delphi_result_resource_pagination.py": 0.3, - "file:tests/test_delphi_storage_codec.py": 0.0, - "file:tests/test_delphi_vote_convention_sites.py": 0.1, - "file:tests/test_delphi_writer.py": 0.2, - "file:tests/test_discrepancy_fixes.py": 0.2, - "file:tests/test_dsn_admission.py": 0.0, - "file:tests/test_dynamodb_consensus_roundtrip.py": 0.0, - "file:tests/test_dynamodb_float_serialization.py": 0.1, - "file:tests/test_edge_cases.py": 0.1, - "file:tests/test_empty_conversation_readers.py": 0.0, - "file:tests/test_engine_rebuild.py": 7.4, - "file:tests/test_entrypoint_boundaries.py": 0.0, - "file:tests/test_env_flags.py": 0.1, - "file:tests/test_generator_vote_copy.py": 0.0, - "file:tests/test_geometry_axis_pair.py": 0.1, - "file:tests/test_group_k_smoother.py": 0.3, - "file:tests/test_group_vote_counts.py": 0.0, - "file:tests/test_import_report.py": 0.7, - "file:tests/test_in_conv_greedy_carry.py": 0.2, - "file:tests/test_job_child_protocol.py": 14.9, - "file:tests/test_large_conv_tick_bench.py": 0.0, - "file:tests/test_legacy_blob_shape.py": 0.3, - "file:tests/test_legacy_kmeans.py": 0.1, - "file:tests/test_light_shadow_triage.py": 0.8, - "file:tests/test_math_pipeline_runs_e2e.py": 1.6, - "file:tests/test_math_writer_numpy_serialization.py": 0.1, - "file:tests/test_minio_access.py": 0.0, - "file:tests/test_mod_ptpt_leak_parity.py": 0.2, - "file:tests/test_mod_update_parity.py": 0.1, - "file:tests/test_most_distal_distance.py": 0.0, - "file:tests/test_narrative_audit.py": 0.0, - "file:tests/test_null_vote_fold.py": 0.4, - "file:tests/test_participant_info.py": 0.0, - "file:tests/test_pca_column_order.py": 0.5, - "file:tests/test_pca_edge_cases.py": 0.0, - "file:tests/test_pca_operation_parity.py": 0.0, - "file:tests/test_pca_unit.py": 0.0, - "file:tests/test_pca_warm_start.py": 0.1, - "file:tests/test_postgres_real_data.py": 0.1, - "file:tests/test_powerit_pca.py": 0.0, - "file:tests/test_priority_unmirror.py": 0.1, - "file:tests/test_probe_reader_snapshot.py": 0.1, - "file:tests/test_prodclone_extract.py": 0.2, - "file:tests/test_production_import.py": 0.6, - "file:tests/test_public_fixture_cli.py": 0.0, - "file:tests/test_queue_noop_executor.py": 0.0, - "file:tests/test_regression.py": 0.0, - "file:tests/test_repness_fallback_precision.py": 0.5, - "file:tests/test_repness_unit.py": 1.1, - "file:tests/test_report_math_env.py": 0.9, - "file:tests/test_representative_payloads.py": 6.2, - "file:tests/test_representative_selection.py": 0.3, - "file:tests/test_reset_conversation.py": 0.0, - "file:tests/test_run_delphi_exit_codes.py": 3.2, - "file:tests/test_run_math_pipeline_main.py": 0.0, - "file:tests/test_sanity_checks.py": 0.6, - "file:tests/test_scan_pacing.py": 0.4, - "file:tests/test_serialization_unfolding.py": 0.0, - "file:tests/test_umap_narrative_pipeline.py": 0.0, - "file:tests/test_validate_service_hook.py": 1.2, - "file:tests/test_vote_convention_source.py": 0.0, - "file:tests/topic_naming/test_job_routing.py": 0.0, - "file:tests/topic_naming/test_model_provider_batch.py": 0.4, - "file:tests/topic_naming/test_topic_naming.py": 0.0, - "group:dataset-biodiversity": 14.0, - "group:dataset-vw": 3.1 + "file:tests/job_graph/test_numerical_stages.py": [ + 0.2, + 0 + ], + "file:tests/poller/recovery/test_atomic_publish.py": [ + 0.1, + 0 + ], + "file:tests/poller/recovery/test_empty_publication.py": [ + 0.0, + 0 + ], + "file:tests/poller/recovery/test_r01_stage_failures.py": [ + 0.0, + 0 + ], + "file:tests/poller/recovery/test_r02_newer_revote_during_retry.py": [ + 0.0, + 0 + ], + "file:tests/poller/recovery/test_r03_park_then_silence.py": [ + 0.0, + 0 + ], + "file:tests/poller/recovery/test_r04_park_unpark_races.py": [ + 0.0, + 0 + ], + "file:tests/poller/recovery/test_r05_mid_batch_restart.py": [ + 0.0, + 0 + ], + "file:tests/poller/recovery/test_r06_lru_in_flight.py": [ + 0.0, + 0 + ], + "file:tests/poller/recovery/test_r07_overlapping_shards.py": [ + 0.4, + 0 + ], + "file:tests/poller/recovery/test_r08_timestamp_boundary.py": [ + 0.0, + 0 + ], + "file:tests/poller/recovery/test_r09_partial_tables_readers.py": [ + 0.1, + 0 + ], + "file:tests/poller/recovery/test_r10_poll_load_failure_fairness.py": [ + 0.0, + 0 + ], + "file:tests/poller/recovery/test_r11_namespace_config.py": [ + 0.1, + 0 + ], + "file:tests/poller/recovery/test_r12_publication_cursor.py": [ + 0.0, + 0 + ], + "file:tests/poller/test_admission.py": [ + 2.7, + 0 + ], + "file:tests/poller/test_backfill.py": [ + 0.7, + 0 + ], + "file:tests/poller/test_backfill_postgres.py": [ + 0, + 56.5 + ], + "file:tests/poller/test_cache_lock.py": [ + 0.0, + 0 + ], + "file:tests/poller/test_capacity.py": [ + 0.3, + 0 + ], + "file:tests/poller/test_capacity_queue.py": [ + 0.2, + 0 + ], + "file:tests/poller/test_capacity_queue_postgres.py": [ + 0, + 29.1 + ], + "file:tests/poller/test_child_process_fence.py": [ + 1.2, + 0 + ], + "file:tests/poller/test_coalescing.py": [ + 0.0, + 0 + ], + "file:tests/poller/test_collect_readiness.py": [ + 232.1, + 0 + ], + "file:tests/poller/test_empty_output.py": [ + 0.4, + 0 + ], + "file:tests/poller/test_enqueue_math_rebuild.py": [ + 0.1, + 0 + ], + "file:tests/poller/test_error_path.py": [ + 0.1, + 0 + ], + "file:tests/poller/test_integration_postgres.py": [ + 0.8, + 0 + ], + "file:tests/poller/test_load_or_init.py": [ + 2.9, + 0 + ], + "file:tests/poller/test_math_writer.py": [ + 0.1, + 0 + ], + "file:tests/poller/test_ops_fixture_lines.py": [ + 0.0, + 0 + ], + "file:tests/poller/test_park_rebuild_recovery.py": [ + 0.1, + 0 + ], + "file:tests/poller/test_poll_once_timeout.py": [ + 0.1, + 0 + ], + "file:tests/poller/test_postgres_client_pid_types.py": [ + 0.0, + 0 + ], + "file:tests/poller/test_postgres_vote_chokepoint.py": [ + 0.0, + 0 + ], + "file:tests/poller/test_promotion.py": [ + 0.2, + 0 + ], + "file:tests/poller/test_promotion_postgres.py": [ + 67.9, + 0 + ], + "file:tests/poller/test_readiness.py": [ + 0.5, + 0 + ], + "file:tests/poller/test_readiness_sources.py": [ + 0.1, + 0 + ], + "file:tests/poller/test_recovery.py": [ + 0.0, + 0 + ], + "file:tests/poller/test_revote_order.py": [ + 0.0, + 0 + ], + "file:tests/poller/test_serialization.py": [ + 0.1, + 0 + ], + "file:tests/poller/test_served_env_guard.py": [ + 0.0, + 0 + ], + "file:tests/poller/test_service.py": [ + 0.1, + 0 + ], + "file:tests/poller/test_single_writer_guards.py": [ + 28.1, + 0 + ], + "file:tests/poller/test_single_writer_lock.py": [ + 160.6, + 0 + ], + "file:tests/poller/test_watermark.py": [ + 0.1, + 0 + ], + "file:tests/projgate/projection_gate_test.py": [ + 0.0, + 0 + ], + "file:tests/projgate/projection_inventory_test.py": [ + 7.8, + 0 + ], + "file:tests/projgate/test_checkout_locator.py": [ + 0.0, + 0 + ], + "file:tests/replay_harness/test_attribution_observer.py": [ + 1.9, + 0 + ], + "file:tests/replay_harness/test_axis_continuity.py": [ + 0.3, + 0 + ], + "file:tests/replay_harness/test_battery_coverage.py": [ + 0.2, + 0 + ], + "file:tests/replay_harness/test_certify.py": [ + 8.5, + 0 + ], + "file:tests/replay_harness/test_certify_cache_keys_p019.py": [ + 0.1, + 0 + ], + "file:tests/replay_harness/test_certify_canonicalization.py": [ + 0.4, + 0 + ], + "file:tests/replay_harness/test_certify_cli.py": [ + 0.1, + 0 + ], + "file:tests/replay_harness/test_certify_strict_gate.py": [ + 57.2, + 0 + ], + "file:tests/replay_harness/test_ci_recordings_manifest.py": [ + 1.0, + 0 + ], + "file:tests/replay_harness/test_cli.py": [ + 4.6, + 0 + ], + "file:tests/replay_harness/test_clj_crosslang.py": [ + 0.4, + 0 + ], + "file:tests/replay_harness/test_clj_moderation_tail.py": [ + 0.0, + 0 + ], + "file:tests/replay_harness/test_decision_ties.py": [ + 0.1, + 0 + ], + "file:tests/replay_harness/test_driver.py": [ + 3.2, + 0 + ], + "file:tests/replay_harness/test_event_ingress.py": [ + 0.2, + 0 + ], + "file:tests/replay_harness/test_polarity_property.py": [ + 53.1, + 0 + ], + "file:tests/replay_harness/test_poller_equiv_compare.py": [ + 0.9, + 0 + ], + "file:tests/replay_harness/test_poller_equiv_envelope.py": [ + 0.9, + 0 + ], + "file:tests/replay_harness/test_poller_equiv_seed.py": [ + 0.2, + 0 + ], + "file:tests/replay_harness/test_real_data_local.py": [ + 0.0, + 0 + ], + "file:tests/replay_harness/test_real_data_mod_events.py": [ + 0.1, + 0 + ], + "file:tests/replay_harness/test_schedule.py": [ + 0.4, + 0 + ], + "file:tests/replay_harness/test_shard_bench.py": [ + 0.5, + 0 + ], + "file:tests/replay_harness/test_stage_oracle_e2e.py": [ + 3.7, + 0 + ], + "file:tests/replay_harness/test_stages.py": [ + 0.8, + 0 + ], + "file:tests/replay_harness/test_stepcompare.py": [ + 0.1, + 0 + ], + "file:tests/replay_harness/test_store.py": [ + 0.2, + 0 + ], + "file:tests/test_501_calculate_comment_extremity.py": [ + 0.0, + 0 + ], + "file:tests/test_after_install_hook.py": [ + 2.3, + 0 + ], + "file:tests/test_aws_credentials.py": [ + 9.0, + 0 + ], + "file:tests/test_base_cluster_lineage.py": [ + 3.3, + 0 + ], + "file:tests/test_batch_id.py": [ + 4.5, + 0 + ], + "file:tests/test_before_install_hook.py": [ + 0.0, + 0 + ], + "file:tests/test_benchmark_boundaries.py": [ + 0.1, + 0 + ], + "file:tests/test_benchmarks_importable.py": [ + 0.0, + 0 + ], + "file:tests/test_bool_flags.py": [ + 171.8, + 0 + ], + "file:tests/test_certify_bundle.py": [ + 44.0, + 0 + ], + "file:tests/test_certify_bundle_integration.py": [ + 0, + 22.4 + ], + "file:tests/test_certify_fixture_config.py": [ + 0.4, + 0 + ], + "file:tests/test_certify_probe_config.py": [ + 7.8, + 0 + ], + "file:tests/test_certify_served_math.py": [ + 2.3, + 0 + ], + "file:tests/test_clj_hash_order.py": [ + 0.0, + 0 + ], + "file:tests/test_clojure_comparer.py": [ + 0.1, + 0 + ], + "file:tests/test_clusters.py": [ + 0.1, + 0 + ], + "file:tests/test_compose_math_env.py": [ + 9.4, + 0 + ], + "file:tests/test_convergence_distance.py": [ + 0.1, + 0 + ], + "file:tests/test_conversation.py": [ + 1.3, + 0 + ], + "file:tests/test_conversation_smoke.py": [ + 9.9, + 0 + ], + "file:tests/test_coordinator_collection.py": [ + 10.7, + 0 + ], + "file:tests/test_corr.py": [ + 0.1, + 0 + ], + "file:tests/test_dataset_selection.py": [ + 1.0, + 0 + ], + "file:tests/test_datasets.py": [ + 0.1, + 0 + ], + "file:tests/test_degenerate_tick_parity.py": [ + 0.6, + 0 + ], + "file:tests/test_delphi_legacy_import.py": [ + 2.9, + 0 + ], + "file:tests/test_delphi_postgres_results.py": [ + 0.0, + 0 + ], + "file:tests/test_delphi_result_resource.py": [ + 0.1, + 0 + ], + "file:tests/test_delphi_result_resource_pagination.py": [ + 0.9, + 0 + ], + "file:tests/test_delphi_storage_codec.py": [ + 0.2, + 0 + ], + "file:tests/test_delphi_vote_convention_sites.py": [ + 0.4, + 0 + ], + "file:tests/test_delphi_writer.py": [ + 0.7, + 0 + ], + "file:tests/test_discrepancy_fixes.py": [ + 0.6, + 0 + ], + "file:tests/test_dsn_admission.py": [ + 0.2, + 0 + ], + "file:tests/test_dynamodb_consensus_roundtrip.py": [ + 0.1, + 0 + ], + "file:tests/test_dynamodb_float_serialization.py": [ + 0.6, + 0 + ], + "file:tests/test_edge_cases.py": [ + 0.2, + 0 + ], + "file:tests/test_empty_conversation_readers.py": [ + 0.1, + 0 + ], + "file:tests/test_engine_rebuild.py": [ + 0, + 13.1 + ], + "file:tests/test_entrypoint_boundaries.py": [ + 0.0, + 0 + ], + "file:tests/test_env_flags.py": [ + 0.4, + 0 + ], + "file:tests/test_generator_vote_copy.py": [ + 0, + 0.1 + ], + "file:tests/test_geometry_axis_pair.py": [ + 0.6, + 0 + ], + "file:tests/test_group_k_smoother.py": [ + 2.0, + 0 + ], + "file:tests/test_group_vote_counts.py": [ + 0.2, + 0 + ], + "file:tests/test_import_report.py": [ + 2.2, + 0 + ], + "file:tests/test_in_conv_greedy_carry.py": [ + 0.6, + 0 + ], + "file:tests/test_job_child_protocol.py": [ + 53.5, + 0 + ], + "file:tests/test_large_conv_tick_bench.py": [ + 0.0, + 0 + ], + "file:tests/test_legacy_blob_shape.py": [ + 1.5, + 0 + ], + "file:tests/test_legacy_kmeans.py": [ + 0.3, + 0 + ], + "file:tests/test_light_shadow_triage.py": [ + 4.0, + 0 + ], + "file:tests/test_math_pipeline_runs_e2e.py": [ + 4.4, + 0 + ], + "file:tests/test_math_writer_numpy_serialization.py": [ + 0.3, + 0 + ], + "file:tests/test_minio_access.py": [ + 0.3, + 0 + ], + "file:tests/test_mod_ptpt_leak_parity.py": [ + 1.4, + 0 + ], + "file:tests/test_mod_update_parity.py": [ + 0.6, + 0 + ], + "file:tests/test_most_distal_distance.py": [ + 0.0, + 0 + ], + "file:tests/test_narrative_audit.py": [ + 0.0, + 0 + ], + "file:tests/test_null_vote_fold.py": [ + 1.1, + 0 + ], + "file:tests/test_participant_info.py": [ + 0.3, + 0 + ], + "file:tests/test_pca_column_order.py": [ + 1.3, + 0 + ], + "file:tests/test_pca_edge_cases.py": [ + 0.0, + 0 + ], + "file:tests/test_pca_operation_parity.py": [ + 0.2, + 0 + ], + "file:tests/test_pca_unit.py": [ + 0.0, + 0 + ], + "file:tests/test_pca_warm_start.py": [ + 0.3, + 0 + ], + "file:tests/test_postgres_real_data.py": [ + 0.9, + 0 + ], + "file:tests/test_powerit_pca.py": [ + 0.2, + 0 + ], + "file:tests/test_priority_unmirror.py": [ + 0.5, + 0 + ], + "file:tests/test_probe_reader_snapshot.py": [ + 0.1, + 0 + ], + "file:tests/test_prodclone_extract.py": [ + 0.2, + 0.1 + ], + "file:tests/test_production_import.py": [ + 1.8, + 0 + ], + "file:tests/test_public_fixture_cli.py": [ + 0.0, + 0 + ], + "file:tests/test_queue_noop_executor.py": [ + 0.1, + 0 + ], + "file:tests/test_regression.py": [ + 0.0, + 0 + ], + "file:tests/test_repness_fallback_precision.py": [ + 2.2, + 0 + ], + "file:tests/test_repness_unit.py": [ + 6.1, + 0 + ], + "file:tests/test_report_math_env.py": [ + 21.9, + 0 + ], + "file:tests/test_representative_payloads.py": [ + 27.9, + 0 + ], + "file:tests/test_representative_selection.py": [ + 0.8, + 0 + ], + "file:tests/test_reset_conversation.py": [ + 0.0, + 0 + ], + "file:tests/test_run_delphi_exit_codes.py": [ + 326.3, + 0 + ], + "file:tests/test_run_math_pipeline_main.py": [ + 0.0, + 0 + ], + "file:tests/test_sanity_checks.py": [ + 2.8, + 0 + ], + "file:tests/test_scan_pacing.py": [ + 1.0, + 0 + ], + "file:tests/test_serialization_unfolding.py": [ + 0.2, + 0 + ], + "file:tests/test_umap_narrative_pipeline.py": [ + 156.7, + 0 + ], + "file:tests/test_validate_service_hook.py": [ + 6.0, + 0 + ], + "file:tests/test_vote_convention_source.py": [ + 0.1, + 0 + ], + "file:tests/topic_naming/test_job_routing.py": [ + 0.0, + 0 + ], + "file:tests/topic_naming/test_model_provider_batch.py": [ + 1.9, + 0 + ], + "file:tests/topic_naming/test_topic_naming.py": [ + 0.1, + 0 + ], + "group:dataset-biodiversity": [ + 38.1, + 0 + ], + "group:dataset-vw": [ + 8.8, + 0 + ] } -} +} \ No newline at end of file diff --git a/ci/local/delphi_python_results.py b/ci/local/delphi_python_results.py index e5335840fd..32b37c7a43 100644 --- a/ci/local/delphi_python_results.py +++ b/ci/local/delphi_python_results.py @@ -5,8 +5,9 @@ combine --out OUT DIR... check that the shards' selections partition the collected tests (each test in exactly one shard), add up the counts, and write OUT/counts.json and - OUT/test-durations.json (seconds per shard unit, - the format of ci/local/delphi-test-durations.json) + OUT/test-durations.json ([parallel, serial] seconds + per shard unit, the format of + ci/local/delphi-test-durations.json) Counts are pytest's own summary numbers (passed, failed, skipped, xfailed, xpassed, error), read from the controller of each pass. "deselected" is left @@ -41,15 +42,18 @@ def fmt(c: Counter) -> str: return ", ".join(f"{c.get(k, 0)} {k}" for k in keys) -def durations(run_dir: Path, units: dict[str, str]) -> Counter: - out: Counter = Counter() +def durations(run_dir: Path, manifest: dict) -> tuple[Counter, Counter]: + """Seconds per unit, split into parallel-pass and serial-marked tests.""" + units, serial = manifest["units"], set(manifest.get("serial", [])) + parallel_s: Counter = Counter() + serial_s: Counter = Counter() for f in sorted(run_dir.glob("results-*.jsonl")): for line in f.read_text().splitlines(): rec = json.loads(line) unit = units.get(rec["nodeid"]) if unit: - out[unit] += rec["duration"] - return out + (serial_s if rec["nodeid"] in serial else parallel_s)[unit] += rec["duration"] + return parallel_s, serial_s def cmd_summary(args) -> int: @@ -68,7 +72,8 @@ def cmd_combine(args) -> int: owner: dict[str, str] = {} problems = [] total: Counter = Counter() - unit_seconds: Counter = Counter() + parallel_s: Counter = Counter() + serial_s: Counter = Counter() for d in map(Path, args.dirs): manifest = json.loads((d / "manifest.json").read_text()) shard = manifest["shard"] @@ -85,7 +90,9 @@ def cmd_combine(args) -> int: c = run_counts(d) print(f"shard {shard}: {len(manifest['selected'])} tests; {fmt(c)}") total += c - unit_seconds += durations(d, manifest["units"]) + p, s = durations(d, manifest) + parallel_s += p + serial_s += s missing = sorted(set(collected or []) - set(owner)) for nodeid in missing[:20]: problems.append(f"{nodeid} was collected but no shard selected it") @@ -94,7 +101,8 @@ def cmd_combine(args) -> int: print(f"all shards: {len(owner)} of {len(collected or [])} collected tests; {fmt(total)}") (out / "counts.json").write_text(json.dumps(dict(total), sort_keys=True, indent=1) + "\n") (out / "test-durations.json").write_text(json.dumps( - {"units": {u: round(s, 1) for u, s in sorted(unit_seconds.items())}}, + {"units": {u: [round(parallel_s[u], 1), round(serial_s[u], 1)] + for u in sorted(set(parallel_s) | set(serial_s))}}, indent=1, sort_keys=True) + "\n") if total.get("failed") or total.get("error"): problems.append(f"{total.get('failed', 0)} failed and {total.get('error', 0)} errors across the shards") diff --git a/delphi/tests/ci_partition.py b/delphi/tests/ci_partition.py index ef0d09d5a4..4d264fb916 100644 --- a/delphi/tests/ci_partition.py +++ b/delphi/tests/ci_partition.py @@ -7,10 +7,13 @@ split by unit: an xdist_group name when the test has one (so a dataset is computed in one shard only), otherwise its module file. Units - are spread over shards by expected seconds + are spread over shards by expected wall seconds (ci/local/delphi-test-durations.json, a per-test default for units it does not list), longest first, so the shards finish together. + The plan depends only on the collection and that + file, never on the machine, so every shard + computes the same plan. DELPHI_TEST_MANIFEST= write the collected and selected node ids (before -m), so the combine step can prove every test ran in exactly one shard. @@ -33,6 +36,9 @@ # Seconds assumed for a test in a unit the durations file does not list. DEFAULT_TEST_SECONDS = 0.5 +# Workers assumed when turning parallel-pass seconds into wall seconds (a +# hosted runner's CPUs). A constant, so the plan is the same on every machine. +PLAN_WORKERS = 4 def base_nodeid(nodeid: str) -> str: @@ -66,16 +72,23 @@ def unit_of(item) -> str: def _load_durations() -> dict[str, float]: + """Expected wall seconds per unit. The file holds [parallel, serial] + seconds per unit: parallel-pass tests share PLAN_WORKERS workers, the + serial pass has the shard to itself.""" path = os.environ.get("DELPHI_TEST_DURATIONS") if not path or not Path(path).is_file(): return {} data = json.loads(Path(path).read_text()) - return {str(k): float(v) for k, v in data.get("units", {}).items()} + out = {} + for unit, value in data.get("units", {}).items(): + parallel, serial = value if isinstance(value, list) else (value, 0.0) + out[str(unit)] = float(parallel) / PLAN_WORKERS + float(serial) + return out def assign_shards(units: dict[str, int], durations: dict[str, float], count: int) -> dict[str, int]: """Longest-expected unit first onto the least-loaded shard (1-based).""" - weight = {u: durations.get(u, n * DEFAULT_TEST_SECONDS) for u, n in units.items()} + weight = {u: durations.get(u, n * DEFAULT_TEST_SECONDS / PLAN_WORKERS) for u, n in units.items()} loads = [0.0] * count out = {} for unit in sorted(units, key=lambda u: (-weight[u], u)): @@ -142,6 +155,8 @@ def pytest_collection_modifyitems(self, session, config, items): "collected": collected, "selected": [base_nodeid(i.nodeid) for i in items], "units": {base_nodeid(i.nodeid): unit_of(i) for i in items}, + "serial": [base_nodeid(i.nodeid) for i in items + if i.get_closest_marker("serial") is not None], })) def pytest_runtest_logreport(self, report): diff --git a/delphi/tests/test_engine_rebuild.py b/delphi/tests/test_engine_rebuild.py index ddfd5ca666..d5fc4d0be8 100644 --- a/delphi/tests/test_engine_rebuild.py +++ b/delphi/tests/test_engine_rebuild.py @@ -165,7 +165,11 @@ def _generated_fixture(cur, zid): """Two opinion camps over ten comments, with passes, gaps, one revote and one moderated-out comment. Stored at agree = -1 (raw = -semantic).""" rng = random.Random(zid) - base = 1_700_000_000_000 + zid * 10_000_000 + # Timestamps in 2020: far older than the start of the test stack's live + # math-python poller (it begins 10 days back), so the poller never computes + # these conversations under its own label while the test checks that only + # the rebuild label has rows. (1.7e12 put them at 2026-10-03, inside that window.) + base = 1_600_000_000_000 + zid * 10_000_000 cur.execute("SET session_replication_role = replica") for table in ("votes", "votes_latest_unique", "comments", "participants", "math_main", "math_bidtopid", "math_ptptstats", "math_ticks", From b6374f908d5ec5d40c9effe57244b0b59b893b07 Mon Sep 17 00:00:00 2001 From: Colin Megill Date: Mon, 12 Oct 2026 01:47:06 -0700 Subject: [PATCH 3/3] CI: the coverage comment step's fallback message no longer breaks its script Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018NVGBuYCk4EZmiz4csUv9s --- .github/workflows/python-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index dcb2fb3323..a8375e9557 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -161,7 +161,7 @@ jobs: if (error.status === 403) { console.log('Note: Could not post coverage comment to PR.'); console.log('This is expected for PRs from forks due to GitHub token permissions.'); - console.log('Coverage report is in the test job's log and in the coverage-report artifact.'); + console.log('Coverage report is in the log of the test job and in the coverage-report artifact.'); } else { console.log(`Unexpected error posting comment: ${error.message}`); }