Skip to content

Commit d3d38ce

Browse files
colinmegillcolinmegill
authored
Promote edge to stable: owner checks on invite routes, connection budget, embeddings tables (#3052)
2 parents efb359b + a008071 commit d3d38ce

81 files changed

Lines changed: 6187 additions & 647 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/cypress-tests.yml‎

Lines changed: 68 additions & 141 deletions
Original file line numberDiff line numberDiff line change
@@ -19,18 +19,27 @@ concurrency:
1919
cancel-in-progress: true
2020

2121
jobs:
22-
cypress-run:
22+
# Every step of the check lives in ci/local/e2e.sh, which runs the same way on
23+
# any Docker host. Hosted CI builds the stack's images once (test-images.yml),
24+
# splits the specs into shards that each run the whole stack from those
25+
# images, and the `cypress-run` job puts the shards back together: it proves
26+
# every spec ran in exactly one shard and adds up the counts per suite.
27+
images:
28+
uses: ./.github/workflows/test-images.yml
29+
with:
30+
# This workflow runs on every push, so it is the one that keeps the cache.
31+
write-cache: true
32+
33+
cypress:
34+
name: cypress shard ${{ matrix.shard }}
35+
needs: images
2336
runs-on: ubuntu-latest
24-
37+
strategy:
38+
fail-fast: false
39+
matrix:
40+
shard: [1, 2, 3, 4, 5, 6]
41+
2542
steps:
26-
27-
- name: 1. Free up disk space on runner
28-
run: |
29-
echo "Starting disk cleanup..."
30-
df -h
31-
sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell
32-
echo "Disk space after cleanup:"
33-
df -h
3443
- name: Checkout code
3544
uses: actions/checkout@v4
3645

@@ -40,156 +49,74 @@ jobs:
4049
curl -JLO "https://dl.filippo.io/mkcert/latest?for=linux/amd64"
4150
chmod +x mkcert-v*-linux-amd64
4251
sudo cp mkcert-v*-linux-amd64 /usr/local/bin/mkcert
43-
52+
4453
# Install CA in system trust store
4554
mkcert -install
4655
47-
- name: Generate certificates
48-
run: |
49-
# Create certificate directory in workspace (not home directory)
50-
mkdir -p ./.simulacrum/certs
51-
cd ./.simulacrum/certs
52-
53-
# Generate certificates for all required hostnames
54-
mkcert -cert-file localhost.pem -key-file localhost-key.pem \
55-
localhost 127.0.0.1 ::1 oidc-simulator host.docker.internal
56-
57-
# Copy mkcert root CA for other containers
58-
cp "$(mkcert -CAROOT)/rootCA.pem" ./rootCA.pem
59-
60-
# List generated files for debugging
61-
echo "Generated certificates:"
62-
ls -la
63-
64-
- name: Set up Docker Buildx
65-
uses: docker/setup-buildx-action@v3
66-
67-
- name: Create test.env with CI-specific paths
68-
run: |
69-
# Update AUTH_CERTS_PATH to use workspace-relative path
70-
sed -i 's|AUTH_CERTS_PATH=.*|AUTH_CERTS_PATH=./.simulacrum/certs|' test.env
71-
72-
# Copy test.env to .env for docker-compose variable interpolation
73-
cp test.env .env
74-
75-
# Show relevant env vars for debugging
76-
echo "Auth-related environment variables:"
77-
grep -E "AUTH_|CERT|DATABASE_URL|POSTGRES_" test.env || true
78-
79-
- name: Build Docker images
80-
run: |
81-
# Build postgres with --no-cache to ensure latest migrations are used
82-
# Docker layer caching can cause old migration files to persist
83-
docker compose -f docker-compose.test.yml --env-file test.env build --no-cache postgres
84-
85-
# Build remaining services (caching OK for these)
86-
docker compose -f docker-compose.test.yml --env-file test.env build
87-
88-
- name: Apply and check test database migrations
89-
run: bash ci/test-migrations.sh
56+
- name: Download the test images
57+
uses: actions/download-artifact@v4
58+
with:
59+
name: test-images
60+
path: ${{ runner.temp }}/test-images
9061

91-
- name: Start services
62+
- name: Load the test images
9263
run: |
93-
# Start all services in detached mode
94-
docker compose -f docker-compose.test.yml --env-file test.env up -d
95-
96-
# Wait for services to be ready
97-
echo "Waiting for services to start..."
98-
sleep 30
99-
100-
# Show running containers
101-
docker compose -f docker-compose.test.yml ps
102-
103-
- name: Check service health
104-
run: |
105-
# Check if key services are responding
106-
echo "Checking postgres..."
107-
docker compose -f docker-compose.test.yml exec -T postgres pg_isready -U postgres || true
108-
109-
echo "Checking oidc-simulator..."
110-
curl -k -f https://localhost:3000/.well-known/jwks.json || true
111-
112-
echo "Checking server API..."
113-
curl -f http://localhost/api/v3/participationInit || true
114-
115-
- name: Show postgres initialization logs
64+
bash ci/local/test-images.sh load "$RUNNER_TEMP/test-images"
65+
rm -rf "$RUNNER_TEMP/test-images"
66+
df -h /
67+
68+
- name: Cypress (shard ${{ matrix.shard }}/${{ strategy.job-total }})
69+
env:
70+
POLIS_TEST_IMAGES: ci
71+
E2E_CHECK_OUT: ${{ github.workspace }}/e2e-results/shard-${{ matrix.shard }}
72+
# The runner is discarded after the job; tearing the stack down only costs time.
73+
E2E_CHECK_KEEP: '1'
74+
run: bash ci/local/e2e.sh --shard ${{ matrix.shard }}/${{ strategy.job-total }}
75+
76+
- name: Upload shard results
11677
if: always()
117-
run: |
118-
echo "=== Postgres container logs (shows migration execution) ==="
119-
docker compose -f docker-compose.test.yml logs postgres | head -200
120-
121-
- name: Show service logs
122-
if: always()
123-
run: |
124-
# Show logs from critical services for debugging
125-
echo "=== NGINX Proxy logs ==="
126-
docker compose -f docker-compose.test.yml logs nginx-proxy || true
127-
128-
echo "=== OIDC Simulator logs ==="
129-
docker compose -f docker-compose.test.yml logs oidc-simulator || true
130-
131-
echo "=== Server logs ==="
132-
docker compose -f docker-compose.test.yml logs server || true
133-
134-
- name: Run auth setup test
135-
run: |
136-
cd e2e
137-
npm install dotenv
138-
139-
# Copy test.env to e2e directory for Cypress to read
140-
cp ../test.env .env
141-
142-
# Update AUTH_CERTS_PATH to be relative to e2e directory
143-
sed -i 's|AUTH_CERTS_PATH=.*|AUTH_CERTS_PATH=../.simulacrum/certs|' .env
144-
145-
# Set environment variables for the test script
146-
export AUTH_CERTS_PATH=../.simulacrum/certs
147-
export CYPRESS_BASE_URL=http://localhost
148-
export AUTH_ISSUER=https://localhost:3000/
149-
150-
# Run the validation script
151-
node test-auth-setup.js
152-
153-
- name: Install Cypress dependencies
154-
run: |
155-
cd e2e
156-
npm ci
157-
158-
- name: Run Cypress E2E tests
159-
run: |
160-
cd e2e
161-
162-
# Set environment variables for Cypress
163-
export NODE_EXTRA_CA_CERTS=$(pwd)/../.simulacrum/certs/rootCA.pem
164-
export CYPRESS_BASE_URL=http://localhost
165-
export AUTH_ISSUER=https://localhost:3000/
166-
export AUTH_CERTS_PATH=$(pwd)/../.simulacrum/certs
167-
168-
# Run all Cypress tests
169-
npm test
170-
171-
- name: Show math engine logs
172-
if: failure()
173-
run: |
174-
docker compose -f docker-compose.test.yml logs --tail 300 math-python || true
78+
uses: actions/upload-artifact@v4
79+
with:
80+
name: e2e-shard-${{ matrix.shard }}
81+
path: |
82+
e2e-results/shard-${{ matrix.shard }}
83+
!e2e-results/shard-${{ matrix.shard }}/stack
84+
if-no-files-found: ignore
17585

17686
- name: Upload Cypress screenshots
17787
if: failure()
17888
uses: actions/upload-artifact@v4
17989
with:
180-
name: cypress-screenshots
90+
name: cypress-screenshots-${{ matrix.shard }}
18191
path: e2e/cypress/screenshots
18292
if-no-files-found: ignore
18393

18494
- name: Upload Cypress videos
18595
if: always()
18696
uses: actions/upload-artifact@v4
18797
with:
188-
name: cypress-videos
98+
name: cypress-videos-${{ matrix.shard }}
18999
path: e2e/cypress/videos
190100
if-no-files-found: ignore
191101

192-
- name: Clean up
193-
if: always()
102+
cypress-run:
103+
needs: cypress
104+
if: always()
105+
runs-on: ubuntu-latest
106+
steps:
107+
- name: Checkout code
108+
uses: actions/checkout@v4
109+
110+
- name: Download shard results
111+
uses: actions/download-artifact@v4
112+
with:
113+
pattern: e2e-shard-*
114+
path: e2e-results
115+
116+
- name: Combine shards (every spec exactly once, counts per suite)
117+
run: bash ci/local/e2e.sh combine e2e-results/e2e-shard-*
118+
119+
- name: Every shard passed
194120
run: |
195-
docker compose -f docker-compose.test.yml down -v
121+
echo "shards: ${{ needs.cypress.result }}"
122+
test "${{ needs.cypress.result }}" = success

‎.github/workflows/embeddings-table.yml‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
name: Embeddings table
2+
3+
on:
4+
pull_request:
5+
paths:
6+
- 'delphi/**'
7+
- 'queue-rs/**'
8+
- 'server/postgres/migrations/**'
9+
- 'scripts/test-embeddings-table.sh'
10+
- '.github/workflows/embeddings-table.yml'
11+
workflow_dispatch:
12+
13+
permissions:
14+
contents: read
15+
16+
concurrency:
17+
group: embeddings-table-${{ github.ref }}
18+
cancel-in-progress: true
19+
20+
jobs:
21+
embeddings-table:
22+
runs-on: ubuntu-24.04
23+
timeout-minutes: 90
24+
env:
25+
COMPOSE_PROJECT_NAME: polis-graph-test-embtable-${{ github.run_id }}-${{ github.run_attempt }}
26+
POLIS_RECOVERY_PG_PORT: '55451'
27+
RECOVERY_PG_PORT: '55451'
28+
steps:
29+
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
30+
with:
31+
persist-credentials: false
32+
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
33+
with:
34+
python-version: '3.12'
35+
- name: Queue toolchain
36+
working-directory: queue-rs
37+
run: rustup show active-toolchain
38+
- name: Real-model embeddings-table graph, write/read-back, re-run and switch-off parity
39+
run: bash scripts/test-embeddings-table.sh
40+
- name: Retain proof logs
41+
if: always()
42+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
43+
with:
44+
name: embeddings-table-proof
45+
retention-days: 7
46+
include-hidden-files: true
47+
path: |
48+
.embtable-proof/${{ env.COMPOSE_PROJECT_NAME }}/*.log
49+
.embtable-proof/${{ env.COMPOSE_PROJECT_NAME }}/*.json

0 commit comments

Comments
 (0)