ref(ci): parallel devservices startup and more robust bootstrap-snuba

.github/actions/setup-devservices/action.yml

@@ -0,0 +1,38 @@
+name: 'Early Devservices'
+description: 'Starts devservices in the background so image pulls overlap with venv setup'
+inputs:
+  mode:
+    description: 'devservices mode (must match the mode passed to setup-sentry)'
+    required: true
+  timeout-minutes:
+    description: 'Maximum minutes for devservices up'
+    required: false
+    default: '10'
+
+runs:
+  using: 'composite'
+  steps:
+    - uses: astral-sh/setup-uv@884ad927a57e558e7a70b92f2bccf9198a4be546 # v6
+      with:
+        version: '0.9.28'
+        enable-cache: false
+
+    - name: Start devservices in background
+      shell: bash --noprofile --norc -euo pipefail {0}
+      run: |
+        DS_VERSION=$(python3 -c "
+        import tomllib
+        with open('uv.lock', 'rb') as f:
+            lock = tomllib.load(f)
+        for pkg in lock['package']:
+            if pkg['name'] == 'devservices':
+                print(pkg['version'])
+                break
+        ")
+        echo "Installing devservices==${DS_VERSION}"
+        uv venv /tmp/ds-venv --python python3 -q
+        uv pip install --python /tmp/ds-venv/bin/python -q \
+          --index-url https://pypi.devinfra.sentry.io/simple \
+          "devservices==${DS_VERSION}"
+        (set +e; timeout ${{ inputs.timeout-minutes }}m /tmp/ds-venv/bin/devservices up --mode ${{ inputs.mode }}; echo $? > /tmp/ds-exit) \
+          > /tmp/ds.log 2>&1 &

.github/actions/setup-devservices/bootstrap-snuba.sh

@@ -0,0 +1,114 @@
+#!/bin/bash
+set -euo pipefail
+
+# Bootstrap per-worker Snuba instances, overlapping the expensive ClickHouse
+# table setup with the devservices health-check wait.
+#
+# Phase 1 (early): As soon as ClickHouse is accepting queries, create per-worker
+#   databases and run `snuba bootstrap --force`. This is the slow part.
+# Phase 2 (after devservices): Stop snuba-snuba-1 and start per-worker API
+#   containers. We must wait for devservices to finish first — stopping the
+#   container while devservices is health-checking it would cause a timeout.
+#
+# Requires: XDIST_WORKERS env var
+# Reads:    /tmp/ds-exit (written by setup-devservices/wait.sh)
+# Writes:   /tmp/snuba-bootstrap-exit
+
+WORKERS=${XDIST_WORKERS:?XDIST_WORKERS must be set}
+
+echo "Waiting for ClickHouse and Snuba container..."
+SECONDS=0
+while true; do
+  if [ $SECONDS -gt 300 ]; then
+    echo "::error::Timed out waiting for Snuba bootstrap prerequisites"
+    echo 1 > /tmp/snuba-bootstrap-exit
+    exit 1
+  fi
+  if curl -sf 'http://localhost:8123/' > /dev/null 2>&1 \
+    && docker inspect snuba-snuba-1 > /dev/null 2>&1; then
+    break
+  fi
+  sleep 2
+done
+echo "Prerequisites ready (${SECONDS}s)"
+
+SNUBA_IMAGE=$(docker inspect snuba-snuba-1 --format '{{.Config.Image}}')
+SNUBA_NETWORK=$(docker inspect snuba-snuba-1 --format '{{range $k, $v := .NetworkSettings.Networks}}{{$k}}{{end}}')
+if [ -z "$SNUBA_IMAGE" ] || [ -z "$SNUBA_NETWORK" ]; then
+  echo "::error::Could not inspect snuba-snuba-1 container"
+  echo 1 > /tmp/snuba-bootstrap-exit
+  exit 1
+fi
+
+SNUBA_ENV=(
+  -e "CLICKHOUSE_HOST=clickhouse" -e "CLICKHOUSE_PORT=9000" -e "CLICKHOUSE_HTTP_PORT=8123"
+  -e "DEFAULT_BROKERS=kafka:9093" -e "REDIS_HOST=redis" -e "REDIS_PORT=6379" -e "REDIS_DB=1"
+  -e "SNUBA_SETTINGS=docker"
+)
+
+# Phase 1: Create databases and run bootstrap (the expensive part).
+# This can safely run while devservices is still health-checking containers.
+echo "Phase 1: bootstrapping ClickHouse databases"
+BOOTSTRAP_PIDS=()
+for i in $(seq 0 $(( WORKERS - 1 ))); do
+  (
+    WORKER_DB="default_gw${i}"
+    curl -sf 'http://localhost:8123/' --data-binary "CREATE DATABASE IF NOT EXISTS ${WORKER_DB}"
+    docker run --rm --network "$SNUBA_NETWORK" \
+      -e "CLICKHOUSE_DATABASE=${WORKER_DB}" "${SNUBA_ENV[@]}" \
+      "$SNUBA_IMAGE" bootstrap --force 2>&1 | tail -3
+  ) &
+  BOOTSTRAP_PIDS+=($!)
+done
+
+for pid in "${BOOTSTRAP_PIDS[@]}"; do
+  wait "$pid" || { echo "ERROR: Snuba bootstrap (PID $pid) failed"; echo 1 > /tmp/snuba-bootstrap-exit; exit 1; }
+done
+echo "Phase 1 done (${SECONDS}s)"
+
+# Phase 2: Wait for devservices to finish, then swap snuba-snuba-1 for per-worker containers.
+while [ ! -f /tmp/ds-exit ]; do sleep 1; done
+DS_RC=$(< /tmp/ds-exit)
+if [ "$DS_RC" -ne 0 ]; then
+  echo "::error::devservices failed (exit $DS_RC), skipping Phase 2"
+  echo 1 > /tmp/snuba-bootstrap-exit
+  exit 1
+fi
+
+docker stop snuba-snuba-1 || true
+
+echo "Phase 2: starting per-worker Snuba API containers"
+GW_PIDS=()
+for i in $(seq 0 $(( WORKERS - 1 ))); do
+  (
+    WORKER_DB="default_gw${i}"
+    WORKER_PORT=$((1230 + i))
+    docker run -d --name "snuba-gw${i}" --network "$SNUBA_NETWORK" \
+      -p "${WORKER_PORT}:1218" \
+      -e "CLICKHOUSE_DATABASE=${WORKER_DB}" "${SNUBA_ENV[@]}" \
+      -e "DEBUG=1" "$SNUBA_IMAGE" api
+
+    for attempt in $(seq 1 30); do
+      if curl -sf "http://127.0.0.1:${WORKER_PORT}/health" > /dev/null 2>&1; then
+        echo "snuba-gw${i} healthy on port ${WORKER_PORT}"
+        break
+      fi
+      if [ "$attempt" -eq 30 ]; then
+        echo "ERROR: snuba-gw${i} failed health check after 30 attempts"
+        docker logs "snuba-gw${i}" 2>&1 | tail -20 || true
+        exit 1
+      fi
+      sleep 2
+    done
+  ) &
+  GW_PIDS+=($!)
+done
+
+RC=0
+for pid in "${GW_PIDS[@]}"; do
+  wait "$pid" || { echo "ERROR: Snuba gateway (PID $pid) failed"; RC=1; }
+done
+
+echo "Snuba bootstrap complete (${SECONDS}s total)"
+echo $RC > /tmp/snuba-bootstrap-exit
+exit $RC

.github/actions/setup-devservices/wait.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+set -euo pipefail
+
+# Wait for the background devservices process started by the setup-devservices action.
+# Usage: wait.sh [timeout_seconds]
+TIMEOUT=${1:-600}
+
+SECONDS=0
+while [ ! -f /tmp/ds-exit ]; do
+  if [ $SECONDS -gt "$TIMEOUT" ]; then
+    echo "::error::Timed out waiting for devservices after ${TIMEOUT}s"
+    cat /tmp/ds.log
+    exit 1
+  fi
+  sleep 2
+done
+
+DS_RC=$(< /tmp/ds-exit)
+if [ "$DS_RC" -ne 0 ]; then
+  echo "::error::devservices up failed (exit $DS_RC)"
+  cat /tmp/ds.log
+  exit 1
+fi
+
+echo "DJANGO_LIVE_TEST_SERVER_ADDRESS=$(docker network inspect bridge --format='{{(index .IPAM.Config 0).Gateway}}')" >> "$GITHUB_ENV"
+docker ps -a

.github/workflows/acceptance.yml

@@ -68,6 +68,12 @@ jobs:
       - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
         name: Checkout sentry
 
+      - run: mkdir -p config/chartcuterie
+
+      - uses: ./.github/actions/setup-devservices
+        with:
+          mode: acceptance-ci
+
       - name: Step configurations
         id: config
         run: |
@@ -98,13 +104,24 @@ jobs:
         id: setup
         with:
           mode: acceptance-ci
+          skip-devservices: 'true'
+
+      - name: Wait for devservices
+        run: |
+          sentry init
+          ./.github/actions/setup-devservices/wait.sh
 
       - name: Run acceptance tests (#${{ steps.setup.outputs.matrix-instance-number }} of ${{ steps.setup.outputs.matrix-instance-total }})
         run: make run-acceptance
 
       - name: Inspect failure
         if: failure()
         run: |
+          if [ -f /tmp/ds.log ]; then
+            echo "--- devservices startup log ---"
+            cat /tmp/ds.log
+          fi
+
           if command -v devservices; then
             devservices logs
           fi