integration-tests.yml

name: Integration Tests (multi-OS hosts)

# Tests the multi-OS SSSD integration stacks introduced in test/integration/.
# Each job brings up one stack with a single Ubuntu 24.04 host to keep CI fast.
# The full 5-OS matrix runs locally via `make integration-all`.
#
# Key things validated that provider-tests.yml does not cover:
#   - Provision endpoint (/api/client/provision) returns valid LDAP bind credentials
#   - entrypoint.sh auto-provisioning path (LDAP_BIND_DN empty → fetch from provision)
#   - SSSD resolves users/groups after auto-provisioned bind creds are written
#   - Challenge create + approve + reject cycle via API (no browser)

on:
  push:
    branches: [dev, main]
    paths:
      - "test/integration/**"
      - "test/testclient/**"
      - "Dockerfile"
      - ".dockerignore"
      - "cmd/**"
      - "internal/**"
      - "go.mod"
      - "go.sum"
      - ".github/workflows/integration-tests.yml"
  pull_request:
    branches: [dev, main]
    paths:
      - "test/integration/**"
      - "test/testclient/**"
      - "Dockerfile"
      - ".dockerignore"
      - "cmd/**"
      - "internal/**"
      - "go.mod"
      - "go.sum"
      - ".github/workflows/integration-tests.yml"
  workflow_dispatch:
    inputs:
      stack:
        description: "Stack to test (full-mode | lldap-dex | vault-escrow | all)"
        required: false
        default: "all"

concurrency:
  group: integration-tests-${{ github.ref }}
  cancel-in-progress: true

env:
  FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true

jobs:
  # ── PocketID full-mode + embedded LDAP + SSSD auto-provisioning ──────────────
  full-mode:
    name: full-mode (PocketID + embedded LDAP + provision)
    runs-on: ubuntu-latest
    if: >
      github.event_name != 'workflow_dispatch' ||
      github.event.inputs.stack == 'all' ||
      github.event.inputs.stack == 'full-mode'
    timeout-minutes: 25

    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3

      # Bring up pocketid + identree + a single ubuntu2404 host + syslog receiver.
      # The other four OS hosts are skipped in CI (tested locally with make integration-all).
      - name: Start full-mode stack (pocketid + identree + syslog + ubuntu2404)
        run: |
          docker compose -f test/integration/full-mode/docker-compose.yml \
            up --build -d pocketid syslog identree host-ubuntu2404
        env:
          COMMIT: ${{ github.sha }}
          VERSION: ${{ github.ref_name }}

      - name: Wait for identree to be healthy
        run: |
          timeout 120 bash -c \
            'until curl -sf http://localhost:8110/healthz; do sleep 3; done'
          echo "identree ready."

      - name: Create OIDC client in PocketID
        id: oidc
        run: |
          # Create the OIDC client and extract its ID + secret
          RESULT=$(curl -sf -X POST http://localhost:1413/api/oidc/clients \
            -H "Content-Type: application/json" \
            -H "X-API-KEY: integ-full-static-key" \
            -d '{"name":"identree-ci","callbackURLs":["http://localhost:8110/auth/callback"],"logoutCallbackURLs":[],"isPublic":false,"pkceEnabled":false}')
          CLIENT_ID=$(echo "$RESULT" | python3 -c "import sys,json; print(json.load(sys.stdin)['id'])")
          SECRET_RESULT=$(curl -sf -X POST \
            "http://localhost:1413/api/oidc/clients/${CLIENT_ID}/secret" \
            -H "Content-Type: application/json" \
            -H "X-API-KEY: integ-full-static-key" -d '')
          CLIENT_SECRET=$(echo "$SECRET_RESULT" | python3 -c "import sys,json; print(json.load(sys.stdin)['secret'])")
          echo "client_id=${CLIENT_ID}" >> "$GITHUB_OUTPUT"
          echo "client_secret=${CLIENT_SECRET}" >> "$GITHUB_OUTPUT"
          echo "OIDC client: ${CLIENT_ID}"

      - name: Restart identree with OIDC credentials
        run: |
          OIDC_CLIENT_ID="${{ steps.oidc.outputs.client_id }}" \
          OIDC_CLIENT_SECRET="${{ steps.oidc.outputs.client_secret }}" \
          docker compose -f test/integration/full-mode/docker-compose.yml \
            up -d identree
          timeout 60 bash -c \
            'until curl -sf http://localhost:8110/healthz; do sleep 3; done'

      - name: Populate PocketID with 25 users
        run: |
          POCKETID_URL=http://localhost:1413 \
          POCKETID_API_KEY=integ-full-static-key \
          IDENTREE_URL=http://localhost:8110 \
          bash test/integration/scripts/create-users-pocketid.sh

      - name: Wait for identree LDAP to sync users from PocketID
        run: |
          echo "Waiting for identree LDAP to sync..."
          timeout 60 bash -c '
            until curl -sf http://localhost:8110/healthz >/dev/null && \
              docker logs integ-full-identree 2>&1 | grep -q "users=25"; do
              sleep 3
            done' || true
          # Give one extra refresh cycle
          sleep 20
          echo "LDAP sync complete (or timed out — checking below)"

      - name: Verify provision endpoint returns valid LDAP bind credentials
        run: |
          curl -sf http://localhost:8110/api/client/provision \
            -H "X-Shared-Secret: integ-full-shared-secret-abc123456" \
            -H "X-Hostname: ci-test-host" \
            -o /tmp/provision-resp.json
          echo "Response:"
          python3 -m json.tool /tmp/provision-resp.json
          python3 << 'PY'
          import json
          d = json.load(open('/tmp/provision-resp.json'))
          assert d.get('ldap_url'), "missing ldap_url"
          assert d.get('base_dn'), "missing base_dn"
          assert d.get('bind_dn'), "missing bind_dn"
          assert d.get('bind_password'), "missing bind_password"
          assert 'ci-test-host' in d['bind_dn'], f"hostname not in bind_dn: {d['bind_dn']}"
          assert len(d['bind_password']) == 64, f"bind_password wrong length: {len(d['bind_password'])}"
          print("Provision endpoint: OK")
          PY

      - name: Wait for SSSD auto-provisioning on ubuntu2404 host
        run: |
          echo "Waiting for SSSD to resolve LDAP users via auto-provisioned bind creds..."
          timeout 120 bash -c '
            until docker exec integ-full-ubuntu2404 getent passwd alice 2>/dev/null | grep -q alice; do
              sleep 5
            done'
          echo "SSSD resolution: OK"
          docker exec integ-full-ubuntu2404 getent passwd alice
          docker exec integ-full-ubuntu2404 getent passwd sam
          docker exec integ-full-ubuntu2404 getent group developers
          docker exec integ-full-ubuntu2404 getent group admins

      - name: Run challenge create + approve + reject cycle
        run: |
          # 1. Get admin session via DEV_LOGIN
          COOKIE=$(curl -sc /tmp/ci-cookies.txt \
            "http://localhost:8110/dev/login?user=sam&role=admin" \
            -o /dev/null -w '%{http_code}')
          SESSION=$(grep 'pam_session' /tmp/ci-cookies.txt | awk '{print $NF}')
          echo "Session cookie obtained (HTTP ${COOKIE})"

          python3 << 'PY'
          import hmac, hashlib, time, urllib.request, json, os

          BASE = "http://localhost:8110"
          SECRET = "integ-full-shared-secret-abc123456"
          ADMIN = "sam"
          SESSION = open('/tmp/ci-cookies.txt').read().split('pam_session\t')[-1].strip()

          def derive_key(secret, ctx):
              return hmac.new(secret.encode(), ctx.encode(), hashlib.sha256).digest()

          def csrf(username, ts):
              key = derive_key(SECRET, "csrf")
              return hmac.new(key, f"csrf:{username}:{ts}".encode(), hashlib.sha256).hexdigest()

          def api(path, method="GET", body=None, headers=None):
              url = BASE + path
              data = json.dumps(body).encode() if body else None
              h = {"Content-Type": "application/json", **(headers or {})}
              req = urllib.request.Request(url, data=data, headers=h, method=method)
              with urllib.request.urlopen(req) as r:
                  return json.loads(r.read()) if r.status not in (204, 303) else {}

          def form_post(path, fields):
              body = "&".join(f"{k}={v}" for k, v in fields.items()).encode()
              req = urllib.request.Request(
                  BASE + path, data=body,
                  headers={"Content-Type": "application/x-www-form-urlencoded",
                           "Cookie": f"pam_session={SESSION}"},
                  method="POST")
              try:
                  urllib.request.urlopen(req)
                  return 200
              except urllib.error.HTTPError as e:
                  return e.code

          # Create challenges for 3 users
          created = []
          for user in ["alice", "bob", "sam"]:
              resp = api("/api/challenge", "POST",
                  {"username": user, "hostname": "prod-ubuntu24-01", "reason": "CI test"},
                  {"X-Shared-Secret": SECRET})
              cid = resp.get("challenge_id", "")
              status = resp.get("status", "pending")
              print(f"  Created: {user} → {cid[:8]} ({status})")
              if status == "pending" and cid:
                  created.append(cid)

          print(f"Pending challenges: {len(created)}")
          assert len(created) >= 1, "Expected at least 1 pending challenge"

          # Approve half, reject half
          approved = rejected = 0
          for i, cid in enumerate(created):
              ts = str(int(time.time()))
              tok = csrf(ADMIN, ts)
              fields = {"username": ADMIN, "csrf_token": tok, "csrf_ts": ts}
              if i % 2 == 0:
                  fields["challenge_id"] = cid
                  rc = form_post("/api/challenges/approve", fields)
                  print(f"  Approve {cid[:8]}: HTTP {rc}")
                  assert rc in (200, 303), f"approve failed: {rc}"
                  approved += 1
              else:
                  fields["challenge_id"] = cid
                  rc = form_post("/api/challenges/reject", fields)
                  print(f"  Reject  {cid[:8]}: HTTP {rc}")
                  assert rc in (200, 303), f"reject failed: {rc}"
                  rejected += 1

          print(f"Approved: {approved}  Rejected: {rejected}  — OK")
          PY

      - name: Verify agent heartbeat round-trip
        run: |
          echo "==> Triggering 'identree heartbeat' from the host container..."
          docker exec integ-full-ubuntu2404 identree heartbeat
          echo "==> Verifying /api/agents lists the host with non-empty os_info..."
          # The dev_login session captured earlier wrote /tmp/ci-cookies.txt.
          SESSION=$(grep 'pam_session' /tmp/ci-cookies.txt | awk '{print $NF}')
          curl -sf -H "Cookie: pam_session=${SESSION}" http://localhost:8110/api/agents \
            | tee /tmp/agents.json | python3 -c "
          import json,sys
          d=json.load(sys.stdin)
          agents=d.get('agents',[])
          print(f'  agents reported: {len(agents)}')
          host = next((a for a in agents if a['hostname']=='prod-ubuntu24-01'), None)
          assert host, f'prod-ubuntu24-01 not in {[a[\"hostname\"] for a in agents]}'
          assert 'Ubuntu' in host['os_info'], f'expected Ubuntu in os_info, got {host[\"os_info\"]}'
          assert host['status']=='green', f'expected status=green, got {host[\"status\"]}'
          print(f'  prod-ubuntu24-01: {host[\"os_info\"]} (status={host[\"status\"]}) — OK')
          "

      - name: Verify audit events — JSON log (stdout)
        run: |
          echo "==> Checking audit events in identree stdout..."
          CREATED=$(docker logs integ-full-identree 2>/dev/null | grep -c '"event":"challenge_created"' || true)
          APPROVED=$(docker logs integ-full-identree 2>/dev/null | grep -c '"event":"challenge_approved"' || true)
          echo "  JSON log: challenge_created=${CREATED} challenge_approved=${APPROVED}"
          [ "$CREATED" -ge 1 ] || { echo "FAIL: no challenge_created in stdout"; exit 1; }
          # Verify valid JSON structure
          docker logs integ-full-identree 2>/dev/null | grep '"event":"challenge_created"' | head -1 | python3 -c "
          import sys, json
          e = json.load(sys.stdin)
          assert e['source'] == 'identree', f'bad source: {e[\"source\"]}'
          assert e['username'], 'missing username'
          assert e['timestamp'], 'missing timestamp'
          print('  JSON log structure: OK')
          "

      - name: Verify audit events — syslog (RFC 5424)
        run: |
          echo "==> Checking syslog receiver..."
          SYSLOG=$(docker exec integ-full-syslog cat /var/log/audit.log 2>/dev/null || echo "")
          LINES=$(echo "$SYSLOG" | grep -c 'identree' || true)
          echo "  Syslog lines containing 'identree': ${LINES}"
          [ "$LINES" -ge 1 ] || { echo "FAIL: no identree events in syslog"; exit 1; }
          # Verify RFC 5424 prefix
          echo "$SYSLOG" | head -1 | grep -q '<86>1 ' || { echo "FAIL: missing RFC 5424 prefix"; exit 1; }
          echo "$SYSLOG" | head -1 | grep -q 'event=' || { echo "FAIL: missing SD event param"; exit 1; }
          echo "  Syslog format: OK"

      - name: "Test: identree setup --sssd on fresh container"
        run: |
          # 1. Determine the compose default network name
          NETWORK=$(docker inspect integ-full-identree --format '{{range $k, $v := .NetworkSettings.Networks}}{{$k}}{{end}}')
          echo "Using network: ${NETWORK}"

          # 2. Run a fresh ubuntu:24.04 container on the same network
          docker run -d --name integ-full-setup-test \
            --hostname setup-test-01 \
            --network "$NETWORK" \
            ubuntu:24.04 sleep infinity

          # 3. Install SSSD packages
          docker exec integ-full-setup-test bash -c '
            export DEBIAN_FRONTEND=noninteractive
            apt-get update -qq
            apt-get install -y -qq sssd sssd-ldap libpam-sss libnss-sss libsss-sudo sudo >/dev/null
          '

          # 4. Copy the identree binary from the identree container.
          # The binary may be a symlink in the container, so use cat to
          # dereference it and get the actual file content.
          docker exec integ-full-identree cat /usr/local/bin/identree > /tmp/identree-ci-bin
          chmod 755 /tmp/identree-ci-bin
          docker cp /tmp/identree-ci-bin integ-full-setup-test:/usr/local/bin/identree

          # 5. Write /etc/identree/client.conf
          docker exec integ-full-setup-test bash -c 'mkdir -p /etc/identree && printf "IDENTREE_SERVER_URL=http://identree:8090\nIDENTREE_SHARED_SECRET=integ-full-shared-secret-abc123456\n" > /etc/identree/client.conf && chmod 600 /etc/identree/client.conf'

          # 6. Run identree setup --sssd --hostname setup-test-01
          docker exec integ-full-setup-test identree setup --sssd --hostname setup-test-01

          # 7. Verify sssd.conf was created
          docker exec integ-full-setup-test test -f /etc/sssd/sssd.conf
          echo "OK: /etc/sssd/sssd.conf exists"

          # 8. Verify sssd.conf contains sudo_provider = ldap
          docker exec integ-full-setup-test grep -q 'sudo_provider.*=.*ldap' /etc/sssd/sssd.conf
          echo "OK: sssd.conf contains sudo_provider = ldap"

          # 9. Verify sssd.conf contains ldap_sudo_search_base
          docker exec integ-full-setup-test grep -q 'ldap_sudo_search_base' /etc/sssd/sssd.conf
          echo "OK: sssd.conf contains ldap_sudo_search_base"

          # 10. Verify PAM sudo has pam_exec.so
          docker exec integ-full-setup-test grep -q 'pam_exec.so' /etc/pam.d/sudo
          echo "OK: /etc/pam.d/sudo contains pam_exec.so"

          # 11. Verify nsswitch.conf has sss in passwd, group, and sudoers lines
          docker exec integ-full-setup-test grep -E '^passwd:' /etc/nsswitch.conf | grep -q sss
          echo "OK: nsswitch.conf passwd has sss"
          docker exec integ-full-setup-test grep -E '^group:' /etc/nsswitch.conf | grep -q sss
          echo "OK: nsswitch.conf group has sss"
          docker exec integ-full-setup-test grep -E '^sudoers:' /etc/nsswitch.conf | grep -q sss
          echo "OK: nsswitch.conf sudoers has sss"

          # 12. Clear stale SSSD state and start SSSD
          docker exec integ-full-setup-test bash -c '
            rm -rf /var/lib/sss/db/* /var/lib/sss/mc/* /run/sssd.pid 2>/dev/null || true
            sssd -D
          '
          echo "SSSD started, waiting for user resolution..."
          timeout 60 bash -c '
            until docker exec integ-full-setup-test getent passwd alice 2>/dev/null | grep -q alice; do
              sleep 3
            done'
          echo "OK: getent passwd alice resolved via identree setup --sssd"
          docker exec integ-full-setup-test getent passwd alice

      - name: "Test: /install.sh endpoint"
        run: |
          # 1. Fetch the static install script
          curl -sf http://localhost:8110/install.sh -o /tmp/install-script.sh
          echo "OK: fetched /install.sh ($(wc -c < /tmp/install-script.sh) bytes)"

          # 2. Verify it's a valid bash script (static installer, no embedded secrets)
          bash -n /tmp/install-script.sh
          echo "OK: install.sh passes bash -n syntax check"

          # 3. Verify it contains identree setup invocation
          grep -q 'identree' /tmp/install-script.sh || \
            { echo "FAIL: install.sh missing identree reference"; exit 1; }
          echo "OK: install.sh references identree"

          # 4. Verify the config endpoint returns deployment-specific values
          CONFIG=$(curl -sf http://localhost:8110/install-config.json \
            -H "X-Shared-Secret: integ-full-shared-secret-abc123456" || echo "")
          echo "$CONFIG" | python3 -c "import sys,json; d=json.load(sys.stdin); assert d.get('server_url'), 'missing server_url'" || \
            { echo "FAIL: /install-config.json missing server_url"; exit 1; }
          echo "OK: /install-config.json returns server_url"

          # 5. Verify signature endpoint exists
          curl -sf http://localhost:8110/install.sh.sig -o /dev/null || \
            { echo "FAIL: /install.sh.sig not available"; exit 1; }
          echo "OK: /install.sh.sig available"

      - name: Capture logs on failure
        if: failure()
        run: |
          echo "=== pocketid logs ==="
          docker logs integ-full-pocketid 2>&1 | tail -30
          echo "=== identree logs ==="
          docker logs integ-full-identree 2>&1 | tail -50
          echo "=== ubuntu2404 sssd.conf ==="
          docker exec integ-full-ubuntu2404 cat /etc/sssd/sssd.conf 2>/dev/null || true
          echo "=== ubuntu2404 sssd log ==="
          docker exec integ-full-ubuntu2404 cat /var/log/sssd/sssd_LDAP.log 2>/dev/null | tail -30 || true
          echo "=== ubuntu2404 entrypoint output ==="
          docker logs integ-full-ubuntu2404 2>&1 | tail -40
          echo "=== setup-test sssd.conf ==="
          docker exec integ-full-setup-test cat /etc/sssd/sssd.conf 2>/dev/null || true
          echo "=== setup-test nsswitch.conf ==="
          docker exec integ-full-setup-test cat /etc/nsswitch.conf 2>/dev/null || true
          echo "=== setup-test pam.d/sudo ==="
          docker exec integ-full-setup-test cat /etc/pam.d/sudo 2>/dev/null || true
          echo "=== setup-test sssd log ==="
          docker exec integ-full-setup-test cat /var/log/sssd/sssd_identree.log 2>/dev/null | tail -30 || true

      - name: Tear down
        if: always()
        run: |
          docker rm -f integ-full-setup-test 2>/dev/null || true
          docker compose -f test/integration/full-mode/docker-compose.yml down -v

  # ── lldap + Dex bridge mode ───────────────────────────────────────────────────
  lldap-dex:
    name: lldap-dex (bridge mode + static sudo)
    runs-on: ubuntu-latest
    if: >
      github.event_name != 'workflow_dispatch' ||
      github.event.inputs.stack == 'all' ||
      github.event.inputs.stack == 'lldap-dex'
    timeout-minutes: 20

    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3

      - name: Start lldap-dex stack (lldap + dex + loki + identree + ubuntu2404)
        run: |
          docker compose -f test/integration/lldap-dex/docker-compose.yml \
            up --build -d lldap dex loki identree host-ubuntu2404
        env:
          COMMIT: ${{ github.sha }}
          VERSION: ${{ github.ref_name }}

      - name: Wait for identree to be healthy
        run: |
          timeout 120 bash -c \
            'until curl -sf http://localhost:8111/healthz; do sleep 3; done'
          echo "identree ready."

      - name: Create 25 users in lldap
        run: |
          LLDAP_URL=http://localhost:17175 \
          CLIENT=integ-lldap-dex-ubuntu2404 \
          bash test/integration/scripts/create-users-lldap.sh

      - name: Wait for SSSD to resolve LDAP users
        run: |
          timeout 90 bash -c '
            until docker exec integ-lldap-dex-ubuntu2404 getent passwd alice 2>/dev/null | grep -q alice; do
              sleep 5
            done'
          echo "SSSD resolution: OK"
          docker exec integ-lldap-dex-ubuntu2404 getent passwd alice
          docker exec integ-lldap-dex-ubuntu2404 getent passwd sam

      - name: Run challenge create + approve + reject cycle
        run: |
          curl -sc /tmp/ci-lldap-cookies.txt \
            "http://localhost:8111/dev/login?user=sam&role=admin" \
            -o /dev/null

          python3 << 'PY'
          import hmac, hashlib, time, urllib.request, json

          BASE = "http://localhost:8111"
          SECRET = "integ-lldap-dex-shared-secret-xyz"
          ADMIN = "sam"
          SESSION = open('/tmp/ci-lldap-cookies.txt').read().split('pam_session\t')[-1].strip()

          def derive_key(secret, ctx):
              return hmac.new(secret.encode(), ctx.encode(), hashlib.sha256).digest()

          def csrf(username, ts):
              key = derive_key(SECRET, "csrf")
              return hmac.new(key, f"csrf:{username}:{ts}".encode(), hashlib.sha256).hexdigest()

          def create_challenge(user, host, reason):
              req = urllib.request.Request(
                  BASE + "/api/challenge",
                  data=json.dumps({"username": user, "hostname": host, "reason": reason}).encode(),
                  headers={"Content-Type": "application/json", "X-Shared-Secret": SECRET},
                  method="POST")
              with urllib.request.urlopen(req) as r:
                  return json.loads(r.read())

          def form_post(path, fields):
              body = "&".join(f"{k}={v}" for k, v in fields.items()).encode()
              req = urllib.request.Request(
                  BASE + path, data=body,
                  headers={"Content-Type": "application/x-www-form-urlencoded",
                           "Cookie": f"pam_session={SESSION}"},
                  method="POST")
              try:
                  urllib.request.urlopen(req); return 200
              except urllib.error.HTTPError as e:
                  return e.code

          # Create 5 challenges across different users
          users = ["alice", "bob", "carol", "sam", "tina"]
          created = []
          for user in users:
              resp = create_challenge(user, "dev-ubuntu24-01", "CI lldap-dex test")
              cid = resp.get("challenge_id", "")
              status = resp.get("status", "pending")
              print(f"  {user}: {cid[:8]} ({status})")
              if status == "pending" and cid:
                  created.append((user, cid))

          approved = rejected = 0
          for i, (user, cid) in enumerate(created):
              ts = str(int(time.time()))
              tok = csrf(ADMIN, ts)
              fields = {"username": ADMIN, "csrf_token": tok, "csrf_ts": ts, "challenge_id": cid}
              if i % 2 == 0:
                  rc = form_post("/api/challenges/approve", fields)
                  assert rc in (200, 303), f"approve {user} failed: {rc}"
                  approved += 1
              else:
                  rc = form_post("/api/challenges/reject", fields)
                  assert rc in (200, 303), f"reject {user} failed: {rc}"
                  rejected += 1

          print(f"Approved: {approved}  Rejected: {rejected}  — OK")
          PY

      - name: Verify audit events — Loki
        run: |
          echo "==> Waiting for Loki to ingest events..."
          sleep 8  # Loki batches on a 5s timer; give extra headroom
          START=$(date -u -d '10 minutes ago' +%s 2>/dev/null || date -u -v-10M +%s)
          curl -sf "http://localhost:3161/loki/api/v1/query_range" \
            --data-urlencode 'query={app="identree"}' \
            --data-urlencode 'limit=50' \
            --data-urlencode "start=${START}000000000" \
            -o /tmp/loki-resp.json || echo '{}' > /tmp/loki-resp.json
          python3 << 'PYEOF'
          import json
          d = json.load(open('/tmp/loki-resp.json'))
          results = d.get('data', {}).get('result', [])
          total = sum(len(s.get('values', [])) for s in results)
          print(f"  Loki: {total} audit log entries")
          assert total >= 1, "FAIL: no audit events found in Loki"
          first_line = results[0]['values'][0][1]
          evt = json.loads(first_line)
          assert evt.get('source') == 'identree', f"bad source: {evt.get('source')}"
          assert evt.get('event'), "missing event field"
          assert evt.get('username'), "missing username field"
          print(f"  Loki event structure: OK (event={evt['event']} user={evt['username']})")
          PYEOF

      - name: Capture logs on failure
        if: failure()
        run: |
          echo "=== lldap logs ==="
          docker logs integ-lldap-dex-lldap 2>&1 | tail -30
          echo "=== dex logs ==="
          docker logs integ-lldap-dex-dex 2>&1 | tail -30
          echo "=== identree logs ==="
          docker logs integ-lldap-dex-identree 2>&1 | tail -50
          echo "=== loki logs ==="
          docker logs integ-lldap-dex-loki 2>&1 | tail -20
          echo "=== ubuntu2404 sssd.conf ==="
          docker exec integ-lldap-dex-ubuntu2404 cat /etc/sssd/sssd.conf 2>/dev/null || true
          echo "=== ubuntu2404 entrypoint output ==="
          docker logs integ-lldap-dex-ubuntu2404 2>&1 | tail -40

      - name: Tear down
        if: always()
        run: docker compose -f test/integration/lldap-dex/docker-compose.yml down -v

  # ── Vault escrow ──────────────────────────────────────────────────────────────
  vault-escrow:
    name: vault-escrow (Vault KV v2 escrow)
    runs-on: ubuntu-latest
    if: >
      github.event_name != 'workflow_dispatch' ||
      github.event.inputs.stack == 'all' ||
      github.event.inputs.stack == 'vault-escrow'
    timeout-minutes: 20

    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3

      - name: Start vault-escrow stack (vault + lldap + dex + splunk-hec + identree + ubuntu2404)
        run: |
          docker compose -f test/integration/vault-escrow/docker-compose.yml \
            up --build -d vault lldap dex splunk-hec identree host-ubuntu2404
        env:
          COMMIT: ${{ github.sha }}
          VERSION: ${{ github.ref_name }}

      - name: Wait for identree to be healthy
        run: |
          timeout 120 bash -c \
            'until curl -sf http://localhost:8114/healthz; do sleep 3; done'
          echo "identree ready."

      - name: Create 25 users in lldap
        run: |
          LLDAP_URL=http://localhost:17177 \
          CLIENT=integ-vault-ubuntu2404 \
          bash test/integration/scripts/create-users-lldap.sh

      - name: Wait for SSSD to resolve LDAP users
        run: |
          timeout 90 bash -c '
            until docker exec integ-vault-ubuntu2404 getent passwd alice 2>/dev/null | grep -q alice; do
              sleep 5
            done'
          echo "SSSD resolution: OK"
          docker exec integ-vault-ubuntu2404 getent passwd alice

      - name: Trigger break-glass escrow write to Vault
        run: |
          # rotate-breakglass writes the break-glass password into Vault
          docker exec integ-vault-ubuntu2404 identree rotate-breakglass || true
          sleep 3
          # Verify the secret was written to Vault
          echo "==> Verifying break-glass secret in Vault..."
          HOSTNAME=$(docker exec integ-vault-ubuntu2404 hostname)
          curl -sf "http://localhost:8201/v1/secret/data/identree/${HOSTNAME}" \
            -H "X-Vault-Token: integ-vault-test-token" | python3 -c "
          import sys, json
          d = json.load(sys.stdin)
          pw = d.get('data', {}).get('data', {}).get('password', '')
          assert pw, 'password missing from Vault secret'
          print(f'Vault secret OK (password length: {len(pw)})')
          " || echo "WARNING: Vault secret not yet present (rotate-breakglass may not have run)"

      - name: Run challenge create + approve + reject cycle
        run: |
          curl -sc /tmp/ci-vault-cookies.txt \
            "http://localhost:8114/dev/login?user=sam&role=admin" \
            -o /dev/null

          python3 << 'PY'
          import hmac, hashlib, time, urllib.request, json

          BASE = "http://localhost:8114"
          SECRET = "integ-vault-escrow-shared-secret-xyz"
          ADMIN = "sam"
          SESSION = open('/tmp/ci-vault-cookies.txt').read().split('pam_session\t')[-1].strip()

          def derive_key(secret, ctx):
              return hmac.new(secret.encode(), ctx.encode(), hashlib.sha256).digest()

          def csrf(username, ts):
              key = derive_key(SECRET, "csrf")
              return hmac.new(key, f"csrf:{username}:{ts}".encode(), hashlib.sha256).hexdigest()

          def create_challenge(user, host, reason):
              req = urllib.request.Request(
                  BASE + "/api/challenge",
                  data=json.dumps({"username": user, "hostname": host, "reason": reason}).encode(),
                  headers={"Content-Type": "application/json", "X-Shared-Secret": SECRET},
                  method="POST")
              with urllib.request.urlopen(req) as r:
                  return json.loads(r.read())

          def form_post(path, fields):
              body = "&".join(f"{k}={v}" for k, v in fields.items()).encode()
              req = urllib.request.Request(
                  BASE + path, data=body,
                  headers={"Content-Type": "application/x-www-form-urlencoded",
                           "Cookie": f"pam_session={SESSION}"},
                  method="POST")
              try:
                  urllib.request.urlopen(req); return 200
              except urllib.error.HTTPError as e:
                  return e.code

          users = ["alice", "bob", "carol", "sam", "tina"]
          created = []
          for user in users:
              resp = create_challenge(user, "vault-ubuntu24-01", "CI vault-escrow test")
              cid = resp.get("challenge_id", "")
              status = resp.get("status", "pending")
              print(f"  {user}: {cid[:8] if cid else '(empty)'} ({status})")
              if status == "pending" and cid:
                  created.append((user, cid))

          approved = rejected = 0
          for i, (user, cid) in enumerate(created):
              ts = str(int(time.time()))
              tok = csrf(ADMIN, ts)
              fields = {"username": ADMIN, "csrf_token": tok, "csrf_ts": ts, "challenge_id": cid}
              if i % 2 == 0:
                  rc = form_post("/api/challenges/approve", fields)
                  assert rc in (200, 303), f"approve {user} failed: {rc}"
                  approved += 1
              else:
                  rc = form_post("/api/challenges/reject", fields)
                  assert rc in (200, 303), f"reject {user} failed: {rc}"
                  rejected += 1

          print(f"Approved: {approved}  Rejected: {rejected}  — OK")
          PY

      - name: Verify audit events — Splunk HEC
        run: |
          echo "==> Waiting for Splunk HEC batch flush..."
          sleep 8  # Splunk HEC batches on a 5s timer
          docker exec integ-vault-splunk-hec cat /var/log/hec.log > /tmp/hec.log 2>/dev/null || true
          EVENTS=$(grep -c 'identree:audit' /tmp/hec.log || true)
          echo "  Splunk HEC: ${EVENTS} batches received"
          [ "$EVENTS" -ge 1 ] || { echo "FAIL: no audit events in Splunk HEC"; exit 1; }
          python3 << 'PYEOF'
          import json
          with open('/tmp/hec.log') as f:
              log = f.read()
          for line in log.strip().split('\n'):
              if not line.strip():
                  continue
              decoder = json.JSONDecoder()
              obj, _ = decoder.raw_decode(line)
              assert obj.get('sourcetype') == 'identree:audit', f"bad sourcetype: {obj.get('sourcetype')}"
              assert obj.get('source') == 'identree', f"bad source: {obj.get('source')}"
              evt = obj.get('event', {})
              assert evt.get('event'), "missing event field in payload"
              assert evt.get('username'), "missing username field in payload"
              print(f"  Splunk HEC structure: OK (event={evt['event']} user={evt['username']})")
              break
          PYEOF

      - name: Capture logs on failure
        if: failure()
        run: |
          echo "=== vault logs ==="
          docker logs integ-vault-vault 2>&1 | tail -30
          echo "=== lldap logs ==="
          docker logs integ-vault-lldap 2>&1 | tail -30
          echo "=== dex logs ==="
          docker logs integ-vault-dex 2>&1 | tail -30
          echo "=== identree logs ==="
          docker logs integ-vault-identree 2>&1 | tail -50
          echo "=== splunk-hec log ==="
          docker exec integ-vault-splunk-hec cat /var/log/hec.log 2>/dev/null | tail -10 || true
          echo "=== ubuntu2404 sssd.conf ==="
          docker exec integ-vault-ubuntu2404 cat /etc/sssd/sssd.conf 2>/dev/null || true
          echo "=== ubuntu2404 entrypoint output ==="
          docker logs integ-vault-ubuntu2404 2>&1 | tail -40

      - name: Tear down
        if: always()
        run: docker compose -f test/integration/vault-escrow/docker-compose.yml down -v