From 6b478a92b5fbf0d6ce689fb4bcaaa51546075007 Mon Sep 17 00:00:00 2001 From: "Lutar, Stephen P." Date: Thu, 13 Aug 2026 08:32:29 -0400 Subject: [PATCH 1/6] fix(ci): align product smoke with runtime API Probe the current Alloy Runtime API on its canonical loopback port. Verify build identity, readiness, missing and invalid API keys, and tenant reads. Make the product smoke a blocking P0 gate with a per-run loopback credential. Workcell: RUNTIME-AUDIT-P1-20260813 Signed-off-by: Lutar, Stephen P. --- .github/workflows/audit-full.yml | 8 + scripts/audit-full.js | 17 +- scripts/qa/smoke-product-mode.js | 730 +++++++++---------------------- 3 files changed, 227 insertions(+), 528 deletions(-) diff --git a/.github/workflows/audit-full.yml b/.github/workflows/audit-full.yml index 65c4028c9..48d1004cf 100644 --- a/.github/workflows/audit-full.yml +++ b/.github/workflows/audit-full.yml @@ -58,6 +58,11 @@ jobs: BASE_PATH: / - name: Boot web apps + runtime API for the qa-site smoke + env: + # Per-run credential for the ephemeral loopback runtime only. This is + # not a repository or environment secret and grants no external access. + ALLOY_API_KEY: runtime-audit-${{ github.run_id }}-${{ github.run_attempt }} + COMMIT_SHA: ${{ github.sha }} run: | set -uo pipefail # 'serve' is a pinned root devDependency installed by the frozen pnpm @@ -158,6 +163,9 @@ jobs: env: CI: 'true' NODE_ENV: test + API_BASE_URL: http://127.0.0.1:8080 + # Matches the per-run credential on the loopback runtime boot step. + SMOKE_API_KEY: runtime-audit-${{ github.run_id }}-${{ github.run_attempt }} # Cap turbo fan-out so the full-repo typecheck (composite tsc builds, # each allowed --max-old-space-size=6144) does not exhaust the 16 GB # runner and get OOM-killed mid-run ("The operation was canceled"). diff --git a/scripts/audit-full.js b/scripts/audit-full.js index 1e49358a6..dea423f71 100644 --- a/scripts/audit-full.js +++ b/scripts/audit-full.js @@ -7,9 +7,10 @@ * * Priority semantics * P0 — Blocking: boot failure, typecheck, lint, unit tests, build, broken core - * public routes. Pipeline aborts on first P0 failure. Exit code 1. + * public routes, runtime readiness, and API-key enforcement. Pipeline + * aborts on first P0 failure. Exit code 1. * P1 — Advisory: mocks, copy, deps, design-system, broken links, a11y, brand, - * docs claims, smoke, E2E. Recorded but do NOT block. Exit code 0. + * docs claims, E2E. Recorded but do NOT block. Exit code 0. * * Evidence layout * artifacts/audit/evidence/// @@ -103,6 +104,12 @@ const STEPS = [ cmd: 'pnpm run qa:site', priority: 'P0', }, + { + id: 'smoke-product-mode', + label: 'Smoke: product mode', + cmd: 'pnpm run smoke:product-mode', + priority: 'P0', + }, // Advisory checks — failures recorded and reported, but do not block merges. { id: 'audit-mocks', @@ -146,12 +153,6 @@ const STEPS = [ cmd: 'pnpm run brand:check', priority: 'P1', }, - { - id: 'smoke-product-mode', - label: 'Smoke: product mode', - cmd: 'pnpm run smoke:product-mode', - priority: 'P1', - }, { id: 'docs-claims-check', label: 'Docs: claims check', diff --git a/scripts/qa/smoke-product-mode.js b/scripts/qa/smoke-product-mode.js index f86b4e1dc..0806c98bc 100644 --- a/scripts/qa/smoke-product-mode.js +++ b/scripts/qa/smoke-product-mode.js @@ -1,572 +1,262 @@ #!/usr/bin/env node /** - * smoke-product-mode.js — Product-Mode Readiness Runner + * Product-mode smoke for the current Alloy Runtime API contract. * - * Validates that the platform is ready to operate in production/demo mode: - * 1. Critical environment variables exist - * 2. API server boots and responds to health check - * 3. Auth endpoints are reachable - * 4. Core trust routes load without error - * 5. Health endpoint reports real dependency status (not optimistic stub) - * 6. Demo data sentinel — confirms demo data is not treated as production data - * 7. No production-blocking errors in health response - * - * Usage: - * node scripts/qa/smoke-product-mode.js - * BASE_URL=https://szlholdings.com node scripts/qa/smoke-product-mode.js - * BASE_URL=http://localhost:5000 node scripts/qa/smoke-product-mode.js - * - * Exit codes: - * 0 — All checks passed - * 1 — One or more Sev 0 or Sev 1 checks failed + * The target server is already running. This probe verifies build identity, + * dependency readiness, fail-closed API-key enforcement, and an authenticated, + * tenant-scoped read without calling any mutation endpoint. */ -const BASE_URL = process.env.BASE_URL ?? process.env.API_BASE_URL ?? 'http://localhost:5000'; -const TIMEOUT_MS = parseInt(process.env.SMOKE_TIMEOUT_MS ?? '10000', 10); -const NODE_ENV = process.env.NODE_ENV ?? 'development'; -const IS_PRODUCTION = NODE_ENV === 'production'; - -const _COLORS = { - green: '\x1b[32m', - red: '\x1b[31m', - yellow: '\x1b[33m', - cyan: '\x1b[36m', - bold: '\x1b[1m', - reset: '\x1b[0m', -}; - -const pass = (_msg) => {}; -const fail = (_msg) => {}; -const warn = (_msg) => {}; -const info = (_msg) => {}; -const header = (_msg) => {}; - -const results = { sev0: [], sev1: [], sev2: [], skipped: [] }; - -function recordSev0(name, message) { - results.sev0.push({ name, message }); -} -function recordSev1(name, message) { - results.sev1.push({ name, message }); -} -function recordSev2(name, message) { - results.sev2.push({ name, message }); -} -function recordSkip(name, reason) { - results.skipped.push({ name, reason }); +import { appendFileSync } from 'node:fs'; +import { artifactUrl } from '../lib/artifact-ports.js'; + +const API_BASE_URL = ( + process.env.API_BASE_URL ?? + process.env.BASE_URL ?? + artifactUrl('api-server') +).replace(/\/+$/, ''); +const SMOKE_API_KEY = process.env.SMOKE_API_KEY; +const INVALID_SMOKE_API_KEY = `${SMOKE_API_KEY ?? 'unset'}-deliberately-invalid`; +const EXPECTED_GIT_SHA = process.env.GITHUB_SHA?.trim() || null; +const SMOKE_TENANT_ID = `runtime-audit-${process.env.GITHUB_RUN_ID ?? process.pid}`; +const parsedTimeout = Number.parseInt(process.env.SMOKE_TIMEOUT_MS ?? '10000', 10); +const TIMEOUT_MS = Number.isFinite(parsedTimeout) && parsedTimeout > 0 ? parsedTimeout : 10_000; +const EXPECTED_DEPENDENCIES = ['memory-store', 'run-registry', 'workflow-runtime']; + +const checks = []; +let livenessGitSha = null; + +function writeStdout(message) { + process.stdout.write(`${message}\n`); } -async function fetchWithTimeout(url, options = {}) { - const controller = new AbortController(); - const timer = setTimeout(() => controller.abort(), TIMEOUT_MS); - try { - const res = await fetch(url, { ...options, signal: controller.signal }); - return res; - } catch (err) { - if (err.name === 'AbortError') { - throw new Error(`Request timed out after ${TIMEOUT_MS}ms: ${url}`); - } - throw err; - } finally { - clearTimeout(timer); - } +function writeStderr(message) { + process.stderr.write(`${message}\n`); } -// ─── Check 1: Critical Environment Variables ────────────────────────────────── - -header('Check 1: Critical Environment Variables'); - -const REQUIRED_ENV_VARS = [ - { name: 'DATABASE_URL', sev: 0, description: 'PostgreSQL connection string' }, - { name: 'SESSION_SECRET', sev: 0, description: 'Session signing secret' }, -]; - -const RECOMMENDED_ENV_VARS = [ - { name: 'NODE_ENV', sev: 2, description: 'Runtime environment' }, - { name: 'PORT', sev: 2, description: 'Server port' }, -]; - -const PRODUCTION_REQUIRED = [ - { name: 'OBJECT_STORAGE_BUCKET_ID', sev: 1, description: 'Cloud object storage' }, -]; - -for (const { name, sev, description } of REQUIRED_ENV_VARS) { - if (process.env[name]) { - pass(`${name} — present (${description})`); - } else { - fail(`${name} — MISSING (${description})`); - if (sev === 0) recordSev0(`env:${name}`, `Required env var ${name} is missing`); - else recordSev1(`env:${name}`, `Required env var ${name} is missing`); - } +function invariant(condition, message) { + if (!condition) throw new Error(message); } -for (const { name, description } of RECOMMENDED_ENV_VARS) { - if (process.env[name]) { - pass(`${name} — present (${description})`); - } else { - warn(`${name} — not set (${description})`); - recordSev2(`env:${name}`, `Recommended env var ${name} is not set`); - } +function isValidDate(value) { + return typeof value === 'string' && Number.isFinite(Date.parse(value)); } -if (IS_PRODUCTION) { - for (const { name, description } of PRODUCTION_REQUIRED) { - if (process.env[name]) { - pass(`${name} — present (${description}) [production-required]`); - } else { - fail(`${name} — MISSING in production (${description})`); - recordSev1(`env:${name}`, `Production-required env var ${name} is missing`); - } - } -} else { - info(`Skipping production-only env checks (NODE_ENV=${NODE_ENV})`); -} +async function fetchJson(pathname, options = {}) { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), TIMEOUT_MS); -// ─── Check 2: API Server Boot Health ───────────────────────────────────────── - -header('Check 2: API Server Health Endpoint'); - -let healthData = null; -let healthReachable = false; - -try { - const healthUrl = `${BASE_URL}/api/health`; - info(`GET ${healthUrl}`); - const res = await fetchWithTimeout(healthUrl); - - if (res.status === 200) { - pass(`/api/health — HTTP 200`); - healthReachable = true; - try { - healthData = await res.json(); - const status = healthData?.status; - if (status === 'ok') { - pass(`Health status: ${status}`); - } else if (status === 'degraded') { - warn(`Health status: ${status} — check service dependencies`); - recordSev1( - 'health:status', - `Health endpoint reports degraded status: ${JSON.stringify(healthData?.services ?? {})}`, - ); - } else { - warn(`Health status: ${status ?? 'unknown'}`); - } - } catch { - warn(`/api/health returned 200 but body is not valid JSON`); - recordSev2('health:json', 'Health endpoint returned non-JSON body'); - } - } else { - fail(`/api/health — HTTP ${res.status}`); - recordSev0('health:status-code', `/api/health returned HTTP ${res.status}`); - } -} catch (err) { - fail(`/api/health — ${err.message}`); - recordSev0('health:reachable', `API server unreachable: ${err.message}`); -} + try { + const response = await fetch(`${API_BASE_URL}${pathname}`, { + ...options, + signal: controller.signal, + }); + const rawBody = await response.text(); + let body = null; -// ─── Check 3: Readiness Probe (DB-aware) ───────────────────────────────────── - -header('Check 3: Readiness Probe (DB-aware health)'); - -try { - const readyUrl = `${BASE_URL}/api/health/ready`; - info(`GET ${readyUrl}`); - const res = await fetchWithTimeout(readyUrl); - - if (res.status === 200) { - pass(`/api/health/ready — HTTP 200`); - try { - const readyData = await res.json(); - // /api/health/ready uses { checks: { database, server, uptime } } shape - // /api/health uses { services: { database: { status, latencyMs } } } shape - const dbCheck = readyData?.checks?.database; - const dbServiceStatus = readyData?.services?.database?.status; - if (dbCheck === 'connected' || dbCheck === 'ok' || dbServiceStatus === 'ok') { - pass(`Database status in readiness probe: connected`); - } else if (dbCheck && dbCheck !== 'connected' && dbCheck !== 'ok') { - fail(`Database status in readiness probe: ${dbCheck}`); - recordSev1('health:db-degraded', `Readiness probe reports database: ${dbCheck}`); - } else if (!readyData?.checks && !readyData?.services) { - warn(`Readiness probe response has no checks or services key — health may be optimistic`); - recordSev2('health:optimistic', 'Readiness probe lacks per-service status breakdown'); - } - } catch { - warn(`/api/health/ready body is not valid JSON`); - } - } else if (res.status === 503) { - fail(`/api/health/ready — HTTP 503 (not ready)`); - recordSev0('health:not-ready', 'Readiness probe returned 503 — system not ready'); - } else if (res.status === 404) { - // /api/health/ready is not implemented in this API — fall back to /api/health - info( - `/api/health/ready — HTTP 404 (no dedicated readiness probe; using /api/health DB status)`, - ); - if (healthReachable && healthData?.services?.database) { - const dbStatus = healthData.services.database.status; - if (dbStatus === 'ok') { - pass(`Database status from /api/health: ok`); - } else { - fail(`Database status from /api/health: ${dbStatus}`); - recordSev1('health:db-degraded', `DB status from /api/health is ${dbStatus}`); + if (rawBody.trim().length > 0) { + try { + body = JSON.parse(rawBody); + } catch { + throw new Error(`${pathname} returned invalid JSON`); } - } else { - recordSkip( - 'health:ready', - '/api/health/ready not implemented; DB status unavailable from /api/health', - ); - } - } else { - warn(`/api/health/ready — HTTP ${res.status} (unexpected; checking /api/health fallback)`); - if (healthReachable && healthData?.services?.database) { - info(`Falling back to DB status from /api/health: ${healthData.services.database.status}`); } - } -} catch { - warn(`/api/health/ready — connection error; falling back to /api/health DB status`); - if (healthReachable && healthData?.services?.database) { - const dbStatus = healthData.services.database.status; - if (dbStatus === 'ok') { - pass(`Database status from /api/health: ok (fallback check)`); - } else { - fail(`Database status from /api/health: ${dbStatus}`); - recordSev1('health:db-degraded-fallback', `DB status from /api/health is ${dbStatus}`); - } - } else { - recordSkip('health:ready', '/api/health/ready not reachable; DB check skipped'); - } -} -// ─── Check 4: Auth Contract Validation ─────────────────────────────────────── -// -// Validates the auth contract at three levels: -// 4a. /api/auth/user — returns 200 with { user: null } when unauthenticated, -// confirming the auth endpoint is up and not erroring -// 4b. Auth guard enforcement — known protected routes must return 401 without -// credentials; a 200 here means auth is bypassed (Sev 0) -// 4c. Login entry point — GET /api/login must be reachable (302 or 503 ok) -// -// Routes are derived from actual oidc-auth.ts and auth.ts registrations: -// - GET /api/auth/user → user info (200 with {user:null} when anon) -// - GET /api/login → OIDC login redirect (302 or 503 when unconfigured) -// - POST /api/auth/login → credential login (not probed — needs POST body) -// - GET /api/apm/snapshot → protected by authMiddleware() → must 401 anon -// - GET /api/connectors → protected by authMiddleware() → must 401 anon - -header('Check 4: Auth Contract Validation'); - -// 4a: Auth user endpoint — must be reachable and return non-5xx -// Returns { user: null } when unauthenticated (200 is correct behavior here) -try { - const userUrl = `${BASE_URL}/api/auth/user`; - info(`GET ${userUrl}`); - const res = await fetchWithTimeout(userUrl, { method: 'GET' }); - - if (res.status >= 500) { - fail(`/api/auth/user — HTTP ${res.status} (server error on auth endpoint)`); - recordSev0('auth:user-5xx', `/api/auth/user returned ${res.status} — auth system error`); - } else if (res.status === 404) { - fail(`/api/auth/user — HTTP 404 (auth/user endpoint not registered)`); - recordSev1( - 'auth:user-missing', - '/api/auth/user returned 404 — auth endpoint must be registered', - ); - } else if (res.status === 200) { - let body = null; - try { - body = await res.json(); - } catch { - /* ignore */ - } - if (body !== null && typeof body === 'object' && 'user' in body) { - pass(`/api/auth/user — HTTP 200 with {user} shape (unauthenticated returns user:null)`); - } else { - pass(`/api/auth/user — HTTP 200 (auth endpoint up)`); + return { response, body }; + } catch (error) { + if (error instanceof Error && error.name === 'AbortError') { + throw new Error(`${pathname} timed out after ${TIMEOUT_MS}ms`); } - } else { - pass(`/api/auth/user — HTTP ${res.status} (auth endpoint reachable)`); + throw error; + } finally { + clearTimeout(timer); } -} catch (err) { - fail(`/api/auth/user — ${err.message}`); - recordSev0('auth:user-unreachable', `/api/auth/user unreachable: ${err.message}`); } -// 4b: Auth guard enforcement — probe known protected routes without credentials. -// These routes use authMiddleware() with no { required: false } override, so they -// MUST return 401 for unauthenticated requests. A 200 means auth is bypassed (Sev 0). -const PROTECTED_ROUTES_TO_PROBE = [ - { path: '/api/apm/snapshot', description: 'APM snapshot — authMiddleware() enforced' }, - { path: '/api/connectors', description: 'Connectors list — authMiddleware() enforced' }, - { path: '/api/audit/events', description: 'Audit events — authMiddleware() enforced' }, -]; - -let authGuardVerified = false; -for (const { path, description } of PROTECTED_ROUTES_TO_PROBE) { +async function runCheck(id, probe) { + const startedAt = Date.now(); + try { - const url = `${BASE_URL}${path}`; - info(`GET ${url} (without credentials — expecting 401)`); - const res = await fetchWithTimeout(url, { method: 'GET' }); - - if (res.status === 401 || res.status === 403) { - pass( - `${path} — HTTP ${res.status} (auth guard confirmed — unauthenticated request rejected)`, - ); - authGuardVerified = true; - break; - } else if (res.status === 200) { - fail(`${path} — HTTP 200 without credentials (auth guard BYPASSED!)`); - recordSev0( - 'auth:guard-bypassed', - `Protected route ${path} (${description}) returned 200 without credentials — auth middleware is broken`, - ); - break; - } else if (res.status >= 500) { - warn(`${path} — HTTP ${res.status} (server error; trying next route)`); - } else { - info(`${path} — HTTP ${res.status} (unexpected; trying next route)`); - } - } catch { - info(`${path} — not reachable; trying next protected route`); + const detail = await probe(); + const result = { id, passed: true, detail, durationMs: Date.now() - startedAt }; + checks.push(result); + writeStdout(`PASS ${id}: ${detail}`); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + const result = { id, passed: false, detail, durationMs: Date.now() - startedAt }; + checks.push(result); + writeStderr(`FAIL ${id}: ${detail}`); } } -if ( - !authGuardVerified && - results.sev0.filter((r) => r.name === 'auth:guard-bypassed').length === 0 -) { - warn( - 'Auth guard unverifiable — known protected routes all returned non-200/non-401 (API server may not be running)', +writeStdout(`Product-mode smoke target: ${API_BASE_URL}`); +writeStdout(`Expected build SHA: ${EXPECTED_GIT_SHA ?? '(not supplied)'}`); + +await runCheck('liveness-build-identity', async () => { + const { response, body } = await fetchJson('/healthz'); + + invariant(response.status === 200, `/healthz returned HTTP ${response.status}, expected 200`); + invariant(body && typeof body === 'object', '/healthz returned no JSON object'); + invariant( + body.status === 'ok', + `/healthz status is ${JSON.stringify(body.status)}, expected "ok"`, ); - recordSev2( - 'auth:guard-unverifiable', - 'Could not confirm auth guard enforcement — API server may be offline', + invariant( + body.service === 'alloy-runtime-api', + `/healthz service is ${JSON.stringify(body.service)}, expected "alloy-runtime-api"`, + ); + invariant( + typeof body.version === 'string' && body.version.length > 0, + '/healthz version is empty', + ); + invariant(typeof body.gitSha === 'string' && body.gitSha.length > 0, '/healthz gitSha is empty'); + invariant(isValidDate(body.bootTime), '/healthz bootTime is not a valid timestamp'); + invariant( + Number.isFinite(body.uptimeSeconds) && body.uptimeSeconds >= 0, + '/healthz uptimeSeconds is not a finite nonnegative number', ); -} -// 4c: Login entry point — GET /api/login must exist (302 to OIDC or 503 when unconfigured) -try { - const loginUrl = `${BASE_URL}/api/login`; - info(`GET ${loginUrl}`); - const res = await fetchWithTimeout(loginUrl, { method: 'GET', redirect: 'manual' }); - - if (res.status >= 500) { - fail(`/api/login — HTTP ${res.status} (server error on login route)`); - recordSev1('auth:login-5xx', `/api/login returned ${res.status} — login entry point is broken`); - } else if (res.status === 404) { - fail(`/api/login — HTTP 404 (login route not registered)`); - recordSev1( - 'auth:login-missing', - '/api/login returned 404 — OIDC login entry point must be registered', + if (EXPECTED_GIT_SHA) { + invariant( + body.gitSha === EXPECTED_GIT_SHA, + `/healthz gitSha ${body.gitSha} does not match expected ${EXPECTED_GIT_SHA}`, ); - } else if (res.status === 302 || res.status === 301) { - pass(`/api/login — HTTP ${res.status} (login redirects to OIDC provider)`); - } else if (res.status === 503) { - warn(`/api/login — HTTP 503 (OIDC not configured — acceptable in dev/staging without OIDC)`); - } else { - pass(`/api/login — HTTP ${res.status} (login route reachable)`); } -} catch (err) { - warn(`/api/login — ${err.message}`); - recordSev2('auth:login-unreachable', `/api/login not reachable: ${err.message}`); -} -// ─── Check 5: Core Trust Routes ────────────────────────────────────────────── + livenessGitSha = body.gitSha; + return `HTTP 200; service=${body.service}; gitSha=${body.gitSha}`; +}); -header('Check 5: Core Trust Routes'); +await runCheck('dependency-readiness', async () => { + const { response, body } = await fetchJson('/readyz'); -// Trust routes are PUBLIC endpoints (no authMiddleware) that must respond 200. -// /api/status is excluded — it is protected and will always 401 unauthenticated. -// Use /api/auth/providers and /api/health/integrations as canonical public probes. -const TRUST_ROUTES = [ - { path: '/api/auth/providers', description: 'Auth providers list (public, from oidc-auth.ts)' }, - { - path: '/api/health/integrations', - description: 'Integration health snapshot (public, from health-integrations.ts)', - }, - { path: '/api/health', description: 'Health endpoint (repeated, expect cached)' }, -]; - -for (const { path, description } of TRUST_ROUTES) { - try { - const url = `${BASE_URL}${path}`; - const res = await fetchWithTimeout(url); - if (res.status < 400) { - pass(`${path} — HTTP ${res.status} (${description})`); - } else { - warn(`${path} — HTTP ${res.status} (${description})`); - recordSev2(`trust:${path}`, `Trust route ${path} returned ${res.status}`); - } - } catch (err) { - warn(`${path} — not reachable (${err.message})`); - recordSev2(`trust:${path}`, `Trust route ${path} unreachable`); - } -} - -// ─── Check 6: Demo Data Sentinel ───────────────────────────────────────────── + invariant(response.status === 200, `/readyz returned HTTP ${response.status}, expected 200`); + invariant(body && typeof body === 'object', '/readyz returned no JSON object'); + invariant(body.ready === true, `/readyz ready is ${JSON.stringify(body.ready)}, expected true`); + invariant( + body.service === 'alloy-runtime-api', + `/readyz service is ${JSON.stringify(body.service)}, expected "alloy-runtime-api"`, + ); + invariant(isValidDate(body.checkedAt), '/readyz checkedAt is not a valid timestamp'); + invariant(Array.isArray(body.dependencies), '/readyz dependencies is not an array'); + invariant(typeof livenessGitSha === 'string', 'liveness build identity was not established'); + invariant( + body.gitSha === livenessGitSha, + `/readyz gitSha ${JSON.stringify(body.gitSha)} does not match /healthz ${livenessGitSha}`, + ); -header('Check 6: Demo Data Sentinel'); + const dependencyNames = body.dependencies.map((dependency) => dependency?.name); + invariant( + body.dependencies.length === EXPECTED_DEPENDENCIES.length && + new Set(dependencyNames).size === EXPECTED_DEPENDENCIES.length && + EXPECTED_DEPENDENCIES.every((name) => dependencyNames.includes(name)), + `/readyz dependencies are ${JSON.stringify(dependencyNames)}, expected ${JSON.stringify(EXPECTED_DEPENDENCIES)}`, + ); -if (IS_PRODUCTION) { - info('Running in production — checking for demo data leakage indicators'); - try { - if (healthData) { - const version = healthData?.version; - const isDemoVersion = typeof version === 'string' && version.includes('demo'); - if (isDemoVersion) { - fail(`Version string contains 'demo' in production: ${version}`); - recordSev1('demo:version', 'Production health endpoint exposes demo version marker'); - } else { - pass(`Version string does not indicate demo mode: ${version ?? '(not set)'}`); - } - } else { - info('Health data unavailable — skipping version check'); - recordSkip('demo:version', 'Health data not available for demo sentinel check'); - } - } catch { - recordSkip('demo:sentinel', 'Demo data sentinel skipped — health not available'); + for (const dependency of body.dependencies) { + invariant(dependency.ready === true, `${dependency.name} readiness is not true`); + invariant( + Number.isFinite(dependency.latencyMs) && dependency.latencyMs >= 0, + `${dependency.name} latencyMs is not a finite nonnegative number`, + ); + invariant(dependency.detail === 'ok', `${dependency.name} detail is not "ok"`); } -} else { - info(`NODE_ENV=${NODE_ENV} — demo data checks are relaxed in non-production`); - pass('Demo data sentinel skipped for non-production environment'); -} - -// ─── Check 7: Health Endpoint Authenticity ─────────────────────────────────── -header('Check 7: Health Endpoint Authenticity (Anti-Optimism Check)'); + return `HTTP 200; ready=true; dependencies=${dependencyNames.join(',')}`; +}); -if (healthData?.services) { - const db = healthData.services.database; - if (!db) { - warn('Health endpoint does not report database status — may be optimistic'); - recordSev2('health:authenticity', 'Health endpoint lacks database status field'); - } else if (db.status === 'ok' && typeof db.latencyMs !== 'number') { - warn("Health endpoint reports DB 'ok' without a latency measurement — may not be checking DB"); - recordSev2( - 'health:db-check-depth', - 'DB health check lacks latency measurement — verify it performs a real query', - ); - } else { - pass( - `Health endpoint reports database status: ${db.status} (latency: ${db.latencyMs ?? 'N/A'}ms)`, - ); - } +await runCheck('anonymous-api-key-guard', async () => { + const { response, body } = await fetchJson('/v1/workflows', { + headers: { 'X-Tenant-Id': SMOKE_TENANT_ID }, + }); - const auth = healthData.services.auth; - if (auth?.status === 'ok' && auth?.mode === 'missing_secret') { - fail("Health reports auth 'ok' but mode is 'missing_secret' — contradictory status"); - recordSev1('health:auth-contradiction', 'Auth status is ok but session secret is missing'); - } else if (auth) { - pass(`Auth status: ${auth.status} (mode: ${auth.mode ?? 'N/A'})`); - } -} else if (healthReachable) { - warn('Health endpoint reachable but lacks services breakdown — treating as optimistic'); - recordSev2('health:no-services', 'Health endpoint does not include per-service status'); -} else { - recordSkip('health:authenticity', 'Health endpoint not reachable — cannot check authenticity'); -} + invariant( + response.status === 401, + `/v1/workflows returned HTTP ${response.status}, expected 401`, + ); + invariant( + body?.code === 'INVALID_API_KEY', + `/v1/workflows error code is ${JSON.stringify(body?.code)}, expected "INVALID_API_KEY"`, + ); -const totalFailed = results.sev0.length + results.sev1.length; -const totalWarnings = results.sev2.length; + return 'HTTP 401; code=INVALID_API_KEY'; +}); -if (results.sev0.length > 0) { - for (const { name, message } of results.sev0) { - } -} +await runCheck('invalid-api-key-guard', async () => { + const { response, body } = await fetchJson('/v1/workflows', { + headers: { + 'X-Api-Key': INVALID_SMOKE_API_KEY, + 'X-Tenant-Id': SMOKE_TENANT_ID, + }, + }); -if (results.sev1.length > 0) { - for (const { name, message } of results.sev1) { - } -} + invariant( + response.status === 401, + `/v1/workflows returned HTTP ${response.status} for an invalid key, expected 401`, + ); + invariant( + body?.code === 'INVALID_API_KEY', + `/v1/workflows error code for an invalid key is ${JSON.stringify(body?.code)}, expected "INVALID_API_KEY"`, + ); -if (results.sev2.length > 0) { - for (const { name, message } of results.sev2) { - } -} + return 'HTTP 401; code=INVALID_API_KEY'; +}); -if (results.skipped.length > 0) { - for (const { name, reason } of results.skipped) { - } -} +await runCheck('authenticated-tenant-read', async () => { + invariant( + typeof SMOKE_API_KEY === 'string' && SMOKE_API_KEY.length > 0, + 'SMOKE_API_KEY is required for the authenticated product-mode probe', + ); -if (totalFailed === 0 && totalWarnings === 0) { -} else if (totalFailed === 0) { -} else { - if (results.sev0.length > 0) { - } -} + const { response, body } = await fetchJson('/v1/workflows', { + headers: { + 'X-Api-Key': SMOKE_API_KEY, + 'X-Tenant-Id': SMOKE_TENANT_ID, + }, + }); -// ─── GitHub Actions Step Summary ───────────────────────────────────────────── -// When running in GitHub Actions, $GITHUB_STEP_SUMMARY points to a markdown file -// that gets rendered in the PR check view. Writing a summary here makes failures -// easy to read without scrolling through raw logs (where ANSI colors are stripped). + invariant( + response.status === 200, + `/v1/workflows returned HTTP ${response.status}, expected 200`, + ); + invariant(Array.isArray(body?.runs), '/v1/workflows runs is not an array'); + invariant( + body.tenantId === SMOKE_TENANT_ID, + `/v1/workflows tenantId is ${JSON.stringify(body?.tenantId)}, expected ${SMOKE_TENANT_ID}`, + ); -if (process.env.GITHUB_STEP_SUMMARY) { - try { - const fs = await import('node:fs'); - - const overall = - totalFailed === 0 && totalWarnings === 0 - ? '✅ All product-mode checks passed' - : totalFailed === 0 - ? `⚠️ No blocking failures — ${totalWarnings} warning(s) to address` - : `❌ ${totalFailed} blocking failure(s) — platform not ready for release`; - - const lines = []; - lines.push(`## Product-Mode Smoke Test`); - lines.push(''); - lines.push(`**Base URL:** \`${BASE_URL}\` `); - lines.push(`**NODE_ENV:** \`${NODE_ENV}\` `); - lines.push(`**Result:** ${overall}`); - lines.push(''); - lines.push(`| Severity | Count |`); - lines.push(`| --- | --- |`); - lines.push(`| Sev 0 (deployment blocked) | ${results.sev0.length} |`); - lines.push(`| Sev 1 (release blocked) | ${results.sev1.length} |`); - lines.push(`| Sev 2 (warnings) | ${results.sev2.length} |`); - lines.push(`| Skipped | ${results.skipped.length} |`); - lines.push(''); - - const renderRows = (items) => - items.length === 0 - ? '_None_' - : [ - `| Check | Detail |`, - `| --- | --- |`, - ...items.map( - ({ name, message, reason }) => - `| \`${name}\` | ${String(message ?? reason ?? '').replace(/\|/g, '\\|')} |`, - ), - ].join('\n'); - - if (results.sev0.length > 0) { - lines.push(`### ❌ Sev 0 — Deployment Blocked (${results.sev0.length})`); - lines.push(renderRows(results.sev0)); - lines.push(''); - } - if (results.sev1.length > 0) { - lines.push(`### ❌ Sev 1 — Release Blocked (${results.sev1.length})`); - lines.push(renderRows(results.sev1)); - lines.push(''); - } - if (results.sev2.length > 0) { - lines.push(`### ⚠️ Sev 2 — Warnings (${results.sev2.length})`); - lines.push(renderRows(results.sev2)); - lines.push(''); - } - if (results.skipped.length > 0) { - lines.push(`
Skipped checks (${results.skipped.length})`); - lines.push(''); - lines.push(renderRows(results.skipped)); - lines.push(''); - lines.push(`
`); - lines.push(''); - } + return `HTTP 200; tenantId=${body.tenantId}; runs=${body.runs.length}`; +}); + +const passed = checks.every((check) => check.passed); +const result = { + schemaVersion: 1, + baseUrl: API_BASE_URL, + expectedGitSha: EXPECTED_GIT_SHA, + tenantId: SMOKE_TENANT_ID, + passed, + checks, +}; - lines.push(`See \`docs/FAILURE_SEVERITY_POLICY.md\` for severity definitions.`); - lines.push(''); +writeStdout(`PRODUCT_SMOKE_RESULT ${JSON.stringify(result)}`); - fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, lines.join('\n')); - } catch (_err) { - } +if (process.env.GITHUB_STEP_SUMMARY) { + const lines = [ + '## Alloy Runtime product-mode smoke', + '', + `- Target: \`${API_BASE_URL}\``, + `- Expected SHA: \`${EXPECTED_GIT_SHA ?? 'not supplied'}\``, + `- Result: **${passed ? 'PASS' : 'FAIL'}**`, + '', + '| Check | Result | Detail |', + '| --- | --- | --- |', + ...checks.map( + (check) => + `| \`${check.id}\` | ${check.passed ? 'PASS' : 'FAIL'} | ${check.detail.replace(/\|/g, '\\|')} |`, + ), + '', + ]; + appendFileSync(process.env.GITHUB_STEP_SUMMARY, lines.join('\n')); } -process.exit(totalFailed > 0 ? 1 : 0); +process.exitCode = passed ? 0 : 1; From f268afad3a20ea007eb30f522323817fff5bb01d Mon Sep 17 00:00:00 2001 From: "Lutar, Stephen P." Date: Thu, 13 Aug 2026 08:36:37 -0400 Subject: [PATCH 2/6] docs(ci): describe runtime audit gate accurately Classify the product-mode smoke as blocking P0 inside the harness. Correct the CI exit semantics and distinguish hosted evidence from the status contexts currently required by live branch protection. Workcell: RUNTIME-AUDIT-P1-20260813 Signed-off-by: Lutar, Stephen P. --- docs/ops/audit-harness.md | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/docs/ops/audit-harness.md b/docs/ops/audit-harness.md index e3b132ad3..c4ecdefdd 100644 --- a/docs/ops/audit-harness.md +++ b/docs/ops/audit-harness.md @@ -2,10 +2,10 @@ ## Overview -The runtime audit harness provides a single command that boots every product, -runs a comprehensive set of quality checks, captures evidence, and produces a -human-readable summary report. It is the required gate for every change in this -workspace. +The runtime audit harness provides a single command that runs the workspace's +P0 and P1 quality checks, captures evidence, and produces a human-readable +summary report. Its hosted workflow is fail-closed for P0 failures, but it is +not currently a required branch-protection status check. ## Quick start @@ -16,7 +16,7 @@ pnpm audit:full # Fast local iteration (skip install + E2E) pnpm audit:full:fast -# CI mode — same as fast but exits 2 on P1 failures +# CI mode — skips install + E2E; exits 1 on P0 and records P1 as advisory pnpm audit:full:ci ``` @@ -31,6 +31,7 @@ pnpm audit:full:ci | P0 | build | Full recursive build of all artifacts | | P0 | audit:routes | Route registry completeness and classification | | P0 | qa:site | Public routes + links + trust pages + meta + empty-states | +| P0 | smoke:product-mode | Runtime identity, readiness, API-key rejection, and tenant-scoped read | | P1 | audit:mocks | Detect real API calls leaking through mock boundaries | | P1 | audit:copy | UI copy consistency (no placeholder text) | | P1 | audit:deps | Dependency health (missing/circular/unused) | @@ -38,7 +39,6 @@ pnpm audit:full:ci | P1 | audit:broken-links | Internal hyperlink integrity | | P1 | qa:a11y | Accessibility audit (axe-core) | | P1 | brand:check | Brand token and copy compliance | -| P1 | smoke:product-mode | Product-mode isolation smoke tests | | P1 | docs:claims-check | Documented claims vs. codebase reality | | P1 | e2e | Playwright end-to-end tests (skipped with `--skip-e2e`) | @@ -81,10 +81,11 @@ The harness runs as the `Runtime Audit Harness` job in artifact named `audit-evidence-`, retained for 30 days. 4. Prints the `latest/summary.md` to the job log regardless of pass/fail. -P0 failures cause the job to exit non-zero, blocking the merge. P1 failures -are advisory and never block the merge — they appear in the summary report -so they can be tracked and resolved. The existing `ci-gate` job in `ci.yml` -remains unchanged; this workflow is an additive, standalone gate. +P0 failures cause the job to exit non-zero. P1 failures remain advisory and +appear in the summary report. Under the current live branch-protection rules, +Runtime Audit is hosted evidence but is not a required status context, so its +failure does not mechanically prevent a merge. The existing jobs in `ci.yml` +remain unchanged; this workflow is an additive, standalone check. ## Interpreting the summary From 3f2566b40ccdc1c111c561a5187041b24885ae82 Mon Sep 17 00:00:00 2001 From: "Lutar, Stephen P." Date: Thu, 13 Aug 2026 08:39:48 -0400 Subject: [PATCH 3/6] docs(ci): remove stale audit harness claims Describe P0 as harness- and job-blocking rather than branch-required. Match the current Biome lint command, workflow triggers, and build/boot steps. Workcell: RUNTIME-AUDIT-P1-20260813 Signed-off-by: Lutar, Stephen P. --- docs/ops/audit-harness.md | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/docs/ops/audit-harness.md b/docs/ops/audit-harness.md index c4ecdefdd..580889456 100644 --- a/docs/ops/audit-harness.md +++ b/docs/ops/audit-harness.md @@ -26,7 +26,7 @@ pnpm audit:full:ci |----------|------|-------------| | P0 | install | `pnpm install --frozen-lockfile` | | P0 | typecheck | TypeScript compilation across all packages | -| P0 | lint | ESLint across the full workspace | +| P0 | lint | Biome lint across the full workspace | | P0 | test | Unit + proof-chain tests | | P0 | build | Full recursive build of all artifacts | | P0 | audit:routes | Route registry completeness and classification | @@ -42,10 +42,11 @@ pnpm audit:full:ci | P1 | docs:claims-check | Documented claims vs. codebase reality | | P1 | e2e | Playwright end-to-end tests (skipped with `--skip-e2e`) | -**P0 failures are merge-blocking.** A failure in any P0 step aborts the -pipeline immediately and exits with code 1. P1 failures are always recorded -as advisory warnings and never affect the exit code — they appear in the -summary report so they can be tracked and resolved over time. +**P0 failures are harness-blocking.** A failure in any P0 step aborts the +pipeline immediately, exits with code 1, and fails the hosted Runtime Audit +job. P1 failures are always recorded as advisory warnings and never affect the +exit code — they appear in the summary report so they can be tracked and +resolved over time. ## Evidence @@ -72,14 +73,16 @@ it in under 60 seconds. ## CI integration The harness runs as the `Runtime Audit Harness` job in -`.github/workflows/audit-full.yml` on every pull request and push to -`master`/`main`. The job: +`.github/workflows/audit-full.yml` on pull requests and pushes to +`master`/`main` (pushes changing only `replit-sync/**` are ignored), and by +manual dispatch. The job: 1. Installs dependencies. -2. Runs `pnpm audit:full:ci` (all steps except E2E; exits 1 on P0 failure only). -3. Uploads the entire `artifacts/audit/evidence/` tree as a GitHub Actions +2. Builds the workspace artifacts and boots the local product/runtime targets. +3. Runs `pnpm audit:full:ci` (all steps except E2E; exits 1 on P0 failure only). +4. Uploads the entire `artifacts/audit/evidence/` tree as a GitHub Actions artifact named `audit-evidence-`, retained for 30 days. -4. Prints the `latest/summary.md` to the job log regardless of pass/fail. +5. Prints the `latest/summary.md` to the job log regardless of pass/fail. P0 failures cause the job to exit non-zero. P1 failures remain advisory and appear in the summary report. Under the current live branch-protection rules, From 10282cda99f7cdd33ea9c646a14359e3ebf4b222 Mon Sep 17 00:00:00 2001 From: "Lutar, Stephen P." Date: Thu, 13 Aug 2026 08:42:17 -0400 Subject: [PATCH 4/6] docs(ci): name hosted audit steps exactly Use the exact workflow and job names and state that CI skips both the harness install and E2E steps after installing dependencies separately. Workcell: RUNTIME-AUDIT-P1-20260813 Signed-off-by: Lutar, Stephen P. --- docs/ops/audit-harness.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/ops/audit-harness.md b/docs/ops/audit-harness.md index 580889456..cd477e130 100644 --- a/docs/ops/audit-harness.md +++ b/docs/ops/audit-harness.md @@ -72,14 +72,15 @@ it in under 60 seconds. ## CI integration -The harness runs as the `Runtime Audit Harness` job in -`.github/workflows/audit-full.yml` on pull requests and pushes to -`master`/`main` (pushes changing only `replit-sync/**` are ignored), and by -manual dispatch. The job: +The harness runs in the `Runtime Audit (audit:full)` job of the +`Runtime Audit Harness` workflow (`.github/workflows/audit-full.yml`) on pull +requests and pushes to `master`/`main` (pushes changing only +`replit-sync/**` are ignored), and by manual dispatch. The job: 1. Installs dependencies. 2. Builds the workspace artifacts and boots the local product/runtime targets. -3. Runs `pnpm audit:full:ci` (all steps except E2E; exits 1 on P0 failure only). +3. Runs `pnpm audit:full:ci` (the harness marks install and E2E skipped; exits 1 + on P0 failure only). 4. Uploads the entire `artifacts/audit/evidence/` tree as a GitHub Actions artifact named `audit-evidence-`, retained for 30 days. 5. Prints the `latest/summary.md` to the job log regardless of pass/fail. From 87d99dd249d1d29cc0de7498ff9eb3d84f65075e Mon Sep 17 00:00:00 2001 From: "Lutar, Stephen P." Date: Thu, 13 Aug 2026 08:54:10 -0400 Subject: [PATCH 5/6] fix(ci): escape audit summary table cells Escape existing backslashes before Markdown table delimiters and normalize line breaks so runtime check details cannot produce incomplete escaping. Workcell: RUNTIME-AUDIT-P1-20260813 Signed-off-by: Lutar, Stephen P. --- scripts/qa/smoke-product-mode.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/qa/smoke-product-mode.js b/scripts/qa/smoke-product-mode.js index 0806c98bc..f2ff2f090 100644 --- a/scripts/qa/smoke-product-mode.js +++ b/scripts/qa/smoke-product-mode.js @@ -42,6 +42,10 @@ function isValidDate(value) { return typeof value === 'string' && Number.isFinite(Date.parse(value)); } +function escapeMarkdownTableCell(value) { + return String(value).replace(/\\/g, '\\\\').replace(/\|/g, '\\|').replace(/\r?\n/g, ' '); +} + async function fetchJson(pathname, options = {}) { const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), TIMEOUT_MS); @@ -252,7 +256,7 @@ if (process.env.GITHUB_STEP_SUMMARY) { '| --- | --- | --- |', ...checks.map( (check) => - `| \`${check.id}\` | ${check.passed ? 'PASS' : 'FAIL'} | ${check.detail.replace(/\|/g, '\\|')} |`, + `| \`${escapeMarkdownTableCell(check.id)}\` | ${check.passed ? 'PASS' : 'FAIL'} | ${escapeMarkdownTableCell(check.detail)} |`, ), '', ]; From e6a0c534c443eb0ea8a759e15380b6db0d9393eb Mon Sep 17 00:00:00 2001 From: "Lutar, Stephen P." Date: Thu, 13 Aug 2026 08:57:33 -0400 Subject: [PATCH 6/6] fix(ci): normalize every Markdown line ending Flatten bare carriage returns as well as LF and CRLF before writing runtime probe details into the GitHub step-summary table. Workcell: RUNTIME-AUDIT-P1-20260813 Signed-off-by: Lutar, Stephen P. --- scripts/qa/smoke-product-mode.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/qa/smoke-product-mode.js b/scripts/qa/smoke-product-mode.js index f2ff2f090..0942b83ef 100644 --- a/scripts/qa/smoke-product-mode.js +++ b/scripts/qa/smoke-product-mode.js @@ -43,7 +43,10 @@ function isValidDate(value) { } function escapeMarkdownTableCell(value) { - return String(value).replace(/\\/g, '\\\\').replace(/\|/g, '\\|').replace(/\r?\n/g, ' '); + return String(value) + .replace(/\\/g, '\\\\') + .replace(/\|/g, '\\|') + .replace(/\r\n?|\n/g, ' '); } async function fetchJson(pathname, options = {}) {