diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..ef8ee85 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,28 @@ +name: CI +on: + push: + branches: [main] + pull_request: + branches: [main] +jobs: + build-and-test: + name: Build + tests (Node ${{ matrix.node }}) + runs-on: ubuntu-latest + strategy: + matrix: + node: [20, 22] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node }} + cache: npm + - run: npm ci + - run: npm run db:generate + - run: npm test + docker-build: + name: Docker build validation + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: docker build -t ai-pr-bot-ci . diff --git a/Dockerfile b/Dockerfile index 738d923..3001811 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,31 +1,23 @@ -FROM node:18-alpine +# Multi-stage build (fixed 2026-07-18): the previous single-stage image installed with +# --omit=dev and then ran `npm run build` — but tsc is a devDependency, so the image +# could never build. Stage 1 installs everything and compiles; stage 2 ships only +# production deps + dist. -# Install dependencies for Prisma +FROM node:20-alpine AS build RUN apk add --no-cache openssl - WORKDIR /app - -# Copy package files COPY package*.json ./ COPY prisma ./prisma/ - -# Install dependencies -RUN if [ -f package-lock.json ]; then \ - npm ci --omit=dev; \ - else \ - npm install --omit=dev; \ - fi && \ - npx prisma generate && \ - npm cache clean --force - -# Copy application files +RUN npm ci && npx prisma generate COPY . . - -# Build TypeScript RUN npm run build -# Expose port +FROM node:20-alpine +RUN apk add --no-cache openssl +WORKDIR /app +COPY package*.json ./ +COPY prisma ./prisma/ +RUN npm ci --omit=dev && npx prisma generate && npm cache clean --force +COPY --from=build /app/dist ./dist EXPOSE 3001 - -# Start the application CMD ["npm", "start"] diff --git a/README.md b/README.md index aafb218..a770cb3 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,10 @@ -> ⚠️ **Under repair (2026-07-18)** — a portfolio audit found this repo has never compiled as committed (several modules were never pushed). A verified repair is in progress on a dedicated branch; until it lands, treat the code as design reference, not runnable. No claim without a check. +> **Status (2026-07-18): compiles and is CI-gated; the analysis pipeline is NOT yet live-wired.** +> This repo previously never compiled as committed and its demo processor posted canned findings +> without disclosure — both are documented as **finding #1 of the +> [portfolio benchmark](https://github.com/sgharlow/ai-control-framework/tree/main/benchmarks)**, +> the case study for why the verification gates exist. Fixed: history repaired (PR #1), demo +> output now fail-closed behind `DEMO_MODE=true` and prominently labeled, first tests added. +> Claim level: **built** — not live-proven against real Redis/GitHub. [![Workshown](https://img.shields.io/badge/Workshown-member-0b7285)](https://sgharlow.github.io/ai-control-framework/) diff --git a/package.json b/package.json index 39b87b6..c8f3000 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,8 @@ "dev": "tsx watch src/index.ts", "db:migrate": "prisma migrate deploy", "db:generate": "prisma generate", - "db:migrate:dev": "prisma migrate dev" + "db:migrate:dev": "prisma migrate dev", + "test": "npm run build && node --test tests/followups.test.mjs" }, "dependencies": { "@fastify/cors": "^8.4.0", diff --git a/src/lib/ai-service/cost-calculator.ts b/src/lib/ai-service/cost-calculator.ts index 99f7356..5d6244c 100644 --- a/src/lib/ai-service/cost-calculator.ts +++ b/src/lib/ai-service/cost-calculator.ts @@ -5,8 +5,11 @@ * * Pricing constants are OpenAI's published per-1K-token prices (openai.com/api/pricing, * as of the models this repo references in config: 'gpt-4' and 'gpt-4-turbo-preview'). - * Unknown models FALL BACK to gpt-4 pricing (the most expensive listed) — fail-closed: - * the cost-limit guard in EnhancedAIService must overestimate, never underestimate. + * Unknown models FALL BACK to the most expensive listed pricing (computed from the + * table, so it stays true as prices are edited) — fail-closed: the cost-limit guard + * in EnhancedAIService must overestimate, never underestimate. (F2 fix 2026-07-18: + * this previously fell back to gpt-4 while claiming "most expensive" — gpt-4-32k + * costs 2x more; the guarantee is now computed, not asserted.) */ import { TokenUsage, CostCalculation } from './types'; @@ -27,8 +30,11 @@ const MODEL_PRICING: Record = { 'gpt-3.5-turbo': { inputCostPer1k: 0.0005, outputCostPer1k: 0.0015 } }; -/** Fail-closed default for unknown models (see header). */ -const FALLBACK_PRICING: ModelPricing = MODEL_PRICING['gpt-4']; +/** Fail-closed default for unknown models: the most expensive listed pricing, + * computed from the table so the guarantee cannot silently rot (see header). */ +const FALLBACK_PRICING: ModelPricing = Object.values(MODEL_PRICING).reduce((max, p) => + p.inputCostPer1k + p.outputCostPer1k > max.inputCostPer1k + max.outputCostPer1k ? p : max +); export class CostCalculator { /** diff --git a/src/lib/queue-system/processors/pr-processor.ts b/src/lib/queue-system/processors/pr-processor.ts index 17d89b2..929470c 100644 --- a/src/lib/queue-system/processors/pr-processor.ts +++ b/src/lib/queue-system/processors/pr-processor.ts @@ -18,7 +18,26 @@ export class EnhancedDemoProcessorSimple { // contract — without it no GitHub App call can be authenticated, so the job // fails explicitly instead of passing undefined onward. const { repository, pullRequest, installationId } = job.data; - console.log(`[EnhancedDemoProcessor] Starting AI-powered analysis for PR #${pullRequest.number}`); + + // F1 (review of PR #1, 2026-07-18): this processor posts CANNED SAMPLE findings — + // the real analysis pipeline is not yet live-wired. Structurally fail-closed: + // without an explicit DEMO_MODE=true, it posts NOTHING and the job fails with the + // reason, so a default deployment can never present fabricated findings as analysis. + // When enabled, every posted comment is prominently labeled as demo output. + if (process.env.DEMO_MODE !== 'true') { + return { + success: false, + error: { + code: 'DEMO_MODE_DISABLED', + message: + `PR analysis for ${repository.fullName}#${pullRequest.number} refused: the real ` + + `analysis pipeline is not live-wired and this processor only produces labeled ` + + `sample output. Set DEMO_MODE=true to post clearly-labeled demo reviews.` + } + }; + } + + console.log(`[EnhancedDemoProcessor] Starting DEMO analysis for PR #${pullRequest.number} (DEMO_MODE=true)`); if (installationId === undefined) { return { @@ -208,6 +227,8 @@ export class EnhancedDemoProcessorSimple { } private buildInlineComment(finding: any): string { + // Every inline demo comment self-identifies (F1): no fabricated finding may + // present as a real analysis of the code it is attached to. const iconMap = { critical: '🚨', high: '⚠️', @@ -217,7 +238,9 @@ export class EnhancedDemoProcessorSimple { const icon = iconMap[finding.severity as keyof typeof iconMap] || '📝'; - return `${icon} **${finding.severity.toUpperCase()}: ${finding.message}** + return `> ⚠️ **Demo sample — not a real finding about this code** (\`DEMO_MODE\`) + +${icon} **${finding.severity.toUpperCase()}: ${finding.message}** ${finding.details} @@ -248,7 +271,11 @@ ${finding.cwe ? `**Reference:** ${finding.cwe}${finding.owasp ? ` | ${finding.ow const overallScore = this.calculateCodeScore(findings); const scoreEmoji = overallScore >= 80 ? '🟢' : overallScore >= 60 ? '🟡' : '🔴'; - return `## 🤖 AI Code Review Report + return `> ⚠️ **DEMO OUTPUT — these are canned sample findings, not an analysis of this +> pull request's code.** This bot is running in demo mode (\`DEMO_MODE=true\`) to showcase +> the report format; the real analysis pipeline is not yet live-wired. + +## 🤖 AI Code Review Report (demo sample) ### 📊 Analysis Summary - **Files analyzed**: ${filesCount} diff --git a/tests/followups.test.mjs b/tests/followups.test.mjs new file mode 100644 index 0000000..94094f7 --- /dev/null +++ b/tests/followups.test.mjs @@ -0,0 +1,62 @@ +// First tests in this repo (2026-07-18) — node:test over the compiled dist/, zero new +// dependencies. Run via `npm test` (builds first). Covers the two review follow-ups: +// F1 (demo mode is fail-closed) and F2 (unknown-model pricing can never under-count). + +import { test } from 'node:test'; +import assert from 'node:assert/strict'; + +const { CostCalculator } = await import('../dist/lib/ai-service/cost-calculator.js'); +const { EnhancedDemoProcessorSimple } = await import('../dist/lib/queue-system/processors/pr-processor.js'); + +test('F2: unknown model falls back to pricing >= every listed model (fail-closed property)', () => { + const unknown = CostCalculator.calculateCost({ + model: 'totally-unknown-model-x', + promptTokens: 1000, + completionTokens: 1000, + totalTokens: 2000 + }); + const listed = ['gpt-4', 'gpt-4-32k', 'gpt-4-turbo', 'gpt-4o', 'gpt-4o-mini', 'gpt-3.5-turbo']; + for (const model of listed) { + const known = CostCalculator.calculateCost({ + model, + promptTokens: 1000, + completionTokens: 1000, + totalTokens: 2000 + }); + assert.ok( + unknown.totalCost >= known.totalCost, + `fallback ($${unknown.totalCost}) must be >= ${model} ($${known.totalCost})` + ); + } +}); + +test('F2: dated variants resolve by longest prefix, not the shortest', () => { + const dated = CostCalculator.calculateCost({ + model: 'gpt-4-turbo-2024-04-09', + promptTokens: 1000, + completionTokens: 0, + totalTokens: 1000 + }); + assert.equal(dated.inputCostPer1k, 0.01, 'gpt-4-turbo-* must price as gpt-4-turbo, not gpt-4'); +}); + +test('F1: without DEMO_MODE=true the PR processor refuses before any GitHub call', async () => { + delete process.env.DEMO_MODE; + const explodingClient = new Proxy({}, { + get() { + throw new Error('GitHub client must not be touched when demo mode is disabled'); + } + }); + const processor = new EnhancedDemoProcessorSimple(explodingClient).createProcessor(); + const result = await processor({ + data: { + repository: { owner: 'o', name: 'r', fullName: 'o/r' }, + pullRequest: { number: 1, title: 't', author: 'a', baseBranch: 'main', headBranch: 'x', sha: 's' }, + installationId: 123, + action: 'opened', + timestamp: new Date() + } + }); + assert.equal(result.success, false); + assert.equal(result.error.code, 'DEMO_MODE_DISABLED'); +});