diff --git a/.aiwg/bt6-maintainer.yaml b/.aiwg/bt6-maintainer.yaml index 6091468c..1dcafdc1 100644 --- a/.aiwg/bt6-maintainer.yaml +++ b/.aiwg/bt6-maintainer.yaml @@ -19,9 +19,15 @@ delivery: allowedMergeMethods: ["squash"] validation: quick: - - "npm run typecheck" + - "npm run test:pr" + - "COVERAGE_BASE=upstream/main PR_CHANGED_LINE_COVERAGE=50 npm run test:pr-coverage" full: - - "npm test" + - "npm run test:release" + - "npm audit --audit-level=high" + qualityPolicy: + pullRequestChangedLineCoverageFloor: 50 + requireBehaviorTests: true + fullSuiteTrigger: "release-tag-certification" documentation: - "npm run docs:check" researchIntegrity: @@ -107,6 +113,13 @@ research: - "notes/**" provenanceRequired: true citationVerificationRequired: true +releaseEvidence: + artifactType: "source-zip" + hashAlgorithm: "sha256" + provenanceFormat: "slsa-v1" + attestationFormat: "sigstore-bundle" + snapshotOnce: true + verifyBeforePromotion: true security: disclosureUrl: "https://forms.gle/QvKoijJMtEhLG7nf8" sensitiveDataPaths: diff --git a/.aiwg/deployment/ci-cd-scaffold.md b/.aiwg/deployment/ci-cd-scaffold.md index b538cffa..4e767c28 100644 --- a/.aiwg/deployment/ci-cd-scaffold.md +++ b/.aiwg/deployment/ci-cd-scaffold.md @@ -66,14 +66,20 @@ N/A for a transactional application database. Local configuration migrations are ## 10. Verification and Validation -The current pipeline runs dependency installation, lint, typecheck, tests, coverage, doctor, claim verification, anti-fitting, provenance, prompt audit, and smoke checks. Release verification additionally confirms the actual packaged artifact starts, the local health route responds, interface documentation matches behavior, and sensitive/ignored corpora are absent. +The pipeline intentionally has two gates. Pull requests run dependency +installation, lint, typecheck, the deterministic test suite, doctor, and a 50% +changed-executable-line coverage floor. Release tags rerun the complete +per-file coverage contract plus claim verification, anti-fitting, provenance, +prompt, smoke, build, dependency-audit, and package-manifest checks. The tested +tag is archived once as a ZIP, checksum-verified, and bound to a retained +Sigstore provenance bundle. Proposed additions: 1. SDLC artifact/citation/traceability conformance. 2. Cross-document maturity consistency. 3. Network-adapter scope/approval/credential/timeout inventory completeness. -4. Release artifact checksums/signatures and SBOM according to an approved supply-chain threat model. +4. CycloneDX SBOM generation bound to the source ZIP. ## 11. Rollback and Contingency diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 52e66f62..efb3d8a0 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -33,4 +33,5 @@ Exact commands and results: - [ ] Diff is scoped against current `upstream/main` - [ ] No unrelated protected evidence, safety tests, or provider/config files were removed - [ ] Published review history was not force-pushed -- [ ] Exact-head CI must be green before merge +- [ ] Exact-head PR CI is green (including the 50% changed-line coverage floor) +- [ ] I understand PR acceptance is not release certification diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fe0869b2..eb1f1332 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,40 +18,47 @@ concurrency: jobs: test: + name: Pull request gate + if: github.event_name == 'pull_request' || (github.event_name == 'push' && github.ref == 'refs/heads/main') runs-on: ubuntu-latest timeout-minutes: 20 steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + fetch-depth: 0 - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version: 22 cache: npm - run: npm ci - - run: npm run lint - - run: npm run typecheck - - run: npm test - # Enforce the 100% coverage gate on the new multi-language ingest files - # (vitest exits non-zero if a per-file threshold is missed). - - run: npm run test:coverage - - run: npm run doctor - # Honesty gates: prove the headline numbers re-derive and that no - # challenge-fitting has crept in. Public contributors / CI can verify. - - run: npm run verify-claims - - run: npm run test:no-fitting - - run: npm run test:no-self-fitting - # The provenance gate itself, in the compiled engine path (builds first). - - run: npm run test:gate - # The operator doctrine / prompt-pack contract must hold. - - run: npm run prompt:audit - # Non-network smoke: core traps run deterministically; server/live tiers - # auto-skip when no server/LLM is present (they are absent in CI). - - run: npm run smoke + - name: Run PR correctness gate + run: npm run test:pr + - name: Resolve comparison base + env: + EVENT_BASE: ${{ github.event.pull_request.base.sha || github.event.before }} + run: | + coverage_base="$EVENT_BASE" + if ! git rev-parse --verify "${coverage_base}^{commit}" >/dev/null 2>&1; then + coverage_base="$(git rev-parse HEAD^)" + fi + echo "COVERAGE_BASE=$coverage_base" >> "$GITHUB_ENV" + - name: Enforce 50% changed-line coverage floor + env: + PR_CHANGED_LINE_COVERAGE: "50" + run: npm run test:pr-coverage + - name: Verify changed-line coverage policy tool + run: npm run test:changed-coverage-script release: + name: Release certification and ZIP archive if: startsWith(github.ref, 'refs/tags/v') - needs: test runs-on: ubuntu-latest timeout-minutes: 30 + permissions: + artifact-metadata: write + attestations: write + contents: read + id-token: write steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: @@ -61,20 +68,40 @@ jobs: node-version: 22 cache: npm - run: npm ci - - run: npm run test:release - - run: npm audit --audit-level=high - - run: npm run pack:dry-run - - run: npm pack --json > pack-result.json - - name: Record release checksums + - name: Run release-grade test and evidence gates + run: npm run test:release + - name: Reject high-severity dependency vulnerabilities + run: npm audit --audit-level=high + - name: Build deterministic source ZIP from tested tag + run: | + mkdir -p release release-evidence + archive="release/T3MP3ST-${GITHUB_SHA}.zip" + git archive --format=zip --prefix="T3MP3ST-${GITHUB_SHA}/" --output="$archive" "$GITHUB_SHA" + unzip -t "$archive" | tee release-evidence/source-zip-check.txt + sha256sum "$archive" | tee release-evidence/SHA256SUMS + - name: Verify a clean user build from the ZIP + run: | + install_root="$RUNNER_TEMP/t3mp3st-release-smoke" + mkdir -p "$install_root" + unzip -q "release/T3MP3ST-${GITHUB_SHA}.zip" -d "$install_root" + cd "$install_root/T3MP3ST-${GITHUB_SHA}" + npm ci + npm run build + node dist/cli.js --help > /dev/null + - name: Attest tested ZIP provenance + id: provenance + uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2 + with: + subject-checksums: release-evidence/SHA256SUMS + - name: Retain provenance bundle and verify archive checksum run: | - package_file="$(node -e "const fs=require('fs'); const p=JSON.parse(fs.readFileSync('pack-result.json')); process.stdout.write(p[0].filename)")" - sha256sum "$package_file" > SHA256SUMS + cp "${{ steps.provenance.outputs.bundle-path }}" release-evidence/slsa-provenance.sigstore.json + sha256sum -c release-evidence/SHA256SUMS - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: tested-package-${{ github.sha }} + name: tested-source-zip-${{ github.sha }} path: | - *.tgz - SHA256SUMS - pack-result.json + release/ + release-evidence/ if-no-files-found: error retention-days: 14 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d2e1ef2e..732d7cf8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -53,12 +53,10 @@ least 50% coverage; documentation-only and metadata-only changes may mark this not applicable with a reason. Trust-boundary, scope, evidence, provider, installation, and release changes also need the focused checks for that risk. -Before opening a PR: +Before opening a PR, run the same correctness gate used by CI: ```bash -npm run typecheck -npm test -npm run doctor +npm run test:pr ``` For changes that affect claims, run modes, agent/tool execution, target scope, @@ -84,9 +82,11 @@ unrelated provider/config churn, benchmark fixture removals, provenance doc removals, or safety-test removals. If the branch has drifted, recreate it from current `main` and reapply only the intended change. -Maintainers run `npm run test:release`, `npm audit --audit-level=high`, and the -package dry run against the exact release commit before publishing. A green PR -gate is necessary for review and merge, but it is not release certification. +Maintainers run `npm run test:release` and `npm audit --audit-level=high` against +the exact release tag before publishing. The tag workflow creates one +deterministic source ZIP, verifies it, records SHA-256 checksums, and retains a +Sigstore provenance bundle. A green PR gate is necessary for review and merge, +but it is not release certification. Do not include secrets, private tracker content, unlicensed corpora, or uncoordinated vulnerability details. Use the disclosure channel in diff --git a/docs/PULL_REQUEST_DELIVERY.md b/docs/PULL_REQUEST_DELIVERY.md index 334cfadc..573dd9c4 100644 --- a/docs/PULL_REQUEST_DELIVERY.md +++ b/docs/PULL_REQUEST_DELIVERY.md @@ -39,14 +39,17 @@ both human contributors and coding agents. 4. Run the smallest meaningful verification set. - For most code changes: + For most code changes, run the same correctness gate used by CI: ```bash - npm run typecheck - npm test - npm run doctor + npm run test:pr ``` + CI additionally measures changed executable lines against the PR base and + requires at least 50% coverage. This permits incremental review without + claiming release certification. Release tags rerun the complete coverage, + evidence, anti-fitting, smoke, build, audit, and package gates. + For docs-only changes, note why code tests were skipped. For UI-only `docs/index.html` changes, extract or otherwise parse-check the page scripts. For benchmark or headline-claim changes, also run: diff --git a/docs/RELEASE_CHECKLIST.md b/docs/RELEASE_CHECKLIST.md index fd82ad43..d0c1532f 100644 --- a/docs/RELEASE_CHECKLIST.md +++ b/docs/RELEASE_CHECKLIST.md @@ -67,12 +67,14 @@ degrade gracefully — they never fail the core run. ``` - Tag the release only after Sections 1–2 are green. - Push the `v*` tag and wait for the tag workflow. It reruns - `npm run test:release`, the high-severity dependency audit, and the package - dry run against the exact tag, then retains the tested `.tgz`, its manifest, - and `SHA256SUMS` as workflow artifacts. -- Inspect `pack-result.json`: only the allowlisted runtime, scripts, tools, - documentation, examples, and package metadata may ship. Workspace notes, - tests, AIWG/provider deployment internals, secrets, and local artifacts are - release blockers. -- Publish the retained, checksum-matched package artifact. Do not rebuild a - different archive from another checkout after certification. + `npm run test:release`, the high-severity dependency audit, and package dry + run against the exact tag. It then creates one deterministic + `T3MP3ST-.zip` directly from that tested Git object. +- The workflow extracts that ZIP into a clean temporary directory, performs a + locked install and build, and invokes the packaged CLI help path before the + archive can be retained. +- Verify `release-evidence/source-zip-check.txt`, `SHA256SUMS`, and the retained + Sigstore provenance bundle. Workspace notes, secrets, ignored files, and + local artifacts cannot enter a `git archive` snapshot. +- Publish the retained, checksum-matched ZIP. Do not rebuild a different + archive from another checkout after certification. diff --git a/docsite/t3mp3st-docs/content/PULL_REQUEST_DELIVERY.md b/docsite/t3mp3st-docs/content/PULL_REQUEST_DELIVERY.md index 0f3a6d18..11529d65 100644 --- a/docsite/t3mp3st-docs/content/PULL_REQUEST_DELIVERY.md +++ b/docsite/t3mp3st-docs/content/PULL_REQUEST_DELIVERY.md @@ -50,14 +50,17 @@ both human contributors and coding agents. 4. Run the smallest meaningful verification set. - For most code changes: + For most code changes, run the same correctness gate used by CI: ```bash - npm run typecheck - npm test - npm run doctor + npm run test:pr ``` + CI additionally measures changed executable lines against the PR base and + requires at least 50% coverage. This permits incremental review without + claiming release certification. Release tags rerun the complete coverage, + evidence, anti-fitting, smoke, build, audit, and package gates. + For docs-only changes, note why code tests were skipped. For UI-only `docs/index.html` changes, extract or otherwise parse-check the page scripts. For benchmark or headline-claim changes, also run: diff --git a/docsite/t3mp3st-docs/content/RELEASE_CHECKLIST.md b/docsite/t3mp3st-docs/content/RELEASE_CHECKLIST.md index 1f942c10..6be4c530 100644 --- a/docsite/t3mp3st-docs/content/RELEASE_CHECKLIST.md +++ b/docsite/t3mp3st-docs/content/RELEASE_CHECKLIST.md @@ -78,12 +78,14 @@ degrade gracefully — they never fail the core run. ``` - Tag the release only after Sections 1–2 are green. - Push the `v*` tag and wait for the tag workflow. It reruns - `npm run test:release`, the high-severity dependency audit, and the package - dry run against the exact tag, then retains the tested `.tgz`, its manifest, - and `SHA256SUMS` as workflow artifacts. -- Inspect `pack-result.json`: only the allowlisted runtime, scripts, tools, - documentation, examples, and package metadata may ship. Workspace notes, - tests, AIWG/provider deployment internals, secrets, and local artifacts are - release blockers. -- Publish the retained, checksum-matched package artifact. Do not rebuild a - different archive from another checkout after certification. + `npm run test:release`, the high-severity dependency audit, and package dry + run against the exact tag. It then creates one deterministic + `T3MP3ST-.zip` directly from that tested Git object. +- The workflow extracts that ZIP into a clean temporary directory, performs a + locked install and build, and invokes the packaged CLI help path before the + archive can be retained. +- Verify `release-evidence/source-zip-check.txt`, `SHA256SUMS`, and the retained + Sigstore provenance bundle. Workspace notes, secrets, ignored files, and + local artifacts cannot enter a `git archive` snapshot. +- Publish the retained, checksum-matched ZIP. Do not rebuild a different + archive from another checkout after certification. diff --git a/package.json b/package.json index 054dd722..8ea0570b 100755 --- a/package.json +++ b/package.json @@ -75,7 +75,10 @@ "mcp": "tsx src/mcp-server.ts", "mcp:prod": "node dist/mcp-server.js", "test": "vitest run src && node scripts/test-ops-preflight.mjs && node scripts/test-model-matrix.mjs && node scripts/refusal-frontier.mjs --self-test", - "test:release": "npm run lint && npm run typecheck && npm test && npm run test:coverage && npm run doctor && npm run verify-claims && npm run test:no-fitting && npm run test:no-self-fitting && npm run test:gate && npm run prompt:audit && npm run smoke && npm run build", + "test:pr": "npm run lint && npm run typecheck && npm test && npm run doctor", + "test:pr-coverage": "vitest run --coverage --config vitest.pr.config.ts && node scripts/check-changed-line-coverage.mjs", + "test:changed-coverage-script": "node scripts/check-changed-line-coverage.mjs --self-test", + "test:release": "npm run lint && npm run typecheck && npm test && npm run test:coverage && npm run doctor && npm run verify-claims && npm run test:no-fitting && npm run test:no-self-fitting && npm run test:no-phantom-tools && npm run test:gate && npm run prompt:audit && npm run smoke && npm run build && npm run pack:dry-run", "pack:dry-run": "npm pack --dry-run", "test:coverage": "vitest run --coverage", "test:watch": "vitest src", diff --git a/scripts/check-changed-line-coverage.mjs b/scripts/check-changed-line-coverage.mjs new file mode 100644 index 00000000..85299f05 --- /dev/null +++ b/scripts/check-changed-line-coverage.mjs @@ -0,0 +1,64 @@ +#!/usr/bin/env node +import { readFileSync } from 'node:fs'; +import { execFileSync } from 'node:child_process'; + +export function changedLines(diff) { + const files = new Map(); + let current; + for (const line of String(diff).split('\n')) { + if (line.startsWith('+++ b/')) { + current = line.slice(6); + if (!files.has(current)) files.set(current, new Set()); + continue; + } + const match = line.match(/^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@/); + if (!match || !current) continue; + const start = Number(match[1]); + const count = match[2] === undefined ? 1 : Number(match[2]); + for (let n = start; n < start + count; n += 1) files.get(current).add(n); + } + return files; +} + +export function assessChangedCoverage(coverage, changes) { + let coverable = 0; + let covered = 0; + const uncovered = []; + for (const [file, lines] of changes) { + if (!file.startsWith('src/') || file.endsWith('.test.ts') || file.includes('/__tests__/')) continue; + const entry = Object.values(coverage).find((item) => String(item.path || '').replaceAll('\\', '/').endsWith(`/${file}`)); + if (!entry) continue; + for (const line of lines) { + const statementIds = Object.entries(entry.statementMap || {}) + .filter(([, span]) => line >= span.start.line && line <= span.end.line) + .map(([id]) => id); + if (!statementIds.length) continue; + coverable += 1; + if (statementIds.some((id) => Number(entry.s?.[id] || 0) > 0)) covered += 1; + else uncovered.push(`${file}:${line}`); + } + } + return { coverable, covered, percent: coverable ? (covered / coverable) * 100 : 100, uncovered }; +} + +function selfTest() { + const changes = changedLines('+++ b/src/a.ts\n@@ -1 +1,3 @@\n+x\n+y\n+z'); + const coverage = { '/repo/src/a.ts': { path: '/repo/src/a.ts', statementMap: { 0: { start: { line: 1 }, end: { line: 1 } }, 1: { start: { line: 2 }, end: { line: 3 } } }, s: { 0: 1, 1: 0 } } }; + const result = assessChangedCoverage(coverage, changes); + if (result.coverable !== 3 || result.covered !== 1 || Math.round(result.percent) !== 33) throw new Error('changed-line coverage self-test failed'); + console.log('changed-line coverage self-test passed'); +} + +if (process.argv.includes('--self-test')) { + selfTest(); +} else { + const base = process.argv[2] || process.env.COVERAGE_BASE; + const minimum = Number(process.argv[3] || process.env.PR_CHANGED_LINE_COVERAGE || 50); + if (!base) throw new Error('base revision required as argv[2] or COVERAGE_BASE'); + const diff = execFileSync('git', ['diff', '--unified=0', '--no-ext-diff', `${base}...HEAD`, '--', 'src'], { encoding: 'utf8' }); + const coverage = JSON.parse(readFileSync('coverage-pr/coverage-final.json', 'utf8')); + const result = assessChangedCoverage(coverage, changedLines(diff)); + console.log(`Changed executable lines: ${result.covered}/${result.coverable} (${result.percent.toFixed(1)}%; required ${minimum}%)`); + if (result.uncovered.length) console.log(`Uncovered changed lines:\n${result.uncovered.slice(0, 100).join('\n')}`); + if (result.percent < minimum) process.exitCode = 1; +} diff --git a/vitest.pr.config.ts b/vitest.pr.config.ts new file mode 100644 index 00000000..d5c9ab05 --- /dev/null +++ b/vitest.pr.config.ts @@ -0,0 +1,16 @@ +import { defineConfig } from 'vitest/config'; + +// Pull requests use changed-line coverage as a review signal. Release tags use +// vitest.config.ts, whose narrow 100% per-file contract remains unchanged. +export default defineConfig({ + test: { + include: ['src/**/*.test.ts'], + coverage: { + provider: 'v8', + include: ['src/**/*.ts'], + exclude: ['src/**/*.test.ts', 'src/**/__tests__/**', 'src/types/**'], + reporter: ['json'], + reportsDirectory: 'coverage-pr', + }, + }, +});