diff --git a/tests/evals/eval-config.json b/tests/evals/eval-config.json index 6761f34..cbcc6a5 100644 --- a/tests/evals/eval-config.json +++ b/tests/evals/eval-config.json @@ -56,7 +56,7 @@ "agentic-workflow": { "pass_max": 0 } }, "recommendations": { - "must_mention": ["secret scanning", ".env.example", "TDD", "agentic workflow"], + "must_mention": ["secret scanning", ".env.example", "TDD", "agentic workflow", "auto-gen"], "must_not_mention": ["run /setup"] } } @@ -82,7 +82,7 @@ "agentic-workflow": { "pass_min": 1 } }, "recommendations": { - "must_not_mention": ["run /setup", "install linter", "install test runner"] + "must_not_mention": ["run /setup", "install linter", "install test runner", "add auto-generated sections"] } } } diff --git a/tests/evals/fixtures/level-5-autonomous/CLAUDE.md b/tests/evals/fixtures/level-5-autonomous/CLAUDE.md index ee5d728..892dc2c 100644 --- a/tests/evals/fixtures/level-5-autonomous/CLAUDE.md +++ b/tests/evals/fixtures/level-5-autonomous/CLAUDE.md @@ -14,19 +14,29 @@ ## Architecture - -``` + +scripts/ +├── generate-docs-helpers.js # Helper functions for generate-docs.js. +└── generate-docs.js # Auto-generate CLAUDE.md sections from source code. src/ - index.ts — Entry point, starts Express server - app.ts — Express app factory (createApp) - app.test.ts — App integration tests - routes/ - health.ts — GET /health endpoint - health.test.ts — Health endpoint tests - users.ts — CRUD /users endpoints - users.test.ts — Users endpoint tests -``` - +├── routes/ +│ ├── health.test.ts +│ ├── health.ts +│ ├── users.test.ts +│ └── users.ts +├── app.test.ts +├── app.ts +└── index.ts + + +## Key Modules + + +| Module | Purpose | Key Exports | +|--------|---------|-------------| +| `scripts/generate-docs-helpers.js` | Helper functions for generate-docs.js. | `name()` | +| `scripts/generate-docs.js` | Auto-generate CLAUDE.md sections from source code. | `replaceMarkers()`, `validateCrossLinks()`, `buildDocsIndex()`, `checkMarkersAreCurrent()` | + ## Critical Gotchas diff --git a/tests/evals/fixtures/level-5-autonomous/docs/index.md b/tests/evals/fixtures/level-5-autonomous/docs/index.md new file mode 100644 index 0000000..9d3234d --- /dev/null +++ b/tests/evals/fixtures/level-5-autonomous/docs/index.md @@ -0,0 +1,3 @@ +# Documentation Index + +No documentation files found. diff --git a/tests/evals/fixtures/level-5-autonomous/scripts/generate-docs-helpers.js b/tests/evals/fixtures/level-5-autonomous/scripts/generate-docs-helpers.js new file mode 100644 index 0000000..1507f4d --- /dev/null +++ b/tests/evals/fixtures/level-5-autonomous/scripts/generate-docs-helpers.js @@ -0,0 +1,290 @@ +/** + * Helper functions for generate-docs.js. + * + * Extracts JSDoc descriptions, export names, builds directory trees, + * and collects module index data from source files. + */ + +const fs = require('node:fs'); +const path = require('node:path'); + +const SKIP_DIRS = new Set(['node_modules', '.git', 'coverage', 'dist', 'build', 'fixtures', 'results']); + +// --------------------------------------------------------------------------- +// JSDoc & Export Extraction +// --------------------------------------------------------------------------- + +/** + * Extract the first description line from a file's JSDoc comment. + * @param {string} filePath - Absolute path to a .js file + * @returns {string} Description text, or empty string + */ +function extractJSDocDescription(filePath) { + let content; + try { + content = fs.readFileSync(filePath, 'utf-8'); + } catch { + return ''; + } + + // Multi-line first (file-level JSDoc is usually multi-line): /** \n * desc \n */ + const multiLine = content.match(/\/\*\*\s*\n([\s\S]*?)\*\//); + if (multiLine) { + const lines = multiLine[1].split('\n'); + for (const line of lines) { + const cleaned = line.replace(/^\s*\*\s?/, '').trim(); + if (cleaned && !cleaned.startsWith('@')) { + return cleaned; + } + } + } + + // Single-line fallback: /** desc */ + const singleLine = content.match(/\/\*\*\s+(.+?)\s*\*\//); + if (singleLine) { + const text = singleLine[1].replace(/\s*\*\/$/, '').trim(); + if (!text.startsWith('@')) return text; + } + return ''; +} + +/** + * Extract exported names from a CommonJS module (capped at 5). + * Reads `module.exports = { ... }` and `exports.name =` patterns. + * @param {string} filePath - Absolute path to a .js file + * @returns {string[]} Array of export names (max 5) + */ +function extractExports(filePath) { + let content; + try { + content = fs.readFileSync(filePath, 'utf-8'); + } catch { + return []; + } + + const names = new Set(); + + // module.exports = { name1, name2, ... } + const objMatch = content.match(/module\.exports\s*=\s*\{([^}]*)\}/s); + if (objMatch) { + const body = objMatch[1]; + const keyRe = /\b([a-zA-Z_$][\w$]*)\b(?:\s*[,:}\n]|\s*$)/g; + let m; + while ((m = keyRe.exec(body)) !== null) { + names.add(m[1]); + } + } + + // exports.name = ... + const namedRe = /exports\.([a-zA-Z_$][\w$]*)\s*=/g; + let m; + while ((m = namedRe.exec(content)) !== null) { + names.add(m[1]); + } + + return [...names].slice(0, 5); +} + +// --------------------------------------------------------------------------- +// Directory Tree Builder +// --------------------------------------------------------------------------- + +/** + * Build an ASCII directory tree with JSDoc annotations. + * @param {string} rootDir - Project root directory + * @param {string[]} dirs - Top-level directories to include (e.g. ['src/', 'bin/']) + * @returns {string} ASCII tree string + */ +function buildDirectoryTree(rootDir, dirs) { + const lines = []; + for (const dir of dirs) { + const fullPath = path.join(rootDir, dir); + if (!fs.existsSync(fullPath)) { + continue; + } + lines.push(dir); + buildTreeRecursive(fullPath, '', lines); + } + return lines.join('\n'); +} + +/** + * Recursively build tree lines for a directory. + * @param {string} dirPath - Directory to scan + * @param {string} prefix - Line prefix for indentation + * @param {string[]} lines - Accumulator for output lines + */ +function buildTreeRecursive(dirPath, prefix, lines) { + let entries; + try { + entries = fs.readdirSync(dirPath, { withFileTypes: true }); + } catch { + return; + } + + entries = entries.filter(e => { + if (e.name.startsWith('.')) { + return false; + } + if (e.isDirectory() && SKIP_DIRS.has(e.name)) { + return false; + } + return true; + }); + + const sortedDirs = entries.filter(e => e.isDirectory()).sort((a, b) => a.name.localeCompare(b.name)); + const sortedFiles = entries.filter(e => e.isFile()).sort((a, b) => a.name.localeCompare(b.name)); + const sorted = [...sortedDirs, ...sortedFiles]; + + for (let i = 0; i < sorted.length; i++) { + const entry = sorted[i]; + const isLast = i === sorted.length - 1; + const connector = isLast ? '\u2514\u2500\u2500 ' : '\u251c\u2500\u2500 '; + const childPrefix = isLast ? ' ' : '\u2502 '; + + if (entry.isDirectory()) { + lines.push(`${prefix}${connector}${entry.name}/`); + buildTreeRecursive(path.join(dirPath, entry.name), prefix + childPrefix, lines); + } else { + let annotation = ''; + if (entry.name.endsWith('.js')) { + const desc = extractJSDocDescription(path.join(dirPath, entry.name)); + if (desc) { + annotation = ` # ${desc}`; + } + } + lines.push(`${prefix}${connector}${entry.name}${annotation}`); + } + } +} + +// --------------------------------------------------------------------------- +// Source Directory Detection +// --------------------------------------------------------------------------- + +const KNOWN_SOURCE_DIRS = new Set([ + 'src', 'lib', 'app', 'apps', 'packages', 'services', 'modules', + 'cmd', 'internal', 'pkg', 'bin', 'components', + 'scripts', 'tests', 'test', 'spec', +]); + +const SOURCE_EXTENSIONS = new Set([ + '.js', '.ts', '.tsx', '.jsx', '.mjs', '.cjs', + '.py', '.go', '.rs', '.c', '.cpp', '.h', '.hpp', + '.java', '.kt', '.rb', '.php', '.swift', '.cs', +]); + +/** + * Detect source directories in a project by name and content. + * @param {string} rootDir - Project root directory + * @returns {string[]} Array of directory names with trailing slash + */ +function detectSourceDirs(rootDir) { + let entries; + try { + entries = fs.readdirSync(rootDir, { withFileTypes: true }); + } catch { + return []; + } + + const dirs = []; + for (const entry of entries) { + if (!entry.isDirectory() || entry.name.startsWith('.') || SKIP_DIRS.has(entry.name)) { + continue; + } + if (KNOWN_SOURCE_DIRS.has(entry.name)) { + if (dirHasFiles(path.join(rootDir, entry.name), false)) { + dirs.push(`${entry.name}/`); + } + continue; + } + if (dirHasFiles(path.join(rootDir, entry.name), true)) { + dirs.push(`${entry.name}/`); + } + } + return dirs.sort(); +} + +function dirHasFiles(dirPath, sourceOnly) { + try { + for (const e of fs.readdirSync(dirPath, { withFileTypes: true })) { + if (e.isFile()) { + if (!sourceOnly) return true; + if (SOURCE_EXTENSIONS.has(path.extname(e.name).toLowerCase())) return true; + } + if (e.isDirectory() && !SKIP_DIRS.has(e.name) && dirHasFiles(path.join(dirPath, e.name), sourceOnly)) { + return true; + } + } + } catch { /* empty */ } + return false; +} + +// --------------------------------------------------------------------------- +// Module Index Builder +// --------------------------------------------------------------------------- + +/** + * Build a markdown table of modules from all detected source directories. + * @param {string} rootDir - Project root directory + * @returns {string} Markdown table + */ +function buildModuleIndex(rootDir) { + const sourceDirs = detectSourceDirs(rootDir); + const rows = []; + for (const dir of sourceDirs) { + const dirName = dir.replace(/\/$/, ''); + const fullPath = path.join(rootDir, dirName); + collectModules(fullPath, rows, dirName); + } + + const header = '| Module | Purpose | Key Exports |'; + const sep = '|--------|---------|-------------|'; + const dataRows = rows.map(r => `| \`${r.module}\` | ${r.purpose} | ${r.exports} |`); + return [header, sep, ...dataRows].join('\n'); +} + +/** + * Recursively collect module info from a directory. + * @param {string} dirPath - Directory to scan + * @param {Array<{module: string, purpose: string, exports: string}>} rows + * @param {string} [relPrefix=''] - Relative path prefix + */ +function collectModules(dirPath, rows, relPrefix = '') { + let entries; + try { + entries = fs.readdirSync(dirPath, { withFileTypes: true }); + } catch { + return; + } + + const sortedDirs = entries.filter(e => e.isDirectory() && !SKIP_DIRS.has(e.name) && !e.name.startsWith('.')) + .sort((a, b) => a.name.localeCompare(b.name)); + const sortedFiles = entries.filter(e => e.isFile() && e.name.endsWith('.js')) + .sort((a, b) => a.name.localeCompare(b.name)); + + for (const file of sortedFiles) { + const fullPath = path.join(dirPath, file.name); + const modulePath = relPrefix ? `${relPrefix}/${file.name}` : file.name; + const desc = extractJSDocDescription(fullPath); + const exps = extractExports(fullPath); + rows.push({ + module: modulePath, + purpose: desc || '', + exports: exps.map(e => `\`${e}()\``).join(', '), + }); + } + + for (const dir of sortedDirs) { + collectModules(path.join(dirPath, dir.name), rows, relPrefix ? `${relPrefix}/${dir.name}` : dir.name); + } +} + +module.exports = { + SKIP_DIRS, + extractJSDocDescription, + extractExports, + buildDirectoryTree, + detectSourceDirs, + buildModuleIndex, +}; diff --git a/tests/evals/fixtures/level-5-autonomous/scripts/generate-docs.js b/tests/evals/fixtures/level-5-autonomous/scripts/generate-docs.js new file mode 100644 index 0000000..f65f14f --- /dev/null +++ b/tests/evals/fixtures/level-5-autonomous/scripts/generate-docs.js @@ -0,0 +1,284 @@ +#!/usr/bin/env node + +/** + * Auto-generate CLAUDE.md sections from source code. + * + * Two modes: + * - Default (write): Regenerates AUTO markers in CLAUDE.md, writes plans index, auto-stages. + * - --check: Compares generated vs current, validates cross-links, exits 1 if stale. + * + * Usage: + * node scripts/generate-docs.js # Write mode (regenerate + stage) + * node scripts/generate-docs.js --check # Check mode (validate only) + */ + +const fs = require('node:fs'); +const path = require('node:path'); +const { execFileSync } = require('node:child_process'); + +const { + buildDirectoryTree, + detectSourceDirs, + buildModuleIndex, +} = require('./generate-docs-helpers'); + +// --------------------------------------------------------------------------- +// Marker Replacement +// --------------------------------------------------------------------------- + +/** + * Replace content between AUTO markers in a document. + * Markers: `` ... `` + * @param {string} content - Full document content + * @param {string} markerName - Marker identifier + * @param {string} newContent - Replacement content + * @returns {string} Updated document + */ +function replaceMarkers(content, markerName, newContent) { + const open = ``; + const close = ``; + const openIdx = content.indexOf(open); + const closeIdx = content.indexOf(close); + + if (openIdx === -1 || closeIdx === -1) { + return content; + } + + const before = content.slice(0, openIdx + open.length); + const after = content.slice(closeIdx); + return `${before}\n${newContent}\n${after}`; +} + +// --------------------------------------------------------------------------- +// Cross-Link Validation +// --------------------------------------------------------------------------- + +/** + * Validate that markdown cross-links point to existing files. + * Skips http/https URLs and anchor-only links. + * @param {string} markdown - Markdown content + * @param {string} rootDir - Project root for resolving relative paths + * @returns {string[]} Array of error messages for broken links + */ +function validateCrossLinks(markdown, rootDir) { + const errors = []; + const linkRe = /\[([^\]]*)\]\(([^)]+)\)/g; + let match; + + while ((match = linkRe.exec(markdown)) !== null) { + const target = match[2]; + if (target.startsWith('http://') || target.startsWith('https://') || target.startsWith('#')) { + continue; + } + const filePart = target.split('#')[0]; + if (!filePart) { + continue; + } + const resolved = path.resolve(rootDir, filePart); + if (!fs.existsSync(resolved)) { + errors.push(`Broken link: [${match[1]}](${target}) -> ${filePart} not found`); + } + } + + return errors; +} + +// --------------------------------------------------------------------------- +// Plans Index Builder +// --------------------------------------------------------------------------- + +/** + * Generate a markdown index of all docs/ content. + * Scans docs/ recursively, groups by subdirectory, extracts headings. + * @param {string} rootDir - Project root directory + * @returns {string} Markdown index content + */ +function buildDocsIndex(rootDir) { + const docsDir = path.join(rootDir, 'docs'); + const rootFiles = []; + const subdirs = {}; + collectDocsRecursive(docsDir, '', rootFiles, subdirs); + + if (rootFiles.length === 0 && Object.keys(subdirs).length === 0) { + return 'No documentation files found.'; + } + + const sections = []; + if (rootFiles.length > 0) { + for (const f of rootFiles) { + sections.push(formatDocEntry(f.name, f.heading, `docs/${f.name}`)); + } + } + for (const [dir, files] of Object.entries(subdirs).sort()) { + if (sections.length > 0) sections.push(''); + sections.push(`## ${dir}\n`); + for (const f of files) { + sections.push(formatDocEntry(f.name, f.heading, `docs/${dir}/${f.name}`)); + } + } + return sections.join('\n'); +} + +function formatDocEntry(name, heading, linkPath) { + return heading ? `- [${name}](${linkPath}) — ${heading}` : `- [${name}](${linkPath})`; +} + +function collectDocsRecursive(dirPath, relPrefix, rootFiles, subdirs) { + let entries; + try { + entries = fs.readdirSync(dirPath, { withFileTypes: true }); + } catch { + return; + } + const dirs = entries.filter(e => e.isDirectory()).sort((a, b) => a.name.localeCompare(b.name)); + const files = entries + .filter(e => e.isFile() && e.name.endsWith('.md') && e.name !== 'index.md') + .sort((a, b) => a.name.localeCompare(b.name)); + + for (const file of files) { + const heading = extractFirstHeading(path.join(dirPath, file.name)); + const entry = { name: file.name, heading }; + if (relPrefix) { + if (!subdirs[relPrefix]) subdirs[relPrefix] = []; + subdirs[relPrefix].push(entry); + } else { + rootFiles.push(entry); + } + } + for (const dir of dirs) { + const nextPrefix = relPrefix ? `${relPrefix}/${dir.name}` : dir.name; + collectDocsRecursive(path.join(dirPath, dir.name), nextPrefix, rootFiles, subdirs); + } +} + +function extractFirstHeading(filePath) { + try { + const content = fs.readFileSync(filePath, 'utf8'); + const match = content.match(/^#\s+(.+)/m); + return match ? match[1].trim() : ''; + } catch { + return ''; + } +} + +// --------------------------------------------------------------------------- +// Staleness Check +// --------------------------------------------------------------------------- + +/** + * Compare current marker content against freshly generated content. + * @param {string} docContent - Current document content + * @param {Object} generated - Map of markerName -> generated content + * @returns {string[]} Names of stale markers + */ +function checkMarkersAreCurrent(docContent, generated) { + const stale = []; + for (const [name, expected] of Object.entries(generated)) { + const open = ``; + const close = ``; + const openIdx = docContent.indexOf(open); + const closeIdx = docContent.indexOf(close); + + if (openIdx === -1 || closeIdx === -1) { + stale.push(name); + continue; + } + const current = docContent.slice(openIdx + open.length, closeIdx).trim(); + if (current !== expected.trim()) { + stale.push(name); + } + } + return stale; +} + +// --------------------------------------------------------------------------- +// Main Entry Point +// --------------------------------------------------------------------------- + +/** Main: regenerate or check CLAUDE.md auto-generated sections. */ +function main() { + const rootDir = path.resolve(__dirname, '..'); + const docPath = path.join(rootDir, 'CLAUDE.md'); + const checkMode = process.argv.includes('--check'); + + const treeDirs = detectSourceDirs(rootDir); + const tree = buildDirectoryTree(rootDir, treeDirs); + const modules = buildModuleIndex(rootDir); + const docsIndex = buildDocsIndex(rootDir); + const generated = { tree, modules }; + + if (checkMode) { + runCheckMode(docPath, rootDir, generated); + return; + } + + runWriteMode(docPath, rootDir, generated, docsIndex); +} + +/** Check mode: validate markers and cross-links, exit 1 if stale. */ +function runCheckMode(docPath, rootDir, generated) { + let doc; + try { + doc = fs.readFileSync(docPath, 'utf-8'); + } catch { + console.error('Cannot read CLAUDE.md'); + process.exit(1); + } + + const stale = checkMarkersAreCurrent(doc, generated); + const linkErrors = validateCrossLinks(doc, rootDir); + + if (stale.length > 0) { + console.error(`Stale markers: ${stale.join(', ')}`); + } + for (const err of linkErrors) { + console.error(err); + } + if (stale.length > 0 || linkErrors.length > 0) { + console.error('\nRun `node scripts/generate-docs.js` to regenerate.'); + process.exit(1); + } + console.log('All markers are current.'); +} + +/** Write mode: regenerate markers, write docs index, auto-stage. */ +function runWriteMode(docPath, rootDir, generated, docsIndex) { + let doc; + try { + doc = fs.readFileSync(docPath, 'utf-8'); + } catch { + console.error('Cannot read CLAUDE.md'); + process.exit(1); + } + + doc = replaceMarkers(doc, 'tree', generated.tree); + doc = replaceMarkers(doc, 'modules', generated.modules); + fs.writeFileSync(docPath, doc); + + const docsIndexPath = path.join(rootDir, 'docs', 'index.md'); + if (fs.existsSync(path.dirname(docsIndexPath))) { + fs.writeFileSync(docsIndexPath, `# Documentation Index\n\n${docsIndex}\n`); + } + + try { + execFileSync('git', ['add', docPath], { stdio: 'ignore' }); + if (fs.existsSync(docsIndexPath)) { + execFileSync('git', ['add', docsIndexPath], { stdio: 'ignore' }); + } + } catch { + // Not in a git repo - that's fine + } + + console.log('CLAUDE.md markers regenerated.'); +} + +if (require.main === module) { + main(); +} + +module.exports = { + replaceMarkers, + validateCrossLinks, + buildDocsIndex, + checkMarkersAreCurrent, +};