Skip to content

Add auto-doc verification and report structure checks #26

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jrenaldi79 merged 1 commit into from
Mar 24, 2026
Merged

Add auto-doc verification and report structure checks #26

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 34 additions & 18 deletions tests/evals/setup-grader.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

const fs = require('fs');
const path = require('path');
const { execFileSync } = require('child_process');

const args = {};
for (let i = 2; i < process.argv.length; i += 2) {
Expand Down Expand Up @@ -139,27 +140,16 @@ if (pathExists(claudeMdPath)) {

// ─── 6. Settings allow/deny structure ───
if (expected.settings_has_allow_deny) {
const settingsPath = path.join(fixtureDir, '.claude/settings.json');
if (!pathExists(settingsPath)) {
check('Settings has allow/deny lists', false, 'settings.json not found');
const sp = path.join(fixtureDir, '.claude/settings.json');
if (!pathExists(sp)) {
check('Settings has allow/deny', false, 'Not found');
} else {
try {
const settings = JSON.parse(fs.readFileSync(settingsPath, 'utf8'));
const perms = settings.permissions || {};
const hasAllow = Array.isArray(perms.allow) && perms.allow.length > 0;
const hasDeny = Array.isArray(perms.deny) && perms.deny.length > 0;
check(
'Settings has allow list',
hasAllow,
hasAllow ? `${perms.allow.length} entries` : 'Missing or empty',
);
check(
'Settings has deny list',
hasDeny,
hasDeny ? `${perms.deny.length} entries` : 'Missing or empty',
);
const perms = JSON.parse(fs.readFileSync(sp, 'utf8')).permissions || {};
check('Settings has allow list', Array.isArray(perms.allow) && perms.allow.length > 0);
check('Settings has deny list', Array.isArray(perms.deny) && perms.deny.length > 0);
} catch (err) {
check('Settings has allow/deny lists', false, err.message);
check('Settings has allow/deny', false, err.message);
}
}
}
Expand Down Expand Up @@ -209,6 +199,32 @@ if (expected.auto_doc_pipeline) {
}
}

// ─── 8b. Auto-doc: run generate-docs --check to verify markers match reality ───
if (expected.auto_doc_pipeline) {
const scriptPath = path.join(fixtureDir, 'scripts/generate-docs.js');
if (pathExists(scriptPath)) {
try {
execFileSync(process.execPath, [scriptPath, '--check'], {
cwd: fixtureDir, stdio: ['ignore', 'pipe', 'pipe'], timeout: 10000,
});
check('Auto-doc: --check passes (markers match filesystem)', true);
} catch (err) {
const msg = (err.stderr || err.message || '').toString().slice(0, 120);
check('Auto-doc: --check passes (markers match filesystem)', false, msg);
}
}
}

// ─── 8c. Setup report structure checks ───
if (pathExists(reportPath)) {
const reportContent = fs.readFileSync(reportPath, 'utf8');
const sections = ['What Was Installed', 'Verification Results', 'Key Commands'];
for (const s of sections) {
const found = reportContent.toLowerCase().includes(s.toLowerCase());
check(`Report has "${s}" section`, found, found ? '' : 'Not found');
}
}

// ─── 9. Docs index auto-generation ───
if (expected.docs_index_generated) {
const docsDir = path.join(fixtureDir, 'docs');
Expand Down
Loading