From 8705d48e748722ab5fc03fbf8084ed4613eeefa6 Mon Sep 17 00:00:00 2001
From: Claude <noreply@anthropic.com>
Date: Tue, 24 Mar 2026 23:40:48 +0000
Subject: [PATCH] Add auto-doc verification and report structure checks to
 setup grader

Three new checks adapted from readiness eval patterns:
1. Run generate-docs.js --check to verify AUTO markers match the
   actual filesystem (catches hand-written vs script-generated content)
2. Verify setup-report.md has required sections: What Was Installed,
   Verification Results, Key Commands
3. Condensed settings checks to stay under 300 lines

https://claude.ai/code/session_01Hbxy31TkbujzukGFSxLcPw
---
 tests/evals/setup-grader.js | 52 ++++++++++++++++++++++++-------------
 1 file changed, 34 insertions(+), 18 deletions(-)

diff --git a/tests/evals/setup-grader.js b/tests/evals/setup-grader.js
index 5c92512..5df8975 100644
--- a/tests/evals/setup-grader.js
+++ b/tests/evals/setup-grader.js
@@ -5,6 +5,7 @@
 
 const fs = require('fs');
 const path = require('path');
+const { execFileSync } = require('child_process');
 
 const args = {};
 for (let i = 2; i < process.argv.length; i += 2) {
@@ -139,27 +140,16 @@ if (pathExists(claudeMdPath)) {
 
 // ─── 6. Settings allow/deny structure ───
 if (expected.settings_has_allow_deny) {
-  const settingsPath = path.join(fixtureDir, '.claude/settings.json');
-  if (!pathExists(settingsPath)) {
-    check('Settings has allow/deny lists', false, 'settings.json not found');
+  const sp = path.join(fixtureDir, '.claude/settings.json');
+  if (!pathExists(sp)) {
+    check('Settings has allow/deny', false, 'Not found');
   } else {
     try {
-      const settings = JSON.parse(fs.readFileSync(settingsPath, 'utf8'));
-      const perms = settings.permissions || {};
-      const hasAllow = Array.isArray(perms.allow) && perms.allow.length > 0;
-      const hasDeny = Array.isArray(perms.deny) && perms.deny.length > 0;
-      check(
-        'Settings has allow list',
-        hasAllow,
-        hasAllow ? `${perms.allow.length} entries` : 'Missing or empty',
-      );
-      check(
-        'Settings has deny list',
-        hasDeny,
-        hasDeny ? `${perms.deny.length} entries` : 'Missing or empty',
-      );
+      const perms = JSON.parse(fs.readFileSync(sp, 'utf8')).permissions || {};
+      check('Settings has allow list', Array.isArray(perms.allow) && perms.allow.length > 0);
+      check('Settings has deny list', Array.isArray(perms.deny) && perms.deny.length > 0);
     } catch (err) {
-      check('Settings has allow/deny lists', false, err.message);
+      check('Settings has allow/deny', false, err.message);
     }
   }
 }
@@ -209,6 +199,32 @@ if (expected.auto_doc_pipeline) {
   }
 }
 
+// ─── 8b. Auto-doc: run generate-docs --check to verify markers match reality ───
+if (expected.auto_doc_pipeline) {
+  const scriptPath = path.join(fixtureDir, 'scripts/generate-docs.js');
+  if (pathExists(scriptPath)) {
+    try {
+      execFileSync(process.execPath, [scriptPath, '--check'], {
+        cwd: fixtureDir, stdio: ['ignore', 'pipe', 'pipe'], timeout: 10000,
+      });
+      check('Auto-doc: --check passes (markers match filesystem)', true);
+    } catch (err) {
+      const msg = (err.stderr || err.message || '').toString().slice(0, 120);
+      check('Auto-doc: --check passes (markers match filesystem)', false, msg);
+    }
+  }
+}
+
+// ─── 8c. Setup report structure checks ───
+if (pathExists(reportPath)) {
+  const reportContent = fs.readFileSync(reportPath, 'utf8');
+  const sections = ['What Was Installed', 'Verification Results', 'Key Commands'];
+  for (const s of sections) {
+    const found = reportContent.toLowerCase().includes(s.toLowerCase());
+    check(`Report has "${s}" section`, found, found ? '' : 'Not found');
+  }
+}
+
 // ─── 9. Docs index auto-generation ───
 if (expected.docs_index_generated) {
   const docsDir = path.join(fixtureDir, 'docs');