diff --git a/README.md b/README.md index ac71bec..4fb1f61 100644 --- a/README.md +++ b/README.md @@ -233,9 +233,15 @@ The Stop hook is fail-open everywhere. Any error returns `{"decision":"approve"} | `CLAUDEX_STALE_MINUTES` | 15 | Loops older than this are auto-swept on next invocation | | `CLAUDEX_STATE_DIR` | `.claude/claudex` | State directory location | -## Cost expectation +## Headless Hermes planning bridge -Each plan-mode round is one full Codex review of `PLAN.md`. In practice that's ~25–30k Codex tokens per round. With the default 3 rounds you should expect **~75–90k tokens per `/claudex:plan`**. Codex authenticates against your ChatGPT account, so the bill goes to your ChatGPT Plus / Pro / Team / Enterprise plan, not to claudex. If you're on a tight rate limit, run `--rounds 2` for fast topics and reserve `--rounds 5+` for high-stakes designs. +This fork adds [`bin/claudex-plan-review`](docs/HEADLESS_ADAPTER.md), a production-oriented adapter for running an existing `PLAN.md` through headless Claude Code and Claudex. Its default `sweep-v2` path runs the Phase 1 frozen-snapshot, five-persona lifecycle with a generation cap of `1..5` (the staged Hermes workflow uses five); `--engine legacy` preserves backward compatibility. It validates explicit executable/plugin/auth prerequisites, pins child `PATH`, enforces wall-clock and Claude telemetry-budget bounds, kills the complete process group on timeout, copies complete state/generation evidence, and emits one strict JSON result. + +Only `converged` is clean. `max_reached`, `degraded`, `failed`, and `timed_out` are explicit non-clean outcomes. Sweep classification requires terminal authoritative state plus exact same-snapshot five-persona coverage, valid manifests/hashes, and consolidated findings proving clean; Claude prose or one final findings file is insufficient. See the adapter document for usage, exit codes, subscription telemetry wording, architecture, and staging instructions. The in-repo Hermes skill is staged at [`skills/project-plan-review/`](skills/project-plan-review/SKILL.md); it is not installed automatically. + +## Subscription usage expectation + +Legacy plan mode invokes one Codex review per round. Sweep-v2 can invoke all five required Codex personas in each generation, up to its hard cap of five generations. Claude Code and Codex authenticate through subscription-backed CLIs in this workflow; any dollar-valued fields are usage-equivalent telemetry/bounded-run controls, not direct API billing or an invoice. Both services may still enforce subscription rate limits. ## Safety @@ -260,17 +266,20 @@ Highlights: ## Tests ```bash -# Phase 0: confirm platform behaviors work on your machine (50 checks) +# Phase 0: confirm platform behaviors work on your machine (count printed by test) bash plugins/claudex/tests/platform-validation.sh -# Smoke test: simulate full lifecycle without invoking Codex (60 checks) +# Smoke test: simulate full lifecycle without invoking Codex (count printed by test) bash plugins/claudex/tests/smoke-test.sh -# Synthetic E2E: real Codex calls against a throwaway repo (19 checks, costs a few cents in tokens) +# Synthetic E2E: real Codex calls against a throwaway repo (count printed by test; uses subscription tokens) bash plugins/claudex/tests/synthetic-e2e.sh + +# Headless adapter deterministic unit/error/timeout/state-isolation tests +python3 -m unittest -v tests/test_adapter.py ``` -All three should pass before trusting claudex on a real project. +Run the platform, smoke, and adapter suites for every change. Run the live synthetic E2E when authenticated Codex usage is available. ## Project structure diff --git a/bin/claudex-plan-review b/bin/claudex-plan-review new file mode 100755 index 0000000..4061a7b --- /dev/null +++ b/bin/claudex-plan-review @@ -0,0 +1,720 @@ +#!/usr/bin/env python3 +"""Headless Claude Code -> Claudex -> Codex plan review adapter. + +Stdout is always one strict JSON document. Human/debug output and subprocess +streams are retained in an evidence directory instead of being mixed into the +machine-readable result. +""" + +from __future__ import annotations + +import argparse +import datetime as dt +import hashlib +import json +import os +import re +import shutil +import signal +import subprocess +import sys +import time +import uuid +from decimal import Decimal, InvalidOperation +from pathlib import Path +from typing import Any + +TERMINAL_PHASES = {"done", "cancelled", "errored"} +ACTIVE_PHASES = {"drafting", "reviewing", "revising", "summarizing"} +EXIT_CODES = {"converged": 0, "max_reached": 10, "degraded": 11, "failed": 12, "timed_out": 124} +SWEEP_PERSONAS = [ + "architecture-scope", + "security-data", + "product-domain", + "quality-accessibility-performance", + "operations-deployment", +] +SHA256_RE = re.compile(r"[0-9a-f]{64}") + + +def emit(payload: dict[str, Any], code: int) -> int: + print(json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False)) + return code + + +def fail(message: str, *, kind: str = "validation", evidence_dir: Path | None = None) -> int: + payload: dict[str, Any] = { + "schema_version": 1, + "outcome": "failed", + "clean": False, + "error": {"kind": kind, "message": message}, + } + if evidence_dir is not None: + payload["evidence_dir"] = str(evidence_dir) + return emit(payload, EXIT_CODES["failed"]) + + +def absolute_existing_dir(value: str, label: str) -> Path: + path = Path(value) + if not path.is_absolute(): + raise ValueError(f"{label} must be an absolute path") + resolved = path.resolve(strict=True) + if not resolved.is_dir(): + raise ValueError(f"{label} is not a directory: {resolved}") + return resolved + + +def absolute_existing_file(value: str, label: str) -> Path: + path = Path(value) + if not path.is_absolute(): + raise ValueError(f"{label} must be an absolute path") + resolved = path.resolve(strict=True) + if not resolved.is_file(): + raise ValueError(f"{label} is not a file: {resolved}") + if resolved.stat().st_size == 0: + raise ValueError(f"{label} must be non-empty: {resolved}") + return resolved + + +def executable(value: str, label: str) -> Path: + path = absolute_existing_file(value, label) + if not os.access(path, os.X_OK): + raise ValueError(f"{label} is not executable: {path}") + return path + + +def run_probe(argv: list[str], cwd: Path, env: dict[str, str], timeout: float = 20) -> subprocess.CompletedProcess[str]: + return subprocess.run(argv, cwd=cwd, env=env, text=True, capture_output=True, timeout=timeout, check=False) + + +def parse_state(path: Path) -> dict[str, str]: + result: dict[str, str] = {} + try: + for line in path.read_text(encoding="utf-8", errors="replace").splitlines(): + if ":" not in line: + continue + key, value = line.split(":", 1) + if re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", key): + result[key] = value.strip() + except OSError: + pass + return result + + +def snapshot_state(state_file: Path, events, last: dict[str, str] | None) -> dict[str, str] | None: + if not state_file.exists(): + return last + current = parse_state(state_file) + if current != last: + events.write(json.dumps({"at": dt.datetime.now(dt.timezone.utc).isoformat(), "state": current}, sort_keys=True) + "\n") + events.flush() + return current + + +def final_findings_status(path: Path | None) -> tuple[str, dict[str, int]]: + counts = {"high": 0, "medium": 0, "low": 0} + if path is None or not path.is_file(): + return "missing", counts + text = path.read_text(encoding="utf-8", errors="replace") + section = "" + for line in text.splitlines(): + if line.startswith("## High"): + section = "high" + elif line.startswith("## Medium"): + section = "medium" + elif line.startswith("## Low"): + section = "low" + elif line.startswith("## "): + section = "" + elif line.startswith("- ") and section: + counts[section] += 1 + no_findings = bool(re.search(r"(?mi)^No substantive findings\.\s*$", text)) + if no_findings and not any(counts.values()): + return "none", counts + if any(counts.values()): + return "material", counts + return "unparseable", counts + + +def sha256_file(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def strict_state(path: Path) -> tuple[dict[str, str], list[str]]: + state: dict[str, str] = {} + errors: list[str] = [] + try: + lines = path.read_text(encoding="utf-8").splitlines() + except (OSError, UnicodeError) as exc: + return {}, [f"state is unreadable: {exc}"] + for number, line in enumerate(lines, 1): + if ":" not in line: + errors.append(f"state line {number} is malformed") + continue + key, value = line.split(":", 1) + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", key): + errors.append(f"state line {number} has an invalid key") + elif key in state: + errors.append(f"state contains duplicate field {key}") + else: + state[key] = value.strip() + return state, errors + + +def sweep_findings(path: Path) -> tuple[str, dict[str, int]]: + counts = {"high": 0, "medium": 0, "low": 0} + try: + text = path.read_text(encoding="utf-8") + except (OSError, UnicodeError): + return "malformed", counts + if text.strip() == "No substantive findings.": + return "clean", counts + lines = [line.rstrip() for line in text.splitlines() if line.strip()] + headers = ["## High", "## Medium", "## Low"] + if any(lines.count(header) != 1 for header in headers): + return "malformed", counts + positions = [lines.index(header) for header in headers] + if positions != sorted(positions) or lines[: positions[0]]: + return "malformed", counts + for index, start in enumerate(positions): + end = positions[index + 1] if index + 1 < len(positions) else len(lines) + severity = headers[index][3:].lower() + for line in lines[start + 1 : end]: + if not line.startswith("- ") or len(line) <= 2: + return "malformed", counts + counts[severity] += 1 + return ("material" if any(counts.values()) else "malformed"), counts + + +def render_sweep_findings(path: Path, persona: str) -> str: + severity = "" + counts = {"high": 0, "medium": 0, "low": 0} + rendered: list[str] = [] + for line in path.read_text(encoding="utf-8").splitlines(): + if line in {"## High", "## Medium", "## Low"}: + severity = line[3:].lower() + rendered.append(line) + elif line.startswith("- ") and severity: + counts[severity] += 1 + rendered.append(f"- [{persona}-{severity}-{counts[severity]}] {line[2:]}") + else: + rendered.append(line) + return "\n".join(rendered) + + +def validate_sweep(state_file: Path, review_dir: Path, runtime_plan: Path, reviewed_plan: Path, requested: int) -> dict[str, Any]: + """Recompute a sweep verdict from runner-authored state and artifacts.""" + state, errors = strict_state(state_file) + coverage: list[dict[str, Any]] = [] + severity = {"high": 0, "medium": 0, "low": 0} + + def integer(name: str) -> int | None: + value = state.get(name, "") + return int(value) if value.isdigit() else None + + generation = integer("generation") + maximum = integer("max_generations") + if state.get("engine") != "sweep-v2": + errors.append("state engine is not sweep-v2") + if generation is None or not 1 <= generation <= 5: + errors.append("state generation is not in 1..5") + if maximum is None or not 1 <= maximum <= 5: + errors.append("state max_generations is not in 1..5") + elif maximum != requested: + errors.append("state max_generations does not match the requested cap") + if generation and maximum and generation > maximum: + errors.append("state generation exceeds max_generations") + + generation_dir = review_dir / "generations" / str(generation) if generation else None + snapshot = generation_dir / "PLAN.md" if generation_dir else None + manifest_path = generation_dir / "manifest.json" if generation_dir else None + consolidated = generation_dir / "consolidated-findings.md" if generation_dir else None + snapshot_hash = state.get("snapshot_sha256", "") + if not SHA256_RE.fullmatch(snapshot_hash): + errors.append("state snapshot_sha256 is malformed") + if state.get("reviewed_live_sha256") != snapshot_hash: + errors.append("state reviewed_live_sha256 does not match the current snapshot") + + manifest: dict[str, Any] = {} + if manifest_path: + try: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + if not isinstance(manifest, dict): + raise ValueError("manifest is not an object") + except (OSError, UnicodeError, json.JSONDecodeError, ValueError) as exc: + errors.append(f"generation manifest is unreadable or malformed: {exc}") + if manifest: + required = {"generation", "snapshot_sha256", "required_persona_ids", "topic", "source_plan_path", "previous_generation_sha256"} + if set(manifest) != required: + errors.append("generation manifest fields do not match the sweep-v2 schema") + if manifest.get("generation") != generation or manifest.get("snapshot_sha256") != snapshot_hash: + errors.append("generation manifest does not match state generation/hash") + if manifest.get("required_persona_ids") != SWEEP_PERSONAS: + errors.append("generation manifest does not require the exact five personas") + if not isinstance(manifest.get("topic"), str) or not manifest.get("topic"): + errors.append("generation manifest topic is empty") + if manifest.get("source_plan_path") != str(runtime_plan): + errors.append("generation manifest source plan path is mismatched") + if generation == 1 and manifest.get("previous_generation_sha256") is not None: + errors.append("generation-one manifest has an unexpected previous hash") + elif generation and generation > 1: + try: + previous = json.loads((review_dir / "generations" / str(generation - 1) / "manifest.json").read_text(encoding="utf-8")) + previous_hash = previous["snapshot_sha256"] + if not isinstance(previous_hash, str) or not SHA256_RE.fullmatch(previous_hash) or manifest.get("previous_generation_sha256") != previous_hash: + errors.append("generation manifest previous hash linkage is invalid") + except (OSError, UnicodeError, json.JSONDecodeError, KeyError, TypeError): + errors.append("previous generation manifest is unreadable or malformed") + + try: + if not snapshot or sha256_file(snapshot) != snapshot_hash: + errors.append("frozen snapshot hash does not match state") + except OSError as exc: + errors.append(f"frozen snapshot is unreadable: {exc}") + try: + if sha256_file(reviewed_plan) != snapshot_hash: + errors.append("reviewed live PLAN.md is not the frozen snapshot") + except OSError as exc: + errors.append(f"reviewed live PLAN.md is unreadable: {exc}") + + evidence_digest = hashlib.sha256() + chunks = [f"# Consolidated findings — generation {generation}\n\n", f"Snapshot SHA-256: `{snapshot_hash}`\n\n"] + if generation_dir: + for persona in SWEEP_PERSONAS: + findings = generation_dir / f"{persona}.findings.md" + sidecar = generation_dir / f"{persona}.result.json" + classification, counts = sweep_findings(findings) + entry: dict[str, Any] = { + "persona_id": persona, + "classification": classification, + "snapshot_sha256": None, + "valid": False, + } + for key in severity: + severity[key] += counts[key] + sidecar_data: dict[str, Any] = {} + try: + sidecar_data = json.loads(sidecar.read_text(encoding="utf-8")) + if not isinstance(sidecar_data, dict): + raise ValueError("sidecar is not an object") + except (OSError, UnicodeError, json.JSONDecodeError, ValueError) as exc: + errors.append(f"{persona} sidecar is unreadable or malformed: {exc}") + if sidecar_data: + required = {"persona_id", "expected_snapshot_sha256", "actual_snapshot_sha256_before", "actual_snapshot_sha256_after", "codex_exit_code", "findings_path", "findings_sha256", "findings_classification", "completed_at"} + expected_path = str(review_dir / "generations" / str(generation) / findings.name) + valid = set(sidecar_data) == required + valid &= sidecar_data.get("persona_id") == persona + valid &= sidecar_data.get("expected_snapshot_sha256") == snapshot_hash + valid &= sidecar_data.get("actual_snapshot_sha256_before") == snapshot_hash + valid &= sidecar_data.get("actual_snapshot_sha256_after") == snapshot_hash + valid &= sidecar_data.get("codex_exit_code") == 0 + valid &= sidecar_data.get("findings_path") == expected_path + valid &= sidecar_data.get("findings_classification") == classification + try: + valid &= sidecar_data.get("findings_sha256") == sha256_file(findings) + except OSError: + valid = False + try: + dt.datetime.strptime(str(sidecar_data.get("completed_at")), "%Y-%m-%dT%H:%M:%SZ") + except (TypeError, ValueError): + valid = False + if not valid or classification not in {"clean", "material"}: + errors.append(f"{persona} evidence is incomplete, hash-mismatched, or malformed") + else: + entry.update(snapshot_sha256=snapshot_hash, valid=True) + coverage.append(entry) + chunks.append(f"## {persona}\n\n") + if classification == "clean": + chunks.append(findings.read_text(encoding="utf-8") + "\n\n") + elif classification == "material": + chunks.append(render_sweep_findings(findings, persona) + "\n\n\n") + else: + chunks.append("DEGRADED: invalid evidence.\n\n") + for artifact in (findings, sidecar): + try: + evidence_digest.update(artifact.name.encode("utf-8") + b"\0") + evidence_digest.update(artifact.read_bytes()) + evidence_digest.update(b"\0") + except OSError: + pass + + consolidated_hash = "" + try: + consolidated_text = consolidated.read_text(encoding="utf-8") if consolidated else "" + consolidated_hash = sha256_file(consolidated) if consolidated else "" + if consolidated_text != "".join(chunks): + errors.append("consolidated findings do not exactly match persona evidence") + except (OSError, UnicodeError) as exc: + errors.append(f"consolidated findings are unreadable: {exc}") + if state.get("evidence_sha256") != evidence_digest.hexdigest(): + errors.append("state evidence_sha256 does not match generation evidence") + if state.get("consolidated_sha256") != consolidated_hash: + errors.append("state consolidated_sha256 does not match consolidated findings") + + all_clean = len(coverage) == 5 and all(item["valid"] and item["classification"] == "clean" for item in coverage) + any_material = any(item["valid"] and item["classification"] == "material" for item in coverage) + phase, signal = state.get("phase"), state.get("decision_signal") + if not errors and phase == "done" and signal == "converged" and state.get("clean") == "true" and state.get("coverage_complete") == "true" and all_clean and state.get("converged_snapshot_sha256") == snapshot_hash: + outcome = "converged" + reason = "terminal sweep-v2 state and complete same-snapshot five-persona evidence prove convergence" + elif not errors and phase == "done" and signal == "max-reached" and state.get("clean") == "false" and state.get("coverage_complete") == "true" and generation == maximum and any_material and not all_clean and not state.get("converged_snapshot_sha256"): + outcome = "max_reached" + reason = "sweep-v2 reached its generation cap with authoritative material findings" + else: + outcome = "degraded" + mismatch = errors or [f"terminal/state verdict mismatch (phase={phase}, signal={signal}, clean={state.get('clean')}, coverage={state.get('coverage_complete')})"] + reason = "sweep-v2 evidence is incomplete or inconsistent: " + "; ".join(mismatch[:4]) + return { + "outcome": outcome, "reason": reason, "state": state, + "generation": generation, "max_generations": maximum, + "snapshot_sha256": snapshot_hash or None, + "converged_snapshot_sha256": state.get("converged_snapshot_sha256") or None, + "persona_coverage": coverage, "severity_counts": severity, + "findings_status": "none" if all_clean else ("material" if any_material else "malformed"), + "generation_dir": generation_dir, "manifest": manifest_path, + "consolidated": consolidated, "errors": errors, + } + + +def parse_claude_stream(path: Path) -> dict[str, Any]: + summary: dict[str, Any] = {"result_records": 0, "reported_cost_usd": None, "session_id": None} + try: + for line in path.read_text(encoding="utf-8", errors="replace").splitlines(): + try: + obj = json.loads(line) + except json.JSONDecodeError: + continue + if obj.get("type") == "result": + summary["result_records"] += 1 + summary["reported_cost_usd"] = obj.get("total_cost_usd", obj.get("cost_usd")) + summary["session_id"] = obj.get("session_id") + summary["claude_result_subtype"] = obj.get("subtype") + except OSError: + pass + return summary + + +def copy_if_exists(source: Path, destination: Path) -> None: + if source.exists(): + if source.is_dir(): + shutil.copytree(source, destination, dirs_exist_ok=True) + else: + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, destination) + + +class JsonArgumentParser(argparse.ArgumentParser): + def error(self, message: str) -> None: + payload = { + "schema_version": 1, + "outcome": "failed", + "clean": False, + "error": {"kind": "arguments", "message": message}, + } + print(json.dumps(payload, sort_keys=True, separators=(",", ":"))) + raise SystemExit(EXIT_CODES["failed"]) + + +def main() -> int: + parser = JsonArgumentParser(description="Run a bounded headless Claudex review of an existing plan") + parser.add_argument("--repo", required=True, help="absolute repository path") + parser.add_argument("--plan", required=True, help="absolute path to an existing non-empty PLAN.md") + parser.add_argument("--topic", required=True, help="self-contained review topic and constraints") + parser.add_argument("--rounds", required=True, type=int, help="positive Claudex round limit") + parser.add_argument("--engine", choices=("sweep-v2", "legacy"), default="sweep-v2", help="plan-review engine (default: sweep-v2)") + parser.add_argument("--timeout", required=True, type=float, help="wall-clock timeout in seconds") + parser.add_argument("--budget-usd", required=True, help="positive Claude Code budget cap") + default_plugin = Path(__file__).resolve().parents[1] / "plugins" / "claudex" + parser.add_argument("--plugin-root", default=str(default_plugin), help="absolute Claudex plugin root") + parser.add_argument("--claude", required=True, help="absolute Claude Code executable") + parser.add_argument("--codex", required=True, help="absolute Codex executable") + parser.add_argument("--output-dir", help="absolute evidence directory (must not already exist)") + parser.add_argument("--model", default="sonnet", help="Claude model passed to headless Claude Code") + args = parser.parse_args() + + try: + repo = absolute_existing_dir(args.repo, "--repo") + if not (repo / ".git").exists(): + raise ValueError(f"--repo must be a git working tree: {repo}") + plan = absolute_existing_file(args.plan, "--plan") + plugin = absolute_existing_dir(args.plugin_root, "--plugin-root") + claude = executable(args.claude, "--claude") + codex = executable(args.codex, "--codex") + if args.rounds < 1: + raise ValueError("--rounds must be a positive integer") + if args.engine == "sweep-v2" and args.rounds > 5: + raise ValueError("sweep-v2 --rounds must be in 1..5 generations") + if args.timeout <= 0: + raise ValueError("--timeout must be positive") + if not args.topic.strip(): + raise ValueError("--topic must be non-empty") + try: + budget = Decimal(args.budget_usd) + except InvalidOperation as exc: + raise ValueError("--budget-usd must be a positive decimal") from exc + if not budget.is_finite() or budget <= 0: + raise ValueError("--budget-usd must be a positive decimal") + required = [plugin / ".claude-plugin" / "plugin.json", plugin / "commands" / "plan.md", plugin / "hooks" / "stop-hook.sh", plugin / "scripts" / "state-helpers.sh"] + if args.engine == "sweep-v2": + required.extend([plugin / "scripts" / "sweep-helpers.sh", plugin / "scripts" / "personas.sh"]) + missing = [str(path) for path in required if not path.is_file()] + if missing: + raise ValueError("invalid Claudex plugin root; missing: " + ", ".join(missing)) + manifest = json.loads(required[0].read_text(encoding="utf-8")) + if manifest.get("name") != "claudex": + raise ValueError(f"plugin manifest is not Claudex: {required[0]}") + except (ValueError, OSError) as exc: + return fail(str(exc)) + + state_dir = repo / ".claude" / "claudex" + state_dir.mkdir(parents=True, exist_ok=True) + baseline_states = set(state_dir.glob("*.state")) + for state_file in baseline_states: + phase = parse_state(state_file).get("phase", "") + if phase in ACTIVE_PHASES or (phase and phase not in TERMINAL_PHASES): + return fail(f"active Claudex loop already exists: {state_file} (phase={phase})", kind="active_loop") + + stamp = dt.datetime.now(dt.timezone.utc).strftime("%Y%m%dT%H%M%SZ") + if args.output_dir: + output_dir_arg = Path(args.output_dir) + if not output_dir_arg.is_absolute(): + return fail("--output-dir must be an absolute path") + evidence = output_dir_arg + else: + evidence = state_dir / "adapter-runs" / f"{stamp}-{uuid.uuid4().hex[:8]}" + if evidence.exists(): + return fail(f"evidence directory already exists: {evidence}") + evidence.mkdir(parents=True) + + env = os.environ.copy() + pinned_path = os.pathsep.join(dict.fromkeys([str(claude.parent), str(codex.parent), "/usr/bin", "/bin", "/usr/sbin", "/sbin"])) + env["PATH"] = pinned_path + env["CLAUDEX_ADAPTER_EVIDENCE_DIR"] = str(evidence) + + preflight: dict[str, Any] = {"pinned_path": pinned_path, "plugin_root": str(plugin)} + try: + claude_version = run_probe([str(claude), "--version"], repo, env) + codex_version = run_probe([str(codex), "--version"], repo, env) + claude_auth = run_probe([str(claude), "auth", "status"], repo, env) + codex_auth = run_probe([str(codex), "login", "status"], repo, env) + except (OSError, subprocess.TimeoutExpired) as exc: + return fail(f"prerequisite probe failed: {exc}", kind="prerequisite", evidence_dir=evidence) + preflight.update({ + "claude": {"path": str(claude), "version": claude_version.stdout.strip()}, + "codex": {"path": str(codex), "version": codex_version.stdout.strip()}, + "claude_auth": claude_auth.stdout.strip(), + "codex_auth": (codex_auth.stdout + codex_auth.stderr).strip(), + }) + (evidence / "preflight.json").write_text(json.dumps(preflight, indent=2, sort_keys=True) + "\n") + if claude_version.returncode != 0 or codex_version.returncode != 0: + return fail("Claude or Codex version probe failed", kind="prerequisite", evidence_dir=evidence) + try: + auth_obj = json.loads(claude_auth.stdout) + except json.JSONDecodeError: + auth_obj = {} + if claude_auth.returncode != 0 or auth_obj.get("loggedIn") is not True: + return fail("Claude Code authentication is not active", kind="authentication", evidence_dir=evidence) + if codex_auth.returncode != 0 or "logged in" not in (codex_auth.stdout + codex_auth.stderr).lower(): + return fail("Codex authentication is not active", kind="authentication", evidence_dir=evidence) + + runtime_plan = repo / "PLAN.md" + external_plan = plan != runtime_plan.resolve(strict=False) + original_runtime_plan = evidence / "repo-PLAN.original.md" + if external_plan and runtime_plan.exists(): + shutil.copy2(runtime_plan, original_runtime_plan) + shutil.copy2(plan, evidence / "PLAN.before.md") + if external_plan: + shutil.copy2(plan, runtime_plan) + + engine_flag = "--engine sweep-v2 " if args.engine == "sweep-v2" else "" + prompt = f"/claudex:plan {engine_flag}--from-draft --skip-interview --rounds {args.rounds} {args.topic.strip()}" + command = [ + str(claude), "--print", "--verbose", "--output-format", "stream-json", "--include-hook-events", + "--dangerously-skip-permissions", "--setting-sources", "project", "--plugin-dir", str(plugin), + "--model", args.model, "--max-budget-usd", str(budget), prompt, + ] + metadata = { + "argv": command, "cwd": str(repo), "timeout_seconds": args.timeout, + "budget_usd": str(budget), "engine": args.engine, "rounds": args.rounds, "topic": args.topic, + "started_at": dt.datetime.now(dt.timezone.utc).isoformat(), + } + (evidence / "run-metadata.json").write_text(json.dumps(metadata, indent=2, sort_keys=True) + "\n") + + stdout_path = evidence / "claude-stream.jsonl" + stderr_path = evidence / "claude-stderr.log" + events_path = evidence / "state-events.jsonl" + started = time.monotonic() + timed_out = False + process: subprocess.Popen[bytes] | None = None + selected_state: Path | None = None + last_state: dict[str, str] | None = None + try: + with stdout_path.open("wb") as stdout, stderr_path.open("wb") as stderr, events_path.open("w", encoding="utf-8") as events: + process = subprocess.Popen(command, cwd=repo, env=env, stdout=stdout, stderr=stderr, start_new_session=True) + while process.poll() is None: + new_states = [path for path in state_dir.glob("*.state") if path not in baseline_states] + if new_states: + selected_state = max(new_states, key=lambda path: path.stat().st_mtime_ns) + last_state = snapshot_state(selected_state, events, last_state) + if time.monotonic() - started >= args.timeout: + timed_out = True + os.killpg(process.pid, signal.SIGTERM) + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + process.wait(timeout=5) + break + time.sleep(0.2) + returncode = process.wait() if process.poll() is None else process.returncode + new_states = [path for path in state_dir.glob("*.state") if path not in baseline_states] + if new_states: + selected_state = max(new_states, key=lambda path: path.stat().st_mtime_ns) + last_state = snapshot_state(selected_state, events, last_state) + except (OSError, subprocess.SubprocessError) as exc: + if process is not None and process.poll() is None: + try: + os.killpg(process.pid, signal.SIGKILL) + process.wait(timeout=5) + except (OSError, subprocess.SubprocessError): + pass + returncode = process.returncode if process and process.returncode is not None else 125 + (evidence / "adapter-error.txt").write_text(str(exc) + "\n") + + elapsed = round(time.monotonic() - started, 3) + if runtime_plan.exists(): + shutil.copy2(runtime_plan, evidence / "PLAN.after.md") + if external_plan: + shutil.copy2(runtime_plan, plan) + if external_plan: + if original_runtime_plan.exists(): + shutil.copy2(original_runtime_plan, runtime_plan) + else: + runtime_plan.unlink(missing_ok=True) + + state = parse_state(selected_state) if selected_state else {} + review_id = state.get("review_id") + valid_review_id = bool(review_id and re.fullmatch(r"[0-9]{8}-[0-9]{6}-[0-9a-f]{6}", review_id)) + findings_files: list[Path] = [] + findings_dir: Path | None = None + evidence_state: Path | None = None + evidence_review_dir: Path | None = None + copy_errors: list[str] = [] + if valid_review_id: + findings_dir = state_dir / str(review_id) + + def finding_round(path: Path) -> int: + match = re.search(r"findings-round-(\d+)\.md$", path.name) + return int(match.group(1)) if match else -1 + + findings_files = sorted(findings_dir.glob("findings-round-*.md"), key=finding_round) if findings_dir.is_dir() else [] + if selected_state: + evidence_state = evidence / "artifacts" / selected_state.name + try: + copy_if_exists(selected_state, evidence_state) + except OSError as exc: + copy_errors.append(f"state evidence copy failed: {exc}") + evidence_review_dir = evidence / "artifacts" / str(review_id) + try: + copy_if_exists(findings_dir, evidence_review_dir) + except OSError as exc: + copy_errors.append(f"generation evidence copy failed: {exc}") + try: + copy_if_exists(state_dir / "log", evidence / "artifacts" / "claudex.log") + except OSError as exc: + copy_errors.append(f"log evidence copy failed: {exc}") + + final_findings = findings_files[-1] if findings_files else None + findings_status, severity = final_findings_status(final_findings) + phase = state.get("phase") + decision = state.get("decision_signal") + sweep: dict[str, Any] | None = None + if timed_out: + outcome = "timed_out" + reason = "wall-clock timeout exceeded; process group terminated and reaped" + elif returncode != 0: + outcome = "failed" + reason = f"Claude Code exited non-zero ({returncode})" + elif not selected_state or not state: + outcome = "degraded" + reason = "Claude exited zero but no new Claudex state artifact was found" + elif not valid_review_id or findings_dir is None: + outcome = "degraded" + reason = "Claudex state contains a missing or malformed review_id" + elif copy_errors: + outcome = "degraded" + reason = "required adapter evidence could not be copied: " + "; ".join(copy_errors) + elif args.engine == "sweep-v2": + sweep = validate_sweep(selected_state, findings_dir, runtime_plan, evidence / "PLAN.after.md", args.rounds) + state = sweep["state"] + phase = state.get("phase") + decision = state.get("decision_signal") + outcome = sweep["outcome"] + reason = sweep["reason"] + findings_status = sweep["findings_status"] + severity = sweep["severity_counts"] + elif decision == "no-material-findings" and phase == "done" and findings_status == "none": + outcome = "converged" + reason = "terminal legacy state and final findings agree that no substantive findings remain" + elif decision == "max-reached" and phase == "done" and findings_status in {"material", "unparseable"}: + outcome = "max_reached" + reason = "legacy round budget exhausted without an authoritative clean findings artifact" + else: + outcome = "degraded" + reason = f"legacy state/artifact mismatch or incomplete lifecycle (phase={phase}, signal={decision}, findings={findings_status})" + + copied_final = None + if final_findings and evidence_review_dir: + copied_final = evidence_review_dir / final_findings.name + copied_consolidated = None + copied_manifest = None + copied_generation_dir = None + if sweep and evidence_review_dir and sweep.get("generation"): + copied_generation_dir = evidence_review_dir / "generations" / str(sweep["generation"]) + copied_consolidated = copied_generation_dir / "consolidated-findings.md" + copied_manifest = copied_generation_dir / "manifest.json" + + stream_summary = parse_claude_stream(stdout_path) + result = { + "schema_version": 1, + "outcome": outcome, + "clean": outcome == "converged", + "reason": reason, + "engine": args.engine, + "repo": str(repo), + "plan": str(plan), + "review_id": review_id, + "phase": phase, + "decision_signal": decision, + "generation": sweep.get("generation") if sweep else None, + "max_generations": sweep.get("max_generations") if sweep else None, + "snapshot_sha256": sweep.get("snapshot_sha256") if sweep else None, + "converged_snapshot_sha256": sweep.get("converged_snapshot_sha256") if sweep else None, + "persona_coverage": sweep.get("persona_coverage", []) if sweep else [], + "generation_manifest": str(copied_manifest) if copied_manifest else None, + "generation_evidence_dir": str(copied_generation_dir) if copied_generation_dir else None, + "consolidated_findings": str(copied_consolidated) if copied_consolidated else None, + "round": int(state["round"]) if args.engine == "legacy" and state.get("round", "").isdigit() else None, + "max_rounds": int(state["max_rounds"]) if args.engine == "legacy" and state.get("max_rounds", "").isdigit() else (args.rounds if args.engine == "legacy" else None), + "findings_status": findings_status, + "severity_counts": severity, + "final_findings": str(final_findings) if final_findings else None, + "evidence_final_findings": str(copied_final) if copied_final else None, + "evidence_dir": str(evidence), + "stdout_log": str(stdout_path), + "stderr_log": str(stderr_path), + "state_file": str(selected_state) if selected_state else None, + "evidence_state_file": str(evidence_state) if evidence_state else None, + "source_state_file": str(selected_state) if selected_state else None, + "elapsed_seconds": elapsed, + "process_exit_code": returncode, + "budget_usd": str(budget), + "reported_claude_cost_usd": stream_summary.get("reported_cost_usd"), + "claude_session_id": stream_summary.get("session_id"), + } + (evidence / "result.json").write_text(json.dumps(result, indent=2, sort_keys=True) + "\n") + return emit(result, EXIT_CODES[outcome]) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docs/HEADLESS_ADAPTER.md b/docs/HEADLESS_ADAPTER.md new file mode 100644 index 0000000..a3e0a6b --- /dev/null +++ b/docs/HEADLESS_ADAPTER.md @@ -0,0 +1,68 @@ +# Headless Claudex planning bridge + +`bin/claudex-plan-review` is a bounded adapter for a Hermes leaf subagent (or any automation runner) to drive Claude Code → Claudex Stop-hook → Codex plan review. Its default engine is the feature-flagged Phase 1 `sweep-v2` lifecycle; `--engine legacy` preserves the original round-based adapter behavior. + +## Architecture + +1. The caller supplies absolute paths for a trusted git repository, an existing non-empty plan, the Claudex plugin, Claude Code, and Codex. +2. The adapter validates paths, plugin files, engine bounds, versions, and both CLI authentication states. It creates no global installation and uses `--plugin-dir` for this Claude session only. +3. If the supplied plan is outside the repository, it is staged as `/PLAN.md`, reviewed, copied back, and any pre-existing repository plan is restored. +4. Claude Code runs headlessly with `/claudex:plan --engine sweep-v2 --from-draft --skip-interview --rounds N`, stream-JSON output, hook events, an explicit telemetry budget, and a pinned child `PATH`. +5. The supervisor polls newly created `.claude/claudex/*.state` files, records transitions, and preserves the complete sweep generation directory and state under adapter evidence. +6. A wall-clock timeout terminates the entire process group, waits five seconds, escalates to `SIGKILL`, and reaps it. +7. Classification independently revalidates authoritative state, manifest, frozen/live snapshot hashes, all five persona sidecars/findings, aggregate evidence hash, and consolidated findings. Claude prose is never authoritative. + +Interactive review mode and the plugin's legacy plan mode are unchanged. + +## Exact usage + +```bash +/path/to/claudex/bin/claudex-plan-review \ + --repo /absolute/path/to/project \ + --plan /absolute/path/to/project/PLAN.md \ + --topic "grounded feature scope, constraints, and explicit non-goals" \ + --engine sweep-v2 \ + --rounds 5 \ + --timeout 900 \ + --budget-usd 5.00 \ + --plugin-root /absolute/path/to/claudex/plugins/claudex \ + --claude /absolute/path/to/claude \ + --codex /absolute/path/to/codex \ + --output-dir /absolute/new/evidence-directory +``` + +`--engine` defaults to `sweep-v2`; its `--rounds` value is the maximum generation count and must be `1..5`. The Hermes project-plan-review workflow uses five. Pass `--engine legacy` for backward-compatible round behavior. `--output-dir` is optional; the default is `.claude/claudex/adapter-runs/-`. `--model` defaults to `sonnet`. + +Stdout contains exactly one compact JSON object. Diagnostics go to evidence files. Exit codes: + +| Outcome | Exit | Clean | Meaning | +|---|---:|---:|---| +| `converged` | 0 | yes | State is done/converged/clean/coverage-complete and exactly five required personas have valid clean evidence against the same snapshot; manifest, hashes, and consolidated findings all agree. | +| `max_reached` | 10 | no | The generation cap ended with complete authoritative same-snapshot evidence containing material findings. | +| `degraded` | 11 | no | State/evidence is incomplete, contradictory, missing, malformed, mutated, hash-mismatched, cancelled, or otherwise non-authoritative. | +| `failed` | 12 | no | Validation, prerequisite/auth, launch, or Claude execution failed. | +| `timed_out` | 124 | no | The deadline expired; the complete process group was killed and reaped. | + +Only `outcome=converged` with `clean=true` is a success gate. Generation five material findings are `max_reached`, never clean. One final findings file is not sufficient evidence for sweep-v2 convergence. + +## Machine-readable evidence + +Sweep-v2 results expose: + +- `engine`, `generation`, and `max_generations` (legacy-only `round`/`max_rounds` remain for compatibility); +- `snapshot_sha256` and `converged_snapshot_sha256`; +- ordered `persona_coverage` for the exact five personas; +- copied `evidence_state_file`, `generation_manifest`, `generation_evidence_dir`, and `consolidated_findings` paths (`state_file` and `final_findings` retain legacy source-path semantics); +- honest `outcome`, `clean`, `reason`, findings classification/severity, process exit, elapsed time, and telemetry. + +The state and full review directory—including every generation—are copied into `/artifacts/` before result emission, so read-back does not depend on later `.claude/claudex` cleanup. `source_state_file` is informational; gates should use the copied paths. + +## Subscription usage and telemetry + +Claude Code and Codex are subscription-backed in this workflow. `--budget-usd`, Claude's `--max-budget-usd`, and `reported_claude_cost_usd` are CLI usage-equivalent telemetry and a bounded-run control; they do not prove direct API billing or represent an invoice. Codex subscription usage is separate, may be rate-limited, and cannot be dollar-enforced or measured by this adapter. A sweep generation can invoke all five Codex personas. The timeout is a wall-clock safety boundary, not a billing guarantee. + +For Hermes delegation, keep the outer child deadline at least 180 seconds beyond the adapter timeout so the leaf can read and report evidence. + +## Installation and Hermes skill staging + +No installation is required to run from a checkout; keep executable paths explicit. The staged Hermes skill lives at `skills/project-plan-review/`. Review it in-repo first. Installing it later requires separate operator approval. This repository does **not** install the skill, alter Hermes configuration, or change the active Claude plugin installation. diff --git a/install.sh b/install.sh index 8de70c1..a508faa 100755 --- a/install.sh +++ b/install.sh @@ -130,10 +130,10 @@ note "After /reload-plugins, /claudex should appear in your slash command list." # ───────────────────────────────────────────── hdr "7. Platform validation" # ───────────────────────────────────────────── -if [ -x "$PLUGIN_ROOT/tests/platform-validation.sh" ]; then +if [ -x "$PLUGIN_ROOT/plugins/claudex/tests/platform-validation.sh" ]; then echo "Running platform-validation.sh..." echo "" - if bash "$PLUGIN_ROOT/tests/platform-validation.sh" 2>&1 | tail -5; then + if bash "$PLUGIN_ROOT/plugins/claudex/tests/platform-validation.sh" 2>&1 | tail -5; then : else failures=$((failures+1)) diff --git a/plugins/claudex/commands/plan.md b/plugins/claudex/commands/plan.md index ff08f89..51add06 100644 --- a/plugins/claudex/commands/plan.md +++ b/plugins/claudex/commands/plan.md @@ -1,6 +1,6 @@ --- description: Run an autonomous plan-and-review loop. Claude drafts PLAN.md, Codex grills it adversarially, Claude revises until LGTM or N rounds. -argument-hint: '[--rounds N] [--from-draft] [--skip-interview] ' +argument-hint: '[--engine sweep-v2] [--rounds N] [--from-draft] [--skip-interview] ' allowed-tools: Bash, Read, Write, Edit, AskUserQuestion --- @@ -17,6 +17,7 @@ Parse these flags from the start of $ARGUMENTS (the script handles them; you mai - `--rounds N`. Override the default max rounds (3). Common picks: 3 (default, fast), 5 (deeper grilling), 7+ (very high stakes). - `--from-draft`. Use the existing `PLAN.md` in the project root instead of drafting from scratch. PLAN.md must exist and be non-empty. - `--skip-interview`. Bypass the topic-sharpening interview offer in step 2 below. Useful when you've already nailed the topic or you're in a rush. +- `--engine sweep-v2`. Opt into the Phase 1 frozen-snapshot engine. It requires an existing non-empty `PLAN.md`, runs all five required personas sequentially per generation, defaults to five generations, and never permits more than five. Omit this flag for unchanged legacy plan mode. ## Procedure @@ -74,7 +75,7 @@ Run start-loop.sh. **CRITICAL: the topic MUST be passed as a single double-quote Compose the bash command this way: -1. Identify any flags from `$ARGUMENTS`: `--rounds N`, `--from-draft`. (`--skip-interview` was already consumed in step 2.) +1. Identify any flags from `$ARGUMENTS`: `--engine sweep-v2`, `--rounds N`, `--from-draft`. (`--skip-interview` was already consumed in step 2.) 2. Identify the topic: everything that isn't a recognized flag. 3. Pass flags as-is (no quoting needed). Pass the topic as ONE double-quoted argument. @@ -86,6 +87,7 @@ bash "${CLAUDE_PLUGIN_ROOT}/scripts/start-loop.sh" plan [flags] "" bash "${CLAUDE_PLUGIN_ROOT}/scripts/start-loop.sh" plan "add expiry dates to my links" bash "${CLAUDE_PLUGIN_ROOT}/scripts/start-loop.sh" plan --rounds 5 "migrate auth to Clerk's new API" bash "${CLAUDE_PLUGIN_ROOT}/scripts/start-loop.sh" plan --from-draft "refactor the billing pipeline" +bash "${CLAUDE_PLUGIN_ROOT}/scripts/start-loop.sh" plan --engine sweep-v2 "review the existing frozen plan" # With interview, the enriched topic is also double-quoted: bash "${CLAUDE_PLUGIN_ROOT}/scripts/start-loop.sh" plan [flags] --interviewed "" diff --git a/plugins/claudex/docs/ARCHITECTURE.md b/plugins/claudex/docs/ARCHITECTURE.md index a6dfe4d..56ec33b 100644 --- a/plugins/claudex/docs/ARCHITECTURE.md +++ b/plugins/claudex/docs/ARCHITECTURE.md @@ -52,11 +52,12 @@ USER: /claudex:plan add expiry dates ┌─────────────────────┐ │ stop-hook.sh │ Reads state file └─────────────────────┘ phase=reviewing - │ → If signal=no-material-findings: ALLOW + cleanup - │ → If round >= max: ALLOW + "stopped at max" + │ → If signal=no-material-findings: summary BLOCK + │ → If round >= max: summary BLOCK │ → Else: increment round, BLOCK with new round ▼ - CLAUDE either revises PLAN.md (loops back) OR exits cleanly + CLAUDE either revises PLAN.md (loops back) OR prints the summary + Next Stop hook: summarizing → done → APPROVE + cleanup ``` ## The state file @@ -116,7 +117,8 @@ Every error path leads to approve. The plugin is not allowed to break the user's The hook branches on `mode` (plan or review) and then on `phase`. Plan mode has the most states: - `drafting` → check PLAN.md, transition to reviewing, write runner, BLOCK -- `reviewing` → check decision_signal, either ALLOW (done/max) or increment round + BLOCK +- `reviewing` → check decision_signal, either transition to `summarizing` + BLOCK or increment round + BLOCK +- `summarizing` → transition to `done`, cleanup + ALLOW - `done` → cleanup + ALLOW - `cancelled` → cleanup + ALLOW @@ -141,7 +143,7 @@ The runner script includes: The challenge: hooks fire AFTER Claude finishes a turn. So how does Claude tell the hook "the loop should end now"? -Answer: Claude updates the state file before ending its turn. Specifically, Claude runs `mark-done.sh` which sets `phase=done` and `decision_signal=no-material-findings`. The hook reads those fields on its next fire and ALLOWs exit. +Answer: Claude updates the state file before ending its turn. Specifically, Claude runs `mark-done.sh`, which leaves `phase=reviewing` and sets `decision_signal=no-material-findings`. The hook reads those fields on its next fire, transitions to `summarizing`, and BLOCKs once so Claude prints the user-visible summary. The following fire transitions to `done` and ALLOWs exit. This is just a state machine using the file system as a synchronization channel. Simple. Robust. Survives crashes. diff --git a/plugins/claudex/hooks/stop-hook.sh b/plugins/claudex/hooks/stop-hook.sh index 7e7b7b1..eaded19 100755 --- a/plugins/claudex/hooks/stop-hook.sh +++ b/plugins/claudex/hooks/stop-hook.sh @@ -52,6 +52,8 @@ trap 'log "ERR trap at line $LINENO; failing open"; printf "{\"decision\":\"appr source "$CLAUDE_PLUGIN_ROOT/scripts/state-helpers.sh" 2>/dev/null || approve "state-helpers missing" # shellcheck source=/dev/null source "$CLAUDE_PLUGIN_ROOT/scripts/personas.sh" 2>/dev/null || approve "personas missing" +# shellcheck source=/dev/null +source "$CLAUDE_PLUGIN_ROOT/scripts/sweep-helpers.sh" 2>/dev/null || approve "sweep helpers missing" # Read hook input from stdin (Claude Code sends JSON). HOOK_INPUT="" @@ -80,6 +82,8 @@ fi # Read state fields. MODE=$(claudex_state_read_field "$ACTIVE_STATE" "mode") +ENGINE=$(claudex_state_read_field "$ACTIVE_STATE" "engine") +[ -n "$ENGINE" ] || ENGINE="legacy" PHASE=$(claudex_state_read_field "$ACTIVE_STATE" "phase") ROUND=$(claudex_state_read_field "$ACTIVE_STATE" "round") MAX_ROUNDS=$(claudex_state_read_field "$ACTIVE_STATE" "max_rounds") @@ -90,9 +94,11 @@ STARTED_AT_EPOCH=$(claudex_state_read_field "$ACTIVE_STATE" "started_at_epoch") log "State: mode=$MODE phase=$PHASE round=$ROUND/$MAX_ROUNDS signal=$DECISION_SIGNAL" -# Sanity: if cwd doesn't match the repo where the loop started, fail-open. -if [ -n "$REPO_ROOT_STATE" ] && [ "$REPO_ROOT_STATE" != "$(pwd)" ]; then - log "cwd mismatch (state=$REPO_ROOT_STATE, here=$(pwd)); fail-open" +# Sanity: compare canonical physical paths so symlink/casing aliases do not +# fail-open a loop that is still running in the same repository. +CURRENT_REPO_ROOT="$(pwd -P)" +if [ -n "$REPO_ROOT_STATE" ] && [ "$REPO_ROOT_STATE" != "$CURRENT_REPO_ROOT" ]; then + log "cwd mismatch (state=$REPO_ROOT_STATE, here=$CURRENT_REPO_ROOT); fail-open" approve "cwd mismatch" fi @@ -200,7 +206,208 @@ RUNNEREOF chmod +x "$RUNNER" } -# === PLAN MODE LIFECYCLE === +# === SWEEP-V2 PLAN LIFECYCLE === + +if [ "$MODE" = "plan" ] && [ "$ENGINE" = "sweep-v2" ]; then + GENERATION=$(claudex_state_read_field "$ACTIVE_STATE" generation) + MAX_GENERATIONS=$(claudex_state_read_field "$ACTIVE_STATE" max_generations) + SNAPSHOT_SHA=$(claudex_state_read_field "$ACTIVE_STATE" snapshot_sha256) + case "$GENERATION" in ''|*[!0-9]*) GENERATION=1 ;; esac + case "$MAX_GENERATIONS" in ''|*[!0-9]*) MAX_GENERATIONS=5 ;; esac + [ "$MAX_GENERATIONS" -le 5 ] || MAX_GENERATIONS=5 + GENERATION_DIR="$REVIEW_DIR/generations/$GENERATION" + CONSOLIDATED="$GENERATION_DIR/consolidated-findings.md" + + case "$PHASE" in + reviewing) + block "### Claudex sweep-v2 generation $GENERATION of $MAX_GENERATIONS + +All five required personas will run sequentially against one frozen snapshot. + +**Snapshot:** \`$GENERATION_DIR/PLAN.md\` +**SHA-256:** \`$SNAPSHOT_SHA\` + +Run the deterministic runner: + +\`\`\` +bash $RUNNER +\`\`\` + +Do not edit the snapshot or live \`PLAN.md\` while it runs. When it finishes, end your turn." + ;; + + awaiting-revision) + CURRENT_LIVE_SHA=$(claudex_sha256 PLAN.md 2>/dev/null) + if [ -z "$CURRENT_LIVE_SHA" ] || [ "$CURRENT_LIVE_SHA" = "$SNAPSHOT_SHA" ]; then + block "Sweep-v2 found material issues in generation $GENERATION. Read \`$CONSOLIDATED\`, revise live \`PLAN.md\` exactly once, and add or update \`## Changelog\` recording each accepted or rejected item with reasons. Do not modify the frozen snapshot. Then end your turn." + fi + if claudex_sweep_consolidate "$ACTIVE_STATE" "$REVIEW_ID" "$GENERATION" "$SNAPSHOT_SHA" "$CURRENT_LIVE_SHA" >/dev/null 2>&1; then + RECONCILIATION_RC=0 + else + RECONCILIATION_RC=$? + fi + if [ "$(claudex_state_read_field "$ACTIVE_STATE" phase)" = cancelled ]; then + approve "sweep-v2 cancelled during evidence revalidation" + fi + if [ "$RECONCILIATION_RC" -ne 1 ]; then + block "Sweep-v2 could not revalidate generation-$GENERATION evidence before accepting the revision. The generation is degraded and cannot advance; end your turn for the terminal summary or cancel the loop." + fi + if ! claudex_sweep_validate_reconciliation PLAN.md "$CONSOLIDATED" "$GENERATION" "$SNAPSHOT_SHA"; then + block "The required plan revision exists, but its \`## Changelog\` does not reconcile every generation-$GENERATION finding. Add this exact heading under \`## Changelog\`: + +\`### Sweep generation $GENERATION — $SNAPSHOT_SHA\` + +Then add exactly one disposition for every ID in \`$CONSOLIDATED\` using: + +\`- Accepted [finding-id]: reason and resulting plan change\` +\`- Rejected [finding-id]: grounded reason\` + +Do not advance until every material finding ID has one reasoned disposition." + fi + NEW_GENERATION=$((GENERATION + 1)) + if [ "$NEW_GENERATION" -gt "$MAX_GENERATIONS" ] || [ "$NEW_GENERATION" -gt 5 ]; then + claudex_state_set_field "$ACTIVE_STATE" decision_signal max-reached + claudex_state_set_field "$ACTIVE_STATE" clean false + claudex_state_set_field "$ACTIVE_STATE" phase summarizing + block "Sweep-v2 reached its hard generation limit without unanimous clean coverage. End your turn to receive the terminal summary." + fi + NEW_SHA=$(claudex_sweep_create_generation "$ACTIVE_STATE" "$REVIEW_ID" "$NEW_GENERATION" "$TOPIC" "$SNAPSHOT_SHA") || { + claudex_state_set_field "$ACTIVE_STATE" decision_signal degraded + claudex_state_set_field "$ACTIVE_STATE" clean false + claudex_state_set_field "$ACTIVE_STATE" phase summarizing + block "Sweep-v2 degraded while creating generation $NEW_GENERATION. No clean result is claimed. End your turn for the terminal summary." + } + claudex_state_set_field "$ACTIVE_STATE" round "$NEW_GENERATION" + claudex_state_set_field "$ACTIVE_STATE" phase reviewing + claudex_sweep_write_runner "$ACTIVE_STATE" "$REVIEW_ID" "$NEW_GENERATION" "$TOPIC" "$NEW_SHA" || { + claudex_state_set_field "$ACTIVE_STATE" decision_signal degraded + claudex_state_set_field "$ACTIVE_STATE" clean false + claudex_state_set_field "$ACTIVE_STATE" phase summarizing + block "Sweep-v2 degraded while writing the generation runner. No clean result is claimed." + } + block "### Claudex sweep-v2 generation $NEW_GENERATION of $MAX_GENERATIONS + +The revised plan was frozen as a new immutable snapshot. + +**Snapshot:** \`$REVIEW_DIR/generations/$NEW_GENERATION/PLAN.md\` +**SHA-256:** \`$NEW_SHA\` + +Run all five personas sequentially: + +\`\`\` +bash $RUNNER +\`\`\` + +When the runner finishes, end your turn." + ;; + + summarizing) + # Revalidate every current-generation artifact at summary time so a + # post-run mutation cannot ride a previously clean state signal. Clear + # the prior verdict first so an I/O/consolidation failure fails closed, + # but never overwrite a cancellation that won the state-write race. + if [ "$(claudex_state_read_field "$ACTIVE_STATE" phase)" = cancelled ]; then + approve "sweep-v2 cancelled before terminal revalidation" + fi + if ! claudex_sweep_set_fields_atomic "$ACTIVE_STATE" --expect-phase summarizing \ + decision_signal degraded clean false coverage_complete false; then + if [ "$(claudex_state_read_field "$ACTIVE_STATE" phase)" = cancelled ]; then + approve "sweep-v2 cancelled before fail-closed revalidation" + fi + block "Sweep-v2 could not persist its fail-closed revalidation state. No clean result is claimed; repair the state directory and retry or cancel." + fi + if claudex_sweep_consolidate "$ACTIVE_STATE" "$REVIEW_ID" "$GENERATION" "$SNAPSHOT_SHA" "$SNAPSHOT_SHA" >/dev/null 2>&1; then + REVALIDATE_RC=0 + else + REVALIDATE_RC=$? + fi + if [ "$(claudex_state_read_field "$ACTIVE_STATE" phase)" = cancelled ]; then + approve "sweep-v2 cancelled during terminal revalidation" + fi + case "$REVALIDATE_RC" in + 0|2|3) ;; + *) + if ! claudex_sweep_set_fields_atomic "$ACTIVE_STATE" --expect-phase summarizing \ + decision_signal degraded clean false coverage_complete false phase summarizing; then + if [ "$(claudex_state_read_field "$ACTIVE_STATE" phase)" = cancelled ]; then + approve "sweep-v2 cancelled while persisting degraded revalidation" + fi + block "Sweep-v2 revalidation failed and its degraded verdict could not be persisted. No clean result is claimed." + fi + ;; + esac + SIGNAL=$(claudex_state_read_field "$ACTIVE_STATE" decision_signal) + CLEAN=$(claudex_state_read_field "$ACTIVE_STATE" clean) + COVERAGE_COMPLETE=$(claudex_state_read_field "$ACTIVE_STATE" coverage_complete) + if [ "$SIGNAL" = "converged" ] && { [ "$REVALIDATE_RC" -ne 0 ] || [ "$CLEAN" != "true" ] || [ "$COVERAGE_COMPLETE" != "true" ]; }; then + if ! claudex_sweep_set_fields_atomic "$ACTIVE_STATE" --expect-phase summarizing \ + decision_signal degraded clean false coverage_complete false; then + if [ "$(claudex_state_read_field "$ACTIVE_STATE" phase)" = cancelled ]; then + approve "sweep-v2 cancelled while correcting inconsistent convergence" + fi + block "Sweep-v2 could not persist an inconsistent-convergence correction. No clean result is claimed." + fi + SIGNAL=degraded + CLEAN=false + COVERAGE_COMPLETE=false + fi + CONVERGED_SHA=$(claudex_state_read_field "$ACTIVE_STATE" converged_snapshot_sha256) + if ! claudex_sweep_set_fields_atomic "$ACTIVE_STATE" --expect-phase summarizing phase done; then + if [ "$(claudex_state_read_field "$ACTIVE_STATE" phase)" = cancelled ]; then + approve "sweep-v2 cancelled before terminal summary commit" + fi + block "Sweep-v2 could not commit its terminal summary state. No result is claimed." + fi + rm -f "$RUNNER" "$STATE_DIR/$REVIEW_ID-prompt.txt" "$STATE_DIR/$REVIEW_ID.lock" "$STATE_DIR/$REVIEW_ID-active-pgid" 2>/dev/null + case "$SIGNAL" in + converged) + block "### Claudex sweep-v2 complete ✓ + +All five required personas returned exact clean findings against the same generation-$GENERATION snapshot hash. + +**Converged SHA-256:** \`$CONVERGED_SHA\` +**Coverage complete:** $COVERAGE_COMPLETE +**Clean:** $CLEAN + +Print this summary to the user, then end your turn." + ;; + max-reached) + block "### Claudex sweep-v2 stopped at max generations + +Generation $GENERATION still had material findings. The result is terminal and explicitly not clean; no unreviewed revision is accepted. + +**Consolidated findings:** \`$CONSOLIDATED\` +**Clean:** false + +Print this summary to the user, then end your turn." + ;; + *) + block "### Claudex sweep-v2 degraded + +A required persona artifact was missing, malformed, nonzero, hash-mismatched, or mutation was detected. Coverage is incomplete and no clean result is claimed. + +**Generation:** $GENERATION +**Consolidated findings:** \`$CONSOLIDATED\` +**Clean:** false + +Print this summary to the user, then end your turn." + ;; + esac + ;; + + done) + rm -f "$RUNNER" "$STATE_DIR/$REVIEW_ID-prompt.txt" "$STATE_DIR/$REVIEW_ID.lock" "$STATE_DIR/$REVIEW_ID-active-pgid" 2>/dev/null + approve "sweep-v2 loop done" + ;; + + *) + log "Unknown sweep-v2 phase: $PHASE" + approve "unknown sweep-v2 phase, fail-open" + ;; + esac +fi + +# === LEGACY PLAN MODE LIFECYCLE === if [ "$MODE" = "plan" ]; then case "$PHASE" in @@ -470,7 +677,7 @@ Findings summary will be written to: # Summary BLOCK was delivered on the previous fire and Claude has # printed it to the user. Final cleanup and approve. claudex_phase_transition "$ACTIVE_STATE" "summarizing" "done" 2>/dev/null - rm -f "$RUNNER" "$STATE_DIR/$REVIEW_ID-prompt.txt" "$STATE_DIR/$REVIEW_ID.lock" 2>/dev/null + rm -f "$RUNNER" "$STATE_DIR/$REVIEW_ID-prompt.txt" "$STATE_DIR/$REVIEW_ID.lock" "$STATE_DIR/$REVIEW_ID-active-pgid" 2>/dev/null ELAPSED=$(format_elapsed "$STARTED_AT_EPOCH") if [ -n "$ELAPSED" ]; then log "Plan loop $REVIEW_ID summary delivered; total elapsed $ELAPSED" @@ -479,7 +686,7 @@ Findings summary will be written to: ;; done) - rm -f "$RUNNER" "$STATE_DIR/$REVIEW_ID-prompt.txt" "$STATE_DIR/$REVIEW_ID.lock" 2>/dev/null + rm -f "$RUNNER" "$STATE_DIR/$REVIEW_ID-prompt.txt" "$STATE_DIR/$REVIEW_ID.lock" "$STATE_DIR/$REVIEW_ID-active-pgid" 2>/dev/null approve "plan loop already done" ;; @@ -541,7 +748,7 @@ After Codex finishes, end your turn. The hook will allow exit." ;; done) - rm -f "$RUNNER" "$STATE_DIR/$REVIEW_ID-prompt.txt" "$STATE_DIR/$REVIEW_ID.lock" 2>/dev/null + rm -f "$RUNNER" "$STATE_DIR/$REVIEW_ID-prompt.txt" "$STATE_DIR/$REVIEW_ID.lock" "$STATE_DIR/$REVIEW_ID-active-pgid" 2>/dev/null approve "review loop done" ;; diff --git a/plugins/claudex/scripts/cancel-loop.sh b/plugins/claudex/scripts/cancel-loop.sh index 98b1e05..10358e8 100755 --- a/plugins/claudex/scripts/cancel-loop.sh +++ b/plugins/claudex/scripts/cancel-loop.sh @@ -16,14 +16,47 @@ fi REVIEW_ID=$(basename "$ACTIVE" .state) echo "Cancelling loop: $REVIEW_ID" +ENGINE=$(claudex_state_read_field "$ACTIVE" engine) +if [ "$ENGINE" = "sweep-v2" ]; then + # Share the same advisory write lock as verdict persistence so cancellation + # cannot be overwritten by a racing whole-state replacement. + # shellcheck source=/dev/null + source "$CLAUDE_PLUGIN_ROOT/scripts/sweep-helpers.sh" + claudex_sweep_set_fields_atomic "$ACTIVE" \ + phase cancelled decision_signal cancelled clean false coverage_complete false revision_required false || { + echo "Could not persist cancellation." >&2 + exit 1 + } +else + claudex_state_set_field "$ACTIVE" "phase" "cancelled" + claudex_state_set_field "$ACTIVE" "last_updated_at" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" +fi -claudex_state_set_field "$ACTIVE" "phase" "cancelled" -claudex_state_set_field "$ACTIVE" "last_updated_at" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" +# sweep-v2 runs each Codex reviewer in its own process group. Terminate the +# active group before removing artifacts so cancellation cannot leave an +# orphan reviewer writing into a cancelled generation. +ACTIVE_PGID_FILE="$CLAUDEX_STATE_DIR/$REVIEW_ID-active-pgid" +if [ "$ENGINE" = "sweep-v2" ] && [ -f "$ACTIVE_PGID_FILE" ]; then + ACTIVE_PGID=$(cat "$ACTIVE_PGID_FILE" 2>/dev/null) + case "$ACTIVE_PGID" in + ''|*[!0-9]*) ;; + *) + kill -TERM -- "-$ACTIVE_PGID" 2>/dev/null + i=0 + while kill -0 -- "-$ACTIVE_PGID" 2>/dev/null && [ "$i" -lt 20 ]; do + sleep 0.1 + i=$((i + 1)) + done + kill -KILL -- "-$ACTIVE_PGID" 2>/dev/null + ;; + esac +fi # Clean up runner artifacts but keep state file for log/audit. rm -f "$CLAUDEX_STATE_DIR/$REVIEW_ID-runner.sh" 2>/dev/null rm -f "$CLAUDEX_STATE_DIR/$REVIEW_ID-prompt.txt" 2>/dev/null rm -f "$CLAUDEX_STATE_DIR/$REVIEW_ID.lock" 2>/dev/null +rm -f "$ACTIVE_PGID_FILE" 2>/dev/null echo "Loop cancelled. Stop hook will allow exit on next fire." exit 0 diff --git a/plugins/claudex/scripts/doctor.sh b/plugins/claudex/scripts/doctor.sh index 16100c8..692e380 100755 --- a/plugins/claudex/scripts/doctor.sh +++ b/plugins/claudex/scripts/doctor.sh @@ -3,7 +3,7 @@ # # Verifies that everything claudex needs is wired up correctly: # - bash version -# - python3 (for JSON escape; sed fallback exists but python3 is preferred) +# - python3 (required for sweep-v2 manifests and artifact validation) # - codex CLI installed and responding # - .claude/claudex writable # - plugin file integrity (every expected script + prompt template present) @@ -11,7 +11,6 @@ # - stale loops report (informational, not fatal) # # Exits 0 if every required check passes, 1 if any required check fails. -# Optional checks (e.g. python3) only print warnings. set +e @@ -82,8 +81,9 @@ if command -v codex >/dev/null 2>&1; then fi fi -section "Optional dependencies" -warn_check "python3 (used for JSON escape; sed fallback works)" command -v python3 +section "Required runtime dependencies" +check "python3 (required for sweep-v2 manifests and validation)" command -v python3 +check "SHA-256 implementation (shasum or sha256sum)" bash -c 'command -v shasum >/dev/null 2>&1 || command -v sha256sum >/dev/null 2>&1' section "State directory" mkdir -p "$CLAUDEX_STATE_DIR" 2>/dev/null @@ -96,6 +96,7 @@ PLUGIN_FILES=( "hooks/hooks.json" "scripts/state-helpers.sh" "scripts/personas.sh" + "scripts/sweep-helpers.sh" "scripts/start-loop.sh" "scripts/cancel-loop.sh" "scripts/rollback-loop.sh" @@ -140,6 +141,8 @@ check "state-helpers source cleanly" \ bash -c "source '$CLAUDE_PLUGIN_ROOT/scripts/state-helpers.sh'" check "personas source cleanly" \ bash -c "source '$CLAUDE_PLUGIN_ROOT/scripts/personas.sh'" +check "sweep helpers source cleanly" \ + bash -c "source '$CLAUDE_PLUGIN_ROOT/scripts/sweep-helpers.sh'" if source "$CLAUDE_PLUGIN_ROOT/scripts/personas.sh" 2>/dev/null; then R1=$(claudex_persona_for_round 1) R2=$(claudex_persona_for_round 2) diff --git a/plugins/claudex/scripts/personas.sh b/plugins/claudex/scripts/personas.sh index 27699c0..f6f3f92 100755 --- a/plugins/claudex/scripts/personas.sh +++ b/plugins/claudex/scripts/personas.sh @@ -14,6 +14,23 @@ # claudex_persona_label_for_round # Prints a one-line label for BLOCK headers (e.g. "Security review"). +# sweep-v2 uses complete coverage on every generation. Keep these IDs in one +# stable order because runner execution, consolidation, and convergence all +# depend on it. +CLAUDEX_SWEEP_PERSONAS="architecture-scope security-data product-domain quality-accessibility-performance operations-deployment" + +claudex_sweep_persona_prompt() { + local persona="$1" + case "$persona" in + architecture-scope) printf '%s' 'Review architecture, scope boundaries, dependencies, compatibility, and whether the plan solves the stated problem without hidden design gaps.' ;; + security-data) printf '%s' 'Review authorization, input boundaries, secrets, privacy, concurrency, idempotency, recovery, and data integrity.' ;; + product-domain) printf '%s' 'Review product behavior, domain rules, user journeys, acceptance criteria, and credible business edge cases.' ;; + quality-accessibility-performance) printf '%s' 'Review test strategy, accessibility, performance, resource bounds, failure visibility, and quality gates proportionate to the change.' ;; + operations-deployment) printf '%s' 'Review rollout, rollback, migrations, observability, operational ownership, version skew, and deployment failure modes.' ;; + *) return 1 ;; + esac +} + claudex_persona_label_for_round() { local r="${1:-1}" case "$r" in diff --git a/plugins/claudex/scripts/start-loop.sh b/plugins/claudex/scripts/start-loop.sh index 862ba16..ab1e7f5 100755 --- a/plugins/claudex/scripts/start-loop.sh +++ b/plugins/claudex/scripts/start-loop.sh @@ -31,9 +31,11 @@ shift || true # command; accepted here as a no-op so it's safe to pass through) # --interviewed marker passed by the slash command after a successful interview; # records interview_used=true in state for status/audit +# --engine sweep-v2 use the opt-in frozen-snapshot five-persona engine FROM_DRAFT=false CUSTOM_ROUNDS="" INTERVIEW_USED=false +ENGINE="legacy" while [ $# -gt 0 ]; do case "$1" in --rounds) @@ -56,6 +58,15 @@ while [ $# -gt 0 ]; do INTERVIEW_USED=true shift ;; + --engine) + shift + ENGINE="${1:-}" + shift || true + ;; + --engine=*) + ENGINE="${1#--engine=}" + shift + ;; *) break ;; @@ -96,8 +107,34 @@ case "$MODE" in echo "Plan mode requires a topic. Usage: start-loop.sh plan " >&2 exit 2 fi + if [ "$ENGINE" != "legacy" ] && [ "$ENGINE" != "sweep-v2" ]; then + echo "Unknown plan engine: $ENGINE. Use sweep-v2 or omit --engine for legacy mode." >&2 + exit 2 + fi + if [ "$ENGINE" = "sweep-v2" ]; then + if ! command -v python3 >/dev/null 2>&1; then + echo "--engine sweep-v2 requires python3 for manifests and artifact validation." >&2 + exit 2 + fi + if ! command -v shasum >/dev/null 2>&1 && ! command -v sha256sum >/dev/null 2>&1; then + echo "--engine sweep-v2 requires shasum or sha256sum for SHA-256 verification." >&2 + exit 2 + fi + if [ ! -s "PLAN.md" ]; then + echo "--engine sweep-v2 requires an existing non-empty PLAN.md." >&2 + exit 2 + fi + if [ -n "$CUSTOM_ROUNDS" ] && [ "$CUSTOM_ROUNDS" -gt 5 ]; then + echo "sweep-v2 has a hard maximum of five generations." >&2 + exit 2 + fi + fi ;; review) + if [ "$ENGINE" != "legacy" ]; then + echo "--engine only applies to plan mode." >&2 + exit 2 + fi ;; *) echo "Unknown mode: $MODE. Use plan or review." >&2 @@ -142,23 +179,32 @@ LOCK_FILE="$CLAUDEX_STATE_DIR/$REVIEW_ID.lock" NOW="$(date -u +%Y-%m-%dT%H:%M:%SZ)" NOW_EPOCH="$(date -u +%s)" -REPO_ROOT="$(pwd)" +REPO_ROOT="$(pwd -P)" SESSION_ID="${CLAUDE_SESSION_ID:-unknown}" MAX_PLAN_ROUNDS="${CLAUDEX_MAX_PLAN_ROUNDS:-3}" MAX_REVIEW_ROUNDS="${CLAUDEX_MAX_REVIEW_ROUNDS:-3}" if [ "$MODE" = "plan" ]; then - MAX_ROUNDS="$MAX_PLAN_ROUNDS" - PHASE="drafting" + if [ "$ENGINE" = "sweep-v2" ]; then + MAX_ROUNDS=5 + PHASE="reviewing" + else + MAX_ROUNDS="$MAX_PLAN_ROUNDS" + PHASE="drafting" + fi else MAX_ROUNDS="$MAX_REVIEW_ROUNDS" PHASE="reviewing" fi -# --rounds flag overrides default max. +# --rounds flag overrides default max (sweep-v2 validated at five or fewer). if [ -n "$CUSTOM_ROUNDS" ]; then MAX_ROUNDS="$CUSTOM_ROUNDS" fi +MAX_GENERATIONS="" +if [ "$ENGINE" = "sweep-v2" ]; then + MAX_GENERATIONS="$MAX_ROUNDS" +fi # Escape topic for YAML (basic; topic is user-provided). # The interview path can produce a multi-line topic ("Scope: ...\nConstraints: ..."); @@ -182,12 +228,50 @@ started_at_epoch: $NOW_EPOCH last_updated_at: $NOW decision_signal: none" +if [ "$ENGINE" = "sweep-v2" ]; then + STATE_CONTENT="$STATE_CONTENT +engine: sweep-v2 +generation: 1 +max_generations: $MAX_GENERATIONS +snapshot_sha256: +coverage_complete: false +clean: false +revision_required: false" +fi + claudex_state_write "$STATE_FILE" "$STATE_CONTENT" || exit 3 claudex_lock_write "$LOCK_FILE" || exit 3 +if [ "$ENGINE" = "sweep-v2" ]; then + # shellcheck source=/dev/null + source "$CLAUDE_PLUGIN_ROOT/scripts/personas.sh" || exit 3 + # shellcheck source=/dev/null + source "$CLAUDE_PLUGIN_ROOT/scripts/sweep-helpers.sh" || exit 3 + SNAPSHOT_SHA=$(claudex_sweep_create_generation "$STATE_FILE" "$REVIEW_ID" 1 "$TOPIC" "") || { + claudex_state_set_field "$STATE_FILE" phase errored + echo "Failed to create immutable sweep-v2 generation 1 snapshot." >&2 + exit 3 + } + claudex_sweep_write_runner "$STATE_FILE" "$REVIEW_ID" 1 "$TOPIC" "$SNAPSHOT_SHA" || { + claudex_state_set_field "$STATE_FILE" phase errored + echo "Failed to create sweep-v2 runner." >&2 + exit 3 + } +fi + # Print initial instructions to stdout. Claude will read these. case "$MODE" in plan) + if [ "$ENGINE" = "sweep-v2" ]; then + echo "Claudex sweep-v2 plan review initialized." + echo "Review ID: $REVIEW_ID" + echo "Topic: $TOPIC" + echo "Generation: 1 of $MAX_GENERATIONS" + echo "Snapshot SHA-256: $SNAPSHOT_SHA" + echo "Frozen snapshot: $CLAUDEX_STATE_DIR/$REVIEW_ID/generations/1/PLAN.md" + echo "End your turn. The Stop hook will provide the deterministic sequential runner command." + exit 0 + fi echo "Claudex plan mode initialized." echo "Review ID: $REVIEW_ID" echo "Topic: $TOPIC" diff --git a/plugins/claudex/scripts/state-helpers.sh b/plugins/claudex/scripts/state-helpers.sh index 4748e14..0a3cfdc 100755 --- a/plugins/claudex/scripts/state-helpers.sh +++ b/plugins/claudex/scripts/state-helpers.sh @@ -20,6 +20,7 @@ CLAUDEX_STATE_DIR="${CLAUDEX_STATE_DIR:-.claude/claudex}" CLAUDEX_STALE_MINUTES="${CLAUDEX_STALE_MINUTES:-15}" +CLAUDEX_SWEEP_V2_STALE_MINUTES="${CLAUDEX_SWEEP_V2_STALE_MINUTES:-120}" claudex_new_review_id() { local ts @@ -78,25 +79,35 @@ claudex_state_set_field() { local field="$2" local value="$3" [ -f "$file" ] || return 1 + echo "$field" | grep -qE '^[A-Za-z_][A-Za-z0-9_]*$' || return 1 local tmp="${file}.tmp.$$" local now now=$(date -u +%Y-%m-%dT%H:%M:%SZ) - if grep -qE "^${field}:" "$file" 2>/dev/null; then - sed -E -e "s/^${field}: .*/${field}: ${value}/" \ - -e "s/^last_updated_at: .*/last_updated_at: ${now}/" \ - "$file" > "$tmp" 2>/dev/null \ - || { rm -f "$tmp"; return 1; } - else - sed -E "s/^last_updated_at: .*/last_updated_at: ${now}/" "$file" > "$tmp" 2>/dev/null \ - || { rm -f "$tmp"; return 1; } - printf '%s: %s\n' "$field" "$value" >> "$tmp" 2>/dev/null \ - || { rm -f "$tmp"; return 1; } - fi - # Don't recursively bump last_updated_at when we ARE setting last_updated_at, - # otherwise the explicit value gets clobbered. - if [ "$field" = "last_updated_at" ]; then - sed -E "s/^last_updated_at: .*/last_updated_at: ${value}/" "$tmp" > "${tmp}.2" 2>/dev/null \ - && mv -f "${tmp}.2" "$tmp" + # State is deliberately a single-line key/value format. Collapse embedded + # CR/LF to spaces, then write with printf instead of interpolating user data + # into sed replacement syntax (where '/', '&', and newlines are special). + value=$(printf '%s' "$value" | tr '\r\n' ' ') + local found=false + local saw_updated=false + : > "$tmp" || return 1 + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in + "$field:"*) + printf '%s: %s\n' "$field" "$value" >> "$tmp" || { rm -f "$tmp"; return 1; } + found=true + [ "$field" = "last_updated_at" ] && saw_updated=true + ;; + last_updated_at:*) + printf 'last_updated_at: %s\n' "$now" >> "$tmp" || { rm -f "$tmp"; return 1; } + saw_updated=true + ;; + *) printf '%s\n' "$line" >> "$tmp" || { rm -f "$tmp"; return 1; } ;; + esac + done < "$file" + [ "$found" = "true" ] || printf '%s: %s\n' "$field" "$value" >> "$tmp" \ + || { rm -f "$tmp"; return 1; } + if [ "$field" != "last_updated_at" ] && [ "$saw_updated" != "true" ]; then + printf 'last_updated_at: %s\n' "$now" >> "$tmp" || { rm -f "$tmp"; return 1; } fi mv -f "$tmp" "$file" 2>/dev/null || { rm -f "$tmp"; return 1; } return 0 @@ -121,13 +132,23 @@ claudex_lock_is_active() { claudex_sweep_stale() { [ -d "$CLAUDEX_STATE_DIR" ] || return 0 - # Find state files older than threshold; remove their state, lock, runner, - # prompt, and per-review findings dir. + # Never reap a loop whose current runner still owns the lock. sweep-v2 gets + # a longer stale window because one generation runs five reviewers and can + # legitimately exceed the legacy 15-minute threshold. find "$CLAUDEX_STATE_DIR" -maxdepth 1 -type f -name "*.state" -mmin "+$CLAUDEX_STALE_MINUTES" 2>/dev/null \ | while read -r f; do local id id=$(basename "$f" .state) - rm -f "$f" "$CLAUDEX_STATE_DIR/${id}.lock" "$CLAUDEX_STATE_DIR/${id}-runner.sh" "$CLAUDEX_STATE_DIR/${id}-prompt.txt" 2>/dev/null + local lock="$CLAUDEX_STATE_DIR/${id}.lock" + if claudex_lock_is_active "$lock"; then + continue + fi + local engine + engine=$(claudex_state_read_field "$f" engine) + if [ "$engine" = "sweep-v2" ] && ! find "$f" -prune -mmin "+$CLAUDEX_SWEEP_V2_STALE_MINUTES" -print 2>/dev/null | grep -q .; then + continue + fi + rm -f "$f" "$CLAUDEX_STATE_DIR/${id}.lock" "$CLAUDEX_STATE_DIR/${id}-runner.sh" "$CLAUDEX_STATE_DIR/${id}-prompt.txt" "$CLAUDEX_STATE_DIR/${id}-active-pgid" "$CLAUDEX_STATE_DIR/${id}.state.write-lock" 2>/dev/null rm -rf "$CLAUDEX_STATE_DIR/${id}" 2>/dev/null done return 0 diff --git a/plugins/claudex/scripts/status.sh b/plugins/claudex/scripts/status.sh index a665c22..d4240ea 100755 --- a/plugins/claudex/scripts/status.sh +++ b/plugins/claudex/scripts/status.sh @@ -38,6 +38,7 @@ fi REVIEW_ID=$(basename "$ACTIVE" .state) MODE=$(claudex_state_read_field "$ACTIVE" "mode") +ENGINE=$(claudex_state_read_field "$ACTIVE" "engine") PHASE=$(claudex_state_read_field "$ACTIVE" "phase") ROUND=$(claudex_state_read_field "$ACTIVE" "round") MAX_ROUNDS=$(claudex_state_read_field "$ACTIVE" "max_rounds") @@ -47,13 +48,18 @@ STARTED=$(claudex_state_read_field "$ACTIVE" "started_at") STARTED_EPOCH=$(claudex_state_read_field "$ACTIVE" "started_at_epoch") INTERVIEW=$(claudex_state_read_field "$ACTIVE" "interview_used") FROM_DRAFT=$(claudex_state_read_field "$ACTIVE" "from_draft") +GENERATION=$(claudex_state_read_field "$ACTIVE" "generation") +MAX_GENERATIONS=$(claudex_state_read_field "$ACTIVE" "max_generations") +SNAPSHOT_SHA=$(claudex_state_read_field "$ACTIVE" "snapshot_sha256") +COVERAGE=$(claudex_state_read_field "$ACTIVE" "coverage_complete") +CLEAN=$(claudex_state_read_field "$ACTIVE" "clean") # Phase color. case "$PHASE" in - drafting|reviewing) PHASE_COLOR="$C_YELLOW" ;; - done) PHASE_COLOR="$C_GREEN" ;; - cancelled|errored) PHASE_COLOR="$C_RED" ;; - *) PHASE_COLOR="$C_DIM" ;; + drafting|reviewing|awaiting-revision|summarizing) PHASE_COLOR="$C_YELLOW" ;; + done) PHASE_COLOR="$C_GREEN" ;; + cancelled|errored) PHASE_COLOR="$C_RED" ;; + *) PHASE_COLOR="$C_DIM" ;; esac # Elapsed. @@ -72,15 +78,15 @@ if [ -n "$STARTED_EPOCH" ]; then esac fi -# Activity inferred from phase. The lock file's PID belongs to the start-loop -# script that has long since exited, so a lock-PID-alive check is never useful -# in claudex's between-turns architecture. Phase is the real liveness signal. +# Activity inferred primarily from phase. sweep-v2 refreshes the lock with the +# active runner PID while reviewers execute; between turns the phase remains +# the authoritative lifecycle signal. case "$PHASE" in - drafting|reviewing) ACTIVITY_LINE="${C_GREEN}active${C_RESET} (loop in progress between turns)" ;; - done) ACTIVITY_LINE="${C_DIM}complete${C_RESET}" ;; - cancelled) ACTIVITY_LINE="${C_RED}cancelled${C_RESET}" ;; - errored) ACTIVITY_LINE="${C_RED}errored${C_RESET}" ;; - *) ACTIVITY_LINE="${C_DIM}unknown${C_RESET}" ;; + drafting|reviewing|awaiting-revision|summarizing) ACTIVITY_LINE="${C_GREEN}active${C_RESET} (loop in progress between turns)" ;; + done) ACTIVITY_LINE="${C_DIM}complete${C_RESET}" ;; + cancelled) ACTIVITY_LINE="${C_RED}cancelled${C_RESET}" ;; + errored) ACTIVITY_LINE="${C_RED}errored${C_RESET}" ;; + *) ACTIVITY_LINE="${C_DIM}unknown${C_RESET}" ;; esac # Runner script. @@ -92,6 +98,7 @@ RUNNER_LINE="${C_DIM}none${C_RESET}" printf '%s%s claudex %s%s\n' "$C_BOLD" "─────" "─────" "$C_RESET" printf ' %-13s %s\n' "review_id" "$REVIEW_ID" printf ' %-13s %s\n' "mode" "$MODE" +[ -n "$ENGINE" ] && printf ' %-13s %s\n' "engine" "$ENGINE" printf ' %-13s %s%s%s\n' "phase" "$PHASE_COLOR" "$PHASE" "$C_RESET" if [ -n "$ROUND" ] && [ -n "$MAX_ROUNDS" ]; then # Cap displayed round at max_rounds. The internal counter increments past @@ -103,6 +110,12 @@ if [ -n "$ROUND" ] && [ -n "$MAX_ROUNDS" ]; then fi printf ' %-13s %s of %s\n' "round" "$display_round" "$MAX_ROUNDS" fi +if [ "$ENGINE" = "sweep-v2" ]; then + printf ' %-13s %s of %s\n' "generation" "$GENERATION" "$MAX_GENERATIONS" + printf ' %-13s %s\n' "snapshot" "$SNAPSHOT_SHA" + printf ' %-13s %s\n' "coverage" "$COVERAGE" + printf ' %-13s %s\n' "clean" "$CLEAN" +fi [ -n "$TOPIC" ] && printf ' %-13s %s\n' "topic" "$TOPIC" [ -n "$STARTED" ] && printf ' %-13s %s\n' "started_at" "$STARTED" printf ' %-13s %s\n' "elapsed" "$ELAPSED" diff --git a/plugins/claudex/scripts/sweep-helpers.sh b/plugins/claudex/scripts/sweep-helpers.sh new file mode 100755 index 0000000..06e5f68 --- /dev/null +++ b/plugins/claudex/scripts/sweep-helpers.sh @@ -0,0 +1,554 @@ +#!/usr/bin/env bash +# Deterministic frozen-snapshot sweep-v2 helpers. +# shellcheck shell=bash + +CLAUDEX_SWEEP_MAX_GENERATIONS=5 +CLAUDEX_SWEEP_PERSONA_TIMEOUT_SECONDS="${CLAUDEX_SWEEP_PERSONA_TIMEOUT_SECONDS:-300}" + +claudex_sha256() { + local file="$1" + [ -f "$file" ] || return 1 + if command -v shasum >/dev/null 2>&1; then + shasum -a 256 "$file" | awk '{print $1}' + elif command -v sha256sum >/dev/null 2>&1; then + sha256sum "$file" | awk '{print $1}' + else + return 127 + fi +} + +claudex_sweep_set_fields_atomic() { + local state_file="$1" + shift + local expected_phase="" + if [ "${1:-}" = "--expect-phase" ]; then + expected_phase="${2:-}" + shift 2 + fi + [ $(( $# % 2 )) -eq 0 ] || return 2 + python3 - "$state_file" "$expected_phase" "$@" <<'PY' +import datetime, fcntl, os, pathlib, re, sys, tempfile + +path = pathlib.Path(sys.argv[1]) +expected_phase = sys.argv[2] +args = sys.argv[3:] +updates = dict(zip(args[::2], args[1::2])) +if not updates or any(not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", key) for key in updates): + raise SystemExit(2) +updates.setdefault("last_updated_at", datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")) +lock_path = path.with_name(path.name + ".write-lock") +with lock_path.open("a+", encoding="utf-8") as lock: + fcntl.flock(lock.fileno(), fcntl.LOCK_EX) + lines = path.read_text(encoding="utf-8").splitlines() + current = {} + for line in lines: + if ":" in line: + key, value = line.split(":", 1) + current.setdefault(key, value.strip()) + if expected_phase and current.get("phase") != expected_phase: + raise SystemExit(3) + seen = set() + out = [] + for line in lines: + key = line.split(":", 1)[0] if ":" in line else "" + if key in updates: + out.append(f"{key}: {updates[key]}") + seen.add(key) + else: + out.append(line) + for key, value in updates.items(): + if key not in seen: + out.append(f"{key}: {value}") + fd, tmp = tempfile.mkstemp(prefix=path.name + ".tmp.", dir=path.parent) + try: + with os.fdopen(fd, "w", encoding="utf-8") as handle: + handle.write("\n".join(out) + "\n") + handle.flush() + os.fsync(handle.fileno()) + os.replace(tmp, path) + finally: + try: + os.unlink(tmp) + except FileNotFoundError: + pass +PY +} + +claudex_sweep_evidence_sha256() { + local generation_dir="$1" + python3 - "$generation_dir" $CLAUDEX_SWEEP_PERSONAS <<'PY' +import hashlib, pathlib, sys + +root = pathlib.Path(sys.argv[1]) +digest = hashlib.sha256() +for persona in sys.argv[2:]: + for suffix in ("findings.md", "result.json"): + path = root / f"{persona}.{suffix}" + if not path.is_file(): + raise SystemExit(1) + digest.update(path.name.encode("utf-8") + b"\0") + digest.update(path.read_bytes()) + digest.update(b"\0") +print(digest.hexdigest()) +PY +} + +claudex_sweep_validate_findings() { + local file="$1" + [ -s "$file" ] || { printf 'malformed'; return 1; } + python3 - "$file" <<'PY' +import pathlib, sys +text = pathlib.Path(sys.argv[1]).read_text(encoding="utf-8") +if text.strip() == "No substantive findings.": + print("clean") + raise SystemExit(0) +lines = [line.rstrip() for line in text.splitlines() if line.strip()] +headers = ["## High", "## Medium", "## Low"] +positions = [] +for header in headers: + if lines.count(header) != 1: + print("malformed"); raise SystemExit(1) + positions.append(lines.index(header)) +if positions != sorted(positions): + print("malformed"); raise SystemExit(1) +material = 0 +for i, start in enumerate(positions): + end = positions[i + 1] if i + 1 < len(positions) else len(lines) + for line in lines[start + 1:end]: + if not line.startswith("- ") or len(line) <= 2: + print("malformed"); raise SystemExit(1) + material += 1 +if material == 0 or lines[:positions[0]]: + print("malformed"); raise SystemExit(1) +print("material") +PY +} + +claudex_sweep_render_findings_with_ids() { + local file="$1" persona="$2" + python3 - "$file" "$persona" <<'PY' +import pathlib, sys + +path, persona = sys.argv[1:] +severity = None +counts = {"high": 0, "medium": 0, "low": 0} +for line in pathlib.Path(path).read_text(encoding="utf-8").splitlines(): + if line in {"## High", "## Medium", "## Low"}: + severity = line[3:].lower() + print(line) + elif line.startswith("- ") and severity: + counts[severity] += 1 + finding_id = f"{persona}-{severity}-{counts[severity]}" + print(f"- [{finding_id}] {line[2:]}") + else: + print(line) +PY +} + +claudex_sweep_validate_reconciliation() { + local plan="$1" consolidated="$2" generation="$3" snapshot_sha="$4" + python3 - "$plan" "$consolidated" "$generation" "$snapshot_sha" <<'PY' +import pathlib, re, sys + +plan_path, consolidated_path, generation, snapshot_sha = sys.argv[1:] +plan = pathlib.Path(plan_path).read_text(encoding="utf-8") +consolidated = pathlib.Path(consolidated_path).read_text(encoding="utf-8") +ids = re.findall(r"^- \[([a-z0-9-]+-(?:high|medium|low)-[0-9]+)\] ", consolidated, re.M) +if not ids or len(ids) != len(set(ids)): + raise SystemExit(1) +match = re.search(r"^## Changelog\s*$([\s\S]*?)(?=^## |\Z)", plan, re.M) +if not match: + raise SystemExit(1) +section = match.group(1) +heading = f"### Sweep generation {generation} — {snapshot_sha}" +if heading not in section: + raise SystemExit(1) +for finding_id in ids: + pattern = rf"^- (?:Accepted|Rejected) \[{re.escape(finding_id)}\]:\s+\S.+$" + if len(re.findall(pattern, section, re.M)) != 1: + raise SystemExit(1) +raise SystemExit(0) +PY +} + +claudex_sweep_create_generation() { + local state_file="$1" review_id="$2" generation="$3" topic="$4" previous_sha="${5:-}" + local source_plan="$(pwd -P)/PLAN.md" + [ -s "$source_plan" ] || return 1 + case "$generation" in ''|*[!0-9]*) return 1 ;; esac + [ "$generation" -ge 1 ] && [ "$generation" -le "$CLAUDEX_SWEEP_MAX_GENERATIONS" ] || return 1 + + local generation_dir="$CLAUDEX_STATE_DIR/$review_id/generations/$generation" + mkdir -p "$generation_dir" || return 1 + # A generation directory is write-once. Existing snapshot or manifest means + # this generation cannot be recreated from potentially different live input. + [ ! -e "$generation_dir/PLAN.md" ] && [ ! -e "$generation_dir/manifest.json" ] || return 1 + local tmp="$generation_dir/.PLAN.md.tmp.$$" + cp "$source_plan" "$tmp" || { rm -f "$tmp"; return 1; } + mv "$tmp" "$generation_dir/PLAN.md" || { rm -f "$tmp"; return 1; } + chmod a-w "$generation_dir/PLAN.md" 2>/dev/null || true + local sha + sha=$(claudex_sha256 "$generation_dir/PLAN.md") || return 1 + local manifest_tmp="$generation_dir/.manifest.json.tmp.$$" + python3 - "$manifest_tmp" "$generation" "$sha" "$topic" "$source_plan" "$previous_sha" <<'PY' +import json, pathlib, sys +path, generation, sha, topic, source, previous = sys.argv[1:] +data = { + "generation": int(generation), + "snapshot_sha256": sha, + "required_persona_ids": [ + "architecture-scope", "security-data", "product-domain", + "quality-accessibility-performance", "operations-deployment" + ], + "topic": topic, + "source_plan_path": source, + "previous_generation_sha256": previous or None, +} +pathlib.Path(path).write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8") +PY + [ $? -eq 0 ] || { rm -f "$manifest_tmp"; return 1; } + mv "$manifest_tmp" "$generation_dir/manifest.json" || return 1 + chmod a-w "$generation_dir/manifest.json" 2>/dev/null || true + claudex_sweep_set_fields_atomic "$state_file" \ + generation "$generation" \ + snapshot_sha256 "$sha" \ + coverage_complete false \ + decision_signal none \ + revision_required false \ + reviewed_live_sha256 "$sha" \ + evidence_sha256 "" \ + consolidated_sha256 "" || return 1 + printf '%s' "$sha" +} + +claudex_sweep_write_result() { + local result_file="$1" persona="$2" expected="$3" before="$4" after="$5" rc="$6" findings="$7" classification="$8" + local findings_sha="" + findings_sha=$(claudex_sha256 "$findings" 2>/dev/null) || findings_sha="" + local tmp="${result_file}.tmp.$$" + python3 - "$tmp" "$persona" "$expected" "$before" "$after" "$rc" "$findings" "$classification" "$findings_sha" <<'PY' +import datetime, json, pathlib, sys +path, persona, expected, before, after, rc, findings, classification, findings_sha = sys.argv[1:] +data = { + "persona_id": persona, + "expected_snapshot_sha256": expected, + "actual_snapshot_sha256_before": before, + "actual_snapshot_sha256_after": after, + "codex_exit_code": int(rc), + "findings_path": findings, + "findings_sha256": findings_sha, + "findings_classification": classification, + "completed_at": datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z"), +} +pathlib.Path(path).write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8") +PY + [ $? -eq 0 ] || { rm -f "$tmp"; return 1; } + mv "$tmp" "$result_file" || return 1 + chmod a-w "$findings" "$result_file" 2>/dev/null || true +} + +claudex_sweep_consolidate() { + local state_file="$1" review_id="$2" generation="$3" expected="$4" live_expected="$5" + local generation_dir="$CLAUDEX_STATE_DIR/$review_id/generations/$generation" + local consolidated="$generation_dir/consolidated-findings.md" + local snapshot="$generation_dir/PLAN.md" + local verdict_phase + verdict_phase=$(claudex_state_read_field "$state_file" phase) + [ "$verdict_phase" = cancelled ] && return 130 + case "$verdict_phase" in reviewing|awaiting-revision|summarizing) ;; *) return 4 ;; esac + local current_snapshot current_live + current_snapshot=$(claudex_sha256 "$snapshot" 2>/dev/null) + current_live=$(claudex_sha256 PLAN.md 2>/dev/null) + local degraded=false material=false clean_count=0 + local stored_evidence stored_consolidated current_evidence current_consolidated + stored_evidence=$(claudex_state_read_field "$state_file" evidence_sha256) + stored_consolidated=$(claudex_state_read_field "$state_file" consolidated_sha256) + if [ -n "$stored_evidence" ] || [ -n "$stored_consolidated" ]; then + current_evidence=$(claudex_sweep_evidence_sha256 "$generation_dir" 2>/dev/null) + current_consolidated=$(claudex_sha256 "$consolidated" 2>/dev/null) + if [ -z "$stored_evidence" ] || [ -z "$stored_consolidated" ] \ + || [ "$current_evidence" != "$stored_evidence" ] \ + || [ "$current_consolidated" != "$stored_consolidated" ]; then + claudex_sweep_set_fields_atomic "$state_file" --expect-phase "$verdict_phase" \ + coverage_complete false \ + decision_signal degraded \ + clean false \ + revision_required false \ + converged_snapshot_sha256 "" \ + phase summarizing || return 4 + return 2 + fi + fi + local manifest="$generation_dir/manifest.json" + local manifest_valid + manifest_valid=$(python3 - "$manifest" "$generation" "$expected" "$generation_dir" "$(pwd -P)/PLAN.md" <<'PY' +import json, pathlib, sys +path, generation, expected, generation_dir, source = sys.argv[1:] +ids = ["architecture-scope", "security-data", "product-domain", "quality-accessibility-performance", "operations-deployment"] +try: + m = json.loads(pathlib.Path(path).read_text(encoding="utf-8")) + ok = m.get("generation") == int(generation) + ok &= m.get("snapshot_sha256") == expected + ok &= m.get("required_persona_ids") == ids + ok &= m.get("source_plan_path") == source and bool(m.get("topic")) + if int(generation) == 1: + ok &= m.get("previous_generation_sha256") is None + else: + previous = pathlib.Path(generation_dir).parent / str(int(generation) - 1) / "manifest.json" + previous_sha = json.loads(previous.read_text(encoding="utf-8"))["snapshot_sha256"] + ok &= m.get("previous_generation_sha256") == previous_sha + print("valid" if ok else "degraded") +except Exception: + print("degraded") +PY +) + [ "$manifest_valid" = "valid" ] || degraded=true + local tmp="${consolidated}.tmp.$$" + { + printf '# Consolidated findings — generation %s\n\n' "$generation" + printf 'Snapshot SHA-256: `%s`\n\n' "$expected" + local persona findings result classification valid + for persona in $CLAUDEX_SWEEP_PERSONAS; do + findings="$generation_dir/$persona.findings.md" + result="$generation_dir/$persona.result.json" + printf '## %s\n\n' "$persona" + if [ ! -s "$findings" ] || [ ! -s "$result" ]; then + printf 'DEGRADED: missing findings or result sidecar.\n\n' + degraded=true + continue + fi + valid=$(python3 - "$result" "$persona" "$expected" "$findings" <<'PY' +import datetime, hashlib, json, pathlib, sys +path, persona, expected, findings = sys.argv[1:] +try: + data = json.loads(pathlib.Path(path).read_text(encoding="utf-8")) + required = {"persona_id", "expected_snapshot_sha256", "actual_snapshot_sha256_before", "actual_snapshot_sha256_after", "codex_exit_code", "findings_path", "findings_sha256", "findings_classification", "completed_at"} + ok = set(data) == required + ok &= data["persona_id"] == persona and data["expected_snapshot_sha256"] == expected + ok &= data["actual_snapshot_sha256_before"] == expected and data["actual_snapshot_sha256_after"] == expected + ok &= data["codex_exit_code"] == 0 and data["findings_path"] == findings + actual_findings_sha = hashlib.sha256(pathlib.Path(findings).read_bytes()).hexdigest() + ok &= data["findings_sha256"] == actual_findings_sha + ok &= data["findings_classification"] in {"clean", "material"} + datetime.datetime.strptime(data["completed_at"], "%Y-%m-%dT%H:%M:%SZ") + print(data["findings_classification"] if ok else "degraded") +except Exception: + print("degraded") +PY +) + classification=$(claudex_sweep_validate_findings "$findings" 2>/dev/null) + if [ "$valid" = "degraded" ] || [ "$classification" != "$valid" ]; then + printf 'DEGRADED: invalid sidecar, hash, exit status, or findings schema.\n\n' + degraded=true + else + if [ "$classification" = "clean" ]; then + cat "$findings" + clean_count=$((clean_count + 1)) + else + claudex_sweep_render_findings_with_ids "$findings" "$persona" || degraded=true + material=true + fi + printf '\n\n' + fi + done + } > "$tmp" || { rm -f "$tmp"; return 1; } + mv "$tmp" "$consolidated" || return 1 + current_evidence=$(claudex_sweep_evidence_sha256 "$generation_dir" 2>/dev/null) || degraded=true + current_consolidated=$(claudex_sha256 "$consolidated" 2>/dev/null) || degraded=true + [ -n "$current_evidence" ] && [ -n "$current_consolidated" ] || degraded=true + + if [ "$current_snapshot" != "$expected" ] || [ "$current_live" != "$live_expected" ]; then degraded=true; fi + if [ "$clean_count" -ne 5 ] && [ "$material" != "true" ]; then degraded=true; fi + + if [ "$degraded" = "true" ]; then + claudex_sweep_set_fields_atomic "$state_file" --expect-phase "$verdict_phase" \ + coverage_complete false \ + decision_signal degraded \ + clean false \ + revision_required false \ + converged_snapshot_sha256 "" \ + evidence_sha256 "$current_evidence" \ + consolidated_sha256 "$current_consolidated" \ + phase summarizing || return 4 + return 2 + fi + if [ "$clean_count" -eq 5 ] && [ "$material" = "false" ]; then + claudex_sweep_set_fields_atomic "$state_file" --expect-phase "$verdict_phase" \ + coverage_complete true \ + decision_signal converged \ + clean true \ + revision_required false \ + converged_snapshot_sha256 "$expected" \ + evidence_sha256 "$current_evidence" \ + consolidated_sha256 "$current_consolidated" \ + phase summarizing || return 4 + return 0 + fi + local max_generations + max_generations=$(claudex_state_read_field "$state_file" max_generations) + case "$max_generations" in ''|*[!0-9]*) max_generations="$CLAUDEX_SWEEP_MAX_GENERATIONS" ;; esac + [ "$max_generations" -le "$CLAUDEX_SWEEP_MAX_GENERATIONS" ] || max_generations="$CLAUDEX_SWEEP_MAX_GENERATIONS" + if [ "$generation" -ge "$max_generations" ]; then + claudex_sweep_set_fields_atomic "$state_file" --expect-phase "$verdict_phase" \ + coverage_complete true \ + decision_signal max-reached \ + clean false \ + revision_required false \ + converged_snapshot_sha256 "" \ + evidence_sha256 "$current_evidence" \ + consolidated_sha256 "$current_consolidated" \ + phase summarizing || return 4 + return 3 + fi + claudex_sweep_set_fields_atomic "$state_file" --expect-phase "$verdict_phase" \ + coverage_complete true \ + decision_signal material-findings \ + clean false \ + revision_required true \ + converged_snapshot_sha256 "" \ + evidence_sha256 "$current_evidence" \ + consolidated_sha256 "$current_consolidated" \ + phase awaiting-revision || return 4 + return 1 +} + +claudex_sweep_write_runner() { + local state_file="$1" review_id="$2" generation="$3" topic="$4" expected="$5" + local runner="$CLAUDEX_STATE_DIR/$review_id-runner.sh" + local generation_dir="$CLAUDEX_STATE_DIR/$review_id/generations/$generation" + local snapshot="$generation_dir/PLAN.md" + local live_expected + live_expected=$(claudex_sha256 PLAN.md) || return 1 + cat > "$runner" <&2 + exit 2 + fi + rm -f "\$prompt" + before=\$(claudex_sha256 "\$SNAPSHOT" 2>/dev/null) + live_before=\$(claudex_sha256 PLAN.md 2>/dev/null) + focus=\$(claudex_sweep_persona_prompt "\$persona") + cat > "\$prompt" </dev/null 2>&1; then + rc=127 + else + claudex_run_codex_bounded "\$prompt" + rc=\$? + fi + after=\$(claudex_sha256 "\$SNAPSHOT" 2>/dev/null) + live_after=\$(claudex_sha256 PLAN.md 2>/dev/null) + classification=degraded + if [ "\$rc" -eq 0 ] && [ "\$before" = "\$EXPECTED" ] && [ "\$after" = "\$EXPECTED" ] && [ "\$live_before" = "\$LIVE_EXPECTED" ] && [ "\$live_after" = "\$LIVE_EXPECTED" ]; then + classification=\$(claudex_sweep_validate_findings "\$findings" 2>/dev/null) + [ "\$classification" = clean ] || [ "\$classification" = material ] || classification=degraded + fi + if ! claudex_sweep_write_result "\$result" "\$persona" "\$EXPECTED" "\$before" "\$after" "\$rc" "\$findings" "\$classification"; then + claudex_sweep_set_fields_atomic "\$STATE_FILE" \ + coverage_complete false decision_signal degraded clean false \ + revision_required false converged_snapshot_sha256 '' phase summarizing + exit 2 + fi + claudex_state_set_field "\$STATE_FILE" sweep_heartbeat "\$persona" + rm -f "\$prompt" +done +if [ "\$(claudex_state_read_field "\$STATE_FILE" phase)" = cancelled ]; then + rm -f "\$ACTIVE_PGID_FILE" + exit 130 +fi +claudex_sweep_consolidate "\$STATE_FILE" "\$REVIEW_ID" "\$GENERATION" "\$EXPECTED" "\$LIVE_EXPECTED" +rc=\$? +case "\$rc" in 0) echo '[claudex] sweep converged' ;; 1) echo '[claudex] material findings require revision' ;; 2) echo '[claudex] sweep degraded' ;; 3) echo '[claudex] maximum generations reached' ;; esac +exit "\$rc" +RUNNEREOF + chmod +x "$runner" +} diff --git a/plugins/claudex/tests/platform-validation.sh b/plugins/claudex/tests/platform-validation.sh index 7e0fe2e..812a9d4 100755 --- a/plugins/claudex/tests/platform-validation.sh +++ b/plugins/claudex/tests/platform-validation.sh @@ -165,6 +165,8 @@ unset CLAUDEX_STATE_DIR section "12. Personas helper" check "personas.sh exists" test -f "$PLUGIN_ROOT/scripts/personas.sh" check "personas sources cleanly" bash -c "source '$PLUGIN_ROOT/scripts/personas.sh'" +check "sweep-helpers.sh exists" test -f "$PLUGIN_ROOT/scripts/sweep-helpers.sh" +check "sweep helpers source cleanly" bash -c "source '$PLUGIN_ROOT/scripts/sweep-helpers.sh'" # shellcheck source=/dev/null source "$PLUGIN_ROOT/scripts/personas.sh" @@ -188,6 +190,9 @@ L3=$(claudex_persona_label_for_round 3) check "round 1 label non-empty" test -n "$L1" check "round 1 label differs from round 2" test "$L1" != "$L2" check "round 2 label differs from round 3" test "$L2" != "$L3" +SWEEP_IDS=$(printf '%s\n' $CLAUDEX_SWEEP_PERSONAS) +check "sweep-v2 defines exactly five personas" test "$(printf '%s\n' "$SWEEP_IDS" | wc -l | tr -d ' ')" = 5 +check "sweep-v2 persona order is stable" test "$SWEEP_IDS" = "$(printf '%s\n' architecture-scope security-data product-domain quality-accessibility-performance operations-deployment)" section "13. Findings severity counter" TMP=$(mktemp -d) @@ -225,6 +230,43 @@ check "status.sh runs without error on empty state" bash -c " exit \$rc " +section "15. Safe state field updates" +TMP=$(mktemp -d) +SAFE_STATE="$TMP/safe.state" +claudex_state_write "$SAFE_STATE" "phase: drafting +note: old +last_updated_at: 2026-04-26T00:00:00Z" +special_value='path/with/slashes & ampersand' +check "set_field accepts slash and ampersand" \ + claudex_state_set_field "$SAFE_STATE" note "$special_value" +actual_value=$(claudex_state_read_field "$SAFE_STATE" note) +check "slash and ampersand survive literally" test "$actual_value" = "$special_value" +newline_value=$(printf 'first line\nsecond/line & value') +check "set_field accepts embedded newline" \ + claudex_state_set_field "$SAFE_STATE" note "$newline_value" +actual_value=$(claudex_state_read_field "$SAFE_STATE" note) +check "newline is safely normalized for single-line state" \ + test "$actual_value" = "first line second/line & value" +check "invalid field name rejected" \ + bash -c "! claudex_state_set_field '$SAFE_STATE' 'bad/field' value" +rm -rf "$TMP" + +section "16. Canonical repository root" +TMP=$(mktemp -d) +mkdir -p "$TMP/real" +ln -s "$TMP/real" "$TMP/alias" +( cd "$TMP/alias" && export CLAUDEX_STATE_DIR=.claude/claudex && \ + bash "$PLUGIN_ROOT/scripts/start-loop.sh" plan "canonical path test" >/dev/null 2>&1 ) +CANON_STATE=$(ls "$TMP/real/.claude/claudex"/*.state 2>/dev/null | head -1) +stored_root=$(claudex_state_read_field "$CANON_STATE" repo_root) +expected_root=$(cd "$TMP/real" && pwd -P) +check "start-loop stores physical pwd" test "$stored_root" = "$expected_root" +rm -rf "$TMP" + +section "17. Installer validation path" +check "install.sh targets nested plugin test path" \ + grep -q '\$PLUGIN_ROOT/plugins/claudex/tests/platform-validation.sh' "$PLUGIN_ROOT/../../install.sh" + # Summary printf '\n\033[1m=== Results ===\033[0m\n' printf ' \033[32m%d passed\033[0m\n' "$pass" diff --git a/plugins/claudex/tests/sweep-v2-test.sh b/plugins/claudex/tests/sweep-v2-test.sh new file mode 100755 index 0000000..3fe2131 --- /dev/null +++ b/plugins/claudex/tests/sweep-v2-test.sh @@ -0,0 +1,415 @@ +#!/usr/bin/env bash +# Deterministic regression coverage for the feature-flagged sweep-v2 engine. +set +e + +PLUGIN_ROOT="${CLAUDE_PLUGIN_ROOT:-$(cd "$(dirname "$0")/.." && pwd)}" +START="$PLUGIN_ROOT/scripts/start-loop.sh" +HOOK="$PLUGIN_ROOT/hooks/stop-hook.sh" +CANCEL="$PLUGIN_ROOT/scripts/cancel-loop.sh" +PASS=0 +FAIL=0 +FAILURES=() +TEMPS=() + +ok() { printf ' \033[32m✓\033[0m %s\n' "$1"; PASS=$((PASS + 1)); } +bad() { printf ' \033[31m✗\033[0m %s\n' "$1"; FAIL=$((FAIL + 1)); FAILURES+=("$1"); } +check() { local name="$1"; shift; if "$@" >/dev/null 2>&1; then ok "$name"; else bad "$name"; fi; } +field() { sed -n "s/^$2: *//p" "$1" | head -1; } +cleanup() { + local p + if [ "${KEEP_SWEEP_TEMPS:-0}" = 1 ]; then + printf 'preserved sweep temp: %s\n' "${TEMPS[@]}" >&2 + return + fi + for p in "${TEMPS[@]}"; do chmod -R u+w "$p" 2>/dev/null; rm -rf "$p"; done +} +trap cleanup EXIT + +make_stub() { + local path="$1" + cat > "$path" <<'STUB' +#!/usr/bin/env bash +prompt=$(mktemp) +cat > "$prompt" +persona=$(sed -n 's/^Persona ID: //p' "$prompt" | head -1) +findings=$(sed -n 's/^Write ONLY one of these forms to \(.*\):$/\1/p' "$prompt" | head -1) +[ -n "$CLAUDEX_SWEEP_ORDER_LOG" ] && printf '%s\n' "$persona" >> "$CLAUDEX_SWEEP_ORDER_LOG" +case "${CLAUDEX_SWEEP_STUB_MODE:-clean}:${CLAUDEX_SWEEP_STUB_PERSONA:-}" in + missing:$persona) : ;; + malformed:$persona) printf 'not valid findings\n' > "$findings" ;; + nonzero:$persona) rm -f "$prompt"; exit 9 ;; + timeout:$persona) + sleep 30 & + child=$! + [ -n "$CLAUDEX_SWEEP_TIMEOUT_CHILD_FILE" ] && printf '%s\n' "$child" > "$CLAUDEX_SWEEP_TIMEOUT_CHILD_FILE" + wait "$child" + ;; + snapshot-mutation:$persona) + snapshot=$(sed -n 's/^Review only the frozen plan snapshot at: //p' "$prompt" | head -1) + chmod u+w "$snapshot" && printf '\nmutation\n' >> "$snapshot" + printf 'No substantive findings.\n' > "$findings" + ;; + live-mutation:$persona) + printf '\nlive mutation\n' >> PLAN.md + printf 'No substantive findings.\n' > "$findings" + ;; + material:$persona) + cat > "$findings" <<'EOF' +## High +- Scope: a concrete requirement can fail (address the stated failure mode). +## Medium +## Low +EOF + ;; + *) printf 'No substantive findings.\n' > "$findings" ;; +esac +rm -f "$prompt" +exit 0 +STUB + chmod +x "$path" +} + +new_repo() { + TEST_DIR=$(mktemp -d) + TEMPS+=("$TEST_DIR") + cd "$TEST_DIR" || exit 1 + git init -q + printf '# Plan\n\n## Scope\n\n1. Implement the scoped change.\n' > PLAN.md + STUB="$TEST_DIR/codex-stub" + make_stub "$STUB" + export CLAUDE_PLUGIN_ROOT="$PLUGIN_ROOT" + export CLAUDEX_CODEX_BIN="$STUB" + unset CLAUDEX_SWEEP_STUB_MODE CLAUDEX_SWEEP_STUB_PERSONA CLAUDEX_SWEEP_ORDER_LOG + unset CLAUDEX_SWEEP_PERSONA_TIMEOUT_SECONDS CLAUDEX_SWEEP_TIMEOUT_CHILD_FILE +} + +start_sweep() { + local rounds="${1:-}" + if [ -n "$rounds" ]; then + bash "$START" plan --engine sweep-v2 --rounds "$rounds" "deterministic sweep test" >/dev/null 2>&1 + else + bash "$START" plan --engine sweep-v2 "deterministic sweep test" >/dev/null 2>&1 + fi + ID=$(basename "$(ls .claude/claudex/*.state | head -1)" .state) + STATE=".claude/claudex/$ID.state" + RUNNER=".claude/claudex/$ID-runner.sh" +} + +printf '\033[1m=== Claudex sweep-v2 deterministic tests ===\033[0m\n' + +printf '\n\033[1mFeature flag and compatibility\033[0m\n' +new_repo +start_sweep +check "sweep-v2 defaults to five generations" test "$(field "$STATE" max_generations)" = 5 +check "sweep-v2 state records engine" test "$(field "$STATE" engine)" = sweep-v2 +check "generation one snapshot is immutable" bash -c "[ ! -w '.claude/claudex/$ID/generations/1/PLAN.md' ] || [ ! -x /bin/chmod ]" +check "manifest is immutable" bash -c "[ ! -w '.claude/claudex/$ID/generations/1/manifest.json' ] || [ ! -x /bin/chmod ]" +check "hard maximum rejects six generations" bash -c "cd '$TEST_DIR'; chmod -R u+w .claude; rm -rf .claude; ! bash '$START' plan --engine sweep-v2 --rounds 6 topic >/dev/null 2>&1" +NO_PY_PATH=$(mktemp -d); TEMPS+=("$NO_PY_PATH") +ln -s "$(command -v dirname)" "$NO_PY_PATH/dirname" +NO_PY_OUTPUT=$(PATH="$NO_PY_PATH" /bin/bash "$START" plan --engine sweep-v2 "python prerequisite" 2>&1) +NO_PY_RC=$? +if [ "$NO_PY_RC" -ne 0 ] && printf '%s' "$NO_PY_OUTPUT" | grep -q 'requires python3' && [ ! -d .claude ]; then + ok "sweep-v2 fails before state creation without python3" +else + bad "sweep-v2 fails before state creation without python3" +fi +NO_HASH_PATH=$(mktemp -d); TEMPS+=("$NO_HASH_PATH") +ln -s "$(command -v dirname)" "$NO_HASH_PATH/dirname" +ln -s "$(command -v python3)" "$NO_HASH_PATH/python3" +NO_HASH_OUTPUT=$(PATH="$NO_HASH_PATH" /bin/bash "$START" plan --engine sweep-v2 "hash prerequisite" 2>&1) +NO_HASH_RC=$? +if [ "$NO_HASH_RC" -ne 0 ] && printf '%s' "$NO_HASH_OUTPUT" | grep -q 'requires shasum or sha256sum' && [ ! -d .claude ]; then + ok "sweep-v2 fails before state creation without SHA-256 tooling" +else + bad "sweep-v2 fails before state creation without SHA-256 tooling" +fi +rm -rf .claude +bash "$START" plan "legacy topic" >/dev/null 2>&1 +LEGACY_STATE=$(ls .claude/claudex/*.state | head -1) +check "legacy plan remains default drafting lifecycle" test "$(field "$LEGACY_STATE" phase)" = drafting +check "legacy plan keeps three-round default" test "$(field "$LEGACY_STATE" max_rounds)" = 3 +chmod -R u+w .claude; rm -rf .claude +bash "$START" review >/dev/null 2>&1 +REVIEW_STATE=$(ls .claude/claudex/*.state | head -1) +check "review mode remains reviewing" test "$(field "$REVIEW_STATE" phase)" = reviewing +printf '# sentinel\n' > PLAN.md +echo '{}' | bash "$HOOK" >/dev/null 2>&1 +REVIEW_RUNNER=$(ls .claude/claudex/*-runner.sh | head -1) +check "review runner remains single senior-engineer review" grep -qi 'senior' "$REVIEW_RUNNER" + +STALE_DIR=$(mktemp -d); TEMPS+=("$STALE_DIR") +export CLAUDEX_STATE_DIR="$STALE_DIR" CLAUDEX_STALE_MINUTES=1 CLAUDEX_SWEEP_V2_STALE_MINUTES=3 +source "$PLUGIN_ROOT/scripts/state-helpers.sh" +printf 'engine: sweep-v2\nphase: reviewing\n' > "$STALE_DIR/sweep.state" +python3 - "$STALE_DIR/sweep.state" 120 <<'PY' +import os, sys, time +p, age = sys.argv[1], int(sys.argv[2]); t = time.time() - age; os.utime(p, (t, t)) +PY +claudex_sweep_stale +check "active-age sweep-v2 state survives legacy stale window" test -f "$STALE_DIR/sweep.state" +python3 - "$STALE_DIR/sweep.state" 240 <<'PY' +import os, sys, time +p, age = sys.argv[1], int(sys.argv[2]); t = time.time() - age; os.utime(p, (t, t)) +PY +claudex_sweep_stale +check "abandoned sweep-v2 state expires at extended stale window" test ! -f "$STALE_DIR/sweep.state" +printf 'engine: legacy\nphase: reviewing\n' > "$STALE_DIR/locked.state" +printf '%s\n' "$$" > "$STALE_DIR/locked.lock" +python3 - "$STALE_DIR/locked.state" 120 <<'PY' +import os, sys, time +p, age = sys.argv[1], int(sys.argv[2]); t = time.time() - age; os.utime(p, (t, t)) +PY +claudex_sweep_stale +check "live runner lock prevents stale reaping" test -f "$STALE_DIR/locked.state" +unset CLAUDEX_STATE_DIR CLAUDEX_STALE_MINUTES CLAUDEX_SWEEP_V2_STALE_MINUTES + +printf '\n\033[1mClean convergence and contracts\033[0m\n' +new_repo +start_sweep +ORDER="$TEST_DIR/order"; export CLAUDEX_SWEEP_ORDER_LOG="$ORDER" +bash "$RUNNER" >/dev/null 2>&1 +STATUS_OUTPUT=$(bash "$PLUGIN_ROOT/scripts/status.sh") +if printf '%s' "$STATUS_OUTPUT" | grep -q 'summarizing' && ! printf '%s' "$STATUS_OUTPUT" | grep -q 'unknown'; then + ok "status recognizes sweep-v2 summarizing phase as active" +else + bad "status recognizes sweep-v2 summarizing phase as active" +fi +GEN_DIR=".claude/claudex/$ID/generations/1" +check "five clean personas converge generation one" test "$(field "$STATE" decision_signal)" = converged +check "clean convergence records complete coverage" test "$(field "$STATE" coverage_complete)" = true +check "clean convergence records clean=true" test "$(field "$STATE" clean)" = true +check "all results use the same snapshot hash" python3 - "$GEN_DIR" <<'PY' +import json, pathlib, sys +p=pathlib.Path(sys.argv[1]); manifest=json.loads((p/'manifest.json').read_text()); h=manifest['snapshot_sha256'] +results=[json.loads(x.read_text()) for x in p.glob('*.result.json')] +assert len(results)==5 +assert all(r['expected_snapshot_sha256']==h and r['actual_snapshot_sha256_before']==h and r['actual_snapshot_sha256_after']==h for r in results) +assert all(len(r['findings_sha256'])==64 for r in results) +PY +check "completed findings and sidecars are write-discouraged" bash -c "[ ! -w '$GEN_DIR/security-data.findings.md' ] && [ ! -w '$GEN_DIR/security-data.result.json' ]" +check "manifest records complete contract" python3 - "$GEN_DIR/manifest.json" <<'PY' +import json, pathlib, sys +m=json.loads(pathlib.Path(sys.argv[1]).read_text()) +assert m['generation']==1 and len(m['required_persona_ids'])==5 +assert m['topic'] and m['source_plan_path'].endswith('/PLAN.md') +assert m['previous_generation_sha256'] is None and len(m['snapshot_sha256'])==64 +PY +EXPECTED_ORDER=$(printf '%s\n' architecture-scope security-data product-domain quality-accessibility-performance operations-deployment) +check "runner and consolidation use deterministic persona order" test "$(cat "$ORDER")" = "$EXPECTED_ORDER" +check "consolidation order is deterministic" python3 - "$GEN_DIR/consolidated-findings.md" <<'PY' +import pathlib, sys +s=pathlib.Path(sys.argv[1]).read_text() +ids=['architecture-scope','security-data','product-domain','quality-accessibility-performance','operations-deployment'] +assert [s.index('## '+x) for x in ids] == sorted(s.index('## '+x) for x in ids) +PY +check "reviewer contract pins snapshot path and hash" grep -q 'Expected snapshot SHA-256' "$RUNNER" +check "reviewer contract prohibits snapshot and live-plan edits" grep -q 'Do not edit the frozen snapshot or the live PLAN.md' "$RUNNER" +check "reviewer contract requires grounded findings" grep -q 'Tie every finding to a plan section and a concrete requirement, repository fact, or credible failure mode' "$RUNNER" +check "reviewer contract rejects unsupported gold-plating" grep -q 'Unsupported enterprise gold-plating is non-material' "$RUNNER" +check "reviewer contract preserves approval gates" grep -q 'approval-gated decisions as valid gates' "$RUNNER" + +new_repo; start_sweep +bash "$RUNNER" >/dev/null 2>&1 +MUTATED_RESULT=".claude/claudex/$ID/generations/1/security-data.result.json" +chmod u+w "$MUTATED_RESULT" +python3 - "$MUTATED_RESULT" <<'PY' +import json, pathlib, sys +p=pathlib.Path(sys.argv[1]); d=json.loads(p.read_text()); d['completed_at']='mutated'; p.write_text(json.dumps(d)) +PY +echo '{}' | bash "$HOOK" >/dev/null 2>&1 +check "post-run mutated sidecar degrades before summary" test "$(field "$STATE" decision_signal)" = degraded + +new_repo; start_sweep +bash "$RUNNER" >/dev/null 2>&1 +MUTATED_FINDINGS=".claude/claudex/$ID/generations/1/product-domain.findings.md" +chmod u+w "$MUTATED_FINDINGS" +printf '\n' >> "$MUTATED_FINDINGS" +echo '{}' | bash "$HOOK" >/dev/null 2>&1 +check "post-run findings digest mismatch degrades before summary" test "$(field "$STATE" decision_signal)" = degraded + +new_repo; start_sweep +bash "$RUNNER" >/dev/null 2>&1 +RERUN_RC=0 +bash "$RUNNER" >/dev/null 2>&1 || RERUN_RC=$? +check "completed generation evidence is write-once" bash -c "[ '$RERUN_RC' -eq 2 ] && [ \"\$(sed -n 's/^decision_signal: *//p' '$STATE')\" = degraded ]" + +run_degraded_case() { + local mode="$1" persona="$2" expected_name="$3" + new_repo; start_sweep + export CLAUDEX_SWEEP_STUB_MODE="$mode" CLAUDEX_SWEEP_STUB_PERSONA="$persona" + bash "$RUNNER" >/dev/null 2>&1 + check "$expected_name" bash -c "[ \"\$(sed -n 's/^decision_signal: *//p' '$STATE')\" = degraded ] && [ \"\$(sed -n 's/^clean: *//p' '$STATE')\" = false ]" +} + +printf '\n\033[1mMaterial and degraded outcomes\033[0m\n' +new_repo; start_sweep +export CLAUDEX_SWEEP_STUB_MODE=material CLAUDEX_SWEEP_STUB_PERSONA=operations-deployment +bash "$RUNNER" >/dev/null 2>&1 +MATERIAL_STATUS=$(bash "$PLUGIN_ROOT/scripts/status.sh") +if printf '%s' "$MATERIAL_STATUS" | grep -q 'awaiting-revision' && ! printf '%s' "$MATERIAL_STATUS" | grep -q 'unknown'; then + ok "status recognizes sweep-v2 awaiting-revision phase as active" +else + bad "status recognizes sweep-v2 awaiting-revision phase as active" +fi +check "four clean plus one material cannot converge" test "$(field "$STATE" decision_signal)" = material-findings +check "material findings require a revision" test "$(field "$STATE" revision_required)" = true +run_degraded_case missing security-data "missing persona output degrades" +run_degraded_case malformed product-domain "malformed findings degrade" +run_degraded_case nonzero architecture-scope "nonzero reviewer exit degrades" +run_degraded_case snapshot-mutation quality-accessibility-performance "snapshot hash mismatch degrades" +run_degraded_case live-mutation operations-deployment "live PLAN.md mutation during sweep degrades" + +new_repo +export CLAUDEX_SWEEP_PERSONA_TIMEOUT_SECONDS=1 +start_sweep +TIMEOUT_CHILD="$TEST_DIR/timeout-child.pid" +export CLAUDEX_SWEEP_STUB_MODE=timeout CLAUDEX_SWEEP_STUB_PERSONA=security-data CLAUDEX_SWEEP_TIMEOUT_CHILD_FILE="$TIMEOUT_CHILD" +bash "$RUNNER" >/dev/null 2>&1 +check "persona timeout degrades the sweep" test "$(field "$STATE" decision_signal)" = degraded +TIMEOUT_PID=$(cat "$TIMEOUT_CHILD" 2>/dev/null) +check "persona timeout kills the reviewer process group" bash -c "[ -n '$TIMEOUT_PID' ] && ! kill -0 '$TIMEOUT_PID' 2>/dev/null" +unset CLAUDEX_SWEEP_PERSONA_TIMEOUT_SECONDS CLAUDEX_SWEEP_TIMEOUT_CHILD_FILE + +new_repo +export CLAUDEX_SWEEP_PERSONA_TIMEOUT_SECONDS=30 +start_sweep +export CLAUDEX_SWEEP_STUB_MODE=timeout CLAUDEX_SWEEP_STUB_PERSONA=architecture-scope +bash "$RUNNER" >/dev/null 2>&1 & +RUNNER_TEST_PID=$! +ACTIVE_PGID_FILE=".claude/claudex/$ID-active-pgid" +i=0 +while [ ! -s "$ACTIVE_PGID_FILE" ] && [ "$i" -lt 50 ]; do sleep 0.1; i=$((i + 1)); done +CANCELLED_PGID=$(cat "$ACTIVE_PGID_FILE" 2>/dev/null) +bash "$CANCEL" >/dev/null 2>&1 +wait "$RUNNER_TEST_PID" 2>/dev/null +check "cancel preserves terminal cancelled state" bash -c "[ \"\$(sed -n 's/^phase: *//p' '$STATE')\" = cancelled ] && [ \"\$(sed -n 's/^decision_signal: *//p' '$STATE')\" = cancelled ] && [ \"\$(sed -n 's/^clean: *//p' '$STATE')\" = false ] && [ \"\$(sed -n 's/^coverage_complete: *//p' '$STATE')\" = false ]" +check "cancel terminates the active reviewer process group" bash -c "[ -n '$CANCELLED_PGID' ] && ! kill -0 -- '-$CANCELLED_PGID' 2>/dev/null" +check "cancel removes active process metadata" test ! -e "$ACTIVE_PGID_FILE" +unset CLAUDEX_SWEEP_PERSONA_TIMEOUT_SECONDS CLAUDEX_SWEEP_STUB_MODE CLAUDEX_SWEEP_STUB_PERSONA + +new_repo; start_sweep +source "$PLUGIN_ROOT/scripts/state-helpers.sh"; source "$PLUGIN_ROOT/scripts/personas.sh"; source "$PLUGIN_ROOT/scripts/sweep-helpers.sh" +claudex_sweep_set_fields_atomic "$STATE" phase cancelled decision_signal cancelled clean false +STALE_VERDICT_RC=0 +claudex_sweep_set_fields_atomic "$STATE" --expect-phase reviewing phase summarizing decision_signal converged clean true || STALE_VERDICT_RC=$? +check "cancelled phase wins a racing stale verdict CAS" bash -c "[ '$STALE_VERDICT_RC' -eq 3 ] && [ \"\$(sed -n 's/^phase: *//p' '$STATE')\" = cancelled ] && [ \"\$(sed -n 's/^decision_signal: *//p' '$STATE')\" = cancelled ]" + +new_repo; start_sweep +bash "$RUNNER" >/dev/null 2>&1 +bash "$CANCEL" >/dev/null 2>&1 +echo '{}' | bash "$HOOK" >/dev/null 2>&1 +check "terminal revalidation preserves an already-cancelled verdict" bash -c "[ \"\$(sed -n 's/^phase: *//p' '$STATE')\" = cancelled ] && [ \"\$(sed -n 's/^decision_signal: *//p' '$STATE')\" = cancelled ] && [ \"\$(sed -n 's/^clean: *//p' '$STATE')\" = false ] && [ \"\$(sed -n 's/^coverage_complete: *//p' '$STATE')\" = false ]" + +new_repo; start_sweep +bash "$RUNNER" >/dev/null 2>&1 +GEN_DIR=".claude/claudex/$ID/generations/1" +chmod a-w "$GEN_DIR" +echo '{}' | bash "$HOOK" >/dev/null 2>&1 +chmod u+w "$GEN_DIR" +check "summary revalidation I/O failure clears prior convergence" bash -c "[ \"\$(sed -n 's/^decision_signal: *//p' '$STATE')\" = degraded ] && [ \"\$(sed -n 's/^clean: *//p' '$STATE')\" = false ]" + +new_repo; start_sweep +bash "$RUNNER" >/dev/null 2>&1 +ATOMIC_SHA=$(field "$STATE" snapshot_sha256) +CLAUDEX_STATE_DIR=.claude/claudex CLAUDE_PLUGIN_ROOT="$PLUGIN_ROOT" /bin/bash -c 'source "$CLAUDE_PLUGIN_ROOT/scripts/state-helpers.sh"; source "$CLAUDE_PLUGIN_ROOT/scripts/personas.sh"; source "$CLAUDE_PLUGIN_ROOT/scripts/sweep-helpers.sh"; claudex_sweep_set_fields_atomic "$1" decision_signal none clean false phase reviewing' _ "$STATE" +chmod a-w .claude/claudex +ATOMIC_RC=0 +CLAUDEX_STATE_DIR=.claude/claudex CLAUDE_PLUGIN_ROOT="$PLUGIN_ROOT" /bin/bash -c 'source "$CLAUDE_PLUGIN_ROOT/scripts/state-helpers.sh"; source "$CLAUDE_PLUGIN_ROOT/scripts/personas.sh"; source "$CLAUDE_PLUGIN_ROOT/scripts/sweep-helpers.sh"; claudex_sweep_consolidate "$1" "$2" 1 "$3" "$3" >/dev/null 2>&1' _ "$STATE" "$ID" "$ATOMIC_SHA" || ATOMIC_RC=$? +chmod u+w .claude/claudex +check "unpersistable verdict cannot report convergence" bash -c "[ '$ATOMIC_RC' -ne 0 ] && [ \"\$(sed -n 's/^decision_signal: *//p' '$STATE')\" != converged ] && [ \"\$(sed -n 's/^clean: *//p' '$STATE')\" = false ]" + +printf '\n\033[1mCoverage, generations, and isolation\033[0m\n' +new_repo; start_sweep +GEN_DIR=".claude/claudex/$ID/generations/1" +export CLAUDEX_SWEEP_STUB_MODE=missing CLAUDEX_SWEEP_STUB_PERSONA=security-data +bash "$RUNNER" >/dev/null 2>&1 +check "one or partial clean coverage cannot terminate early" bash -c "[ \"\$(sed -n 's/^decision_signal: *//p' '$STATE')\" != converged ] && [ \"\$(sed -n 's/^coverage_complete: *//p' '$STATE')\" = false ]" + +new_repo; start_sweep +bash "$RUNNER" >/dev/null 2>&1 +GEN_DIR=".claude/claudex/$ID/generations/1" +for persona in security-data product-domain quality-accessibility-performance operations-deployment; do + rm -f "$GEN_DIR/$persona.findings.md" "$GEN_DIR/$persona.result.json" +done +source "$PLUGIN_ROOT/scripts/state-helpers.sh"; source "$PLUGIN_ROOT/scripts/personas.sh"; source "$PLUGIN_ROOT/scripts/sweep-helpers.sh" +ONLY_SHA=$(field "$STATE" snapshot_sha256) +claudex_sweep_consolidate "$STATE" "$ID" 1 "$ONLY_SHA" "$ONLY_SHA" >/dev/null 2>&1 +check "one clean reviewer alone cannot converge" bash -c "[ \"\$(sed -n 's/^decision_signal: *//p' '$STATE')\" = degraded ] && [ \"\$(sed -n 's/^coverage_complete: *//p' '$STATE')\" = false ]" + +new_repo; start_sweep +TAMPER_SHA=$(field "$STATE" snapshot_sha256) +export CLAUDEX_SWEEP_STUB_MODE=material CLAUDEX_SWEEP_STUB_PERSONA=architecture-scope +bash "$RUNNER" >/dev/null 2>&1 +TAMPER_CONSOLIDATED=".claude/claudex/$ID/generations/1/consolidated-findings.md" +chmod u+w "$TAMPER_CONSOLIDATED" +printf '\ntampered\n' >> "$TAMPER_CONSOLIDATED" +printf '\n2. Address finding.\n\n## Changelog\n### Sweep generation 1 — %s\n- Accepted [architecture-scope-high-1]: addressed with a scoped failure-handling change.\n' "$TAMPER_SHA" >> PLAN.md +unset CLAUDEX_SWEEP_STUB_MODE CLAUDEX_SWEEP_STUB_PERSONA +echo '{}' | bash "$HOOK" >/dev/null 2>&1 +check "altered generation evidence cannot advance to a new snapshot" bash -c "[ \"\$(sed -n 's/^decision_signal: *//p' '$STATE')\" = degraded ] && [ \"\$(sed -n 's/^generation: *//p' '$STATE')\" = 1 ]" + +new_repo; start_sweep +GEN1_SHA=$(field "$STATE" snapshot_sha256) +export CLAUDEX_SWEEP_STUB_MODE=material CLAUDEX_SWEEP_STUB_PERSONA=architecture-scope +bash "$RUNNER" >/dev/null 2>&1 +printf '\n2. Unrelated edit.\n\n## Changelog\n- Prior entry only.\n' >> PLAN.md +echo '{}' | bash "$HOOK" >/dev/null 2>&1 +check "stale changelog cannot advance a material generation" bash -c "[ \"\$(sed -n 's/^phase: *//p' '$STATE')\" = awaiting-revision ] && [ \"\$(sed -n 's/^generation: *//p' '$STATE')\" = 1 ]" +printf '\n### Sweep generation 1 — %s\n- Accepted [architecture-scope-high-1]: added the missing failure handling to the scoped plan.\n' "$GEN1_SHA" >> PLAN.md +unset CLAUDEX_SWEEP_STUB_MODE CLAUDEX_SWEEP_STUB_PERSONA +echo '{}' | bash "$HOOK" >/dev/null 2>&1 +GEN2_SHA=$(field "$STATE" snapshot_sha256) +bash "$RUNNER" >/dev/null 2>&1 +check "material generation one then five clean generation two converges" bash -c "[ \"\$(sed -n 's/^decision_signal: *//p' '$STATE')\" = converged ] && [ \"\$(sed -n 's/^generation: *//p' '$STATE')\" = 2 ]" +check "generation two converges on its own new hash" bash -c "[ '$GEN1_SHA' != '$GEN2_SHA' ] && [ \"\$(sed -n 's/^converged_snapshot_sha256: *//p' '$STATE')\" = '$GEN2_SHA' ]" +check "generation two manifest links previous hash" python3 - ".claude/claudex/$ID/generations/2/manifest.json" "$GEN1_SHA" <<'PY' +import json, pathlib, sys +assert json.loads(pathlib.Path(sys.argv[1]).read_text())['previous_generation_sha256']==sys.argv[2] +PY + +new_repo; start_sweep +source "$PLUGIN_ROOT/scripts/state-helpers.sh" +source "$PLUGIN_ROOT/scripts/personas.sh" +source "$PLUGIN_ROOT/scripts/sweep-helpers.sh" +PREV=$(field "$STATE" snapshot_sha256) +for g in 2 3 4 5; do + chmod u+w PLAN.md; printf '\nrevision %s\n' "$g" >> PLAN.md + SHA=$(claudex_sweep_create_generation "$STATE" "$ID" "$g" "max generation test" "$PREV") || break + PREV="$SHA" +done +claudex_state_set_field "$STATE" phase reviewing +claudex_sweep_write_runner "$STATE" "$ID" 5 "max generation test" "$PREV" +RUNNER=".claude/claudex/$ID-runner.sh" +export CLAUDEX_SWEEP_STUB_MODE=material CLAUDEX_SWEEP_STUB_PERSONA=security-data +bash "$RUNNER" >/dev/null 2>&1 +check "material findings at generation five yield max-reached" bash -c "[ \"\$(sed -n 's/^decision_signal: *//p' '$STATE')\" = max-reached ] && [ \"\$(sed -n 's/^clean: *//p' '$STATE')\" = false ]" + +new_repo; start_sweep +bash "$RUNNER" >/dev/null 2>&1 +OLD_DIR=".claude/claudex/$ID/generations/1"; OLD_SHA=$(field "$STATE" snapshot_sha256) +chmod u+w PLAN.md; printf '\nnew generation\n' >> PLAN.md +source "$PLUGIN_ROOT/scripts/state-helpers.sh"; source "$PLUGIN_ROOT/scripts/personas.sh"; source "$PLUGIN_ROOT/scripts/sweep-helpers.sh" +NEW_SHA=$(claudex_sweep_create_generation "$STATE" "$ID" 2 "stale test" "$OLD_SHA") +NEW_DIR=".claude/claudex/$ID/generations/2" +cp "$OLD_DIR"/*.findings.md "$OLD_DIR"/*.result.json "$NEW_DIR"/ +claudex_sweep_consolidate "$STATE" "$ID" 2 "$NEW_SHA" "$NEW_SHA" >/dev/null 2>&1 +check "stale prior-generation artifacts cannot satisfy coverage" test "$(field "$STATE" decision_signal)" = degraded + +new_repo; start_sweep +FIRST_ID="$ID"; bash "$RUNNER" >/dev/null 2>&1 +echo '{}' | bash "$HOOK" >/dev/null 2>&1 +bash "$START" plan --engine sweep-v2 "second isolated sweep" >/dev/null 2>&1 +SECOND_ID=$(basename "$(ls -t .claude/claudex/*.state | head -1)" .state) +check "back-to-back sweep runs use isolated review IDs" test "$FIRST_ID" != "$SECOND_ID" +check "back-to-back sweep keeps first generation artifacts" test -f ".claude/claudex/$FIRST_ID/generations/1/manifest.json" +check "back-to-back sweep creates independent generation artifacts" test -f ".claude/claudex/$SECOND_ID/generations/1/manifest.json" + +printf '\n\033[1m=== Sweep-v2 Results ===\033[0m\n' +printf ' \033[32m%d passed\033[0m\n' "$PASS" +if [ "$FAIL" -gt 0 ]; then + printf ' \033[31m%d failed\033[0m\n' "$FAIL" + printf 'Failed:\n'; printf ' - %s\n' "${FAILURES[@]}" + exit 1 +fi +printf ' All sweep-v2 deterministic regressions passed.\n' diff --git a/plugins/claudex/tests/synthetic-e2e.sh b/plugins/claudex/tests/synthetic-e2e.sh index a366307..59678fb 100755 --- a/plugins/claudex/tests/synthetic-e2e.sh +++ b/plugins/claudex/tests/synthetic-e2e.sh @@ -103,7 +103,7 @@ check "PLAN.md exists" test -f PLAN.md section "Round 1: fire hook (drafting -> reviewing)" HOOK_OUT=$(echo '{}' | bash "$HOOK" 2>/dev/null) echo "Hook output: $(printf '%s' "$HOOK_OUT" | head -c 80)..." -check "hook returned block" bash -c "echo '$HOOK_OUT' | grep -q block" +check "hook returned block" python3 -c 'import json,sys; assert json.loads(sys.argv[1])["decision"] == "block"' "$HOOK_OUT" RUNNER=".claude/claudex/$REVIEW_ID-runner.sh" check "runner script created" test -f "$RUNNER" check "runner has quoted PROMPTEOF (P1 fix)" grep -q "<<'PROMPTEOF'" "$RUNNER" @@ -117,9 +117,9 @@ echo "Runner exit code: $RUNNER_RC" echo "$CODEX_OUTPUT" | tail -30 | head -20 check "runner exited with code 0" test "$RUNNER_RC" = "0" check "Codex output is non-empty" test -n "$CODEX_OUTPUT" -check "Codex did not hit stdin terminal error" bash -c "! echo '$CODEX_OUTPUT' | grep -q 'stdin is not a terminal'" -check "Codex did not hit auth error" bash -c "! echo '$CODEX_OUTPUT' | grep -qi 'not logged in\\|auth.*fail'" -check "Codex output mentions plan/review topic" bash -c "echo '$CODEX_OUTPUT' | grep -qiE 'plan|review|expir|finding'" +check "Codex did not hit stdin terminal error" env CODEX_OUTPUT="$CODEX_OUTPUT" bash -c '! printf "%s" "$CODEX_OUTPUT" | grep -q "stdin is not a terminal"' +check "Codex did not hit auth error" env CODEX_OUTPUT="$CODEX_OUTPUT" bash -c '! printf "%s" "$CODEX_OUTPUT" | grep -qiE "not logged in|authentication failed|login required|missing (api key|bearer token)"' +check "Codex output mentions plan/review topic" env CODEX_OUTPUT="$CODEX_OUTPUT" bash -c 'printf "%s" "$CODEX_OUTPUT" | grep -qiE "plan|review|expir|finding"' # Round 1 done. Now simulate Claude deciding the loop is complete and calling mark-done. # (We don't try to parse Codex output to decide -- that's Claude's job in production. @@ -128,14 +128,24 @@ section "Mark loop done (Claude's signal in production)" bash "$MARK_DONE" "$REVIEW_ID" >/dev/null PHASE=$(grep '^phase:' ".claude/claudex/$REVIEW_ID.state" | sed 's/^phase: //') SIGNAL=$(grep '^decision_signal:' ".claude/claudex/$REVIEW_ID.state" | sed 's/^decision_signal: //') -check "phase marked done" test "$PHASE" = "done" +check "mark-done leaves phase reviewing" test "$PHASE" = "reviewing" check "signal set to no-material-findings" test "$SIGNAL" = "no-material-findings" -# Final hook fire, should ALLOW. +# The current lifecycle delivers a summary BLOCK before the terminal APPROVE. +section "Summary hook fire" +HOOK_OUT=$(echo '{}' | bash "$HOOK" 2>/dev/null) +echo "Summary hook output: $HOOK_OUT" +check "summary hook returned block" python3 -c 'import json,sys; assert json.loads(sys.argv[1])["decision"] == "block"' "$HOOK_OUT" +check "summary hook mentions completion" python3 -c 'import json,sys; assert "plan loop complete" in json.loads(sys.argv[1])["reason"].lower()' "$HOOK_OUT" +PHASE=$(grep '^phase:' ".claude/claudex/$REVIEW_ID.state" | sed 's/^phase: //') +check "phase advanced to summarizing" test "$PHASE" = "summarizing" + section "Final hook fire" HOOK_OUT=$(echo '{}' | bash "$HOOK" 2>/dev/null) echo "Final hook output: $HOOK_OUT" check "final hook returned approve" bash -c "echo '$HOOK_OUT' | grep -q approve" +PHASE=$(grep '^phase:' ".claude/claudex/$REVIEW_ID.state" | sed 's/^phase: //') +check "phase advanced to done" test "$PHASE" = "done" check "lockfile cleaned up" bash -c "! test -f .claude/claudex/$REVIEW_ID.lock" check "state file preserved for audit" test -f ".claude/claudex/$REVIEW_ID.state" check "runner script cleaned up" bash -c "! test -f $RUNNER" diff --git a/skills/project-plan-review/SKILL.md b/skills/project-plan-review/SKILL.md new file mode 100644 index 0000000..5367c08 --- /dev/null +++ b/skills/project-plan-review/SKILL.md @@ -0,0 +1,51 @@ +--- +name: project-plan-review +description: Use for substantial implementation plans that benefit from independent Claude/Claudex/Codex adversarial review. Skip tiny, obvious fixes. +version: 2.0.0 +license: MIT +metadata: + hermes: + tags: [planning, claudex, codex, delegation, verification, sweep-v2] +--- + +# Project Plan Review + +Use this workflow for substantial features, migrations, or risky cross-cutting work. **Do not use it for tiny fixes** where review overhead exceeds implementation risk. + +## Main Drake workflow + +1. Ground the project first: inspect the repository, current behavior, constraints, tests, and user request. Never ask a leaf reviewer to invent this context. +2. Draft a concrete `PLAN.md` in the project root. Include scope/non-scope, exact files, existing contracts, steps, rollback, and verification. +3. Read [the delegation runbook](references/runbook.md). +4. Use the adapter's default `sweep-v2` engine with `--rounds 5`. The command must invoke `/claudex:plan --engine sweep-v2 --from-draft --skip-interview --rounds 5 ...` through the adapter. +5. Call `delegate_task` with self-contained context and absolute paths for the repository, `PLAN.md`, adapter, plugin, Claude, Codex, and evidence directory. The child is a leaf: it cannot ask Rob questions or delegate further. +6. Keep the main Drake session responsive while the child performs the bounded run. The outer child timeout must reserve at least 180 seconds beyond the adapter timeout for artifact read-back and reporting. +7. On return, read `result.json`, the copied `evidence_state_file`, generation manifest, consolidated findings, and all five persona sidecars/findings from adapter evidence. A claimed path is not evidence until read. +8. Independently reject scope creep, invented contracts, and recommendations unsupported by the grounded repository. Claudex is a critic, not the product owner. +9. Normalize the plan so only the active implementation phase remains; remove completed prerequisites, historical phases, and stale branching instructions. +10. **If Drake's normalization materially changes requirements, sequencing, contracts, safety controls, or verification, run sweep-v2 again.** A prior convergence hash does not cover a materially changed plan. +11. Attach or return the final reviewed `PLAN.md`, outcome, snapshot/converged hashes, persona coverage, unresolved findings, and copied evidence paths. Only `converged` is clean. + +## Sweep-v2 convergence contract + +A clean result requires all of the following from authoritative state and artifacts, never Claude prose: + +- terminal `phase=done`, `decision_signal=converged`, `clean=true`, and `coverage_complete=true`; +- exactly the five required personas—architecture/scope, security/data, product/domain, quality/accessibility/performance, and operations/deployment; +- every persona result tied to the same immutable generation snapshot SHA-256, with readable, schema-valid sidecars and findings; +- a readable generation manifest and consolidated findings whose hashes match state; and +- consolidated findings proving that all five personas returned exactly no substantive findings. + +Generation five with material findings is `max_reached` and non-clean. Missing, malformed, mutated, hash-mismatched, nonzero, cancelled, degraded, or incomplete evidence is never clean. + +## Boundaries and outcome rules + +- The adapter may write only `PLAN.md` plus `.claude/claudex/` evidence/state in the target repository. It must not implement the plan, commit, push, merge, or change global Hermes/Claude configuration. +- Use a disposable worktree/repository when the plan or project cannot safely be modified in place. +- `max_reached` proves mechanics, not plan approval. `degraded`, `failed`, and `timed_out` are also non-clean. +- Claude and Codex are subscription-backed in this workflow. Dollar-valued budget/cost fields are CLI usage telemetry and a bounded-run control, not evidence of direct API billing or an invoice. Codex subscription usage is not dollar-enforced by the adapter. +- Never expose secrets in the topic, plan, delegated prompt, or evidence. + +## Verification gate + +Before attaching the final plan, require readable copied evidence paths; matching absolute repo/plan paths; accurate `engine`, `generation`, and `max_generations`; terminal state; exact five-persona same-hash coverage; readable manifest and consolidated findings; `clean=true` only with `outcome=converged`; and a final plan whose contracts map to repository evidence or explicit user requirements. Re-review any later material plan change. diff --git a/skills/project-plan-review/evals/eval-prompts.md b/skills/project-plan-review/evals/eval-prompts.md new file mode 100644 index 0000000..ff9284c --- /dev/null +++ b/skills/project-plan-review/evals/eval-prompts.md @@ -0,0 +1,23 @@ +# Evaluation prompts + +Use these prompts to evaluate whether the skill routes correctly and enforces its handoff contract. + +## Eval 1: substantial migration + +> Plan a multi-tenant billing migration across the API, workers, and database. Have Claudex pressure-test it while I keep discussing rollout questions with you. + +Expected: main Drake grounds the repo and drafts `PLAN.md`, delegates a self-contained absolute-path adapter run to a leaf, stays user-facing, reads artifacts back, and rejects invented contracts. + +## Eval 2: tiny fix + +> Fix the typo in the settings button label and tell me what changed. + +Expected: skip this workflow entirely because the change is tiny. + +## Eval 3: non-converged result + +> Review the disaster-recovery plan with two rounds and proceed only if it is genuinely clean. + +Fixture/result: adapter returns `max_reached` with material final findings. + +Expected: main Drake reports non-clean, does not call the plan approved, reads final findings, normalizes the plan if useful, and attaches it with unresolved concerns. diff --git a/skills/project-plan-review/references/runbook.md b/skills/project-plan-review/references/runbook.md new file mode 100644 index 0000000..48c1446 --- /dev/null +++ b/skills/project-plan-review/references/runbook.md @@ -0,0 +1,60 @@ +# Delegation runbook + +## Inputs main Drake must prepare + +Resolve all paths before delegation: + +- `REPO`: trusted absolute git working-tree path +- `PLAN`: absolute existing, non-empty plan path (normally `$REPO/PLAN.md`) +- `ADAPTER`: staged `bin/claudex-plan-review` absolute path +- `PLUGIN_ROOT`: staged `plugins/claudex` absolute path +- `CLAUDE`: vetted absolute Claude Code executable +- `CODEX`: vetted absolute Codex executable +- `EVIDENCE`: new absolute output directory +- `ROUNDS=5` for the standard project-plan-review sweep +- positive `TIMEOUT_SECONDS` and `BUDGET_USD` +- the Hermes delegation wall-clock limit from `delegation.child_timeout_seconds` + +## Timeout budget invariant + +The outer Hermes leaf must outlive the adapter plus artifact read-back and summary. Before calling `delegate_task`, read `delegation.child_timeout_seconds` from the active profile and require one of: + +- `child_timeout_seconds: 0` (no delegation wall-clock cap), or +- `child_timeout_seconds >= TIMEOUT_SECONDS + 180`. + +The 180-second reserve is mandatory. Never give the adapter a timeout equal to or greater than the child timeout. If this invariant is violated, the adapter can finish while Hermes reports the leaf as timed out before artifact verification. A timed-out outer leaf is not proof that the adapter failed: inspect `result.json`, process state, and copied evidence before classifying or retrying. + +## `delegate_task` goal/context template + +Use the available `delegate_task` tool with a prompt equivalent to this, filling every placeholder: + +> You are a leaf execution subagent. Do not ask Rob questions and do not delegate. Run exactly one bounded sweep-v2 plan-review adapter operation, then return the exact JSON result and absolute artifact paths. Repository: ``. Existing plan: ``. Grounded topic/constraints: ``. Adapter: ``. Claudex plugin root: ``. Claude executable: ``. Codex executable: ``. Evidence directory: ``. Run: ` --repo --plan --topic --engine sweep-v2 --rounds 5 --timeout --budget-usd --plugin-root --claude --codex --output-dir `. Preserve stdout exactly. A nonzero exit is an outcome to report, not a reason to improvise. Do not implement, commit, push, install skills/plugins, edit global configuration, or touch files outside the adapter's documented scope. Before returning, verify `result.json`, final plan, copied state, generation manifest, generation evidence directory, and consolidated findings exist. Return outcome, exit code, generation/max-generations, snapshot hashes, persona coverage, and paths; never call a non-converged run clean. + +Do not omit context on the assumption the child can read the parent conversation. It cannot ask the user to fill gaps. + +## Main Drake read-back + +After the child returns: + +1. Read `/result.json`; reject malformed or mismatched results. +2. Require `engine=sweep-v2`, `generation` and `max_generations` in `1..5`, with the normal workflow reporting `max_generations=5`. +3. Read copied `evidence_state_file`, `generation_manifest`, `consolidated_findings`, and every persona finding/sidecar under `generation_evidence_dir`. `state_file` and `final_findings` retain legacy source-path semantics; do not rely on them after cleanup. +4. Read `` again and compare it with the grounded scope and reported snapshot hash. +5. Check outcome invariants: + - `converged`: exit 0 and `clean=true`; state is done/converged/clean/coverage-complete; exactly the five required personas have readable clean evidence tied to one hash; snapshot and converged hashes match; manifest and consolidated hashes validate; consolidated findings prove all five are clean. + - `max_reached`: exit 10 and `clean=false`; the capped generation has complete same-snapshot coverage and material findings. At the standard cap this is generation five. + - `degraded`: exit 11 and `clean=false`; missing, malformed, mutated, hash-mismatched, cancelled, or incomplete state/evidence. + - `failed`: exit 12 and `clean=false`. + - `timed_out`: exit 124 and `clean=false`; process group was terminated and reaped. +6. Reject invented APIs, data models, deployment guarantees, and scope additions unless they map to repository facts or explicit requirements. +7. Normalize the active implementation phase if review churn left historical/completed phases in the plan. +8. If normalization is material—not merely formatting or wording—run a fresh five-generation-cap sweep-v2 review. Convergence is bound to the reported snapshot hash. +9. Attach the final plan and disclose unresolved concerns and telemetry limitations. + +## Safety and telemetry notes + +- Claude Code and Codex are subscription-backed here. `budget_usd` and `reported_claude_cost_usd` are CLI usage-equivalent telemetry/bounded-run controls, not proof of direct API billing or a charged invoice. +- Codex subscription usage is separate, may be rate-limited, and is not dollar-capped by the adapter. Each sweep generation can run all five persona reviews. +- The adapter pins child `PATH` from explicit executable directories and system paths. +- Never point the adapter at an untrusted repository: it runs Claude with bypassed permission prompts so the Stop hook can operate unattended. +- `--engine legacy` exists only for backward compatibility. Drake's project-plan-review workflow uses sweep-v2 with five generations. diff --git a/tests/test_adapter.py b/tests/test_adapter.py new file mode 100644 index 0000000..ab6b285 --- /dev/null +++ b/tests/test_adapter.py @@ -0,0 +1,280 @@ +#!/usr/bin/env python3 +import json +import os +import subprocess +import tempfile +import textwrap +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +ADAPTER = ROOT / "bin" / "claudex-plan-review" +PLUGIN = ROOT / "plugins" / "claudex" +PERSONAS = [ + "architecture-scope", + "security-data", + "product-domain", + "quality-accessibility-performance", + "operations-deployment", +] + + +class AdapterTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.base = Path(self.tmp.name) + self.repo = self.base / "repo" + self.repo.mkdir() + subprocess.run(["git", "init", "-q", str(self.repo)], check=True) + self.plan = self.repo / "PLAN.md" + self.plan.write_text("# Plan\n\n1. Do the thing safely.\n") + self.bin = self.base / "bin" + self.bin.mkdir() + self.claude = self.bin / "claude" + self.codex = self.bin / "codex" + self._write_executable(self.codex, """#!/bin/sh +case "$1 $2" in + "login status") echo 'Logged in using ChatGPT'; exit 0 ;; +esac +echo 'codex-cli 0.test' +""") + self._write_executable(self.claude, r"""#!/usr/bin/env python3 +import datetime, hashlib, json, os, pathlib, re, subprocess, sys, time, uuid +if sys.argv[1:] == ['--version']: + print('2.test'); raise SystemExit(0) +if sys.argv[1:] == ['auth', 'status']: + print(json.dumps({'loggedIn': True, 'authMethod': 'test'})); raise SystemExit(0) +outcome = os.environ.get('FAKE_CLAUDE_OUTCOME', 'sweep_clean') +prompt = sys.argv[-1] +marker = os.environ.get('FAKE_PROMPT_FILE') +if marker: pathlib.Path(marker).write_text(prompt) +if outcome == 'failed': + print('forced failure', file=sys.stderr); raise SystemExit(7) +if outcome == 'timeout': + child = subprocess.Popen(['sleep', '60']) + marker = os.environ.get('FAKE_CHILD_PID_FILE') + if marker: pathlib.Path(marker).write_text(str(child.pid)) + time.sleep(60); raise SystemExit(0) +state_dir = pathlib.Path.cwd() / '.claude' / 'claudex' +state_dir.mkdir(parents=True, exist_ok=True) +rid = '20990101-000000-' + uuid.uuid4().hex[:6] +review_dir = state_dir / rid +review_dir.mkdir() +state_path = state_dir / (rid + '.state') +if outcome.startswith('legacy_'): + if outcome == 'legacy_max': + signal, findings, round_value = 'max-reached', '# Round 1 findings\n\n## High\n- unsafe gap (fix it)\n', '1' + elif outcome == 'legacy_degraded': + signal, findings, round_value = 'no-material-findings', '# Round 1 findings\n\n## High\n- contradiction (fix it)\n', '1' + else: + signal, findings, round_value = 'no-material-findings', '# Round 1 findings\n\nNo substantive findings.\n', '1' + (review_dir / 'findings-round-1.md').write_text(findings) + state_path.write_text(f'''mode: plan\nphase: done\ntopic: test\nround: {round_value}\nmax_rounds: 1\nreview_id: {rid}\nrepo_root: {pathlib.Path.cwd().resolve()}\ndecision_signal: {signal}\n''') +else: + personas = ['architecture-scope','security-data','product-domain','quality-accessibility-performance','operations-deployment'] + match = re.search(r'--rounds (\d+)', prompt) + maximum = int(match.group(1)) if match else 5 + generation = 5 if outcome == 'sweep_max' else 1 + generation_dir = review_dir / 'generations' / str(generation) + generation_dir.mkdir(parents=True) + snapshot = generation_dir / 'PLAN.md' + snapshot.write_bytes((pathlib.Path.cwd() / 'PLAN.md').read_bytes()) + snapshot_hash = hashlib.sha256(snapshot.read_bytes()).hexdigest() + previous_hash = None + if generation > 1: + previous_dir = review_dir / 'generations' / str(generation - 1) + previous_dir.mkdir(parents=True) + (previous_dir / 'manifest.json').write_text(json.dumps({'snapshot_sha256': snapshot_hash})) + previous_hash = snapshot_hash + manifest = { + 'generation': generation, 'snapshot_sha256': snapshot_hash, + 'required_persona_ids': personas, 'topic': 'grounded test topic', + 'source_plan_path': str(pathlib.Path.cwd().resolve() / 'PLAN.md'), + 'previous_generation_sha256': previous_hash, + } + (generation_dir / 'manifest.json').write_text(json.dumps(manifest, indent=2, sort_keys=True) + '\n') + chunks = [f'# Consolidated findings — generation {generation}\n\n', f'Snapshot SHA-256: `{snapshot_hash}`\n\n'] + evidence = hashlib.sha256() + for persona in personas: + findings = generation_dir / f'{persona}.findings.md' + sidecar = generation_dir / f'{persona}.result.json' + material = outcome == 'sweep_max' and persona == 'security-data' + if material: + text = '## High\n- Scope: material generation-five gap (fix it).\n## Medium\n## Low\n' + classification = 'material' + else: + text = 'No substantive findings.\n' + classification = 'clean' + if outcome == 'sweep_malformed' and persona == 'product-domain': + text = 'not valid findings\n' + findings.write_text(text) + findings_hash = hashlib.sha256(findings.read_bytes()).hexdigest() + expected = '0' * 64 if outcome == 'sweep_hash_mismatch' and persona == 'security-data' else snapshot_hash + data = { + 'persona_id': persona, 'expected_snapshot_sha256': expected, + 'actual_snapshot_sha256_before': snapshot_hash, + 'actual_snapshot_sha256_after': snapshot_hash, 'codex_exit_code': 0, + 'findings_path': str(findings), 'findings_sha256': findings_hash, + 'findings_classification': classification, + 'completed_at': '2099-01-01T00:00:00Z', + } + sidecar.write_text(json.dumps(data, indent=2, sort_keys=True) + '\n') + chunks.append(f'## {persona}\n\n') + if material: + chunks.append(f'## High\n- [{persona}-high-1] Scope: material generation-five gap (fix it).\n## Medium\n## Low\n\n\n') + else: + chunks.append('No substantive findings.\n\n\n') + for artifact in (findings, sidecar): + evidence.update(artifact.name.encode() + b'\0') + evidence.update(artifact.read_bytes()) + evidence.update(b'\0') + if outcome == 'sweep_missing': + (generation_dir / 'operations-deployment.result.json').unlink() + consolidated = generation_dir / 'consolidated-findings.md' + consolidated.write_text(''.join(chunks)) + if outcome == 'sweep_mutated_consolidated': + consolidated.write_text(consolidated.read_text() + 'Claude says this is still clean.\n') + signal = 'max-reached' if outcome == 'sweep_max' else ('cancelled' if outcome == 'sweep_cancelled' else ('degraded' if outcome == 'sweep_degraded' else 'converged')) + phase = 'cancelled' if outcome == 'sweep_cancelled' else 'done' + clean = 'false' if outcome in {'sweep_max', 'sweep_cancelled', 'sweep_degraded'} else 'true' + converged_hash = '' if outcome in {'sweep_max', 'sweep_cancelled', 'sweep_degraded'} else snapshot_hash + state_path.write_text(f'''mode: plan\nphase: {phase}\ntopic: test\nround: {generation}\nmax_rounds: {maximum}\nreview_id: {rid}\nrepo_root: {pathlib.Path.cwd().resolve()}\ndecision_signal: {signal}\nengine: sweep-v2\ngeneration: {generation}\nmax_generations: {maximum}\nsnapshot_sha256: {snapshot_hash}\ncoverage_complete: true\nclean: {clean}\nrevision_required: false\nreviewed_live_sha256: {snapshot_hash}\nevidence_sha256: {evidence.hexdigest()}\nconsolidated_sha256: {hashlib.sha256(consolidated.read_bytes()).hexdigest()}\nconverged_snapshot_sha256: {converged_hash}\n''') +print(json.dumps({'type': 'result', 'subtype': 'success', 'total_cost_usd': 0.01, 'session_id': 'fake-session'})) +""") + + def tearDown(self): + self.tmp.cleanup() + + def _write_executable(self, path, content): + path.write_text(textwrap.dedent(content)) + path.chmod(0o755) + + def run_adapter(self, outcome="sweep_clean", timeout="5", extra_env=None, engine="sweep-v2", rounds="1"): + env = os.environ.copy() + env["FAKE_CLAUDE_OUTCOME"] = outcome + if extra_env: + env.update(extra_env) + command = [ + str(ADAPTER), "--repo", str(self.repo.resolve()), "--plan", str(self.plan.resolve()), + "--topic", "review the grounded plan", "--rounds", rounds, "--timeout", timeout, + "--budget-usd", "1.25", "--plugin-root", str(PLUGIN.resolve()), + "--claude", str(self.claude.resolve()), "--codex", str(self.codex.resolve()), + "--engine", engine, + ] + completed = subprocess.run(command, text=True, capture_output=True, env=env, timeout=10) + lines = completed.stdout.splitlines() + self.assertEqual(len(lines), 1, completed.stdout) + return completed, json.loads(lines[0]) + + def test_sweep_clean_requires_exact_same_snapshot_five_persona_coverage(self): + marker = self.base / "prompt.txt" + completed, result = self.run_adapter(extra_env={"FAKE_PROMPT_FILE": str(marker)}) + self.assertEqual(completed.returncode, 0) + self.assertEqual(result["outcome"], "converged") + self.assertTrue(result["clean"]) + self.assertEqual(result["engine"], "sweep-v2") + self.assertEqual((result["generation"], result["max_generations"]), (1, 1)) + self.assertEqual(len(result["persona_coverage"]), 5) + self.assertEqual([item["persona_id"] for item in result["persona_coverage"]], PERSONAS) + self.assertEqual({item["snapshot_sha256"] for item in result["persona_coverage"]}, {result["snapshot_sha256"]}) + self.assertIn("/claudex:plan --engine sweep-v2 --from-draft --skip-interview --rounds 1", marker.read_text()) + for key in ("evidence_state_file", "generation_manifest", "generation_evidence_dir", "consolidated_findings"): + self.assertTrue(Path(result[key]).exists(), key) + self.assertTrue(str(Path(result[key])).startswith(result["evidence_dir"])) + + def test_generation_five_material_findings_are_max_reached(self): + completed, result = self.run_adapter("sweep_max", rounds="5") + self.assertEqual(completed.returncode, 10) + self.assertEqual(result["outcome"], "max_reached") + self.assertFalse(result["clean"]) + self.assertEqual((result["generation"], result["max_generations"]), (5, 5)) + self.assertEqual(result["findings_status"], "material") + + def test_missing_persona_evidence_cannot_be_clean(self): + completed, result = self.run_adapter("sweep_missing") + self.assertEqual(completed.returncode, 11) + self.assertEqual(result["outcome"], "degraded") + self.assertFalse(result["clean"]) + + def test_malformed_findings_cannot_be_clean(self): + completed, result = self.run_adapter("sweep_malformed") + self.assertEqual(completed.returncode, 11) + self.assertFalse(result["clean"]) + self.assertEqual(result["findings_status"], "malformed") + + def test_hash_mismatch_cannot_be_clean(self): + completed, result = self.run_adapter("sweep_hash_mismatch") + self.assertEqual(completed.returncode, 11) + self.assertFalse(result["clean"]) + self.assertIn("hash-mismatched", result["reason"]) + + def test_mutated_consolidated_findings_cannot_be_clean(self): + completed, result = self.run_adapter("sweep_mutated_consolidated") + self.assertEqual(completed.returncode, 11) + self.assertFalse(result["clean"]) + self.assertIn("consolidated findings", result["reason"]) + + def test_degraded_and_cancelled_state_are_never_clean(self): + for fixture in ("sweep_degraded", "sweep_cancelled"): + with self.subTest(fixture=fixture): + completed, result = self.run_adapter(fixture) + self.assertEqual(completed.returncode, 11) + self.assertEqual(result["outcome"], "degraded") + self.assertFalse(result["clean"]) + + def test_legacy_engine_remains_compatible(self): + completed, result = self.run_adapter("legacy_clean", engine="legacy") + self.assertEqual(completed.returncode, 0) + self.assertTrue(result["clean"]) + self.assertEqual(result["engine"], "legacy") + self.assertEqual((result["round"], result["max_rounds"]), (1, 1)) + self.assertIsNone(result["generation"]) + self.assertTrue(Path(result["final_findings"]).is_file()) + + def test_legacy_signal_cannot_override_material_findings(self): + completed, result = self.run_adapter("legacy_degraded", engine="legacy") + self.assertEqual(completed.returncode, 11) + self.assertFalse(result["clean"]) + + def test_nonzero_claude_is_failed(self): + completed, result = self.run_adapter("failed") + self.assertEqual(completed.returncode, 12) + self.assertEqual(result["outcome"], "failed") + self.assertFalse(result["clean"]) + + def test_timeout_kills_and_reaps_process_group(self): + marker = self.base / "child.pid" + completed, result = self.run_adapter("timeout", timeout="0.5", extra_env={"FAKE_CHILD_PID_FILE": str(marker)}) + self.assertEqual(completed.returncode, 124) + self.assertEqual(result["outcome"], "timed_out") + pid = int(marker.read_text()) + probe = subprocess.run(["kill", "-0", str(pid)], capture_output=True) + self.assertNotEqual(probe.returncode, 0, f"child process {pid} survived timeout") + + def test_second_consecutive_run_uses_new_state(self): + first, result1 = self.run_adapter() + second, result2 = self.run_adapter() + self.assertEqual((first.returncode, second.returncode), (0, 0)) + self.assertNotEqual(result1["review_id"], result2["review_id"]) + self.assertNotEqual(result1["state_file"], result2["state_file"]) + + def test_sweep_generation_cap_is_rejected_before_launch(self): + completed, result = self.run_adapter(rounds="6") + self.assertEqual(completed.returncode, 12) + self.assertEqual(result["error"]["kind"], "validation") + self.assertIn("1..5", result["error"]["message"]) + + def test_relative_repo_is_rejected_as_json(self): + completed = subprocess.run([ + str(ADAPTER), "--repo", "relative", "--plan", str(self.plan.resolve()), "--topic", "x", + "--rounds", "1", "--timeout", "1", "--budget-usd", "1", + "--plugin-root", str(PLUGIN.resolve()), "--claude", str(self.claude.resolve()), + "--codex", str(self.codex.resolve()), + ], text=True, capture_output=True) + result = json.loads(completed.stdout) + self.assertEqual(completed.returncode, 12) + self.assertEqual(result["error"]["kind"], "validation") + + +if __name__ == "__main__": + unittest.main()