From c4f8ac5b92f58bb3339734b26851623d5e3cccbe Mon Sep 17 00:00:00 2001 From: robgfl45 Date: Sat, 11 Jul 2026 04:08:45 -0400 Subject: [PATCH 01/10] feat: add headless Hermes plan review bridge (#1) Co-authored-by: robgfl45 <227035225+robgfl45@users.noreply.github.com> --- README.md | 17 +- bin/claudex-plan-review | 407 ++++++++++++++++++ docs/HEADLESS_ADAPTER.md | 61 +++ install.sh | 4 +- plugins/claudex/docs/ARCHITECTURE.md | 12 +- plugins/claudex/hooks/stop-hook.sh | 8 +- plugins/claudex/scripts/start-loop.sh | 2 +- plugins/claudex/scripts/state-helpers.sh | 42 +- plugins/claudex/tests/platform-validation.sh | 37 ++ plugins/claudex/tests/synthetic-e2e.sh | 16 +- skills/project-plan-review/SKILL.md | 37 ++ .../project-plan-review/evals/eval-prompts.md | 23 + .../project-plan-review/references/runbook.md | 46 ++ tests/test_adapter.py | 145 +++++++ 14 files changed, 823 insertions(+), 34 deletions(-) create mode 100755 bin/claudex-plan-review create mode 100644 docs/HEADLESS_ADAPTER.md create mode 100644 skills/project-plan-review/SKILL.md create mode 100644 skills/project-plan-review/evals/eval-prompts.md create mode 100644 skills/project-plan-review/references/runbook.md create mode 100644 tests/test_adapter.py diff --git a/README.md b/README.md index ac71bec..8bcf8e3 100644 --- a/README.md +++ b/README.md @@ -233,6 +233,12 @@ The Stop hook is fail-open everywhere. Any error returns `{"decision":"approve"} | `CLAUDEX_STALE_MINUTES` | 15 | Loops older than this are auto-swept on next invocation | | `CLAUDEX_STATE_DIR` | `.claude/claudex` | State directory location | +## Headless Hermes planning bridge + +This fork adds [`bin/claudex-plan-review`](docs/HEADLESS_ADAPTER.md), a production-oriented adapter for running an existing `PLAN.md` through headless Claude Code, the Claudex Stop-hook lifecycle, and real Codex reviews from a Hermes leaf subagent. It validates explicit executable/plugin/auth prerequisites, pins child `PATH`, enforces wall-clock and Claude budget bounds, kills the complete process group on timeout, preserves evidence, and emits one strict JSON result. + +Only `converged` is clean. `max_reached`, `degraded`, `failed`, and `timed_out` are explicit non-clean outcomes. Classification is based on Claudex state and findings artifacts, never Claude's prose tally. See the adapter document for exact usage, exit codes, costs, architecture, and staging instructions. The in-repo Hermes skill is staged at [`skills/project-plan-review/`](skills/project-plan-review/SKILL.md); it is not installed automatically. + ## Cost expectation Each plan-mode round is one full Codex review of `PLAN.md`. In practice that's ~25–30k Codex tokens per round. With the default 3 rounds you should expect **~75–90k tokens per `/claudex:plan`**. Codex authenticates against your ChatGPT account, so the bill goes to your ChatGPT Plus / Pro / Team / Enterprise plan, not to claudex. If you're on a tight rate limit, run `--rounds 2` for fast topics and reserve `--rounds 5+` for high-stakes designs. @@ -260,17 +266,20 @@ Highlights: ## Tests ```bash -# Phase 0: confirm platform behaviors work on your machine (50 checks) +# Phase 0: confirm platform behaviors work on your machine (count printed by test) bash plugins/claudex/tests/platform-validation.sh -# Smoke test: simulate full lifecycle without invoking Codex (60 checks) +# Smoke test: simulate full lifecycle without invoking Codex (count printed by test) bash plugins/claudex/tests/smoke-test.sh -# Synthetic E2E: real Codex calls against a throwaway repo (19 checks, costs a few cents in tokens) +# Synthetic E2E: real Codex calls against a throwaway repo (count printed by test; uses subscription tokens) bash plugins/claudex/tests/synthetic-e2e.sh + +# Headless adapter deterministic unit/error/timeout/state-isolation tests +python3 -m unittest -v tests/test_adapter.py ``` -All three should pass before trusting claudex on a real project. +Run the platform, smoke, and adapter suites for every change. Run the live synthetic E2E when authenticated Codex usage is available. ## Project structure diff --git a/bin/claudex-plan-review b/bin/claudex-plan-review new file mode 100755 index 0000000..6e6ceed --- /dev/null +++ b/bin/claudex-plan-review @@ -0,0 +1,407 @@ +#!/usr/bin/env python3 +"""Headless Claude Code -> Claudex -> Codex plan review adapter. + +Stdout is always one strict JSON document. Human/debug output and subprocess +streams are retained in an evidence directory instead of being mixed into the +machine-readable result. +""" + +from __future__ import annotations + +import argparse +import datetime as dt +import json +import os +import re +import shutil +import signal +import subprocess +import sys +import time +import uuid +from decimal import Decimal, InvalidOperation +from pathlib import Path +from typing import Any + +TERMINAL_PHASES = {"done", "cancelled", "errored"} +ACTIVE_PHASES = {"drafting", "reviewing", "revising", "summarizing"} +EXIT_CODES = {"converged": 0, "max_reached": 10, "degraded": 11, "failed": 12, "timed_out": 124} + + +def emit(payload: dict[str, Any], code: int) -> int: + print(json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False)) + return code + + +def fail(message: str, *, kind: str = "validation", evidence_dir: Path | None = None) -> int: + payload: dict[str, Any] = { + "schema_version": 1, + "outcome": "failed", + "clean": False, + "error": {"kind": kind, "message": message}, + } + if evidence_dir is not None: + payload["evidence_dir"] = str(evidence_dir) + return emit(payload, EXIT_CODES["failed"]) + + +def absolute_existing_dir(value: str, label: str) -> Path: + path = Path(value) + if not path.is_absolute(): + raise ValueError(f"{label} must be an absolute path") + resolved = path.resolve(strict=True) + if not resolved.is_dir(): + raise ValueError(f"{label} is not a directory: {resolved}") + return resolved + + +def absolute_existing_file(value: str, label: str) -> Path: + path = Path(value) + if not path.is_absolute(): + raise ValueError(f"{label} must be an absolute path") + resolved = path.resolve(strict=True) + if not resolved.is_file(): + raise ValueError(f"{label} is not a file: {resolved}") + if resolved.stat().st_size == 0: + raise ValueError(f"{label} must be non-empty: {resolved}") + return resolved + + +def executable(value: str, label: str) -> Path: + path = absolute_existing_file(value, label) + if not os.access(path, os.X_OK): + raise ValueError(f"{label} is not executable: {path}") + return path + + +def run_probe(argv: list[str], cwd: Path, env: dict[str, str], timeout: float = 20) -> subprocess.CompletedProcess[str]: + return subprocess.run(argv, cwd=cwd, env=env, text=True, capture_output=True, timeout=timeout, check=False) + + +def parse_state(path: Path) -> dict[str, str]: + result: dict[str, str] = {} + try: + for line in path.read_text(encoding="utf-8", errors="replace").splitlines(): + if ":" not in line: + continue + key, value = line.split(":", 1) + if re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", key): + result[key] = value.strip() + except OSError: + pass + return result + + +def snapshot_state(state_file: Path, events, last: dict[str, str] | None) -> dict[str, str] | None: + if not state_file.exists(): + return last + current = parse_state(state_file) + if current != last: + events.write(json.dumps({"at": dt.datetime.now(dt.timezone.utc).isoformat(), "state": current}, sort_keys=True) + "\n") + events.flush() + return current + + +def final_findings_status(path: Path | None) -> tuple[str, dict[str, int]]: + counts = {"high": 0, "medium": 0, "low": 0} + if path is None or not path.is_file(): + return "missing", counts + text = path.read_text(encoding="utf-8", errors="replace") + section = "" + for line in text.splitlines(): + if line.startswith("## High"): + section = "high" + elif line.startswith("## Medium"): + section = "medium" + elif line.startswith("## Low"): + section = "low" + elif line.startswith("## "): + section = "" + elif line.startswith("- ") and section: + counts[section] += 1 + no_findings = bool(re.search(r"(?mi)^No substantive findings\.\s*$", text)) + if no_findings and not any(counts.values()): + return "none", counts + if any(counts.values()): + return "material", counts + return "unparseable", counts + + +def parse_claude_stream(path: Path) -> dict[str, Any]: + summary: dict[str, Any] = {"result_records": 0, "reported_cost_usd": None, "session_id": None} + try: + for line in path.read_text(encoding="utf-8", errors="replace").splitlines(): + try: + obj = json.loads(line) + except json.JSONDecodeError: + continue + if obj.get("type") == "result": + summary["result_records"] += 1 + summary["reported_cost_usd"] = obj.get("total_cost_usd", obj.get("cost_usd")) + summary["session_id"] = obj.get("session_id") + summary["claude_result_subtype"] = obj.get("subtype") + except OSError: + pass + return summary + + +def copy_if_exists(source: Path, destination: Path) -> None: + if source.exists(): + if source.is_dir(): + shutil.copytree(source, destination, dirs_exist_ok=True) + else: + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, destination) + + +class JsonArgumentParser(argparse.ArgumentParser): + def error(self, message: str) -> None: + payload = { + "schema_version": 1, + "outcome": "failed", + "clean": False, + "error": {"kind": "arguments", "message": message}, + } + print(json.dumps(payload, sort_keys=True, separators=(",", ":"))) + raise SystemExit(EXIT_CODES["failed"]) + + +def main() -> int: + parser = JsonArgumentParser(description="Run a bounded headless Claudex review of an existing plan") + parser.add_argument("--repo", required=True, help="absolute repository path") + parser.add_argument("--plan", required=True, help="absolute path to an existing non-empty PLAN.md") + parser.add_argument("--topic", required=True, help="self-contained review topic and constraints") + parser.add_argument("--rounds", required=True, type=int, help="positive Claudex round limit") + parser.add_argument("--timeout", required=True, type=float, help="wall-clock timeout in seconds") + parser.add_argument("--budget-usd", required=True, help="positive Claude Code budget cap") + default_plugin = Path(__file__).resolve().parents[1] / "plugins" / "claudex" + parser.add_argument("--plugin-root", default=str(default_plugin), help="absolute Claudex plugin root") + parser.add_argument("--claude", required=True, help="absolute Claude Code executable") + parser.add_argument("--codex", required=True, help="absolute Codex executable") + parser.add_argument("--output-dir", help="absolute evidence directory (must not already exist)") + parser.add_argument("--model", default="sonnet", help="Claude model passed to headless Claude Code") + args = parser.parse_args() + + try: + repo = absolute_existing_dir(args.repo, "--repo") + if not (repo / ".git").exists(): + raise ValueError(f"--repo must be a git working tree: {repo}") + plan = absolute_existing_file(args.plan, "--plan") + plugin = absolute_existing_dir(args.plugin_root, "--plugin-root") + claude = executable(args.claude, "--claude") + codex = executable(args.codex, "--codex") + if args.rounds < 1: + raise ValueError("--rounds must be a positive integer") + if args.timeout <= 0: + raise ValueError("--timeout must be positive") + if not args.topic.strip(): + raise ValueError("--topic must be non-empty") + try: + budget = Decimal(args.budget_usd) + except InvalidOperation as exc: + raise ValueError("--budget-usd must be a positive decimal") from exc + if not budget.is_finite() or budget <= 0: + raise ValueError("--budget-usd must be a positive decimal") + required = [plugin / ".claude-plugin" / "plugin.json", plugin / "commands" / "plan.md", plugin / "hooks" / "stop-hook.sh", plugin / "scripts" / "state-helpers.sh"] + missing = [str(path) for path in required if not path.is_file()] + if missing: + raise ValueError("invalid Claudex plugin root; missing: " + ", ".join(missing)) + manifest = json.loads(required[0].read_text(encoding="utf-8")) + if manifest.get("name") != "claudex": + raise ValueError(f"plugin manifest is not Claudex: {required[0]}") + except (ValueError, OSError) as exc: + return fail(str(exc)) + + state_dir = repo / ".claude" / "claudex" + state_dir.mkdir(parents=True, exist_ok=True) + baseline_states = set(state_dir.glob("*.state")) + for state_file in baseline_states: + phase = parse_state(state_file).get("phase", "") + if phase in ACTIVE_PHASES or (phase and phase not in TERMINAL_PHASES): + return fail(f"active Claudex loop already exists: {state_file} (phase={phase})", kind="active_loop") + + stamp = dt.datetime.now(dt.timezone.utc).strftime("%Y%m%dT%H%M%SZ") + if args.output_dir: + output_dir_arg = Path(args.output_dir) + if not output_dir_arg.is_absolute(): + return fail("--output-dir must be an absolute path") + evidence = output_dir_arg + else: + evidence = state_dir / "adapter-runs" / f"{stamp}-{uuid.uuid4().hex[:8]}" + if evidence.exists(): + return fail(f"evidence directory already exists: {evidence}") + evidence.mkdir(parents=True) + + env = os.environ.copy() + pinned_path = os.pathsep.join(dict.fromkeys([str(claude.parent), str(codex.parent), "/usr/bin", "/bin", "/usr/sbin", "/sbin"])) + env["PATH"] = pinned_path + env["CLAUDEX_ADAPTER_EVIDENCE_DIR"] = str(evidence) + + preflight: dict[str, Any] = {"pinned_path": pinned_path, "plugin_root": str(plugin)} + try: + claude_version = run_probe([str(claude), "--version"], repo, env) + codex_version = run_probe([str(codex), "--version"], repo, env) + claude_auth = run_probe([str(claude), "auth", "status"], repo, env) + codex_auth = run_probe([str(codex), "login", "status"], repo, env) + except (OSError, subprocess.TimeoutExpired) as exc: + return fail(f"prerequisite probe failed: {exc}", kind="prerequisite", evidence_dir=evidence) + preflight.update({ + "claude": {"path": str(claude), "version": claude_version.stdout.strip()}, + "codex": {"path": str(codex), "version": codex_version.stdout.strip()}, + "claude_auth": claude_auth.stdout.strip(), + "codex_auth": (codex_auth.stdout + codex_auth.stderr).strip(), + }) + (evidence / "preflight.json").write_text(json.dumps(preflight, indent=2, sort_keys=True) + "\n") + if claude_version.returncode != 0 or codex_version.returncode != 0: + return fail("Claude or Codex version probe failed", kind="prerequisite", evidence_dir=evidence) + try: + auth_obj = json.loads(claude_auth.stdout) + except json.JSONDecodeError: + auth_obj = {} + if claude_auth.returncode != 0 or auth_obj.get("loggedIn") is not True: + return fail("Claude Code authentication is not active", kind="authentication", evidence_dir=evidence) + if codex_auth.returncode != 0 or "logged in" not in (codex_auth.stdout + codex_auth.stderr).lower(): + return fail("Codex authentication is not active", kind="authentication", evidence_dir=evidence) + + runtime_plan = repo / "PLAN.md" + external_plan = plan != runtime_plan.resolve(strict=False) + original_runtime_plan = evidence / "repo-PLAN.original.md" + if external_plan and runtime_plan.exists(): + shutil.copy2(runtime_plan, original_runtime_plan) + shutil.copy2(plan, evidence / "PLAN.before.md") + if external_plan: + shutil.copy2(plan, runtime_plan) + + prompt = f"/claudex:plan --from-draft --skip-interview --rounds {args.rounds} {args.topic.strip()}" + command = [ + str(claude), "--print", "--verbose", "--output-format", "stream-json", "--include-hook-events", + "--dangerously-skip-permissions", "--setting-sources", "project", "--plugin-dir", str(plugin), + "--model", args.model, "--max-budget-usd", str(budget), prompt, + ] + metadata = { + "argv": command, "cwd": str(repo), "timeout_seconds": args.timeout, + "budget_usd": str(budget), "rounds": args.rounds, "topic": args.topic, + "started_at": dt.datetime.now(dt.timezone.utc).isoformat(), + } + (evidence / "run-metadata.json").write_text(json.dumps(metadata, indent=2, sort_keys=True) + "\n") + + stdout_path = evidence / "claude-stream.jsonl" + stderr_path = evidence / "claude-stderr.log" + events_path = evidence / "state-events.jsonl" + started = time.monotonic() + timed_out = False + process: subprocess.Popen[bytes] | None = None + selected_state: Path | None = None + last_state: dict[str, str] | None = None + try: + with stdout_path.open("wb") as stdout, stderr_path.open("wb") as stderr, events_path.open("w", encoding="utf-8") as events: + process = subprocess.Popen(command, cwd=repo, env=env, stdout=stdout, stderr=stderr, start_new_session=True) + while process.poll() is None: + new_states = [path for path in state_dir.glob("*.state") if path not in baseline_states] + if new_states: + selected_state = max(new_states, key=lambda path: path.stat().st_mtime_ns) + last_state = snapshot_state(selected_state, events, last_state) + if time.monotonic() - started >= args.timeout: + timed_out = True + os.killpg(process.pid, signal.SIGTERM) + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + process.wait(timeout=5) + break + time.sleep(0.2) + returncode = process.wait() if process.poll() is None else process.returncode + new_states = [path for path in state_dir.glob("*.state") if path not in baseline_states] + if new_states: + selected_state = max(new_states, key=lambda path: path.stat().st_mtime_ns) + last_state = snapshot_state(selected_state, events, last_state) + except (OSError, subprocess.SubprocessError) as exc: + if process is not None and process.poll() is None: + try: + os.killpg(process.pid, signal.SIGKILL) + process.wait(timeout=5) + except (OSError, subprocess.SubprocessError): + pass + returncode = process.returncode if process and process.returncode is not None else 125 + (evidence / "adapter-error.txt").write_text(str(exc) + "\n") + + elapsed = round(time.monotonic() - started, 3) + if runtime_plan.exists(): + shutil.copy2(runtime_plan, evidence / "PLAN.after.md") + if external_plan: + shutil.copy2(runtime_plan, plan) + if external_plan: + if original_runtime_plan.exists(): + shutil.copy2(original_runtime_plan, runtime_plan) + else: + runtime_plan.unlink(missing_ok=True) + + state = parse_state(selected_state) if selected_state else {} + review_id = state.get("review_id") + findings_files: list[Path] = [] + if review_id: + findings_dir = state_dir / review_id + def finding_round(path: Path) -> int: + match = re.search(r"findings-round-(\d+)\.md$", path.name) + return int(match.group(1)) if match else -1 + findings_files = sorted(findings_dir.glob("findings-round-*.md"), key=finding_round) if findings_dir.is_dir() else [] + copy_if_exists(selected_state, evidence / "artifacts" / selected_state.name) + copy_if_exists(findings_dir, evidence / "artifacts" / review_id) + copy_if_exists(state_dir / "log", evidence / "artifacts" / "claudex.log") + + final_findings = findings_files[-1] if findings_files else None + findings_status, severity = final_findings_status(final_findings) + phase = state.get("phase") + decision = state.get("decision_signal") + if timed_out: + outcome = "timed_out" + reason = "wall-clock timeout exceeded; process group terminated and reaped" + elif returncode != 0: + outcome = "failed" + reason = f"Claude Code exited non-zero ({returncode})" + elif not selected_state or not state: + outcome = "degraded" + reason = "Claude exited zero but no new Claudex state artifact was found" + elif decision == "no-material-findings" and phase == "done" and findings_status == "none": + outcome = "converged" + reason = "terminal state and final findings agree that no substantive findings remain" + elif decision == "max-reached" and phase == "done" and findings_status in {"material", "unparseable"}: + outcome = "max_reached" + reason = "round budget exhausted without an authoritative clean findings artifact" + else: + outcome = "degraded" + reason = f"state/artifact mismatch or incomplete lifecycle (phase={phase}, signal={decision}, findings={findings_status})" + + stream_summary = parse_claude_stream(stdout_path) + result = { + "schema_version": 1, + "outcome": outcome, + "clean": outcome == "converged", + "reason": reason, + "repo": str(repo), + "plan": str(plan), + "review_id": review_id, + "phase": phase, + "decision_signal": decision, + "round": int(state["round"]) if state.get("round", "").isdigit() else None, + "max_rounds": int(state["max_rounds"]) if state.get("max_rounds", "").isdigit() else args.rounds, + "findings_status": findings_status, + "severity_counts": severity, + "final_findings": str(final_findings) if final_findings else None, + "evidence_dir": str(evidence), + "stdout_log": str(stdout_path), + "stderr_log": str(stderr_path), + "state_file": str(selected_state) if selected_state else None, + "elapsed_seconds": elapsed, + "process_exit_code": returncode, + "budget_usd": str(budget), + "reported_claude_cost_usd": stream_summary.get("reported_cost_usd"), + "claude_session_id": stream_summary.get("session_id"), + } + (evidence / "result.json").write_text(json.dumps(result, indent=2, sort_keys=True) + "\n") + return emit(result, EXIT_CODES[outcome]) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docs/HEADLESS_ADAPTER.md b/docs/HEADLESS_ADAPTER.md new file mode 100644 index 0000000..26d7f85 --- /dev/null +++ b/docs/HEADLESS_ADAPTER.md @@ -0,0 +1,61 @@ +# Headless Claudex planning bridge + +`bin/claudex-plan-review` is a bounded adapter for a Hermes leaf subagent (or any automation runner) to drive the existing Claude Code → Claudex Stop-hook → Codex plan-review lifecycle. + +## Architecture + +1. The caller supplies absolute paths for a trusted git repository, an existing non-empty plan, the Claudex plugin, Claude Code, and Codex. +2. The adapter validates paths, plugin files, versions, and both CLI authentication states. It creates no global installation and uses `--plugin-dir` for this Claude session only. +3. If the supplied plan is outside the repository, it is staged as `/PLAN.md`, reviewed, copied back, and any pre-existing repository plan is restored. +4. Claude Code runs headlessly with `/claudex:plan --from-draft --skip-interview`, explicit rounds and Claude budget, stream-JSON output, hook events, and a pinned child `PATH` whose first entries are the supplied executable directories. +5. The supervisor polls newly created `.claude/claudex/*.state` files, records state transitions, and preserves state, findings, logs, plans, and raw Claude streams. +6. A wall-clock timeout terminates the entire process group, waits five seconds, escalates to `SIGKILL`, and reaps it. +7. Classification uses state plus the final findings artifact. Claude's prose summary is never authoritative. + +The existing interactive review mode is unchanged. + +## Exact usage + +```bash +/path/to/claudex/bin/claudex-plan-review \ + --repo /absolute/path/to/project \ + --plan /absolute/path/to/project/PLAN.md \ + --topic "grounded feature scope, constraints, and explicit non-goals" \ + --rounds 3 \ + --timeout 900 \ + --budget-usd 5.00 \ + --plugin-root /absolute/path/to/claudex/plugins/claudex \ + --claude /absolute/path/to/claude \ + --codex /absolute/path/to/codex \ + --output-dir /absolute/new/evidence-directory +``` + +`--output-dir` is optional; the default is `.claude/claudex/adapter-runs/-`. `--model` defaults to `sonnet`. The repository must be trusted because unattended Claude runs with permission prompts bypassed so the plugin hook can execute. + +Stdout contains exactly one compact JSON object. Diagnostics go to evidence files. Exit codes and semantics: + +| Outcome | Exit | Clean | Meaning | +|---|---:|---:|---| +| `converged` | 0 | yes | State is terminal with `no-material-findings`, and the final findings artifact independently contains no substantive findings or severity bullets. | +| `max_reached` | 10 | no | The round budget ended with unresolved/non-clean findings evidence. Mechanics worked; approval did not occur. | +| `degraded` | 11 | no | Claude exited zero, but state/artifacts are incomplete or contradictory. | +| `failed` | 12 | no | Validation, prerequisite/auth, launch, or Claude execution failed. | +| `timed_out` | 124 | no | Wall-clock deadline expired; the process group was killed and reaped. | + +Only `outcome=converged` with `clean=true` is a success gate. + +## Cost and budgets + +Each round invokes one full Codex review, historically about 25–30k Codex tokens per plan round. `--max-budget-usd` bounds the headless Claude Code side and the adapter reports Claude's emitted dollar cost when available. Codex uses the configured ChatGPT subscription; its usage is separate, may be rate-limited, and cannot be dollar-capped or measured by this adapter. The timeout is a wall-clock safety boundary, not a billing guarantee. + +## Evidence and failure handling + +Evidence includes `run-metadata.json`, `preflight.json`, `claude-stream.jsonl`, `claude-stderr.log`, `state-events.jsonl`, before/after plan copies, Claudex state/findings/log copies, and `result.json`. On any non-clean outcome, inspect `reason`, state, final findings, and stderr; never infer convergence from Claude's final prose. + +The adapter refuses an existing active Claudex loop. Terminal prior states are baselined so consecutive runs select only the new run's state. + +## Installation and Hermes skill staging + +No installation is required to run from a checkout; keep executable paths explicit. To put the command on a controlled local `PATH`, symlink or copy `bin/claudex-plan-review` yourself and continue to pass `--plugin-root`. + +The staged Hermes skill lives at `skills/project-plan-review/`. Review it in-repo first. To install later, outside this change and only with operator approval, copy that whole directory into the active Hermes profile's skills directory, then start a fresh Hermes session. This repository does **not** install the skill, alter Hermes configuration, or change the active Claude plugin installation. diff --git a/install.sh b/install.sh index 8de70c1..a508faa 100755 --- a/install.sh +++ b/install.sh @@ -130,10 +130,10 @@ note "After /reload-plugins, /claudex should appear in your slash command list." # ───────────────────────────────────────────── hdr "7. Platform validation" # ───────────────────────────────────────────── -if [ -x "$PLUGIN_ROOT/tests/platform-validation.sh" ]; then +if [ -x "$PLUGIN_ROOT/plugins/claudex/tests/platform-validation.sh" ]; then echo "Running platform-validation.sh..." echo "" - if bash "$PLUGIN_ROOT/tests/platform-validation.sh" 2>&1 | tail -5; then + if bash "$PLUGIN_ROOT/plugins/claudex/tests/platform-validation.sh" 2>&1 | tail -5; then : else failures=$((failures+1)) diff --git a/plugins/claudex/docs/ARCHITECTURE.md b/plugins/claudex/docs/ARCHITECTURE.md index a6dfe4d..56ec33b 100644 --- a/plugins/claudex/docs/ARCHITECTURE.md +++ b/plugins/claudex/docs/ARCHITECTURE.md @@ -52,11 +52,12 @@ USER: /claudex:plan add expiry dates ┌─────────────────────┐ │ stop-hook.sh │ Reads state file └─────────────────────┘ phase=reviewing - │ → If signal=no-material-findings: ALLOW + cleanup - │ → If round >= max: ALLOW + "stopped at max" + │ → If signal=no-material-findings: summary BLOCK + │ → If round >= max: summary BLOCK │ → Else: increment round, BLOCK with new round ▼ - CLAUDE either revises PLAN.md (loops back) OR exits cleanly + CLAUDE either revises PLAN.md (loops back) OR prints the summary + Next Stop hook: summarizing → done → APPROVE + cleanup ``` ## The state file @@ -116,7 +117,8 @@ Every error path leads to approve. The plugin is not allowed to break the user's The hook branches on `mode` (plan or review) and then on `phase`. Plan mode has the most states: - `drafting` → check PLAN.md, transition to reviewing, write runner, BLOCK -- `reviewing` → check decision_signal, either ALLOW (done/max) or increment round + BLOCK +- `reviewing` → check decision_signal, either transition to `summarizing` + BLOCK or increment round + BLOCK +- `summarizing` → transition to `done`, cleanup + ALLOW - `done` → cleanup + ALLOW - `cancelled` → cleanup + ALLOW @@ -141,7 +143,7 @@ The runner script includes: The challenge: hooks fire AFTER Claude finishes a turn. So how does Claude tell the hook "the loop should end now"? -Answer: Claude updates the state file before ending its turn. Specifically, Claude runs `mark-done.sh` which sets `phase=done` and `decision_signal=no-material-findings`. The hook reads those fields on its next fire and ALLOWs exit. +Answer: Claude updates the state file before ending its turn. Specifically, Claude runs `mark-done.sh`, which leaves `phase=reviewing` and sets `decision_signal=no-material-findings`. The hook reads those fields on its next fire, transitions to `summarizing`, and BLOCKs once so Claude prints the user-visible summary. The following fire transitions to `done` and ALLOWs exit. This is just a state machine using the file system as a synchronization channel. Simple. Robust. Survives crashes. diff --git a/plugins/claudex/hooks/stop-hook.sh b/plugins/claudex/hooks/stop-hook.sh index 7e7b7b1..4fc67aa 100755 --- a/plugins/claudex/hooks/stop-hook.sh +++ b/plugins/claudex/hooks/stop-hook.sh @@ -90,9 +90,11 @@ STARTED_AT_EPOCH=$(claudex_state_read_field "$ACTIVE_STATE" "started_at_epoch") log "State: mode=$MODE phase=$PHASE round=$ROUND/$MAX_ROUNDS signal=$DECISION_SIGNAL" -# Sanity: if cwd doesn't match the repo where the loop started, fail-open. -if [ -n "$REPO_ROOT_STATE" ] && [ "$REPO_ROOT_STATE" != "$(pwd)" ]; then - log "cwd mismatch (state=$REPO_ROOT_STATE, here=$(pwd)); fail-open" +# Sanity: compare canonical physical paths so symlink/casing aliases do not +# fail-open a loop that is still running in the same repository. +CURRENT_REPO_ROOT="$(pwd -P)" +if [ -n "$REPO_ROOT_STATE" ] && [ "$REPO_ROOT_STATE" != "$CURRENT_REPO_ROOT" ]; then + log "cwd mismatch (state=$REPO_ROOT_STATE, here=$CURRENT_REPO_ROOT); fail-open" approve "cwd mismatch" fi diff --git a/plugins/claudex/scripts/start-loop.sh b/plugins/claudex/scripts/start-loop.sh index 862ba16..b1de854 100755 --- a/plugins/claudex/scripts/start-loop.sh +++ b/plugins/claudex/scripts/start-loop.sh @@ -142,7 +142,7 @@ LOCK_FILE="$CLAUDEX_STATE_DIR/$REVIEW_ID.lock" NOW="$(date -u +%Y-%m-%dT%H:%M:%SZ)" NOW_EPOCH="$(date -u +%s)" -REPO_ROOT="$(pwd)" +REPO_ROOT="$(pwd -P)" SESSION_ID="${CLAUDE_SESSION_ID:-unknown}" MAX_PLAN_ROUNDS="${CLAUDEX_MAX_PLAN_ROUNDS:-3}" MAX_REVIEW_ROUNDS="${CLAUDEX_MAX_REVIEW_ROUNDS:-3}" diff --git a/plugins/claudex/scripts/state-helpers.sh b/plugins/claudex/scripts/state-helpers.sh index 4748e14..44ab77e 100755 --- a/plugins/claudex/scripts/state-helpers.sh +++ b/plugins/claudex/scripts/state-helpers.sh @@ -78,25 +78,35 @@ claudex_state_set_field() { local field="$2" local value="$3" [ -f "$file" ] || return 1 + echo "$field" | grep -qE '^[A-Za-z_][A-Za-z0-9_]*$' || return 1 local tmp="${file}.tmp.$$" local now now=$(date -u +%Y-%m-%dT%H:%M:%SZ) - if grep -qE "^${field}:" "$file" 2>/dev/null; then - sed -E -e "s/^${field}: .*/${field}: ${value}/" \ - -e "s/^last_updated_at: .*/last_updated_at: ${now}/" \ - "$file" > "$tmp" 2>/dev/null \ - || { rm -f "$tmp"; return 1; } - else - sed -E "s/^last_updated_at: .*/last_updated_at: ${now}/" "$file" > "$tmp" 2>/dev/null \ - || { rm -f "$tmp"; return 1; } - printf '%s: %s\n' "$field" "$value" >> "$tmp" 2>/dev/null \ - || { rm -f "$tmp"; return 1; } - fi - # Don't recursively bump last_updated_at when we ARE setting last_updated_at, - # otherwise the explicit value gets clobbered. - if [ "$field" = "last_updated_at" ]; then - sed -E "s/^last_updated_at: .*/last_updated_at: ${value}/" "$tmp" > "${tmp}.2" 2>/dev/null \ - && mv -f "${tmp}.2" "$tmp" + # State is deliberately a single-line key/value format. Collapse embedded + # CR/LF to spaces, then write with printf instead of interpolating user data + # into sed replacement syntax (where '/', '&', and newlines are special). + value=$(printf '%s' "$value" | tr '\r\n' ' ') + local found=false + local saw_updated=false + : > "$tmp" || return 1 + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in + "$field:"*) + printf '%s: %s\n' "$field" "$value" >> "$tmp" || { rm -f "$tmp"; return 1; } + found=true + [ "$field" = "last_updated_at" ] && saw_updated=true + ;; + last_updated_at:*) + printf 'last_updated_at: %s\n' "$now" >> "$tmp" || { rm -f "$tmp"; return 1; } + saw_updated=true + ;; + *) printf '%s\n' "$line" >> "$tmp" || { rm -f "$tmp"; return 1; } ;; + esac + done < "$file" + [ "$found" = "true" ] || printf '%s: %s\n' "$field" "$value" >> "$tmp" \ + || { rm -f "$tmp"; return 1; } + if [ "$field" != "last_updated_at" ] && [ "$saw_updated" != "true" ]; then + printf 'last_updated_at: %s\n' "$now" >> "$tmp" || { rm -f "$tmp"; return 1; } fi mv -f "$tmp" "$file" 2>/dev/null || { rm -f "$tmp"; return 1; } return 0 diff --git a/plugins/claudex/tests/platform-validation.sh b/plugins/claudex/tests/platform-validation.sh index 7e0fe2e..032d172 100755 --- a/plugins/claudex/tests/platform-validation.sh +++ b/plugins/claudex/tests/platform-validation.sh @@ -225,6 +225,43 @@ check "status.sh runs without error on empty state" bash -c " exit \$rc " +section "15. Safe state field updates" +TMP=$(mktemp -d) +SAFE_STATE="$TMP/safe.state" +claudex_state_write "$SAFE_STATE" "phase: drafting +note: old +last_updated_at: 2026-04-26T00:00:00Z" +special_value='path/with/slashes & ampersand' +check "set_field accepts slash and ampersand" \ + claudex_state_set_field "$SAFE_STATE" note "$special_value" +actual_value=$(claudex_state_read_field "$SAFE_STATE" note) +check "slash and ampersand survive literally" test "$actual_value" = "$special_value" +newline_value=$(printf 'first line\nsecond/line & value') +check "set_field accepts embedded newline" \ + claudex_state_set_field "$SAFE_STATE" note "$newline_value" +actual_value=$(claudex_state_read_field "$SAFE_STATE" note) +check "newline is safely normalized for single-line state" \ + test "$actual_value" = "first line second/line & value" +check "invalid field name rejected" \ + bash -c "! claudex_state_set_field '$SAFE_STATE' 'bad/field' value" +rm -rf "$TMP" + +section "16. Canonical repository root" +TMP=$(mktemp -d) +mkdir -p "$TMP/real" +ln -s "$TMP/real" "$TMP/alias" +( cd "$TMP/alias" && export CLAUDEX_STATE_DIR=.claude/claudex && \ + bash "$PLUGIN_ROOT/scripts/start-loop.sh" plan "canonical path test" >/dev/null 2>&1 ) +CANON_STATE=$(ls "$TMP/real/.claude/claudex"/*.state 2>/dev/null | head -1) +stored_root=$(claudex_state_read_field "$CANON_STATE" repo_root) +expected_root=$(cd "$TMP/real" && pwd -P) +check "start-loop stores physical pwd" test "$stored_root" = "$expected_root" +rm -rf "$TMP" + +section "17. Installer validation path" +check "install.sh targets nested plugin test path" \ + grep -q '\$PLUGIN_ROOT/plugins/claudex/tests/platform-validation.sh' "$PLUGIN_ROOT/../../install.sh" + # Summary printf '\n\033[1m=== Results ===\033[0m\n' printf ' \033[32m%d passed\033[0m\n' "$pass" diff --git a/plugins/claudex/tests/synthetic-e2e.sh b/plugins/claudex/tests/synthetic-e2e.sh index a366307..2800532 100755 --- a/plugins/claudex/tests/synthetic-e2e.sh +++ b/plugins/claudex/tests/synthetic-e2e.sh @@ -103,7 +103,7 @@ check "PLAN.md exists" test -f PLAN.md section "Round 1: fire hook (drafting -> reviewing)" HOOK_OUT=$(echo '{}' | bash "$HOOK" 2>/dev/null) echo "Hook output: $(printf '%s' "$HOOK_OUT" | head -c 80)..." -check "hook returned block" bash -c "echo '$HOOK_OUT' | grep -q block" +check "hook returned block" python3 -c 'import json,sys; assert json.loads(sys.argv[1])["decision"] == "block"' "$HOOK_OUT" RUNNER=".claude/claudex/$REVIEW_ID-runner.sh" check "runner script created" test -f "$RUNNER" check "runner has quoted PROMPTEOF (P1 fix)" grep -q "<<'PROMPTEOF'" "$RUNNER" @@ -128,14 +128,24 @@ section "Mark loop done (Claude's signal in production)" bash "$MARK_DONE" "$REVIEW_ID" >/dev/null PHASE=$(grep '^phase:' ".claude/claudex/$REVIEW_ID.state" | sed 's/^phase: //') SIGNAL=$(grep '^decision_signal:' ".claude/claudex/$REVIEW_ID.state" | sed 's/^decision_signal: //') -check "phase marked done" test "$PHASE" = "done" +check "mark-done leaves phase reviewing" test "$PHASE" = "reviewing" check "signal set to no-material-findings" test "$SIGNAL" = "no-material-findings" -# Final hook fire, should ALLOW. +# The current lifecycle delivers a summary BLOCK before the terminal APPROVE. +section "Summary hook fire" +HOOK_OUT=$(echo '{}' | bash "$HOOK" 2>/dev/null) +echo "Summary hook output: $HOOK_OUT" +check "summary hook returned block" python3 -c 'import json,sys; assert json.loads(sys.argv[1])["decision"] == "block"' "$HOOK_OUT" +check "summary hook mentions completion" python3 -c 'import json,sys; assert "plan loop complete" in json.loads(sys.argv[1])["reason"].lower()' "$HOOK_OUT" +PHASE=$(grep '^phase:' ".claude/claudex/$REVIEW_ID.state" | sed 's/^phase: //') +check "phase advanced to summarizing" test "$PHASE" = "summarizing" + section "Final hook fire" HOOK_OUT=$(echo '{}' | bash "$HOOK" 2>/dev/null) echo "Final hook output: $HOOK_OUT" check "final hook returned approve" bash -c "echo '$HOOK_OUT' | grep -q approve" +PHASE=$(grep '^phase:' ".claude/claudex/$REVIEW_ID.state" | sed 's/^phase: //') +check "phase advanced to done" test "$PHASE" = "done" check "lockfile cleaned up" bash -c "! test -f .claude/claudex/$REVIEW_ID.lock" check "state file preserved for audit" test -f ".claude/claudex/$REVIEW_ID.state" check "runner script cleaned up" bash -c "! test -f $RUNNER" diff --git a/skills/project-plan-review/SKILL.md b/skills/project-plan-review/SKILL.md new file mode 100644 index 0000000..14d72f8 --- /dev/null +++ b/skills/project-plan-review/SKILL.md @@ -0,0 +1,37 @@ +--- +name: project-plan-review +description: Use for substantial implementation plans that benefit from independent Claude/Claudex/Codex adversarial review. Skip tiny, obvious fixes. +version: 1.0.0 +license: MIT +metadata: + hermes: + tags: [planning, claudex, codex, delegation, verification] +--- + +# Project Plan Review + +Use this workflow for substantial features, migrations, or risky cross-cutting work. **Do not use it for tiny fixes** where the review overhead exceeds the implementation risk. + +## Main Drake workflow + +1. Ground the project first: inspect the repository, current behavior, constraints, tests, and user request. Never ask a leaf reviewer to invent this context. +2. Draft a concrete `PLAN.md` in the project root. Include scope/non-scope, exact files, contracts that already exist, steps, rollback, and verification. +3. Read [the delegation runbook](references/runbook.md). +4. Call `delegate_task` with a self-contained goal/context that includes absolute paths for the repository, `PLAN.md`, adapter, Claude, Codex, and desired evidence directory; include rounds, timeout, and budget. The child is a leaf: it cannot ask Rob questions or delegate further. +5. Keep the main Drake session responsive while the child performs the bounded adapter run. Do not transfer user-facing ownership to the child. +6. On return, verify every returned file exists and read back `result.json`, final `PLAN.md`, state, and final findings. A claimed path is not evidence until read. +7. Independently reject scope creep, invented contracts, and recommendations unsupported by the grounded repository. Claudex is a critic, not the product owner. +8. Normalize the plan so only the active implementation phase remains; remove completed prerequisites, historical phases, and stale branching instructions. +9. Attach or return the final reviewed `PLAN.md`, outcome, unresolved findings, and evidence paths. Only `converged` is clean. + +## Boundaries and outcome rules + +- The adapter may write only `PLAN.md` plus `.claude/claudex/` evidence/state in the target repository. It must not implement the plan, commit, push, merge, or change global Hermes/Claude configuration. +- Use a disposable worktree/repository when the plan or project cannot safely be modified in place. +- `max_reached` proves mechanics, not plan approval. `degraded`, `failed`, and `timed_out` are also non-clean. +- Findings artifacts and Claudex state override Claude prose. Main Drake still independently validates all accepted recommendations. +- Never expose secrets in the topic, plan, delegated prompt, or evidence. + +## Verification gate + +Before attaching the final plan, require: readable `result.json`; matching absolute repo/plan paths; a terminal state; readable final findings; `clean=true` only with `outcome=converged`; and a final plan whose proposed contracts map to repository evidence or explicit user requirements. diff --git a/skills/project-plan-review/evals/eval-prompts.md b/skills/project-plan-review/evals/eval-prompts.md new file mode 100644 index 0000000..ff9284c --- /dev/null +++ b/skills/project-plan-review/evals/eval-prompts.md @@ -0,0 +1,23 @@ +# Evaluation prompts + +Use these prompts to evaluate whether the skill routes correctly and enforces its handoff contract. + +## Eval 1: substantial migration + +> Plan a multi-tenant billing migration across the API, workers, and database. Have Claudex pressure-test it while I keep discussing rollout questions with you. + +Expected: main Drake grounds the repo and drafts `PLAN.md`, delegates a self-contained absolute-path adapter run to a leaf, stays user-facing, reads artifacts back, and rejects invented contracts. + +## Eval 2: tiny fix + +> Fix the typo in the settings button label and tell me what changed. + +Expected: skip this workflow entirely because the change is tiny. + +## Eval 3: non-converged result + +> Review the disaster-recovery plan with two rounds and proceed only if it is genuinely clean. + +Fixture/result: adapter returns `max_reached` with material final findings. + +Expected: main Drake reports non-clean, does not call the plan approved, reads final findings, normalizes the plan if useful, and attaches it with unresolved concerns. diff --git a/skills/project-plan-review/references/runbook.md b/skills/project-plan-review/references/runbook.md new file mode 100644 index 0000000..3181676 --- /dev/null +++ b/skills/project-plan-review/references/runbook.md @@ -0,0 +1,46 @@ +# Delegation runbook + +## Inputs main Drake must prepare + +Resolve all paths before delegation: + +- `REPO`: trusted absolute git working-tree path +- `PLAN`: absolute existing, non-empty plan path (normally `$REPO/PLAN.md`) +- `ADAPTER`: staged `bin/claudex-plan-review` absolute path +- `PLUGIN_ROOT`: staged `plugins/claudex` absolute path +- `CLAUDE`: vetted absolute Claude Code executable +- `CODEX`: vetted absolute Codex executable +- `EVIDENCE`: new absolute output directory +- positive `ROUNDS`, `TIMEOUT_SECONDS`, and `BUDGET_USD` + +## `delegate_task` goal/context template + +Use the available `delegate_task` tool with a prompt equivalent to this, filling every placeholder: + +> You are a leaf execution subagent. Do not ask Rob questions and do not delegate. Run exactly one bounded plan-review adapter operation, then return the exact JSON result and absolute artifact paths. Repository: ``. Existing plan: ``. Grounded topic/constraints: ``. Adapter: ``. Claudex plugin root: ``. Claude executable: ``. Codex executable: ``. Evidence directory: ``. Run: ` --repo --plan --topic --rounds --timeout --budget-usd --plugin-root --claude --codex --output-dir `. Preserve stdout exactly. A nonzero exit is an outcome to report, not a reason to improvise. Do not implement, commit, push, install skills/plugins, edit global configuration, or touch files outside the adapter's documented scope. Before returning, verify `result.json`, the final plan, state file, and final findings paths exist. Return outcome, exit code, and paths; never call a non-converged run clean. + +Do not omit context on the assumption the child can read the parent conversation. It cannot ask the user to fill gaps. + +## Main Drake read-back + +After the child returns: + +1. Read `/result.json`; reject malformed or mismatched results. +2. Read the returned `state_file` and `final_findings` when present. +3. Read `` again from disk and compare it with the grounded scope. +4. Check outcome invariants: + - `converged`: exit 0, `clean=true`, state `phase=done`, signal `no-material-findings`, final findings says exactly no substantive findings and has no severity bullets. + - `max_reached`: exit 10, `clean=false`; unresolved findings remain or final artifact is not authoritative-clean. + - `degraded`: exit 11, `clean=false`; lifecycle/artifact mismatch or incomplete evidence. + - `failed`: exit 12, `clean=false`. + - `timed_out`: exit 124, `clean=false`; process group was terminated/reaped. +5. Reject invented APIs, data models, deployment guarantees, and scope additions unless they map to repository facts or explicit requirements. +6. Rewrite/normalize the active implementation phase if review churn left historical or completed phases in the plan. +7. Attach the final plan and disclose unresolved concerns and cost reporting limitations. + +## Safety notes + +- Claude's budget cap covers Claude Code API spend reported by Claude. Codex subscription usage is separate and is not dollar-enforced by the adapter. +- The adapter pins child `PATH` from explicit executable directories and system paths. +- Never point the adapter at an untrusted repository: it runs Claude with bypassed permission prompts so the Stop hook can operate unattended. +- Prefer two to three rounds for normal work. Increase only when the risk justifies additional Codex reviews. diff --git a/tests/test_adapter.py b/tests/test_adapter.py new file mode 100644 index 0000000..d679e02 --- /dev/null +++ b/tests/test_adapter.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +import json +import os +import subprocess +import tempfile +import textwrap +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +ADAPTER = ROOT / "bin" / "claudex-plan-review" +PLUGIN = ROOT / "plugins" / "claudex" + + +class AdapterTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.base = Path(self.tmp.name) + self.repo = self.base / "repo" + self.repo.mkdir() + subprocess.run(["git", "init", "-q", str(self.repo)], check=True) + self.plan = self.repo / "PLAN.md" + self.plan.write_text("# Plan\n\n1. Do the thing safely.\n") + self.bin = self.base / "bin" + self.bin.mkdir() + self.claude = self.bin / "claude" + self.codex = self.bin / "codex" + self._write_executable(self.codex, """#!/bin/sh +case "$1 $2" in + "login status") echo 'Logged in using ChatGPT'; exit 0 ;; +esac +echo 'codex-cli 0.test' +""") + self._write_executable(self.claude, """#!/usr/bin/env python3 +import json, os, pathlib, subprocess, sys, time, uuid +if sys.argv[1:] == ['--version']: + print('2.test'); raise SystemExit(0) +if sys.argv[1:] == ['auth', 'status']: + print(json.dumps({'loggedIn': True, 'authMethod': 'test'})); raise SystemExit(0) +outcome = os.environ.get('FAKE_CLAUDE_OUTCOME', 'converged') +if outcome == 'failed': + print('forced failure', file=sys.stderr); raise SystemExit(7) +if outcome == 'timeout': + child = subprocess.Popen(['sleep', '60']) + marker = os.environ.get('FAKE_CHILD_PID_FILE') + if marker: pathlib.Path(marker).write_text(str(child.pid)) + time.sleep(60); raise SystemExit(0) +state_dir = pathlib.Path.cwd() / '.claude' / 'claudex' +state_dir.mkdir(parents=True, exist_ok=True) +rid = '20990101-000000-' + uuid.uuid4().hex[:6] +review_dir = state_dir / rid +review_dir.mkdir() +if outcome == 'max_reached': + signal, findings, round_value = 'max-reached', '# Round 1 findings\\n\\n## High\\n- unsafe gap (fix it)\\n', '2' +elif outcome == 'degraded': + signal, findings, round_value = 'no-material-findings', '# Round 1 findings\\n\\n## High\\n- contradiction (fix it)\\n', '1' +else: + signal, findings, round_value = 'no-material-findings', '# Round 1 findings\\n\\nNo substantive findings.\\n', '1' +(review_dir / 'findings-round-1.md').write_text(findings) +state = state_dir / (rid + '.state') +state.write_text(f'''mode: plan\nphase: done\ntopic: test\nround: {round_value}\nmax_rounds: 1\nreview_id: {rid}\nrepo_root: {pathlib.Path.cwd().resolve()}\ndecision_signal: {signal}\n''') +print(json.dumps({'type': 'result', 'subtype': 'success', 'total_cost_usd': 0.01, 'session_id': 'fake-session'})) +""") + + def tearDown(self): + self.tmp.cleanup() + + def _write_executable(self, path, content): + path.write_text(textwrap.dedent(content)) + path.chmod(0o755) + + def run_adapter(self, outcome="converged", timeout="5", extra_env=None): + env = os.environ.copy() + env["FAKE_CLAUDE_OUTCOME"] = outcome + if extra_env: + env.update(extra_env) + command = [ + str(ADAPTER), "--repo", str(self.repo.resolve()), "--plan", str(self.plan.resolve()), + "--topic", "review the grounded plan", "--rounds", "1", "--timeout", timeout, + "--budget-usd", "1.25", "--plugin-root", str(PLUGIN.resolve()), + "--claude", str(self.claude.resolve()), "--codex", str(self.codex.resolve()), + ] + completed = subprocess.run(command, text=True, capture_output=True, env=env, timeout=10) + lines = completed.stdout.splitlines() + self.assertEqual(len(lines), 1, completed.stdout) + return completed, json.loads(lines[0]) + + def test_converged_is_only_clean_success(self): + completed, result = self.run_adapter("converged") + self.assertEqual(completed.returncode, 0) + self.assertEqual(result["outcome"], "converged") + self.assertTrue(result["clean"]) + self.assertEqual(result["findings_status"], "none") + self.assertTrue(Path(result["evidence_dir"], "result.json").is_file()) + + def test_max_reached_is_honest_and_nonzero(self): + completed, result = self.run_adapter("max_reached") + self.assertEqual(completed.returncode, 10) + self.assertEqual(result["outcome"], "max_reached") + self.assertFalse(result["clean"]) + + def test_prose_or_signal_cannot_override_material_findings(self): + completed, result = self.run_adapter("degraded") + self.assertEqual(completed.returncode, 11) + self.assertEqual(result["outcome"], "degraded") + self.assertFalse(result["clean"]) + self.assertEqual(result["findings_status"], "material") + + def test_nonzero_claude_is_failed(self): + completed, result = self.run_adapter("failed") + self.assertEqual(completed.returncode, 12) + self.assertEqual(result["outcome"], "failed") + self.assertFalse(result["clean"]) + + def test_timeout_kills_and_reaps_process_group(self): + marker = self.base / "child.pid" + completed, result = self.run_adapter("timeout", timeout="0.5", extra_env={"FAKE_CHILD_PID_FILE": str(marker)}) + self.assertEqual(completed.returncode, 124) + self.assertEqual(result["outcome"], "timed_out") + self.assertFalse(result["clean"]) + pid = int(marker.read_text()) + probe = subprocess.run(["kill", "-0", str(pid)], capture_output=True) + self.assertNotEqual(probe.returncode, 0, f"child process {pid} survived timeout") + + def test_second_consecutive_run_uses_new_state(self): + first, result1 = self.run_adapter("converged") + second, result2 = self.run_adapter("converged") + self.assertEqual((first.returncode, second.returncode), (0, 0)) + self.assertNotEqual(result1["review_id"], result2["review_id"]) + self.assertNotEqual(result1["state_file"], result2["state_file"]) + + def test_relative_repo_is_rejected_as_json(self): + completed = subprocess.run([ + str(ADAPTER), "--repo", "relative", "--plan", str(self.plan.resolve()), "--topic", "x", + "--rounds", "1", "--timeout", "1", "--budget-usd", "1", + "--plugin-root", str(PLUGIN.resolve()), "--claude", str(self.claude.resolve()), + "--codex", str(self.codex.resolve()), + ], text=True, capture_output=True) + result = json.loads(completed.stdout) + self.assertEqual(completed.returncode, 12) + self.assertEqual(result["error"]["kind"], "validation") + + +if __name__ == "__main__": + unittest.main() From 9889b583db20dc596cf0b050099135d196631981 Mon Sep 17 00:00:00 2001 From: robgfl45 Date: Sat, 11 Jul 2026 11:45:33 -0400 Subject: [PATCH 02/10] fix: align delegation and adapter timeout budgets (#2) Co-authored-by: robgfl45 <227035225+robgfl45@users.noreply.github.com> --- skills/project-plan-review/references/runbook.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/skills/project-plan-review/references/runbook.md b/skills/project-plan-review/references/runbook.md index 3181676..a1a069f 100644 --- a/skills/project-plan-review/references/runbook.md +++ b/skills/project-plan-review/references/runbook.md @@ -12,6 +12,16 @@ Resolve all paths before delegation: - `CODEX`: vetted absolute Codex executable - `EVIDENCE`: new absolute output directory - positive `ROUNDS`, `TIMEOUT_SECONDS`, and `BUDGET_USD` +- the Hermes delegation wall-clock limit from `delegation.child_timeout_seconds` + +## Timeout budget invariant + +The outer Hermes leaf must outlive the adapter plus artifact read-back and summary. Before calling `delegate_task`, read `delegation.child_timeout_seconds` from the active profile and require one of: + +- `child_timeout_seconds: 0` (no delegation wall-clock cap), or +- `child_timeout_seconds >= TIMEOUT_SECONDS + 180`. + +Never give the adapter a timeout equal to or greater than the child timeout. If this invariant is violated, the adapter can finish successfully while Hermes reports the leaf as timed out before it returns its summary. For a normal three-round review, use an adapter timeout of 900 seconds and a child timeout of at least 1080 seconds. A timed-out outer leaf is not proof that the adapter failed: inspect `result.json`, process state, and evidence directly before classifying the run or retrying. ## `delegate_task` goal/context template From 9b5ce201f9c48136c76f7e2f345fdd6e0f490126 Mon Sep 17 00:00:00 2001 From: robgfl45 <227035225+robgfl45@users.noreply.github.com> Date: Sat, 11 Jul 2026 12:14:00 -0400 Subject: [PATCH 03/10] feat: add frozen snapshot sweep engine --- plugins/claudex/commands/plan.md | 6 +- plugins/claudex/hooks/stop-hook.sh | 139 ++++++++- plugins/claudex/scripts/doctor.sh | 3 + plugins/claudex/scripts/personas.sh | 17 ++ plugins/claudex/scripts/start-loop.sh | 82 ++++- plugins/claudex/scripts/status.sh | 13 + plugins/claudex/scripts/sweep-helpers.sh | 301 +++++++++++++++++++ plugins/claudex/tests/platform-validation.sh | 5 + plugins/claudex/tests/sweep-v2-test.sh | 252 ++++++++++++++++ 9 files changed, 812 insertions(+), 6 deletions(-) create mode 100755 plugins/claudex/scripts/sweep-helpers.sh create mode 100755 plugins/claudex/tests/sweep-v2-test.sh diff --git a/plugins/claudex/commands/plan.md b/plugins/claudex/commands/plan.md index ff08f89..51add06 100644 --- a/plugins/claudex/commands/plan.md +++ b/plugins/claudex/commands/plan.md @@ -1,6 +1,6 @@ --- description: Run an autonomous plan-and-review loop. Claude drafts PLAN.md, Codex grills it adversarially, Claude revises until LGTM or N rounds. -argument-hint: '[--rounds N] [--from-draft] [--skip-interview] ' +argument-hint: '[--engine sweep-v2] [--rounds N] [--from-draft] [--skip-interview] ' allowed-tools: Bash, Read, Write, Edit, AskUserQuestion --- @@ -17,6 +17,7 @@ Parse these flags from the start of $ARGUMENTS (the script handles them; you mai - `--rounds N`. Override the default max rounds (3). Common picks: 3 (default, fast), 5 (deeper grilling), 7+ (very high stakes). - `--from-draft`. Use the existing `PLAN.md` in the project root instead of drafting from scratch. PLAN.md must exist and be non-empty. - `--skip-interview`. Bypass the topic-sharpening interview offer in step 2 below. Useful when you've already nailed the topic or you're in a rush. +- `--engine sweep-v2`. Opt into the Phase 1 frozen-snapshot engine. It requires an existing non-empty `PLAN.md`, runs all five required personas sequentially per generation, defaults to five generations, and never permits more than five. Omit this flag for unchanged legacy plan mode. ## Procedure @@ -74,7 +75,7 @@ Run start-loop.sh. **CRITICAL: the topic MUST be passed as a single double-quote Compose the bash command this way: -1. Identify any flags from `$ARGUMENTS`: `--rounds N`, `--from-draft`. (`--skip-interview` was already consumed in step 2.) +1. Identify any flags from `$ARGUMENTS`: `--engine sweep-v2`, `--rounds N`, `--from-draft`. (`--skip-interview` was already consumed in step 2.) 2. Identify the topic: everything that isn't a recognized flag. 3. Pass flags as-is (no quoting needed). Pass the topic as ONE double-quoted argument. @@ -86,6 +87,7 @@ bash "${CLAUDE_PLUGIN_ROOT}/scripts/start-loop.sh" plan [flags] "" bash "${CLAUDE_PLUGIN_ROOT}/scripts/start-loop.sh" plan "add expiry dates to my links" bash "${CLAUDE_PLUGIN_ROOT}/scripts/start-loop.sh" plan --rounds 5 "migrate auth to Clerk's new API" bash "${CLAUDE_PLUGIN_ROOT}/scripts/start-loop.sh" plan --from-draft "refactor the billing pipeline" +bash "${CLAUDE_PLUGIN_ROOT}/scripts/start-loop.sh" plan --engine sweep-v2 "review the existing frozen plan" # With interview, the enriched topic is also double-quoted: bash "${CLAUDE_PLUGIN_ROOT}/scripts/start-loop.sh" plan [flags] --interviewed "" diff --git a/plugins/claudex/hooks/stop-hook.sh b/plugins/claudex/hooks/stop-hook.sh index 4fc67aa..9a764aa 100755 --- a/plugins/claudex/hooks/stop-hook.sh +++ b/plugins/claudex/hooks/stop-hook.sh @@ -52,6 +52,8 @@ trap 'log "ERR trap at line $LINENO; failing open"; printf "{\"decision\":\"appr source "$CLAUDE_PLUGIN_ROOT/scripts/state-helpers.sh" 2>/dev/null || approve "state-helpers missing" # shellcheck source=/dev/null source "$CLAUDE_PLUGIN_ROOT/scripts/personas.sh" 2>/dev/null || approve "personas missing" +# shellcheck source=/dev/null +source "$CLAUDE_PLUGIN_ROOT/scripts/sweep-helpers.sh" 2>/dev/null || approve "sweep helpers missing" # Read hook input from stdin (Claude Code sends JSON). HOOK_INPUT="" @@ -80,6 +82,8 @@ fi # Read state fields. MODE=$(claudex_state_read_field "$ACTIVE_STATE" "mode") +ENGINE=$(claudex_state_read_field "$ACTIVE_STATE" "engine") +[ -n "$ENGINE" ] || ENGINE="legacy" PHASE=$(claudex_state_read_field "$ACTIVE_STATE" "phase") ROUND=$(claudex_state_read_field "$ACTIVE_STATE" "round") MAX_ROUNDS=$(claudex_state_read_field "$ACTIVE_STATE" "max_rounds") @@ -202,7 +206,140 @@ RUNNEREOF chmod +x "$RUNNER" } -# === PLAN MODE LIFECYCLE === +# === SWEEP-V2 PLAN LIFECYCLE === + +if [ "$MODE" = "plan" ] && [ "$ENGINE" = "sweep-v2" ]; then + GENERATION=$(claudex_state_read_field "$ACTIVE_STATE" generation) + MAX_GENERATIONS=$(claudex_state_read_field "$ACTIVE_STATE" max_generations) + SNAPSHOT_SHA=$(claudex_state_read_field "$ACTIVE_STATE" snapshot_sha256) + COVERAGE_COMPLETE=$(claudex_state_read_field "$ACTIVE_STATE" coverage_complete) + case "$GENERATION" in ''|*[!0-9]*) GENERATION=1 ;; esac + case "$MAX_GENERATIONS" in ''|*[!0-9]*) MAX_GENERATIONS=5 ;; esac + [ "$MAX_GENERATIONS" -le 5 ] || MAX_GENERATIONS=5 + GENERATION_DIR="$REVIEW_DIR/generations/$GENERATION" + CONSOLIDATED="$GENERATION_DIR/consolidated-findings.md" + + case "$PHASE" in + reviewing) + block "### Claudex sweep-v2 generation $GENERATION of $MAX_GENERATIONS + +All five required personas will run sequentially against one frozen snapshot. + +**Snapshot:** \`$GENERATION_DIR/PLAN.md\` +**SHA-256:** \`$SNAPSHOT_SHA\` + +Run the deterministic runner: + +\`\`\` +bash $RUNNER +\`\`\` + +Do not edit the snapshot or live \`PLAN.md\` while it runs. When it finishes, end your turn." + ;; + + awaiting-revision) + CURRENT_LIVE_SHA=$(claudex_sha256 PLAN.md 2>/dev/null) + if [ -z "$CURRENT_LIVE_SHA" ] || [ "$CURRENT_LIVE_SHA" = "$SNAPSHOT_SHA" ]; then + block "Sweep-v2 found material issues in generation $GENERATION. Read \`$CONSOLIDATED\`, revise live \`PLAN.md\` exactly once, and add or update \`## Changelog\` recording each accepted or rejected item with reasons. Do not modify the frozen snapshot. Then end your turn." + fi + if ! grep -qE '^## Changelog[[:space:]]*$' PLAN.md 2>/dev/null; then + block "The required plan revision exists, but \`PLAN.md\` has no \`## Changelog\`. Record accepted and rejected consolidated findings with reasons, then end your turn." + fi + NEW_GENERATION=$((GENERATION + 1)) + if [ "$NEW_GENERATION" -gt "$MAX_GENERATIONS" ] || [ "$NEW_GENERATION" -gt 5 ]; then + claudex_state_set_field "$ACTIVE_STATE" decision_signal max-reached + claudex_state_set_field "$ACTIVE_STATE" clean false + claudex_state_set_field "$ACTIVE_STATE" phase summarizing + block "Sweep-v2 reached its hard generation limit without unanimous clean coverage. End your turn to receive the terminal summary." + fi + NEW_SHA=$(claudex_sweep_create_generation "$ACTIVE_STATE" "$REVIEW_ID" "$NEW_GENERATION" "$TOPIC" "$SNAPSHOT_SHA") || { + claudex_state_set_field "$ACTIVE_STATE" decision_signal degraded + claudex_state_set_field "$ACTIVE_STATE" clean false + claudex_state_set_field "$ACTIVE_STATE" phase summarizing + block "Sweep-v2 degraded while creating generation $NEW_GENERATION. No clean result is claimed. End your turn for the terminal summary." + } + claudex_state_set_field "$ACTIVE_STATE" round "$NEW_GENERATION" + claudex_state_set_field "$ACTIVE_STATE" phase reviewing + claudex_sweep_write_runner "$ACTIVE_STATE" "$REVIEW_ID" "$NEW_GENERATION" "$TOPIC" "$NEW_SHA" || { + claudex_state_set_field "$ACTIVE_STATE" decision_signal degraded + claudex_state_set_field "$ACTIVE_STATE" clean false + claudex_state_set_field "$ACTIVE_STATE" phase summarizing + block "Sweep-v2 degraded while writing the generation runner. No clean result is claimed." + } + block "### Claudex sweep-v2 generation $NEW_GENERATION of $MAX_GENERATIONS + +The revised plan was frozen as a new immutable snapshot. + +**Snapshot:** \`$REVIEW_DIR/generations/$NEW_GENERATION/PLAN.md\` +**SHA-256:** \`$NEW_SHA\` + +Run all five personas sequentially: + +\`\`\` +bash $RUNNER +\`\`\` + +When the runner finishes, end your turn." + ;; + + summarizing) + # Revalidate every current-generation artifact at summary time so a + # post-run mutation cannot ride a previously clean state signal. + claudex_sweep_consolidate "$ACTIVE_STATE" "$REVIEW_ID" "$GENERATION" "$SNAPSHOT_SHA" "$SNAPSHOT_SHA" >/dev/null 2>&1 + SIGNAL=$(claudex_state_read_field "$ACTIVE_STATE" decision_signal) + CLEAN=$(claudex_state_read_field "$ACTIVE_STATE" clean) + CONVERGED_SHA=$(claudex_state_read_field "$ACTIVE_STATE" converged_snapshot_sha256) + claudex_state_set_field "$ACTIVE_STATE" phase done + rm -f "$RUNNER" "$STATE_DIR/$REVIEW_ID-prompt.txt" "$STATE_DIR/$REVIEW_ID.lock" 2>/dev/null + case "$SIGNAL" in + converged) + block "### Claudex sweep-v2 complete ✓ + +All five required personas returned exact clean findings against the same generation-$GENERATION snapshot hash. + +**Converged SHA-256:** \`$CONVERGED_SHA\` +**Coverage complete:** $COVERAGE_COMPLETE +**Clean:** $CLEAN + +Print this summary to the user, then end your turn." + ;; + max-reached) + block "### Claudex sweep-v2 stopped at max generations + +Generation $GENERATION still had material findings. The result is terminal and explicitly not clean; no unreviewed revision is accepted. + +**Consolidated findings:** \`$CONSOLIDATED\` +**Clean:** false + +Print this summary to the user, then end your turn." + ;; + *) + block "### Claudex sweep-v2 degraded + +A required persona artifact was missing, malformed, nonzero, hash-mismatched, or mutation was detected. Coverage is incomplete and no clean result is claimed. + +**Generation:** $GENERATION +**Consolidated findings:** \`$CONSOLIDATED\` +**Clean:** false + +Print this summary to the user, then end your turn." + ;; + esac + ;; + + done) + rm -f "$RUNNER" "$STATE_DIR/$REVIEW_ID-prompt.txt" "$STATE_DIR/$REVIEW_ID.lock" 2>/dev/null + approve "sweep-v2 loop done" + ;; + + *) + log "Unknown sweep-v2 phase: $PHASE" + approve "unknown sweep-v2 phase, fail-open" + ;; + esac +fi + +# === LEGACY PLAN MODE LIFECYCLE === if [ "$MODE" = "plan" ]; then case "$PHASE" in diff --git a/plugins/claudex/scripts/doctor.sh b/plugins/claudex/scripts/doctor.sh index 16100c8..03cad9a 100755 --- a/plugins/claudex/scripts/doctor.sh +++ b/plugins/claudex/scripts/doctor.sh @@ -96,6 +96,7 @@ PLUGIN_FILES=( "hooks/hooks.json" "scripts/state-helpers.sh" "scripts/personas.sh" + "scripts/sweep-helpers.sh" "scripts/start-loop.sh" "scripts/cancel-loop.sh" "scripts/rollback-loop.sh" @@ -140,6 +141,8 @@ check "state-helpers source cleanly" \ bash -c "source '$CLAUDE_PLUGIN_ROOT/scripts/state-helpers.sh'" check "personas source cleanly" \ bash -c "source '$CLAUDE_PLUGIN_ROOT/scripts/personas.sh'" +check "sweep helpers source cleanly" \ + bash -c "source '$CLAUDE_PLUGIN_ROOT/scripts/sweep-helpers.sh'" if source "$CLAUDE_PLUGIN_ROOT/scripts/personas.sh" 2>/dev/null; then R1=$(claudex_persona_for_round 1) R2=$(claudex_persona_for_round 2) diff --git a/plugins/claudex/scripts/personas.sh b/plugins/claudex/scripts/personas.sh index 27699c0..f6f3f92 100755 --- a/plugins/claudex/scripts/personas.sh +++ b/plugins/claudex/scripts/personas.sh @@ -14,6 +14,23 @@ # claudex_persona_label_for_round # Prints a one-line label for BLOCK headers (e.g. "Security review"). +# sweep-v2 uses complete coverage on every generation. Keep these IDs in one +# stable order because runner execution, consolidation, and convergence all +# depend on it. +CLAUDEX_SWEEP_PERSONAS="architecture-scope security-data product-domain quality-accessibility-performance operations-deployment" + +claudex_sweep_persona_prompt() { + local persona="$1" + case "$persona" in + architecture-scope) printf '%s' 'Review architecture, scope boundaries, dependencies, compatibility, and whether the plan solves the stated problem without hidden design gaps.' ;; + security-data) printf '%s' 'Review authorization, input boundaries, secrets, privacy, concurrency, idempotency, recovery, and data integrity.' ;; + product-domain) printf '%s' 'Review product behavior, domain rules, user journeys, acceptance criteria, and credible business edge cases.' ;; + quality-accessibility-performance) printf '%s' 'Review test strategy, accessibility, performance, resource bounds, failure visibility, and quality gates proportionate to the change.' ;; + operations-deployment) printf '%s' 'Review rollout, rollback, migrations, observability, operational ownership, version skew, and deployment failure modes.' ;; + *) return 1 ;; + esac +} + claudex_persona_label_for_round() { local r="${1:-1}" case "$r" in diff --git a/plugins/claudex/scripts/start-loop.sh b/plugins/claudex/scripts/start-loop.sh index b1de854..2385b12 100755 --- a/plugins/claudex/scripts/start-loop.sh +++ b/plugins/claudex/scripts/start-loop.sh @@ -31,9 +31,11 @@ shift || true # command; accepted here as a no-op so it's safe to pass through) # --interviewed marker passed by the slash command after a successful interview; # records interview_used=true in state for status/audit +# --engine sweep-v2 use the opt-in frozen-snapshot five-persona engine FROM_DRAFT=false CUSTOM_ROUNDS="" INTERVIEW_USED=false +ENGINE="legacy" while [ $# -gt 0 ]; do case "$1" in --rounds) @@ -56,6 +58,15 @@ while [ $# -gt 0 ]; do INTERVIEW_USED=true shift ;; + --engine) + shift + ENGINE="${1:-}" + shift || true + ;; + --engine=*) + ENGINE="${1#--engine=}" + shift + ;; *) break ;; @@ -96,8 +107,26 @@ case "$MODE" in echo "Plan mode requires a topic. Usage: start-loop.sh plan " >&2 exit 2 fi + if [ "$ENGINE" != "legacy" ] && [ "$ENGINE" != "sweep-v2" ]; then + echo "Unknown plan engine: $ENGINE. Use sweep-v2 or omit --engine for legacy mode." >&2 + exit 2 + fi + if [ "$ENGINE" = "sweep-v2" ]; then + if [ ! -s "PLAN.md" ]; then + echo "--engine sweep-v2 requires an existing non-empty PLAN.md." >&2 + exit 2 + fi + if [ -n "$CUSTOM_ROUNDS" ] && [ "$CUSTOM_ROUNDS" -gt 5 ]; then + echo "sweep-v2 has a hard maximum of five generations." >&2 + exit 2 + fi + fi ;; review) + if [ "$ENGINE" != "legacy" ]; then + echo "--engine only applies to plan mode." >&2 + exit 2 + fi ;; *) echo "Unknown mode: $MODE. Use plan or review." >&2 @@ -148,17 +177,26 @@ MAX_PLAN_ROUNDS="${CLAUDEX_MAX_PLAN_ROUNDS:-3}" MAX_REVIEW_ROUNDS="${CLAUDEX_MAX_REVIEW_ROUNDS:-3}" if [ "$MODE" = "plan" ]; then - MAX_ROUNDS="$MAX_PLAN_ROUNDS" - PHASE="drafting" + if [ "$ENGINE" = "sweep-v2" ]; then + MAX_ROUNDS=5 + PHASE="reviewing" + else + MAX_ROUNDS="$MAX_PLAN_ROUNDS" + PHASE="drafting" + fi else MAX_ROUNDS="$MAX_REVIEW_ROUNDS" PHASE="reviewing" fi -# --rounds flag overrides default max. +# --rounds flag overrides default max (sweep-v2 validated at five or fewer). if [ -n "$CUSTOM_ROUNDS" ]; then MAX_ROUNDS="$CUSTOM_ROUNDS" fi +MAX_GENERATIONS="" +if [ "$ENGINE" = "sweep-v2" ]; then + MAX_GENERATIONS="$MAX_ROUNDS" +fi # Escape topic for YAML (basic; topic is user-provided). # The interview path can produce a multi-line topic ("Scope: ...\nConstraints: ..."); @@ -182,12 +220,50 @@ started_at_epoch: $NOW_EPOCH last_updated_at: $NOW decision_signal: none" +if [ "$ENGINE" = "sweep-v2" ]; then + STATE_CONTENT="$STATE_CONTENT +engine: sweep-v2 +generation: 1 +max_generations: $MAX_GENERATIONS +snapshot_sha256: +coverage_complete: false +clean: false +revision_required: false" +fi + claudex_state_write "$STATE_FILE" "$STATE_CONTENT" || exit 3 claudex_lock_write "$LOCK_FILE" || exit 3 +if [ "$ENGINE" = "sweep-v2" ]; then + # shellcheck source=/dev/null + source "$CLAUDE_PLUGIN_ROOT/scripts/personas.sh" || exit 3 + # shellcheck source=/dev/null + source "$CLAUDE_PLUGIN_ROOT/scripts/sweep-helpers.sh" || exit 3 + SNAPSHOT_SHA=$(claudex_sweep_create_generation "$STATE_FILE" "$REVIEW_ID" 1 "$TOPIC" "") || { + claudex_state_set_field "$STATE_FILE" phase errored + echo "Failed to create immutable sweep-v2 generation 1 snapshot." >&2 + exit 3 + } + claudex_sweep_write_runner "$STATE_FILE" "$REVIEW_ID" 1 "$TOPIC" "$SNAPSHOT_SHA" || { + claudex_state_set_field "$STATE_FILE" phase errored + echo "Failed to create sweep-v2 runner." >&2 + exit 3 + } +fi + # Print initial instructions to stdout. Claude will read these. case "$MODE" in plan) + if [ "$ENGINE" = "sweep-v2" ]; then + echo "Claudex sweep-v2 plan review initialized." + echo "Review ID: $REVIEW_ID" + echo "Topic: $TOPIC" + echo "Generation: 1 of $MAX_GENERATIONS" + echo "Snapshot SHA-256: $SNAPSHOT_SHA" + echo "Frozen snapshot: $CLAUDEX_STATE_DIR/$REVIEW_ID/generations/1/PLAN.md" + echo "End your turn. The Stop hook will provide the deterministic sequential runner command." + exit 0 + fi echo "Claudex plan mode initialized." echo "Review ID: $REVIEW_ID" echo "Topic: $TOPIC" diff --git a/plugins/claudex/scripts/status.sh b/plugins/claudex/scripts/status.sh index a665c22..6778868 100755 --- a/plugins/claudex/scripts/status.sh +++ b/plugins/claudex/scripts/status.sh @@ -38,6 +38,7 @@ fi REVIEW_ID=$(basename "$ACTIVE" .state) MODE=$(claudex_state_read_field "$ACTIVE" "mode") +ENGINE=$(claudex_state_read_field "$ACTIVE" "engine") PHASE=$(claudex_state_read_field "$ACTIVE" "phase") ROUND=$(claudex_state_read_field "$ACTIVE" "round") MAX_ROUNDS=$(claudex_state_read_field "$ACTIVE" "max_rounds") @@ -47,6 +48,11 @@ STARTED=$(claudex_state_read_field "$ACTIVE" "started_at") STARTED_EPOCH=$(claudex_state_read_field "$ACTIVE" "started_at_epoch") INTERVIEW=$(claudex_state_read_field "$ACTIVE" "interview_used") FROM_DRAFT=$(claudex_state_read_field "$ACTIVE" "from_draft") +GENERATION=$(claudex_state_read_field "$ACTIVE" "generation") +MAX_GENERATIONS=$(claudex_state_read_field "$ACTIVE" "max_generations") +SNAPSHOT_SHA=$(claudex_state_read_field "$ACTIVE" "snapshot_sha256") +COVERAGE=$(claudex_state_read_field "$ACTIVE" "coverage_complete") +CLEAN=$(claudex_state_read_field "$ACTIVE" "clean") # Phase color. case "$PHASE" in @@ -92,6 +98,7 @@ RUNNER_LINE="${C_DIM}none${C_RESET}" printf '%s%s claudex %s%s\n' "$C_BOLD" "─────" "─────" "$C_RESET" printf ' %-13s %s\n' "review_id" "$REVIEW_ID" printf ' %-13s %s\n' "mode" "$MODE" +[ -n "$ENGINE" ] && printf ' %-13s %s\n' "engine" "$ENGINE" printf ' %-13s %s%s%s\n' "phase" "$PHASE_COLOR" "$PHASE" "$C_RESET" if [ -n "$ROUND" ] && [ -n "$MAX_ROUNDS" ]; then # Cap displayed round at max_rounds. The internal counter increments past @@ -103,6 +110,12 @@ if [ -n "$ROUND" ] && [ -n "$MAX_ROUNDS" ]; then fi printf ' %-13s %s of %s\n' "round" "$display_round" "$MAX_ROUNDS" fi +if [ "$ENGINE" = "sweep-v2" ]; then + printf ' %-13s %s of %s\n' "generation" "$GENERATION" "$MAX_GENERATIONS" + printf ' %-13s %s\n' "snapshot" "$SNAPSHOT_SHA" + printf ' %-13s %s\n' "coverage" "$COVERAGE" + printf ' %-13s %s\n' "clean" "$CLEAN" +fi [ -n "$TOPIC" ] && printf ' %-13s %s\n' "topic" "$TOPIC" [ -n "$STARTED" ] && printf ' %-13s %s\n' "started_at" "$STARTED" printf ' %-13s %s\n' "elapsed" "$ELAPSED" diff --git a/plugins/claudex/scripts/sweep-helpers.sh b/plugins/claudex/scripts/sweep-helpers.sh new file mode 100755 index 0000000..86c5991 --- /dev/null +++ b/plugins/claudex/scripts/sweep-helpers.sh @@ -0,0 +1,301 @@ +#!/usr/bin/env bash +# Deterministic frozen-snapshot sweep-v2 helpers. +# shellcheck shell=bash + +CLAUDEX_SWEEP_MAX_GENERATIONS=5 + +claudex_sha256() { + local file="$1" + [ -f "$file" ] || return 1 + if command -v shasum >/dev/null 2>&1; then + shasum -a 256 "$file" | awk '{print $1}' + else + sha256sum "$file" | awk '{print $1}' + fi +} + + +claudex_sweep_validate_findings() { + local file="$1" + [ -s "$file" ] || { printf 'malformed'; return 1; } + python3 - "$file" <<'PY' +import pathlib, sys +text = pathlib.Path(sys.argv[1]).read_text(encoding="utf-8") +if text.strip() == "No substantive findings.": + print("clean") + raise SystemExit(0) +lines = [line.rstrip() for line in text.splitlines() if line.strip()] +headers = ["## High", "## Medium", "## Low"] +positions = [] +for header in headers: + if lines.count(header) != 1: + print("malformed"); raise SystemExit(1) + positions.append(lines.index(header)) +if positions != sorted(positions): + print("malformed"); raise SystemExit(1) +material = 0 +for i, start in enumerate(positions): + end = positions[i + 1] if i + 1 < len(positions) else len(lines) + for line in lines[start + 1:end]: + if not line.startswith("- ") or len(line) <= 2: + print("malformed"); raise SystemExit(1) + material += 1 +if material == 0 or lines[:positions[0]]: + print("malformed"); raise SystemExit(1) +print("material") +PY +} + +claudex_sweep_create_generation() { + local state_file="$1" review_id="$2" generation="$3" topic="$4" previous_sha="${5:-}" + local source_plan="$(pwd -P)/PLAN.md" + [ -s "$source_plan" ] || return 1 + case "$generation" in ''|*[!0-9]*) return 1 ;; esac + [ "$generation" -ge 1 ] && [ "$generation" -le "$CLAUDEX_SWEEP_MAX_GENERATIONS" ] || return 1 + + local generation_dir="$CLAUDEX_STATE_DIR/$review_id/generations/$generation" + mkdir -p "$generation_dir" || return 1 + # A generation directory is write-once. Existing snapshot or manifest means + # this generation cannot be recreated from potentially different live input. + [ ! -e "$generation_dir/PLAN.md" ] && [ ! -e "$generation_dir/manifest.json" ] || return 1 + local tmp="$generation_dir/.PLAN.md.tmp.$$" + cp "$source_plan" "$tmp" || { rm -f "$tmp"; return 1; } + mv "$tmp" "$generation_dir/PLAN.md" || { rm -f "$tmp"; return 1; } + chmod a-w "$generation_dir/PLAN.md" 2>/dev/null || true + local sha + sha=$(claudex_sha256 "$generation_dir/PLAN.md") || return 1 + local manifest_tmp="$generation_dir/.manifest.json.tmp.$$" + python3 - "$manifest_tmp" "$generation" "$sha" "$topic" "$source_plan" "$previous_sha" <<'PY' +import json, pathlib, sys +path, generation, sha, topic, source, previous = sys.argv[1:] +data = { + "generation": int(generation), + "snapshot_sha256": sha, + "required_persona_ids": [ + "architecture-scope", "security-data", "product-domain", + "quality-accessibility-performance", "operations-deployment" + ], + "topic": topic, + "source_plan_path": source, + "previous_generation_sha256": previous or None, +} +pathlib.Path(path).write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8") +PY + [ $? -eq 0 ] || { rm -f "$manifest_tmp"; return 1; } + mv "$manifest_tmp" "$generation_dir/manifest.json" || return 1 + chmod a-w "$generation_dir/manifest.json" 2>/dev/null || true + claudex_state_set_field "$state_file" generation "$generation" || return 1 + claudex_state_set_field "$state_file" snapshot_sha256 "$sha" || return 1 + claudex_state_set_field "$state_file" coverage_complete false || return 1 + claudex_state_set_field "$state_file" decision_signal none || return 1 + claudex_state_set_field "$state_file" revision_required false || return 1 + claudex_state_set_field "$state_file" reviewed_live_sha256 "$sha" || return 1 + printf '%s' "$sha" +} + +claudex_sweep_write_result() { + local result_file="$1" persona="$2" expected="$3" before="$4" after="$5" rc="$6" findings="$7" classification="$8" + local tmp="${result_file}.tmp.$$" + python3 - "$tmp" "$persona" "$expected" "$before" "$after" "$rc" "$findings" "$classification" <<'PY' +import datetime, json, pathlib, sys +path, persona, expected, before, after, rc, findings, classification = sys.argv[1:] +data = { + "persona_id": persona, + "expected_snapshot_sha256": expected, + "actual_snapshot_sha256_before": before, + "actual_snapshot_sha256_after": after, + "codex_exit_code": int(rc), + "findings_path": findings, + "findings_classification": classification, + "completed_at": datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z"), +} +pathlib.Path(path).write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8") +PY + [ $? -eq 0 ] || { rm -f "$tmp"; return 1; } + mv "$tmp" "$result_file" +} + +claudex_sweep_consolidate() { + local state_file="$1" review_id="$2" generation="$3" expected="$4" live_expected="$5" + local generation_dir="$CLAUDEX_STATE_DIR/$review_id/generations/$generation" + local consolidated="$generation_dir/consolidated-findings.md" + local snapshot="$generation_dir/PLAN.md" + local current_snapshot current_live + current_snapshot=$(claudex_sha256 "$snapshot" 2>/dev/null) + current_live=$(claudex_sha256 PLAN.md 2>/dev/null) + local degraded=false material=false clean_count=0 + local manifest="$generation_dir/manifest.json" + local manifest_valid + manifest_valid=$(python3 - "$manifest" "$generation" "$expected" "$generation_dir" "$(pwd -P)/PLAN.md" <<'PY' +import json, pathlib, sys +path, generation, expected, generation_dir, source = sys.argv[1:] +ids = ["architecture-scope", "security-data", "product-domain", "quality-accessibility-performance", "operations-deployment"] +try: + m = json.loads(pathlib.Path(path).read_text(encoding="utf-8")) + ok = m.get("generation") == int(generation) + ok &= m.get("snapshot_sha256") == expected + ok &= m.get("required_persona_ids") == ids + ok &= m.get("source_plan_path") == source and bool(m.get("topic")) + if int(generation) == 1: + ok &= m.get("previous_generation_sha256") is None + else: + previous = pathlib.Path(generation_dir).parent / str(int(generation) - 1) / "manifest.json" + previous_sha = json.loads(previous.read_text(encoding="utf-8"))["snapshot_sha256"] + ok &= m.get("previous_generation_sha256") == previous_sha + print("valid" if ok else "degraded") +except Exception: + print("degraded") +PY +) + [ "$manifest_valid" = "valid" ] || degraded=true + local tmp="${consolidated}.tmp.$$" + { + printf '# Consolidated findings — generation %s\n\n' "$generation" + printf 'Snapshot SHA-256: `%s`\n\n' "$expected" + local persona findings result classification valid + for persona in $CLAUDEX_SWEEP_PERSONAS; do + findings="$generation_dir/$persona.findings.md" + result="$generation_dir/$persona.result.json" + printf '## %s\n\n' "$persona" + if [ ! -s "$findings" ] || [ ! -s "$result" ]; then + printf 'DEGRADED: missing findings or result sidecar.\n\n' + degraded=true + continue + fi + valid=$(python3 - "$result" "$persona" "$expected" "$findings" <<'PY' +import datetime, json, pathlib, sys +path, persona, expected, findings = sys.argv[1:] +try: + data = json.loads(pathlib.Path(path).read_text(encoding="utf-8")) + required = {"persona_id", "expected_snapshot_sha256", "actual_snapshot_sha256_before", "actual_snapshot_sha256_after", "codex_exit_code", "findings_path", "findings_classification", "completed_at"} + ok = set(data) == required + ok &= data["persona_id"] == persona and data["expected_snapshot_sha256"] == expected + ok &= data["actual_snapshot_sha256_before"] == expected and data["actual_snapshot_sha256_after"] == expected + ok &= data["codex_exit_code"] == 0 and data["findings_path"] == findings + ok &= data["findings_classification"] in {"clean", "material"} + datetime.datetime.strptime(data["completed_at"], "%Y-%m-%dT%H:%M:%SZ") + print(data["findings_classification"] if ok else "degraded") +except Exception: + print("degraded") +PY +) + classification=$(claudex_sweep_validate_findings "$findings" 2>/dev/null) + if [ "$valid" = "degraded" ] || [ "$classification" != "$valid" ]; then + printf 'DEGRADED: invalid sidecar, hash, exit status, or findings schema.\n\n' + degraded=true + else + cat "$findings" + printf '\n\n' + if [ "$classification" = "clean" ]; then clean_count=$((clean_count + 1)); else material=true; fi + fi + done + } > "$tmp" || { rm -f "$tmp"; return 1; } + mv "$tmp" "$consolidated" || return 1 + + if [ "$current_snapshot" != "$expected" ] || [ "$current_live" != "$live_expected" ]; then degraded=true; fi + if [ "$clean_count" -ne 5 ] && [ "$material" != "true" ]; then degraded=true; fi + + if [ "$degraded" = "true" ]; then + claudex_state_set_field "$state_file" coverage_complete false + claudex_state_set_field "$state_file" decision_signal degraded + claudex_state_set_field "$state_file" clean false + claudex_state_set_field "$state_file" phase summarizing + return 2 + fi + claudex_state_set_field "$state_file" coverage_complete true + if [ "$clean_count" -eq 5 ] && [ "$material" = "false" ]; then + claudex_state_set_field "$state_file" decision_signal converged + claudex_state_set_field "$state_file" clean true + claudex_state_set_field "$state_file" converged_snapshot_sha256 "$expected" + claudex_state_set_field "$state_file" phase summarizing + return 0 + fi + claudex_state_set_field "$state_file" clean false + local max_generations + max_generations=$(claudex_state_read_field "$state_file" max_generations) + case "$max_generations" in ''|*[!0-9]*) max_generations="$CLAUDEX_SWEEP_MAX_GENERATIONS" ;; esac + [ "$max_generations" -le "$CLAUDEX_SWEEP_MAX_GENERATIONS" ] || max_generations="$CLAUDEX_SWEEP_MAX_GENERATIONS" + if [ "$generation" -ge "$max_generations" ]; then + claudex_state_set_field "$state_file" decision_signal max-reached + claudex_state_set_field "$state_file" revision_required false + claudex_state_set_field "$state_file" phase summarizing + return 3 + fi + claudex_state_set_field "$state_file" decision_signal material-findings + claudex_state_set_field "$state_file" revision_required true + claudex_state_set_field "$state_file" phase awaiting-revision + return 1 +} + +claudex_sweep_write_runner() { + local state_file="$1" review_id="$2" generation="$3" topic="$4" expected="$5" + local runner="$CLAUDEX_STATE_DIR/$review_id-runner.sh" + local generation_dir="$CLAUDEX_STATE_DIR/$review_id/generations/$generation" + local snapshot="$generation_dir/PLAN.md" + local live_expected + live_expected=$(claudex_sha256 PLAN.md) || return 1 + cat > "$runner" </dev/null) + live_before=\$(claudex_sha256 PLAN.md 2>/dev/null) + focus=\$(claudex_sweep_persona_prompt "\$persona") + cat > "\$prompt" </dev/null 2>&1; then + rc=127 + else + "\$CODEX_BIN" exec --dangerously-bypass-approvals-and-sandbox < "\$prompt" + rc=\$? + fi + after=\$(claudex_sha256 "\$SNAPSHOT" 2>/dev/null) + live_after=\$(claudex_sha256 PLAN.md 2>/dev/null) + classification=degraded + if [ "\$rc" -eq 0 ] && [ "\$before" = "\$EXPECTED" ] && [ "\$after" = "\$EXPECTED" ] && [ "\$live_before" = "\$LIVE_EXPECTED" ] && [ "\$live_after" = "\$LIVE_EXPECTED" ]; then + classification=\$(claudex_sweep_validate_findings "\$findings" 2>/dev/null) + [ "\$classification" = clean ] || [ "\$classification" = material ] || classification=degraded + fi + claudex_sweep_write_result "\$result" "\$persona" "\$EXPECTED" "\$before" "\$after" "\$rc" "\$findings" "\$classification" + rm -f "\$prompt" +done +claudex_sweep_consolidate "\$STATE_FILE" "\$REVIEW_ID" "\$GENERATION" "\$EXPECTED" "\$LIVE_EXPECTED" +rc=\$? +case "\$rc" in 0) echo '[claudex] sweep converged' ;; 1) echo '[claudex] material findings require revision' ;; 2) echo '[claudex] sweep degraded' ;; 3) echo '[claudex] maximum generations reached' ;; esac +exit "\$rc" +RUNNEREOF + chmod +x "$runner" +} diff --git a/plugins/claudex/tests/platform-validation.sh b/plugins/claudex/tests/platform-validation.sh index 032d172..812a9d4 100755 --- a/plugins/claudex/tests/platform-validation.sh +++ b/plugins/claudex/tests/platform-validation.sh @@ -165,6 +165,8 @@ unset CLAUDEX_STATE_DIR section "12. Personas helper" check "personas.sh exists" test -f "$PLUGIN_ROOT/scripts/personas.sh" check "personas sources cleanly" bash -c "source '$PLUGIN_ROOT/scripts/personas.sh'" +check "sweep-helpers.sh exists" test -f "$PLUGIN_ROOT/scripts/sweep-helpers.sh" +check "sweep helpers source cleanly" bash -c "source '$PLUGIN_ROOT/scripts/sweep-helpers.sh'" # shellcheck source=/dev/null source "$PLUGIN_ROOT/scripts/personas.sh" @@ -188,6 +190,9 @@ L3=$(claudex_persona_label_for_round 3) check "round 1 label non-empty" test -n "$L1" check "round 1 label differs from round 2" test "$L1" != "$L2" check "round 2 label differs from round 3" test "$L2" != "$L3" +SWEEP_IDS=$(printf '%s\n' $CLAUDEX_SWEEP_PERSONAS) +check "sweep-v2 defines exactly five personas" test "$(printf '%s\n' "$SWEEP_IDS" | wc -l | tr -d ' ')" = 5 +check "sweep-v2 persona order is stable" test "$SWEEP_IDS" = "$(printf '%s\n' architecture-scope security-data product-domain quality-accessibility-performance operations-deployment)" section "13. Findings severity counter" TMP=$(mktemp -d) diff --git a/plugins/claudex/tests/sweep-v2-test.sh b/plugins/claudex/tests/sweep-v2-test.sh new file mode 100755 index 0000000..f1585c4 --- /dev/null +++ b/plugins/claudex/tests/sweep-v2-test.sh @@ -0,0 +1,252 @@ +#!/usr/bin/env bash +# Deterministic regression coverage for the feature-flagged sweep-v2 engine. +set +e + +PLUGIN_ROOT="${CLAUDE_PLUGIN_ROOT:-$(cd "$(dirname "$0")/.." && pwd)}" +START="$PLUGIN_ROOT/scripts/start-loop.sh" +HOOK="$PLUGIN_ROOT/hooks/stop-hook.sh" +PASS=0 +FAIL=0 +FAILURES=() +TEMPS=() + +ok() { printf ' \033[32m✓\033[0m %s\n' "$1"; PASS=$((PASS + 1)); } +bad() { printf ' \033[31m✗\033[0m %s\n' "$1"; FAIL=$((FAIL + 1)); FAILURES+=("$1"); } +check() { local name="$1"; shift; if "$@" >/dev/null 2>&1; then ok "$name"; else bad "$name"; fi; } +field() { sed -n "s/^$2: *//p" "$1" | head -1; } +cleanup() { local p; for p in "${TEMPS[@]}"; do chmod -R u+w "$p" 2>/dev/null; rm -rf "$p"; done; } +trap cleanup EXIT + +make_stub() { + local path="$1" + cat > "$path" <<'STUB' +#!/usr/bin/env bash +prompt=$(mktemp) +cat > "$prompt" +persona=$(sed -n 's/^Persona ID: //p' "$prompt" | head -1) +findings=$(sed -n 's/^Write ONLY one of these forms to \(.*\):$/\1/p' "$prompt" | head -1) +[ -n "$CLAUDEX_SWEEP_ORDER_LOG" ] && printf '%s\n' "$persona" >> "$CLAUDEX_SWEEP_ORDER_LOG" +case "${CLAUDEX_SWEEP_STUB_MODE:-clean}:${CLAUDEX_SWEEP_STUB_PERSONA:-}" in + missing:$persona) : ;; + malformed:$persona) printf 'not valid findings\n' > "$findings" ;; + nonzero:$persona) rm -f "$prompt"; exit 9 ;; + snapshot-mutation:$persona) + snapshot=$(sed -n 's/^Review only the frozen plan snapshot at: //p' "$prompt" | head -1) + chmod u+w "$snapshot" && printf '\nmutation\n' >> "$snapshot" + printf 'No substantive findings.\n' > "$findings" + ;; + live-mutation:$persona) + printf '\nlive mutation\n' >> PLAN.md + printf 'No substantive findings.\n' > "$findings" + ;; + material:$persona) + cat > "$findings" <<'EOF' +## High +- Scope: a concrete requirement can fail (address the stated failure mode). +## Medium +## Low +EOF + ;; + *) printf 'No substantive findings.\n' > "$findings" ;; +esac +rm -f "$prompt" +exit 0 +STUB + chmod +x "$path" +} + +new_repo() { + TEST_DIR=$(mktemp -d) + TEMPS+=("$TEST_DIR") + cd "$TEST_DIR" || exit 1 + git init -q + printf '# Plan\n\n## Scope\n\n1. Implement the scoped change.\n' > PLAN.md + STUB="$TEST_DIR/codex-stub" + make_stub "$STUB" + export CLAUDE_PLUGIN_ROOT="$PLUGIN_ROOT" + export CLAUDEX_CODEX_BIN="$STUB" + unset CLAUDEX_SWEEP_STUB_MODE CLAUDEX_SWEEP_STUB_PERSONA CLAUDEX_SWEEP_ORDER_LOG +} + +start_sweep() { + local rounds="${1:-}" + if [ -n "$rounds" ]; then + bash "$START" plan --engine sweep-v2 --rounds "$rounds" "deterministic sweep test" >/dev/null 2>&1 + else + bash "$START" plan --engine sweep-v2 "deterministic sweep test" >/dev/null 2>&1 + fi + ID=$(basename "$(ls .claude/claudex/*.state | head -1)" .state) + STATE=".claude/claudex/$ID.state" + RUNNER=".claude/claudex/$ID-runner.sh" +} + +printf '\033[1m=== Claudex sweep-v2 deterministic tests ===\033[0m\n' + +printf '\n\033[1mFeature flag and compatibility\033[0m\n' +new_repo +start_sweep +check "sweep-v2 defaults to five generations" test "$(field "$STATE" max_generations)" = 5 +check "sweep-v2 state records engine" test "$(field "$STATE" engine)" = sweep-v2 +check "generation one snapshot is immutable" bash -c "[ ! -w '.claude/claudex/$ID/generations/1/PLAN.md' ] || [ ! -x /bin/chmod ]" +check "manifest is immutable" bash -c "[ ! -w '.claude/claudex/$ID/generations/1/manifest.json' ] || [ ! -x /bin/chmod ]" +check "hard maximum rejects six generations" bash -c "cd '$TEST_DIR'; chmod -R u+w .claude; rm -rf .claude; ! bash '$START' plan --engine sweep-v2 --rounds 6 topic >/dev/null 2>&1" +rm -rf .claude +bash "$START" plan "legacy topic" >/dev/null 2>&1 +LEGACY_STATE=$(ls .claude/claudex/*.state | head -1) +check "legacy plan remains default drafting lifecycle" test "$(field "$LEGACY_STATE" phase)" = drafting +check "legacy plan keeps three-round default" test "$(field "$LEGACY_STATE" max_rounds)" = 3 +chmod -R u+w .claude; rm -rf .claude +bash "$START" review >/dev/null 2>&1 +REVIEW_STATE=$(ls .claude/claudex/*.state | head -1) +check "review mode remains reviewing" test "$(field "$REVIEW_STATE" phase)" = reviewing +printf '# sentinel\n' > PLAN.md +echo '{}' | bash "$HOOK" >/dev/null 2>&1 +REVIEW_RUNNER=$(ls .claude/claudex/*-runner.sh | head -1) +check "review runner remains single senior-engineer review" grep -qi 'senior' "$REVIEW_RUNNER" + +printf '\n\033[1mClean convergence and contracts\033[0m\n' +new_repo +start_sweep +ORDER="$TEST_DIR/order"; export CLAUDEX_SWEEP_ORDER_LOG="$ORDER" +bash "$RUNNER" >/dev/null 2>&1 +GEN_DIR=".claude/claudex/$ID/generations/1" +check "five clean personas converge generation one" test "$(field "$STATE" decision_signal)" = converged +check "clean convergence records complete coverage" test "$(field "$STATE" coverage_complete)" = true +check "clean convergence records clean=true" test "$(field "$STATE" clean)" = true +check "all results use the same snapshot hash" python3 - "$GEN_DIR" <<'PY' +import json, pathlib, sys +p=pathlib.Path(sys.argv[1]); manifest=json.loads((p/'manifest.json').read_text()); h=manifest['snapshot_sha256'] +results=[json.loads(x.read_text()) for x in p.glob('*.result.json')] +assert len(results)==5 +assert all(r['expected_snapshot_sha256']==h and r['actual_snapshot_sha256_before']==h and r['actual_snapshot_sha256_after']==h for r in results) +PY +check "manifest records complete contract" python3 - "$GEN_DIR/manifest.json" <<'PY' +import json, pathlib, sys +m=json.loads(pathlib.Path(sys.argv[1]).read_text()) +assert m['generation']==1 and len(m['required_persona_ids'])==5 +assert m['topic'] and m['source_plan_path'].endswith('/PLAN.md') +assert m['previous_generation_sha256'] is None and len(m['snapshot_sha256'])==64 +PY +EXPECTED_ORDER=$(printf '%s\n' architecture-scope security-data product-domain quality-accessibility-performance operations-deployment) +check "runner and consolidation use deterministic persona order" test "$(cat "$ORDER")" = "$EXPECTED_ORDER" +check "consolidation order is deterministic" python3 - "$GEN_DIR/consolidated-findings.md" <<'PY' +import pathlib, sys +s=pathlib.Path(sys.argv[1]).read_text() +ids=['architecture-scope','security-data','product-domain','quality-accessibility-performance','operations-deployment'] +assert [s.index('## '+x) for x in ids] == sorted(s.index('## '+x) for x in ids) +PY +check "reviewer contract pins snapshot path and hash" grep -q 'Expected snapshot SHA-256' "$RUNNER" +check "reviewer contract prohibits snapshot and live-plan edits" grep -q 'Do not edit the frozen snapshot or the live PLAN.md' "$RUNNER" +check "reviewer contract requires grounded findings" grep -q 'Tie every finding to a plan section and a concrete requirement, repository fact, or credible failure mode' "$RUNNER" +check "reviewer contract rejects unsupported gold-plating" grep -q 'Unsupported enterprise gold-plating is non-material' "$RUNNER" +check "reviewer contract preserves approval gates" grep -q 'approval-gated decisions as valid gates' "$RUNNER" + +new_repo; start_sweep +bash "$RUNNER" >/dev/null 2>&1 +MUTATED_RESULT=".claude/claudex/$ID/generations/1/security-data.result.json" +python3 - "$MUTATED_RESULT" <<'PY' +import json, pathlib, sys +p=pathlib.Path(sys.argv[1]); d=json.loads(p.read_text()); d['completed_at']='mutated'; p.write_text(json.dumps(d)) +PY +echo '{}' | bash "$HOOK" >/dev/null 2>&1 +check "post-run mutated sidecar degrades before summary" test "$(field "$STATE" decision_signal)" = degraded + +run_degraded_case() { + local mode="$1" persona="$2" expected_name="$3" + new_repo; start_sweep + export CLAUDEX_SWEEP_STUB_MODE="$mode" CLAUDEX_SWEEP_STUB_PERSONA="$persona" + bash "$RUNNER" >/dev/null 2>&1 + check "$expected_name" bash -c "[ \"\$(sed -n 's/^decision_signal: *//p' '$STATE')\" = degraded ] && [ \"\$(sed -n 's/^clean: *//p' '$STATE')\" = false ]" +} + +printf '\n\033[1mMaterial and degraded outcomes\033[0m\n' +new_repo; start_sweep +export CLAUDEX_SWEEP_STUB_MODE=material CLAUDEX_SWEEP_STUB_PERSONA=operations-deployment +bash "$RUNNER" >/dev/null 2>&1 +check "four clean plus one material cannot converge" test "$(field "$STATE" decision_signal)" = material-findings +check "material findings require a revision" test "$(field "$STATE" revision_required)" = true +run_degraded_case missing security-data "missing persona output degrades" +run_degraded_case malformed product-domain "malformed findings degrade" +run_degraded_case nonzero architecture-scope "nonzero reviewer exit degrades" +run_degraded_case snapshot-mutation quality-accessibility-performance "snapshot hash mismatch degrades" +run_degraded_case live-mutation operations-deployment "live PLAN.md mutation during sweep degrades" + +printf '\n\033[1mCoverage, generations, and isolation\033[0m\n' +new_repo; start_sweep +GEN_DIR=".claude/claudex/$ID/generations/1" +export CLAUDEX_SWEEP_STUB_MODE=missing CLAUDEX_SWEEP_STUB_PERSONA=security-data +bash "$RUNNER" >/dev/null 2>&1 +check "one or partial clean coverage cannot terminate early" bash -c "[ \"\$(sed -n 's/^decision_signal: *//p' '$STATE')\" != converged ] && [ \"\$(sed -n 's/^coverage_complete: *//p' '$STATE')\" = false ]" + +new_repo; start_sweep +bash "$RUNNER" >/dev/null 2>&1 +GEN_DIR=".claude/claudex/$ID/generations/1" +for persona in security-data product-domain quality-accessibility-performance operations-deployment; do + rm -f "$GEN_DIR/$persona.findings.md" "$GEN_DIR/$persona.result.json" +done +source "$PLUGIN_ROOT/scripts/state-helpers.sh"; source "$PLUGIN_ROOT/scripts/personas.sh"; source "$PLUGIN_ROOT/scripts/sweep-helpers.sh" +ONLY_SHA=$(field "$STATE" snapshot_sha256) +claudex_sweep_consolidate "$STATE" "$ID" 1 "$ONLY_SHA" "$ONLY_SHA" >/dev/null 2>&1 +check "one clean reviewer alone cannot converge" bash -c "[ \"\$(sed -n 's/^decision_signal: *//p' '$STATE')\" = degraded ] && [ \"\$(sed -n 's/^coverage_complete: *//p' '$STATE')\" = false ]" + +new_repo; start_sweep +GEN1_SHA=$(field "$STATE" snapshot_sha256) +export CLAUDEX_SWEEP_STUB_MODE=material CLAUDEX_SWEEP_STUB_PERSONA=architecture-scope +bash "$RUNNER" >/dev/null 2>&1 +printf '\n2. Address the finding.\n\n## Changelog\n- Accepted architecture-scope finding: added the missing failure handling.\n' >> PLAN.md +unset CLAUDEX_SWEEP_STUB_MODE CLAUDEX_SWEEP_STUB_PERSONA +echo '{}' | bash "$HOOK" >/dev/null 2>&1 +GEN2_SHA=$(field "$STATE" snapshot_sha256) +bash "$RUNNER" >/dev/null 2>&1 +check "material generation one then five clean generation two converges" bash -c "[ \"\$(sed -n 's/^decision_signal: *//p' '$STATE')\" = converged ] && [ \"\$(sed -n 's/^generation: *//p' '$STATE')\" = 2 ]" +check "generation two converges on its own new hash" bash -c "[ '$GEN1_SHA' != '$GEN2_SHA' ] && [ \"\$(sed -n 's/^converged_snapshot_sha256: *//p' '$STATE')\" = '$GEN2_SHA' ]" +check "generation two manifest links previous hash" python3 - ".claude/claudex/$ID/generations/2/manifest.json" "$GEN1_SHA" <<'PY' +import json, pathlib, sys +assert json.loads(pathlib.Path(sys.argv[1]).read_text())['previous_generation_sha256']==sys.argv[2] +PY + +new_repo; start_sweep +source "$PLUGIN_ROOT/scripts/state-helpers.sh" +source "$PLUGIN_ROOT/scripts/personas.sh" +source "$PLUGIN_ROOT/scripts/sweep-helpers.sh" +PREV=$(field "$STATE" snapshot_sha256) +for g in 2 3 4 5; do + chmod u+w PLAN.md; printf '\nrevision %s\n' "$g" >> PLAN.md + SHA=$(claudex_sweep_create_generation "$STATE" "$ID" "$g" "max generation test" "$PREV") || break + PREV="$SHA" +done +claudex_state_set_field "$STATE" phase reviewing +claudex_sweep_write_runner "$STATE" "$ID" 5 "max generation test" "$PREV" +RUNNER=".claude/claudex/$ID-runner.sh" +export CLAUDEX_SWEEP_STUB_MODE=material CLAUDEX_SWEEP_STUB_PERSONA=security-data +bash "$RUNNER" >/dev/null 2>&1 +check "material findings at generation five yield max-reached" bash -c "[ \"\$(sed -n 's/^decision_signal: *//p' '$STATE')\" = max-reached ] && [ \"\$(sed -n 's/^clean: *//p' '$STATE')\" = false ]" + +new_repo; start_sweep +bash "$RUNNER" >/dev/null 2>&1 +OLD_DIR=".claude/claudex/$ID/generations/1"; OLD_SHA=$(field "$STATE" snapshot_sha256) +chmod u+w PLAN.md; printf '\nnew generation\n' >> PLAN.md +source "$PLUGIN_ROOT/scripts/state-helpers.sh"; source "$PLUGIN_ROOT/scripts/personas.sh"; source "$PLUGIN_ROOT/scripts/sweep-helpers.sh" +NEW_SHA=$(claudex_sweep_create_generation "$STATE" "$ID" 2 "stale test" "$OLD_SHA") +NEW_DIR=".claude/claudex/$ID/generations/2" +cp "$OLD_DIR"/*.findings.md "$OLD_DIR"/*.result.json "$NEW_DIR"/ +claudex_sweep_consolidate "$STATE" "$ID" 2 "$NEW_SHA" "$NEW_SHA" >/dev/null 2>&1 +check "stale prior-generation artifacts cannot satisfy coverage" test "$(field "$STATE" decision_signal)" = degraded + +new_repo; start_sweep +FIRST_ID="$ID"; bash "$RUNNER" >/dev/null 2>&1 +echo '{}' | bash "$HOOK" >/dev/null 2>&1 +bash "$START" plan --engine sweep-v2 "second isolated sweep" >/dev/null 2>&1 +SECOND_ID=$(basename "$(ls -t .claude/claudex/*.state | head -1)" .state) +check "back-to-back sweep runs use isolated review IDs" test "$FIRST_ID" != "$SECOND_ID" +check "back-to-back sweep keeps first generation artifacts" test -f ".claude/claudex/$FIRST_ID/generations/1/manifest.json" +check "back-to-back sweep creates independent generation artifacts" test -f ".claude/claudex/$SECOND_ID/generations/1/manifest.json" + +printf '\n\033[1m=== Sweep-v2 Results ===\033[0m\n' +printf ' \033[32m%d passed\033[0m\n' "$PASS" +if [ "$FAIL" -gt 0 ]; then + printf ' \033[31m%d failed\033[0m\n' "$FAIL" + printf 'Failed:\n'; printf ' - %s\n' "${FAILURES[@]}" + exit 1 +fi +printf ' All sweep-v2 deterministic regressions passed.\n' From 898cdb367c29159076af95ec977262a9007ddab4 Mon Sep 17 00:00:00 2001 From: robgfl45 <227035225+robgfl45@users.noreply.github.com> Date: Sat, 11 Jul 2026 12:44:22 -0400 Subject: [PATCH 04/10] fix: harden sweep convergence lifecycle --- plugins/claudex/hooks/stop-hook.sh | 52 +++++++-- plugins/claudex/scripts/cancel-loop.sh | 22 ++++ plugins/claudex/scripts/doctor.sh | 7 +- plugins/claudex/scripts/start-loop.sh | 4 + plugins/claudex/scripts/state-helpers.sh | 17 ++- plugins/claudex/scripts/status.sh | 24 ++--- plugins/claudex/scripts/sweep-helpers.sh | 131 +++++++++++++++++++++-- plugins/claudex/tests/sweep-v2-test.sh | 107 +++++++++++++++++- 8 files changed, 328 insertions(+), 36 deletions(-) diff --git a/plugins/claudex/hooks/stop-hook.sh b/plugins/claudex/hooks/stop-hook.sh index 9a764aa..4648c59 100755 --- a/plugins/claudex/hooks/stop-hook.sh +++ b/plugins/claudex/hooks/stop-hook.sh @@ -242,8 +242,17 @@ Do not edit the snapshot or live \`PLAN.md\` while it runs. When it finishes, en if [ -z "$CURRENT_LIVE_SHA" ] || [ "$CURRENT_LIVE_SHA" = "$SNAPSHOT_SHA" ]; then block "Sweep-v2 found material issues in generation $GENERATION. Read \`$CONSOLIDATED\`, revise live \`PLAN.md\` exactly once, and add or update \`## Changelog\` recording each accepted or rejected item with reasons. Do not modify the frozen snapshot. Then end your turn." fi - if ! grep -qE '^## Changelog[[:space:]]*$' PLAN.md 2>/dev/null; then - block "The required plan revision exists, but \`PLAN.md\` has no \`## Changelog\`. Record accepted and rejected consolidated findings with reasons, then end your turn." + if ! claudex_sweep_validate_reconciliation PLAN.md "$CONSOLIDATED" "$GENERATION" "$SNAPSHOT_SHA"; then + block "The required plan revision exists, but its \`## Changelog\` does not reconcile every generation-$GENERATION finding. Add this exact heading under \`## Changelog\`: + +\`### Sweep generation $GENERATION — $SNAPSHOT_SHA\` + +Then add exactly one disposition for every ID in \`$CONSOLIDATED\` using: + +\`- Accepted [finding-id]: reason and resulting plan change\` +\`- Rejected [finding-id]: grounded reason\` + +Do not advance until every material finding ID has one reasoned disposition." fi NEW_GENERATION=$((GENERATION + 1)) if [ "$NEW_GENERATION" -gt "$MAX_GENERATIONS" ] || [ "$NEW_GENERATION" -gt 5 ]; then @@ -284,13 +293,40 @@ When the runner finishes, end your turn." summarizing) # Revalidate every current-generation artifact at summary time so a - # post-run mutation cannot ride a previously clean state signal. + # post-run mutation cannot ride a previously clean state signal. Clear + # the prior verdict first so an I/O/consolidation failure fails closed. + if ! claudex_state_set_field "$ACTIVE_STATE" decision_signal degraded \ + || ! claudex_state_set_field "$ACTIVE_STATE" clean false \ + || ! claudex_state_set_field "$ACTIVE_STATE" coverage_complete false; then + block "Sweep-v2 could not persist its fail-closed revalidation state. No clean result is claimed; repair the state directory and retry or cancel." + fi claudex_sweep_consolidate "$ACTIVE_STATE" "$REVIEW_ID" "$GENERATION" "$SNAPSHOT_SHA" "$SNAPSHOT_SHA" >/dev/null 2>&1 + REVALIDATE_RC=$? + case "$REVALIDATE_RC" in + 0|2|3) ;; + *) + if ! claudex_state_set_field "$ACTIVE_STATE" decision_signal degraded \ + || ! claudex_state_set_field "$ACTIVE_STATE" clean false \ + || ! claudex_state_set_field "$ACTIVE_STATE" coverage_complete false \ + || ! claudex_state_set_field "$ACTIVE_STATE" phase summarizing; then + block "Sweep-v2 revalidation failed and its degraded verdict could not be persisted. No clean result is claimed." + fi + ;; + esac SIGNAL=$(claudex_state_read_field "$ACTIVE_STATE" decision_signal) CLEAN=$(claudex_state_read_field "$ACTIVE_STATE" clean) + COVERAGE_COMPLETE=$(claudex_state_read_field "$ACTIVE_STATE" coverage_complete) + if [ "$SIGNAL" = "converged" ] && { [ "$REVALIDATE_RC" -ne 0 ] || [ "$CLEAN" != "true" ] || [ "$COVERAGE_COMPLETE" != "true" ]; }; then + claudex_state_set_field "$ACTIVE_STATE" decision_signal degraded + claudex_state_set_field "$ACTIVE_STATE" clean false + claudex_state_set_field "$ACTIVE_STATE" coverage_complete false + SIGNAL=degraded + CLEAN=false + COVERAGE_COMPLETE=false + fi CONVERGED_SHA=$(claudex_state_read_field "$ACTIVE_STATE" converged_snapshot_sha256) claudex_state_set_field "$ACTIVE_STATE" phase done - rm -f "$RUNNER" "$STATE_DIR/$REVIEW_ID-prompt.txt" "$STATE_DIR/$REVIEW_ID.lock" 2>/dev/null + rm -f "$RUNNER" "$STATE_DIR/$REVIEW_ID-prompt.txt" "$STATE_DIR/$REVIEW_ID.lock" "$STATE_DIR/$REVIEW_ID-active-pgid" 2>/dev/null case "$SIGNAL" in converged) block "### Claudex sweep-v2 complete ✓ @@ -328,7 +364,7 @@ Print this summary to the user, then end your turn." ;; done) - rm -f "$RUNNER" "$STATE_DIR/$REVIEW_ID-prompt.txt" "$STATE_DIR/$REVIEW_ID.lock" 2>/dev/null + rm -f "$RUNNER" "$STATE_DIR/$REVIEW_ID-prompt.txt" "$STATE_DIR/$REVIEW_ID.lock" "$STATE_DIR/$REVIEW_ID-active-pgid" 2>/dev/null approve "sweep-v2 loop done" ;; @@ -609,7 +645,7 @@ Findings summary will be written to: # Summary BLOCK was delivered on the previous fire and Claude has # printed it to the user. Final cleanup and approve. claudex_phase_transition "$ACTIVE_STATE" "summarizing" "done" 2>/dev/null - rm -f "$RUNNER" "$STATE_DIR/$REVIEW_ID-prompt.txt" "$STATE_DIR/$REVIEW_ID.lock" 2>/dev/null + rm -f "$RUNNER" "$STATE_DIR/$REVIEW_ID-prompt.txt" "$STATE_DIR/$REVIEW_ID.lock" "$STATE_DIR/$REVIEW_ID-active-pgid" 2>/dev/null ELAPSED=$(format_elapsed "$STARTED_AT_EPOCH") if [ -n "$ELAPSED" ]; then log "Plan loop $REVIEW_ID summary delivered; total elapsed $ELAPSED" @@ -618,7 +654,7 @@ Findings summary will be written to: ;; done) - rm -f "$RUNNER" "$STATE_DIR/$REVIEW_ID-prompt.txt" "$STATE_DIR/$REVIEW_ID.lock" 2>/dev/null + rm -f "$RUNNER" "$STATE_DIR/$REVIEW_ID-prompt.txt" "$STATE_DIR/$REVIEW_ID.lock" "$STATE_DIR/$REVIEW_ID-active-pgid" 2>/dev/null approve "plan loop already done" ;; @@ -680,7 +716,7 @@ After Codex finishes, end your turn. The hook will allow exit." ;; done) - rm -f "$RUNNER" "$STATE_DIR/$REVIEW_ID-prompt.txt" "$STATE_DIR/$REVIEW_ID.lock" 2>/dev/null + rm -f "$RUNNER" "$STATE_DIR/$REVIEW_ID-prompt.txt" "$STATE_DIR/$REVIEW_ID.lock" "$STATE_DIR/$REVIEW_ID-active-pgid" 2>/dev/null approve "review loop done" ;; diff --git a/plugins/claudex/scripts/cancel-loop.sh b/plugins/claudex/scripts/cancel-loop.sh index 98b1e05..1a1940e 100755 --- a/plugins/claudex/scripts/cancel-loop.sh +++ b/plugins/claudex/scripts/cancel-loop.sh @@ -20,10 +20,32 @@ echo "Cancelling loop: $REVIEW_ID" claudex_state_set_field "$ACTIVE" "phase" "cancelled" claudex_state_set_field "$ACTIVE" "last_updated_at" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" +# sweep-v2 runs each Codex reviewer in its own process group. Terminate the +# active group before removing artifacts so cancellation cannot leave an +# orphan reviewer writing into a cancelled generation. +ENGINE=$(claudex_state_read_field "$ACTIVE" engine) +ACTIVE_PGID_FILE="$CLAUDEX_STATE_DIR/$REVIEW_ID-active-pgid" +if [ "$ENGINE" = "sweep-v2" ] && [ -f "$ACTIVE_PGID_FILE" ]; then + ACTIVE_PGID=$(cat "$ACTIVE_PGID_FILE" 2>/dev/null) + case "$ACTIVE_PGID" in + ''|*[!0-9]*) ;; + *) + kill -TERM -- "-$ACTIVE_PGID" 2>/dev/null + i=0 + while kill -0 -- "-$ACTIVE_PGID" 2>/dev/null && [ "$i" -lt 20 ]; do + sleep 0.1 + i=$((i + 1)) + done + kill -KILL -- "-$ACTIVE_PGID" 2>/dev/null + ;; + esac +fi + # Clean up runner artifacts but keep state file for log/audit. rm -f "$CLAUDEX_STATE_DIR/$REVIEW_ID-runner.sh" 2>/dev/null rm -f "$CLAUDEX_STATE_DIR/$REVIEW_ID-prompt.txt" 2>/dev/null rm -f "$CLAUDEX_STATE_DIR/$REVIEW_ID.lock" 2>/dev/null +rm -f "$ACTIVE_PGID_FILE" 2>/dev/null echo "Loop cancelled. Stop hook will allow exit on next fire." exit 0 diff --git a/plugins/claudex/scripts/doctor.sh b/plugins/claudex/scripts/doctor.sh index 03cad9a..a763f2f 100755 --- a/plugins/claudex/scripts/doctor.sh +++ b/plugins/claudex/scripts/doctor.sh @@ -3,7 +3,7 @@ # # Verifies that everything claudex needs is wired up correctly: # - bash version -# - python3 (for JSON escape; sed fallback exists but python3 is preferred) +# - python3 (required for sweep-v2 manifests and artifact validation) # - codex CLI installed and responding # - .claude/claudex writable # - plugin file integrity (every expected script + prompt template present) @@ -11,7 +11,6 @@ # - stale loops report (informational, not fatal) # # Exits 0 if every required check passes, 1 if any required check fails. -# Optional checks (e.g. python3) only print warnings. set +e @@ -82,8 +81,8 @@ if command -v codex >/dev/null 2>&1; then fi fi -section "Optional dependencies" -warn_check "python3 (used for JSON escape; sed fallback works)" command -v python3 +section "Required runtime dependencies" +check "python3 (required for sweep-v2 manifests and validation)" command -v python3 section "State directory" mkdir -p "$CLAUDEX_STATE_DIR" 2>/dev/null diff --git a/plugins/claudex/scripts/start-loop.sh b/plugins/claudex/scripts/start-loop.sh index 2385b12..16e039c 100755 --- a/plugins/claudex/scripts/start-loop.sh +++ b/plugins/claudex/scripts/start-loop.sh @@ -112,6 +112,10 @@ case "$MODE" in exit 2 fi if [ "$ENGINE" = "sweep-v2" ]; then + if ! command -v python3 >/dev/null 2>&1; then + echo "--engine sweep-v2 requires python3 for manifests and artifact validation." >&2 + exit 2 + fi if [ ! -s "PLAN.md" ]; then echo "--engine sweep-v2 requires an existing non-empty PLAN.md." >&2 exit 2 diff --git a/plugins/claudex/scripts/state-helpers.sh b/plugins/claudex/scripts/state-helpers.sh index 44ab77e..7841fe9 100755 --- a/plugins/claudex/scripts/state-helpers.sh +++ b/plugins/claudex/scripts/state-helpers.sh @@ -20,6 +20,7 @@ CLAUDEX_STATE_DIR="${CLAUDEX_STATE_DIR:-.claude/claudex}" CLAUDEX_STALE_MINUTES="${CLAUDEX_STALE_MINUTES:-15}" +CLAUDEX_SWEEP_V2_STALE_MINUTES="${CLAUDEX_SWEEP_V2_STALE_MINUTES:-120}" claudex_new_review_id() { local ts @@ -131,13 +132,23 @@ claudex_lock_is_active() { claudex_sweep_stale() { [ -d "$CLAUDEX_STATE_DIR" ] || return 0 - # Find state files older than threshold; remove their state, lock, runner, - # prompt, and per-review findings dir. + # Never reap a loop whose current runner still owns the lock. sweep-v2 gets + # a longer stale window because one generation runs five reviewers and can + # legitimately exceed the legacy 15-minute threshold. find "$CLAUDEX_STATE_DIR" -maxdepth 1 -type f -name "*.state" -mmin "+$CLAUDEX_STALE_MINUTES" 2>/dev/null \ | while read -r f; do local id id=$(basename "$f" .state) - rm -f "$f" "$CLAUDEX_STATE_DIR/${id}.lock" "$CLAUDEX_STATE_DIR/${id}-runner.sh" "$CLAUDEX_STATE_DIR/${id}-prompt.txt" 2>/dev/null + local lock="$CLAUDEX_STATE_DIR/${id}.lock" + if claudex_lock_is_active "$lock"; then + continue + fi + local engine + engine=$(claudex_state_read_field "$f" engine) + if [ "$engine" = "sweep-v2" ] && ! find "$f" -prune -mmin "+$CLAUDEX_SWEEP_V2_STALE_MINUTES" -print 2>/dev/null | grep -q .; then + continue + fi + rm -f "$f" "$CLAUDEX_STATE_DIR/${id}.lock" "$CLAUDEX_STATE_DIR/${id}-runner.sh" "$CLAUDEX_STATE_DIR/${id}-prompt.txt" "$CLAUDEX_STATE_DIR/${id}-active-pgid" 2>/dev/null rm -rf "$CLAUDEX_STATE_DIR/${id}" 2>/dev/null done return 0 diff --git a/plugins/claudex/scripts/status.sh b/plugins/claudex/scripts/status.sh index 6778868..d4240ea 100755 --- a/plugins/claudex/scripts/status.sh +++ b/plugins/claudex/scripts/status.sh @@ -56,10 +56,10 @@ CLEAN=$(claudex_state_read_field "$ACTIVE" "clean") # Phase color. case "$PHASE" in - drafting|reviewing) PHASE_COLOR="$C_YELLOW" ;; - done) PHASE_COLOR="$C_GREEN" ;; - cancelled|errored) PHASE_COLOR="$C_RED" ;; - *) PHASE_COLOR="$C_DIM" ;; + drafting|reviewing|awaiting-revision|summarizing) PHASE_COLOR="$C_YELLOW" ;; + done) PHASE_COLOR="$C_GREEN" ;; + cancelled|errored) PHASE_COLOR="$C_RED" ;; + *) PHASE_COLOR="$C_DIM" ;; esac # Elapsed. @@ -78,15 +78,15 @@ if [ -n "$STARTED_EPOCH" ]; then esac fi -# Activity inferred from phase. The lock file's PID belongs to the start-loop -# script that has long since exited, so a lock-PID-alive check is never useful -# in claudex's between-turns architecture. Phase is the real liveness signal. +# Activity inferred primarily from phase. sweep-v2 refreshes the lock with the +# active runner PID while reviewers execute; between turns the phase remains +# the authoritative lifecycle signal. case "$PHASE" in - drafting|reviewing) ACTIVITY_LINE="${C_GREEN}active${C_RESET} (loop in progress between turns)" ;; - done) ACTIVITY_LINE="${C_DIM}complete${C_RESET}" ;; - cancelled) ACTIVITY_LINE="${C_RED}cancelled${C_RESET}" ;; - errored) ACTIVITY_LINE="${C_RED}errored${C_RESET}" ;; - *) ACTIVITY_LINE="${C_DIM}unknown${C_RESET}" ;; + drafting|reviewing|awaiting-revision|summarizing) ACTIVITY_LINE="${C_GREEN}active${C_RESET} (loop in progress between turns)" ;; + done) ACTIVITY_LINE="${C_DIM}complete${C_RESET}" ;; + cancelled) ACTIVITY_LINE="${C_RED}cancelled${C_RESET}" ;; + errored) ACTIVITY_LINE="${C_RED}errored${C_RESET}" ;; + *) ACTIVITY_LINE="${C_DIM}unknown${C_RESET}" ;; esac # Runner script. diff --git a/plugins/claudex/scripts/sweep-helpers.sh b/plugins/claudex/scripts/sweep-helpers.sh index 86c5991..4065658 100755 --- a/plugins/claudex/scripts/sweep-helpers.sh +++ b/plugins/claudex/scripts/sweep-helpers.sh @@ -3,6 +3,7 @@ # shellcheck shell=bash CLAUDEX_SWEEP_MAX_GENERATIONS=5 +CLAUDEX_SWEEP_PERSONA_TIMEOUT_SECONDS="${CLAUDEX_SWEEP_PERSONA_TIMEOUT_SECONDS:-300}" claudex_sha256() { local file="$1" @@ -46,6 +47,53 @@ print("material") PY } +claudex_sweep_render_findings_with_ids() { + local file="$1" persona="$2" + python3 - "$file" "$persona" <<'PY' +import pathlib, sys + +path, persona = sys.argv[1:] +severity = None +counts = {"high": 0, "medium": 0, "low": 0} +for line in pathlib.Path(path).read_text(encoding="utf-8").splitlines(): + if line in {"## High", "## Medium", "## Low"}: + severity = line[3:].lower() + print(line) + elif line.startswith("- ") and severity: + counts[severity] += 1 + finding_id = f"{persona}-{severity}-{counts[severity]}" + print(f"- [{finding_id}] {line[2:]}") + else: + print(line) +PY +} + +claudex_sweep_validate_reconciliation() { + local plan="$1" consolidated="$2" generation="$3" snapshot_sha="$4" + python3 - "$plan" "$consolidated" "$generation" "$snapshot_sha" <<'PY' +import pathlib, re, sys + +plan_path, consolidated_path, generation, snapshot_sha = sys.argv[1:] +plan = pathlib.Path(plan_path).read_text(encoding="utf-8") +consolidated = pathlib.Path(consolidated_path).read_text(encoding="utf-8") +ids = re.findall(r"^- \[([a-z0-9-]+-(?:high|medium|low)-[0-9]+)\] ", consolidated, re.M) +if not ids or len(ids) != len(set(ids)): + raise SystemExit(1) +match = re.search(r"^## Changelog\s*$([\s\S]*?)(?=^## |\Z)", plan, re.M) +if not match: + raise SystemExit(1) +section = match.group(1) +heading = f"### Sweep generation {generation} — {snapshot_sha}" +if heading not in section: + raise SystemExit(1) +for finding_id in ids: + pattern = rf"^- (?:Accepted|Rejected) \[{re.escape(finding_id)}\]:\s+\S.+$" + if len(re.findall(pattern, section, re.M)) != 1: + raise SystemExit(1) +raise SystemExit(0) +PY +} + claudex_sweep_create_generation() { local state_file="$1" review_id="$2" generation="$3" topic="$4" previous_sha="${5:-}" local source_plan="$(pwd -P)/PLAN.md" @@ -95,10 +143,12 @@ PY claudex_sweep_write_result() { local result_file="$1" persona="$2" expected="$3" before="$4" after="$5" rc="$6" findings="$7" classification="$8" + local findings_sha="" + findings_sha=$(claudex_sha256 "$findings" 2>/dev/null) || findings_sha="" local tmp="${result_file}.tmp.$$" - python3 - "$tmp" "$persona" "$expected" "$before" "$after" "$rc" "$findings" "$classification" <<'PY' + python3 - "$tmp" "$persona" "$expected" "$before" "$after" "$rc" "$findings" "$classification" "$findings_sha" <<'PY' import datetime, json, pathlib, sys -path, persona, expected, before, after, rc, findings, classification = sys.argv[1:] +path, persona, expected, before, after, rc, findings, classification, findings_sha = sys.argv[1:] data = { "persona_id": persona, "expected_snapshot_sha256": expected, @@ -106,13 +156,15 @@ data = { "actual_snapshot_sha256_after": after, "codex_exit_code": int(rc), "findings_path": findings, + "findings_sha256": findings_sha, "findings_classification": classification, "completed_at": datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z"), } pathlib.Path(path).write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8") PY [ $? -eq 0 ] || { rm -f "$tmp"; return 1; } - mv "$tmp" "$result_file" + mv "$tmp" "$result_file" || return 1 + chmod a-w "$findings" "$result_file" 2>/dev/null || true } claudex_sweep_consolidate() { @@ -163,15 +215,17 @@ PY continue fi valid=$(python3 - "$result" "$persona" "$expected" "$findings" <<'PY' -import datetime, json, pathlib, sys +import datetime, hashlib, json, pathlib, sys path, persona, expected, findings = sys.argv[1:] try: data = json.loads(pathlib.Path(path).read_text(encoding="utf-8")) - required = {"persona_id", "expected_snapshot_sha256", "actual_snapshot_sha256_before", "actual_snapshot_sha256_after", "codex_exit_code", "findings_path", "findings_classification", "completed_at"} + required = {"persona_id", "expected_snapshot_sha256", "actual_snapshot_sha256_before", "actual_snapshot_sha256_after", "codex_exit_code", "findings_path", "findings_sha256", "findings_classification", "completed_at"} ok = set(data) == required ok &= data["persona_id"] == persona and data["expected_snapshot_sha256"] == expected ok &= data["actual_snapshot_sha256_before"] == expected and data["actual_snapshot_sha256_after"] == expected ok &= data["codex_exit_code"] == 0 and data["findings_path"] == findings + actual_findings_sha = hashlib.sha256(pathlib.Path(findings).read_bytes()).hexdigest() + ok &= data["findings_sha256"] == actual_findings_sha ok &= data["findings_classification"] in {"clean", "material"} datetime.datetime.strptime(data["completed_at"], "%Y-%m-%dT%H:%M:%SZ") print(data["findings_classification"] if ok else "degraded") @@ -184,9 +238,14 @@ PY printf 'DEGRADED: invalid sidecar, hash, exit status, or findings schema.\n\n' degraded=true else - cat "$findings" + if [ "$classification" = "clean" ]; then + cat "$findings" + clean_count=$((clean_count + 1)) + else + claudex_sweep_render_findings_with_ids "$findings" "$persona" || degraded=true + material=true + fi printf '\n\n' - if [ "$classification" = "clean" ]; then clean_count=$((clean_count + 1)); else material=true; fi fi done } > "$tmp" || { rm -f "$tmp"; return 1; } @@ -248,11 +307,62 @@ EXPECTED=$(printf '%q' "$expected") LIVE_EXPECTED=$(printf '%q' "$live_expected") GENERATION_DIR=$(printf '%q' "$generation_dir") SNAPSHOT=$(printf '%q' "$snapshot") +PERSONA_TIMEOUT=$(printf '%q' "$CLAUDEX_SWEEP_PERSONA_TIMEOUT_SECONDS") +ACTIVE_PGID_FILE=$(printf '%q' "$CLAUDEX_STATE_DIR/$review_id-active-pgid") source "\$CLAUDE_PLUGIN_ROOT/scripts/state-helpers.sh" source "\$CLAUDE_PLUGIN_ROOT/scripts/personas.sh" source "\$CLAUDE_PLUGIN_ROOT/scripts/sweep-helpers.sh" CODEX_BIN="\${CLAUDEX_CODEX_BIN:-codex}" +claudex_run_codex_bounded() { + python3 - "\$CODEX_BIN" "\$1" "\$PERSONA_TIMEOUT" "\$ACTIVE_PGID_FILE" <<'PY' +import os, pathlib, signal, subprocess, sys + +codex_bin, prompt_path, timeout_raw, active_pgid_path = sys.argv[1:] +try: + timeout = int(timeout_raw) + if timeout < 1: + raise ValueError +except ValueError: + raise SystemExit(125) + +with open(prompt_path, "rb") as prompt: + process = subprocess.Popen( + [codex_bin, "exec", "--dangerously-bypass-approvals-and-sandbox"], + stdin=prompt, + start_new_session=True, + ) +pgid_path = pathlib.Path(active_pgid_path) +pgid_path.write_text(str(process.pid) + "\n", encoding="utf-8") +exit_code = 125 +try: + try: + exit_code = process.wait(timeout=timeout) + except subprocess.TimeoutExpired: + try: + os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: + pass + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait() + exit_code = 124 +finally: + try: + if pgid_path.read_text(encoding="utf-8").strip() == str(process.pid): + pgid_path.unlink() + except FileNotFoundError: + pass +raise SystemExit(exit_code) +PY +} +claudex_lock_write "\$CLAUDEX_STATE_DIR/\$REVIEW_ID.lock" for persona in \$CLAUDEX_SWEEP_PERSONAS; do + [ "\$(claudex_state_read_field "\$STATE_FILE" phase)" = cancelled ] && break findings="\$GENERATION_DIR/\$persona.findings.md" result="\$GENERATION_DIR/\$persona.result.json" prompt="\$GENERATION_DIR/.\$persona.prompt.txt" @@ -279,7 +389,7 @@ PROMPTEOF elif ! command -v "\$CODEX_BIN" >/dev/null 2>&1; then rc=127 else - "\$CODEX_BIN" exec --dangerously-bypass-approvals-and-sandbox < "\$prompt" + claudex_run_codex_bounded "\$prompt" rc=\$? fi after=\$(claudex_sha256 "\$SNAPSHOT" 2>/dev/null) @@ -290,8 +400,13 @@ PROMPTEOF [ "\$classification" = clean ] || [ "\$classification" = material ] || classification=degraded fi claudex_sweep_write_result "\$result" "\$persona" "\$EXPECTED" "\$before" "\$after" "\$rc" "\$findings" "\$classification" + claudex_state_set_field "\$STATE_FILE" sweep_heartbeat "\$persona" rm -f "\$prompt" done +if [ "\$(claudex_state_read_field "\$STATE_FILE" phase)" = cancelled ]; then + rm -f "\$ACTIVE_PGID_FILE" + exit 130 +fi claudex_sweep_consolidate "\$STATE_FILE" "\$REVIEW_ID" "\$GENERATION" "\$EXPECTED" "\$LIVE_EXPECTED" rc=\$? case "\$rc" in 0) echo '[claudex] sweep converged' ;; 1) echo '[claudex] material findings require revision' ;; 2) echo '[claudex] sweep degraded' ;; 3) echo '[claudex] maximum generations reached' ;; esac diff --git a/plugins/claudex/tests/sweep-v2-test.sh b/plugins/claudex/tests/sweep-v2-test.sh index f1585c4..d30bcb3 100755 --- a/plugins/claudex/tests/sweep-v2-test.sh +++ b/plugins/claudex/tests/sweep-v2-test.sh @@ -5,6 +5,7 @@ set +e PLUGIN_ROOT="${CLAUDE_PLUGIN_ROOT:-$(cd "$(dirname "$0")/.." && pwd)}" START="$PLUGIN_ROOT/scripts/start-loop.sh" HOOK="$PLUGIN_ROOT/hooks/stop-hook.sh" +CANCEL="$PLUGIN_ROOT/scripts/cancel-loop.sh" PASS=0 FAIL=0 FAILURES=() @@ -30,6 +31,12 @@ case "${CLAUDEX_SWEEP_STUB_MODE:-clean}:${CLAUDEX_SWEEP_STUB_PERSONA:-}" in missing:$persona) : ;; malformed:$persona) printf 'not valid findings\n' > "$findings" ;; nonzero:$persona) rm -f "$prompt"; exit 9 ;; + timeout:$persona) + sleep 30 & + child=$! + [ -n "$CLAUDEX_SWEEP_TIMEOUT_CHILD_FILE" ] && printf '%s\n' "$child" > "$CLAUDEX_SWEEP_TIMEOUT_CHILD_FILE" + wait "$child" + ;; snapshot-mutation:$persona) snapshot=$(sed -n 's/^Review only the frozen plan snapshot at: //p' "$prompt" | head -1) chmod u+w "$snapshot" && printf '\nmutation\n' >> "$snapshot" @@ -66,6 +73,7 @@ new_repo() { export CLAUDE_PLUGIN_ROOT="$PLUGIN_ROOT" export CLAUDEX_CODEX_BIN="$STUB" unset CLAUDEX_SWEEP_STUB_MODE CLAUDEX_SWEEP_STUB_PERSONA CLAUDEX_SWEEP_ORDER_LOG + unset CLAUDEX_SWEEP_PERSONA_TIMEOUT_SECONDS CLAUDEX_SWEEP_TIMEOUT_CHILD_FILE } start_sweep() { @@ -90,6 +98,15 @@ check "sweep-v2 state records engine" test "$(field "$STATE" engine)" = sweep-v2 check "generation one snapshot is immutable" bash -c "[ ! -w '.claude/claudex/$ID/generations/1/PLAN.md' ] || [ ! -x /bin/chmod ]" check "manifest is immutable" bash -c "[ ! -w '.claude/claudex/$ID/generations/1/manifest.json' ] || [ ! -x /bin/chmod ]" check "hard maximum rejects six generations" bash -c "cd '$TEST_DIR'; chmod -R u+w .claude; rm -rf .claude; ! bash '$START' plan --engine sweep-v2 --rounds 6 topic >/dev/null 2>&1" +NO_PY_PATH=$(mktemp -d); TEMPS+=("$NO_PY_PATH") +ln -s "$(command -v dirname)" "$NO_PY_PATH/dirname" +NO_PY_OUTPUT=$(PATH="$NO_PY_PATH" /bin/bash "$START" plan --engine sweep-v2 "python prerequisite" 2>&1) +NO_PY_RC=$? +if [ "$NO_PY_RC" -ne 0 ] && printf '%s' "$NO_PY_OUTPUT" | grep -q 'requires python3' && [ ! -d .claude ]; then + ok "sweep-v2 fails before state creation without python3" +else + bad "sweep-v2 fails before state creation without python3" +fi rm -rf .claude bash "$START" plan "legacy topic" >/dev/null 2>&1 LEGACY_STATE=$(ls .claude/claudex/*.state | head -1) @@ -104,11 +121,43 @@ echo '{}' | bash "$HOOK" >/dev/null 2>&1 REVIEW_RUNNER=$(ls .claude/claudex/*-runner.sh | head -1) check "review runner remains single senior-engineer review" grep -qi 'senior' "$REVIEW_RUNNER" +STALE_DIR=$(mktemp -d); TEMPS+=("$STALE_DIR") +export CLAUDEX_STATE_DIR="$STALE_DIR" CLAUDEX_STALE_MINUTES=1 CLAUDEX_SWEEP_V2_STALE_MINUTES=3 +source "$PLUGIN_ROOT/scripts/state-helpers.sh" +printf 'engine: sweep-v2\nphase: reviewing\n' > "$STALE_DIR/sweep.state" +python3 - "$STALE_DIR/sweep.state" 120 <<'PY' +import os, sys, time +p, age = sys.argv[1], int(sys.argv[2]); t = time.time() - age; os.utime(p, (t, t)) +PY +claudex_sweep_stale +check "active-age sweep-v2 state survives legacy stale window" test -f "$STALE_DIR/sweep.state" +python3 - "$STALE_DIR/sweep.state" 240 <<'PY' +import os, sys, time +p, age = sys.argv[1], int(sys.argv[2]); t = time.time() - age; os.utime(p, (t, t)) +PY +claudex_sweep_stale +check "abandoned sweep-v2 state expires at extended stale window" test ! -f "$STALE_DIR/sweep.state" +printf 'engine: legacy\nphase: reviewing\n' > "$STALE_DIR/locked.state" +printf '%s\n' "$$" > "$STALE_DIR/locked.lock" +python3 - "$STALE_DIR/locked.state" 120 <<'PY' +import os, sys, time +p, age = sys.argv[1], int(sys.argv[2]); t = time.time() - age; os.utime(p, (t, t)) +PY +claudex_sweep_stale +check "live runner lock prevents stale reaping" test -f "$STALE_DIR/locked.state" +unset CLAUDEX_STATE_DIR CLAUDEX_STALE_MINUTES CLAUDEX_SWEEP_V2_STALE_MINUTES + printf '\n\033[1mClean convergence and contracts\033[0m\n' new_repo start_sweep ORDER="$TEST_DIR/order"; export CLAUDEX_SWEEP_ORDER_LOG="$ORDER" bash "$RUNNER" >/dev/null 2>&1 +STATUS_OUTPUT=$(bash "$PLUGIN_ROOT/scripts/status.sh") +if printf '%s' "$STATUS_OUTPUT" | grep -q 'summarizing' && ! printf '%s' "$STATUS_OUTPUT" | grep -q 'unknown'; then + ok "status recognizes sweep-v2 summarizing phase as active" +else + bad "status recognizes sweep-v2 summarizing phase as active" +fi GEN_DIR=".claude/claudex/$ID/generations/1" check "five clean personas converge generation one" test "$(field "$STATE" decision_signal)" = converged check "clean convergence records complete coverage" test "$(field "$STATE" coverage_complete)" = true @@ -119,7 +168,9 @@ p=pathlib.Path(sys.argv[1]); manifest=json.loads((p/'manifest.json').read_text() results=[json.loads(x.read_text()) for x in p.glob('*.result.json')] assert len(results)==5 assert all(r['expected_snapshot_sha256']==h and r['actual_snapshot_sha256_before']==h and r['actual_snapshot_sha256_after']==h for r in results) +assert all(len(r['findings_sha256'])==64 for r in results) PY +check "completed findings and sidecars are write-discouraged" bash -c "[ ! -w '$GEN_DIR/security-data.findings.md' ] && [ ! -w '$GEN_DIR/security-data.result.json' ]" check "manifest records complete contract" python3 - "$GEN_DIR/manifest.json" <<'PY' import json, pathlib, sys m=json.loads(pathlib.Path(sys.argv[1]).read_text()) @@ -144,6 +195,7 @@ check "reviewer contract preserves approval gates" grep -q 'approval-gated decis new_repo; start_sweep bash "$RUNNER" >/dev/null 2>&1 MUTATED_RESULT=".claude/claudex/$ID/generations/1/security-data.result.json" +chmod u+w "$MUTATED_RESULT" python3 - "$MUTATED_RESULT" <<'PY' import json, pathlib, sys p=pathlib.Path(sys.argv[1]); d=json.loads(p.read_text()); d['completed_at']='mutated'; p.write_text(json.dumps(d)) @@ -151,6 +203,14 @@ PY echo '{}' | bash "$HOOK" >/dev/null 2>&1 check "post-run mutated sidecar degrades before summary" test "$(field "$STATE" decision_signal)" = degraded +new_repo; start_sweep +bash "$RUNNER" >/dev/null 2>&1 +MUTATED_FINDINGS=".claude/claudex/$ID/generations/1/product-domain.findings.md" +chmod u+w "$MUTATED_FINDINGS" +printf '\n' >> "$MUTATED_FINDINGS" +echo '{}' | bash "$HOOK" >/dev/null 2>&1 +check "post-run findings digest mismatch degrades before summary" test "$(field "$STATE" decision_signal)" = degraded + run_degraded_case() { local mode="$1" persona="$2" expected_name="$3" new_repo; start_sweep @@ -163,6 +223,12 @@ printf '\n\033[1mMaterial and degraded outcomes\033[0m\n' new_repo; start_sweep export CLAUDEX_SWEEP_STUB_MODE=material CLAUDEX_SWEEP_STUB_PERSONA=operations-deployment bash "$RUNNER" >/dev/null 2>&1 +MATERIAL_STATUS=$(bash "$PLUGIN_ROOT/scripts/status.sh") +if printf '%s' "$MATERIAL_STATUS" | grep -q 'awaiting-revision' && ! printf '%s' "$MATERIAL_STATUS" | grep -q 'unknown'; then + ok "status recognizes sweep-v2 awaiting-revision phase as active" +else + bad "status recognizes sweep-v2 awaiting-revision phase as active" +fi check "four clean plus one material cannot converge" test "$(field "$STATE" decision_signal)" = material-findings check "material findings require a revision" test "$(field "$STATE" revision_required)" = true run_degraded_case missing security-data "missing persona output degrades" @@ -171,6 +237,42 @@ run_degraded_case nonzero architecture-scope "nonzero reviewer exit degrades" run_degraded_case snapshot-mutation quality-accessibility-performance "snapshot hash mismatch degrades" run_degraded_case live-mutation operations-deployment "live PLAN.md mutation during sweep degrades" +new_repo +export CLAUDEX_SWEEP_PERSONA_TIMEOUT_SECONDS=1 +start_sweep +TIMEOUT_CHILD="$TEST_DIR/timeout-child.pid" +export CLAUDEX_SWEEP_STUB_MODE=timeout CLAUDEX_SWEEP_STUB_PERSONA=security-data CLAUDEX_SWEEP_TIMEOUT_CHILD_FILE="$TIMEOUT_CHILD" +bash "$RUNNER" >/dev/null 2>&1 +check "persona timeout degrades the sweep" test "$(field "$STATE" decision_signal)" = degraded +TIMEOUT_PID=$(cat "$TIMEOUT_CHILD" 2>/dev/null) +check "persona timeout kills the reviewer process group" bash -c "[ -n '$TIMEOUT_PID' ] && ! kill -0 '$TIMEOUT_PID' 2>/dev/null" +unset CLAUDEX_SWEEP_PERSONA_TIMEOUT_SECONDS CLAUDEX_SWEEP_TIMEOUT_CHILD_FILE + +new_repo +export CLAUDEX_SWEEP_PERSONA_TIMEOUT_SECONDS=30 +start_sweep +export CLAUDEX_SWEEP_STUB_MODE=timeout CLAUDEX_SWEEP_STUB_PERSONA=architecture-scope +bash "$RUNNER" >/dev/null 2>&1 & +RUNNER_TEST_PID=$! +ACTIVE_PGID_FILE=".claude/claudex/$ID-active-pgid" +i=0 +while [ ! -s "$ACTIVE_PGID_FILE" ] && [ "$i" -lt 50 ]; do sleep 0.1; i=$((i + 1)); done +CANCELLED_PGID=$(cat "$ACTIVE_PGID_FILE" 2>/dev/null) +bash "$CANCEL" >/dev/null 2>&1 +wait "$RUNNER_TEST_PID" 2>/dev/null +check "cancel preserves terminal cancelled state" test "$(field "$STATE" phase)" = cancelled +check "cancel terminates the active reviewer process group" bash -c "[ -n '$CANCELLED_PGID' ] && ! kill -0 -- '-$CANCELLED_PGID' 2>/dev/null" +check "cancel removes active process metadata" test ! -e "$ACTIVE_PGID_FILE" +unset CLAUDEX_SWEEP_PERSONA_TIMEOUT_SECONDS CLAUDEX_SWEEP_STUB_MODE CLAUDEX_SWEEP_STUB_PERSONA + +new_repo; start_sweep +bash "$RUNNER" >/dev/null 2>&1 +GEN_DIR=".claude/claudex/$ID/generations/1" +chmod a-w "$GEN_DIR" +echo '{}' | bash "$HOOK" >/dev/null 2>&1 +chmod u+w "$GEN_DIR" +check "summary revalidation I/O failure clears prior convergence" bash -c "[ \"\$(sed -n 's/^decision_signal: *//p' '$STATE')\" = degraded ] && [ \"\$(sed -n 's/^clean: *//p' '$STATE')\" = false ]" + printf '\n\033[1mCoverage, generations, and isolation\033[0m\n' new_repo; start_sweep GEN_DIR=".claude/claudex/$ID/generations/1" @@ -193,7 +295,10 @@ new_repo; start_sweep GEN1_SHA=$(field "$STATE" snapshot_sha256) export CLAUDEX_SWEEP_STUB_MODE=material CLAUDEX_SWEEP_STUB_PERSONA=architecture-scope bash "$RUNNER" >/dev/null 2>&1 -printf '\n2. Address the finding.\n\n## Changelog\n- Accepted architecture-scope finding: added the missing failure handling.\n' >> PLAN.md +printf '\n2. Unrelated edit.\n\n## Changelog\n- Prior entry only.\n' >> PLAN.md +echo '{}' | bash "$HOOK" >/dev/null 2>&1 +check "stale changelog cannot advance a material generation" bash -c "[ \"\$(sed -n 's/^phase: *//p' '$STATE')\" = awaiting-revision ] && [ \"\$(sed -n 's/^generation: *//p' '$STATE')\" = 1 ]" +printf '\n### Sweep generation 1 — %s\n- Accepted [architecture-scope-high-1]: added the missing failure handling to the scoped plan.\n' "$GEN1_SHA" >> PLAN.md unset CLAUDEX_SWEEP_STUB_MODE CLAUDEX_SWEEP_STUB_PERSONA echo '{}' | bash "$HOOK" >/dev/null 2>&1 GEN2_SHA=$(field "$STATE" snapshot_sha256) From b1c7aa2fd8bc0ed3fde98310a46a337f4a0cf94d Mon Sep 17 00:00:00 2001 From: robgfl45 <227035225+robgfl45@users.noreply.github.com> Date: Sat, 11 Jul 2026 12:53:16 -0400 Subject: [PATCH 05/10] chore: retrigger repository checks From 279ddc0d562c2d9c53d50d64c2103b27d686e772 Mon Sep 17 00:00:00 2001 From: robgfl45 <227035225+robgfl45@users.noreply.github.com> Date: Sat, 11 Jul 2026 13:16:36 -0400 Subject: [PATCH 06/10] fix: address Cubic sweep integrity findings --- plugins/claudex/hooks/stop-hook.sh | 16 ++- plugins/claudex/scripts/doctor.sh | 1 + plugins/claudex/scripts/start-loop.sh | 4 + plugins/claudex/scripts/sweep-helpers.sh | 168 +++++++++++++++++++---- plugins/claudex/tests/sweep-v2-test.sh | 47 ++++++- plugins/claudex/tests/synthetic-e2e.sh | 6 +- 6 files changed, 210 insertions(+), 32 deletions(-) diff --git a/plugins/claudex/hooks/stop-hook.sh b/plugins/claudex/hooks/stop-hook.sh index 4648c59..b7816f6 100755 --- a/plugins/claudex/hooks/stop-hook.sh +++ b/plugins/claudex/hooks/stop-hook.sh @@ -212,7 +212,6 @@ if [ "$MODE" = "plan" ] && [ "$ENGINE" = "sweep-v2" ]; then GENERATION=$(claudex_state_read_field "$ACTIVE_STATE" generation) MAX_GENERATIONS=$(claudex_state_read_field "$ACTIVE_STATE" max_generations) SNAPSHOT_SHA=$(claudex_state_read_field "$ACTIVE_STATE" snapshot_sha256) - COVERAGE_COMPLETE=$(claudex_state_read_field "$ACTIVE_STATE" coverage_complete) case "$GENERATION" in ''|*[!0-9]*) GENERATION=1 ;; esac case "$MAX_GENERATIONS" in ''|*[!0-9]*) MAX_GENERATIONS=5 ;; esac [ "$MAX_GENERATIONS" -le 5 ] || MAX_GENERATIONS=5 @@ -242,6 +241,14 @@ Do not edit the snapshot or live \`PLAN.md\` while it runs. When it finishes, en if [ -z "$CURRENT_LIVE_SHA" ] || [ "$CURRENT_LIVE_SHA" = "$SNAPSHOT_SHA" ]; then block "Sweep-v2 found material issues in generation $GENERATION. Read \`$CONSOLIDATED\`, revise live \`PLAN.md\` exactly once, and add or update \`## Changelog\` recording each accepted or rejected item with reasons. Do not modify the frozen snapshot. Then end your turn." fi + if claudex_sweep_consolidate "$ACTIVE_STATE" "$REVIEW_ID" "$GENERATION" "$SNAPSHOT_SHA" "$CURRENT_LIVE_SHA" >/dev/null 2>&1; then + RECONCILIATION_RC=0 + else + RECONCILIATION_RC=$? + fi + if [ "$RECONCILIATION_RC" -ne 1 ]; then + block "Sweep-v2 could not revalidate generation-$GENERATION evidence before accepting the revision. The generation is degraded and cannot advance; end your turn for the terminal summary or cancel the loop." + fi if ! claudex_sweep_validate_reconciliation PLAN.md "$CONSOLIDATED" "$GENERATION" "$SNAPSHOT_SHA"; then block "The required plan revision exists, but its \`## Changelog\` does not reconcile every generation-$GENERATION finding. Add this exact heading under \`## Changelog\`: @@ -300,8 +307,11 @@ When the runner finishes, end your turn." || ! claudex_state_set_field "$ACTIVE_STATE" coverage_complete false; then block "Sweep-v2 could not persist its fail-closed revalidation state. No clean result is claimed; repair the state directory and retry or cancel." fi - claudex_sweep_consolidate "$ACTIVE_STATE" "$REVIEW_ID" "$GENERATION" "$SNAPSHOT_SHA" "$SNAPSHOT_SHA" >/dev/null 2>&1 - REVALIDATE_RC=$? + if claudex_sweep_consolidate "$ACTIVE_STATE" "$REVIEW_ID" "$GENERATION" "$SNAPSHOT_SHA" "$SNAPSHOT_SHA" >/dev/null 2>&1; then + REVALIDATE_RC=0 + else + REVALIDATE_RC=$? + fi case "$REVALIDATE_RC" in 0|2|3) ;; *) diff --git a/plugins/claudex/scripts/doctor.sh b/plugins/claudex/scripts/doctor.sh index a763f2f..692e380 100755 --- a/plugins/claudex/scripts/doctor.sh +++ b/plugins/claudex/scripts/doctor.sh @@ -83,6 +83,7 @@ fi section "Required runtime dependencies" check "python3 (required for sweep-v2 manifests and validation)" command -v python3 +check "SHA-256 implementation (shasum or sha256sum)" bash -c 'command -v shasum >/dev/null 2>&1 || command -v sha256sum >/dev/null 2>&1' section "State directory" mkdir -p "$CLAUDEX_STATE_DIR" 2>/dev/null diff --git a/plugins/claudex/scripts/start-loop.sh b/plugins/claudex/scripts/start-loop.sh index 16e039c..ab1e7f5 100755 --- a/plugins/claudex/scripts/start-loop.sh +++ b/plugins/claudex/scripts/start-loop.sh @@ -116,6 +116,10 @@ case "$MODE" in echo "--engine sweep-v2 requires python3 for manifests and artifact validation." >&2 exit 2 fi + if ! command -v shasum >/dev/null 2>&1 && ! command -v sha256sum >/dev/null 2>&1; then + echo "--engine sweep-v2 requires shasum or sha256sum for SHA-256 verification." >&2 + exit 2 + fi if [ ! -s "PLAN.md" ]; then echo "--engine sweep-v2 requires an existing non-empty PLAN.md." >&2 exit 2 diff --git a/plugins/claudex/scripts/sweep-helpers.sh b/plugins/claudex/scripts/sweep-helpers.sh index 4065658..bb143c8 100755 --- a/plugins/claudex/scripts/sweep-helpers.sh +++ b/plugins/claudex/scripts/sweep-helpers.sh @@ -10,11 +10,72 @@ claudex_sha256() { [ -f "$file" ] || return 1 if command -v shasum >/dev/null 2>&1; then shasum -a 256 "$file" | awk '{print $1}' - else + elif command -v sha256sum >/dev/null 2>&1; then sha256sum "$file" | awk '{print $1}' + else + return 127 fi } +claudex_sweep_set_fields_atomic() { + local state_file="$1" + shift + [ $(( $# % 2 )) -eq 0 ] || return 2 + python3 - "$state_file" "$@" <<'PY' +import datetime, os, pathlib, re, sys, tempfile + +path = pathlib.Path(sys.argv[1]) +args = sys.argv[2:] +updates = dict(zip(args[::2], args[1::2])) +if not updates or any(not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", key) for key in updates): + raise SystemExit(2) +updates.setdefault("last_updated_at", datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")) +lines = path.read_text(encoding="utf-8").splitlines() +seen = set() +out = [] +for line in lines: + key = line.split(":", 1)[0] if ":" in line else "" + if key in updates: + out.append(f"{key}: {updates[key]}") + seen.add(key) + else: + out.append(line) +for key, value in updates.items(): + if key not in seen: + out.append(f"{key}: {value}") +fd, tmp = tempfile.mkstemp(prefix=path.name + ".tmp.", dir=path.parent) +try: + with os.fdopen(fd, "w", encoding="utf-8") as handle: + handle.write("\n".join(out) + "\n") + handle.flush() + os.fsync(handle.fileno()) + os.replace(tmp, path) +finally: + try: + os.unlink(tmp) + except FileNotFoundError: + pass +PY +} + +claudex_sweep_evidence_sha256() { + local generation_dir="$1" + python3 - "$generation_dir" $CLAUDEX_SWEEP_PERSONAS <<'PY' +import hashlib, pathlib, sys + +root = pathlib.Path(sys.argv[1]) +digest = hashlib.sha256() +for persona in sys.argv[2:]: + for suffix in ("findings.md", "result.json"): + path = root / f"{persona}.{suffix}" + if not path.is_file(): + raise SystemExit(1) + digest.update(path.name.encode("utf-8") + b"\0") + digest.update(path.read_bytes()) + digest.update(b"\0") +print(digest.hexdigest()) +PY +} claudex_sweep_validate_findings() { local file="$1" @@ -132,12 +193,15 @@ PY [ $? -eq 0 ] || { rm -f "$manifest_tmp"; return 1; } mv "$manifest_tmp" "$generation_dir/manifest.json" || return 1 chmod a-w "$generation_dir/manifest.json" 2>/dev/null || true - claudex_state_set_field "$state_file" generation "$generation" || return 1 - claudex_state_set_field "$state_file" snapshot_sha256 "$sha" || return 1 - claudex_state_set_field "$state_file" coverage_complete false || return 1 - claudex_state_set_field "$state_file" decision_signal none || return 1 - claudex_state_set_field "$state_file" revision_required false || return 1 - claudex_state_set_field "$state_file" reviewed_live_sha256 "$sha" || return 1 + claudex_sweep_set_fields_atomic "$state_file" \ + generation "$generation" \ + snapshot_sha256 "$sha" \ + coverage_complete false \ + decision_signal none \ + revision_required false \ + reviewed_live_sha256 "$sha" \ + evidence_sha256 "" \ + consolidated_sha256 "" || return 1 printf '%s' "$sha" } @@ -176,6 +240,25 @@ claudex_sweep_consolidate() { current_snapshot=$(claudex_sha256 "$snapshot" 2>/dev/null) current_live=$(claudex_sha256 PLAN.md 2>/dev/null) local degraded=false material=false clean_count=0 + local stored_evidence stored_consolidated current_evidence current_consolidated + stored_evidence=$(claudex_state_read_field "$state_file" evidence_sha256) + stored_consolidated=$(claudex_state_read_field "$state_file" consolidated_sha256) + if [ -n "$stored_evidence" ] || [ -n "$stored_consolidated" ]; then + current_evidence=$(claudex_sweep_evidence_sha256 "$generation_dir" 2>/dev/null) + current_consolidated=$(claudex_sha256 "$consolidated" 2>/dev/null) + if [ -z "$stored_evidence" ] || [ -z "$stored_consolidated" ] \ + || [ "$current_evidence" != "$stored_evidence" ] \ + || [ "$current_consolidated" != "$stored_consolidated" ]; then + claudex_sweep_set_fields_atomic "$state_file" \ + coverage_complete false \ + decision_signal degraded \ + clean false \ + revision_required false \ + converged_snapshot_sha256 "" \ + phase summarizing || return 4 + return 2 + fi + fi local manifest="$generation_dir/manifest.json" local manifest_valid manifest_valid=$(python3 - "$manifest" "$generation" "$expected" "$generation_dir" "$(pwd -P)/PLAN.md" <<'PY' @@ -250,39 +333,62 @@ PY done } > "$tmp" || { rm -f "$tmp"; return 1; } mv "$tmp" "$consolidated" || return 1 + current_evidence=$(claudex_sweep_evidence_sha256 "$generation_dir" 2>/dev/null) || degraded=true + current_consolidated=$(claudex_sha256 "$consolidated" 2>/dev/null) || degraded=true + [ -n "$current_evidence" ] && [ -n "$current_consolidated" ] || degraded=true if [ "$current_snapshot" != "$expected" ] || [ "$current_live" != "$live_expected" ]; then degraded=true; fi if [ "$clean_count" -ne 5 ] && [ "$material" != "true" ]; then degraded=true; fi if [ "$degraded" = "true" ]; then - claudex_state_set_field "$state_file" coverage_complete false - claudex_state_set_field "$state_file" decision_signal degraded - claudex_state_set_field "$state_file" clean false - claudex_state_set_field "$state_file" phase summarizing + claudex_sweep_set_fields_atomic "$state_file" \ + coverage_complete false \ + decision_signal degraded \ + clean false \ + revision_required false \ + converged_snapshot_sha256 "" \ + evidence_sha256 "$current_evidence" \ + consolidated_sha256 "$current_consolidated" \ + phase summarizing || return 4 return 2 fi - claudex_state_set_field "$state_file" coverage_complete true if [ "$clean_count" -eq 5 ] && [ "$material" = "false" ]; then - claudex_state_set_field "$state_file" decision_signal converged - claudex_state_set_field "$state_file" clean true - claudex_state_set_field "$state_file" converged_snapshot_sha256 "$expected" - claudex_state_set_field "$state_file" phase summarizing + claudex_sweep_set_fields_atomic "$state_file" \ + coverage_complete true \ + decision_signal converged \ + clean true \ + revision_required false \ + converged_snapshot_sha256 "$expected" \ + evidence_sha256 "$current_evidence" \ + consolidated_sha256 "$current_consolidated" \ + phase summarizing || return 4 return 0 fi - claudex_state_set_field "$state_file" clean false local max_generations max_generations=$(claudex_state_read_field "$state_file" max_generations) case "$max_generations" in ''|*[!0-9]*) max_generations="$CLAUDEX_SWEEP_MAX_GENERATIONS" ;; esac [ "$max_generations" -le "$CLAUDEX_SWEEP_MAX_GENERATIONS" ] || max_generations="$CLAUDEX_SWEEP_MAX_GENERATIONS" if [ "$generation" -ge "$max_generations" ]; then - claudex_state_set_field "$state_file" decision_signal max-reached - claudex_state_set_field "$state_file" revision_required false - claudex_state_set_field "$state_file" phase summarizing + claudex_sweep_set_fields_atomic "$state_file" \ + coverage_complete true \ + decision_signal max-reached \ + clean false \ + revision_required false \ + converged_snapshot_sha256 "" \ + evidence_sha256 "$current_evidence" \ + consolidated_sha256 "$current_consolidated" \ + phase summarizing || return 4 return 3 fi - claudex_state_set_field "$state_file" decision_signal material-findings - claudex_state_set_field "$state_file" revision_required true - claudex_state_set_field "$state_file" phase awaiting-revision + claudex_sweep_set_fields_atomic "$state_file" \ + coverage_complete true \ + decision_signal material-findings \ + clean false \ + revision_required true \ + converged_snapshot_sha256 "" \ + evidence_sha256 "$current_evidence" \ + consolidated_sha256 "$current_consolidated" \ + phase awaiting-revision || return 4 return 1 } @@ -366,7 +472,14 @@ for persona in \$CLAUDEX_SWEEP_PERSONAS; do findings="\$GENERATION_DIR/\$persona.findings.md" result="\$GENERATION_DIR/\$persona.result.json" prompt="\$GENERATION_DIR/.\$persona.prompt.txt" - rm -f "\$findings" "\$result" "\$prompt" + if [ -e "\$findings" ] || [ -e "\$result" ]; then + claudex_sweep_set_fields_atomic "\$STATE_FILE" \ + coverage_complete false decision_signal degraded clean false \ + revision_required false converged_snapshot_sha256 '' phase summarizing + echo "[claudex] refusing to replace existing generation evidence for \$persona" >&2 + exit 2 + fi + rm -f "\$prompt" before=\$(claudex_sha256 "\$SNAPSHOT" 2>/dev/null) live_before=\$(claudex_sha256 PLAN.md 2>/dev/null) focus=\$(claudex_sweep_persona_prompt "\$persona") @@ -399,7 +512,12 @@ PROMPTEOF classification=\$(claudex_sweep_validate_findings "\$findings" 2>/dev/null) [ "\$classification" = clean ] || [ "\$classification" = material ] || classification=degraded fi - claudex_sweep_write_result "\$result" "\$persona" "\$EXPECTED" "\$before" "\$after" "\$rc" "\$findings" "\$classification" + if ! claudex_sweep_write_result "\$result" "\$persona" "\$EXPECTED" "\$before" "\$after" "\$rc" "\$findings" "\$classification"; then + claudex_sweep_set_fields_atomic "\$STATE_FILE" \ + coverage_complete false decision_signal degraded clean false \ + revision_required false converged_snapshot_sha256 '' phase summarizing + exit 2 + fi claudex_state_set_field "\$STATE_FILE" sweep_heartbeat "\$persona" rm -f "\$prompt" done diff --git a/plugins/claudex/tests/sweep-v2-test.sh b/plugins/claudex/tests/sweep-v2-test.sh index d30bcb3..f0b8ea7 100755 --- a/plugins/claudex/tests/sweep-v2-test.sh +++ b/plugins/claudex/tests/sweep-v2-test.sh @@ -15,7 +15,14 @@ ok() { printf ' \033[32m✓\033[0m %s\n' "$1"; PASS=$((PASS + 1)); } bad() { printf ' \033[31m✗\033[0m %s\n' "$1"; FAIL=$((FAIL + 1)); FAILURES+=("$1"); } check() { local name="$1"; shift; if "$@" >/dev/null 2>&1; then ok "$name"; else bad "$name"; fi; } field() { sed -n "s/^$2: *//p" "$1" | head -1; } -cleanup() { local p; for p in "${TEMPS[@]}"; do chmod -R u+w "$p" 2>/dev/null; rm -rf "$p"; done; } +cleanup() { + local p + if [ "${KEEP_SWEEP_TEMPS:-0}" = 1 ]; then + printf 'preserved sweep temp: %s\n' "${TEMPS[@]}" >&2 + return + fi + for p in "${TEMPS[@]}"; do chmod -R u+w "$p" 2>/dev/null; rm -rf "$p"; done +} trap cleanup EXIT make_stub() { @@ -107,6 +114,16 @@ if [ "$NO_PY_RC" -ne 0 ] && printf '%s' "$NO_PY_OUTPUT" | grep -q 'requires pyth else bad "sweep-v2 fails before state creation without python3" fi +NO_HASH_PATH=$(mktemp -d); TEMPS+=("$NO_HASH_PATH") +ln -s "$(command -v dirname)" "$NO_HASH_PATH/dirname" +ln -s "$(command -v python3)" "$NO_HASH_PATH/python3" +NO_HASH_OUTPUT=$(PATH="$NO_HASH_PATH" /bin/bash "$START" plan --engine sweep-v2 "hash prerequisite" 2>&1) +NO_HASH_RC=$? +if [ "$NO_HASH_RC" -ne 0 ] && printf '%s' "$NO_HASH_OUTPUT" | grep -q 'requires shasum or sha256sum' && [ ! -d .claude ]; then + ok "sweep-v2 fails before state creation without SHA-256 tooling" +else + bad "sweep-v2 fails before state creation without SHA-256 tooling" +fi rm -rf .claude bash "$START" plan "legacy topic" >/dev/null 2>&1 LEGACY_STATE=$(ls .claude/claudex/*.state | head -1) @@ -211,6 +228,12 @@ printf '\n' >> "$MUTATED_FINDINGS" echo '{}' | bash "$HOOK" >/dev/null 2>&1 check "post-run findings digest mismatch degrades before summary" test "$(field "$STATE" decision_signal)" = degraded +new_repo; start_sweep +bash "$RUNNER" >/dev/null 2>&1 +RERUN_RC=0 +bash "$RUNNER" >/dev/null 2>&1 || RERUN_RC=$? +check "completed generation evidence is write-once" bash -c "[ '$RERUN_RC' -eq 2 ] && [ \"\$(sed -n 's/^decision_signal: *//p' '$STATE')\" = degraded ]" + run_degraded_case() { local mode="$1" persona="$2" expected_name="$3" new_repo; start_sweep @@ -273,6 +296,16 @@ echo '{}' | bash "$HOOK" >/dev/null 2>&1 chmod u+w "$GEN_DIR" check "summary revalidation I/O failure clears prior convergence" bash -c "[ \"\$(sed -n 's/^decision_signal: *//p' '$STATE')\" = degraded ] && [ \"\$(sed -n 's/^clean: *//p' '$STATE')\" = false ]" +new_repo; start_sweep +bash "$RUNNER" >/dev/null 2>&1 +ATOMIC_SHA=$(field "$STATE" snapshot_sha256) +CLAUDEX_STATE_DIR=.claude/claudex CLAUDE_PLUGIN_ROOT="$PLUGIN_ROOT" /bin/bash -c 'source "$CLAUDE_PLUGIN_ROOT/scripts/state-helpers.sh"; source "$CLAUDE_PLUGIN_ROOT/scripts/personas.sh"; source "$CLAUDE_PLUGIN_ROOT/scripts/sweep-helpers.sh"; claudex_sweep_set_fields_atomic "$1" decision_signal none clean false phase reviewing' _ "$STATE" +chmod a-w .claude/claudex +ATOMIC_RC=0 +CLAUDEX_STATE_DIR=.claude/claudex CLAUDE_PLUGIN_ROOT="$PLUGIN_ROOT" /bin/bash -c 'source "$CLAUDE_PLUGIN_ROOT/scripts/state-helpers.sh"; source "$CLAUDE_PLUGIN_ROOT/scripts/personas.sh"; source "$CLAUDE_PLUGIN_ROOT/scripts/sweep-helpers.sh"; claudex_sweep_consolidate "$1" "$2" 1 "$3" "$3" >/dev/null 2>&1' _ "$STATE" "$ID" "$ATOMIC_SHA" || ATOMIC_RC=$? +chmod u+w .claude/claudex +check "unpersistable verdict cannot report convergence" bash -c "[ '$ATOMIC_RC' -ne 0 ] && [ \"\$(sed -n 's/^decision_signal: *//p' '$STATE')\" != converged ] && [ \"\$(sed -n 's/^clean: *//p' '$STATE')\" = false ]" + printf '\n\033[1mCoverage, generations, and isolation\033[0m\n' new_repo; start_sweep GEN_DIR=".claude/claudex/$ID/generations/1" @@ -291,6 +324,18 @@ ONLY_SHA=$(field "$STATE" snapshot_sha256) claudex_sweep_consolidate "$STATE" "$ID" 1 "$ONLY_SHA" "$ONLY_SHA" >/dev/null 2>&1 check "one clean reviewer alone cannot converge" bash -c "[ \"\$(sed -n 's/^decision_signal: *//p' '$STATE')\" = degraded ] && [ \"\$(sed -n 's/^coverage_complete: *//p' '$STATE')\" = false ]" +new_repo; start_sweep +TAMPER_SHA=$(field "$STATE" snapshot_sha256) +export CLAUDEX_SWEEP_STUB_MODE=material CLAUDEX_SWEEP_STUB_PERSONA=architecture-scope +bash "$RUNNER" >/dev/null 2>&1 +TAMPER_CONSOLIDATED=".claude/claudex/$ID/generations/1/consolidated-findings.md" +chmod u+w "$TAMPER_CONSOLIDATED" +printf '\ntampered\n' >> "$TAMPER_CONSOLIDATED" +printf '\n2. Address finding.\n\n## Changelog\n### Sweep generation 1 — %s\n- Accepted [architecture-scope-high-1]: addressed with a scoped failure-handling change.\n' "$TAMPER_SHA" >> PLAN.md +unset CLAUDEX_SWEEP_STUB_MODE CLAUDEX_SWEEP_STUB_PERSONA +echo '{}' | bash "$HOOK" >/dev/null 2>&1 +check "altered generation evidence cannot advance to a new snapshot" bash -c "[ \"\$(sed -n 's/^decision_signal: *//p' '$STATE')\" = degraded ] && [ \"\$(sed -n 's/^generation: *//p' '$STATE')\" = 1 ]" + new_repo; start_sweep GEN1_SHA=$(field "$STATE" snapshot_sha256) export CLAUDEX_SWEEP_STUB_MODE=material CLAUDEX_SWEEP_STUB_PERSONA=architecture-scope diff --git a/plugins/claudex/tests/synthetic-e2e.sh b/plugins/claudex/tests/synthetic-e2e.sh index 2800532..d453e8c 100755 --- a/plugins/claudex/tests/synthetic-e2e.sh +++ b/plugins/claudex/tests/synthetic-e2e.sh @@ -117,9 +117,9 @@ echo "Runner exit code: $RUNNER_RC" echo "$CODEX_OUTPUT" | tail -30 | head -20 check "runner exited with code 0" test "$RUNNER_RC" = "0" check "Codex output is non-empty" test -n "$CODEX_OUTPUT" -check "Codex did not hit stdin terminal error" bash -c "! echo '$CODEX_OUTPUT' | grep -q 'stdin is not a terminal'" -check "Codex did not hit auth error" bash -c "! echo '$CODEX_OUTPUT' | grep -qi 'not logged in\\|auth.*fail'" -check "Codex output mentions plan/review topic" bash -c "echo '$CODEX_OUTPUT' | grep -qiE 'plan|review|expir|finding'" +check "Codex did not hit stdin terminal error" env CODEX_OUTPUT="$CODEX_OUTPUT" bash -c '! printf "%s" "$CODEX_OUTPUT" | grep -q "stdin is not a terminal"' +check "Codex did not hit auth error" env CODEX_OUTPUT="$CODEX_OUTPUT" bash -c '! printf "%s" "$CODEX_OUTPUT" | grep -qiE "not logged in|auth.*fail"' +check "Codex output mentions plan/review topic" env CODEX_OUTPUT="$CODEX_OUTPUT" bash -c 'printf "%s" "$CODEX_OUTPUT" | grep -qiE "plan|review|expir|finding"' # Round 1 done. Now simulate Claude deciding the loop is complete and calling mark-done. # (We don't try to parse Codex output to decide -- that's Claude's job in production. From 657bb7b888bd3a8aefd1487a6c3de67ff1e00fa9 Mon Sep 17 00:00:00 2001 From: robgfl45 <227035225+robgfl45@users.noreply.github.com> Date: Sat, 11 Jul 2026 13:26:07 -0400 Subject: [PATCH 07/10] fix: preserve cancellation across verdict races --- plugins/claudex/hooks/stop-hook.sh | 6 ++ plugins/claudex/scripts/cancel-loop.sh | 19 ++++-- plugins/claudex/scripts/state-helpers.sh | 2 +- plugins/claudex/scripts/sweep-helpers.sh | 84 +++++++++++++++--------- plugins/claudex/tests/sweep-v2-test.sh | 7 ++ plugins/claudex/tests/synthetic-e2e.sh | 2 +- 6 files changed, 82 insertions(+), 38 deletions(-) diff --git a/plugins/claudex/hooks/stop-hook.sh b/plugins/claudex/hooks/stop-hook.sh index b7816f6..72d094a 100755 --- a/plugins/claudex/hooks/stop-hook.sh +++ b/plugins/claudex/hooks/stop-hook.sh @@ -246,6 +246,9 @@ Do not edit the snapshot or live \`PLAN.md\` while it runs. When it finishes, en else RECONCILIATION_RC=$? fi + if [ "$(claudex_state_read_field "$ACTIVE_STATE" phase)" = cancelled ]; then + approve "sweep-v2 cancelled during evidence revalidation" + fi if [ "$RECONCILIATION_RC" -ne 1 ]; then block "Sweep-v2 could not revalidate generation-$GENERATION evidence before accepting the revision. The generation is degraded and cannot advance; end your turn for the terminal summary or cancel the loop." fi @@ -312,6 +315,9 @@ When the runner finishes, end your turn." else REVALIDATE_RC=$? fi + if [ "$(claudex_state_read_field "$ACTIVE_STATE" phase)" = cancelled ]; then + approve "sweep-v2 cancelled during terminal revalidation" + fi case "$REVALIDATE_RC" in 0|2|3) ;; *) diff --git a/plugins/claudex/scripts/cancel-loop.sh b/plugins/claudex/scripts/cancel-loop.sh index 1a1940e..8e11b42 100755 --- a/plugins/claudex/scripts/cancel-loop.sh +++ b/plugins/claudex/scripts/cancel-loop.sh @@ -16,14 +16,25 @@ fi REVIEW_ID=$(basename "$ACTIVE" .state) echo "Cancelling loop: $REVIEW_ID" - -claudex_state_set_field "$ACTIVE" "phase" "cancelled" -claudex_state_set_field "$ACTIVE" "last_updated_at" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" +ENGINE=$(claudex_state_read_field "$ACTIVE" engine) +if [ "$ENGINE" = "sweep-v2" ]; then + # Share the same advisory write lock as verdict persistence so cancellation + # cannot be overwritten by a racing whole-state replacement. + # shellcheck source=/dev/null + source "$CLAUDE_PLUGIN_ROOT/scripts/sweep-helpers.sh" + claudex_sweep_set_fields_atomic "$ACTIVE" \ + phase cancelled decision_signal cancelled clean false revision_required false || { + echo "Could not persist cancellation." >&2 + exit 1 + } +else + claudex_state_set_field "$ACTIVE" "phase" "cancelled" + claudex_state_set_field "$ACTIVE" "last_updated_at" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" +fi # sweep-v2 runs each Codex reviewer in its own process group. Terminate the # active group before removing artifacts so cancellation cannot leave an # orphan reviewer writing into a cancelled generation. -ENGINE=$(claudex_state_read_field "$ACTIVE" engine) ACTIVE_PGID_FILE="$CLAUDEX_STATE_DIR/$REVIEW_ID-active-pgid" if [ "$ENGINE" = "sweep-v2" ] && [ -f "$ACTIVE_PGID_FILE" ]; then ACTIVE_PGID=$(cat "$ACTIVE_PGID_FILE" 2>/dev/null) diff --git a/plugins/claudex/scripts/state-helpers.sh b/plugins/claudex/scripts/state-helpers.sh index 7841fe9..0a3cfdc 100755 --- a/plugins/claudex/scripts/state-helpers.sh +++ b/plugins/claudex/scripts/state-helpers.sh @@ -148,7 +148,7 @@ claudex_sweep_stale() { if [ "$engine" = "sweep-v2" ] && ! find "$f" -prune -mmin "+$CLAUDEX_SWEEP_V2_STALE_MINUTES" -print 2>/dev/null | grep -q .; then continue fi - rm -f "$f" "$CLAUDEX_STATE_DIR/${id}.lock" "$CLAUDEX_STATE_DIR/${id}-runner.sh" "$CLAUDEX_STATE_DIR/${id}-prompt.txt" "$CLAUDEX_STATE_DIR/${id}-active-pgid" 2>/dev/null + rm -f "$f" "$CLAUDEX_STATE_DIR/${id}.lock" "$CLAUDEX_STATE_DIR/${id}-runner.sh" "$CLAUDEX_STATE_DIR/${id}-prompt.txt" "$CLAUDEX_STATE_DIR/${id}-active-pgid" "$CLAUDEX_STATE_DIR/${id}.state.write-lock" 2>/dev/null rm -rf "$CLAUDEX_STATE_DIR/${id}" 2>/dev/null done return 0 diff --git a/plugins/claudex/scripts/sweep-helpers.sh b/plugins/claudex/scripts/sweep-helpers.sh index bb143c8..06e5f68 100755 --- a/plugins/claudex/scripts/sweep-helpers.sh +++ b/plugins/claudex/scripts/sweep-helpers.sh @@ -20,41 +20,57 @@ claudex_sha256() { claudex_sweep_set_fields_atomic() { local state_file="$1" shift + local expected_phase="" + if [ "${1:-}" = "--expect-phase" ]; then + expected_phase="${2:-}" + shift 2 + fi [ $(( $# % 2 )) -eq 0 ] || return 2 - python3 - "$state_file" "$@" <<'PY' -import datetime, os, pathlib, re, sys, tempfile + python3 - "$state_file" "$expected_phase" "$@" <<'PY' +import datetime, fcntl, os, pathlib, re, sys, tempfile path = pathlib.Path(sys.argv[1]) -args = sys.argv[2:] +expected_phase = sys.argv[2] +args = sys.argv[3:] updates = dict(zip(args[::2], args[1::2])) if not updates or any(not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", key) for key in updates): raise SystemExit(2) updates.setdefault("last_updated_at", datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")) -lines = path.read_text(encoding="utf-8").splitlines() -seen = set() -out = [] -for line in lines: - key = line.split(":", 1)[0] if ":" in line else "" - if key in updates: - out.append(f"{key}: {updates[key]}") - seen.add(key) - else: - out.append(line) -for key, value in updates.items(): - if key not in seen: - out.append(f"{key}: {value}") -fd, tmp = tempfile.mkstemp(prefix=path.name + ".tmp.", dir=path.parent) -try: - with os.fdopen(fd, "w", encoding="utf-8") as handle: - handle.write("\n".join(out) + "\n") - handle.flush() - os.fsync(handle.fileno()) - os.replace(tmp, path) -finally: +lock_path = path.with_name(path.name + ".write-lock") +with lock_path.open("a+", encoding="utf-8") as lock: + fcntl.flock(lock.fileno(), fcntl.LOCK_EX) + lines = path.read_text(encoding="utf-8").splitlines() + current = {} + for line in lines: + if ":" in line: + key, value = line.split(":", 1) + current.setdefault(key, value.strip()) + if expected_phase and current.get("phase") != expected_phase: + raise SystemExit(3) + seen = set() + out = [] + for line in lines: + key = line.split(":", 1)[0] if ":" in line else "" + if key in updates: + out.append(f"{key}: {updates[key]}") + seen.add(key) + else: + out.append(line) + for key, value in updates.items(): + if key not in seen: + out.append(f"{key}: {value}") + fd, tmp = tempfile.mkstemp(prefix=path.name + ".tmp.", dir=path.parent) try: - os.unlink(tmp) - except FileNotFoundError: - pass + with os.fdopen(fd, "w", encoding="utf-8") as handle: + handle.write("\n".join(out) + "\n") + handle.flush() + os.fsync(handle.fileno()) + os.replace(tmp, path) + finally: + try: + os.unlink(tmp) + except FileNotFoundError: + pass PY } @@ -236,6 +252,10 @@ claudex_sweep_consolidate() { local generation_dir="$CLAUDEX_STATE_DIR/$review_id/generations/$generation" local consolidated="$generation_dir/consolidated-findings.md" local snapshot="$generation_dir/PLAN.md" + local verdict_phase + verdict_phase=$(claudex_state_read_field "$state_file" phase) + [ "$verdict_phase" = cancelled ] && return 130 + case "$verdict_phase" in reviewing|awaiting-revision|summarizing) ;; *) return 4 ;; esac local current_snapshot current_live current_snapshot=$(claudex_sha256 "$snapshot" 2>/dev/null) current_live=$(claudex_sha256 PLAN.md 2>/dev/null) @@ -249,7 +269,7 @@ claudex_sweep_consolidate() { if [ -z "$stored_evidence" ] || [ -z "$stored_consolidated" ] \ || [ "$current_evidence" != "$stored_evidence" ] \ || [ "$current_consolidated" != "$stored_consolidated" ]; then - claudex_sweep_set_fields_atomic "$state_file" \ + claudex_sweep_set_fields_atomic "$state_file" --expect-phase "$verdict_phase" \ coverage_complete false \ decision_signal degraded \ clean false \ @@ -341,7 +361,7 @@ PY if [ "$clean_count" -ne 5 ] && [ "$material" != "true" ]; then degraded=true; fi if [ "$degraded" = "true" ]; then - claudex_sweep_set_fields_atomic "$state_file" \ + claudex_sweep_set_fields_atomic "$state_file" --expect-phase "$verdict_phase" \ coverage_complete false \ decision_signal degraded \ clean false \ @@ -353,7 +373,7 @@ PY return 2 fi if [ "$clean_count" -eq 5 ] && [ "$material" = "false" ]; then - claudex_sweep_set_fields_atomic "$state_file" \ + claudex_sweep_set_fields_atomic "$state_file" --expect-phase "$verdict_phase" \ coverage_complete true \ decision_signal converged \ clean true \ @@ -369,7 +389,7 @@ PY case "$max_generations" in ''|*[!0-9]*) max_generations="$CLAUDEX_SWEEP_MAX_GENERATIONS" ;; esac [ "$max_generations" -le "$CLAUDEX_SWEEP_MAX_GENERATIONS" ] || max_generations="$CLAUDEX_SWEEP_MAX_GENERATIONS" if [ "$generation" -ge "$max_generations" ]; then - claudex_sweep_set_fields_atomic "$state_file" \ + claudex_sweep_set_fields_atomic "$state_file" --expect-phase "$verdict_phase" \ coverage_complete true \ decision_signal max-reached \ clean false \ @@ -380,7 +400,7 @@ PY phase summarizing || return 4 return 3 fi - claudex_sweep_set_fields_atomic "$state_file" \ + claudex_sweep_set_fields_atomic "$state_file" --expect-phase "$verdict_phase" \ coverage_complete true \ decision_signal material-findings \ clean false \ diff --git a/plugins/claudex/tests/sweep-v2-test.sh b/plugins/claudex/tests/sweep-v2-test.sh index f0b8ea7..ea0a314 100755 --- a/plugins/claudex/tests/sweep-v2-test.sh +++ b/plugins/claudex/tests/sweep-v2-test.sh @@ -288,6 +288,13 @@ check "cancel terminates the active reviewer process group" bash -c "[ -n '$CANC check "cancel removes active process metadata" test ! -e "$ACTIVE_PGID_FILE" unset CLAUDEX_SWEEP_PERSONA_TIMEOUT_SECONDS CLAUDEX_SWEEP_STUB_MODE CLAUDEX_SWEEP_STUB_PERSONA +new_repo; start_sweep +source "$PLUGIN_ROOT/scripts/state-helpers.sh"; source "$PLUGIN_ROOT/scripts/personas.sh"; source "$PLUGIN_ROOT/scripts/sweep-helpers.sh" +claudex_sweep_set_fields_atomic "$STATE" phase cancelled decision_signal cancelled clean false +STALE_VERDICT_RC=0 +claudex_sweep_set_fields_atomic "$STATE" --expect-phase reviewing phase summarizing decision_signal converged clean true || STALE_VERDICT_RC=$? +check "cancelled phase wins a racing stale verdict CAS" bash -c "[ '$STALE_VERDICT_RC' -eq 3 ] && [ \"\$(sed -n 's/^phase: *//p' '$STATE')\" = cancelled ] && [ \"\$(sed -n 's/^decision_signal: *//p' '$STATE')\" = cancelled ]" + new_repo; start_sweep bash "$RUNNER" >/dev/null 2>&1 GEN_DIR=".claude/claudex/$ID/generations/1" diff --git a/plugins/claudex/tests/synthetic-e2e.sh b/plugins/claudex/tests/synthetic-e2e.sh index d453e8c..59678fb 100755 --- a/plugins/claudex/tests/synthetic-e2e.sh +++ b/plugins/claudex/tests/synthetic-e2e.sh @@ -118,7 +118,7 @@ echo "$CODEX_OUTPUT" | tail -30 | head -20 check "runner exited with code 0" test "$RUNNER_RC" = "0" check "Codex output is non-empty" test -n "$CODEX_OUTPUT" check "Codex did not hit stdin terminal error" env CODEX_OUTPUT="$CODEX_OUTPUT" bash -c '! printf "%s" "$CODEX_OUTPUT" | grep -q "stdin is not a terminal"' -check "Codex did not hit auth error" env CODEX_OUTPUT="$CODEX_OUTPUT" bash -c '! printf "%s" "$CODEX_OUTPUT" | grep -qiE "not logged in|auth.*fail"' +check "Codex did not hit auth error" env CODEX_OUTPUT="$CODEX_OUTPUT" bash -c '! printf "%s" "$CODEX_OUTPUT" | grep -qiE "not logged in|authentication failed|login required|missing (api key|bearer token)"' check "Codex output mentions plan/review topic" env CODEX_OUTPUT="$CODEX_OUTPUT" bash -c 'printf "%s" "$CODEX_OUTPUT" | grep -qiE "plan|review|expir|finding"' # Round 1 done. Now simulate Claude deciding the loop is complete and calling mark-done. From e1d51d4b9e9d87c5a523343a56cb7aedc78585bc Mon Sep 17 00:00:00 2001 From: robgfl45 <227035225+robgfl45@users.noreply.github.com> Date: Sat, 11 Jul 2026 13:32:17 -0400 Subject: [PATCH 08/10] fix: keep cancellation authoritative in summaries --- plugins/claudex/hooks/stop-hook.sh | 40 ++++++++++++++++++-------- plugins/claudex/tests/sweep-v2-test.sh | 6 ++++ 2 files changed, 34 insertions(+), 12 deletions(-) diff --git a/plugins/claudex/hooks/stop-hook.sh b/plugins/claudex/hooks/stop-hook.sh index 72d094a..eaded19 100755 --- a/plugins/claudex/hooks/stop-hook.sh +++ b/plugins/claudex/hooks/stop-hook.sh @@ -304,10 +304,16 @@ When the runner finishes, end your turn." summarizing) # Revalidate every current-generation artifact at summary time so a # post-run mutation cannot ride a previously clean state signal. Clear - # the prior verdict first so an I/O/consolidation failure fails closed. - if ! claudex_state_set_field "$ACTIVE_STATE" decision_signal degraded \ - || ! claudex_state_set_field "$ACTIVE_STATE" clean false \ - || ! claudex_state_set_field "$ACTIVE_STATE" coverage_complete false; then + # the prior verdict first so an I/O/consolidation failure fails closed, + # but never overwrite a cancellation that won the state-write race. + if [ "$(claudex_state_read_field "$ACTIVE_STATE" phase)" = cancelled ]; then + approve "sweep-v2 cancelled before terminal revalidation" + fi + if ! claudex_sweep_set_fields_atomic "$ACTIVE_STATE" --expect-phase summarizing \ + decision_signal degraded clean false coverage_complete false; then + if [ "$(claudex_state_read_field "$ACTIVE_STATE" phase)" = cancelled ]; then + approve "sweep-v2 cancelled before fail-closed revalidation" + fi block "Sweep-v2 could not persist its fail-closed revalidation state. No clean result is claimed; repair the state directory and retry or cancel." fi if claudex_sweep_consolidate "$ACTIVE_STATE" "$REVIEW_ID" "$GENERATION" "$SNAPSHOT_SHA" "$SNAPSHOT_SHA" >/dev/null 2>&1; then @@ -321,10 +327,11 @@ When the runner finishes, end your turn." case "$REVALIDATE_RC" in 0|2|3) ;; *) - if ! claudex_state_set_field "$ACTIVE_STATE" decision_signal degraded \ - || ! claudex_state_set_field "$ACTIVE_STATE" clean false \ - || ! claudex_state_set_field "$ACTIVE_STATE" coverage_complete false \ - || ! claudex_state_set_field "$ACTIVE_STATE" phase summarizing; then + if ! claudex_sweep_set_fields_atomic "$ACTIVE_STATE" --expect-phase summarizing \ + decision_signal degraded clean false coverage_complete false phase summarizing; then + if [ "$(claudex_state_read_field "$ACTIVE_STATE" phase)" = cancelled ]; then + approve "sweep-v2 cancelled while persisting degraded revalidation" + fi block "Sweep-v2 revalidation failed and its degraded verdict could not be persisted. No clean result is claimed." fi ;; @@ -333,15 +340,24 @@ When the runner finishes, end your turn." CLEAN=$(claudex_state_read_field "$ACTIVE_STATE" clean) COVERAGE_COMPLETE=$(claudex_state_read_field "$ACTIVE_STATE" coverage_complete) if [ "$SIGNAL" = "converged" ] && { [ "$REVALIDATE_RC" -ne 0 ] || [ "$CLEAN" != "true" ] || [ "$COVERAGE_COMPLETE" != "true" ]; }; then - claudex_state_set_field "$ACTIVE_STATE" decision_signal degraded - claudex_state_set_field "$ACTIVE_STATE" clean false - claudex_state_set_field "$ACTIVE_STATE" coverage_complete false + if ! claudex_sweep_set_fields_atomic "$ACTIVE_STATE" --expect-phase summarizing \ + decision_signal degraded clean false coverage_complete false; then + if [ "$(claudex_state_read_field "$ACTIVE_STATE" phase)" = cancelled ]; then + approve "sweep-v2 cancelled while correcting inconsistent convergence" + fi + block "Sweep-v2 could not persist an inconsistent-convergence correction. No clean result is claimed." + fi SIGNAL=degraded CLEAN=false COVERAGE_COMPLETE=false fi CONVERGED_SHA=$(claudex_state_read_field "$ACTIVE_STATE" converged_snapshot_sha256) - claudex_state_set_field "$ACTIVE_STATE" phase done + if ! claudex_sweep_set_fields_atomic "$ACTIVE_STATE" --expect-phase summarizing phase done; then + if [ "$(claudex_state_read_field "$ACTIVE_STATE" phase)" = cancelled ]; then + approve "sweep-v2 cancelled before terminal summary commit" + fi + block "Sweep-v2 could not commit its terminal summary state. No result is claimed." + fi rm -f "$RUNNER" "$STATE_DIR/$REVIEW_ID-prompt.txt" "$STATE_DIR/$REVIEW_ID.lock" "$STATE_DIR/$REVIEW_ID-active-pgid" 2>/dev/null case "$SIGNAL" in converged) diff --git a/plugins/claudex/tests/sweep-v2-test.sh b/plugins/claudex/tests/sweep-v2-test.sh index ea0a314..c7769d9 100755 --- a/plugins/claudex/tests/sweep-v2-test.sh +++ b/plugins/claudex/tests/sweep-v2-test.sh @@ -295,6 +295,12 @@ STALE_VERDICT_RC=0 claudex_sweep_set_fields_atomic "$STATE" --expect-phase reviewing phase summarizing decision_signal converged clean true || STALE_VERDICT_RC=$? check "cancelled phase wins a racing stale verdict CAS" bash -c "[ '$STALE_VERDICT_RC' -eq 3 ] && [ \"\$(sed -n 's/^phase: *//p' '$STATE')\" = cancelled ] && [ \"\$(sed -n 's/^decision_signal: *//p' '$STATE')\" = cancelled ]" +new_repo; start_sweep +bash "$RUNNER" >/dev/null 2>&1 +bash "$CANCEL" >/dev/null 2>&1 +echo '{}' | bash "$HOOK" >/dev/null 2>&1 +check "terminal revalidation preserves an already-cancelled verdict" bash -c "[ \"\$(sed -n 's/^phase: *//p' '$STATE')\" = cancelled ] && [ \"\$(sed -n 's/^decision_signal: *//p' '$STATE')\" = cancelled ] && [ \"\$(sed -n 's/^clean: *//p' '$STATE')\" = false ]" + new_repo; start_sweep bash "$RUNNER" >/dev/null 2>&1 GEN_DIR=".claude/claudex/$ID/generations/1" From 2fa2a974bdd14c08d85a8bf2f778939065c01805 Mon Sep 17 00:00:00 2001 From: robgfl45 <227035225+robgfl45@users.noreply.github.com> Date: Sat, 11 Jul 2026 13:39:30 -0400 Subject: [PATCH 09/10] fix: clear stale coverage on sweep cancellation --- plugins/claudex/scripts/cancel-loop.sh | 2 +- plugins/claudex/tests/sweep-v2-test.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/claudex/scripts/cancel-loop.sh b/plugins/claudex/scripts/cancel-loop.sh index 8e11b42..10358e8 100755 --- a/plugins/claudex/scripts/cancel-loop.sh +++ b/plugins/claudex/scripts/cancel-loop.sh @@ -23,7 +23,7 @@ if [ "$ENGINE" = "sweep-v2" ]; then # shellcheck source=/dev/null source "$CLAUDE_PLUGIN_ROOT/scripts/sweep-helpers.sh" claudex_sweep_set_fields_atomic "$ACTIVE" \ - phase cancelled decision_signal cancelled clean false revision_required false || { + phase cancelled decision_signal cancelled clean false coverage_complete false revision_required false || { echo "Could not persist cancellation." >&2 exit 1 } diff --git a/plugins/claudex/tests/sweep-v2-test.sh b/plugins/claudex/tests/sweep-v2-test.sh index c7769d9..3fe2131 100755 --- a/plugins/claudex/tests/sweep-v2-test.sh +++ b/plugins/claudex/tests/sweep-v2-test.sh @@ -283,7 +283,7 @@ while [ ! -s "$ACTIVE_PGID_FILE" ] && [ "$i" -lt 50 ]; do sleep 0.1; i=$((i + 1) CANCELLED_PGID=$(cat "$ACTIVE_PGID_FILE" 2>/dev/null) bash "$CANCEL" >/dev/null 2>&1 wait "$RUNNER_TEST_PID" 2>/dev/null -check "cancel preserves terminal cancelled state" test "$(field "$STATE" phase)" = cancelled +check "cancel preserves terminal cancelled state" bash -c "[ \"\$(sed -n 's/^phase: *//p' '$STATE')\" = cancelled ] && [ \"\$(sed -n 's/^decision_signal: *//p' '$STATE')\" = cancelled ] && [ \"\$(sed -n 's/^clean: *//p' '$STATE')\" = false ] && [ \"\$(sed -n 's/^coverage_complete: *//p' '$STATE')\" = false ]" check "cancel terminates the active reviewer process group" bash -c "[ -n '$CANCELLED_PGID' ] && ! kill -0 -- '-$CANCELLED_PGID' 2>/dev/null" check "cancel removes active process metadata" test ! -e "$ACTIVE_PGID_FILE" unset CLAUDEX_SWEEP_PERSONA_TIMEOUT_SECONDS CLAUDEX_SWEEP_STUB_MODE CLAUDEX_SWEEP_STUB_PERSONA @@ -299,7 +299,7 @@ new_repo; start_sweep bash "$RUNNER" >/dev/null 2>&1 bash "$CANCEL" >/dev/null 2>&1 echo '{}' | bash "$HOOK" >/dev/null 2>&1 -check "terminal revalidation preserves an already-cancelled verdict" bash -c "[ \"\$(sed -n 's/^phase: *//p' '$STATE')\" = cancelled ] && [ \"\$(sed -n 's/^decision_signal: *//p' '$STATE')\" = cancelled ] && [ \"\$(sed -n 's/^clean: *//p' '$STATE')\" = false ]" +check "terminal revalidation preserves an already-cancelled verdict" bash -c "[ \"\$(sed -n 's/^phase: *//p' '$STATE')\" = cancelled ] && [ \"\$(sed -n 's/^decision_signal: *//p' '$STATE')\" = cancelled ] && [ \"\$(sed -n 's/^clean: *//p' '$STATE')\" = false ] && [ \"\$(sed -n 's/^coverage_complete: *//p' '$STATE')\" = false ]" new_repo; start_sweep bash "$RUNNER" >/dev/null 2>&1 From e31296a2a826b161c5bb6c448e81db5a0394152b Mon Sep 17 00:00:00 2001 From: robgfl45 <227035225+robgfl45@users.noreply.github.com> Date: Sat, 11 Jul 2026 13:59:19 -0400 Subject: [PATCH 10/10] feat: integrate sweep-v2 plan review adapter --- README.md | 8 +- bin/claudex-plan-review | 337 +++++++++++++++++- docs/HEADLESS_ADAPTER.md | 51 +-- skills/project-plan-review/SKILL.md | 38 +- .../project-plan-review/references/runbook.md | 42 ++- tests/test_adapter.py | 185 ++++++++-- 6 files changed, 567 insertions(+), 94 deletions(-) diff --git a/README.md b/README.md index 8bcf8e3..4fb1f61 100644 --- a/README.md +++ b/README.md @@ -235,13 +235,13 @@ The Stop hook is fail-open everywhere. Any error returns `{"decision":"approve"} ## Headless Hermes planning bridge -This fork adds [`bin/claudex-plan-review`](docs/HEADLESS_ADAPTER.md), a production-oriented adapter for running an existing `PLAN.md` through headless Claude Code, the Claudex Stop-hook lifecycle, and real Codex reviews from a Hermes leaf subagent. It validates explicit executable/plugin/auth prerequisites, pins child `PATH`, enforces wall-clock and Claude budget bounds, kills the complete process group on timeout, preserves evidence, and emits one strict JSON result. +This fork adds [`bin/claudex-plan-review`](docs/HEADLESS_ADAPTER.md), a production-oriented adapter for running an existing `PLAN.md` through headless Claude Code and Claudex. Its default `sweep-v2` path runs the Phase 1 frozen-snapshot, five-persona lifecycle with a generation cap of `1..5` (the staged Hermes workflow uses five); `--engine legacy` preserves backward compatibility. It validates explicit executable/plugin/auth prerequisites, pins child `PATH`, enforces wall-clock and Claude telemetry-budget bounds, kills the complete process group on timeout, copies complete state/generation evidence, and emits one strict JSON result. -Only `converged` is clean. `max_reached`, `degraded`, `failed`, and `timed_out` are explicit non-clean outcomes. Classification is based on Claudex state and findings artifacts, never Claude's prose tally. See the adapter document for exact usage, exit codes, costs, architecture, and staging instructions. The in-repo Hermes skill is staged at [`skills/project-plan-review/`](skills/project-plan-review/SKILL.md); it is not installed automatically. +Only `converged` is clean. `max_reached`, `degraded`, `failed`, and `timed_out` are explicit non-clean outcomes. Sweep classification requires terminal authoritative state plus exact same-snapshot five-persona coverage, valid manifests/hashes, and consolidated findings proving clean; Claude prose or one final findings file is insufficient. See the adapter document for usage, exit codes, subscription telemetry wording, architecture, and staging instructions. The in-repo Hermes skill is staged at [`skills/project-plan-review/`](skills/project-plan-review/SKILL.md); it is not installed automatically. -## Cost expectation +## Subscription usage expectation -Each plan-mode round is one full Codex review of `PLAN.md`. In practice that's ~25–30k Codex tokens per round. With the default 3 rounds you should expect **~75–90k tokens per `/claudex:plan`**. Codex authenticates against your ChatGPT account, so the bill goes to your ChatGPT Plus / Pro / Team / Enterprise plan, not to claudex. If you're on a tight rate limit, run `--rounds 2` for fast topics and reserve `--rounds 5+` for high-stakes designs. +Legacy plan mode invokes one Codex review per round. Sweep-v2 can invoke all five required Codex personas in each generation, up to its hard cap of five generations. Claude Code and Codex authenticate through subscription-backed CLIs in this workflow; any dollar-valued fields are usage-equivalent telemetry/bounded-run controls, not direct API billing or an invoice. Both services may still enforce subscription rate limits. ## Safety diff --git a/bin/claudex-plan-review b/bin/claudex-plan-review index 6e6ceed..4061a7b 100755 --- a/bin/claudex-plan-review +++ b/bin/claudex-plan-review @@ -10,6 +10,7 @@ from __future__ import annotations import argparse import datetime as dt +import hashlib import json import os import re @@ -26,6 +27,14 @@ from typing import Any TERMINAL_PHASES = {"done", "cancelled", "errored"} ACTIVE_PHASES = {"drafting", "reviewing", "revising", "summarizing"} EXIT_CODES = {"converged": 0, "max_reached": 10, "degraded": 11, "failed": 12, "timed_out": 124} +SWEEP_PERSONAS = [ + "architecture-scope", + "security-data", + "product-domain", + "quality-accessibility-performance", + "operations-deployment", +] +SHA256_RE = re.compile(r"[0-9a-f]{64}") def emit(payload: dict[str, Any], code: int) -> int: @@ -127,6 +136,246 @@ def final_findings_status(path: Path | None) -> tuple[str, dict[str, int]]: return "unparseable", counts +def sha256_file(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def strict_state(path: Path) -> tuple[dict[str, str], list[str]]: + state: dict[str, str] = {} + errors: list[str] = [] + try: + lines = path.read_text(encoding="utf-8").splitlines() + except (OSError, UnicodeError) as exc: + return {}, [f"state is unreadable: {exc}"] + for number, line in enumerate(lines, 1): + if ":" not in line: + errors.append(f"state line {number} is malformed") + continue + key, value = line.split(":", 1) + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", key): + errors.append(f"state line {number} has an invalid key") + elif key in state: + errors.append(f"state contains duplicate field {key}") + else: + state[key] = value.strip() + return state, errors + + +def sweep_findings(path: Path) -> tuple[str, dict[str, int]]: + counts = {"high": 0, "medium": 0, "low": 0} + try: + text = path.read_text(encoding="utf-8") + except (OSError, UnicodeError): + return "malformed", counts + if text.strip() == "No substantive findings.": + return "clean", counts + lines = [line.rstrip() for line in text.splitlines() if line.strip()] + headers = ["## High", "## Medium", "## Low"] + if any(lines.count(header) != 1 for header in headers): + return "malformed", counts + positions = [lines.index(header) for header in headers] + if positions != sorted(positions) or lines[: positions[0]]: + return "malformed", counts + for index, start in enumerate(positions): + end = positions[index + 1] if index + 1 < len(positions) else len(lines) + severity = headers[index][3:].lower() + for line in lines[start + 1 : end]: + if not line.startswith("- ") or len(line) <= 2: + return "malformed", counts + counts[severity] += 1 + return ("material" if any(counts.values()) else "malformed"), counts + + +def render_sweep_findings(path: Path, persona: str) -> str: + severity = "" + counts = {"high": 0, "medium": 0, "low": 0} + rendered: list[str] = [] + for line in path.read_text(encoding="utf-8").splitlines(): + if line in {"## High", "## Medium", "## Low"}: + severity = line[3:].lower() + rendered.append(line) + elif line.startswith("- ") and severity: + counts[severity] += 1 + rendered.append(f"- [{persona}-{severity}-{counts[severity]}] {line[2:]}") + else: + rendered.append(line) + return "\n".join(rendered) + + +def validate_sweep(state_file: Path, review_dir: Path, runtime_plan: Path, reviewed_plan: Path, requested: int) -> dict[str, Any]: + """Recompute a sweep verdict from runner-authored state and artifacts.""" + state, errors = strict_state(state_file) + coverage: list[dict[str, Any]] = [] + severity = {"high": 0, "medium": 0, "low": 0} + + def integer(name: str) -> int | None: + value = state.get(name, "") + return int(value) if value.isdigit() else None + + generation = integer("generation") + maximum = integer("max_generations") + if state.get("engine") != "sweep-v2": + errors.append("state engine is not sweep-v2") + if generation is None or not 1 <= generation <= 5: + errors.append("state generation is not in 1..5") + if maximum is None or not 1 <= maximum <= 5: + errors.append("state max_generations is not in 1..5") + elif maximum != requested: + errors.append("state max_generations does not match the requested cap") + if generation and maximum and generation > maximum: + errors.append("state generation exceeds max_generations") + + generation_dir = review_dir / "generations" / str(generation) if generation else None + snapshot = generation_dir / "PLAN.md" if generation_dir else None + manifest_path = generation_dir / "manifest.json" if generation_dir else None + consolidated = generation_dir / "consolidated-findings.md" if generation_dir else None + snapshot_hash = state.get("snapshot_sha256", "") + if not SHA256_RE.fullmatch(snapshot_hash): + errors.append("state snapshot_sha256 is malformed") + if state.get("reviewed_live_sha256") != snapshot_hash: + errors.append("state reviewed_live_sha256 does not match the current snapshot") + + manifest: dict[str, Any] = {} + if manifest_path: + try: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + if not isinstance(manifest, dict): + raise ValueError("manifest is not an object") + except (OSError, UnicodeError, json.JSONDecodeError, ValueError) as exc: + errors.append(f"generation manifest is unreadable or malformed: {exc}") + if manifest: + required = {"generation", "snapshot_sha256", "required_persona_ids", "topic", "source_plan_path", "previous_generation_sha256"} + if set(manifest) != required: + errors.append("generation manifest fields do not match the sweep-v2 schema") + if manifest.get("generation") != generation or manifest.get("snapshot_sha256") != snapshot_hash: + errors.append("generation manifest does not match state generation/hash") + if manifest.get("required_persona_ids") != SWEEP_PERSONAS: + errors.append("generation manifest does not require the exact five personas") + if not isinstance(manifest.get("topic"), str) or not manifest.get("topic"): + errors.append("generation manifest topic is empty") + if manifest.get("source_plan_path") != str(runtime_plan): + errors.append("generation manifest source plan path is mismatched") + if generation == 1 and manifest.get("previous_generation_sha256") is not None: + errors.append("generation-one manifest has an unexpected previous hash") + elif generation and generation > 1: + try: + previous = json.loads((review_dir / "generations" / str(generation - 1) / "manifest.json").read_text(encoding="utf-8")) + previous_hash = previous["snapshot_sha256"] + if not isinstance(previous_hash, str) or not SHA256_RE.fullmatch(previous_hash) or manifest.get("previous_generation_sha256") != previous_hash: + errors.append("generation manifest previous hash linkage is invalid") + except (OSError, UnicodeError, json.JSONDecodeError, KeyError, TypeError): + errors.append("previous generation manifest is unreadable or malformed") + + try: + if not snapshot or sha256_file(snapshot) != snapshot_hash: + errors.append("frozen snapshot hash does not match state") + except OSError as exc: + errors.append(f"frozen snapshot is unreadable: {exc}") + try: + if sha256_file(reviewed_plan) != snapshot_hash: + errors.append("reviewed live PLAN.md is not the frozen snapshot") + except OSError as exc: + errors.append(f"reviewed live PLAN.md is unreadable: {exc}") + + evidence_digest = hashlib.sha256() + chunks = [f"# Consolidated findings — generation {generation}\n\n", f"Snapshot SHA-256: `{snapshot_hash}`\n\n"] + if generation_dir: + for persona in SWEEP_PERSONAS: + findings = generation_dir / f"{persona}.findings.md" + sidecar = generation_dir / f"{persona}.result.json" + classification, counts = sweep_findings(findings) + entry: dict[str, Any] = { + "persona_id": persona, + "classification": classification, + "snapshot_sha256": None, + "valid": False, + } + for key in severity: + severity[key] += counts[key] + sidecar_data: dict[str, Any] = {} + try: + sidecar_data = json.loads(sidecar.read_text(encoding="utf-8")) + if not isinstance(sidecar_data, dict): + raise ValueError("sidecar is not an object") + except (OSError, UnicodeError, json.JSONDecodeError, ValueError) as exc: + errors.append(f"{persona} sidecar is unreadable or malformed: {exc}") + if sidecar_data: + required = {"persona_id", "expected_snapshot_sha256", "actual_snapshot_sha256_before", "actual_snapshot_sha256_after", "codex_exit_code", "findings_path", "findings_sha256", "findings_classification", "completed_at"} + expected_path = str(review_dir / "generations" / str(generation) / findings.name) + valid = set(sidecar_data) == required + valid &= sidecar_data.get("persona_id") == persona + valid &= sidecar_data.get("expected_snapshot_sha256") == snapshot_hash + valid &= sidecar_data.get("actual_snapshot_sha256_before") == snapshot_hash + valid &= sidecar_data.get("actual_snapshot_sha256_after") == snapshot_hash + valid &= sidecar_data.get("codex_exit_code") == 0 + valid &= sidecar_data.get("findings_path") == expected_path + valid &= sidecar_data.get("findings_classification") == classification + try: + valid &= sidecar_data.get("findings_sha256") == sha256_file(findings) + except OSError: + valid = False + try: + dt.datetime.strptime(str(sidecar_data.get("completed_at")), "%Y-%m-%dT%H:%M:%SZ") + except (TypeError, ValueError): + valid = False + if not valid or classification not in {"clean", "material"}: + errors.append(f"{persona} evidence is incomplete, hash-mismatched, or malformed") + else: + entry.update(snapshot_sha256=snapshot_hash, valid=True) + coverage.append(entry) + chunks.append(f"## {persona}\n\n") + if classification == "clean": + chunks.append(findings.read_text(encoding="utf-8") + "\n\n") + elif classification == "material": + chunks.append(render_sweep_findings(findings, persona) + "\n\n\n") + else: + chunks.append("DEGRADED: invalid evidence.\n\n") + for artifact in (findings, sidecar): + try: + evidence_digest.update(artifact.name.encode("utf-8") + b"\0") + evidence_digest.update(artifact.read_bytes()) + evidence_digest.update(b"\0") + except OSError: + pass + + consolidated_hash = "" + try: + consolidated_text = consolidated.read_text(encoding="utf-8") if consolidated else "" + consolidated_hash = sha256_file(consolidated) if consolidated else "" + if consolidated_text != "".join(chunks): + errors.append("consolidated findings do not exactly match persona evidence") + except (OSError, UnicodeError) as exc: + errors.append(f"consolidated findings are unreadable: {exc}") + if state.get("evidence_sha256") != evidence_digest.hexdigest(): + errors.append("state evidence_sha256 does not match generation evidence") + if state.get("consolidated_sha256") != consolidated_hash: + errors.append("state consolidated_sha256 does not match consolidated findings") + + all_clean = len(coverage) == 5 and all(item["valid"] and item["classification"] == "clean" for item in coverage) + any_material = any(item["valid"] and item["classification"] == "material" for item in coverage) + phase, signal = state.get("phase"), state.get("decision_signal") + if not errors and phase == "done" and signal == "converged" and state.get("clean") == "true" and state.get("coverage_complete") == "true" and all_clean and state.get("converged_snapshot_sha256") == snapshot_hash: + outcome = "converged" + reason = "terminal sweep-v2 state and complete same-snapshot five-persona evidence prove convergence" + elif not errors and phase == "done" and signal == "max-reached" and state.get("clean") == "false" and state.get("coverage_complete") == "true" and generation == maximum and any_material and not all_clean and not state.get("converged_snapshot_sha256"): + outcome = "max_reached" + reason = "sweep-v2 reached its generation cap with authoritative material findings" + else: + outcome = "degraded" + mismatch = errors or [f"terminal/state verdict mismatch (phase={phase}, signal={signal}, clean={state.get('clean')}, coverage={state.get('coverage_complete')})"] + reason = "sweep-v2 evidence is incomplete or inconsistent: " + "; ".join(mismatch[:4]) + return { + "outcome": outcome, "reason": reason, "state": state, + "generation": generation, "max_generations": maximum, + "snapshot_sha256": snapshot_hash or None, + "converged_snapshot_sha256": state.get("converged_snapshot_sha256") or None, + "persona_coverage": coverage, "severity_counts": severity, + "findings_status": "none" if all_clean else ("material" if any_material else "malformed"), + "generation_dir": generation_dir, "manifest": manifest_path, + "consolidated": consolidated, "errors": errors, + } + + def parse_claude_stream(path: Path) -> dict[str, Any]: summary: dict[str, Any] = {"result_records": 0, "reported_cost_usd": None, "session_id": None} try: @@ -172,6 +421,7 @@ def main() -> int: parser.add_argument("--plan", required=True, help="absolute path to an existing non-empty PLAN.md") parser.add_argument("--topic", required=True, help="self-contained review topic and constraints") parser.add_argument("--rounds", required=True, type=int, help="positive Claudex round limit") + parser.add_argument("--engine", choices=("sweep-v2", "legacy"), default="sweep-v2", help="plan-review engine (default: sweep-v2)") parser.add_argument("--timeout", required=True, type=float, help="wall-clock timeout in seconds") parser.add_argument("--budget-usd", required=True, help="positive Claude Code budget cap") default_plugin = Path(__file__).resolve().parents[1] / "plugins" / "claudex" @@ -192,6 +442,8 @@ def main() -> int: codex = executable(args.codex, "--codex") if args.rounds < 1: raise ValueError("--rounds must be a positive integer") + if args.engine == "sweep-v2" and args.rounds > 5: + raise ValueError("sweep-v2 --rounds must be in 1..5 generations") if args.timeout <= 0: raise ValueError("--timeout must be positive") if not args.topic.strip(): @@ -203,6 +455,8 @@ def main() -> int: if not budget.is_finite() or budget <= 0: raise ValueError("--budget-usd must be a positive decimal") required = [plugin / ".claude-plugin" / "plugin.json", plugin / "commands" / "plan.md", plugin / "hooks" / "stop-hook.sh", plugin / "scripts" / "state-helpers.sh"] + if args.engine == "sweep-v2": + required.extend([plugin / "scripts" / "sweep-helpers.sh", plugin / "scripts" / "personas.sh"]) missing = [str(path) for path in required if not path.is_file()] if missing: raise ValueError("invalid Claudex plugin root; missing: " + ", ".join(missing)) @@ -272,7 +526,8 @@ def main() -> int: if external_plan: shutil.copy2(plan, runtime_plan) - prompt = f"/claudex:plan --from-draft --skip-interview --rounds {args.rounds} {args.topic.strip()}" + engine_flag = "--engine sweep-v2 " if args.engine == "sweep-v2" else "" + prompt = f"/claudex:plan {engine_flag}--from-draft --skip-interview --rounds {args.rounds} {args.topic.strip()}" command = [ str(claude), "--print", "--verbose", "--output-format", "stream-json", "--include-hook-events", "--dangerously-skip-permissions", "--setting-sources", "project", "--plugin-dir", str(plugin), @@ -280,7 +535,7 @@ def main() -> int: ] metadata = { "argv": command, "cwd": str(repo), "timeout_seconds": args.timeout, - "budget_usd": str(budget), "rounds": args.rounds, "topic": args.topic, + "budget_usd": str(budget), "engine": args.engine, "rounds": args.rounds, "topic": args.topic, "started_at": dt.datetime.now(dt.timezone.utc).isoformat(), } (evidence / "run-metadata.json").write_text(json.dumps(metadata, indent=2, sort_keys=True) + "\n") @@ -339,21 +594,41 @@ def main() -> int: state = parse_state(selected_state) if selected_state else {} review_id = state.get("review_id") + valid_review_id = bool(review_id and re.fullmatch(r"[0-9]{8}-[0-9]{6}-[0-9a-f]{6}", review_id)) findings_files: list[Path] = [] - if review_id: - findings_dir = state_dir / review_id + findings_dir: Path | None = None + evidence_state: Path | None = None + evidence_review_dir: Path | None = None + copy_errors: list[str] = [] + if valid_review_id: + findings_dir = state_dir / str(review_id) + def finding_round(path: Path) -> int: match = re.search(r"findings-round-(\d+)\.md$", path.name) return int(match.group(1)) if match else -1 + findings_files = sorted(findings_dir.glob("findings-round-*.md"), key=finding_round) if findings_dir.is_dir() else [] - copy_if_exists(selected_state, evidence / "artifacts" / selected_state.name) - copy_if_exists(findings_dir, evidence / "artifacts" / review_id) - copy_if_exists(state_dir / "log", evidence / "artifacts" / "claudex.log") + if selected_state: + evidence_state = evidence / "artifacts" / selected_state.name + try: + copy_if_exists(selected_state, evidence_state) + except OSError as exc: + copy_errors.append(f"state evidence copy failed: {exc}") + evidence_review_dir = evidence / "artifacts" / str(review_id) + try: + copy_if_exists(findings_dir, evidence_review_dir) + except OSError as exc: + copy_errors.append(f"generation evidence copy failed: {exc}") + try: + copy_if_exists(state_dir / "log", evidence / "artifacts" / "claudex.log") + except OSError as exc: + copy_errors.append(f"log evidence copy failed: {exc}") final_findings = findings_files[-1] if findings_files else None findings_status, severity = final_findings_status(final_findings) phase = state.get("phase") decision = state.get("decision_signal") + sweep: dict[str, Any] | None = None if timed_out: outcome = "timed_out" reason = "wall-clock timeout exceeded; process group terminated and reaped" @@ -363,15 +638,41 @@ def main() -> int: elif not selected_state or not state: outcome = "degraded" reason = "Claude exited zero but no new Claudex state artifact was found" + elif not valid_review_id or findings_dir is None: + outcome = "degraded" + reason = "Claudex state contains a missing or malformed review_id" + elif copy_errors: + outcome = "degraded" + reason = "required adapter evidence could not be copied: " + "; ".join(copy_errors) + elif args.engine == "sweep-v2": + sweep = validate_sweep(selected_state, findings_dir, runtime_plan, evidence / "PLAN.after.md", args.rounds) + state = sweep["state"] + phase = state.get("phase") + decision = state.get("decision_signal") + outcome = sweep["outcome"] + reason = sweep["reason"] + findings_status = sweep["findings_status"] + severity = sweep["severity_counts"] elif decision == "no-material-findings" and phase == "done" and findings_status == "none": outcome = "converged" - reason = "terminal state and final findings agree that no substantive findings remain" + reason = "terminal legacy state and final findings agree that no substantive findings remain" elif decision == "max-reached" and phase == "done" and findings_status in {"material", "unparseable"}: outcome = "max_reached" - reason = "round budget exhausted without an authoritative clean findings artifact" + reason = "legacy round budget exhausted without an authoritative clean findings artifact" else: outcome = "degraded" - reason = f"state/artifact mismatch or incomplete lifecycle (phase={phase}, signal={decision}, findings={findings_status})" + reason = f"legacy state/artifact mismatch or incomplete lifecycle (phase={phase}, signal={decision}, findings={findings_status})" + + copied_final = None + if final_findings and evidence_review_dir: + copied_final = evidence_review_dir / final_findings.name + copied_consolidated = None + copied_manifest = None + copied_generation_dir = None + if sweep and evidence_review_dir and sweep.get("generation"): + copied_generation_dir = evidence_review_dir / "generations" / str(sweep["generation"]) + copied_consolidated = copied_generation_dir / "consolidated-findings.md" + copied_manifest = copied_generation_dir / "manifest.json" stream_summary = parse_claude_stream(stdout_path) result = { @@ -379,20 +680,32 @@ def main() -> int: "outcome": outcome, "clean": outcome == "converged", "reason": reason, + "engine": args.engine, "repo": str(repo), "plan": str(plan), "review_id": review_id, "phase": phase, "decision_signal": decision, - "round": int(state["round"]) if state.get("round", "").isdigit() else None, - "max_rounds": int(state["max_rounds"]) if state.get("max_rounds", "").isdigit() else args.rounds, + "generation": sweep.get("generation") if sweep else None, + "max_generations": sweep.get("max_generations") if sweep else None, + "snapshot_sha256": sweep.get("snapshot_sha256") if sweep else None, + "converged_snapshot_sha256": sweep.get("converged_snapshot_sha256") if sweep else None, + "persona_coverage": sweep.get("persona_coverage", []) if sweep else [], + "generation_manifest": str(copied_manifest) if copied_manifest else None, + "generation_evidence_dir": str(copied_generation_dir) if copied_generation_dir else None, + "consolidated_findings": str(copied_consolidated) if copied_consolidated else None, + "round": int(state["round"]) if args.engine == "legacy" and state.get("round", "").isdigit() else None, + "max_rounds": int(state["max_rounds"]) if args.engine == "legacy" and state.get("max_rounds", "").isdigit() else (args.rounds if args.engine == "legacy" else None), "findings_status": findings_status, "severity_counts": severity, "final_findings": str(final_findings) if final_findings else None, + "evidence_final_findings": str(copied_final) if copied_final else None, "evidence_dir": str(evidence), "stdout_log": str(stdout_path), "stderr_log": str(stderr_path), "state_file": str(selected_state) if selected_state else None, + "evidence_state_file": str(evidence_state) if evidence_state else None, + "source_state_file": str(selected_state) if selected_state else None, "elapsed_seconds": elapsed, "process_exit_code": returncode, "budget_usd": str(budget), diff --git a/docs/HEADLESS_ADAPTER.md b/docs/HEADLESS_ADAPTER.md index 26d7f85..a3e0a6b 100644 --- a/docs/HEADLESS_ADAPTER.md +++ b/docs/HEADLESS_ADAPTER.md @@ -1,18 +1,18 @@ # Headless Claudex planning bridge -`bin/claudex-plan-review` is a bounded adapter for a Hermes leaf subagent (or any automation runner) to drive the existing Claude Code → Claudex Stop-hook → Codex plan-review lifecycle. +`bin/claudex-plan-review` is a bounded adapter for a Hermes leaf subagent (or any automation runner) to drive Claude Code → Claudex Stop-hook → Codex plan review. Its default engine is the feature-flagged Phase 1 `sweep-v2` lifecycle; `--engine legacy` preserves the original round-based adapter behavior. ## Architecture 1. The caller supplies absolute paths for a trusted git repository, an existing non-empty plan, the Claudex plugin, Claude Code, and Codex. -2. The adapter validates paths, plugin files, versions, and both CLI authentication states. It creates no global installation and uses `--plugin-dir` for this Claude session only. +2. The adapter validates paths, plugin files, engine bounds, versions, and both CLI authentication states. It creates no global installation and uses `--plugin-dir` for this Claude session only. 3. If the supplied plan is outside the repository, it is staged as `/PLAN.md`, reviewed, copied back, and any pre-existing repository plan is restored. -4. Claude Code runs headlessly with `/claudex:plan --from-draft --skip-interview`, explicit rounds and Claude budget, stream-JSON output, hook events, and a pinned child `PATH` whose first entries are the supplied executable directories. -5. The supervisor polls newly created `.claude/claudex/*.state` files, records state transitions, and preserves state, findings, logs, plans, and raw Claude streams. +4. Claude Code runs headlessly with `/claudex:plan --engine sweep-v2 --from-draft --skip-interview --rounds N`, stream-JSON output, hook events, an explicit telemetry budget, and a pinned child `PATH`. +5. The supervisor polls newly created `.claude/claudex/*.state` files, records transitions, and preserves the complete sweep generation directory and state under adapter evidence. 6. A wall-clock timeout terminates the entire process group, waits five seconds, escalates to `SIGKILL`, and reaps it. -7. Classification uses state plus the final findings artifact. Claude's prose summary is never authoritative. +7. Classification independently revalidates authoritative state, manifest, frozen/live snapshot hashes, all five persona sidecars/findings, aggregate evidence hash, and consolidated findings. Claude prose is never authoritative. -The existing interactive review mode is unchanged. +Interactive review mode and the plugin's legacy plan mode are unchanged. ## Exact usage @@ -21,7 +21,8 @@ The existing interactive review mode is unchanged. --repo /absolute/path/to/project \ --plan /absolute/path/to/project/PLAN.md \ --topic "grounded feature scope, constraints, and explicit non-goals" \ - --rounds 3 \ + --engine sweep-v2 \ + --rounds 5 \ --timeout 900 \ --budget-usd 5.00 \ --plugin-root /absolute/path/to/claudex/plugins/claudex \ @@ -30,32 +31,38 @@ The existing interactive review mode is unchanged. --output-dir /absolute/new/evidence-directory ``` -`--output-dir` is optional; the default is `.claude/claudex/adapter-runs/-`. `--model` defaults to `sonnet`. The repository must be trusted because unattended Claude runs with permission prompts bypassed so the plugin hook can execute. +`--engine` defaults to `sweep-v2`; its `--rounds` value is the maximum generation count and must be `1..5`. The Hermes project-plan-review workflow uses five. Pass `--engine legacy` for backward-compatible round behavior. `--output-dir` is optional; the default is `.claude/claudex/adapter-runs/-`. `--model` defaults to `sonnet`. -Stdout contains exactly one compact JSON object. Diagnostics go to evidence files. Exit codes and semantics: +Stdout contains exactly one compact JSON object. Diagnostics go to evidence files. Exit codes: | Outcome | Exit | Clean | Meaning | |---|---:|---:|---| -| `converged` | 0 | yes | State is terminal with `no-material-findings`, and the final findings artifact independently contains no substantive findings or severity bullets. | -| `max_reached` | 10 | no | The round budget ended with unresolved/non-clean findings evidence. Mechanics worked; approval did not occur. | -| `degraded` | 11 | no | Claude exited zero, but state/artifacts are incomplete or contradictory. | +| `converged` | 0 | yes | State is done/converged/clean/coverage-complete and exactly five required personas have valid clean evidence against the same snapshot; manifest, hashes, and consolidated findings all agree. | +| `max_reached` | 10 | no | The generation cap ended with complete authoritative same-snapshot evidence containing material findings. | +| `degraded` | 11 | no | State/evidence is incomplete, contradictory, missing, malformed, mutated, hash-mismatched, cancelled, or otherwise non-authoritative. | | `failed` | 12 | no | Validation, prerequisite/auth, launch, or Claude execution failed. | -| `timed_out` | 124 | no | Wall-clock deadline expired; the process group was killed and reaped. | +| `timed_out` | 124 | no | The deadline expired; the complete process group was killed and reaped. | -Only `outcome=converged` with `clean=true` is a success gate. +Only `outcome=converged` with `clean=true` is a success gate. Generation five material findings are `max_reached`, never clean. One final findings file is not sufficient evidence for sweep-v2 convergence. -## Cost and budgets +## Machine-readable evidence -Each round invokes one full Codex review, historically about 25–30k Codex tokens per plan round. `--max-budget-usd` bounds the headless Claude Code side and the adapter reports Claude's emitted dollar cost when available. Codex uses the configured ChatGPT subscription; its usage is separate, may be rate-limited, and cannot be dollar-capped or measured by this adapter. The timeout is a wall-clock safety boundary, not a billing guarantee. +Sweep-v2 results expose: -## Evidence and failure handling +- `engine`, `generation`, and `max_generations` (legacy-only `round`/`max_rounds` remain for compatibility); +- `snapshot_sha256` and `converged_snapshot_sha256`; +- ordered `persona_coverage` for the exact five personas; +- copied `evidence_state_file`, `generation_manifest`, `generation_evidence_dir`, and `consolidated_findings` paths (`state_file` and `final_findings` retain legacy source-path semantics); +- honest `outcome`, `clean`, `reason`, findings classification/severity, process exit, elapsed time, and telemetry. -Evidence includes `run-metadata.json`, `preflight.json`, `claude-stream.jsonl`, `claude-stderr.log`, `state-events.jsonl`, before/after plan copies, Claudex state/findings/log copies, and `result.json`. On any non-clean outcome, inspect `reason`, state, final findings, and stderr; never infer convergence from Claude's final prose. +The state and full review directory—including every generation—are copied into `/artifacts/` before result emission, so read-back does not depend on later `.claude/claudex` cleanup. `source_state_file` is informational; gates should use the copied paths. -The adapter refuses an existing active Claudex loop. Terminal prior states are baselined so consecutive runs select only the new run's state. +## Subscription usage and telemetry -## Installation and Hermes skill staging +Claude Code and Codex are subscription-backed in this workflow. `--budget-usd`, Claude's `--max-budget-usd`, and `reported_claude_cost_usd` are CLI usage-equivalent telemetry and a bounded-run control; they do not prove direct API billing or represent an invoice. Codex subscription usage is separate, may be rate-limited, and cannot be dollar-enforced or measured by this adapter. A sweep generation can invoke all five Codex personas. The timeout is a wall-clock safety boundary, not a billing guarantee. + +For Hermes delegation, keep the outer child deadline at least 180 seconds beyond the adapter timeout so the leaf can read and report evidence. -No installation is required to run from a checkout; keep executable paths explicit. To put the command on a controlled local `PATH`, symlink or copy `bin/claudex-plan-review` yourself and continue to pass `--plugin-root`. +## Installation and Hermes skill staging -The staged Hermes skill lives at `skills/project-plan-review/`. Review it in-repo first. To install later, outside this change and only with operator approval, copy that whole directory into the active Hermes profile's skills directory, then start a fresh Hermes session. This repository does **not** install the skill, alter Hermes configuration, or change the active Claude plugin installation. +No installation is required to run from a checkout; keep executable paths explicit. The staged Hermes skill lives at `skills/project-plan-review/`. Review it in-repo first. Installing it later requires separate operator approval. This repository does **not** install the skill, alter Hermes configuration, or change the active Claude plugin installation. diff --git a/skills/project-plan-review/SKILL.md b/skills/project-plan-review/SKILL.md index 14d72f8..5367c08 100644 --- a/skills/project-plan-review/SKILL.md +++ b/skills/project-plan-review/SKILL.md @@ -1,37 +1,51 @@ --- name: project-plan-review description: Use for substantial implementation plans that benefit from independent Claude/Claudex/Codex adversarial review. Skip tiny, obvious fixes. -version: 1.0.0 +version: 2.0.0 license: MIT metadata: hermes: - tags: [planning, claudex, codex, delegation, verification] + tags: [planning, claudex, codex, delegation, verification, sweep-v2] --- # Project Plan Review -Use this workflow for substantial features, migrations, or risky cross-cutting work. **Do not use it for tiny fixes** where the review overhead exceeds the implementation risk. +Use this workflow for substantial features, migrations, or risky cross-cutting work. **Do not use it for tiny fixes** where review overhead exceeds implementation risk. ## Main Drake workflow 1. Ground the project first: inspect the repository, current behavior, constraints, tests, and user request. Never ask a leaf reviewer to invent this context. -2. Draft a concrete `PLAN.md` in the project root. Include scope/non-scope, exact files, contracts that already exist, steps, rollback, and verification. +2. Draft a concrete `PLAN.md` in the project root. Include scope/non-scope, exact files, existing contracts, steps, rollback, and verification. 3. Read [the delegation runbook](references/runbook.md). -4. Call `delegate_task` with a self-contained goal/context that includes absolute paths for the repository, `PLAN.md`, adapter, Claude, Codex, and desired evidence directory; include rounds, timeout, and budget. The child is a leaf: it cannot ask Rob questions or delegate further. -5. Keep the main Drake session responsive while the child performs the bounded adapter run. Do not transfer user-facing ownership to the child. -6. On return, verify every returned file exists and read back `result.json`, final `PLAN.md`, state, and final findings. A claimed path is not evidence until read. -7. Independently reject scope creep, invented contracts, and recommendations unsupported by the grounded repository. Claudex is a critic, not the product owner. -8. Normalize the plan so only the active implementation phase remains; remove completed prerequisites, historical phases, and stale branching instructions. -9. Attach or return the final reviewed `PLAN.md`, outcome, unresolved findings, and evidence paths. Only `converged` is clean. +4. Use the adapter's default `sweep-v2` engine with `--rounds 5`. The command must invoke `/claudex:plan --engine sweep-v2 --from-draft --skip-interview --rounds 5 ...` through the adapter. +5. Call `delegate_task` with self-contained context and absolute paths for the repository, `PLAN.md`, adapter, plugin, Claude, Codex, and evidence directory. The child is a leaf: it cannot ask Rob questions or delegate further. +6. Keep the main Drake session responsive while the child performs the bounded run. The outer child timeout must reserve at least 180 seconds beyond the adapter timeout for artifact read-back and reporting. +7. On return, read `result.json`, the copied `evidence_state_file`, generation manifest, consolidated findings, and all five persona sidecars/findings from adapter evidence. A claimed path is not evidence until read. +8. Independently reject scope creep, invented contracts, and recommendations unsupported by the grounded repository. Claudex is a critic, not the product owner. +9. Normalize the plan so only the active implementation phase remains; remove completed prerequisites, historical phases, and stale branching instructions. +10. **If Drake's normalization materially changes requirements, sequencing, contracts, safety controls, or verification, run sweep-v2 again.** A prior convergence hash does not cover a materially changed plan. +11. Attach or return the final reviewed `PLAN.md`, outcome, snapshot/converged hashes, persona coverage, unresolved findings, and copied evidence paths. Only `converged` is clean. + +## Sweep-v2 convergence contract + +A clean result requires all of the following from authoritative state and artifacts, never Claude prose: + +- terminal `phase=done`, `decision_signal=converged`, `clean=true`, and `coverage_complete=true`; +- exactly the five required personas—architecture/scope, security/data, product/domain, quality/accessibility/performance, and operations/deployment; +- every persona result tied to the same immutable generation snapshot SHA-256, with readable, schema-valid sidecars and findings; +- a readable generation manifest and consolidated findings whose hashes match state; and +- consolidated findings proving that all five personas returned exactly no substantive findings. + +Generation five with material findings is `max_reached` and non-clean. Missing, malformed, mutated, hash-mismatched, nonzero, cancelled, degraded, or incomplete evidence is never clean. ## Boundaries and outcome rules - The adapter may write only `PLAN.md` plus `.claude/claudex/` evidence/state in the target repository. It must not implement the plan, commit, push, merge, or change global Hermes/Claude configuration. - Use a disposable worktree/repository when the plan or project cannot safely be modified in place. - `max_reached` proves mechanics, not plan approval. `degraded`, `failed`, and `timed_out` are also non-clean. -- Findings artifacts and Claudex state override Claude prose. Main Drake still independently validates all accepted recommendations. +- Claude and Codex are subscription-backed in this workflow. Dollar-valued budget/cost fields are CLI usage telemetry and a bounded-run control, not evidence of direct API billing or an invoice. Codex subscription usage is not dollar-enforced by the adapter. - Never expose secrets in the topic, plan, delegated prompt, or evidence. ## Verification gate -Before attaching the final plan, require: readable `result.json`; matching absolute repo/plan paths; a terminal state; readable final findings; `clean=true` only with `outcome=converged`; and a final plan whose proposed contracts map to repository evidence or explicit user requirements. +Before attaching the final plan, require readable copied evidence paths; matching absolute repo/plan paths; accurate `engine`, `generation`, and `max_generations`; terminal state; exact five-persona same-hash coverage; readable manifest and consolidated findings; `clean=true` only with `outcome=converged`; and a final plan whose contracts map to repository evidence or explicit user requirements. Re-review any later material plan change. diff --git a/skills/project-plan-review/references/runbook.md b/skills/project-plan-review/references/runbook.md index a1a069f..48c1446 100644 --- a/skills/project-plan-review/references/runbook.md +++ b/skills/project-plan-review/references/runbook.md @@ -11,7 +11,8 @@ Resolve all paths before delegation: - `CLAUDE`: vetted absolute Claude Code executable - `CODEX`: vetted absolute Codex executable - `EVIDENCE`: new absolute output directory -- positive `ROUNDS`, `TIMEOUT_SECONDS`, and `BUDGET_USD` +- `ROUNDS=5` for the standard project-plan-review sweep +- positive `TIMEOUT_SECONDS` and `BUDGET_USD` - the Hermes delegation wall-clock limit from `delegation.child_timeout_seconds` ## Timeout budget invariant @@ -21,13 +22,13 @@ The outer Hermes leaf must outlive the adapter plus artifact read-back and summa - `child_timeout_seconds: 0` (no delegation wall-clock cap), or - `child_timeout_seconds >= TIMEOUT_SECONDS + 180`. -Never give the adapter a timeout equal to or greater than the child timeout. If this invariant is violated, the adapter can finish successfully while Hermes reports the leaf as timed out before it returns its summary. For a normal three-round review, use an adapter timeout of 900 seconds and a child timeout of at least 1080 seconds. A timed-out outer leaf is not proof that the adapter failed: inspect `result.json`, process state, and evidence directly before classifying the run or retrying. +The 180-second reserve is mandatory. Never give the adapter a timeout equal to or greater than the child timeout. If this invariant is violated, the adapter can finish while Hermes reports the leaf as timed out before artifact verification. A timed-out outer leaf is not proof that the adapter failed: inspect `result.json`, process state, and copied evidence before classifying or retrying. ## `delegate_task` goal/context template Use the available `delegate_task` tool with a prompt equivalent to this, filling every placeholder: -> You are a leaf execution subagent. Do not ask Rob questions and do not delegate. Run exactly one bounded plan-review adapter operation, then return the exact JSON result and absolute artifact paths. Repository: ``. Existing plan: ``. Grounded topic/constraints: ``. Adapter: ``. Claudex plugin root: ``. Claude executable: ``. Codex executable: ``. Evidence directory: ``. Run: ` --repo --plan --topic --rounds --timeout --budget-usd --plugin-root --claude --codex --output-dir `. Preserve stdout exactly. A nonzero exit is an outcome to report, not a reason to improvise. Do not implement, commit, push, install skills/plugins, edit global configuration, or touch files outside the adapter's documented scope. Before returning, verify `result.json`, the final plan, state file, and final findings paths exist. Return outcome, exit code, and paths; never call a non-converged run clean. +> You are a leaf execution subagent. Do not ask Rob questions and do not delegate. Run exactly one bounded sweep-v2 plan-review adapter operation, then return the exact JSON result and absolute artifact paths. Repository: ``. Existing plan: ``. Grounded topic/constraints: ``. Adapter: ``. Claudex plugin root: ``. Claude executable: ``. Codex executable: ``. Evidence directory: ``. Run: ` --repo --plan --topic --engine sweep-v2 --rounds 5 --timeout --budget-usd --plugin-root --claude --codex --output-dir `. Preserve stdout exactly. A nonzero exit is an outcome to report, not a reason to improvise. Do not implement, commit, push, install skills/plugins, edit global configuration, or touch files outside the adapter's documented scope. Before returning, verify `result.json`, final plan, copied state, generation manifest, generation evidence directory, and consolidated findings exist. Return outcome, exit code, generation/max-generations, snapshot hashes, persona coverage, and paths; never call a non-converged run clean. Do not omit context on the assumption the child can read the parent conversation. It cannot ask the user to fill gaps. @@ -36,21 +37,24 @@ Do not omit context on the assumption the child can read the parent conversation After the child returns: 1. Read `/result.json`; reject malformed or mismatched results. -2. Read the returned `state_file` and `final_findings` when present. -3. Read `` again from disk and compare it with the grounded scope. -4. Check outcome invariants: - - `converged`: exit 0, `clean=true`, state `phase=done`, signal `no-material-findings`, final findings says exactly no substantive findings and has no severity bullets. - - `max_reached`: exit 10, `clean=false`; unresolved findings remain or final artifact is not authoritative-clean. - - `degraded`: exit 11, `clean=false`; lifecycle/artifact mismatch or incomplete evidence. - - `failed`: exit 12, `clean=false`. - - `timed_out`: exit 124, `clean=false`; process group was terminated/reaped. -5. Reject invented APIs, data models, deployment guarantees, and scope additions unless they map to repository facts or explicit requirements. -6. Rewrite/normalize the active implementation phase if review churn left historical or completed phases in the plan. -7. Attach the final plan and disclose unresolved concerns and cost reporting limitations. - -## Safety notes - -- Claude's budget cap covers Claude Code API spend reported by Claude. Codex subscription usage is separate and is not dollar-enforced by the adapter. +2. Require `engine=sweep-v2`, `generation` and `max_generations` in `1..5`, with the normal workflow reporting `max_generations=5`. +3. Read copied `evidence_state_file`, `generation_manifest`, `consolidated_findings`, and every persona finding/sidecar under `generation_evidence_dir`. `state_file` and `final_findings` retain legacy source-path semantics; do not rely on them after cleanup. +4. Read `` again and compare it with the grounded scope and reported snapshot hash. +5. Check outcome invariants: + - `converged`: exit 0 and `clean=true`; state is done/converged/clean/coverage-complete; exactly the five required personas have readable clean evidence tied to one hash; snapshot and converged hashes match; manifest and consolidated hashes validate; consolidated findings prove all five are clean. + - `max_reached`: exit 10 and `clean=false`; the capped generation has complete same-snapshot coverage and material findings. At the standard cap this is generation five. + - `degraded`: exit 11 and `clean=false`; missing, malformed, mutated, hash-mismatched, cancelled, or incomplete state/evidence. + - `failed`: exit 12 and `clean=false`. + - `timed_out`: exit 124 and `clean=false`; process group was terminated and reaped. +6. Reject invented APIs, data models, deployment guarantees, and scope additions unless they map to repository facts or explicit requirements. +7. Normalize the active implementation phase if review churn left historical/completed phases in the plan. +8. If normalization is material—not merely formatting or wording—run a fresh five-generation-cap sweep-v2 review. Convergence is bound to the reported snapshot hash. +9. Attach the final plan and disclose unresolved concerns and telemetry limitations. + +## Safety and telemetry notes + +- Claude Code and Codex are subscription-backed here. `budget_usd` and `reported_claude_cost_usd` are CLI usage-equivalent telemetry/bounded-run controls, not proof of direct API billing or a charged invoice. +- Codex subscription usage is separate, may be rate-limited, and is not dollar-capped by the adapter. Each sweep generation can run all five persona reviews. - The adapter pins child `PATH` from explicit executable directories and system paths. - Never point the adapter at an untrusted repository: it runs Claude with bypassed permission prompts so the Stop hook can operate unattended. -- Prefer two to three rounds for normal work. Increase only when the risk justifies additional Codex reviews. +- `--engine legacy` exists only for backward compatibility. Drake's project-plan-review workflow uses sweep-v2 with five generations. diff --git a/tests/test_adapter.py b/tests/test_adapter.py index d679e02..ab6b285 100644 --- a/tests/test_adapter.py +++ b/tests/test_adapter.py @@ -10,6 +10,13 @@ ROOT = Path(__file__).resolve().parents[1] ADAPTER = ROOT / "bin" / "claudex-plan-review" PLUGIN = ROOT / "plugins" / "claudex" +PERSONAS = [ + "architecture-scope", + "security-data", + "product-domain", + "quality-accessibility-performance", + "operations-deployment", +] class AdapterTests(unittest.TestCase): @@ -31,13 +38,16 @@ def setUp(self): esac echo 'codex-cli 0.test' """) - self._write_executable(self.claude, """#!/usr/bin/env python3 -import json, os, pathlib, subprocess, sys, time, uuid + self._write_executable(self.claude, r"""#!/usr/bin/env python3 +import datetime, hashlib, json, os, pathlib, re, subprocess, sys, time, uuid if sys.argv[1:] == ['--version']: print('2.test'); raise SystemExit(0) if sys.argv[1:] == ['auth', 'status']: print(json.dumps({'loggedIn': True, 'authMethod': 'test'})); raise SystemExit(0) -outcome = os.environ.get('FAKE_CLAUDE_OUTCOME', 'converged') +outcome = os.environ.get('FAKE_CLAUDE_OUTCOME', 'sweep_clean') +prompt = sys.argv[-1] +marker = os.environ.get('FAKE_PROMPT_FILE') +if marker: pathlib.Path(marker).write_text(prompt) if outcome == 'failed': print('forced failure', file=sys.stderr); raise SystemExit(7) if outcome == 'timeout': @@ -50,15 +60,85 @@ def setUp(self): rid = '20990101-000000-' + uuid.uuid4().hex[:6] review_dir = state_dir / rid review_dir.mkdir() -if outcome == 'max_reached': - signal, findings, round_value = 'max-reached', '# Round 1 findings\\n\\n## High\\n- unsafe gap (fix it)\\n', '2' -elif outcome == 'degraded': - signal, findings, round_value = 'no-material-findings', '# Round 1 findings\\n\\n## High\\n- contradiction (fix it)\\n', '1' +state_path = state_dir / (rid + '.state') +if outcome.startswith('legacy_'): + if outcome == 'legacy_max': + signal, findings, round_value = 'max-reached', '# Round 1 findings\n\n## High\n- unsafe gap (fix it)\n', '1' + elif outcome == 'legacy_degraded': + signal, findings, round_value = 'no-material-findings', '# Round 1 findings\n\n## High\n- contradiction (fix it)\n', '1' + else: + signal, findings, round_value = 'no-material-findings', '# Round 1 findings\n\nNo substantive findings.\n', '1' + (review_dir / 'findings-round-1.md').write_text(findings) + state_path.write_text(f'''mode: plan\nphase: done\ntopic: test\nround: {round_value}\nmax_rounds: 1\nreview_id: {rid}\nrepo_root: {pathlib.Path.cwd().resolve()}\ndecision_signal: {signal}\n''') else: - signal, findings, round_value = 'no-material-findings', '# Round 1 findings\\n\\nNo substantive findings.\\n', '1' -(review_dir / 'findings-round-1.md').write_text(findings) -state = state_dir / (rid + '.state') -state.write_text(f'''mode: plan\nphase: done\ntopic: test\nround: {round_value}\nmax_rounds: 1\nreview_id: {rid}\nrepo_root: {pathlib.Path.cwd().resolve()}\ndecision_signal: {signal}\n''') + personas = ['architecture-scope','security-data','product-domain','quality-accessibility-performance','operations-deployment'] + match = re.search(r'--rounds (\d+)', prompt) + maximum = int(match.group(1)) if match else 5 + generation = 5 if outcome == 'sweep_max' else 1 + generation_dir = review_dir / 'generations' / str(generation) + generation_dir.mkdir(parents=True) + snapshot = generation_dir / 'PLAN.md' + snapshot.write_bytes((pathlib.Path.cwd() / 'PLAN.md').read_bytes()) + snapshot_hash = hashlib.sha256(snapshot.read_bytes()).hexdigest() + previous_hash = None + if generation > 1: + previous_dir = review_dir / 'generations' / str(generation - 1) + previous_dir.mkdir(parents=True) + (previous_dir / 'manifest.json').write_text(json.dumps({'snapshot_sha256': snapshot_hash})) + previous_hash = snapshot_hash + manifest = { + 'generation': generation, 'snapshot_sha256': snapshot_hash, + 'required_persona_ids': personas, 'topic': 'grounded test topic', + 'source_plan_path': str(pathlib.Path.cwd().resolve() / 'PLAN.md'), + 'previous_generation_sha256': previous_hash, + } + (generation_dir / 'manifest.json').write_text(json.dumps(manifest, indent=2, sort_keys=True) + '\n') + chunks = [f'# Consolidated findings — generation {generation}\n\n', f'Snapshot SHA-256: `{snapshot_hash}`\n\n'] + evidence = hashlib.sha256() + for persona in personas: + findings = generation_dir / f'{persona}.findings.md' + sidecar = generation_dir / f'{persona}.result.json' + material = outcome == 'sweep_max' and persona == 'security-data' + if material: + text = '## High\n- Scope: material generation-five gap (fix it).\n## Medium\n## Low\n' + classification = 'material' + else: + text = 'No substantive findings.\n' + classification = 'clean' + if outcome == 'sweep_malformed' and persona == 'product-domain': + text = 'not valid findings\n' + findings.write_text(text) + findings_hash = hashlib.sha256(findings.read_bytes()).hexdigest() + expected = '0' * 64 if outcome == 'sweep_hash_mismatch' and persona == 'security-data' else snapshot_hash + data = { + 'persona_id': persona, 'expected_snapshot_sha256': expected, + 'actual_snapshot_sha256_before': snapshot_hash, + 'actual_snapshot_sha256_after': snapshot_hash, 'codex_exit_code': 0, + 'findings_path': str(findings), 'findings_sha256': findings_hash, + 'findings_classification': classification, + 'completed_at': '2099-01-01T00:00:00Z', + } + sidecar.write_text(json.dumps(data, indent=2, sort_keys=True) + '\n') + chunks.append(f'## {persona}\n\n') + if material: + chunks.append(f'## High\n- [{persona}-high-1] Scope: material generation-five gap (fix it).\n## Medium\n## Low\n\n\n') + else: + chunks.append('No substantive findings.\n\n\n') + for artifact in (findings, sidecar): + evidence.update(artifact.name.encode() + b'\0') + evidence.update(artifact.read_bytes()) + evidence.update(b'\0') + if outcome == 'sweep_missing': + (generation_dir / 'operations-deployment.result.json').unlink() + consolidated = generation_dir / 'consolidated-findings.md' + consolidated.write_text(''.join(chunks)) + if outcome == 'sweep_mutated_consolidated': + consolidated.write_text(consolidated.read_text() + 'Claude says this is still clean.\n') + signal = 'max-reached' if outcome == 'sweep_max' else ('cancelled' if outcome == 'sweep_cancelled' else ('degraded' if outcome == 'sweep_degraded' else 'converged')) + phase = 'cancelled' if outcome == 'sweep_cancelled' else 'done' + clean = 'false' if outcome in {'sweep_max', 'sweep_cancelled', 'sweep_degraded'} else 'true' + converged_hash = '' if outcome in {'sweep_max', 'sweep_cancelled', 'sweep_degraded'} else snapshot_hash + state_path.write_text(f'''mode: plan\nphase: {phase}\ntopic: test\nround: {generation}\nmax_rounds: {maximum}\nreview_id: {rid}\nrepo_root: {pathlib.Path.cwd().resolve()}\ndecision_signal: {signal}\nengine: sweep-v2\ngeneration: {generation}\nmax_generations: {maximum}\nsnapshot_sha256: {snapshot_hash}\ncoverage_complete: true\nclean: {clean}\nrevision_required: false\nreviewed_live_sha256: {snapshot_hash}\nevidence_sha256: {evidence.hexdigest()}\nconsolidated_sha256: {hashlib.sha256(consolidated.read_bytes()).hexdigest()}\nconverged_snapshot_sha256: {converged_hash}\n''') print(json.dumps({'type': 'result', 'subtype': 'success', 'total_cost_usd': 0.01, 'session_id': 'fake-session'})) """) @@ -69,42 +149,92 @@ def _write_executable(self, path, content): path.write_text(textwrap.dedent(content)) path.chmod(0o755) - def run_adapter(self, outcome="converged", timeout="5", extra_env=None): + def run_adapter(self, outcome="sweep_clean", timeout="5", extra_env=None, engine="sweep-v2", rounds="1"): env = os.environ.copy() env["FAKE_CLAUDE_OUTCOME"] = outcome if extra_env: env.update(extra_env) command = [ str(ADAPTER), "--repo", str(self.repo.resolve()), "--plan", str(self.plan.resolve()), - "--topic", "review the grounded plan", "--rounds", "1", "--timeout", timeout, + "--topic", "review the grounded plan", "--rounds", rounds, "--timeout", timeout, "--budget-usd", "1.25", "--plugin-root", str(PLUGIN.resolve()), "--claude", str(self.claude.resolve()), "--codex", str(self.codex.resolve()), + "--engine", engine, ] completed = subprocess.run(command, text=True, capture_output=True, env=env, timeout=10) lines = completed.stdout.splitlines() self.assertEqual(len(lines), 1, completed.stdout) return completed, json.loads(lines[0]) - def test_converged_is_only_clean_success(self): - completed, result = self.run_adapter("converged") + def test_sweep_clean_requires_exact_same_snapshot_five_persona_coverage(self): + marker = self.base / "prompt.txt" + completed, result = self.run_adapter(extra_env={"FAKE_PROMPT_FILE": str(marker)}) self.assertEqual(completed.returncode, 0) self.assertEqual(result["outcome"], "converged") self.assertTrue(result["clean"]) - self.assertEqual(result["findings_status"], "none") - self.assertTrue(Path(result["evidence_dir"], "result.json").is_file()) + self.assertEqual(result["engine"], "sweep-v2") + self.assertEqual((result["generation"], result["max_generations"]), (1, 1)) + self.assertEqual(len(result["persona_coverage"]), 5) + self.assertEqual([item["persona_id"] for item in result["persona_coverage"]], PERSONAS) + self.assertEqual({item["snapshot_sha256"] for item in result["persona_coverage"]}, {result["snapshot_sha256"]}) + self.assertIn("/claudex:plan --engine sweep-v2 --from-draft --skip-interview --rounds 1", marker.read_text()) + for key in ("evidence_state_file", "generation_manifest", "generation_evidence_dir", "consolidated_findings"): + self.assertTrue(Path(result[key]).exists(), key) + self.assertTrue(str(Path(result[key])).startswith(result["evidence_dir"])) - def test_max_reached_is_honest_and_nonzero(self): - completed, result = self.run_adapter("max_reached") + def test_generation_five_material_findings_are_max_reached(self): + completed, result = self.run_adapter("sweep_max", rounds="5") self.assertEqual(completed.returncode, 10) self.assertEqual(result["outcome"], "max_reached") self.assertFalse(result["clean"]) + self.assertEqual((result["generation"], result["max_generations"]), (5, 5)) + self.assertEqual(result["findings_status"], "material") - def test_prose_or_signal_cannot_override_material_findings(self): - completed, result = self.run_adapter("degraded") + def test_missing_persona_evidence_cannot_be_clean(self): + completed, result = self.run_adapter("sweep_missing") self.assertEqual(completed.returncode, 11) self.assertEqual(result["outcome"], "degraded") self.assertFalse(result["clean"]) - self.assertEqual(result["findings_status"], "material") + + def test_malformed_findings_cannot_be_clean(self): + completed, result = self.run_adapter("sweep_malformed") + self.assertEqual(completed.returncode, 11) + self.assertFalse(result["clean"]) + self.assertEqual(result["findings_status"], "malformed") + + def test_hash_mismatch_cannot_be_clean(self): + completed, result = self.run_adapter("sweep_hash_mismatch") + self.assertEqual(completed.returncode, 11) + self.assertFalse(result["clean"]) + self.assertIn("hash-mismatched", result["reason"]) + + def test_mutated_consolidated_findings_cannot_be_clean(self): + completed, result = self.run_adapter("sweep_mutated_consolidated") + self.assertEqual(completed.returncode, 11) + self.assertFalse(result["clean"]) + self.assertIn("consolidated findings", result["reason"]) + + def test_degraded_and_cancelled_state_are_never_clean(self): + for fixture in ("sweep_degraded", "sweep_cancelled"): + with self.subTest(fixture=fixture): + completed, result = self.run_adapter(fixture) + self.assertEqual(completed.returncode, 11) + self.assertEqual(result["outcome"], "degraded") + self.assertFalse(result["clean"]) + + def test_legacy_engine_remains_compatible(self): + completed, result = self.run_adapter("legacy_clean", engine="legacy") + self.assertEqual(completed.returncode, 0) + self.assertTrue(result["clean"]) + self.assertEqual(result["engine"], "legacy") + self.assertEqual((result["round"], result["max_rounds"]), (1, 1)) + self.assertIsNone(result["generation"]) + self.assertTrue(Path(result["final_findings"]).is_file()) + + def test_legacy_signal_cannot_override_material_findings(self): + completed, result = self.run_adapter("legacy_degraded", engine="legacy") + self.assertEqual(completed.returncode, 11) + self.assertFalse(result["clean"]) def test_nonzero_claude_is_failed(self): completed, result = self.run_adapter("failed") @@ -117,18 +247,23 @@ def test_timeout_kills_and_reaps_process_group(self): completed, result = self.run_adapter("timeout", timeout="0.5", extra_env={"FAKE_CHILD_PID_FILE": str(marker)}) self.assertEqual(completed.returncode, 124) self.assertEqual(result["outcome"], "timed_out") - self.assertFalse(result["clean"]) pid = int(marker.read_text()) probe = subprocess.run(["kill", "-0", str(pid)], capture_output=True) self.assertNotEqual(probe.returncode, 0, f"child process {pid} survived timeout") def test_second_consecutive_run_uses_new_state(self): - first, result1 = self.run_adapter("converged") - second, result2 = self.run_adapter("converged") + first, result1 = self.run_adapter() + second, result2 = self.run_adapter() self.assertEqual((first.returncode, second.returncode), (0, 0)) self.assertNotEqual(result1["review_id"], result2["review_id"]) self.assertNotEqual(result1["state_file"], result2["state_file"]) + def test_sweep_generation_cap_is_rejected_before_launch(self): + completed, result = self.run_adapter(rounds="6") + self.assertEqual(completed.returncode, 12) + self.assertEqual(result["error"]["kind"], "validation") + self.assertIn("1..5", result["error"]["message"]) + def test_relative_repo_is_rejected_as_json(self): completed = subprocess.run([ str(ADAPTER), "--repo", "relative", "--plan", str(self.plan.resolve()), "--topic", "x",