diff --git a/README.md b/README.md index 1f396c9..4d73d91 100644 --- a/README.md +++ b/README.md @@ -139,6 +139,20 @@ flightrecorder validate \ --strict ``` +Audit evidence before handoff: + +```bash +flightrecorder audit \ + --runs runs \ + --fail-on-privacy \ + --fail-on-failed +``` + +`audit --fail-on-privacy` blocks generated artifacts that contain public-repo +privacy findings such as local workspace paths or personal contact strings. Add +`--forbid-text` with `--fail-on-leak` for project-specific canaries or secrets +that must never appear in evidence. + Run a deterministic offline harness packet without launching Hermes or a model provider: diff --git a/flightrecorder/cli.py b/flightrecorder/cli.py index 2c4df8d..f5379bb 100644 --- a/flightrecorder/cli.py +++ b/flightrecorder/cli.py @@ -722,6 +722,8 @@ def cmd_audit(args: argparse.Namespace) -> int: print(rendered, end="") if args.fail_on_leak and summary["leaks"]: return 1 + if args.fail_on_privacy and summary["privacy_findings"]: + return 1 if args.fail_on_failed and summary["failed"] > 0: return 1 return 0 @@ -2231,6 +2233,7 @@ def _parser() -> argparse.ArgumentParser: audit.add_argument("--out") audit.add_argument("--forbid-text", action="append", default=[], help="Literal text that must not appear in generated artifacts") audit.add_argument("--fail-on-leak", action="store_true", help="Exit nonzero if forbidden text is found") + audit.add_argument("--fail-on-privacy", action="store_true", help="Exit nonzero if generated artifacts contain public-repo privacy findings") audit.add_argument("--fail-on-failed", action="store_true", help="Exit nonzero if any scorecard failed") audit.set_defaults(func=cmd_audit) @@ -4990,6 +4993,43 @@ def _read_scorecard_ref(path: Path) -> tuple[dict[str, Any], str]: return _read_json(score_path), _display_path(score_path) +PUBLIC_REPO_PRIVACY_PATTERNS: tuple[tuple[str, re.Pattern[str]], ...] = ( + ("posix_home_path", re.compile("/" + "Users/|" + "/" + r"home/[^\s\"'<>]+")), + ("windows_home_path", re.compile(r"[A-Za-z]:[\\/]" + "Users" + r"[\\/]")), + ("local_workspace_path", re.compile("Documents/" + "GitHub")), + ("private_codex_worktree", re.compile(r"\." + "codex-" + "goal-" + "worktrees")), + ("private_codex_automation", re.compile(r"\." + "codex/" + "automations")), + ("email_address", re.compile(r"\b[A-Za-z0-9._%+-]+" + "@" + r"[A-Za-z0-9.-]+[.][A-Za-z]{2,}\b")), +) + + +def _audit_privacy_findings(path: Path, text: str) -> list[dict[str, Any]]: + findings: list[dict[str, Any]] = [] + seen: set[tuple[str, int]] = set() + for name, pattern in PUBLIC_REPO_PRIVACY_PATTERNS: + for match in pattern.finditer(text): + if name == "email_address" and _is_example_email_address(match.group(0)): + continue + line = text.count("\n", 0, match.start()) + 1 + key = (name, line) + if key in seen: + continue + seen.add(key) + findings.append( + { + "path": _display_path(path), + "line": line, + "pattern": name, + } + ) + return findings + + +def _is_example_email_address(value: str) -> bool: + domain = value.partition("@")[2].lower() + return domain.startswith("example.") or domain.endswith(".example") + + def _audit_runs(runs_dir: Path, forbidden_text: list[str]) -> dict[str, Any]: if not runs_dir.exists(): raise FileNotFoundError(f"Runs directory not found: {runs_dir}") @@ -4998,6 +5038,7 @@ def _audit_runs(runs_dir: Path, forbidden_text: list[str]) -> dict[str, Any]: scorecards: list[dict[str, Any]] = [] leaks: list[dict[str, str]] = [] + privacy_findings: list[dict[str, Any]] = [] for score_path in sorted(runs_dir.glob("*/scorecard.json")): scorecard = _read_json(score_path) scorecards.append( @@ -5011,23 +5052,25 @@ def _audit_runs(runs_dir: Path, forbidden_text: list[str]) -> dict[str, Any]: ) needles = [needle for needle in forbidden_text if needle] - if needles and runs_dir.exists(): - for path in sorted(runs_dir.rglob("*")): - if not path.is_file(): - continue - text = path.read_text(encoding="utf-8", errors="ignore") + for path in sorted(runs_dir.rglob("*")): + if not path.is_file(): + continue + text = path.read_text(encoding="utf-8", errors="ignore") + privacy_findings.extend(_audit_privacy_findings(path, text)) + if needles: for needle in needles: if needle in text: - leaks.append({"path": str(path), "text": needle}) + leaks.append({"path": _display_path(path), "text": needle}) passed = sum(1 for item in scorecards if item["passed"]) failed = len(scorecards) - passed return { - "runs_dir": str(runs_dir), + "runs_dir": _display_path(runs_dir), "total": len(scorecards), "passed": passed, "failed": failed, "leaks": leaks, + "privacy_findings": privacy_findings, "scorecards": scorecards, } diff --git a/release_check.sh b/release_check.sh index 4df0faa..0d24097 100755 --- a/release_check.sh +++ b/release_check.sh @@ -1045,7 +1045,7 @@ summary = { "hermes_root": "/tmp/hermes-agent", "hermes_git_commit": "abcdef123456", "hermes_git_dirty": False, - "flight_recorder_root": str(Path.cwd()), + "flight_recorder_root": "", "flight_recorder_git_commit": "123456abcdef", "flight_recorder_git_dirty": False, }, @@ -1392,7 +1392,7 @@ assert bundle["metrics"]["compare_export"]["regressed_rule_counts"] == {} assert bundle["metrics"]["compare_export"]["new_critical_failure_counts"] == {} assert bundle["metrics"]["training_export"]["trainer_view_source_fingerprint_coverage"]["unverified"] == 0 assert bundle["metrics"]["live_smoke_summary"]["chat_completion_request_count"] == 1 -assert bundle["metrics"]["live_smoke_summary"]["flight_recorder_root"] == str(Path.cwd()) +assert bundle["metrics"]["live_smoke_summary"]["flight_recorder_root"] == "" assert bundle["metrics"]["trace_observability"]["final_answer_rate"] == 1.0 assert bundle["metrics"]["review_export"]["item_count"] >= 6 assert bundle["metrics"]["reviewed_export"]["reviewed_label_count"] == bundle["metrics"]["review_export"]["item_count"] @@ -1461,7 +1461,8 @@ PY --runs runs \ --forbid-text hfr_fixture_secret_value_123 \ --forbid-text DEMO_API_KEY=hfr_fixture \ - --fail-on-leak >/dev/null + --fail-on-leak \ + --fail-on-privacy >/dev/null INSTALL_DIR="$(mktemp -d)" VENV_DIR="$(mktemp -d)" diff --git a/tests/test_cli_report.py b/tests/test_cli_report.py index eb7efa4..0ec1455 100644 --- a/tests/test_cli_report.py +++ b/tests/test_cli_report.py @@ -1327,11 +1327,35 @@ def test_audit_command_summarizes_runs_and_can_fail_on_leak(self): self.assertEqual(audit["passed"], 1) self.assertEqual(audit["failed"], 1) self.assertEqual(audit["leaks"], []) + self.assertEqual(audit["privacy_findings"], []) (bad / "leak.txt").write_text("do-not-ship", encoding="utf-8") leak_code = run_cli(["audit", "--runs", str(runs), "--forbid-text", "do-not-ship", "--fail-on-leak"]) self.assertEqual(leak_code, 1) + example_email = "customer" + "@" + "example.com" + (bad / "example_email.txt").write_text(example_email, encoding="utf-8") + example_out = Path(tmp) / "example_audit.json" + example_code = run_cli(["audit", "--runs", str(runs), "--out", str(example_out), "--fail-on-privacy"]) + self.assertEqual(example_code, 0) + example_audit = json.loads(example_out.read_text(encoding="utf-8")) + self.assertEqual(example_audit["privacy_findings"], []) + + private_path = "/" + "Users/alice/" + "Documents/" + "GitHub/private" + (bad / "local_path.txt").write_text(f"workspace={private_path}", encoding="utf-8") + privacy_out = Path(tmp) / "privacy_audit.json" + + privacy_code = run_cli( + ["audit", "--runs", str(runs), "--out", str(privacy_out), "--fail-on-privacy"] + ) + + self.assertEqual(privacy_code, 1) + privacy_audit = json.loads(privacy_out.read_text(encoding="utf-8")) + privacy_patterns = {finding["pattern"] for finding in privacy_audit["privacy_findings"]} + self.assertIn("posix_home_path", privacy_patterns) + self.assertIn("local_workspace_path", privacy_patterns) + self.assertNotIn("alice", json.dumps(privacy_audit)) + def test_audit_command_can_fail_when_any_scorecard_failed(self): with tempfile.TemporaryDirectory() as tmp: runs = Path(tmp) / "runs"