diff --git a/bin/fm-crosscheck-slack-service.sh b/bin/fm-crosscheck-slack-service.sh new file mode 100755 index 00000000000..ce9ddaf6d44 --- /dev/null +++ b/bin/fm-crosscheck-slack-service.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash +# Install and operate the central macOS Crosscheck Slack listener. +# +# Usage: fm-crosscheck-slack-service.sh install|start|stop|restart|status|uninstall +# +set -eu + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +FM_ROOT="${FM_ROOT_OVERRIDE:-$(cd "$SCRIPT_DIR/.." && pwd)}" +FM_HOME="${FM_HOME:-$FM_ROOT}" +CONFIG="${FM_CROSSCHECK_SLACK_CONFIG:-$FM_HOME/config/crosscheck-slack.json}" +LABEL=com.firstmate.crosscheck-slack +DOMAIN="gui/$(id -u)" +AGENT_DIR="$HOME/Library/LaunchAgents" +PLIST="$AGENT_DIR/$LABEL.plist" +LOG_DIR="$FM_HOME/logs" +STDOUT_LOG="$LOG_DIR/crosscheck-slack.log" +STDERR_LOG="$LOG_DIR/crosscheck-slack.error.log" +WRAPPER="$FM_ROOT/bin/fm-crosscheck-slack.sh" + +usage() { + echo "usage: $0 install|start|stop|restart|status|uninstall" >&2 + exit 2 +} + +write_plist() { + # shellcheck source=bin/fm-crosscheck-python-lib.sh + . "$SCRIPT_DIR/fm-crosscheck-python-lib.sh" + interpreter=$(fm_crosscheck_resolve_python) + interpreter=$("$interpreter" -c 'import os, sys; print(os.path.abspath(sys.executable))') + mkdir -p "$AGENT_DIR" "$LOG_DIR" + temporary=$(mktemp "$AGENT_DIR/.$LABEL.XXXXXX") + trap 'rm -f "$temporary"' EXIT + plutil -create xml1 "$temporary" + plutil -insert Label -string "$LABEL" "$temporary" + plutil -insert ProgramArguments -array "$temporary" + plutil -insert ProgramArguments.0 -string "$WRAPPER" "$temporary" + plutil -insert ProgramArguments.1 -string run "$temporary" + plutil -insert ProgramArguments.2 -string --config "$temporary" + plutil -insert ProgramArguments.3 -string "$CONFIG" "$temporary" + plutil -insert ProgramArguments.4 -string --keychain-only "$temporary" + plutil -insert EnvironmentVariables -dictionary "$temporary" + plutil -insert EnvironmentVariables.HOME -string "$HOME" "$temporary" + plutil -insert EnvironmentVariables.PATH -string "$PATH" "$temporary" + plutil -insert EnvironmentVariables.FM_CROSSCHECK_PYTHON -string "$interpreter" "$temporary" + plutil -insert EnvironmentVariables.FM_HOME -string "$FM_HOME" "$temporary" + plutil -insert EnvironmentVariables.FM_CROSSCHECK_SLACK_CONFIG \ + -string "$CONFIG" "$temporary" + plutil -insert RunAtLoad -bool true "$temporary" + plutil -insert KeepAlive -bool true "$temporary" + plutil -insert ThrottleInterval -integer 10 "$temporary" + plutil -insert StandardOutPath -string "$STDOUT_LOG" "$temporary" + plutil -insert StandardErrorPath -string "$STDERR_LOG" "$temporary" + chmod 600 "$temporary" + validate_plist "$temporary" + mv "$temporary" "$PLIST" + trap - EXIT +} + +validate_plist() { + # shellcheck source=bin/fm-crosscheck-python-lib.sh + . "$SCRIPT_DIR/fm-crosscheck-python-lib.sh" + validator_python=$(fm_crosscheck_resolve_python) + "$validator_python" - "$1" <<'PY' +import plistlib +import subprocess +import sys + +with open(sys.argv[1], "rb") as handle: + agent = plistlib.load(handle) +command = agent["ProgramArguments"] +if "--keychain-only" not in command: + raise SystemExit("error: reinstall the service to require Keychain credentials") +environment = agent["EnvironmentVariables"] +for arguments in ([command[0], "--selftest", command[3]], [command[0], "preflight", *command[2:]]): + result = subprocess.run(arguments, env=environment, timeout=60, check=False) + if result.returncode: + raise SystemExit(result.returncode) +PY +} + +loaded() { + launchctl print "$DOMAIN/$LABEL" >/dev/null 2>&1 +} + +stop_service() { + if loaded; then + launchctl bootout "$DOMAIN/$LABEL" + fi +} + +case "${1:-}" in + install) + [ -x "$WRAPPER" ] || { + echo "error: Crosscheck Slack wrapper is not executable at $WRAPPER" >&2 + exit 1 + } + write_plist + echo "installed: $PLIST" + echo "listener remains stopped until '$0 start' passes credential preflight" + ;; + start) + [ -f "$PLIST" ] || { + echo "error: service is not installed at $PLIST" >&2 + exit 1 + } + validate_plist "$PLIST" + stop_service + launchctl bootstrap "$DOMAIN" "$PLIST" + launchctl enable "$DOMAIN/$LABEL" + launchctl kickstart -k "$DOMAIN/$LABEL" + echo "started: $LABEL" + ;; + stop) + stop_service + echo "stopped: $LABEL" + ;; + restart) + "$0" stop + "$0" start + ;; + status) + if loaded; then + launchctl print "$DOMAIN/$LABEL" + else + echo "stopped: $LABEL" + exit 3 + fi + ;; + uninstall) + stop_service + if [ -f "$PLIST" ]; then + rm "$PLIST" + fi + echo "uninstalled: $LABEL (logs and durable review state retained)" + ;; + *) usage ;; +esac diff --git a/bin/fm-crosscheck-slack.py b/bin/fm-crosscheck-slack.py index add0a2b3e47..3593d370817 100755 --- a/bin/fm-crosscheck-slack.py +++ b/bin/fm-crosscheck-slack.py @@ -13,24 +13,17 @@ this process ever takes from a mention is one pull-request URL, which is validated against the repository allowlist before any credentialed tool sees it. Nothing from Slack or from the PR is ever executed. -- Tokens come only from environment variables named by the config file. - They are never stored in the config, never written to state, never placed +- Credential sources and service startup requirements are owned by + docs/crosscheck-slack.md. + Tokens are never stored in the config, never written to state, never placed in a child process environment (except the GitHub read credential, whose entire job is to be the crosscheck subprocess's read credential), and every log line passes through a redactor that knows every secret value. -- A missing token environment variable is a startup refusal that names the - exact variable, so the ready-to-flip posture is explicit: the owner - supplies tokens later and nothing else changes. -- AUTHORSHIP ASSERTION, stated loudly: this lane stages task metadata as - model=human-authored, which satisfies the crosscheck gate's - model-separation screen for EVERY reviewer. That is only true because - the lane asserts human authorship: submissions come from engineers in - Slack, and any PR whose head branch matches an `agent_branch_prefixes` - entry (default `fm/`) is refused in thread and redirected to the - ordinary crosscheck lane, which carries true author metadata. This lane - must never be pointed at agent-authored pull requests; the - model-separation guarantee for Slack reviews rests on that assertion - and on the branch screen, not on the gate's own screen. +- Authorship comes only from an HMAC-signed Firstmate/no-mistakes + attestation bound to the exact repository, pull request, and head SHA. + Slack text, the submitter, branch names, and caller-supplied model text + carry no authorship authority. Missing, conflicting, or unverifiable + provenance is a visible fail-closed refusal. - Every Slack event id is claimed durably before any work starts, so a retried delivery of the same event never starts a second review. - Team usage is metered per submitter per UTC day in a durable JSON ledger @@ -55,7 +48,10 @@ import dataclasses import datetime as dt import fcntl +import functools import hashlib +import hmac +import importlib.util import json import os from pathlib import Path @@ -64,7 +60,9 @@ import secrets import socket import ssl +import stat import struct +import subprocess import sys import tempfile import threading @@ -92,13 +90,11 @@ "github_token_env", "daily_budget_usd", "daily_request_cap", + "provenance_key_file", "state_dir", } -# Optional keys carry their own defaults; agent_branch_prefixes defaults to -# the fleet's own branch prefix so the authorship screen is on out of the box. -OPTIONAL_CONFIG_KEYS = {"agent_branch_prefixes"} +OPTIONAL_CONFIG_KEYS = {"keychain_services"} CONFIG_KEYS = REQUIRED_CONFIG_KEYS | OPTIONAL_CONFIG_KEYS -DEFAULT_AGENT_BRANCH_PREFIXES = ("fm/",) ENV_NAME_RE = re.compile(r"^[A-Z][A-Z0-9_]*$") REPO_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$") CHANNEL_RE = re.compile(r"^[A-Z0-9]{1,32}$") @@ -126,7 +122,15 @@ WS_GUID = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11" WEB_API_TIMEOUT_SECONDS = 30 REVIEW_QUEUE_LIMIT = 8 +REVIEW_WORKERS = 4 RECONNECT_BACKOFF_CAP_SECONDS = 60.0 +LAUNCH_PROVENANCE_SCHEMA = "firstmate.crosscheck-author-launch.v1" +PROVENANCE_SCHEMA = "firstmate.crosscheck-authorship.v2" +PROVENANCE_SIGNATURE_ALGORITHM = "hmac-sha256" +MAX_PROVENANCE_BYTES = 64 * 1024 +MAX_TASK_META_BYTES = 64 * 1024 +TASK_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$") +SHA_RE = re.compile(r"^[0-9a-f]{40}$") class SlackExposureError(RuntimeError): @@ -173,11 +177,12 @@ class Config: app_token_env: str bot_token_env: str github_token_env: str + keychain_services: dict[str, str] channel_allowlist: tuple[str, ...] repo_allowlist: tuple[str, ...] daily_budget_usd: float | None daily_request_cap: int | None - agent_branch_prefixes: tuple[str, ...] + provenance_key_file: Path state_dir: Path @@ -204,21 +209,21 @@ def default_config_path() -> Path: return Path(home) / "config" / "crosscheck-slack.json" -def expand_state_dir(raw: str) -> Path: +def expand_host_path(raw: str, label: str) -> Path: if raw == "$FM_HOME" or raw.startswith("$FM_HOME/"): home = os.environ.get("FM_HOME", "") require( home != "", - "state_dir references $FM_HOME but FM_HOME is not set in the environment", + f"{label} references $FM_HOME but FM_HOME is not set in the environment", ) raw = home + raw[len("$FM_HOME"):] require( "$" not in raw, - "state_dir supports only a literal leading $FM_HOME reference; " + f"{label} supports only a literal leading $FM_HOME reference; " f"other substitutions are refused: {raw!r}", ) path = Path(raw) - require(path.is_absolute(), f"state_dir must resolve to an absolute path, got {raw!r}") + require(path.is_absolute(), f"{label} must resolve to an absolute path, got {raw!r}") return path @@ -247,6 +252,27 @@ def load_config(path: Path) -> Config: "configuration app_token_env, bot_token_env, and github_token_env " "must name three distinct environment variables", ) + keychain_raw = value.get("keychain_services", {}) + require( + isinstance(keychain_raw, dict), + "configuration keychain_services must be an object when present", + ) + require( + set(keychain_raw) <= {"app_token", "bot_token", "github_token"}, + "configuration keychain_services has an unknown credential role", + ) + keychain_services: dict[str, str] = {} + for role, service in keychain_raw.items(): + service = require_string(service, f"configuration keychain_services.{role}") + require( + re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]{0,127}", service) is not None, + f"configuration keychain_services.{role} is not a safe service name", + ) + keychain_services[role] = service + require( + len(set(keychain_services.values())) == len(keychain_services), + "configuration keychain_services must name distinct services", + ) channels_raw = value.get("channel_allowlist") require( @@ -300,49 +326,133 @@ def load_config(path: Path) -> Config: ) cap = cap_raw - if "agent_branch_prefixes" in value: - prefixes_raw = value.get("agent_branch_prefixes") - require( - isinstance(prefixes_raw, list), - "configuration agent_branch_prefixes must be an array of branch prefixes " - "(an empty array deliberately disables the agent-branch screen)", - ) - prefixes: list[str] = [] - for index, prefix in enumerate(prefixes_raw): - prefix = require_string(prefix, f"configuration agent_branch_prefixes[{index}]") - prefixes.append(prefix) - agent_branch_prefixes = tuple(prefixes) - else: - agent_branch_prefixes = DEFAULT_AGENT_BRANCH_PREFIXES - - state_dir = expand_state_dir(require_string(value.get("state_dir"), "configuration state_dir")) + state_dir = expand_host_path( + require_string(value.get("state_dir"), "configuration state_dir"), + "state_dir", + ) + provenance_key_file = expand_host_path( + require_string( + value.get("provenance_key_file"), + "configuration provenance_key_file", + ), + "provenance_key_file", + ) return Config( path=path, app_token_env=env_names["app_token_env"], bot_token_env=env_names["bot_token_env"], github_token_env=env_names["github_token_env"], + keychain_services=keychain_services, channel_allowlist=tuple(channels), repo_allowlist=tuple(repos), daily_budget_usd=budget, daily_request_cap=cap, - agent_branch_prefixes=agent_branch_prefixes, + provenance_key_file=provenance_key_file, state_dir=state_dir, ) -def required_token(env_name: str, role: str) -> str: +def required_token( + env_name: str, + role: str, + keychain_service: str | None = None, +) -> str: value = os.environ.get(env_name) + if (value is None or value == "") and keychain_service: + security = Path("/usr/bin/security") + if security.is_file(): + try: + result = subprocess.run( + [ + str(security), + "find-generic-password", + "-s", + keychain_service, + "-w", + ], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + timeout=10, + check=False, + ) + except (OSError, subprocess.TimeoutExpired): + result = None + if result is not None and result.returncode == 0: + try: + value = result.stdout.decode("utf-8", "strict").rstrip("\r\n") + except UnicodeDecodeError: + value = None if value is None or value == "": + fallback = ( + f" or install it in macOS Keychain service {keychain_service}" + if keychain_service + else "" + ) refuse( f"cannot start: the {role} environment variable {env_name} is not set. " - f"Export {env_name} with the credential and start again; the token is " + f"Export {env_name} with the credential{fallback} and start again; the token is " "never stored in the config file" ) register_secret(value) return value +def load_provenance_key(path: Path) -> bytes: + """Load the coordinator-only HMAC key without following links.""" + + try: + stat_result = path.lstat() + except FileNotFoundError as exc: + raise SlackExposureError( + f"provenance signing key is missing at {path}" + ) from exc + except OSError as exc: + raise SlackExposureError( + f"provenance signing key inspection failed at {path}: {exc}" + ) from exc + require(not path.is_symlink(), f"provenance signing key must not be a symlink: {path}") + require( + (stat_result.st_mode & 0o077) == 0, + f"provenance signing key permissions must be owner-only at {path}", + ) + try: + raw = path.read_text(encoding="ascii").strip() + except (OSError, UnicodeError) as exc: + raise SlackExposureError( + f"provenance signing key is unreadable at {path}: {exc}" + ) from exc + require( + re.fullmatch(r"[0-9a-f]{64}", raw) is not None, + f"provenance signing key at {path} must contain exactly 32 bytes as lowercase hex", + ) + register_secret(raw) + return bytes.fromhex(raw) + + +def provenance_key_id(key: bytes) -> str: + return hashlib.sha256(key).hexdigest()[:16] + + +def configured_credentials(config: Config) -> tuple[str, str, str]: + app_token = required_token( + config.app_token_env, + "Slack app-level (Socket Mode) token", + config.keychain_services.get("app_token"), + ) + bot_token = required_token( + config.bot_token_env, + "Slack bot token", + config.keychain_services.get("bot_token"), + ) + github_token = required_token( + config.github_token_env, + "GitHub read credential", + config.keychain_services.get("github_token"), + ) + return app_token, bot_token, github_token + + # --- durable event dedupe ----------------------------------------------------- @@ -539,15 +649,47 @@ def _locked(self) -> Any: fcntl.flock(handle, fcntl.LOCK_EX) return handle - def begin(self, submitter: str, pr_url: str, event_id: str) -> str: + def begin( + self, + submitter: str, + pr_url: str, + event_id: str, + *, + request_cap: int | None = None, + budget_usd: float | None = None, + ) -> tuple[str | None, str | None, float]: + """Atomically enforce both bounds and record one admitted request. + + Returns (request_id, refusal, observed), where refusal is `cap` or + `budget` and observed is the count or spend that reached the bound. + The check and append share one lock so concurrent listener workers + cannot overrun a per-engineer cap. + """ + day = self._day_fn() require( REQUEST_DAY_RE.fullmatch(day) is not None, f"meter day function returned a non-day value: {day!r}", ) - request_id = f"req-{day}-{secrets.token_hex(6)}" with self._locked(): ledger = self._load(day) + submitter_rows = [ + record + for record in ledger["requests"] + if record.get("submitter") == submitter + ] + count = len(submitter_rows) + if request_cap is not None and count >= request_cap: + return None, "cap", float(count) + spent = sum( + float(cost) + for record in submitter_rows + for cost in [record.get("estimated_usd")] + if isinstance(cost, (int, float)) and not isinstance(cost, bool) + ) + if budget_usd is not None and spent >= budget_usd: + return None, "budget", spent + request_id = f"req-{day}-{secrets.token_hex(6)}" ledger["requests"].append( { "id": request_id, @@ -564,7 +706,7 @@ def begin(self, submitter: str, pr_url: str, event_id: str) -> str: } ) self._write(day, ledger) - return request_id + return request_id, None, 0.0 def finish( self, @@ -659,17 +801,47 @@ def repo_of(pr_url: str) -> str: return f"{match.group(1)}/{match.group(2)}".lower() -BRANCH_NAME_RE = re.compile(r"^[^\s]{1,255}$") +@dataclasses.dataclass(frozen=True) +class PrSnapshot: + pr_url: str + repository: str + number: int + head_sha: str -def fetch_head_branch(pr_url: str, github_token: str) -> str: - """Return the PR's head branch name via the GitHub API, fail closed. +@dataclasses.dataclass(frozen=True) +class AuthorshipProvenance: + harness: str + model: str + model_family: str + task_id: str + task_generation: str + author_kind: str + author_account_identity: str | None + launch_attestation_path: Path + launch_attestation_sha256: str + attestation_path: Path + attestation_sha256: str - Called only AFTER the repository allowlist admitted the URL, so the read - credential is never pointed outside the allowlist. Used by the - agent-branch screen; any failure raises so the caller refuses to review - rather than reviewing with an unverified authorship assertion. - """ + +@dataclasses.dataclass(frozen=True) +class LaunchProvenance: + harness: str + model: str + model_family: str + task_id: str + task_generation: str + author_account_identity: str | None + worktree: Path + git_dir_identity: str + launch_head: str + launch_ref: str + attestation_path: Path + attestation_sha256: str + + +def fetch_pr_snapshot(pr_url: str, github_token: str) -> PrSnapshot: + """Return the allowlisted PR's exact current head, fail closed.""" match = PR_LINK_RE.fullmatch(pr_url) require(match is not None, f"internal error: unparseable PR URL {pr_url!r}") @@ -687,25 +859,670 @@ def fetch_head_branch(pr_url: str, github_token: str) -> str: with urllib.request.urlopen(request, timeout=WEB_API_TIMEOUT_SECONDS) as response: body = response.read(1024 * 1024) except (urllib.error.URLError, OSError) as exc: - raise SlackExposureError(f"head-branch lookup failed for {pr_url}: {exc}") from exc + raise SlackExposureError(f"PR head lookup failed for {pr_url}: {exc}") from exc try: value = json.loads(body.decode("utf-8")) except (UnicodeDecodeError, json.JSONDecodeError) as exc: raise SlackExposureError( - f"head-branch lookup returned malformed JSON for {pr_url}" + f"PR head lookup returned malformed JSON for {pr_url}" ) from exc head = value.get("head") if isinstance(value, dict) else None - ref = head.get("ref") if isinstance(head, dict) else None - if not isinstance(ref, str) or BRANCH_NAME_RE.fullmatch(ref) is None: - raise SlackExposureError(f"head-branch lookup returned no usable ref for {pr_url}") - return ref + sha = head.get("sha") if isinstance(head, dict) else None + if not isinstance(sha, str) or SHA_RE.fullmatch(sha) is None: + raise SlackExposureError(f"PR head lookup returned no exact SHA for {pr_url}") + return PrSnapshot( + pr_url=pr_url, + repository=repo_of(pr_url), + number=int(match.group(3)), + head_sha=sha, + ) + + +def canonical_json(value: Any) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8") + + +@functools.lru_cache(maxsize=1) +def crosscheck_core() -> Any: + module_path = BIN_DIR / "fm-crosscheck.py" + spec = importlib.util.spec_from_file_location("fm_crosscheck_core_identity", module_path) + require(spec is not None and spec.loader is not None, "Crosscheck model-family owner is unavailable") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@functools.lru_cache(maxsize=128) +def crosscheck_model_family(model: str) -> str: + """Use the core gate's exact family classifier, never a Slack-local guess.""" + + module = crosscheck_core() + family = module.model_family(model) + require(isinstance(family, str) and family != "", "Crosscheck model family is empty") + return family + + +def attestation_path(state_dir: Path, snapshot: PrSnapshot) -> Path: + identity = f"{snapshot.repository}#{snapshot.number}@{snapshot.head_sha}" + name = hashlib.sha256(identity.encode("utf-8")).hexdigest() + ".json" + return state_dir / "provenance" / name + + +def launch_attestation_path( + state_dir: Path, task_id: str, task_generation: str +) -> Path: + identity = f"{task_id}@{task_generation}" + name = hashlib.sha256(identity.encode("utf-8")).hexdigest() + ".json" + return state_dir / "launch-provenance" / name + + +def git_inspect_worktree(worktree: Path) -> dict[str, str]: + """Return the exact physical Git identity of one clean task worktree.""" + + require(worktree.is_absolute(), f"task worktree must be absolute: {worktree}") + try: + metadata = worktree.lstat() + except OSError as exc: + raise SlackExposureError(f"task worktree is unavailable at {worktree}: {exc}") from exc + require( + stat.S_ISDIR(metadata.st_mode) and not worktree.is_symlink(), + f"task worktree must be a real directory at {worktree}", + ) + resolved = worktree.resolve() + + def git(*arguments: str) -> str: + try: + result = run_bounded( + ["git", "-C", str(resolved), *arguments], + timeout_seconds=30, + maximum_output_bytes=1024 * 1024, + env={ + "PATH": os.environ.get("PATH", "/usr/bin:/bin"), + "HOME": os.environ.get("HOME", "/var/empty"), + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_TERMINAL_PROMPT": "0", + "GIT_ASKPASS": "/usr/bin/false", + "SSH_ASKPASS": "/usr/bin/false", + }, + ) + except BoundedIOError as exc: + raise SlackExposureError( + f"task worktree Git inspection exceeded its bounds at {resolved}: {exc}" + ) from exc + diagnostic = result.stderr.decode("utf-8", "replace").strip() + require( + result.returncode == 0, + f"task worktree Git inspection failed at {resolved}: " + f"{clamp(redact(diagnostic or 'no diagnostic'), 300)}", + ) + return result.stdout.decode("utf-8", "strict").strip() + + top = Path(git("rev-parse", "--show-toplevel")).resolve() + require(top == resolved, f"task worktree top differs from its recorded path: {top}") + head = git("rev-parse", "HEAD") + require(SHA_RE.fullmatch(head) is not None, f"task worktree has no exact HEAD at {resolved}") + git_dir_raw = Path(git("rev-parse", "--git-dir")) + git_dir = git_dir_raw if git_dir_raw.is_absolute() else resolved / git_dir_raw + git_dir = git_dir.resolve() + try: + git_dir_stat = git_dir.stat() + except OSError as exc: + raise SlackExposureError( + f"task worktree Git directory is unavailable at {git_dir}: {exc}" + ) from exc + require(stat.S_ISDIR(git_dir_stat.st_mode), f"task Git directory is not a directory at {git_dir}") + ref_result = git("rev-parse", "--symbolic-full-name", "HEAD") + return { + "worktree": str(resolved), + "head": head, + "ref": ref_result if ref_result != "HEAD" else "DETACHED", + "git_dir_identity": f"{git_dir_stat.st_dev}:{git_dir_stat.st_ino}", + "tracked_status": git("status", "--porcelain=v1", "--untracked-files=no"), + } + + +def git_head_descends_from(worktree: Path, ancestor: str) -> bool: + """Prove the current task head retains the commit it launched from.""" + + require(SHA_RE.fullmatch(ancestor) is not None, "launch head is not an exact SHA") + try: + result = run_bounded( + ["git", "-C", str(worktree.resolve()), "merge-base", "--is-ancestor", ancestor, "HEAD"], + timeout_seconds=30, + maximum_output_bytes=1024 * 1024, + env={ + "PATH": os.environ.get("PATH", "/usr/bin:/bin"), + "HOME": os.environ.get("HOME", "/var/empty"), + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_TERMINAL_PROMPT": "0", + "GIT_ASKPASS": "/usr/bin/false", + "SSH_ASKPASS": "/usr/bin/false", + }, + ) + except BoundedIOError as exc: + raise SlackExposureError( + f"task worktree ancestry inspection exceeded its bounds at {worktree}: {exc}" + ) from exc + if result.returncode == 0: + return True + if result.returncode == 1: + return False + diagnostic = result.stderr.decode("utf-8", "replace").strip() + raise SlackExposureError( + f"task worktree ancestry inspection failed at {worktree}: " + f"{clamp(redact(diagnostic or 'no diagnostic'), 300)}" + ) + + +def launch_account_identity(harness: str, account_home: Path) -> str: + """Derive the author account through Crosscheck's identity owner.""" + + core = crosscheck_core() + if harness == "codex": + return core.account_identity(harness, account_home) + require(harness == "pi", f"unsupported OpenAI author harness {harness!r}") + credential_path = account_home.resolve() / "auth.json" + try: + credential = read_bounded_json(credential_path, maximum_bytes=1024 * 1024) + except BoundedIOError as exc: + raise SlackExposureError( + f"Pi author credential is unreadable at {credential_path}: {exc}" + ) from exc + require( + isinstance(credential, dict) and len(credential) == 1, + f"Pi author credential at {credential_path} must contain exactly one profile", + ) + entry = next(iter(credential.values())) + return core.account_identity_from_credential( + "pi", {"openai-codex": entry}, str(credential_path) + ) + + +def parse_task_identity(meta_path: Path) -> dict[str, str]: + try: + metadata_stat = meta_path.lstat() + require( + stat.S_ISREG(metadata_stat.st_mode), + f"task metadata is not a regular file at {meta_path}", + ) + require( + 0 < metadata_stat.st_size <= MAX_TASK_META_BYTES, + f"task metadata at {meta_path} exceeds its byte bound", + ) + lines = meta_path.read_text(encoding="utf-8").splitlines() + except (OSError, UnicodeError) as exc: + raise SlackExposureError(f"task metadata is unreadable at {meta_path}: {exc}") from exc + values: dict[str, list[str]] = {} + for line in lines: + if "=" not in line: + continue + key, value = line.split("=", 1) + if key in { + "harness", + "model", + "generation_id", + "worktree", + "pr", + "pr_head", + }: + values.setdefault(key, []).append(value) + result: dict[str, str] = {} + for key in ("harness", "model", "generation_id", "worktree", "pr"): + found = values.get(key, []) + require( + len(found) == 1 and found[0] != "", + f"task metadata at {meta_path} must contain exactly one nonempty {key}", + ) + result[key] = found[0] + heads = values.get("pr_head", []) + require(heads and SHA_RE.fullmatch(heads[-1]) is not None, f"task metadata at {meta_path} has no exact PR head") + result["pr_head"] = heads[-1] + return result + + +def require_provenance_string(value: Any, label: str, maximum: int = 512) -> str: + require( + isinstance(value, str) + and 0 < len(value) <= maximum + and all(character.isprintable() for character in value), + f"{label} must be a bounded printable string", + ) + return value + + +def signed_attestation( + payload: dict[str, Any], key: bytes, *, schema: str = PROVENANCE_SCHEMA +) -> dict[str, Any]: + signed_body = {"schema": schema, "payload": payload} + signature = hmac.new(key, canonical_json(signed_body), hashlib.sha256).hexdigest() + return { + "schema": schema, + "payload": payload, + "signature": { + "algorithm": PROVENANCE_SIGNATURE_ALGORITHM, + "key_id": provenance_key_id(key), + "value": signature, + }, + } + + +def write_new_attestation(destination: Path, document: dict[str, Any]) -> None: + destination.parent.mkdir(parents=True, exist_ok=True) + encoded = json.dumps(document, indent=2, sort_keys=True) + "\n" + try: + descriptor = os.open( + str(destination), os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600 + ) + except FileExistsError: + raise + try: + with os.fdopen(descriptor, "w", encoding="utf-8") as handle: + handle.write(encoded) + except BaseException: + try: + destination.unlink() + except OSError: + pass + raise + + +def issue_launch_attestation( + config: Config, + key: bytes, + task_id: str, + task_generation: str, + worktree: Path, + harness: str, + model: str, + account_home: Path | None, +) -> Path: + """Capture the author identity before the task's agent process starts.""" + + require(TASK_ID_RE.fullmatch(task_id) is not None, f"task id validation rejected {task_id!r}") + for value, label in ( + (task_generation, "task generation"), + (harness, "author harness"), + (model, "author model"), + ): + require_provenance_string(value, label) + require( + harness != "human" and model != "human-authored", + "Firstmate task launch provenance cannot classify human authorship", + ) + require(model != "default", "author model is unresolved at task launch") + worktree_identity = git_inspect_worktree(worktree) + family = crosscheck_model_family(model) + account_identity: str | None = None + if family == "openai": + require( + harness in {"pi", "codex"}, + f"codex-family author model {model!r} has unsupported harness {harness!r}", + ) + require(account_home is not None, "codex-family author has no launch-bound account home") + try: + account_identity = launch_account_identity(harness, account_home) + except Exception as exc: + raise SlackExposureError( + f"codex-family author account identity is unverifiable: {exc}" + ) from exc + require_provenance_string(account_identity, "author account identity") + destination = launch_attestation_path(config.state_dir, task_id, task_generation) + payload: dict[str, Any] = { + "issued_at": utc_now(), + "issuer": "firstmate-spawn", + "task": { + "task_id": task_id, + "task_generation": task_generation, + }, + "author": { + "harness": harness, + "model": model, + "model_family": family, + "account_identity": account_identity, + }, + "launch": { + "worktree": worktree_identity["worktree"], + "git_dir_identity": worktree_identity["git_dir_identity"], + "head_sha": worktree_identity["head"], + "ref": worktree_identity["ref"], + }, + } + document = signed_attestation(payload, key, schema=LAUNCH_PROVENANCE_SCHEMA) + if destination.exists(): + existing = verify_launch_attestation(config, key, task_id, task_generation) + require( + existing.harness == harness + and existing.model == model + and existing.author_account_identity == account_identity + and existing.worktree == Path(worktree_identity["worktree"]) + and existing.git_dir_identity == worktree_identity["git_dir_identity"] + and existing.launch_head == worktree_identity["head"] + and existing.launch_ref == worktree_identity["ref"], + f"conflicting launch provenance already exists at {destination}", + ) + return destination + try: + write_new_attestation(destination, document) + except FileExistsError: + existing = verify_launch_attestation(config, key, task_id, task_generation) + require( + existing.harness == harness + and existing.model == model + and existing.author_account_identity == account_identity + and existing.worktree == Path(worktree_identity["worktree"]) + and existing.git_dir_identity == worktree_identity["git_dir_identity"] + and existing.launch_head == worktree_identity["head"] + and existing.launch_ref == worktree_identity["ref"], + f"conflicting launch provenance won a concurrent issue at {destination}", + ) + return destination + + +def verify_launch_attestation( + config: Config, + key: bytes, + task_id: str, + task_generation: str, +) -> LaunchProvenance: + path = launch_attestation_path(config.state_dir, task_id, task_generation) + try: + raw_document = path.read_bytes() + except OSError as exc: + raise SlackExposureError( + f"no launch-bound Firstmate authorship attestation exists for task " + f"{task_id} generation {task_generation}: {exc}" + ) from exc + require(len(raw_document) <= MAX_PROVENANCE_BYTES, f"launch attestation at {path} exceeds its byte bound") + try: + document = read_bounded_json(path, maximum_bytes=MAX_PROVENANCE_BYTES) + raw_value = json.loads(raw_document.decode("utf-8")) + except (BoundedIOError, UnicodeDecodeError, json.JSONDecodeError, ValueError) as exc: + raise SlackExposureError(f"launch attestation is malformed at {path}: {exc}") from exc + require(raw_value == document, f"launch attestation changed while it was verified at {path}") + require(isinstance(document, dict), f"launch attestation at {path} must be an object") + require(set(document) == {"schema", "payload", "signature"}, f"launch attestation at {path} has an unknown shape") + require(document.get("schema") == LAUNCH_PROVENANCE_SCHEMA, f"launch attestation at {path} has an unknown schema") + payload = document.get("payload") + signature = document.get("signature") + require(isinstance(payload, dict) and isinstance(signature, dict), f"launch attestation at {path} is incomplete") + require( + set(payload) == {"issued_at", "issuer", "task", "author", "launch"} + and payload.get("issuer") == "firstmate-spawn" + and isinstance(payload.get("issued_at"), str), + f"launch attestation at {path} has unknown issuer metadata", + ) + require( + signature.get("algorithm") == PROVENANCE_SIGNATURE_ALGORITHM + and signature.get("key_id") == provenance_key_id(key) + and isinstance(signature.get("value"), str), + f"launch attestation at {path} has untrusted signature metadata", + ) + expected = hmac.new( + key, + canonical_json({"schema": LAUNCH_PROVENANCE_SCHEMA, "payload": payload}), + hashlib.sha256, + ).hexdigest() + require(hmac.compare_digest(signature["value"], expected), f"launch attestation signature verification failed at {path}") + task = payload.get("task") + author = payload.get("author") + launch = payload.get("launch") + require( + task == {"task_id": task_id, "task_generation": task_generation}, + f"launch attestation at {path} conflicts with the requested task generation", + ) + require(isinstance(author, dict) and set(author) == {"harness", "model", "model_family", "account_identity"}, f"launch attestation at {path} has an unknown author shape") + require(isinstance(launch, dict) and set(launch) == {"worktree", "git_dir_identity", "head_sha", "ref"}, f"launch attestation at {path} has an unknown worktree shape") + for field in ("harness", "model", "model_family"): + require_provenance_string(author.get(field), f"launch attestation at {path} field {field}") + require( + author["harness"] != "human" and author["model"] != "human-authored", + f"launch attestation at {path} cannot establish human authorship", + ) + require(author["model_family"] == crosscheck_model_family(author["model"]), f"launch attestation at {path} conflicts on model family") + account = author.get("account_identity") + if author["model_family"] == "openai": + require_provenance_string(account, f"launch attestation at {path} account identity") + else: + require(account is None or isinstance(account, str), f"launch attestation at {path} has malformed account identity") + worktree = Path(require_provenance_string(launch.get("worktree"), f"launch attestation at {path} worktree", 4096)) + require(worktree.is_absolute(), f"launch attestation at {path} has a relative worktree") + git_dir_identity = require_provenance_string(launch.get("git_dir_identity"), f"launch attestation at {path} Git identity") + launch_head = require_provenance_string(launch.get("head_sha"), f"launch attestation at {path} head") + require(SHA_RE.fullmatch(launch_head) is not None, f"launch attestation at {path} has no exact launch head") + launch_ref = require_provenance_string(launch.get("ref"), f"launch attestation at {path} ref") + return LaunchProvenance( + harness=author["harness"], + model=author["model"], + model_family=author["model_family"], + task_id=task_id, + task_generation=task_generation, + author_account_identity=account, + worktree=worktree, + git_dir_identity=git_dir_identity, + launch_head=launch_head, + launch_ref=launch_ref, + attestation_path=path, + attestation_sha256=hashlib.sha256(raw_document).hexdigest(), + ) + + +def issue_task_attestation( + config: Config, + key: bytes, + task_id: str, + pr_url: str, + head_sha: str, +) -> Path: + """Mint exact-head provenance from a launch-bound Firstmate task record.""" + + require(TASK_ID_RE.fullmatch(task_id) is not None, f"task id validation rejected {task_id!r}") + require(PR_LINK_RE.fullmatch(pr_url) is not None, f"PR URL validation rejected {pr_url!r}") + require(SHA_RE.fullmatch(head_sha) is not None, f"PR head validation rejected {head_sha!r}") + fm_home = Path(os.environ.get("FM_HOME", "")) + require(fm_home.is_absolute(), "FM_HOME is required to issue task provenance") + meta_path = Path(os.environ.get("FM_STATE_OVERRIDE") or fm_home / "state") / f"{task_id}.meta" + identity = parse_task_identity(meta_path) + require(identity["pr"].rstrip("/") == pr_url.rstrip("/"), "task metadata PR does not match the attestation subject") + require(identity["pr_head"] == head_sha, "task metadata head does not match the attestation subject") + require( + identity["harness"] != "human" and identity["model"] != "human-authored", + "Firstmate task metadata cannot establish human authorship; a trusted upstream producer is required", + ) + launch = verify_launch_attestation( + config, key, task_id, identity["generation_id"] + ) + require( + launch.harness == identity["harness"] + and launch.model == identity["model"], + "task metadata conflicts with launch-bound author identity", + ) + current = git_inspect_worktree(Path(identity["worktree"])) + require( + Path(current["worktree"]) == launch.worktree, + "task metadata worktree conflicts with launch-bound provenance", + ) + require( + current["git_dir_identity"] == launch.git_dir_identity, + "task worktree Git identity changed after author launch", + ) + require( + git_head_descends_from(Path(current["worktree"]), launch.launch_head), + "task worktree HEAD does not descend from its launch-bound head", + ) + require( + current["tracked_status"] == "", + "task worktree has uncommitted tracked changes; exact-head authorship is not attestable", + ) + require( + current["head"] == head_sha, + "task worktree HEAD does not equal the live PR head; another worktree or later author produced it", + ) + snapshot = PrSnapshot(pr_url, repo_of(pr_url), int(PR_LINK_RE.fullmatch(pr_url).group(3)), head_sha) # type: ignore[union-attr] + destination = attestation_path(config.state_dir, snapshot) + if destination.exists(): + existing = verify_attestation(config, key, snapshot) + require( + existing.harness == launch.harness + and existing.model == launch.model + and existing.task_id == task_id + and existing.task_generation == identity["generation_id"] + and existing.author_account_identity == launch.author_account_identity + and existing.launch_attestation_sha256 == launch.attestation_sha256, + f"conflicting provenance already exists at {destination}", + ) + return destination + payload: dict[str, Any] = { + "issued_at": utc_now(), + "issuer": "firstmate-pr-check", + "subject": { + "repository": snapshot.repository, + "pull_request": snapshot.number, + "pr_url": snapshot.pr_url.rstrip("/"), + "head_sha": snapshot.head_sha, + }, + "author": { + "kind": "agent", + "harness": launch.harness, + "model": launch.model, + "model_family": launch.model_family, + "task_id": task_id, + "task_generation": identity["generation_id"], + "account_identity": launch.author_account_identity, + }, + "origin": { + "schema": LAUNCH_PROVENANCE_SCHEMA, + "sha256": launch.attestation_sha256, + }, + } + document = signed_attestation(payload, key) + try: + write_new_attestation(destination, document) + except FileExistsError: + existing = verify_attestation(config, key, snapshot) + require( + existing.harness == launch.harness + and existing.model == launch.model + and existing.task_id == task_id + and existing.task_generation == identity["generation_id"] + and existing.author_account_identity == launch.author_account_identity + and existing.launch_attestation_sha256 == launch.attestation_sha256, + f"conflicting provenance won a concurrent issue at {destination}", + ) + return destination + return destination -def matched_agent_prefix(branch: str, prefixes: tuple[str, ...]) -> str | None: - for prefix in prefixes: - if branch.startswith(prefix): - return prefix - return None +def verify_attestation(config: Config, key: bytes, snapshot: PrSnapshot) -> AuthorshipProvenance: + path = attestation_path(config.state_dir, snapshot) + try: + raw_document = path.read_bytes() + except OSError as exc: + raise SlackExposureError( + f"no verifiable authorship attestation exists for this exact PR head: {exc}" + ) from exc + require( + len(raw_document) <= MAX_PROVENANCE_BYTES, + f"authorship attestation at {path} exceeds its byte bound", + ) + try: + document = read_bounded_json(path, maximum_bytes=MAX_PROVENANCE_BYTES) + except BoundedIOError as exc: + raise SlackExposureError(f"no verifiable authorship attestation exists for this exact PR head: {exc}") from exc + try: + raw_value = json.loads(raw_document.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError, ValueError) as exc: + raise SlackExposureError( + f"authorship attestation changed or became malformed at {path}: {exc}" + ) from exc + require( + raw_value == document, + f"authorship attestation changed while it was being verified at {path}", + ) + require(isinstance(document, dict), f"authorship attestation at {path} must be an object") + require(set(document) == {"schema", "payload", "signature"}, f"authorship attestation at {path} has an unknown shape") + require(document.get("schema") == PROVENANCE_SCHEMA, f"authorship attestation at {path} has an unknown schema") + payload = document.get("payload") + signature = document.get("signature") + require(isinstance(payload, dict) and isinstance(signature, dict), f"authorship attestation at {path} is incomplete") + require( + set(payload) == {"issued_at", "issuer", "subject", "author", "origin"} + and payload.get("issuer") == "firstmate-pr-check" + and isinstance(payload.get("issued_at"), str), + f"authorship attestation at {path} has unknown issuer metadata", + ) + require( + signature.get("algorithm") == PROVENANCE_SIGNATURE_ALGORITHM + and signature.get("key_id") == provenance_key_id(key) + and isinstance(signature.get("value"), str), + f"authorship attestation at {path} has untrusted signature metadata", + ) + expected = hmac.new( + key, + canonical_json({"schema": PROVENANCE_SCHEMA, "payload": payload}), + hashlib.sha256, + ).hexdigest() + require(hmac.compare_digest(signature["value"], expected), f"authorship attestation signature verification failed at {path}") + subject = payload.get("subject") + author = payload.get("author") + origin = payload.get("origin") + require(isinstance(subject, dict) and isinstance(author, dict) and isinstance(origin, dict), f"authorship attestation at {path} has no subject, author, or origin") + expected_subject = { + "repository": snapshot.repository, + "pull_request": snapshot.number, + "pr_url": snapshot.pr_url.rstrip("/"), + "head_sha": snapshot.head_sha, + } + require(subject == expected_subject, f"authorship attestation at {path} conflicts with the live PR head") + required_author = {"kind", "harness", "model", "model_family", "task_id", "task_generation", "account_identity"} + require(set(author) == required_author, f"authorship attestation at {path} has an unknown author shape") + for field in ("kind", "harness", "model", "model_family", "task_id", "task_generation"): + require_provenance_string( + author[field], f"authorship attestation at {path} field {field}" + ) + require(author["kind"] in {"agent", "human"}, f"authorship attestation at {path} has unknown author kind") + expected_kind = "human" if author["harness"] == "human" and author["model"] == "human-authored" else "agent" + require(author["kind"] == expected_kind, f"authorship attestation at {path} conflicts on author kind") + require(author["model_family"] == crosscheck_model_family(author["model"]), f"authorship attestation at {path} conflicts on model family") + account = author["account_identity"] + if author["model_family"] == "openai" or account is not None: + require_provenance_string( + account, f"authorship attestation at {path} account identity" + ) + require( + TASK_ID_RE.fullmatch(author["task_id"]) is not None, + f"authorship attestation at {path} has malformed task identity", + ) + require( + origin.get("schema") == LAUNCH_PROVENANCE_SCHEMA + and set(origin) == {"schema", "sha256"} + and isinstance(origin.get("sha256"), str) + and re.fullmatch(r"[0-9a-f]{64}", origin["sha256"]) is not None, + f"authorship attestation at {path} has untrusted launch provenance", + ) + launch = verify_launch_attestation( + config, key, author["task_id"], author["task_generation"] + ) + require( + launch.attestation_sha256 == origin["sha256"] + and launch.harness == author["harness"] + and launch.model == author["model"] + and launch.model_family == author["model_family"] + and launch.author_account_identity == account, + f"authorship attestation at {path} conflicts with launch-bound provenance", + ) + return AuthorshipProvenance( + harness=author["harness"], + model=author["model"], + model_family=author["model_family"], + task_id=author["task_id"], + task_generation=author["task_generation"], + author_kind=author["kind"], + author_account_identity=account, + launch_attestation_path=launch.attestation_path, + launch_attestation_sha256=launch.attestation_sha256, + attestation_path=path, + attestation_sha256=hashlib.sha256(raw_document).hexdigest(), + ) # --- lane naming ----------------------------------------------------------------- @@ -810,6 +1627,7 @@ def render_verdict_reply( ledger: dict[str, Any], lane: str, report_path: Path, + task_id: str, ) -> str: # Every ledger-derived value is escaped uniformly: state, lane, head, # summary, severities, and titles all pass through escape_slack, so no @@ -818,10 +1636,14 @@ def render_verdict_reply( lines = [ f"Crosscheck {state} for {pr_url}", f"Lane: {escape_slack(clamp(lane, 120))}", + f"Task ID: {escape_slack(task_id)}", ] head = run.get("head_sha") - if isinstance(head, str) and head: - lines.append(f"Reviewed head: {escape_slack(clamp(head, 64))}") + require( + isinstance(head, str) and SHA_RE.fullmatch(head) is not None, + "the Crosscheck run has no exact reviewed head", + ) + lines.append(f"Reviewed head: {head}") summary = run.get("summary") if isinstance(summary, str) and summary.strip(): lines.append(f"Summary: {escape_slack(clamp(summary.strip(), MAX_SUMMARY_CHARS))}") @@ -847,7 +1669,57 @@ def render_verdict_reply( lines.append("No active findings for this head.") # The report is a file on the listener's host, not a link anyone remote # can open; label it so nobody reads it as one. - lines.append(f"Host report path for the operator: {report_path}") + lines.append(f"Durable artifact: {report_path}") + return clamp("\n".join(lines), MAX_REPLY_CHARS) + + +def render_stale_reply( + pr_url: str, + reviewed_head: str, + current_head: str, + lane: str, + task_id: str, + report_path: Path, +) -> str: + return clamp( + "\n".join( + [ + f"Crosscheck STALE for {pr_url}", + f"Lane: {escape_slack(clamp(lane, 120))}", + f"Task ID: {escape_slack(task_id)}", + f"Reviewed head: {reviewed_head}", + f"Current head: {current_head}", + f"Durable artifact: {report_path}", + "The PR head changed during review, so the earlier verdict does not apply. Request a new review for the current head.", + ] + ), + MAX_REPLY_CHARS, + ) + + +def render_nonverdict_reply( + pr_url: str, + state: str, + reviewed_head: str, + lane: str, + task_id: str, + report_path: Path, + summary: Any, +) -> str: + lines = [ + f"Crosscheck {escape_slack(clamp(state.upper(), 40))} for {pr_url}", + f"Lane: {escape_slack(clamp(lane, 120))}", + f"Task ID: {escape_slack(task_id)}", + f"Reviewed head: {reviewed_head}", + f"Durable artifact: {report_path}", + ] + if isinstance(summary, str) and summary.strip(): + lines.append( + f"Diagnostic: {escape_slack(clamp(summary.strip(), MAX_SUMMARY_CHARS))}" + ) + lines.append( + "No admitted verdict was produced; this is a review or infrastructure failure, not CLEAR." + ) return clamp("\n".join(lines), MAX_REPLY_CHARS) @@ -911,23 +1783,65 @@ def scrubbed_child_environment(config: Config, github_token: str) -> dict[str, s return child -def make_run_review(config: Config, github_token: str) -> Callable[[str], ReviewOutcome]: +def make_run_review( + config: Config, + github_token: str, + current_snapshot: Callable[[str], PrSnapshot] | None = None, +) -> Callable[[str, PrSnapshot, AuthorshipProvenance], ReviewOutcome]: fm_home_raw = os.environ.get("FM_HOME", "") if not fm_home_raw: raise SlackExposureError("FM_HOME is required to run crosscheck reviews") fm_home = Path(fm_home_raw).resolve() state = Path(os.environ.get("FM_STATE_OVERRIDE") or fm_home / "state") data = Path(os.environ.get("FM_DATA_OVERRIDE") or fm_home / "data") + resolve_current_snapshot = current_snapshot or ( + lambda pr_url: fetch_pr_snapshot(pr_url, github_token) + ) - def run_review(pr_url: str) -> ReviewOutcome: + def run_review( + pr_url: str, + snapshot: PrSnapshot, + provenance: AuthorshipProvenance, + ) -> ReviewOutcome: task_id = f"slack-{secrets.token_hex(6)}" state.mkdir(parents=True, exist_ok=True) - # The crosscheck gate derives the author's model family from task - # metadata; a Slack-submitted engineer PR is human-authored, so every - # reviewer model family is structurally separate from the author. + meta_lines = [ + f"harness={provenance.harness}", + f"model={provenance.model}", + f"author_model_family={provenance.model_family}", + f"author_task_id={provenance.task_id}", + f"author_task_generation={provenance.task_generation}", + f"author_attestation_sha256={provenance.attestation_sha256}", + f"author_launch_attestation_sha256={provenance.launch_attestation_sha256}", + ] + if provenance.author_account_identity: + meta_lines.append( + f"author_account_identity={provenance.author_account_identity}" + ) (state / f"{task_id}.meta").write_text( - "harness=slack-team\nmodel=human-authored\n", encoding="utf-8" + "\n".join(meta_lines) + "\n", + encoding="utf-8", + ) + task_data = data / task_id + task_data.mkdir(parents=True, exist_ok=True) + attestation_copy = task_data / "authorship-attestation.json" + attestation_bytes = provenance.attestation_path.read_bytes() + require( + hashlib.sha256(attestation_bytes).hexdigest() + == provenance.attestation_sha256, + "authorship attestation changed after verification", + ) + attestation_copy.write_bytes(attestation_bytes) + os.chmod(attestation_copy, 0o600) + launch_copy = task_data / "author-launch-attestation.json" + launch_bytes = provenance.launch_attestation_path.read_bytes() + require( + hashlib.sha256(launch_bytes).hexdigest() + == provenance.launch_attestation_sha256, + "author launch attestation changed after verification", ) + launch_copy.write_bytes(launch_bytes) + os.chmod(launch_copy, 0o600) argv = crosscheck_argv(task_id, pr_url) try: result = run_bounded( @@ -947,7 +1861,7 @@ def run_review(pr_url: str) -> ReviewOutcome: task_id=task_id, ) ledger_path = data / task_id / "crosscheck-ledger.json" - report_path = data / task_id / "crosscheck.md" + report_path = task_data / "crosscheck.md" if not ledger_path.is_file(): stderr_tail = result.stderr.decode("utf-8", "replace")[-800:].strip() return ReviewOutcome( @@ -989,10 +1903,99 @@ def run_review(pr_url: str) -> ReviewOutcome: run = runs[-1] lane = lane_name(run.get("reviewer")) tokens, estimated = usage_from_run(run) - reply = render_verdict_reply(pr_url, run, ledger, lane, report_path) + reviewed_head = run.get("head_sha") + if not isinstance(reviewed_head, str) or SHA_RE.fullmatch(reviewed_head) is None: + return ReviewOutcome( + ok=False, + state="tool-failure", + lane=lane, + reply_text=render_failure_reply( + pr_url, "the Crosscheck ledger recorded no exact reviewed head" + ), + tokens=tokens, + estimated_usd=estimated, + task_id=task_id, + ) + if reviewed_head != snapshot.head_sha: + return ReviewOutcome( + ok=False, + state="tool-failure", + lane=lane, + reply_text=render_failure_reply( + pr_url, + "the Crosscheck ledger head conflicts with the exact head admitted from Slack", + ), + tokens=tokens, + estimated_usd=estimated, + task_id=task_id, + ) + try: + current = resolve_current_snapshot(pr_url) + except SlackExposureError as exc: + return ReviewOutcome( + ok=False, + state="tool-failure", + lane=lane, + reply_text=render_failure_reply( + pr_url, f"post-review head verification failed: {exc}" + ), + tokens=tokens, + estimated_usd=estimated, + task_id=task_id, + ) + if current.head_sha != reviewed_head: + return ReviewOutcome( + ok=False, + state="stale", + lane=lane, + reply_text=render_stale_reply( + pr_url, + reviewed_head, + current.head_sha, + lane, + task_id, + report_path, + ), + tokens=tokens, + estimated_usd=estimated, + task_id=task_id, + ) + run_state = str(run.get("state") or "unknown") + if run_state not in {"clear", "blocking"}: + return ReviewOutcome( + ok=False, + state=run_state, + lane=lane, + reply_text=render_nonverdict_reply( + pr_url, + run_state, + reviewed_head, + lane, + task_id, + report_path, + run.get("summary"), + ), + tokens=tokens, + estimated_usd=estimated, + task_id=task_id, + ) + try: + reply = render_verdict_reply( + pr_url, run, ledger, lane, report_path, task_id + ) + except SlackExposureError as exc: + return ReviewOutcome( + ok=False, + state="tool-failure", + lane=lane, + reply_text=render_failure_reply(pr_url, str(exc)), + tokens=tokens, + estimated_usd=estimated, + task_id=task_id, + ) return ReviewOutcome( - ok=True, - state=str(run.get("state") or "unknown"), + ok=run_state in {"clear", "blocking"}, + state=run_state, lane=lane, reply_text=reply, tokens=tokens, @@ -1006,11 +2009,13 @@ def run_review(pr_url: str) -> ReviewOutcome: # --- the event-handling core (driven directly by the tests) ------------------------- -def _unwired_head_branch(pr_url: str) -> str: - # Fail closed: a context built without a head-branch resolver must - # refuse to review, never silently skip the agent-branch screen. +def _unwired_pr_snapshot(pr_url: str) -> PrSnapshot: + raise SlackExposureError(f"PR head resolver is not wired; refusing to review {pr_url}") + + +def _unwired_provenance(snapshot: PrSnapshot) -> AuthorshipProvenance: raise SlackExposureError( - f"head-branch resolver is not wired; refusing to review {pr_url}" + f"authorship provenance resolver is not wired for {snapshot.pr_url}" ) @@ -1021,10 +2026,9 @@ class MentionContext: deduper: EventDeduper post: Callable[[str, str, str], None] # channel, thread_ts, text react: Callable[[str, str, str], None] # channel, ts, emoji name - run_review: Callable[[str], ReviewOutcome] - # Returns the PR's head branch name; raises on any failure so the - # agent-branch screen fails closed instead of reviewing unverified. - head_branch: Callable[[str], str] = _unwired_head_branch + run_review: Callable[[str, PrSnapshot, AuthorshipProvenance], ReviewOutcome] + pr_snapshot: Callable[[str], PrSnapshot] = _unwired_pr_snapshot + provenance: Callable[[PrSnapshot], AuthorshipProvenance] = _unwired_provenance log: Callable[[str], None] = log @@ -1058,131 +2062,115 @@ def handle_mention(event_id: str, event: dict[str, Any], ctx: MentionContext) -> # reply on redelivery; a second review is never started. stored = ctx.deduper.undelivered_reply(event_id) if stored is not None: - ctx.post(channel, thread_ts, stored) - ctx.deduper.mark_delivered(event_id) - ctx.log(f"redelivered stored verdict for event {event_id}") - return "redelivered" + return _deliver_final_reply( + ctx, event_id, channel, thread_ts, stored, "redelivered" + ) ctx.log(f"duplicate delivery of event {event_id}; not starting a second review") return "duplicate" + def terminal_reply(action: str, reply: str) -> str: + return _deliver_final_reply( + ctx, event_id, channel, thread_ts, reply, action + ) + if channel not in ctx.config.channel_allowlist: - ctx.post( - channel, - thread_ts, + return terminal_reply( + "channel-refused", "This channel is not enabled for crosscheck reviews. " "Ask the owner to add it to the channel allowlist.", ) - return "channel-refused" links = extract_pr_links(text) if not links: - ctx.post(channel, thread_ts, usage_reply()) - return "no-link" + return terminal_reply("no-link", usage_reply()) if len(links) > 1: - ctx.post( - channel, - thread_ts, + return terminal_reply( + "multiple-links", "One pull-request link per request, please; I received " f"{len(links)}. Mention me once per PR.", ) - return "multiple-links" pr_url = links[0] repo = repo_of(pr_url) if repo not in ctx.config.repo_allowlist: allowed = ", ".join(ctx.config.repo_allowlist) - ctx.post( - channel, - thread_ts, + return terminal_reply( + "repo-refused", f"Refusing to review {pr_url}: repository {repo} is not in the " f"crosscheck repository allowlist ({allowed}). The bot's read " "credential is never pointed at repositories outside the allowlist.", ) - return "repo-refused" - - # Authorship screen: this lane stages human-authored task metadata, so - # agent-authored PRs must be refused here and reviewed through the - # ordinary crosscheck lane, which carries true author metadata. The - # lookup runs only after the allowlist admitted the repository, and any - # lookup failure fails closed. - if ctx.config.agent_branch_prefixes: - try: - branch = ctx.head_branch(pr_url) - except Exception as exc: - ctx.post( - channel, - thread_ts, - f"Refusing to review {pr_url}: the head branch could not be " - f"verified ({escape_slack(clamp(redact(str(exc)), 300))}), and " - "this lane only reviews once its human-authorship screen has " - "run. Retry, or use the ordinary crosscheck lane.", - ) - return "branch-screen-failed" - prefix = matched_agent_prefix(branch, ctx.config.agent_branch_prefixes) - if prefix is not None: - ctx.post( - channel, - thread_ts, - f"Refusing to review {pr_url}: its head branch " - f"{escape_slack(clamp(branch, 120))} matches the agent-branch " - f"prefix {escape_slack(prefix)}. Agent-authored pull requests " - "must go through the ordinary crosscheck lane " - "(bin/fm-crosscheck.sh), which carries true author metadata; " - "the Slack lane asserts human authorship and its " - "model-separation guarantee rests on that assertion.", - ) - return "agent-branch-refused" + + try: + snapshot = ctx.pr_snapshot(pr_url) + except Exception as exc: + return terminal_reply( + "head-refused", + f"Refusing to review {pr_url}: its exact current head could not be " + f"verified ({escape_slack(clamp(redact(str(exc)), 300))}). No review started.", + ) + try: + provenance = ctx.provenance(snapshot) + except Exception as exc: + return terminal_reply( + "provenance-refused", + f"Refusing to review {pr_url} at {snapshot.head_sha}: no trustworthy " + "Firstmate/no-mistakes authorship attestation matches this exact head " + f"({escape_slack(clamp(redact(str(exc)), 300))}). Branch names, Slack " + "identity, and message text cannot assert the author model.", + ) # The request-count cap is the bound that actually binds today. The USD # bound remains a forward contract; observational Crosscheck telemetry # does not activate a spend cap in this change. - cap = ctx.config.daily_request_cap - if cap is not None: - count = ctx.meter.submitter_day_count(user) - if count >= cap: - ctx.post( - channel, - thread_ts, - f"Daily crosscheck request cap reached: {count} of {cap} " - "requests recorded for you today, so this review is not " - "starting. The bound resets at midnight UTC.", - ) - return "cap-refused" - - budget = ctx.config.daily_budget_usd - if budget is not None: - spent = ctx.meter.submitter_day_usd(user) - if spent >= budget: - ctx.post( - channel, - thread_ts, - f"Daily crosscheck budget reached: ${spent:.2f} of ${budget:.2f} " - "recorded for you today, so this review is not starting. " - "The bound resets at midnight UTC.", - ) - return "budget-refused" + request_id, refusal, observed = ctx.meter.begin( + user, + pr_url, + event_id, + request_cap=ctx.config.daily_request_cap, + budget_usd=ctx.config.daily_budget_usd, + ) + if refusal == "cap": + return terminal_reply( + "cap-refused", + f"Daily crosscheck request cap reached: {int(observed)} of " + f"{ctx.config.daily_request_cap} requests recorded for you today, " + "so this review is not starting. The bound resets at midnight UTC.", + ) + if refusal == "budget": + return terminal_reply( + "budget-refused", + f"Daily crosscheck budget reached: ${observed:.2f} of " + f"${ctx.config.daily_budget_usd:.2f} recorded for you today, so this " + "review is not starting. The bound resets at midnight UTC.", + ) + assert request_id is not None try: ctx.react(channel, ts, "hourglass_flowing_sand") except Exception as exc: # a failed reaction never blocks the review ctx.log(f"reaction failed for event {event_id}: {exc}") - ctx.post( - channel, - thread_ts, - f"Review started for {pr_url}. Findings will land in this thread " - "with the reviewing lane named.", - ) - - request_id = ctx.meter.begin(user, pr_url, event_id) try: - outcome = ctx.run_review(pr_url) + ctx.post( + channel, + thread_ts, + f"Review started for {pr_url} at {snapshot.head_sha}. Authorship was " + f"verified from task {provenance.task_id}; findings will land in this thread.", + ) + except Exception as exc: + ctx.log( + f"start acknowledgement failed for event {event_id} " + f"({type(exc).__name__}: {exc}); review continues" + ) + try: + outcome = ctx.run_review(pr_url, snapshot, provenance) except Exception as exc: ctx.meter.finish(request_id, "tool-failure", None, None, None) reply = render_failure_reply(pr_url, f"unexpected {type(exc).__name__}: {exc}") return _deliver_final_reply(ctx, event_id, channel, thread_ts, reply, "failed") ctx.meter.finish( request_id, - outcome.state if outcome.ok else "tool-failure", + outcome.state, outcome.lane, outcome.tokens, outcome.estimated_usd, @@ -1193,6 +2181,33 @@ def handle_mention(event_id: str, event: dict[str, Any], ctx: MentionContext) -> ) +def revalidate_reply(ctx: MentionContext, reply: str) -> str: + lines = reply.splitlines() + if len(lines) < 4 or not lines[0].startswith("Crosscheck "): + return reply + if lines[0].startswith("Crosscheck STALE for "): + return reply + if not lines[3].startswith("Reviewed head: "): + return reply + links = extract_pr_links(lines[0]) + if len(links) != 1 or repo_of(links[0]) not in ctx.config.repo_allowlist: + return render_failure_reply("", "Stored verdict has no allowlisted PR subject") + pr_url = links[0] + reviewed_head = lines[3].removeprefix("Reviewed head: ") + try: + require(SHA_RE.fullmatch(reviewed_head) is not None, "Stored verdict has no exact reviewed head") + current = ctx.pr_snapshot(pr_url) + require(SHA_RE.fullmatch(current.head_sha) is not None, "Live PR lookup returned no exact head") + except Exception as exc: + return render_failure_reply(pr_url, f"Cannot validate verdict head before delivery: {exc}") + if current.head_sha != reviewed_head: + lines[0] = f"Crosscheck STALE for {pr_url}" + lines.insert(4, f"Current head at delivery: {current.head_sha}") + lines.insert(5, "The reviewed verdict does not apply to the current head. Request a new review.") + return clamp("\n".join(lines), MAX_REPLY_CHARS) + return reply + + def _deliver_final_reply( ctx: MentionContext, event_id: str, @@ -1208,6 +2223,7 @@ def _deliver_final_reply( of silently deduping (and never runs a second review). """ + reply = revalidate_reply(ctx, reply) ctx.deduper.store_reply(event_id, reply) try: ctx.post(channel, thread_ts, reply) @@ -1446,15 +2462,15 @@ def close(self) -> None: class SocketModeService: def __init__(self, config: Config) -> None: self.config = config - app_token = required_token(config.app_token_env, "Slack app-level (Socket Mode) token") - bot_token = required_token(config.bot_token_env, "Slack bot token") - github_token = required_token(config.github_token_env, "GitHub read credential") + app_token, bot_token, github_token = configured_credentials(config) + provenance_key = load_provenance_key(config.provenance_key_file) self.web = SlackWebClient(bot_token=bot_token, app_token=app_token) self.config.state_dir.mkdir(parents=True, exist_ok=True) self.meter = DailyMeter(config.state_dir / "meter") self.deduper = EventDeduper(config.state_dir / "events") self.run_review = make_run_review(config, github_token) self._github_token = github_token + self._provenance_key = provenance_key removed = sweep_state(config.state_dir) if removed: log(f"retention sweep removed {len(removed)} aged state file(s) at startup") @@ -1464,7 +2480,14 @@ def __init__(self, config: Config) -> None: self.queue: queue.Queue[tuple[str, dict[str, Any]]] = queue.Queue( maxsize=REVIEW_QUEUE_LIMIT ) - self.worker = threading.Thread(target=self._drain, name="crosscheck-slack-worker", daemon=True) + self.workers = tuple( + threading.Thread( + target=self._drain, + name=f"crosscheck-slack-worker-{index + 1}", + daemon=True, + ) + for index in range(REVIEW_WORKERS) + ) def _sweep_loop(self) -> None: # One retention pass per day (checked every 6 hours), plus the pass @@ -1486,7 +2509,10 @@ def _context(self) -> MentionContext: post=self.web.post_message, react=self.web.add_reaction, run_review=self.run_review, - head_branch=lambda pr_url: fetch_head_branch(pr_url, self._github_token), + pr_snapshot=lambda pr_url: fetch_pr_snapshot(pr_url, self._github_token), + provenance=lambda snapshot: verify_attestation( + self.config, self._provenance_key, snapshot + ), ) def _drain(self) -> None: @@ -1549,7 +2575,8 @@ def _handle_envelope(self, ws: WebSocketClient, raw: bytes) -> None: self._enqueue(event_id, event) def serve_forever(self) -> NoReturn: - self.worker.start() + for worker in self.workers: + worker.start() self._sweeper.start() backoff = 1.0 while True: @@ -1580,6 +2607,7 @@ def serve_forever(self) -> NoReturn: def selftest(config_path: Path) -> int: config = load_config(config_path) + key = load_provenance_key(config.provenance_key_file) lines = [ f"config: {config.path}", "schema: valid", @@ -1591,12 +2619,8 @@ def selftest(config_path: Path) -> int: f"({'set' if os.environ.get(config.github_token_env) else 'UNSET'})", f"channel_allowlist: {len(config.channel_allowlist)} channel(s)", f"repo_allowlist: {', '.join(config.repo_allowlist)}", - "agent_branch_prefixes: " - + ( - ", ".join(config.agent_branch_prefixes) - if config.agent_branch_prefixes - else "(empty: agent-branch screen deliberately disabled)" - ), + f"provenance_key_file: {config.provenance_key_file} (valid, key id {provenance_key_id(key)})", + f"review_workers: {REVIEW_WORKERS} (shared core FIFO lanes)", "daily_request_cap: " + ( f"{config.daily_request_cap} (the binding control today)" @@ -1638,8 +2662,34 @@ def build_parser() -> argparse.ArgumentParser: subparsers = parser.add_subparsers(dest="command", required=True) run = subparsers.add_parser("run", help="start the Socket Mode listener") run.add_argument("--config", default="", help="config path (default: $FM_HOME/config/crosscheck-slack.json)") + run.add_argument("--keychain-only", action="store_true", help="require service-accessible Keychain credentials; ignore token environment") check = subparsers.add_parser("selftest", help="validate config shape and exit") check.add_argument("config", nargs="?", default="", help="config path override") + preflight = subparsers.add_parser( + "preflight", + help="validate config, provenance key, and central credentials", + ) + preflight.add_argument("--config", default="", help="config path override") + preflight.add_argument("--keychain-only", action="store_true", help="require service-accessible Keychain credentials; ignore token environment") + attest = subparsers.add_parser( + "attest-task", + help="sign exact-head authorship from a Firstmate task record", + ) + attest.add_argument("task_id") + attest.add_argument("pr_url") + attest.add_argument("head_sha") + attest.add_argument("--config", default="", help="config path override") + launch = subparsers.add_parser( + "attest-launch", + help="capture a task's author identity before its agent starts", + ) + launch.add_argument("task_id") + launch.add_argument("task_generation") + launch.add_argument("worktree") + launch.add_argument("harness") + launch.add_argument("model") + launch.add_argument("--account-home", default="") + launch.add_argument("--config", default="", help="config path override") return parser @@ -1647,6 +2697,48 @@ def main() -> int: assert_supported_interpreter() args = build_parser().parse_args() try: + if getattr(args, "keychain_only", False): + path = Path(args.config) if args.config else default_config_path() + config = load_config(path) + require( + set(config.keychain_services) == {"app_token", "bot_token", "github_token"}, + "service requires Keychain services for all three credentials", + ) + for name in (config.app_token_env, config.bot_token_env, config.github_token_env): + os.environ.pop(name, None) + if args.command == "attest-launch": + path = Path(args.config) if args.config else default_config_path() + config = load_config(path) + key = load_provenance_key(config.provenance_key_file) + account_home = Path(args.account_home) if args.account_home else None + destination = issue_launch_attestation( + config, + key, + args.task_id, + args.task_generation, + Path(args.worktree), + args.harness, + args.model, + account_home, + ) + print(f"launch-attested: {destination}") + return 0 + if args.command == "attest-task": + path = Path(args.config) if args.config else default_config_path() + config = load_config(path) + key = load_provenance_key(config.provenance_key_file) + destination = issue_task_attestation( + config, key, args.task_id, args.pr_url, args.head_sha + ) + print(f"attested: {destination}") + return 0 + if args.command == "preflight": + path = Path(args.config) if args.config else default_config_path() + config = load_config(path) + load_provenance_key(config.provenance_key_file) + configured_credentials(config) + print("preflight: config, provenance key, and central credentials are ready") + return 0 if args.command == "selftest": path = Path(args.config) if args.config else default_config_path() return selftest(path) @@ -1655,7 +2747,7 @@ def main() -> int: service = SocketModeService(config) log( f"starting Socket Mode listener; repos: {', '.join(config.repo_allowlist)}; " - f"channels: {len(config.channel_allowlist)}; request cap: " + f"channels: {len(config.channel_allowlist)}; workers: {REVIEW_WORKERS}; request cap: " + ( f"{config.daily_request_cap}/submitter/day" if config.daily_request_cap is not None diff --git a/bin/fm-crosscheck-slack.sh b/bin/fm-crosscheck-slack.sh index bf1a6ce083a..41925d12567 100755 --- a/bin/fm-crosscheck-slack.sh +++ b/bin/fm-crosscheck-slack.sh @@ -2,13 +2,15 @@ # Slack Socket Mode exposure of the crosscheck gate for team engineers (R10). # # Usage: -# fm-crosscheck-slack.sh run [--config ] +# fm-crosscheck-slack.sh run [--config ] [--keychain-only] +# fm-crosscheck-slack.sh preflight [--config ] [--keychain-only] # fm-crosscheck-slack.sh --selftest [] +# fm-crosscheck-slack.sh attest-launch [--account-home ] [--config ] +# fm-crosscheck-slack.sh attest-task [--config ] # -# `run` starts the resident Socket Mode listener; it refuses to start when -# any token environment variable named by $FM_HOME/config/crosscheck-slack.json -# is unset, naming the exact variable (the ready-to-flip posture). -# `--selftest` validates the config shape and exits without touching Slack. +# Credential sources and service startup requirements are owned by +# docs/crosscheck-slack.md. `--selftest` validates config and the provenance +# key; `preflight` also checks credential loading, without remote authentication. # # The interpreter floor and the reason for it are owned by # bin/fm-crosscheck-python-lib.sh: this listener parses hostile JSON (Slack diff --git a/bin/fm-crosscheck.py b/bin/fm-crosscheck.py index c4409827e13..17d6f4b681b 100755 --- a/bin/fm-crosscheck.py +++ b/bin/fm-crosscheck.py @@ -1577,7 +1577,7 @@ def parse_meta(path: Path) -> dict[str, str] | None: if not line or "=" not in line: continue key, value = line.split("=", 1) - if key in {"harness", "model"}: + if key in {"harness", "model", "author_account_identity"}: require( key not in result, f"task metadata at {path} duplicates {key} at line {line_number}", @@ -7149,12 +7149,15 @@ def persist_azure_result( snapshot_value=snapshot_value, ledger=ledger, config=config, - # Task metadata carries no upstream authorship - # account record; reviewer independence stays - # structural (the dedicated Crosscheck account - # pool) and the adapter's same-account refusal - # arms once an authorship identity exists. - author_account_identity="", + # Direct tasks may not carry an upstream author + # account identity. Signed Slack provenance does, + # when Firstmate captured one, which arms the + # Azure adapter's same-account refusal. + author_account_identity=( + meta.get("author_account_identity", "") + if meta is not None + else "" + ), # The compartment lane owns create/stage/boot/ # collect; it measures them into this same timer # so one run record carries the whole clock. diff --git a/bin/fm-pr-check.sh b/bin/fm-pr-check.sh index c7ddf8ed179..acfda113993 100755 --- a/bin/fm-pr-check.sh +++ b/bin/fm-pr-check.sh @@ -3,6 +3,9 @@ # state/.meta when available, then arms the watcher's merge poll by writing # state/.check.sh, which prints one line when the PR is merged or its lookup # fails (the watcher's check contract: output = wake, silence = keep sleeping). +# With central Slack config installed, then binds the live PR head to the signed +# launch record created before the task agent started. Issuance failure exits +# nonzero after poll setup. # Usage: fm-pr-check.sh set -eu @@ -99,4 +102,12 @@ chmod +x "$CHECK_TMP" mv "$CHECK_TMP" "$STATE/$ID.check.sh" fm_account_meta_lock_release "$META_LOCK" trap - EXIT +SLACK_CONFIG=${FM_CROSSCHECK_SLACK_CONFIG:-$FM_HOME/config/crosscheck-slack.json} +if [ -f "$SLACK_CONFIG" ]; then + "$FM_ROOT/bin/fm-crosscheck-slack.sh" attest-task \ + "$ID" "$URL" "$PR_HEAD" --config "$SLACK_CONFIG" || { + echo "error: could not issue exact-head Crosscheck authorship provenance for $ID" >&2 + exit 1 + } +fi echo "armed: state/$ID.check.sh polls $URL" diff --git a/bin/fm-spawn.sh b/bin/fm-spawn.sh index 0eb9e910eef..2a3dca316ff 100755 --- a/bin/fm-spawn.sh +++ b/bin/fm-spawn.sh @@ -3849,6 +3849,25 @@ finally: fi PI_AUTHOR_ACCOUNT_HOME= +CLOUD_AUTHOR_ACCOUNT_HOME= + +capture_crosscheck_author_launch() { + local slack_config account_home='' args + [ "$KIND" != secondmate ] || return 0 + [ "$RAW_LAUNCH" != 1 ] || return 0 + slack_config=${FM_CROSSCHECK_SLACK_CONFIG:-$FM_HOME/config/crosscheck-slack.json} + [ -f "$slack_config" ] || return 0 + case "$HARNESS" in + pi) account_home=${CLOUD_AUTHOR_ACCOUNT_HOME:-$PI_AUTHOR_ACCOUNT_HOME} ;; + codex) account_home=${DIRECT_ACCOUNT_HOME:-${CODEX_HOME:-$HOME/.codex}} ;; + esac + args=(attest-launch "$ID" "$SPAWN_GENERATION_ID" "$WT" "$HARNESS" "${MODEL:-default}" --config "$slack_config") + [ -z "$account_home" ] || args+=(--account-home "$account_home") + "$FM_ROOT/bin/fm-crosscheck-slack.sh" "${args[@]}" >/dev/null || { + echo "error: could not capture launch-bound Crosscheck authorship provenance for $ID" >&2 + exit 1 + } +} # prepare_launch_environment: every step the launch-command construction below # The PATH a crewmate's tool commands run with. A harness executes tool commands @@ -4106,6 +4125,9 @@ if [ "$DIRECT_ACCOUNT_ROUTING" = 1 ]; then } fi fi +if [ "$SPAWN_CLOUD" != azure ]; then + capture_crosscheck_author_launch +fi } # build_launch_command: resolve LAUNCH (the full harness launch command) and its @@ -4614,6 +4636,7 @@ spawn_cloud_bind_account_snapshot() { # echo "error: the controller named no assignment-private provider-account home for $ID; refusing to stage a pooled credential" >&2 return 1 } + CLOUD_AUTHOR_ACCOUNT_HOME=$leased [ -d "$leased" ] && [ -f "$leased/auth.json" ] || { echo "error: provider-account snapshot home '$leased' holds no credential for $ID" >&2 return 1 @@ -4930,6 +4953,7 @@ spawn_cloud_dispatch() { echo "error: cloud placement for $ID could not stage its provider-account snapshot" >&2 return 1 } + capture_crosscheck_author_launch if ! spawn_cloud_lifecycle reconcile --apply \ --confirm-subscription "${FM_AZURE_SUBSCRIPTION_ID:-}" --json \ > "$STATE/$ID.worker-reconcile.json" 2>&1; then diff --git a/docs/azure-requirements.md b/docs/azure-requirements.md index a50fe305d93..6e0f1278ea9 100644 --- a/docs/azure-requirements.md +++ b/docs/azure-requirements.md @@ -667,10 +667,10 @@ This is deliberately weaker than the original sentence, and it is weaker in the available: the original was never provable, so leaving it in place would have meant marking R6 DONE on an assertion. -Making authorship genuinely recordable is the alternative, and it is a real change rather than a -doc edit: it needs an authorship identity captured at task creation and carried into the ledger, -which is the same `author_account_identity` field the Azure adapter's same-account refusal is -already waiting on. Worth doing, out of scope here, and the amendment above does not depend on it. +Making authorship genuinely recordable was the alternative, and at the time of this amendment it +was out of scope. Resumed R10 later added a signed pre-agent launch record plus an exact-head +binding for centrally configured Firstmate tasks. That newer producer does not retroactively make +the historical R6 declaration runs into authorship evidence. **Declaring a codex author is the SAFE error; declaring a claude author is the dangerous one.** A codex declaration can only narrow eligibility: against a codex-family reviewer it forces the @@ -965,8 +965,9 @@ An accepted Pi-harness review on the current model image is proven by Azure revi The former outstanding dependency is now complete: R4 cell `azv-c1bb1c5ff906` reached `close` with its worktree disk released, as recorded in [`docs/evidence/azure-r4-live-acceptance-2026-08-23/evidence.json`](evidence/azure-r4-live-acceptance-2026-08-23/evidence.json). -R6's second declaration leg was already complete, so the tracked evidence now covers every -non-dropped R1-R10 requirement. +R6's second declaration leg was already complete, so that tracked evidence closes the previously +accepted Azure core through R9. +Resumed R10 has its own Slack activation and live-team acceptance below. The landing path used to be recorded here as unprovable synthetically, and that was correct behavior rather than a gap to route around. @@ -980,60 +981,42 @@ did. ## R10. Crosscheck is exposed to team engineers through Slack -Status: DROPPED by the owner, 2026-08-21. -The build (the Slack lane, `bin/fm-crosscheck-slack.sh`) remains in-tree, inert without its -tokens; no acceptance is owed. -Everything below is retained as history of what was built and what turning it on would have -taken, and none of it is outstanding work. - -Where the lane stood before it was dropped: BUILT, ready-to-flip; awaiting the three owner inputs -(the Slack app's two tokens, the GitHub read credential, and the metering numbers). -Directed by the owner 2026-08-19; builds after R6. -The lane is `bin/fm-crosscheck-slack.sh` and its owning document is `docs/crosscheck-slack.md`; a -missing token environment variable refuses startup naming the exact variable, so flipping it on is -supplying tokens, not changing code. -Live acceptance would still have required those inputs and an engineer other than the owner. - -What DK would have had to click: create a Slack app in the workspace with Socket Mode on; -mint the app-level token with `connections:write` and the bot token with -`app_mentions:read`, `chat:write`, `channels:history`, and `reactions:write`; -subscribe it to the `app_mention` event, install it, and invite the bot to the -channel(s) going into `channel_allowlist`; mint a read-only GitHub credential -scoped to the allowlisted repositories; export the three values under the -environment variable names in `$FM_HOME/config/crosscheck-slack.json`; and -pick the two metering numbers: `daily_request_cap` (the control that binds -today, counting each submitter's started reviews per day) and -`daily_budget_usd` (the USD bound, which binds only once the crosscheck -ledger records per-review cost; today it does not). Null for either stays -unmetered pass-through, still ledgered. Details and the run recipe: -`docs/crosscheck-slack.md`. - -The owner's former v1 shape: an engineer tags the crosscheck bot in a Slack channel with a pull -request link; the GLM lane reviews it; the bot posts the findings as a thread reply on the -engineer's own message. No engineer wires up a harness or touches an endpoint, and the same path -works for deliberate on-demand use. Cursor Bugbot continues to run for engineers' pull requests -(it stays disabled on the owner's), so this lane complements rather than replaces it. - -Constraints the built lane was intended to honor: - -- The listener uses Slack Socket Mode, so no public inbound endpoint is added to the private - lane posture. It is a resident process; where it runs is decided at build time and its - standing cost is recorded under C3. -- v1 accepts pull request links only, and only for repositories in the organization allowlist. - The bot's repository read credential must never be pointed at a repository outside that - allowlist, because a review pulls untrusted content into a credentialed context. -- Every thread reply names the lane that produced it (GLM, or the pi-codex fallback), the same - visibility R6 requires, so engineers and the owner can always see what is serving. -- The intended design called for per-submitter metering under a daily cost bound (C3), with a - refusal in the thread instead of a silently dropped request. The in-tree build binds the daily - request cap, but its USD bound never became effective because the crosscheck ledger records no - per-review cost. That unfinished control is retained as history, not work still owed after the - drop. - -Acceptance, as it stood before the drop and no longer owed: an engineer other than the owner tags -the bot with a pull request link and receives threaded findings produced by the GLM lane, with the -lane named in the reply, the request metered, and an out-of-allowlist link refused with a clear -message. +Status: DEFERRED 2026-08-21, RESUMED 2026-08-26; implementation complete, central activation and live-team acceptance pending. + +Historical explanation: the owner dropped R10 from the August 21 critical path only to ship the +core Crosscheck lane sooner. +The original listener remained in-tree and inert without credentials. +That deferral was not a rejection of Slack access, and this resumed work preserves that history. + +The resumed lane is owned by `docs/crosscheck-slack.md`. +It keeps outbound-only Slack Socket Mode and the existing direct CLI, while four listener workers +enter the same central four-lane FIFO allocator as direct requests. +Exact channel and repository allowlists, durable event dedupe, atomic per-engineer daily caps, +visible saturation, central reports, and a launchd restart owner remain required. + +The original build's branch-prefix authorship screen and unconditional `model=human-authored` +staging are retired as unsafe. +For managed agent work, the resumed lane requires a signed Firstmate launch attestation created +before the agent starts and a second signed record that binds the same worktree, harness, exact +model, model family, account identity, task identity, and generation to the exact PR head. +Mutable task metadata cannot establish that provenance by itself. +Slack submitter identity, branch names, and free text carry no authorship authority. +Missing, conflicting, or unverifiable provenance fails closed in the request thread. +Human authorship is accepted only from a separate trustworthy exact-head producer; until a +no-mistakes human producer exists, such requests remain unclassified and fail closed rather than +being guessed from commit or PR metadata. + +Activation still requires the Slack app's two credential values, one repository-scoped read-only +GitHub credential, exact approved channel IDs, exact approved repositories, and a binding daily +request cap on the coordinator. +Credential names and locations are recorded in `docs/crosscheck-slack.md`; values never enter this +document, Slack replies, logs, state, or artifacts. + +Acceptance: an internal engineer other than the owner tags the bot in an approved channel with one +allowlisted PR URL and receives an admitted exact-head CLEAR or findings reply in the same thread. +The reply names the reviewed SHA, reviewer lane, task ID, and durable artifact; the meter records +the engineer; duplicate delivery starts exactly one review; head movement invalidates the verdict; +out-of-allowlist and unverified-provenance requests refuse; infrastructure failure never reads CLEAR. ## C1. Crosscheck completes in 20 to 30 minutes @@ -1164,11 +1147,11 @@ The adjacent cooldown gap is also closed: a release-proved worker whose VM was d operator-side (surrender's dark-compute gate) now gets `cooldown_started_at` stamped on first observation, so `delete-compute` becomes due instead of waiting forever. -Dropped R10's intended metering and standing-cost booking: the inert Slack lane was designed to -run its listener on the operator Mac, with approximately zero standing Azure cost. Its -per-submitter daily request ledger remains in-tree under `$FM_HOME/state/crosscheck-slack`; -`daily_request_cap` is implemented, while `daily_budget_usd` would bind only if that ledger ever -recorded per-review cost, which it does not. No listener or acceptance is owed after the drop. +Resumed R10 runs its outbound Socket Mode listener on the coordinator Mac with approximately zero +standing Azure cost. +Its per-submitter daily request ledger lives under `$FM_HOME/state/crosscheck-slack`. +`daily_request_cap` is binding and atomically enforced across concurrent workers. +`daily_budget_usd` remains optional and binds only when a review exposes compatible cost data. Acceptance: a day cannot cross the bound without an explicit operator override, and a worker whose task ended deallocates unattended. @@ -1197,7 +1180,8 @@ not a claim of real-time metering. 7. C1, instrumented 2026-08-20; the measured local-lane runs are above the band rather than inside it, and the compartment lane's phases wait on that lane being switched back on. 8. R9, done 2026-08-23 after R4's final cell close completed the tracked proof set. -9. R10, the Slack team exposure, dropped by the owner on 2026-08-21 and no longer ordered work. +9. R10, deferred on 2026-08-21 to ship core Crosscheck sooner and resumed on 2026-08-26 for + central Slack activation and live team acceptance. ## Standing constraints diff --git a/docs/crosscheck-slack.md b/docs/crosscheck-slack.md index 0978b9e4a8c..a018469d221 100644 --- a/docs/crosscheck-slack.md +++ b/docs/crosscheck-slack.md @@ -1,30 +1,114 @@ -# Crosscheck over Slack (R10) - -This document owns the Slack team exposure of the crosscheck gate: the -listener `bin/fm-crosscheck-slack.py` (launched through -`bin/fm-crosscheck-slack.sh`), its configuration, its metering ledger, and -the operator recipe. The review itself is owned by `bin/fm-crosscheck.py` -and `docs/crosscheck.md`; the requirement text is R10 in -`docs/azure-requirements.md`. - -## What it does - -An engineer tags the crosscheck bot in an allowlisted Slack channel with a -GitHub pull-request link. The bot validates the repository against the -allowlist, checks the submitter's daily meter, acks in thread ("Review -started"), runs `bin/fm-crosscheck.sh run ` as a bounded -subprocess, and posts the findings as a thread reply on the engineer's own -message, naming the lane that produced the review (the cross-family primary deployment or -"pi-codex fallback (degraded)"). Tool failures are posted honestly as -failures, never as verdicts. Cursor Bugbot continues to run for engineers' -pull requests; this lane complements it. - -The listener uses Slack Socket Mode over an outbound websocket, so no -public inbound endpoint is added to the private lane posture. The websocket -client is a minimal RFC 6455 implementation over the standard library; the -repo carries no third-party Python dependencies and none was added. - -## Configuration: `$FM_HOME/config/crosscheck-slack.json` +# Crosscheck for Slack + +This is the centrally operated R10 access lane for internal engineers. + +An engineer needs only Slack. +In an approved channel, tag the Crosscheck bot with exactly one GitHub pull request URL. +The bot acknowledges the exact head it admitted and posts CLEAR, BLOCKING findings, STALE, or TOOL FAILURE in the same thread. +Every admitted result names the reviewed head SHA, reviewer lane, Crosscheck task ID, and exact durable report path on the coordinator. + +The direct command remains supported for Firstmate and operators: + +```sh +FM_HOME=/Users/dongkeun/firstmate-home bin/fm-crosscheck.sh run +``` + +The Slack lane is an access adapter around that command. +It does not implement a second reviewer, lane allocator, or evidence policy. +Do not create a Crosscheck agent skill for this lane. + +## Engineer request + +In an approved Slack channel: + +```text +@Crosscheck https://github.com/ORG/REPOSITORY/pull/123 +``` + +Use one pull request URL per mention. +No Firstmate checkout, `FM_HOME`, Azure configuration, GitHub login, or provider credential is needed on the engineer's machine. + +The coordinator owns Slack Socket Mode, the exact channel and repository allowlists, the GitHub read credential, signed authorship provenance, the reviewer roster, Azure access, queueing, metering, reports, and restarts. + +## Network and capacity shape + +The listener uses Slack Socket Mode over an outbound websocket. +It opens no public inbound endpoint. + +Four listener workers admit requests concurrently. +Each worker invokes the supported `fm-crosscheck.sh run` wrapper with the same central `FM_HOME` used by direct CLI callers. +Both entry paths therefore enter Crosscheck's existing four-lane durable FIFO allocator. +The Slack adapter has a bounded eight-request waiting queue in front of those workers. +When that queue is full, the bot posts a visible refusal in the request thread. + +The listener never translates a queue, GitHub, reviewer, Azure, cleanup, ledger, or Slack delivery failure into CLEAR. + +## Authorship provenance + +The old R10 build inferred authorship from branch prefixes and staged every request as `model=human-authored`. +That was unsafe and is retired. + +The active Firstmate producer uses two coordinator-signed records. +`fm-spawn` writes `firstmate.crosscheck-author-launch.v1` before the agent process starts. +That immutable launch record binds: + +- Originating Firstmate task ID and task generation. +- Exact task worktree, Git directory identity, branch ref, and launch head. +- Author harness, exact model, model family, and required captured account identity for OpenAI-family agents. + +`fm-pr-check` later writes `firstmate.crosscheck-authorship.v2` only when the same worktree and Git identity remain, the current head descends from the launch head, tracked files are clean, and current HEAD equals the live PR head. +That exact-head record binds: + +- Exact repository and pull request number. +- Exact 40-character head SHA. +- The complete launch-bound author identity. +- The SHA-256 digest of the signed launch record. + +Both records authenticate their schema and payload with the coordinator provenance key. +Mutable task metadata can only agree with the signed launch identity; it cannot replace it. +A head produced in another worktree, a changed branch identity, or uncommitted tracked source is refused. + +`bin/fm-spawn.sh` creates the launch record automatically for a managed task when central Slack configuration is installed, and `bin/fm-pr-check.sh` emits the exact-head record after it resolves the live PR head. +The exact-head command below is a pipeline interface, not a way for a caller to claim a model: + +```sh +bin/fm-crosscheck-slack.sh attest-task --config +``` + +The Slack listener fetches the live PR head with its read-only GitHub credential, verifies the exact-head signature, recomputes the model family through Crosscheck's own classifier, and stages the verified harness and model for the core family-separation screen. +Both signed attestations are copied into the review's durable artifact directory. +Agent attestations require a captured author account identity, which also arms the Azure adapter's same-account refusal. + +Slack identity, branch names, PR text, and caller-supplied free text carry no authorship authority. +Missing, malformed, tampered, conflicting, or wrong-head provenance gets a clear threaded refusal and starts no review. +Firstmate launch records deliberately cannot claim human authorship. +A human-authored PR may be classified as human only after a separate trusted producer, such as no-mistakes, supplies verifiable exact-head evidence from its own author boundary. +No such human producer is inferred from commit metadata, Slack identity, an unsigned PR-body marker, or absence of a Firstmate record; without it, the request fails closed as unclassified. + +## Exact-head response contract + +The listener resolves the current PR head before admission and binds provenance to it. +After Crosscheck returns, it requires the ledger's reviewed head to equal the admitted head. +It then fetches the live head again. + +If the head moved during review, the bot posts STALE with both SHAs, the lane, task ID, and durable artifact. +The older verdict does not apply to the new head and the reply never says CLEAR. + +An admitted response has this shape: + +```text +Crosscheck CLEAR for https://github.com/ORG/REPOSITORY/pull/123 +Lane: glm-5p2 primary +Task ID: slack-0123456789ab +Reviewed head: 0123456789abcdef0123456789abcdef01234567 +Summary: ... +No active findings for this head. +Durable artifact: /coordinator/fm-home/data/slack-0123456789ab/crosscheck.md +``` + +## Central configuration + +The default path is `$FM_HOME/config/crosscheck-slack.json`. ```json { @@ -33,173 +117,117 @@ repo carries no third-party Python dependencies and none was added. "channel_allowlist": ["C0123ABCDEF"], "repo_allowlist": ["Ruby-Labs/relvino"], "github_token_env": "FM_GITHUB_READ_TOKEN", + "keychain_services": { + "app_token": "firstmate-crosscheck-slack-app", + "bot_token": "firstmate-crosscheck-slack-bot", + "github_token": "firstmate-crosscheck-github-read" + }, "daily_budget_usd": null, - "daily_request_cap": null, - "agent_branch_prefixes": ["fm/"], + "daily_request_cap": 10, + "provenance_key_file": "$FM_HOME/config/crosscheck-slack-provenance.key", "state_dir": "$FM_HOME/state/crosscheck-slack" } ``` -- Tokens come ONLY from the environment variables named here. They are - never stored in this file, never written to state, and never logged; - every emitted line passes a redactor that knows every resolved secret. -- A missing token environment variable refuses startup with an exact - message naming the variable. That is the ready-to-flip posture: the - config and code land first, the owner supplies tokens later, nothing - else changes. -- `repo_allowlist` is exact `owner/name` matching (case-insensitive). The - bot's repository read credential is never pointed at a repository outside - this list, because a review pulls untrusted content into a credentialed - context. Out-of-allowlist links get a threaded refusal naming the - repository and the allowlist. -- `channel_allowlist` bounds where the bot works; mentions elsewhere get a - threaded "not enabled" refusal and no review. -- `daily_request_cap` is the metering control that BINDS TODAY: a - per-submitter, per-UTC-day cap on started reviews, needing no cost data. - At the cap the bot says so in the thread instead of silently dropping the - request. Null = uncapped, still ledgered. -- `daily_budget_usd` is a forward contract, stated plainly: it binds only - once the crosscheck ledger records per-review cost, which today's ledger - schema does not, so a submitter's recorded day total stays 0.0 and this - bound cannot fire until that lands. When cost data exists, a submitter at - the bound gets the same in-thread reply. Null = unmetered pass-through, - still ledgered. -- `agent_branch_prefixes` (optional; default `["fm/"]`) is the - human-authorship screen described below. An empty array deliberately - disables it; do that only if agent work never reaches the allowlisted - repositories. -- `state_dir` supports a literal leading `$FM_HOME` and nothing else. - -## THE AUTHORSHIP ASSERTION (read this before pointing the lane anywhere) - -This lane stages every review's task metadata as `model=human-authored`, -which satisfies the crosscheck gate's model-separation screen for EVERY -reviewer. That is only sound because the lane asserts human authorship: -submissions come from engineers in Slack, and before any review the bot -fetches the PR's head branch (with the same read credential, only after -the repository allowlist admitted the URL) and REFUSES in thread any PR -whose branch matches an `agent_branch_prefixes` entry, directing it to the -ordinary crosscheck lane (`bin/fm-crosscheck.sh`), which carries true -author metadata. A branch-lookup failure also refuses (fail closed). The -model-separation guarantee for Slack reviews therefore rests on this -assertion plus the branch screen, not on the gate's own screen; the lane -must never be pointed at agent-authored pull requests, and an -agent-authored PR on a non-agent-prefixed branch is outside what this -screen can catch, which is exactly why the refusal message names the -ordinary lane. - -`bin/fm-crosscheck-slack.sh --selftest [config-path]` validates the config -shape (and reports which token variables are set, values never shown) and -exits without touching Slack. - -## The three owner inputs - -The lane is built and tested; it goes live when the owner supplies exactly -these three things. - -1. **Slack app (Socket Mode) and its two tokens.** Create a Slack app in - the workspace; enable Socket Mode; create an app-level token with the - `connections:write` scope (this is `app_token_env`, an `xapp-...` - value). Under OAuth, grant the bot token scopes `app_mentions:read`, - `chat:write`, `channels:history`, and `reactions:write`; subscribe the - app to the `app_mention` bot event; install the app to the workspace - (the `xoxb-...` value is `bot_token_env`); invite the bot to each - allowlisted channel. -2. **The GitHub organization read credential**, exported as the variable - named by `github_token_env`. A fine-grained PAT with read-only access to - exactly the allowlisted repositories is the right shape; the listener - hands it to the crosscheck subprocess (also as `GH_TOKEN`) and to - nothing else. -3. **The two metering numbers**, both DK inputs recorded under C3: - `daily_request_cap` (the control that binds today; a per-submitter daily - count of started reviews) and `daily_budget_usd` (the USD bound, which - binds only once the crosscheck ledger records per-review cost). Until - they are set the lane runs unmetered pass-through with full ledgering. - -## Run recipe +`channel_allowlist` contains exact Slack channel IDs. +`repo_allowlist` contains exact case-insensitive `owner/name` repositories. +The GitHub credential is never used before the repository allowlist admits the URL. + +`daily_request_cap` is a binding per-engineer, per-UTC-day cap on started reviews. +Admission and ledger append occur under one lock, so concurrent workers cannot overrun it. +`daily_budget_usd` remains optional and binds only to cost values the Crosscheck ledger actually exposes. +Null disables that bound without disabling request logging. + +The three environment variables remain supported for foreground operation. +The central macOS service requires all three `keychain_services`; environment-only credentials are supported only for foreground operation. +Only Keychain service names appear in config or launchd state. +Credential values never appear there. + +The provenance key is a 32-byte random key encoded as 64 lowercase hex characters in an owner-only regular file. +It must not be a symlink and must have no group or other permission bits. +The listener logs only its nonsecret key ID. + +## Credentials and app permissions + +The Slack app must have Socket Mode enabled. +Its app-level token needs `connections:write`. +Its bot token needs `app_mentions:read`, `chat:write`, `channels:history`, and `reactions:write`. +Subscribe the app to `app_mention`, install it to the workspace, and invite it only to approved channels. + +The GitHub credential must be a read-only GitHub App installation token or fine-grained credential limited to the repositories in `repo_allowlist`. +It needs pull request metadata and repository contents read access. +It needs no write scope. + +Slack, GitHub, provider, and Azure credential values stay only on the coordinator host. +The Crosscheck child receives only the GitHub read credential and required `FM_*` runtime configuration. +Slack credentials never enter the child environment. +Every process log and Slack error path passes through the registered secret redactor. + +## Durable state and metering + +Slack event markers live under `/events`. +The first process to create an event claim owns it, so concurrent duplicate delivery starts exactly one review. +The final reply is stored before posting and marked delivered afterward. +If Slack delivery fails, a redelivery revalidates the stored reviewed head before posting without rerunning the review. +A moved head produces STALE, and an unavailable head lookup produces a tool failure. + +Per-engineer request records live under `/meter/.json`. +Each record includes the Slack user ID, PR URL, event ID, start and finish times, state, lane, available token data, and available cost data. +Request records stay bound to the UTC day on which they started, including reviews that cross midnight. + +Signed launch records live under `/launch-provenance`, and exact-head records live under `/provenance`. +Review reports and both copied attestations live under `$FM_HOME/data/`. + +Event artifacts expire after 14 days and meter files after 90 days. +Review artifacts follow the central Crosscheck retention owner. + +## Install, restart, and inspect + +Validate central configuration and the provenance key without contacting Slack: + +```sh +FM_HOME=/Users/dongkeun/firstmate-home \ + bin/fm-crosscheck-slack.sh --selftest +``` + +Check that all three central credentials can be loaded without printing them (this does not authenticate against Slack or GitHub): + +```sh +FM_HOME=/Users/dongkeun/firstmate-home \ + bin/fm-crosscheck-slack.sh preflight --keychain-only +``` + +Install the macOS launch agent without starting an uncredentialed listener: + +```sh +FM_HOME=/Users/dongkeun/firstmate-home \ + bin/fm-crosscheck-slack-service.sh install +``` + +Operate it centrally: ```sh -export FM_SLACK_APP_TOKEN=... # from owner input 1 -export FM_SLACK_BOT_TOKEN=... # from owner input 1 -export FM_GITHUB_READ_TOKEN=... # from owner input 2 -bin/fm-crosscheck-slack.sh --selftest # config shape check -bin/fm-crosscheck-slack.sh run # resident listener +FM_HOME=/Users/dongkeun/firstmate-home bin/fm-crosscheck-slack-service.sh start +FM_HOME=/Users/dongkeun/firstmate-home bin/fm-crosscheck-slack-service.sh status +FM_HOME=/Users/dongkeun/firstmate-home bin/fm-crosscheck-slack-service.sh restart +FM_HOME=/Users/dongkeun/firstmate-home bin/fm-crosscheck-slack-service.sh stop ``` -Where it runs is an operator decision recorded under C3. The v1 -recommendation is the always-on local mac that already hosts the fleet: -the listener is a single low-CPU resident process whose standing cost is -the machine staying awake plus one Socket Mode connection, and colocating -it with `$FM_HOME` gives it the crosscheck gate, the reviewer roster, and -the state directory with no new credential distribution. Moving it to a -cloud VM later changes only where the three environment variables live; -that standing cost, when chosen, is booked under C3 alongside the -per-submitter metering this listener already records. - -Operational bounds: reviews run one at a time off a bounded queue (depth -8; overflow gets a threaded refusal), each review subprocess has a wall -clock bound (default 5400 seconds, `FM_CROSSCHECK_SLACK_REVIEW_TIMEOUT_SECONDS` -to override) and an output ceiling, and the websocket reconnects with -capped backoff. - -## Metering ledger (the C3 hook) - -`/meter/.json` records every request: submitter, PR -URL, event id, start/finish timestamps, outcome status, the lane that -served it, token usage when the crosscheck output exposes it, and -estimated USD when derivable, else null. Writes are atomic under an -advisory lock. Honest limit, stated plainly: today's crosscheck ledger -schema does not record token usage or cost, so `estimated_usd` stays null, -recorded spend stays 0.0, and the USD bound cannot bind until the lane -records cost; `daily_request_cap` is the binding control today. The -per-request records are already durable, so C3 can attach a per-review -price or a usage-derived cost without a schema change here. A request is -metered against the UTC day it STARTED: the request id encodes the origin -day and completion finalizes the origin day's file, so a review crossing -midnight neither orphans its start record nor lands a misattributed -completion in the next day's ledger. - -Event dedupe markers live in `/events/`; a redelivered Slack -event id never starts a second review. The rendered final reply is stored -durably BEFORE it is posted and marked delivered after the post succeeds, -so a redelivery of an event whose verdict was produced but never delivered -re-posts the stored verdict instead of being silently dropped. Retention: -a sweep at startup and daily removes event artifacts older than 14 days -and meter day-files older than 90 days; fresh files are never touched. - -## Lane naming - -Every thread reply names the lane that produced the review, the same -visibility R6 requires. The bot reads the `reviewer` object of the latest -ledger run: an explicit `lane` marker (with its `degraded` flag) is passed -through verbatim when present; when the run predates that marker, the lane -is derived from the reviewer profile that ran: a GLM model names -"GLM-5.2 primary" and a pi/codex profile names "pi-codex fallback -(degraded)". Coupling note: the explicit marker is being added by the R6 -GLM roster work; once that lands, the derived path only serves ledgers -written before it. - -## Prompt-injection posture - -Slack text and PR content are data, never instructions. The only value the -bot extracts from a mention is one pull-request URL, validated against the -allowlist before any credentialed tool sees it; v1 accepts pull-request -links only, one per request. Nothing from Slack or from the PR is executed; -replies render findings as escaped plain text. The crosscheck subprocess -runs with a scrubbed environment that carries the GitHub read credential -and the FM_* configuration and never the Slack tokens. Task metadata is -staged as `harness=slack-team`, `model=human-authored`; that staging is -sound only under the authorship assertion and branch screen above. -Replies are posted with Slack `mrkdwn` disabled, so hostile finding text -cannot render fake emphasis, code spans, or links. - -## Tests - -`tests/fm-crosscheck-slack.test.sh` (hermetic) drives the real -event-handling core with parsed events and a fixture crosscheck binary: -link extraction, allowlist refusal text, durable dedupe, meter -accumulation and the bound-reached reply, lane naming passthrough, the -missing-token startup refusal, and a whole-artifact grep proving no token -value reaches any log or ledger. Live Slack is never contacted; live -acceptance (an engineer other than the owner, per R10) waits on the three -owner inputs. +The launch agent is `~/Library/LaunchAgents/com.firstmate.crosscheck-slack.plist`. +Logs are `$FM_HOME/logs/crosscheck-slack.log` and `$FM_HOME/logs/crosscheck-slack.error.log`. +The launch agent contains no credential values. +The launch agent persists the resolved absolute Python interpreter, executable PATH, HOME, and service configuration. +Install and start execute selftest and credential preflight with exactly the emitted environment and require Keychain access, ignoring inherited token variables. +Installation refuses before replacing an existing plist if validation fails. +The listener also requires Keychain credentials on every launch, including launchd restarts. +Reinstall the launch agent after moving the interpreter or changing tool locations. + +## Activation and live acceptance + +Before activation, supply the coordinator inputs in [Central configuration](#central-configuration) and satisfy [Credentials and app permissions](#credentials-and-app-permissions). +Install the central configuration before spawning an agent whose PR must be reviewable, because provenance begins at agent launch and cannot be reconstructed later. +The launchd credential checks and lifecycle are owned by [Install, restart, and inspect](#install-restart-and-inspect). + +The live acceptance request must come from an internal engineer other than Dongkeun in an approved channel. +Record the request thread, exact admitted and returned SHA, provenance task, reviewer lane, Slack task ID, durable artifact, meter row, dedupe result, and service status in the R10 evidence directory. +Never copy credential values into that evidence. diff --git a/docs/scripts.md b/docs/scripts.md index a260273da41..61fb21e4b6d 100644 --- a/docs/scripts.md +++ b/docs/scripts.md @@ -103,8 +103,9 @@ The shared no-mistakes gate refusal used by every directly invocable mutating co | `fm-github-pr.py` | Adapt observed gh-axi TOON PR reads through a fail-closed read-only CLI | | `fm-crosscheck.sh` | Run, verify, time, or atomically merge through the durable exact-head PR finding ledger | | `fm-crosscheck.py` | Validate and execute crosscheck reviewer evidence and finding lifecycle transitions | -| `fm-crosscheck-slack.sh` | Launch or selftest the Slack Socket Mode crosscheck exposure (docs/crosscheck-slack.md) | -| `fm-crosscheck-slack.py` | Listen for Slack mentions and run allowlisted, metered, lane-named crosscheck reviews | +| `fm-crosscheck-slack.sh` | Run, preflight, selftest, or issue exact-head provenance for the Slack Crosscheck lane | +| `fm-crosscheck-slack.py` | Serve allowlisted, metered, exact-head Slack reviews through the shared core lanes | +| `fm-crosscheck-slack-service.sh` | Install and operate the credential-free macOS launchd wrapper for the central listener | | `fm-pr-check.sh` | Record `pr=` and `pr_head=` for a PR-ready task, then arm the watcher's merge poll | | `fm-pr-merge.sh` | Require exact-head crosscheck, record PR metadata, and atomically merge or enqueue the reviewed SHA | | `fm-promote.sh` | Promote a scout task in place to a protected ship task | diff --git a/tests/fm-crosscheck-slack.test.sh b/tests/fm-crosscheck-slack.test.sh index 99aeb2629b0..9fd44be0868 100755 --- a/tests/fm-crosscheck-slack.test.sh +++ b/tests/fm-crosscheck-slack.test.sh @@ -21,6 +21,7 @@ PYTHON="$(fm_crosscheck_resolve_python)" || fail "no supported python interprete BOT_PY="$ROOT/bin/fm-crosscheck-slack.py" BOT_SH="$ROOT/bin/fm-crosscheck-slack.sh" +SERVICE_SH="$ROOT/bin/fm-crosscheck-slack-service.sh" fm_test_tmproot_into TMP_ROOT fm-crosscheck-slack-tests @@ -41,6 +42,10 @@ mkdir -p "$HOMEDIR/config" "$HOMEDIR/state" "$HOMEDIR/data" "$OUTDIR" APP_TOKEN='xapp-1-SECRETAPP-cafef00dcafef00d' BOT_TOKEN='xoxb-SECRETBOT-deadbeefdeadbeef' GH_TOKEN_VALUE='ghp_SECRETGITHUB0123456789abcdef' +PROVENANCE_KEY_FILE="$HOMEDIR/config/crosscheck-slack-provenance.key" +printf '%s\n' '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef' \ + > "$PROVENANCE_KEY_FILE" +chmod 600 "$PROVENANCE_KEY_FILE" CONFIG_MAIN="$TMP_ROOT/config-main.json" CONFIG_BUDGET="$TMP_ROOT/config-budget.json" @@ -57,6 +62,7 @@ write_config() { # "github_token_env": "FM_TEST_GITHUB_READ_TOKEN", "daily_budget_usd": $2, "daily_request_cap": $3, + "provenance_key_file": "\$FM_HOME/config/crosscheck-slack-provenance.key", "state_dir": "\$FM_HOME/state/crosscheck-slack" } JSON @@ -81,7 +87,7 @@ fi task=$2 url=$3 meta="$FM_HOME/state/$task.meta" -if ! grep -q '^harness=slack-team$' "$meta" || ! grep -q '^model=human-authored$' "$meta"; then +if ! grep -q '^harness=pi$' "$meta" || ! grep -q '^model=openai-codex-8/gpt-5.6-sol$' "$meta"; then echo "fixture: task metadata missing or wrong at $meta" >&2 exit 67 fi @@ -107,7 +113,7 @@ cat > "$dest/crosscheck-ledger.json" < FM_FIXTURE_REVIEWER_JSON="${FM_FIXTURE_REVIEWER_JSON:-}" \ FM_FIXTURE_USAGE_JSON="${FM_FIXTURE_USAGE_JSON:-}" \ FM_FIXTURE_FINDINGS_JSON="${FM_FIXTURE_FINDINGS_JSON:-}" \ + FM_FIXTURE_HEAD_SHA="${FMT_HEAD_SHA:-1111111111111111111111111111111111111111}" \ FMT_BOT_PY="$BOT_PY" \ FMT_POSTS="$POSTS" \ FMT_REACTS="$REACTS" \ FMT_EVENT_ID="$1" \ - FMT_HEAD_BRANCH="${FMT_HEAD_BRANCH:-}" \ + FMT_HEAD_SHA="${FMT_HEAD_SHA:-}" \ + FMT_HEAD_AFTER="${FMT_HEAD_AFTER:-}" \ + FMT_PROVENANCE_MODE="${FMT_PROVENANCE_MODE:-}" \ + FMT_SOURCE_TASK="${FMT_SOURCE_TASK:-}" \ + FMT_SOURCE_HARNESS="${FMT_SOURCE_HARNESS:-}" \ + FMT_SOURCE_MODEL="${FMT_SOURCE_MODEL:-}" \ + FMT_SOURCE_GENERATION="${FMT_SOURCE_GENERATION:-}" \ FMT_FAIL_VERDICT_POST="${FMT_FAIL_VERDICT_POST:-}" \ "$PYTHON" "$DRIVER" mention "$3" > "$out" 2>&1 local code=$? @@ -397,8 +553,10 @@ test_selftest_validates_config_shape() { "selftest did not explain the null budget" assert_contains "$output" "daily_request_cap: null (uncapped" \ "selftest did not report the null request cap" - assert_contains "$output" "agent_branch_prefixes: fm/" \ - "selftest did not report the default agent-branch screen" + assert_contains "$output" "provenance_key_file:" \ + "selftest did not validate the provenance key" + assert_contains "$output" "review_workers: 4" \ + "selftest did not report four shared-capacity workers" bad="$TMP_ROOT/config-bad.json" "$PYTHON" -c 'import json, sys @@ -456,6 +614,77 @@ test_missing_token_env_refuses_start() { pass "a missing token environment variable refuses startup naming the exact variable" } +test_attestation_cli_derives_and_signs_author_identity() { + agent_task=attest-agent-task + agent_pr=https://github.com/Ruby-Labs/goodrepo/pull/77 + agent_worktree="$TMP_ROOT/attest-agent-worktree" + agent_account="$TMP_ROOT/attest-agent-account" + mkdir -p "$agent_worktree" "$agent_account" + git -C "$agent_worktree" init -q + git -C "$agent_worktree" config user.email fixture@example.com + git -C "$agent_worktree" config user.name Fixture + printf 'launch\n' > "$agent_worktree/value.txt" + git -C "$agent_worktree" add value.txt + git -C "$agent_worktree" commit -qm 'fixture launch' + agent_head=$(git -C "$agent_worktree" rev-parse HEAD) + printf '%s\n' '{"openai-codex":{"accountId":"fixture-author-account"}}' \ + > "$agent_account/auth.json" + output=$(perl -e 'alarm 60; exec @ARGV' -- \ + env FM_HOME="$HOMEDIR" "$BOT_SH" attest-launch \ + "$agent_task" spawn:attest-agent "$agent_worktree" pi \ + openai-codex-8/gpt-5.6-sol --account-home "$agent_account" \ + --config "$CONFIG_MAIN" 2>&1) \ + || fail "agent launch attestation command failed: $output" + assert_contains "$output" "launch-attested:" \ + "agent launch attestation was not written" + cat > "$HOMEDIR/state/$agent_task.meta" <&1) \ + || fail "agent attestation command failed: $output" + agent_path=${output#attested: } + assert_present "$agent_path" "agent attestation was not written" + shape=$($PYTHON -c 'import json, sys +value=json.load(open(sys.argv[1])) +author=value["payload"]["author"] +print(author["kind"], author["harness"], author["model_family"], author["task_id"])' \ + "$agent_path") + [ "$shape" = "agent pi openai $agent_task" ] \ + || fail "agent provenance fields were not derived correctly: $shape" + origin_shape=$($PYTHON -c 'import json, sys +value=json.load(open(sys.argv[1])) +print(value["schema"], value["payload"]["origin"]["schema"])' "$agent_path") + [ "$origin_shape" = "firstmate.crosscheck-authorship.v2 firstmate.crosscheck-author-launch.v1" ] \ + || fail "exact-head attestation did not bind launch provenance: $origin_shape" + + human_task=attest-human-task + human_pr=https://github.com/Ruby-Labs/goodrepo/pull/78 + human_head=$agent_head + cat > "$HOMEDIR/state/$human_task.meta" <&1); then + fail "unsigned human task metadata was accepted as trusted provenance" + fi + assert_contains "$output" "cannot establish human authorship" \ + "human provenance refusal did not name the missing trusted producer" + pass "the attestation CLI binds agent identity to launch evidence and refuses unsigned human classification" +} + test_mention_without_link_gets_usage_reply() { event="$TMP_ROOT/event-nolink.json" write_event "$event" C0TESTCHAN U0ALICE 1755640000.000100 '<@U0BOT> hello there' @@ -511,7 +740,7 @@ test_channel_outside_allowlist_is_refused() { pass "a mention outside the channel allowlist is refused without a review" } -test_completed_review_names_the_lane_and_writes_gate_metadata() { +test_completed_review_names_lane_head_task_and_artifact() { before=$(fixture_run_count) event="$TMP_ROOT/event-clear.json" write_event "$event" C0TESTCHAN U0ALICE 1755640004.000100 "$GOOD_PR_TEXT" @@ -523,9 +752,13 @@ test_completed_review_names_the_lane_and_writes_gate_metadata() { assert_contains "$reply" "Crosscheck CLEAR" "verdict reply missing state" assert_contains "$reply" "Lane: glm-5p2 primary" \ "verdict reply did not name the cross-family lane" + assert_contains "$reply" "Reviewed head: 1111111111111111111111111111111111111111" \ + "verdict reply did not name the exact reviewed head" + assert_contains "$reply" "Task ID: slack-" \ + "verdict reply did not name the durable task" assert_contains "$reply" "crosscheck.md" "verdict reply did not point at the full report" - assert_contains "$reply" "Host report path for the operator:" \ - "report path was not labeled as host-local" + assert_contains "$reply" "Durable artifact:" \ + "report path was not labeled as the durable artifact" after=$(fixture_run_count) [ "$after" = $((before + 1)) ] || fail "expected exactly one crosscheck invocation" assert_grep "Review started" "$POSTS" "review start ack missing" @@ -685,38 +918,59 @@ test_request_cap_binds_today() { pass "the per-submitter daily request cap binds today with no cost data and is announced in thread" } -test_agent_branch_is_refused_to_the_ordinary_lane() { +test_missing_or_tampered_provenance_fails_closed() { before=$(fixture_run_count) - event="$TMP_ROOT/event-agentbranch.json" + event="$TMP_ROOT/event-provenance-missing.json" write_event "$event" C0TESTCHAN U0ALICE 1755640040.000100 "$GOOD_PR_TEXT" - FMT_HEAD_BRANCH="fm/r10-something" \ - run_mention ev-agentbranch-1 "$CONFIG_MAIN" "$event" agentbranch \ - || fail "agent-branch mention errored: $RUN_MENTION_OUTPUT" - assert_contains "$RUN_MENTION_OUTPUT" "action: agent-branch-refused" \ - "agent-prefixed branch was not refused" + FMT_HEAD_SHA="2222222222222222222222222222222222222222" \ + FMT_PROVENANCE_MODE=missing \ + run_mention ev-provenance-missing "$CONFIG_MAIN" "$event" provenance-missing \ + || fail "missing-provenance mention errored: $RUN_MENTION_OUTPUT" + assert_contains "$RUN_MENTION_OUTPUT" "action: provenance-refused" \ + "missing provenance was not refused" reply=$(last_post_text) - assert_contains "$reply" "matches the agent-branch prefix fm/" \ - "refusal did not name the matched prefix" - assert_contains "$reply" "ordinary crosscheck lane" \ - "refusal did not name the ordinary lane" - assert_contains "$reply" "asserts human authorship" \ - "refusal did not state the authorship assertion" + assert_contains "$reply" "no trustworthy Firstmate/no-mistakes authorship attestation" \ + "missing provenance refusal did not name the trust requirement" + assert_contains "$reply" "Branch names, Slack identity, and message text cannot assert" \ + "refusal did not reject caller-controlled authorship signals" after=$(fixture_run_count) - [ "$after" = "$before" ] || fail "an agent-branch PR reached the crosscheck CLI" + [ "$after" = "$before" ] || fail "a PR without provenance reached the crosscheck CLI" - # A failed branch lookup also refuses, fail closed, without a review. - event="$TMP_ROOT/event-branchfail.json" + event="$TMP_ROOT/event-provenance-tampered.json" write_event "$event" C0TESTCHAN U0ALICE 1755640041.000100 "$GOOD_PR_TEXT" - FMT_HEAD_BRANCH="ERROR" \ - run_mention ev-branchfail-1 "$CONFIG_MAIN" "$event" branchfail \ - || fail "branch-lookup-failure mention errored: $RUN_MENTION_OUTPUT" - assert_contains "$RUN_MENTION_OUTPUT" "action: branch-screen-failed" \ - "a failed branch lookup did not fail closed" - assert_contains "$(last_post_text)" "could not be verified" \ - "branch-lookup failure reply missing" + FMT_HEAD_SHA="3333333333333333333333333333333333333333" \ + FMT_PROVENANCE_MODE=tampered \ + FMT_SOURCE_TASK=source-tampered \ + run_mention ev-provenance-tampered "$CONFIG_MAIN" "$event" provenance-tampered \ + || fail "tampered-provenance mention errored: $RUN_MENTION_OUTPUT" + assert_contains "$RUN_MENTION_OUTPUT" "action: provenance-refused" \ + "tampered provenance was not refused" + assert_contains "$(last_post_text)" "signature verification failed" \ + "tampered provenance refusal did not name signature verification" after=$(fixture_run_count) - [ "$after" = "$before" ] || fail "an unverified branch reached the crosscheck CLI" - pass "an agent-prefixed branch is refused to the ordinary lane and lookup failure fails closed" + [ "$after" = "$before" ] || fail "a PR with tampered provenance reached the crosscheck CLI" + pass "missing and tampered exact-head provenance fail closed without a review" +} + +test_head_change_invalidates_the_verdict() { + event="$TMP_ROOT/event-stale.json" + write_event "$event" C0TESTCHAN U0ALICE 1755640042.000100 "$GOOD_PR_TEXT" + FMT_HEAD_SHA="4444444444444444444444444444444444444444" \ + FMT_HEAD_AFTER="5555555555555555555555555555555555555555" \ + FMT_SOURCE_TASK=source-stale \ + run_mention ev-stale-1 "$CONFIG_MAIN" "$event" stale \ + || fail "head-change mention errored: $RUN_MENTION_OUTPUT" + assert_contains "$RUN_MENTION_OUTPUT" "action: failed" \ + "head change did not invalidate the review" + reply=$(last_post_text) + assert_contains "$reply" "Crosscheck STALE" "head change was not reported as stale" + assert_contains "$reply" "Reviewed head: 4444444444444444444444444444444444444444" \ + "stale reply lost the reviewed head" + assert_contains "$reply" "Current head: 5555555555555555555555555555555555555555" \ + "stale reply lost the new head" + assert_not_contains "$reply" "Crosscheck CLEAR" \ + "a stale review was translated into CLEAR" + pass "a PR head change invalidates the earlier verdict visibly" } test_day_rollover_finalizes_the_origin_day() { @@ -759,6 +1013,177 @@ test_undelivered_verdict_is_reposted_on_redelivery() { pass "a produced-but-undelivered verdict is re-posted on redelivery without a second review" } +test_redelivery_revalidates_head() { + local next_head event before reply + for next_head in 2222222222222222222222222222222222222222 ERROR; do + event="$TMP_ROOT/redelivery-$next_head.json" + write_event "$event" C0TESTCHAN U0ALICE 1755640014.000100 "$GOOD_PR_TEXT" + before=$(fixture_run_count) + FMT_FAIL_VERDICT_POST=1 run_mention "ev-retry-$next_head" "$CONFIG_MAIN" "$event" "retry-first-$next_head" \ + || fail "initial review failed: $RUN_MENTION_OUTPUT" + assert_contains "$RUN_MENTION_OUTPUT" "undelivered:completed:clear" "verdict was not stored" + FMT_HEAD_SHA="$next_head" run_mention "ev-retry-$next_head" "$CONFIG_MAIN" "$event" "retry-second-$next_head" \ + || fail "redelivery failed: $RUN_MENTION_OUTPUT" + reply=$(last_post_text) + assert_not_contains "$reply" "Crosscheck CLEAR" "redelivery emitted an unverified CLEAR" + if [ "$next_head" = ERROR ]; then + assert_contains "$reply" "TOOL FAILURE" "lookup failure did not invalidate verdict" + else + assert_contains "$reply" "Crosscheck STALE" "changed head did not invalidate verdict" + assert_contains "$reply" "$next_head" "stale verdict omitted current head" + fi + [ "$(fixture_run_count)" = $((before + 1)) ] || fail "redelivery started another review" + done + pass "redelivery checks current head and never translates lookup failure to CLEAR" +} + +test_agent_account_is_required() { + output=$(env FM_HOME="$HOMEDIR" FMT_BOT_PY="$BOT_PY" \ + FM_CROSSCHECK_SLACK_CONFIG="$CONFIG_MAIN" "$PYTHON" - <<'PYTEST' +import importlib.util +import os +import sys +from pathlib import Path +spec = importlib.util.spec_from_file_location("slack_account_test", os.environ["FMT_BOT_PY"]) +mod = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = mod +spec.loader.exec_module(mod) +config = mod.load_config(Path(os.environ["FM_CROSSCHECK_SLACK_CONFIG"])) +key = mod.load_provenance_key(config.provenance_key_file) +url = "https://github.com/Ruby-Labs/goodrepo/pull/91" +head = "9" * 40 +task = "account-required" +generation = "spawn:account" +worktree = Path(os.environ["FM_HOME"]) / "account-required-worktree" +worktree.mkdir() +account_home = Path(os.environ["FM_HOME"]) / "account-required-home" +account_home.mkdir() +identity = { + "worktree": str(worktree.resolve()), + "head": head, + "ref": "refs/heads/codex/account-required", + "git_dir_identity": "fixture:account-required", + "tracked_status": "", +} +mod.git_inspect_worktree = lambda _worktree: dict(identity) +mod.git_head_descends_from = lambda _worktree, _ancestor: True +try: + mod.issue_launch_attestation( + config, key, task, generation, worktree, "pi", + "openai-codex-8/gpt-5.6-sol", None, + ) +except mod.SlackExposureError: + pass +else: + raise AssertionError("launch issuer accepted absent account identity") +(account_home / "auth.json").write_text( + '{"openai-codex":{"accountId":"fixture-author-account"}}' +) +mod.issue_launch_attestation( + config, key, task, generation, worktree, "pi", + "openai-codex-8/gpt-5.6-sol", account_home, +) +meta = Path(os.environ["FM_HOME"]) / "state" / f"{task}.meta" +meta.write_text( + f"harness=pi\nmodel=openai-codex-8/gpt-5.6-sol\n" + f"generation_id={generation}\nworktree={worktree.resolve()}\n" + f"pr={url}\npr_head={head}\n" +) +path = mod.issue_task_attestation(config, key, task, url, head) +import json +payload = json.loads(path.read_text())["payload"] +payload["author"]["account_identity"] = None +path.write_text(json.dumps(mod.signed_attestation(payload, key))) +snapshot = mod.PrSnapshot(url, mod.repo_of(url), 91, head) +try: + mod.verify_attestation(config, key, snapshot) +except mod.SlackExposureError: + pass +else: + raise AssertionError("verifier accepted signed null account identity") +PYTEST + ) || fail "required account regression failed: $output" + pass "issuer and verifier reject missing agent account identity" +} + +test_exact_head_cannot_be_rebound_to_another_worktree() { + output=$(env FM_HOME="$HOMEDIR" FMT_BOT_PY="$BOT_PY" \ + FM_CROSSCHECK_SLACK_CONFIG="$CONFIG_MAIN" "$PYTHON" - <<'PYTEST' +import importlib.util +import json +import os +from pathlib import Path +import subprocess +import sys + +spec = importlib.util.spec_from_file_location("slack_worktree_test", os.environ["FMT_BOT_PY"]) +mod = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = mod +spec.loader.exec_module(mod) +config = mod.load_config(Path(os.environ["FM_CROSSCHECK_SLACK_CONFIG"])) +key = mod.load_provenance_key(config.provenance_key_file) +root = Path(os.environ["FM_HOME"]) / "worktree-binding" +root.mkdir(exist_ok=True) + +def repository(name, value): + path = root / name + path.mkdir() + subprocess.run(["git", "-C", str(path), "init", "-q"], check=True) + subprocess.run(["git", "-C", str(path), "config", "user.email", "fixture@example.com"], check=True) + subprocess.run(["git", "-C", str(path), "config", "user.name", "Fixture"], check=True) + (path / "value.txt").write_text(value) + subprocess.run(["git", "-C", str(path), "add", "value.txt"], check=True) + subprocess.run(["git", "-C", str(path), "commit", "-qm", f"fixture {name}"], check=True) + head = subprocess.check_output(["git", "-C", str(path), "rev-parse", "HEAD"], text=True).strip() + return path.resolve(), head + +author_worktree, author_head = repository("author", "author\n") +other_worktree, other_head = repository("other", "other\n") +account_home = root / "account" +account_home.mkdir() +(account_home / "auth.json").write_text(json.dumps({ + "openai-codex": {"accountId": "fixture-author-account"} +})) +task = "worktree-bound" +generation = "spawn:worktree-bound" +url = "https://github.com/Ruby-Labs/goodrepo/pull/92" +mod.issue_launch_attestation( + config, key, task, generation, author_worktree, "pi", + "openai-codex-8/gpt-5.6-sol", account_home, +) +meta = Path(os.environ["FM_HOME"]) / "state" / f"{task}.meta" + +def write_meta(worktree, head): + meta.write_text( + "harness=pi\nmodel=openai-codex-8/gpt-5.6-sol\n" + f"generation_id={generation}\nworktree={worktree}\n" + f"pr={url}\npr_head={head}\n" + ) + +write_meta(other_worktree, other_head) +try: + mod.issue_task_attestation(config, key, task, url, other_head) +except mod.SlackExposureError as exc: + assert "worktree conflicts" in str(exc) +else: + raise AssertionError("another worktree was rebound to the original task") + +write_meta(author_worktree, other_head) +try: + mod.issue_task_attestation(config, key, task, url, other_head) +except mod.SlackExposureError as exc: + assert "does not equal the live PR head" in str(exc) +else: + raise AssertionError("another head was attributed to the original task worktree") + +write_meta(author_worktree, author_head) +path = mod.issue_task_attestation(config, key, task, url, author_head) +assert path.is_file() +PYTEST + ) || fail "worktree-bound provenance regression failed: $output" + pass "launch provenance cannot be rebound to a head produced in another worktree" +} + test_retention_sweep_removes_only_aged_state() { sweep_dir="$TMP_ROOT/sweep-state" output=$(perl -e 'alarm 60; exec @ARGV' -- \ @@ -777,6 +1202,128 @@ test_posts_disable_mrkdwn() { pass "chat.postMessage payloads disable mrkdwn and redact registered secrets" } +test_four_workers_and_concurrent_cap_are_binding() { + concurrency_dir="$TMP_ROOT/concurrency-meter" + output=$(perl -e 'alarm 60; exec @ARGV' -- \ + env FM_HOME="$HOMEDIR" \ + FM_TEST_SLACK_APP_TOKEN="$APP_TOKEN" \ + FM_TEST_SLACK_BOT_TOKEN="$BOT_TOKEN" \ + FM_TEST_GITHUB_READ_TOKEN="$GH_TOKEN_VALUE" \ + FM_CROSSCHECK_SLACK_CONFIG="$CONFIG_MAIN" \ + FMT_BOT_PY="$BOT_PY" FMT_METER_DIR="$concurrency_dir" \ + "$PYTHON" "$DRIVER" concurrency-unit 2>&1) \ + || fail "concurrency unit failed: $output" + assert_contains "$output" "concurrency-ok" "concurrency unit did not complete" + pass "four Slack workers expose shared capacity and concurrent requests cannot overrun the engineer cap" +} + +test_service_install_contains_no_credentials() { + output=$("$PYTHON" - "$TMP_ROOT" "$ROOT" "$HOMEDIR" "$CONFIG_MAIN" "$SERVICE_SH" <<'PYTEST' +import json +import os +from pathlib import Path +import plistlib +import subprocess +import sys + +root, repo, fm_home, original_config, service = map(Path, sys.argv[1:]) +home = root / "service-home" +fixture = root / "service-fixture" +bin_dir = fixture / "bin" +bin_dir.mkdir(parents=True) +home.mkdir() +config_path = fixture / "config.json" +config = json.loads(original_config.read_text()) +config["keychain_services"] = dict(app_token="fixture-app", bot_token="fixture-bot", github_token="fixture-github") +config_path.write_text(json.dumps(config)) +ready = fixture / "keychain-ready" +launch_log = fixture / "launch.log" +wrapper = bin_dir / "fm-crosscheck-slack.sh" +wrapper.write_text(f"#!{sys.executable}\n" + f""" +import importlib.util +import os +from pathlib import Path +import subprocess +import sys +spec = importlib.util.spec_from_file_location('slack_service_fixture', {str(repo / 'bin/fm-crosscheck-slack.py')!r}) +mod = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = mod +spec.loader.exec_module(mod) +real_run = subprocess.run +real_is_file = Path.is_file +def is_file(path): + return True if str(path) == '/usr/bin/security' else real_is_file(path) +def run(command, **kwargs): + if command[0] == '/usr/bin/security': + assert not any(os.environ.get(name) for name in {tuple(config[k] for k in ('app_token_env', 'bot_token_env', 'github_token_env'))!r}) + if not Path({str(ready)!r}).exists(): + return subprocess.CompletedProcess(command, 1, b'') + return subprocess.CompletedProcess(command, 0, b'fixture-keychain-secret') + return real_run(command, **kwargs) +subprocess.run = run +Path.is_file = is_file +if sys.argv[1] == '--selftest': + sys.argv[1] = 'selftest' +sys.exit(mod.main()) +""") +wrapper.chmod(0o755) +launchctl = bin_dir / "launchctl" +launchctl.write_text(f"#!{sys.executable}\n" + f""" +from pathlib import Path +import sys +with Path({str(launch_log)!r}).open('a') as handle: + handle.write(' '.join(sys.argv[1:]) + '\\n') +sys.exit(1 if sys.argv[1] == 'print' else 0) +""") +launchctl.chmod(0o755) +environment = dict(os.environ, HOME=str(home), FM_HOME=str(fm_home), FM_ROOT_OVERRIDE=str(fixture), FM_CROSSCHECK_SLACK_CONFIG=str(config_path), FM_CROSSCHECK_PYTHON=sys.executable, PATH=str(bin_dir) + os.pathsep + os.environ['PATH']) +for name in ('app_token_env', 'bot_token_env', 'github_token_env'): + environment[config[name]] = 'fixture-inherited-secret' +plist = home / 'Library/LaunchAgents/com.firstmate.crosscheck-slack.plist' +def invoke(operation): + result = subprocess.run([str(service), operation], env=environment, capture_output=True, text=True, timeout=60) + assert 'fixture-inherited-secret' not in result.stdout + result.stderr + assert 'fixture-keychain-secret' not in result.stdout + result.stderr + return result +result = invoke('install') +assert result.returncode != 0, 'install accepted shell credentials with unavailable Keychain' +assert not plist.exists() +assert not launch_log.exists() +ready.touch() +result = invoke('install') +assert result.returncode == 0, result.stdout + result.stderr +assert not launch_log.exists(), 'install started listener' +agent = plistlib.loads(plist.read_bytes()) +assert set(agent['EnvironmentVariables']) == {'HOME', 'FM_HOME', 'FM_CROSSCHECK_SLACK_CONFIG', 'FM_CROSSCHECK_PYTHON', 'PATH'} +assert '--keychain-only' in agent['ProgramArguments'] +assert 'fixture-inherited-secret' not in plist.read_text() +assert 'fixture-keychain-secret' not in plist.read_text() +command = agent['ProgramArguments'] +result = subprocess.run([command[0], 'preflight', *command[2:]], env=agent['EnvironmentVariables'], capture_output=True, text=True, timeout=30) +assert result.returncode == 0, result.stdout + result.stderr +result = invoke('start') +assert result.returncode == 0, result.stdout + result.stderr +assert 'bootstrap' in launch_log.read_text() +launch_log.unlink() +ready.unlink() +result = invoke('start') +assert result.returncode != 0, 'start accepted unavailable Keychain' +assert not launch_log.exists(), 'failed validation mutated service state' +previous = plist.read_bytes() +result = invoke('install') +assert result.returncode != 0 +assert plist.read_bytes() == previous, 'failed reinstall replaced plist' +ready.touch() +del config['keychain_services'] +config_path.write_text(json.dumps(config)) +result = invoke('install') +assert result.returncode != 0, 'service accepted environment-only configuration' +assert plist.read_bytes() == previous +PYTEST + ) || fail "launch-agent credential contract failed: $output" + pass "service validates emitted environment and refuses unavailable Keychain without mutating launch state" +} + test_tool_failure_is_reported_honestly() { event="$TMP_ROOT/event-fail.json" write_event "$event" C0TESTCHAN U0ALICE 1755640012.000100 "$GOOD_PR_TEXT" @@ -824,26 +1371,41 @@ UNITS=( test_pr_link_extraction test_selftest_validates_config_shape test_missing_token_env_refuses_start + test_attestation_cli_derives_and_signs_author_identity test_mention_without_link_gets_usage_reply test_multiple_links_are_refused test_out_of_allowlist_repo_is_refused test_channel_outside_allowlist_is_refused - test_completed_review_names_the_lane_and_writes_gate_metadata + test_completed_review_names_lane_head_task_and_artifact test_lane_naming_covers_fallback_and_explicit_marker test_duplicate_event_id_starts_one_review test_usd_meter_forward_contract_with_fixture_injected_usage test_production_shaped_ledger_never_trips_usd_bound test_request_cap_binds_today - test_agent_branch_is_refused_to_the_ordinary_lane + test_missing_or_tampered_provenance_fails_closed + test_head_change_invalidates_the_verdict test_day_rollover_finalizes_the_origin_day test_undelivered_verdict_is_reposted_on_redelivery + test_redelivery_revalidates_head + test_agent_account_is_required + test_exact_head_cannot_be_rebound_to_another_worktree test_retention_sweep_removes_only_aged_state test_posts_disable_mrkdwn + test_four_workers_and_concurrent_cap_are_binding + test_service_install_contains_no_credentials test_tool_failure_is_reported_honestly test_blocking_verdict_reply_names_state_and_findings test_tokens_never_reach_logs_or_ledgers ) +case "${FM_SLACK_TEST_GROUP:-all}" in + service-repair) UNITS=(test_service_install_contains_no_credentials) ;; + repair) UNITS=(test_agent_account_is_required test_exact_head_cannot_be_rebound_to_another_worktree test_redelivery_revalidates_head test_service_install_contains_no_credentials) ;; + compatibility) UNITS=(test_attestation_cli_derives_and_signs_author_identity test_exact_head_cannot_be_rebound_to_another_worktree test_completed_review_names_lane_head_task_and_artifact test_undelivered_verdict_is_reposted_on_redelivery test_head_change_invalidates_the_verdict test_four_workers_and_concurrent_cap_are_binding) ;; + all) ;; + *) fail "unknown Slack test group" ;; +esac + for unit in "${UNITS[@]}"; do declare -F "$unit" >/dev/null || fail "registered unit is not a defined function: $unit" "$unit" diff --git a/tests/fm-spawn-dispatch-profile.test.sh b/tests/fm-spawn-dispatch-profile.test.sh index 89e1daf6270..7f382def88d 100755 --- a/tests/fm-spawn-dispatch-profile.test.sh +++ b/tests/fm-spawn-dispatch-profile.test.sh @@ -621,6 +621,54 @@ PY pass "Pi snapshots remain bound without recording or requiring author identity" } +test_crosscheck_launch_provenance_uses_the_real_pi_snapshot() { + local rec id source out status attest shape leased_worktree exact_head exact_attest + id=profile-pi-crosscheck-provenance-z25 + rec=$(make_spawn_case profile-pi-crosscheck-provenance pi "$id") + read_case_record "$rec" + source="$CASE_DIR/pi-source" + make_pi_account_source "$source" account-crosscheck + printf '%s\n' \ + '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef' \ + > "$HOME_DIR/config/crosscheck-slack-provenance.key" + chmod 600 "$HOME_DIR/config/crosscheck-slack-provenance.key" + cat > "$HOME_DIR/config/crosscheck-slack.json" < "$leased_worktree/crosscheck-provenance-fixture.txt" + git -C "$leased_worktree" add crosscheck-provenance-fixture.txt + git -C "$leased_worktree" commit -qm 'test: author after launch' + exact_head=$(git -C "$leased_worktree" rev-parse HEAD) + printf 'pr=https://github.com/ruby-labs/firstmate/pull/999\npr_head=%s\n' \ + "$exact_head" >> "$HOME_DIR/state/$id.meta" + out=$(FM_HOME="$HOME_DIR" "$ROOT/bin/fm-crosscheck-slack.sh" attest-task \ + "$id" https://github.com/ruby-labs/firstmate/pull/999 "$exact_head" \ + --config "$HOME_DIR/config/crosscheck-slack.json" 2>&1) + status=$? + expect_code 0 "$status" "exact-head provenance should accept the task's post-launch branch: $out" + exact_attest=${out#attested: } + [ -f "$exact_attest" ] || fail "exact-head provenance artifact is missing after branch creation" + pass "fm-spawn captures signed Pi author and worktree provenance from the real launch snapshot" +} + test_batch_forwards_shared_profile_flags() { local rec id1 id2 out status id1=profile-batch-a-z9 @@ -669,6 +717,10 @@ if [ "${FM_TEST_FOCUSED:-}" = pi-author-snapshot ]; then test_pi_author_account_snapshot_binds_launch_and_recovery exit 0 fi +if [ "${FM_TEST_FOCUSED:-}" = crosscheck-author-launch ]; then + test_crosscheck_launch_provenance_uses_the_real_pi_snapshot + exit 0 +fi test_no_profile_resolves_claude_model_anchor test_active_dispatch_profile_requires_explicit_harness_for_ship @@ -688,6 +740,7 @@ test_pi_omits_invalid_max_effort test_pi_crewmate_carries_autonomy_flags test_pi_secondmate_approves_without_excluding_tools test_pi_author_account_snapshot_binds_launch_and_recovery +test_crosscheck_launch_provenance_uses_the_real_pi_snapshot test_batch_forwards_shared_profile_flags test_active_dispatch_profile_does_not_block_secondmate_launch