diff --git a/cmd/gortex/init_doctor.go b/cmd/gortex/doctor.go similarity index 70% rename from cmd/gortex/init_doctor.go rename to cmd/gortex/doctor.go index 6de74466e..db6baf70a 100644 --- a/cmd/gortex/init_doctor.go +++ b/cmd/gortex/doctor.go @@ -13,28 +13,84 @@ import ( "github.com/zzet/gortex/internal/agents" "github.com/zzet/gortex/internal/daemon" + "github.com/zzet/gortex/internal/doctor" ) -// initDoctorCmd implements `gortex init doctor`. It re-runs every -// adapter's Detect + Plan against the current environment and -// reports drift — files we'd like to write, files that already -// exist, files whose contents look wrong. Zero writes: doctor -// never modifies disk. +// doctorCmd implements `gortex doctor`. It re-runs every adapter's +// Detect + Plan against the current environment and reports drift — +// files we'd like to write, files that already exist, files whose +// contents look wrong — and then goes past what a config file can +// prove: whether the hooks those files declare have actually run, +// whether the agent is calling Gortex tools, and what the savings +// ledger recorded. Zero writes: doctor never modifies disk. +// +// The runtime half exists because an agent config is not evidence. +// Codex, for one, records trust against each hook's hash and skips new +// or changed hooks until the user approves them — so a fully populated +// config.toml and a completely inert integration look identical on +// disk. Only the invocation log tells them apart. // // This is the zero-op form of the installer and the primary // support-channel tool. Output defaults to a human-readable table; // --json emits a machine-readable report for CI consumers. -var initDoctorCmd = &cobra.Command{ +var doctorCmd = &cobra.Command{ Use: "doctor", - Short: "Report the configured state of every Gortex integration without writing anything", - Args: cobra.NoArgs, - RunE: runInitDoctor, + Short: "Diagnose the Gortex install: config, hook activity, agent adoption, savings", + Long: `Reports the machine-wide state of every Gortex integration without +writing anything. + +Static half — per-adapter Detect + Plan: which files exist, which are +missing, whether an MCP stanza is stale. + +Runtime half — the evidence a config file cannot carry: + + hook activity did the hook processes actually run, did they inject + context, did they reach the daemon? An event configured + with zero invocations is the signature of a hook the + agent is skipping (Codex requires per-hook trust via + /hooks before a new or changed hook runs). + adoption for agents that keep session transcripts, how often the + model called a Gortex tool versus shelling out. + savings what the ledger recorded, and how much of it carries no + client or model attribution. + +Examples: + + gortex doctor # last 7 days + gortex doctor --days 30 + gortex doctor --redact # hash repo paths before sharing + gortex doctor --json # machine-readable + +Exit status is 1 when a blocker is found, so CI can gate on it.`, + Args: cobra.NoArgs, + RunE: runDoctor, +} + +// initDoctorCmd keeps `gortex init doctor` working for anyone with it in a +// script. The name always oversold its scope — doctor reports machine-wide +// state, not this repo's — so the canonical spelling is `gortex doctor`. +var initDoctorCmd = &cobra.Command{ + Use: "doctor", + Short: "Deprecated alias for `gortex doctor`", + Args: cobra.NoArgs, + Deprecated: "use `gortex doctor` (doctor reports machine-wide state, not one repo's).", + RunE: runDoctor, } -var initDoctorJSON bool +var ( + doctorJSON bool + doctorDays int + doctorRedact bool +) func init() { - initDoctorCmd.Flags().BoolVar(&initDoctorJSON, "json", false, "emit a structured JSON report on stdout") + for _, c := range []*cobra.Command{doctorCmd, initDoctorCmd} { + c.Flags().BoolVar(&doctorJSON, "json", false, "emit a structured JSON report on stdout") + c.Flags().IntVar(&doctorDays, "days", 7, "look-back window for hook activity, adoption, and savings") + c.Flags().BoolVar(&doctorRedact, "redact", false, "hash repo paths and branch names so the report can be shared") + } + silenceDoctorErrors(doctorCmd, initDoctorCmd) + rootCmd.AddCommand(doctorCmd) initCmd.AddCommand(initDoctorCmd) } @@ -84,7 +140,7 @@ type DoctorFileStatus struct { StanzaHint string `json:"stanza_hint,omitempty"` } -func runInitDoctor(cmd *cobra.Command, _ []string) error { +func runDoctor(cmd *cobra.Command, _ []string) error { home, _ := os.UserHomeDir() root, err := filepath.Abs(".") if err != nil { @@ -108,13 +164,33 @@ func runInitDoctor(cmd *cobra.Command, _ []string) error { reports = append(reports, inspectAdapter(a, env)) } - if initDoctorJSON { + runtime := collectRuntime(home, doctorDays) + + if doctorJSON { enc := json.NewEncoder(cmd.OutOrStdout()) enc.SetIndent("", " ") - return enc.Encode(map[string]any{"environment": doctorEnv, "agents": reports}) + if err := enc.Encode(map[string]any{ + "environment": doctorEnv, + "agents": reports, + "runtime": runtime, + }); err != nil { + return err + } + return doctorExit(runtime) } printDoctorEnvironment(cmd.OutOrStdout(), doctorEnv) printDoctorHuman(cmd.OutOrStdout(), reports) + printDoctorRuntime(cmd.OutOrStdout(), runtime) + return doctorExit(runtime) +} + +// doctorExit turns a blocker into a non-zero status without printing a +// second error line — the findings already said what is wrong, and cobra +// would otherwise echo the error on top of them. +func doctorExit(r doctorRuntime) error { + if doctor.HasBlocker(r.Findings) { + return errDoctorBlocker + } return nil } @@ -232,7 +308,7 @@ func inspectAdapter(a agents.Adapter, env agents.Env) DoctorAgentReport { // and the daemon handshake. These two lines answer "is the integration // actually live", not just "is the config file present". func printDoctorEnvironment(w io.Writer, env DoctorEnvironment) { - fmt.Fprintln(w, "Gortex init doctor — environment:") + fmt.Fprintln(w, "Gortex doctor — environment:") if env.BinaryOnPath { fmt.Fprintf(w, " %s gortex on PATH: %s\n", glyphCheck, env.BinaryPath) } else { @@ -260,7 +336,7 @@ const ( // agent, with a nested file list. Columns line up for copy-paste // into issue reports. func printDoctorHuman(w io.Writer, reports []DoctorAgentReport) { - fmt.Fprintln(w, "Gortex init doctor — observed state of every adapter's planned files:") + fmt.Fprintln(w, "Gortex doctor — observed state of every adapter's planned files:") fmt.Fprintln(w) for _, r := range reports { diff --git a/cmd/gortex/doctor_runtime.go b/cmd/gortex/doctor_runtime.go new file mode 100644 index 000000000..c4ed02355 --- /dev/null +++ b/cmd/gortex/doctor_runtime.go @@ -0,0 +1,355 @@ +package main + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "sort" + "time" + + "github.com/spf13/cobra" + + "github.com/zzet/gortex/internal/agents/codex" + "github.com/zzet/gortex/internal/doctor" + "github.com/zzet/gortex/internal/hooks" + "github.com/zzet/gortex/internal/savings" +) + +// doctor_runtime.go is the half of `gortex doctor` that reads evidence rather +// than configuration: did the hooks run, is the agent calling Gortex tools, +// and what did the savings ledger record. Everything here is read-only and +// degrades to "no data" — doctor has to work on exactly the broken machines +// that make people run it. + +// errDoctorBlocker makes the command exit non-zero when a blocker is found. +// SilenceUsage/SilenceErrors on the command keep cobra from printing it: the +// findings block is the message. +var errDoctorBlocker = errors.New("doctor found a blocking problem") + +type doctorRuntime struct { + Window string `json:"window"` + Since time.Time `json:"since"` + Redacted bool `json:"redacted"` + Codex codex.InstallState `json:"codex"` + Hooks hooks.EffectivenessSummary `json:"hooks"` + Adoption doctor.Adoption `json:"adoption"` + Savings doctorSavings `json:"savings"` + Findings []doctor.Finding `json:"findings"` +} + +type doctorSavings struct { + Available bool `json:"available"` + Error string `json:"error,omitempty"` + Events int `json:"events"` + Saved int64 `json:"saved"` + Returned int64 `json:"returned"` + NoClient int `json:"events_without_client"` + NoModel int `json:"events_without_model"` + ByTool []doctorDimTotal `json:"by_tool,omitempty"` + ByClient []doctorDimTotal `json:"by_client,omitempty"` + ByModel []doctorDimTotal `json:"by_model,omitempty"` +} + +type doctorDimTotal struct { + Name string `json:"name"` + Calls int64 `json:"calls"` + Saved int64 `json:"saved"` +} + +func collectRuntime(home string, days int) doctorRuntime { + if days < 1 { + days = 1 + } + now := time.Now() + since := now.Add(-time.Duration(days) * 24 * time.Hour) + + state := codex.Inspect(home) + activity, err := hooks.ReadEffectiveness(since) + if err != nil { + // A log we cannot read is not fatal; the other sections still carry + // signal, and the empty summary renders as "no hook has ever run". + activity.Path = hooks.EffectivenessLogPath() + } + adoption := doctor.ScanCodexSessions(doctor.CodexHome(), since, 10) + + out := doctorRuntime{ + Window: fmt.Sprintf("last %dd", days), + Since: since, + Redacted: doctorRedact, + Codex: state, + Hooks: activity, + Adoption: adoption, + Savings: collectSavings(since), + } + out.Findings = doctor.Diagnose(doctor.AgentHooks{ + Agent: codex.Name, + Configured: state.Hooks, + RequiresTrust: true, + TrustRemedy: codex.TrustRemedy, + InstructionsPath: state.InstructionsPath, + InstructionsWired: state.InstructionsWired, + InstructionsShadowed: state.ShadowedBy, + }, activity, adoption, now) + + if doctorRedact { + out.Adoption = redactAdoption(out.Adoption) + } + return out +} + +func collectSavings(since time.Time) doctorSavings { + out := doctorSavings{} + store, err := savings.Open(savings.DefaultDBPath()) + if err != nil { + out.Error = err.Error() + return out + } + defer store.Close() + out.Available = true + + events, err := store.EventsSince(since) + if err != nil { + out.Error = err.Error() + return out + } + out.Events = len(events) + for _, e := range events { + out.Saved += e.Saved + out.Returned += e.Returned + if e.Client == "" { + out.NoClient++ + } + if e.Model == "" { + out.NoModel++ + } + } + if tools, err := store.ToolTotals(since); err == nil { + for _, t := range tools { + out.ByTool = append(out.ByTool, doctorDimTotal{Name: t.Tool, Calls: t.CallsCounted, Saved: t.TokensSaved}) + } + } + if clients, err := store.ClientTotals(since); err == nil { + for _, c := range clients { + out.ByClient = append(out.ByClient, doctorDimTotal{Name: c.Name, Calls: c.CallsCounted, Saved: c.TokensSaved}) + } + } + if models, err := store.ModelTotals(since); err == nil { + for _, m := range models { + out.ByModel = append(out.ByModel, doctorDimTotal{Name: m.Name, Calls: m.CallsCounted, Saved: m.TokensSaved}) + } + } + return out +} + +// redactAdoption hashes the workspace identifiers so the report can be pasted +// into an issue. Tool names, counts, and timings are never sensitive and are +// always left intact — redacting them would defeat the point of sharing it. +func redactAdoption(a doctor.Adoption) doctor.Adoption { + a.Root = redactPath(a.Root) + for i := range a.Sessions { + a.Sessions[i].CWD = redactPath(a.Sessions[i].CWD) + a.Sessions[i].Branch = redactPath(a.Sessions[i].Branch) + } + return a +} + +func redactPath(value string) string { + if value == "" { + return "" + } + sum := sha256.Sum256([]byte(value)) + return "…/<" + hex.EncodeToString(sum[:])[:8] + ">" +} + +func printDoctorRuntime(w io.Writer, r doctorRuntime) { + fmt.Fprintf(w, "Gortex doctor — runtime evidence (%s):\n\n", r.Window) + + fmt.Fprintln(w, " codex install") + if !r.Codex.ConfigPresent { + fmt.Fprintf(w, " %s %s missing\n", glyphCross, r.Codex.ConfigPath) + } else { + fmt.Fprintf(w, " %s mcp_servers.gortex\n", mark(r.Codex.MCPServer)) + for _, event := range codex.HookEvents { + fmt.Fprintf(w, " %s hook %-17s %d configured\n", mark(r.Codex.Hooks[event] > 0), event, r.Codex.Hooks[event]) + } + } + fmt.Fprintf(w, " %s rule block in %s\n", mark(r.Codex.InstructionsWired), r.Codex.InstructionsPath) + if r.Codex.ShadowedBy != "" { + fmt.Fprintf(w, " %s shadowed by %s\n", glyphCross, r.Codex.ShadowedBy) + } + fmt.Fprintln(w) + + fmt.Fprintln(w, " hook activity (did the hook processes actually run?)") + if !r.Hooks.Present { + fmt.Fprintf(w, " no %s — no hook has ever run on this machine\n", r.Hooks.Path) + } else { + fmt.Fprintf(w, " %-18s %7s %9s %10s %s\n", "event", "runs", "injected", "daemon-up", "last seen") + for _, event := range doctorEventOrder(r.Hooks) { + stats := r.Hooks.Events[event] + daemon := "n/a" + if stats.DaemonKnown > 0 { + daemon = fmt.Sprintf("%d/%d", stats.DaemonUp, stats.DaemonKnown) + } + last := "never" + if !stats.LastSeen.IsZero() { + last = stats.LastSeen.Format(time.RFC3339)[:19] + } + fmt.Fprintf(w, " %-18s %7d %9d %10s %s\n", event, stats.Runs, stats.Emitted, daemon, last) + } + fmt.Fprintf(w, " %d row(s) in window, %d in the whole log\n", r.Hooks.WindowRows, r.Hooks.TotalRows) + } + fmt.Fprintln(w, " note: this log has no agent dimension — on a machine running more than") + fmt.Fprintln(w, " one agent, these counts are the sum.") + fmt.Fprintln(w) + + fmt.Fprintln(w, " codex adoption (what the model actually called)") + if r.Adoption.FilesFound == 0 { + fmt.Fprintf(w, " no session transcripts under %s\n", r.Adoption.Root) + } else { + fmt.Fprintf(w, " sessions with tool calls %d of %d on disk\n", r.Adoption.InWindow, r.Adoption.FilesFound) + fmt.Fprintf(w, " gortex MCP calls %d\n", r.Adoption.GortexCalls) + fmt.Fprintf(w, " other MCP calls %d\n", r.Adoption.OtherMCP) + fmt.Fprintf(w, " shell calls %d (%d read/search shaped)\n", r.Adoption.ShellCalls, r.Adoption.ShellReads) + if len(r.Adoption.Tools) > 0 { + fmt.Fprintf(w, " gortex tools used %s\n", topCounts(r.Adoption.Tools, 8)) + } + if len(r.Adoption.Models) > 0 { + fmt.Fprintf(w, " models %s\n", topCounts(r.Adoption.Models, 4)) + } + } + fmt.Fprintln(w, " note: Codex does not persist hook-injected context into transcripts, so") + fmt.Fprintln(w, " this section cannot prove a hook fired — hook activity above does.") + fmt.Fprintln(w) + + fmt.Fprintln(w, " savings ledger") + switch { + case !r.Savings.Available: + fmt.Fprintf(w, " unavailable: %s\n", r.Savings.Error) + case r.Savings.Events == 0: + fmt.Fprintln(w, " no recorded events in window") + default: + baseline := r.Savings.Saved + r.Savings.Returned + pct := 0.0 + if baseline > 0 { + pct = 100 * float64(r.Savings.Saved) / float64(baseline) + } + fmt.Fprintf(w, " events %d\n", r.Savings.Events) + fmt.Fprintf(w, " saved / baseline %d / %d (%.1f%%)\n", r.Savings.Saved, baseline, pct) + fmt.Fprintf(w, " without client / model %d / %d\n", r.Savings.NoClient, r.Savings.NoModel) + for label, rows := range map[string][]doctorDimTotal{ + "by tool": r.Savings.ByTool, "by client": r.Savings.ByClient, "by model": r.Savings.ByModel, + } { + for i, row := range rows { + if i >= 6 { + break + } + fmt.Fprintf(w, " %-10s %-24s %6d calls %10d saved\n", label, doctorTruncate(row.Name, 24), row.Calls, row.Saved) + label = "" + } + } + } + fmt.Fprintln(w, " note: only the read-family tools write here (saved = whole-file tokens −") + fmt.Fprintln(w, " returned). explore / search / relations / trace record nothing, and") + fmt.Fprintln(w, " reading a whole file records 0 — so this describes those calls only.") + fmt.Fprintln(w) + + fmt.Fprintln(w, " findings") + for _, f := range r.Findings { + glyph := "·" + switch f.Severity { + case doctor.SeverityBlocker: + glyph = glyphCross + case doctor.SeverityWarn: + glyph = "!" + case doctor.SeverityOK: + glyph = glyphCheck + } + fmt.Fprintf(w, " %s %-7s %s\n", glyph, f.Severity, f.Summary) + if f.Remedy != "" { + fmt.Fprintf(w, " → %s\n", f.Remedy) + } + } + if !r.Redacted { + fmt.Fprintln(w, "\n sharing this? re-run with --redact to hash repo paths and branch names.") + } + fmt.Fprintln(w) +} + +// doctorEventOrder lists the adapter's own events first, in lifecycle order, +// then anything else the log saw — so the events a reader is looking for are +// never buried under LocalizationTerminal rows. +func doctorEventOrder(s hooks.EffectivenessSummary) []string { + seen := map[string]bool{} + out := make([]string, 0, len(s.Events)+len(codex.HookEvents)) + for _, event := range codex.HookEvents { + out = append(out, event) + seen[event] = true + } + for _, event := range s.EventNames() { + if !seen[event] { + out = append(out, event) + } + } + return out +} + +func topCounts(counts map[string]int, limit int) string { + type kv struct { + name string + n int + } + rows := make([]kv, 0, len(counts)) + for name, n := range counts { + rows = append(rows, kv{name, n}) + } + sort.Slice(rows, func(i, j int) bool { + if rows[i].n != rows[j].n { + return rows[i].n > rows[j].n + } + return rows[i].name < rows[j].name + }) + if len(rows) > limit { + rows = rows[:limit] + } + parts := make([]string, 0, len(rows)) + for _, row := range rows { + parts = append(parts, fmt.Sprintf("%s×%d", row.name, row.n)) + } + return joinComma(parts) +} + +func joinComma(parts []string) string { + out := "" + for i, p := range parts { + if i > 0 { + out += ", " + } + out += p + } + return out +} + +func doctorTruncate(s string, n int) string { + if len(s) <= n { + return s + } + return s[:n] +} + +func mark(ok bool) string { + if ok { + return glyphCheck + } + return glyphCross +} + +// silenceDoctorErrors keeps the blocker exit status from printing a second +// error line under the findings block. +func silenceDoctorErrors(cmds ...*cobra.Command) { + for _, c := range cmds { + c.SilenceUsage = true + c.SilenceErrors = true + } +} diff --git a/cmd/gortex/init_doctor_test.go b/cmd/gortex/doctor_test.go similarity index 100% rename from cmd/gortex/init_doctor_test.go rename to cmd/gortex/doctor_test.go diff --git a/docs/agents.md b/docs/agents.md index 9a6c0ac7f..7c0e21170 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -12,7 +12,8 @@ machine. Nineteen adapters ship today. hooks where supported, per-agent marker-guarded community-routing blocks, and `.claude/skills/generated/` per-community SKILL.md. -Run `gortex init doctor` to see what's currently configured. Both +Run `gortex doctor` to see what's currently configured — and, past what a +config file can prove, whether the hooks it declares are actually running. Both commands accept `--agents=` to constrain setup and `--agents-skip=` to exclude an adapter. @@ -137,8 +138,9 @@ gortex init --force # overwrite merge-preserved keys gortex init --hooks-only # refresh supported agent hooks only # Observe-only -gortex init doctor # read-only state report -gortex init doctor --json # machine-readable report +gortex doctor # config + hook activity + adoption +gortex doctor --json # machine-readable report +gortex doctor --redact # safe to paste into an issue ``` ## Adapter contract @@ -247,6 +249,14 @@ installer says so. > `SessionStart` never fires, which is the surface that puts the Gortex rule > in front of the model. +To tell the two states apart, run `gortex doctor`. It correlates the Codex +config, the hook-invocation log, the Codex session transcripts, and the savings +ledger into one verdict — *hooks configured but none has run* is the +untrusted-hook signature, reported as a blocker with the `/hooks` remedy. +`--redact` hashes repo paths and branch names so the report can be pasted into +an issue, `--json` emits it machine-readably, and the exit status is non-zero +when a blocker is found. + When hooks are enabled (the default), Codex receives user-level hooks. The default posture remains advisory. A team can opt into `deny`, `rewrite`, or `suppress` by setting diff --git a/docs/cli.md b/docs/cli.md index e540cb477..d7c91f652 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -4,7 +4,7 @@ gortex install One-time machine-wide setup (user-level MCP, skills, hooks, daemon wiring) gortex init [path] Per-repo setup (.mcp.json, hooks, community routing, per-community SKILL.md) gortex init --dry-run-intake Emit a privacy-safe intake manifest and exit before parsing/writes -gortex init doctor Zero-op drift report across all detected agents (human or --json) +gortex doctor Zero-op state report: adapter drift, hook activity, adoption, savings (human or --json) gortex mcp [flags] Start the MCP stdio server (auto-detects daemon; --no-daemon / --proxy; --server adds HTTP API) gortex daemon start [flags] Start the daemon; --http-addr serves the HTTP/JSON API under /v1/* plus the MCP /mcp transport (--http-auth-token, --cors-origin) gortex daemon start / stop / restart / reload / status / logs / install-service / service-status / uninstall-service / server (multi-server roster) diff --git a/docs/skills.md b/docs/skills.md index 3740300e5..1a6e9d587 100644 --- a/docs/skills.md +++ b/docs/skills.md @@ -35,7 +35,7 @@ The trade-off versus an MCP install — push notifications and per-session overl Tool-usage guidance for agents that have a user-level surface (Claude Code, Antigravity) lives once per user; for the rest, MCP tool descriptions carry the teaching and `gortex init` adds only a per-repo community-routing block — no more duplicated instructions blocks in every repo. - **Adapter matrix + per-agent schema notes:** [`agents.md`](agents.md) -- **Audit what's currently configured:** `gortex init doctor` (zero-op; `--json` for CI consumers) +- **Audit what's currently configured:** `gortex doctor` (zero-op; `--json` for CI consumers) - **Constrain setup:** `gortex init --agents=claude-code,cursor` or `--agents-skip=antigravity` (same flags accepted by `gortex install`) - **CI / scripted install:** `gortex install --yes --json` then `gortex init --yes --json --dry-run` diff --git a/internal/agents/codex/inspect.go b/internal/agents/codex/inspect.go new file mode 100644 index 000000000..9726fed3d --- /dev/null +++ b/internal/agents/codex/inspect.go @@ -0,0 +1,111 @@ +package codex + +import ( + "os" + "path/filepath" + "strings" + + toml "github.com/pelletier/go-toml/v2" +) + +// inspect.go is the read-only counterpart to Apply: it reports what is +// actually on disk so `gortex doctor` can compare intent against reality. +// It reuses the same hook-entry predicates the writer uses, so a hook this +// package would install and a hook it recognises can never drift apart. + +// InstallState is the observed state of the Codex integration. +// +// It answers "what did we write", not "is it working" — Codex gates hook +// execution behind per-hook trust, so a fully populated config here says +// nothing about whether any of it runs. That question is answered by the +// hook invocation log, and the two are only meaningful together. +type InstallState struct { + ConfigPath string `json:"config_path"` + ConfigPresent bool `json:"config_present"` + MCPServer bool `json:"mcp_server"` + Hooks map[string]int `json:"hooks"` + // InstructionsPath is the file Codex will actually load; ShadowedBy is + // set when an override file takes precedence over the one Gortex writes, + // which turns a correctly-installed rule block into dead text. + InstructionsPath string `json:"instructions_path,omitempty"` + InstructionsWired bool `json:"instructions_wired"` + ShadowedBy string `json:"shadowed_by,omitempty"` +} + +// HookEvents are the lifecycle events this adapter installs, in the order +// they matter for adoption: SessionStart is what states the Gortex rule for +// the session, so its absence explains a silent integration on its own. +var HookEvents = []string{"SessionStart", "UserPromptSubmit", "PreToolUse", "PostToolUse"} + +// TrustRemedy is how a user approves hooks Codex is skipping. +const TrustRemedy = "run `/hooks` inside Codex, review the gortex entries, and trust them" + +// Inspect reads the Codex config and instructions file under home. Every +// failure degrades to "not configured" rather than an error: doctor's job is +// to describe a broken machine, not to fail on one. +func Inspect(home string) InstallState { + state := InstallState{Hooks: map[string]int{}} + if home == "" { + return state + } + for _, event := range HookEvents { + state.Hooks[event] = 0 + } + state.ConfigPath = filepath.Join(home, ".codex", "config.toml") + + if data, err := os.ReadFile(state.ConfigPath); err == nil { + state.ConfigPresent = true + root := map[string]any{} + if toml.Unmarshal(data, &root) == nil { + if servers, ok := root["mcp_servers"].(map[string]any); ok { + _, state.MCPServer = servers["gortex"] + } + countGortexHooks(root, state.Hooks) + } + } + + // Codex prefers the override file in its home and then ignores AGENTS.md + // entirely, so report the file it will really read. + base := filepath.Join(home, ".codex", codexGlobalInstructionsFile) + override := filepath.Join(home, ".codex", codexGlobalInstructionsOverrideFile) + state.InstructionsPath = base + if _, err := os.Stat(override); err == nil { + if _, err := os.Stat(base); err == nil { + state.ShadowedBy = override + } + state.InstructionsPath = override + } + if body, err := os.ReadFile(state.InstructionsPath); err == nil { + text := string(body) + state.InstructionsWired = strings.Contains(text, "gortex:rules:start") || + strings.Contains(text, "MANDATORY: Use Gortex MCP") + } + return state +} + +// countGortexHooks tallies Gortex-authored entries per event, reusing the +// writer's own recognisers so a hook shape we install is always a hook shape +// we can find again. +func countGortexHooks(root map[string]any, out map[string]int) { + hooks, ok := root["hooks"].(map[string]any) + if !ok { + return + } + recognisers := map[string]func(any) bool{ + "SessionStart": codexHookEntryIsGortexSessionStart, + "UserPromptSubmit": codexHookEntryIsGortexUserPromptSubmit, + "PreToolUse": codexHookEntryIsGortexPreToolUse, + "PostToolUse": codexHookEntryIsGortexPostToolUse, + } + for event, isGortex := range recognisers { + entries, ok := codexHookList(hooks[event]) + if !ok { + continue + } + for _, entry := range entries { + if isGortex(entry) { + out[event]++ + } + } + } +} diff --git a/internal/agents/codex/inspect_test.go b/internal/agents/codex/inspect_test.go new file mode 100644 index 000000000..fbb654e4e --- /dev/null +++ b/internal/agents/codex/inspect_test.go @@ -0,0 +1,126 @@ +package codex + +import ( + "os" + "path/filepath" + "testing" + + "github.com/zzet/gortex/internal/agents" +) + +// TestInspectSeesWhatApplyWrote is the drift fence between the writer and the +// reader. `gortex doctor` reports "hooks configured" from Inspect; if Apply's +// hook shape ever changes without Inspect following, doctor would quietly +// report a correctly-installed integration as unconfigured — turning the +// diagnostic into a second bug rather than a tool for finding the first. +func TestInspectSeesWhatApplyWrote(t *testing.T) { + env := codexGlobalEnv(t) + env.InstallGlobalInstructions = true + + before := Inspect(env.Home) + if before.ConfigPresent { + t.Fatalf("fixture home should start empty, got %+v", before) + } + for _, event := range HookEvents { + if before.Hooks[event] != 0 { + t.Fatalf("%s counted %d hooks before install", event, before.Hooks[event]) + } + } + + if _, err := New().Apply(env, agents.ApplyOpts{}); err != nil { + t.Fatalf("apply: %v", err) + } + + after := Inspect(env.Home) + if !after.ConfigPresent { + t.Fatal("config.toml not seen after install") + } + if !after.MCPServer { + t.Fatal("mcp_servers.gortex not seen after install") + } + for _, event := range HookEvents { + if after.Hooks[event] == 0 { + t.Errorf("%s: Apply wrote a hook that Inspect cannot see", event) + } + } + if !after.InstructionsWired { + t.Error("rule block written by Apply is not visible to Inspect") + } + if after.ShadowedBy != "" { + t.Errorf("unexpected shadowing: %s", after.ShadowedBy) + } +} + +// TestInspectReportsOverrideShadowing pins the case where the rule block is +// installed correctly and still never reaches the model: Codex reads +// AGENTS.override.md in its home and ignores AGENTS.md entirely. +func TestInspectReportsOverrideShadowing(t *testing.T) { + env := codexGlobalEnv(t) + env.InstallGlobalInstructions = true + if _, err := New().Apply(env, agents.ApplyOpts{}); err != nil { + t.Fatalf("apply: %v", err) + } + override := filepath.Join(env.Home, ".codex", codexGlobalInstructionsOverrideFile) + if err := os.WriteFile(override, []byte("# my own rules\n"), 0o644); err != nil { + t.Fatal(err) + } + + state := Inspect(env.Home) + if state.ShadowedBy != override { + t.Fatalf("ShadowedBy=%q want %q", state.ShadowedBy, override) + } + if state.InstructionsPath != override { + t.Fatalf("InstructionsPath=%q should be the file Codex actually reads", state.InstructionsPath) + } + if state.InstructionsWired { + t.Error("the override has no Gortex block, so the rule is not wired") + } +} + +// TestInspectIgnoresForeignHooks keeps doctor from crediting Gortex for a +// hook somebody else installed — which would mask a missing Gortex hook. +func TestInspectIgnoresForeignHooks(t *testing.T) { + env := codexGlobalEnv(t) + path := filepath.Join(env.Home, ".codex", "config.toml") + config := "" + + "[[hooks.SessionStart]]\n" + + "matcher = \"startup\"\n\n" + + "[[hooks.SessionStart.hooks]]\n" + + "type = \"command\"\n" + + "command = \"/usr/local/bin/some-other-tool notify\"\n" + if err := os.WriteFile(path, []byte(config), 0o644); err != nil { + t.Fatal(err) + } + + state := Inspect(env.Home) + if !state.ConfigPresent { + t.Fatal("config.toml not seen") + } + if state.Hooks["SessionStart"] != 0 { + t.Fatalf("counted a non-Gortex hook: %d", state.Hooks["SessionStart"]) + } +} + +// TestInspectSurvivesBrokenConfig — doctor exists to describe broken +// machines, so unreadable or malformed input must degrade, never panic. +func TestInspectSurvivesBrokenConfig(t *testing.T) { + env := codexGlobalEnv(t) + path := filepath.Join(env.Home, ".codex", "config.toml") + if err := os.WriteFile(path, []byte("this is not { valid toml ][\n"), 0o644); err != nil { + t.Fatal(err) + } + state := Inspect(env.Home) + if !state.ConfigPresent { + t.Fatal("a malformed config still exists on disk") + } + if state.MCPServer { + t.Error("nothing parseable, so nothing should be claimed") + } + + if got := Inspect(""); got.ConfigPresent { + t.Errorf("empty home should report nothing, got %+v", got) + } + if got := Inspect(filepath.Join(env.Home, "nope")); got.ConfigPresent { + t.Errorf("missing home should report nothing, got %+v", got) + } +} diff --git a/internal/doctor/findings.go b/internal/doctor/findings.go new file mode 100644 index 000000000..6814b02ed --- /dev/null +++ b/internal/doctor/findings.go @@ -0,0 +1,197 @@ +package doctor + +import ( + "fmt" + "sort" + "time" + + "github.com/zzet/gortex/internal/hooks" +) + +// Severity orders findings by how much they cost the user. +type Severity string + +const ( + // SeverityBlocker means the integration is not working. Doctor exits + // non-zero when one is present so CI can gate on it. + SeverityBlocker Severity = "BLOCKER" + // SeverityWarn means it works but is degraded or half-wired. + SeverityWarn Severity = "WARN" + SeverityOK Severity = "OK" + SeverityInfo Severity = "INFO" +) + +// Finding is one diagnosis. Remedy is the exact command or step that fixes +// it — a finding the reader cannot act on is a complaint, not a diagnosis. +type Finding struct { + Severity Severity `json:"severity"` + Summary string `json:"summary"` + Remedy string `json:"remedy,omitempty"` +} + +// AgentHooks is what an adapter's config declares, per lifecycle event. +type AgentHooks struct { + // Agent is the adapter name ("codex"). + Agent string + // Configured counts declared hook entries per event name. + Configured map[string]int + // RequiresTrust marks agents that gate hook execution behind an explicit + // per-hook approval, where "declared but never ran" is the expected + // signature of an unapproved hook rather than a broken install. + RequiresTrust bool + // TrustRemedy is how the user approves them. + TrustRemedy string + // InstructionsPath is the agent's user-level rules file, and + // InstructionsWired reports whether Gortex's block is in it. + InstructionsPath string + InstructionsWired bool + // InstructionsShadowed is set when the agent will read a different file + // instead, making the block above dead text. + InstructionsShadowed string +} + +// Diagnose turns the collected evidence into ordered findings, most +// actionable first. It takes plain data so the whole decision table is +// testable without a machine that has Codex, hooks, or a daemon on it. +func Diagnose(agent AgentHooks, activity hooks.EffectivenessSummary, adoption Adoption, now time.Time) []Finding { + var findings []Finding + add := func(sev Severity, remedy, format string, args ...any) { + findings = append(findings, Finding{Severity: sev, Summary: fmt.Sprintf(format, args...), Remedy: remedy}) + } + + var configured int + events := make([]string, 0, len(agent.Configured)) + for event, count := range agent.Configured { + configured += count + if count > 0 { + events = append(events, event) + } + } + sort.Strings(events) + + var ran int + for _, event := range events { + ran += activity.Events[event].Runs + } + + switch { + case configured == 0: + add(SeverityWarn, "gortex install", + "%s has no Gortex lifecycle hooks configured.", agent.Agent) + case ran == 0 && agent.RequiresTrust: + // The headline case. Every declared hook is on disk and none has + // run, which for a trust-gating agent means the hooks were never + // approved — or were re-approved away by an upgrade that changed + // their definitions, since trust is recorded against the hook hash. + add(SeverityBlocker, agent.TrustRemedy, + "%d %s hook(s) are configured but none has run in this window — %s skips new or changed hooks until they are trusted.", + configured, agent.Agent, agent.Agent) + case ran == 0: + add(SeverityBlocker, "check the agent's hook configuration and that `gortex` is on its PATH", + "%d %s hook(s) are configured but none has run in this window.", configured, agent.Agent) + } + + // A per-event zero while other events ran is a narrower version of the + // same problem: one hook's definition changed, so only its trust lapsed. + if ran > 0 { + for _, event := range events { + stats := activity.Events[event] + if stats.Runs > 0 { + continue + } + remedy := "re-run `gortex install`" + if agent.RequiresTrust { + remedy = agent.TrustRemedy + } + add(SeverityBlocker, remedy, + "%s is configured but never ran, while other hooks did%s.", event, lastSeenSuffix(stats.LastSeen, now)) + } + } + + for _, event := range events { + stats := activity.Events[event] + if stats.Runs == 0 { + continue + } + if stats.DaemonKnown > 0 && stats.DaemonUp == 0 { + add(SeverityBlocker, "gortex daemon start --detach", + "%s ran %d time(s) and never reached the daemon.", event, stats.Runs) + continue + } + if stats.Emitted == 0 { + add(SeverityWarn, "", + "%s ran %d time(s) and injected context 0 times — it fires but has nothing to say.", event, stats.Runs) + } + } + + if agent.InstructionsShadowed != "" { + add(SeverityWarn, fmt.Sprintf("move the Gortex block into %s, or remove it", agent.InstructionsShadowed), + "%s reads %s instead of %s, so any rule block in the latter is ignored.", + agent.Agent, agent.InstructionsShadowed, agent.InstructionsPath) + } else if agent.InstructionsPath != "" && !agent.InstructionsWired { + add(SeverityWarn, "gortex install", + "No Gortex rule block in %s — the agent loads that file into every session, so without it a skipped or silent hook leaves the session with no standing rule.", + agent.InstructionsPath) + } + + switch share := adoption.GortexShare(); { + case adoption.FilesFound == 0: + add(SeverityInfo, "", "No %s session transcripts found — adoption could not be measured.", agent.Agent) + case share < 0: + add(SeverityInfo, "", "No tool calls in the window — adoption could not be measured.") + case adoption.GortexCalls == 0: + add(SeverityBlocker, "gortex install", + "Across %d session(s) the model made %d shell call(s) and zero Gortex tool calls.", + adoption.InWindow, adoption.ShellCalls) + case share < 50: + add(SeverityWarn, "", + "Gortex tools are %.0f%% of tool calls (%d Gortex vs %d shell, %d read/search shaped).", + share, adoption.GortexCalls, adoption.ShellCalls, adoption.ShellReads) + default: + add(SeverityOK, "", + "Gortex tools are %.0f%% of tool calls (%d Gortex vs %d shell).", + share, adoption.GortexCalls, adoption.ShellCalls) + } + + sort.SliceStable(findings, func(i, j int) bool { + return severityRank(findings[i].Severity) < severityRank(findings[j].Severity) + }) + return findings +} + +// HasBlocker reports whether any finding is a blocker — the command's exit +// status. +func HasBlocker(findings []Finding) bool { + for _, f := range findings { + if f.Severity == SeverityBlocker { + return true + } + } + return false +} + +func severityRank(s Severity) int { + switch s { + case SeverityBlocker: + return 0 + case SeverityWarn: + return 1 + case SeverityInfo: + return 2 + default: + return 3 + } +} + +// lastSeenSuffix turns a never/stale timestamp into a clause, so "never ran" +// can distinguish "not in this window" from "not once, ever". +func lastSeenSuffix(last, now time.Time) string { + if last.IsZero() { + return " (not once, ever)" + } + days := int(now.Sub(last).Hours() / 24) + if days <= 0 { + return "" + } + return fmt.Sprintf(" (last ran %dd ago)", days) +} diff --git a/internal/doctor/findings_test.go b/internal/doctor/findings_test.go new file mode 100644 index 000000000..9313ea801 --- /dev/null +++ b/internal/doctor/findings_test.go @@ -0,0 +1,217 @@ +package doctor + +import ( + "strings" + "testing" + "time" + + "github.com/zzet/gortex/internal/hooks" +) + +func codexAgent() AgentHooks { + return AgentHooks{ + Agent: "codex", + Configured: map[string]int{"SessionStart": 1, "UserPromptSubmit": 1, "PreToolUse": 2, "PostToolUse": 1}, + RequiresTrust: true, + TrustRemedy: "run `/hooks` inside Codex", + InstructionsPath: "/home/u/.codex/AGENTS.md", + InstructionsWired: true, + } +} + +func activity(events map[string]hooks.EventActivity) hooks.EffectivenessSummary { + if events == nil { + events = map[string]hooks.EventActivity{} + } + return hooks.EffectivenessSummary{Present: true, Events: events} +} + +func findingWith(t *testing.T, findings []Finding, substr string) Finding { + t.Helper() + for _, f := range findings { + if strings.Contains(f.Summary, substr) { + return f + } + } + t.Fatalf("no finding containing %q in %+v", substr, findings) + return Finding{} +} + +func assertNoFindingWith(t *testing.T, findings []Finding, substr string) { + t.Helper() + for _, f := range findings { + if strings.Contains(f.Summary, substr) { + t.Fatalf("unexpected finding %q in %+v", substr, findings) + } + } +} + +// TestConfiguredButNeverRanIsABlocker is the reported failure: every hook is +// on disk and none of them runs, because Codex skips hooks it has not been +// told to trust. Config alone cannot distinguish this from a healthy install, +// so the finding has to carry the remedy. +func TestConfiguredButNeverRanIsABlocker(t *testing.T) { + findings := Diagnose(codexAgent(), activity(nil), Adoption{}, time.Now()) + + got := findingWith(t, findings, "none has run") + if got.Severity != SeverityBlocker { + t.Errorf("severity=%s want BLOCKER", got.Severity) + } + if !strings.Contains(got.Remedy, "/hooks") { + t.Errorf("remedy should name /hooks, got %q", got.Remedy) + } + if !HasBlocker(findings) { + t.Error("HasBlocker should be true so the command exits non-zero") + } + if findings[0].Severity != SeverityBlocker { + t.Errorf("blockers must sort first, got %+v", findings) + } +} + +// TestNeverRanWithoutTrustGateAvoidsTheTrustRemedy keeps the Codex-specific +// explanation from being handed to agents that do not gate on trust — a +// wrong remedy costs more than a vague one. +func TestNeverRanWithoutTrustGateAvoidsTheTrustRemedy(t *testing.T) { + agent := codexAgent() + agent.Agent = "claude-code" + agent.RequiresTrust = false + agent.TrustRemedy = "" + + findings := Diagnose(agent, activity(nil), Adoption{}, time.Now()) + got := findingWith(t, findings, "none has run") + if got.Severity != SeverityBlocker { + t.Errorf("severity=%s want BLOCKER", got.Severity) + } + if strings.Contains(got.Summary, "trusted") { + t.Errorf("trust wording leaked to a non-gating agent: %q", got.Summary) + } +} + +// TestSingleStaleHookIsReported covers the upgrade case: trust is recorded +// per hook hash, so changing one hook's definition silences only that hook. +func TestSingleStaleHookIsReported(t *testing.T) { + now := time.Now() + findings := Diagnose(codexAgent(), activity(map[string]hooks.EventActivity{ + "SessionStart": {Runs: 0, LastSeen: now.Add(-72 * time.Hour)}, + "UserPromptSubmit": {Runs: 5, Emitted: 5}, + "PreToolUse": {Runs: 40, Emitted: 12}, + "PostToolUse": {Runs: 3, Emitted: 1}, + }), Adoption{GortexCalls: 10, ShellCalls: 1}, now) + + got := findingWith(t, findings, "SessionStart is configured but never ran") + if got.Severity != SeverityBlocker { + t.Errorf("severity=%s want BLOCKER", got.Severity) + } + if !strings.Contains(got.Summary, "3d ago") { + t.Errorf("should date the last run so a stale hook is distinguishable: %q", got.Summary) + } + assertNoFindingWith(t, findings, "none has run") +} + +// TestHookThatRunsButNeverInjectsIsAWarn separates "never fires" from "fires +// and has nothing to say" — the same symptom, entirely different causes. +func TestHookThatRunsButNeverInjectsIsAWarn(t *testing.T) { + findings := Diagnose(codexAgent(), activity(map[string]hooks.EventActivity{ + "SessionStart": {Runs: 4, Emitted: 4, DaemonKnown: 4, DaemonUp: 4}, + "UserPromptSubmit": {Runs: 9, Emitted: 0, DaemonKnown: 9, DaemonUp: 9}, + "PreToolUse": {Runs: 9, Emitted: 3, DaemonKnown: 9, DaemonUp: 9}, + "PostToolUse": {Runs: 9, Emitted: 3, DaemonKnown: 9, DaemonUp: 9}, + }), Adoption{GortexCalls: 20, ShellCalls: 2}, time.Now()) + + got := findingWith(t, findings, "UserPromptSubmit ran 9") + if got.Severity != SeverityWarn { + t.Errorf("severity=%s want WARN", got.Severity) + } + if HasBlocker(findings) { + t.Errorf("a silent-but-live hook is not a blocker: %+v", findings) + } +} + +// TestUnreachableDaemonOutranksSilence: a hook that cannot reach the daemon +// is why it is silent, so reporting both would bury the cause under a +// symptom. +func TestUnreachableDaemonOutranksSilence(t *testing.T) { + findings := Diagnose(codexAgent(), activity(map[string]hooks.EventActivity{ + "SessionStart": {Runs: 3, Emitted: 0, DaemonKnown: 3, DaemonUp: 0}, + "UserPromptSubmit": {Runs: 3, Emitted: 3, DaemonKnown: 3, DaemonUp: 3}, + "PreToolUse": {Runs: 3, Emitted: 3, DaemonKnown: 3, DaemonUp: 3}, + "PostToolUse": {Runs: 3, Emitted: 3, DaemonKnown: 3, DaemonUp: 3}, + }), Adoption{GortexCalls: 5, ShellCalls: 1}, time.Now()) + + got := findingWith(t, findings, "never reached the daemon") + if got.Severity != SeverityBlocker { + t.Errorf("severity=%s want BLOCKER", got.Severity) + } + if !strings.Contains(got.Remedy, "daemon start") { + t.Errorf("remedy should start the daemon, got %q", got.Remedy) + } + assertNoFindingWith(t, findings, "SessionStart ran 3 time(s) and injected context 0") +} + +func TestInstructionsFindings(t *testing.T) { + healthy := activity(map[string]hooks.EventActivity{"SessionStart": {Runs: 2, Emitted: 2}}) + + t.Run("missing block", func(t *testing.T) { + agent := codexAgent() + agent.InstructionsWired = false + got := findingWith(t, Diagnose(agent, healthy, Adoption{}, time.Now()), "No Gortex rule block") + if got.Severity != SeverityWarn || got.Remedy != "gortex install" { + t.Errorf("got %+v", got) + } + }) + + t.Run("shadowed block", func(t *testing.T) { + agent := codexAgent() + agent.InstructionsShadowed = "/home/u/.codex/AGENTS.override.md" + findings := Diagnose(agent, healthy, Adoption{}, time.Now()) + got := findingWith(t, findings, "instead of") + if got.Severity != SeverityWarn { + t.Errorf("severity=%s want WARN", got.Severity) + } + // A wired-but-shadowed block must not also be reported as missing: + // the fix is different and the pair reads as contradictory. + assertNoFindingWith(t, findings, "No Gortex rule block") + }) +} + +func TestAdoptionFindings(t *testing.T) { + healthy := activity(map[string]hooks.EventActivity{ + "SessionStart": {Runs: 1, Emitted: 1}, "UserPromptSubmit": {Runs: 1, Emitted: 1}, + "PreToolUse": {Runs: 1, Emitted: 1}, "PostToolUse": {Runs: 1, Emitted: 1}, + }) + + cases := []struct { + name string + adoption Adoption + want Severity + substr string + }{ + {"all shell", Adoption{FilesFound: 3, InWindow: 3, ShellCalls: 20}, SeverityBlocker, "zero Gortex tool calls"}, + {"mostly shell", Adoption{FilesFound: 3, InWindow: 3, GortexCalls: 3, ShellCalls: 20, ShellReads: 15}, SeverityWarn, "13% of tool calls"}, + {"healthy", Adoption{FilesFound: 3, InWindow: 3, GortexCalls: 90, ShellCalls: 10}, SeverityOK, "90% of tool calls"}, + {"no transcripts", Adoption{}, SeverityInfo, "could not be measured"}, + {"no tool calls", Adoption{FilesFound: 4}, SeverityInfo, "could not be measured"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := findingWith(t, Diagnose(codexAgent(), healthy, tc.adoption, time.Now()), tc.substr) + if got.Severity != tc.want { + t.Errorf("severity=%s want %s (%q)", got.Severity, tc.want, got.Summary) + } + }) + } +} + +// TestNoHooksConfiguredIsNotATrustProblem: with nothing installed there is +// nothing to trust, so the remedy is the installer. +func TestNoHooksConfiguredIsNotATrustProblem(t *testing.T) { + agent := codexAgent() + agent.Configured = map[string]int{} + + findings := Diagnose(agent, activity(nil), Adoption{}, time.Now()) + got := findingWith(t, findings, "no Gortex lifecycle hooks") + if got.Severity != SeverityWarn || got.Remedy != "gortex install" { + t.Errorf("got %+v", got) + } + assertNoFindingWith(t, findings, "trusted") +} diff --git a/internal/doctor/sessions.go b/internal/doctor/sessions.go new file mode 100644 index 000000000..35d86195c --- /dev/null +++ b/internal/doctor/sessions.go @@ -0,0 +1,253 @@ +// Package doctor holds the runtime half of `gortex doctor`: the probes that +// answer questions an agent's config file cannot. +// +// An install is two claims — "Gortex is wired in" and "the agent is using +// it" — and only the first is visible on disk. Worse, some agents make the +// first claim unverifiable too: Codex records trust against each hook's hash +// and skips new or changed hooks until the user approves them in /hooks, so a +// fully populated config.toml and a completely inert integration are +// byte-identical. The probes here read the evidence instead: the hook +// invocation log (internal/hooks) and the agent's own session transcripts. +package doctor + +import ( + "bufio" + "encoding/json" + "io/fs" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "time" +) + +// shellTools are the Codex tool names that run a command. A read- or +// search-shaped call here is the behaviour the Gortex rule exists to replace, +// so it is the denominator adoption is measured against. +var shellTools = map[string]bool{ + "exec_command": true, + "shell": true, + "local_shell_call": true, + "container.exec": true, + "bash": true, +} + +// readLike matches the shell commands that read or search source. Deliberately +// generous: over-counting a `ls` as a read costs nothing, while missing a +// `rg` would hide the exact behaviour being diagnosed. +var readLike = regexp.MustCompile(`\b(rg|grep|egrep|fgrep|ag|ack|cat|head|tail|sed|awk|less|more|find|fd|ls|tree)\b`) + +// SessionRow is one agent session's tool-call tally. +type SessionRow struct { + Started string `json:"started,omitempty"` + Model string `json:"model,omitempty"` + CWD string `json:"cwd,omitempty"` + Branch string `json:"branch,omitempty"` + CLIVersion string `json:"cli_version,omitempty"` + Gortex int `json:"gortex_calls"` + OtherMCP int `json:"other_mcp_calls"` + Shell int `json:"shell_calls"` + ShellRead int `json:"shell_read_calls"` +} + +// Adoption is what the agent's transcripts say about tool choice. +// +// It deliberately carries no "did the hook fire" signal: Codex does not +// persist hook-injected context into its rollouts, so an absent orientation +// block proves nothing and must not be reported as if it did. Hook liveness +// comes from the invocation log alone. +type Adoption struct { + Root string `json:"root"` + FilesFound int `json:"files_found"` + InWindow int `json:"sessions_in_window"` + GortexCalls int `json:"gortex_calls"` + OtherMCP int `json:"other_mcp_calls"` + ShellCalls int `json:"shell_calls"` + ShellReads int `json:"shell_read_calls"` + Tools map[string]int `json:"tools,omitempty"` + Models map[string]int `json:"models,omitempty"` + CLIVersions map[string]int `json:"cli_versions,omitempty"` + Sessions []SessionRow `json:"sessions,omitempty"` +} + +// GortexShare is the fraction of graph-vs-shell tool calls that went to +// Gortex, in percent. Returns -1 when there were no calls to compare. +func (a Adoption) GortexShare() float64 { + total := a.GortexCalls + a.ShellCalls + if total == 0 { + return -1 + } + return 100 * float64(a.GortexCalls) / float64(total) +} + +// CodexHome resolves Codex's config root the way Codex does. +func CodexHome() string { + if v := strings.TrimSpace(os.Getenv("CODEX_HOME")); v != "" { + return v + } + home, err := os.UserHomeDir() + if err != nil { + return "" + } + return filepath.Join(home, ".codex") +} + +// maxSessionFiles bounds the scan. Rollouts accumulate indefinitely and a +// single one can reach hundreds of megabytes, so doctor reads the newest +// files and stops — a diagnostic that takes a minute does not get run. +const maxSessionFiles = 200 + +// ScanCodexSessions tallies tool calls across Codex rollout transcripts +// (~/.codex/sessions/YYYY/MM/DD/rollout-*.jsonl, one JSON object per line). +// Sessions with no tool calls are skipped: they say nothing about tool choice +// and would dilute the ratio. +// +// Every failure mode degrades to "less data", never an error: a missing +// directory means Codex was never run, and an unreadable or half-written +// rollout is skipped. Neither should take down a diagnostic whose whole job +// is to work on a broken machine. +func ScanCodexSessions(codexHome string, since time.Time, limit int) Adoption { + root := filepath.Join(codexHome, "sessions") + out := Adoption{ + Root: root, + Tools: map[string]int{}, + Models: map[string]int{}, + CLIVersions: map[string]int{}, + } + if codexHome == "" { + return out + } + + type candidate struct { + path string + mod time.Time + } + var files []candidate + _ = filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return nil // unreadable subtree: skip, don't abort + } + if d.IsDir() || !strings.HasSuffix(path, ".jsonl") { + return nil + } + info, statErr := d.Info() + if statErr != nil { + return nil + } + files = append(files, candidate{path: path, mod: info.ModTime()}) + return nil + }) + out.FilesFound = len(files) + sort.Slice(files, func(i, j int) bool { return files[i].mod.After(files[j].mod) }) + if len(files) > maxSessionFiles { + files = files[:maxSessionFiles] + } + + for _, file := range files { + if file.mod.Before(since) { + continue // rollouts are append-only: an old mtime means an old session + } + row, ok := scanCodexRollout(file.path) + if !ok { + continue + } + out.InWindow++ + out.GortexCalls += row.Gortex + out.OtherMCP += row.OtherMCP + out.ShellCalls += row.Shell + out.ShellReads += row.ShellRead + if row.Model != "" { + out.Models[row.Model]++ + } + if row.CLIVersion != "" { + out.CLIVersions[row.CLIVersion]++ + } + for name, n := range row.tools { + out.Tools[name] += n + } + if len(out.Sessions) < limit { + out.Sessions = append(out.Sessions, row.SessionRow) + } + } + return out +} + +type rollupRow struct { + SessionRow + tools map[string]int +} + +// maxRolloutLine bounds a single transcript line. Tool results are inlined +// into rollouts, so one line can carry a whole file; bufio's default 64 KiB +// would silently truncate the scan mid-file. +const maxRolloutLine = 8 << 20 + +func scanCodexRollout(path string) (rollupRow, bool) { + handle, err := os.Open(path) + if err != nil { + return rollupRow{}, false + } + defer handle.Close() + + row := rollupRow{tools: map[string]int{}} + scanner := bufio.NewScanner(handle) + scanner.Buffer(make([]byte, 0, 64<<10), maxRolloutLine) + for scanner.Scan() { + var rec struct { + Type string `json:"type"` + Payload struct { + Type string `json:"type"` + // function_call + Name string `json:"name"` + Namespace string `json:"namespace"` + Arguments string `json:"arguments"` + // session_meta + CWD string `json:"cwd"` + CLIVersion string `json:"cli_version"` + Timestamp string `json:"timestamp"` + Git struct { + Branch string `json:"branch"` + } `json:"git"` + // turn_context + Model string `json:"model"` + } `json:"payload"` + } + if json.Unmarshal(scanner.Bytes(), &rec) != nil { + continue + } + payload := rec.Payload + switch { + case rec.Type == "session_meta": + row.CWD = payload.CWD + row.CLIVersion = payload.CLIVersion + row.Started = payload.Timestamp + row.Branch = payload.Git.Branch + case rec.Type == "turn_context": + if payload.Model != "" { + row.Model = payload.Model + } + case payload.Type == "function_call": + switch { + case strings.Contains(payload.Namespace, "gortex") || strings.Contains(payload.Name, "gortex"): + row.Gortex++ + name := payload.Name + if name == "" { + name = payload.Namespace + } + row.tools[name]++ + case payload.Namespace != "": + row.OtherMCP++ + case shellTools[payload.Name]: + row.Shell++ + if readLike.MatchString(payload.Arguments) { + row.ShellRead++ + } + } + } + } + if row.Gortex == 0 && row.Shell == 0 && row.OtherMCP == 0 { + return rollupRow{}, false + } + return row, true +} diff --git a/internal/doctor/sessions_test.go b/internal/doctor/sessions_test.go new file mode 100644 index 000000000..ba4fff297 --- /dev/null +++ b/internal/doctor/sessions_test.go @@ -0,0 +1,177 @@ +package doctor + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// rollout writes a Codex-shaped transcript: one JSON object per line, tool +// calls carried as function_call payloads with a namespace for MCP tools. +func rollout(t *testing.T, home, day, name string, lines []map[string]any, mod time.Time) { + t.Helper() + dir := filepath.Join(home, "sessions", day) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + path := filepath.Join(dir, name) + var buf strings.Builder + for _, line := range lines { + blob, err := json.Marshal(line) + if err != nil { + t.Fatal(err) + } + buf.Write(blob) + buf.WriteByte('\n') + } + if err := os.WriteFile(path, []byte(buf.String()), 0o644); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(path, mod, mod); err != nil { + t.Fatal(err) + } +} + +func call(namespace, name, args string) map[string]any { + return map[string]any{"type": "response_item", "payload": map[string]any{ + "type": "function_call", "namespace": namespace, "name": name, "arguments": args, + }} +} + +func TestScanCodexSessionsClassifiesToolCalls(t *testing.T) { + home := t.TempDir() + now := time.Now() + rollout(t, home, "2026/07/26", "rollout-a.jsonl", []map[string]any{ + {"type": "session_meta", "payload": map[string]any{ + "cwd": "/repo", "cli_version": "0.145.0", "timestamp": "2026-07-26T09:11:55Z", + "git": map[string]any{"branch": "main"}, + }}, + {"type": "turn_context", "payload": map[string]any{"model": "gpt-5.6-sol"}}, + call("mcp__gortex", "explore", `{"operation":"task"}`), + call("mcp__gortex", "search", `{"operation":"symbols"}`), + call("mcp__other", "lookup", `{}`), + call("", "exec_command", `{"command":"rg TODO ."}`), + call("", "exec_command", `{"command":"go build ./..."}`), + }, now) + + got := ScanCodexSessions(home, now.Add(-24*time.Hour), 10) + + if got.FilesFound != 1 || got.InWindow != 1 { + t.Fatalf("files=%d inWindow=%d want 1/1", got.FilesFound, got.InWindow) + } + if got.GortexCalls != 2 { + t.Errorf("gortex=%d want 2", got.GortexCalls) + } + if got.OtherMCP != 1 { + t.Errorf("otherMCP=%d want 1 (a non-Gortex MCP call is not a shell call)", got.OtherMCP) + } + if got.ShellCalls != 2 { + t.Errorf("shell=%d want 2", got.ShellCalls) + } + if got.ShellReads != 1 { + t.Errorf("shellReads=%d want 1 — only the rg is a read/search", got.ShellReads) + } + if got.Tools["explore"] != 1 || got.Tools["search"] != 1 { + t.Errorf("tools=%v", got.Tools) + } + if got.Models["gpt-5.6-sol"] != 1 { + t.Errorf("models=%v", got.Models) + } + if share := got.GortexShare(); share < 49 || share > 51 { + t.Errorf("share=%.1f want ~50", share) + } + if len(got.Sessions) != 1 || got.Sessions[0].Branch != "main" || got.Sessions[0].CWD != "/repo" { + t.Errorf("session row=%+v", got.Sessions) + } +} + +// TestScanCodexSessionsWindowAndEmpties: sessions outside the window and +// sessions with no tool calls must not enter the ratio — a week of idle +// transcripts would otherwise read as an adoption collapse. +func TestScanCodexSessionsWindowAndEmpties(t *testing.T) { + home := t.TempDir() + now := time.Now() + rollout(t, home, "2026/07/26", "recent.jsonl", []map[string]any{ + call("mcp__gortex", "read", `{}`), + }, now) + rollout(t, home, "2026/06/01", "old.jsonl", []map[string]any{ + call("", "exec_command", `{"command":"grep -r x ."}`), + }, now.Add(-30*24*time.Hour)) + rollout(t, home, "2026/07/26", "chatonly.jsonl", []map[string]any{ + {"type": "response_item", "payload": map[string]any{"type": "message"}}, + }, now) + + got := ScanCodexSessions(home, now.Add(-24*time.Hour), 10) + + if got.FilesFound != 3 { + t.Errorf("filesFound=%d want 3 (every transcript on disk is counted)", got.FilesFound) + } + if got.InWindow != 1 { + t.Errorf("inWindow=%d want 1", got.InWindow) + } + if got.ShellCalls != 0 { + t.Errorf("shell=%d — the out-of-window session leaked in", got.ShellCalls) + } + if got.GortexCalls != 1 { + t.Errorf("gortex=%d want 1", got.GortexCalls) + } +} + +// TestScanCodexSessionsSurvivesGarbage — a half-written final line is normal +// for an append-only transcript from a killed process, and must not blind the +// scan to the good lines before it. +func TestScanCodexSessionsSurvivesGarbage(t *testing.T) { + home := t.TempDir() + now := time.Now() + dir := filepath.Join(home, "sessions", "2026/07/26") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + path := filepath.Join(dir, "torn.jsonl") + body := `{"type":"response_item","payload":{"type":"function_call","namespace":"mcp__gortex","name":"read"}}` + "\n" + + "not json at all\n" + `{"type":"response_item","payload":{"type":"functi` + if err := os.WriteFile(path, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(path, now, now); err != nil { + t.Fatal(err) + } + + got := ScanCodexSessions(home, now.Add(-24*time.Hour), 10) + if got.GortexCalls != 1 { + t.Errorf("gortex=%d want 1 — the good line before the garbage still counts", got.GortexCalls) + } +} + +func TestScanCodexSessionsMissingHome(t *testing.T) { + if got := ScanCodexSessions("", time.Now().Add(-time.Hour), 10); got.FilesFound != 0 { + t.Errorf("empty home should yield nothing, got %+v", got) + } + got := ScanCodexSessions(filepath.Join(t.TempDir(), "absent"), time.Now().Add(-time.Hour), 10) + if got.FilesFound != 0 || got.GortexShare() != -1 { + t.Errorf("missing sessions dir should yield nothing, got %+v", got) + } +} + +// TestScanCodexSessionsLimitsListedRows keeps the report readable while still +// aggregating every session in the window. +func TestScanCodexSessionsLimitsListedRows(t *testing.T) { + home := t.TempDir() + now := time.Now() + for i := 0; i < 5; i++ { + rollout(t, home, "2026/07/26", "s"+string(rune('a'+i))+".jsonl", []map[string]any{ + call("mcp__gortex", "read", `{}`), + }, now) + } + + got := ScanCodexSessions(home, now.Add(-24*time.Hour), 2) + if got.InWindow != 5 || got.GortexCalls != 5 { + t.Errorf("aggregate should cover every session: %+v", got) + } + if len(got.Sessions) != 2 { + t.Errorf("listed rows=%d want 2", len(got.Sessions)) + } +} diff --git a/internal/hooks/telemetry_read.go b/internal/hooks/telemetry_read.go new file mode 100644 index 000000000..8d42f683a --- /dev/null +++ b/internal/hooks/telemetry_read.go @@ -0,0 +1,129 @@ +package hooks + +import ( + "encoding/json" + "os" + "sort" + "strings" + "time" +) + +// Reading side of the effectiveness log. It lives next to the writer so the +// record shape has exactly one definition — a diagnostic that silently +// mis-parses its own telemetry is worse than no diagnostic. +// +// The log is the only evidence that separates "hook installed" from "hook +// running": every invocation appends a row whether or not it had anything to +// inject, so an event with zero rows did not run. That distinction is +// invisible in an agent's config file, which is what makes an untrusted or +// disabled hook look like a working one. + +// EffectivenessLogPath is where hook invocations are recorded. +func EffectivenessLogPath() string { return hookEffectivenessPath() } + +// EventActivity is one event's rollup over the requested window. +type EventActivity struct { + Event string `json:"event"` + // Runs counts invocations; Emitted counts the subset that injected + // context. Runs>0 with Emitted==0 is a hook that fires and has nothing + // to say — a different problem from one that never fires. + Runs int `json:"runs"` + Emitted int `json:"emitted"` + // DaemonKnown counts rows that reported reachability at all; hooks doing + // no daemon I/O omit it rather than report a false "down". + DaemonKnown int `json:"daemon_known"` + DaemonUp int `json:"daemon_up"` + // TotalMS across Runs, for a mean the caller can render. + TotalMS int64 `json:"total_ms"` + LastSeen time.Time `json:"last_seen,omitempty"` +} + +// EffectivenessSummary is the whole log reduced to per-event rollups. +// LastSeen is tracked across the entire log, not just the window, so a +// caller can say "last ran 9 days ago" instead of only "not in window". +type EffectivenessSummary struct { + Path string `json:"path"` + Present bool `json:"present"` + WindowRows int `json:"window_rows"` + TotalRows int `json:"total_rows"` + Events map[string]EventActivity `json:"events"` +} + +// Ran reports whether the event was invoked at least once in the window. +func (s EffectivenessSummary) Ran(event string) bool { + return s.Events[event].Runs > 0 +} + +// EventNames lists the events present in the summary, sorted. +func (s EffectivenessSummary) EventNames() []string { + out := make([]string, 0, len(s.Events)) + for name := range s.Events { + out = append(out, name) + } + sort.Strings(out) + return out +} + +// ReadEffectiveness aggregates the log, counting rows with ts >= since. A +// missing log is not an error: it means no hook has ever run on this machine, +// which is itself the answer the caller is looking for. +// +// Malformed lines are skipped rather than failing the read — the log is +// append-only from short-lived processes, so a torn final line during a crash +// must not blind the diagnostic to the 40,000 good rows before it. +func ReadEffectiveness(since time.Time) (EffectivenessSummary, error) { + path := EffectivenessLogPath() + summary := EffectivenessSummary{Path: path, Events: map[string]EventActivity{}} + if path == "" { + return summary, nil + } + data, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return summary, nil + } + return summary, err + } + summary.Present = true + + for _, line := range strings.Split(string(data), "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + var rec hookEffectiveness + if json.Unmarshal([]byte(line), &rec) != nil { + continue + } + summary.TotalRows++ + stamp, err := time.Parse(time.RFC3339Nano, rec.Timestamp) + if err != nil { + continue + } + event := rec.Event + if event == "" { + event = "?" + } + activity := summary.Events[event] + activity.Event = event + if stamp.After(activity.LastSeen) { + activity.LastSeen = stamp + } + if !stamp.Before(since) { + summary.WindowRows++ + activity.Runs++ + activity.TotalMS += rec.DurationMS + if rec.EmittedContext { + activity.Emitted++ + } + if rec.DaemonReachable != nil { + activity.DaemonKnown++ + if *rec.DaemonReachable { + activity.DaemonUp++ + } + } + } + summary.Events[event] = activity + } + return summary, nil +} diff --git a/internal/hooks/telemetry_read_test.go b/internal/hooks/telemetry_read_test.go new file mode 100644 index 000000000..be84f9336 --- /dev/null +++ b/internal/hooks/telemetry_read_test.go @@ -0,0 +1,145 @@ +package hooks + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func writeEffectivenessLog(t *testing.T, rows []hookEffectiveness) string { + t.Helper() + path := filepath.Join(t.TempDir(), "hook-effectiveness.jsonl") + var buf strings.Builder + for _, row := range rows { + blob, err := json.Marshal(row) + if err != nil { + t.Fatal(err) + } + buf.Write(blob) + buf.WriteByte('\n') + } + if err := os.WriteFile(path, []byte(buf.String()), 0o644); err != nil { + t.Fatal(err) + } + t.Setenv("GORTEX_HOOK_EFFECTIVENESS_LOG", path) + return path +} + +func stamp(at time.Time) string { return at.UTC().Format(time.RFC3339Nano) } + +func boolPtr(v bool) *bool { return &v } + +func TestReadEffectivenessAggregatesWithinWindow(t *testing.T) { + now := time.Now() + writeEffectivenessLog(t, []hookEffectiveness{ + {Timestamp: stamp(now.Add(-30 * time.Minute)), Event: "SessionStart", EmittedContext: true, DaemonReachable: boolPtr(true), DurationMS: 40}, + {Timestamp: stamp(now.Add(-20 * time.Minute)), Event: "UserPromptSubmit", EmittedContext: false, DaemonReachable: boolPtr(true), DurationMS: 10}, + {Timestamp: stamp(now.Add(-10 * time.Minute)), Event: "UserPromptSubmit", EmittedContext: false, DaemonReachable: boolPtr(false), DurationMS: 12}, + // Outside the window: must not count toward runs, but must still + // move LastSeen so a caller can say "last ran N days ago" instead of + // only "not in window". + {Timestamp: stamp(now.Add(-72 * time.Hour)), Event: "PostToolUse", EmittedContext: true, DaemonReachable: boolPtr(true)}, + }) + + got, err := ReadEffectiveness(now.Add(-time.Hour)) + if err != nil { + t.Fatalf("read: %v", err) + } + if !got.Present { + t.Fatal("log exists, Present should be true") + } + if got.TotalRows != 4 { + t.Errorf("totalRows=%d want 4", got.TotalRows) + } + if got.WindowRows != 3 { + t.Errorf("windowRows=%d want 3", got.WindowRows) + } + + ups := got.Events["UserPromptSubmit"] + if ups.Runs != 2 || ups.Emitted != 0 { + t.Errorf("UserPromptSubmit runs=%d emitted=%d want 2/0", ups.Runs, ups.Emitted) + } + if ups.DaemonKnown != 2 || ups.DaemonUp != 1 { + t.Errorf("daemon up=%d/%d want 1/2", ups.DaemonUp, ups.DaemonKnown) + } + if ups.TotalMS != 22 { + t.Errorf("totalMS=%d want 22", ups.TotalMS) + } + if !got.Ran("SessionStart") { + t.Error("SessionStart ran in the window") + } + if got.Ran("PostToolUse") { + t.Error("PostToolUse is outside the window and must not count as ran") + } + if last := got.Events["PostToolUse"].LastSeen; last.IsZero() { + t.Error("LastSeen must be tracked across the whole log, not just the window") + } +} + +// TestReadEffectivenessMissingLogIsNotAnError: no log means no hook has ever +// run, which is the answer doctor is looking for — not a failure. +func TestReadEffectivenessMissingLogIsNotAnError(t *testing.T) { + t.Setenv("GORTEX_HOOK_EFFECTIVENESS_LOG", filepath.Join(t.TempDir(), "absent.jsonl")) + + got, err := ReadEffectiveness(time.Now().Add(-time.Hour)) + if err != nil { + t.Fatalf("missing log should not error: %v", err) + } + if got.Present || got.TotalRows != 0 || got.Ran("SessionStart") { + t.Errorf("got %+v", got) + } +} + +// TestReadEffectivenessSkipsMalformedRows — the log is appended by +// short-lived processes, so a torn line during a crash is expected and must +// not blind the reader to the rows around it. +func TestReadEffectivenessSkipsMalformedRows(t *testing.T) { + now := time.Now() + path := writeEffectivenessLog(t, []hookEffectiveness{ + {Timestamp: stamp(now.Add(-time.Minute)), Event: "SessionStart", EmittedContext: true}, + }) + handle, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0o644) + if err != nil { + t.Fatal(err) + } + if _, err := handle.WriteString("garbage\n{\"ts\":\"not-a-time\",\"event\":\"SessionStart\"}\n{\"ts\":\"" + stamp(now) + "\",\"even"); err != nil { + t.Fatal(err) + } + handle.Close() + + got, err := ReadEffectiveness(now.Add(-time.Hour)) + if err != nil { + t.Fatalf("read: %v", err) + } + if got.Events["SessionStart"].Runs != 1 { + t.Errorf("runs=%d want 1 — the good row survives the garbage", got.Events["SessionStart"].Runs) + } +} + +// TestEventNamesSorted keeps the rendered table stable across runs. +func TestEventNamesSorted(t *testing.T) { + now := time.Now() + writeEffectivenessLog(t, []hookEffectiveness{ + {Timestamp: stamp(now), Event: "UserPromptSubmit"}, + {Timestamp: stamp(now), Event: "PostToolUse"}, + {Timestamp: stamp(now), Event: "SessionStart"}, + }) + + got, err := ReadEffectiveness(now.Add(-time.Hour)) + if err != nil { + t.Fatal(err) + } + names := got.EventNames() + want := []string{"PostToolUse", "SessionStart", "UserPromptSubmit"} + if len(names) != len(want) { + t.Fatalf("names=%v want %v", names, want) + } + for i := range want { + if names[i] != want[i] { + t.Fatalf("names=%v want %v", names, want) + } + } +}