diff --git a/README.md b/README.md index 93f197e6a..701699ec2 100644 --- a/README.md +++ b/README.md @@ -363,6 +363,7 @@ local Amp thread JSON files. | Forge | `~/.forge/` | | Gemini CLI | `~/.gemini/` | | Goose | `~/.local/share/goose/sessions/` (macOS and Linux), `%APPDATA%\\Block\\goose\\data\\sessions\\` (Windows) | +| Crush | `~/.local/share/crush/projects.json` registry pointing at per-project `~//.crush/crush.db` stores (macOS and Linux), `%LOCALAPPDATA%\\crush\\projects.json` (Windows) | | gptme | `~/.local/share/gptme/logs/` | | Grok | `~/.grok/sessions/` | | Hermes Agent | `~/.hermes/sessions/` | @@ -418,6 +419,14 @@ models, token usage, and recorded costs. Set `GOOSE_PATH_ROOT` to a Goose path root (sessions are read from `/data/sessions/`), or `agents.goose.dirs` to one or more data or sessions directories. +Crush sessions are read from each project's SQLite `.crush/crush.db`, including +transcript content, thinking, tool calls and results, session relationships, +models, and recorded session costs. The project registry lives at +`~/.local/share/crush/projects.json` (macOS and Linux) or +`%LOCALAPPDATA%\crush\projects.json` (Windows). Set `CRUSH_DIR` or +`agents.crush.dirs` to one or more Crush data directories, `.crush` +directories, or `crush.db` files. + Each directory can be overridden with an environment variable. See the [configuration docs](https://agentsview.io/configuration/) for details. Cursor attribution stats are a live, machine-local read from diff --git a/docs/changelog.md b/docs/changelog.md index df0a327d5..3d28e2108 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -31,6 +31,12 @@ description: Release history for AgentsView **Improvements** +- Sync sessions faster by reusing Git repository lookups for repeated working + directories within each sync operation. OpenCode and its Kilo and MiMoCode + forks, Command Code, and Kiro CLI now respect disabled filesystem discovery + during remote imports and capture. These sessions use recorded paths to name + projects, so their project names may change when re-parsed. (#1759) + - Expand a **Cursor CLI** session in the sidebar to read the transcripts of the subagents it delegated to. Transcripts stored in a parent session's `subagents` directory are now discovered, synced, and linked to the @@ -96,6 +102,9 @@ description: Release history for AgentsView **Bug fixes** +- Keep remotely imported project names independent of local repositories when + importing changed files, matching full-import behavior. (#1759) + - Generate insights and session analyses when opening AgentsView over HTTP outside localhost. Starting a report no longer requires a browser UUID API that is unavailable on those origins. (#1742) diff --git a/docs/configuration.md b/docs/configuration.md index c3198f3a6..f89ff24c0 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -392,6 +392,7 @@ keeps its default directories. | Codex | `~/.codex/sessions/` and `~/.codex/archived_sessions/` | JSONL per session | | Command Code | `~/.commandcode/projects/` | JSONL per session, optional `.meta.json` sidecar | | Copilot CLI | `~/.copilot/` | JSONL per session under `session-state/` | +| Crush | (platform-specific, see below) | Per-project SQLite `crush.db` with transcripts, tool activity, relationships, and recorded session costs | | Devin CLI | `~/.local/share/devin/` (Linux), `~/Library/Application Support/devin/` (macOS) | Local CLI data rooted at the directory that contains `cli/`; session data is discovered under `/cli/...` | | Evener | `~/.local/state/evener/` (or `$XDG_STATE_HOME/evener/`) | Semantic v2 `*.transcript.jsonl` and optional `*.meta.json` | | Cortex Code | `~/.snowflake/cortex/conversations/` | JSON / JSONL per session | @@ -501,6 +502,19 @@ directory. directly to that sessions directory, its parent data directory, or the database file. +**Crush default directories** are: + +- **macOS and Linux:** `~/.local/share/crush/` +- **Windows:** `%LOCALAPPDATA%/crush/` + +AgentsView reads `projects.json` in that directory and discovers one SQLite +`crush.db` per project under each listed `/.crush/` data directory. +Set `CRUSH_DIR` or `agents.crush.dirs` to override the default with one or more +directories: each entry may be a Crush data directory (containing +`projects.json`), a `/.crush` directory, or a `crush.db` file. A +project added after AgentsView starts is picked up by the next scheduled +reconciliation pass. + Omnigent sessions are read from `~/.omnigent/chat.db`. Set `OMNIGENT_DIR` or `agents.omnigent.dirs` to override the default directory. AgentsView creates one session per conversation and supports the split text-ID and current binary-UUID @@ -833,6 +847,7 @@ export CODEX_SESSIONS_DIR=~/custom/codex export CODEX_HOME=~/custom/codex-home # re-roots the default sessions/ paths export COMMANDCODE_PROJECTS_DIR=~/custom/commandcode export COPILOT_DIR=~/custom/copilot +export CRUSH_DIR=~/custom/crush export DEVIN_DIR=~/Library/Application\ Support/devin export CORTEX_DIR=~/custom/cortex export CURSOR_PROJECTS_DIR=~/custom/cursor diff --git a/docs/internal/session-format-sources.md b/docs/internal/session-format-sources.md index d4eba72fd..1c8002cae 100644 --- a/docs/internal/session-format-sources.md +++ b/docs/internal/session-format-sources.md @@ -110,14 +110,14 @@ add an archived or maintained mirror without replacing the original identity. - **Format:** Project-scoped JSONL transcripts, including subagent JSONL, with `user`, `assistant`, `system`, and progress records. -- **Title evidence (2026-09-13):** A local corpus measure sampled 768 files - and found 12,261 `ai-title` records, with a mean of 15.96 records per file - and a maximum of 454. No sampled `aiTitle` value was empty. `custom-title` +- **Title evidence (2026-09-13):** A local corpus measure sampled 768 files and + found 12,261 `ai-title` records, with a mean of 15.96 records per file and a + maximum of 454. No sampled `aiTitle` value was empty. `custom-title` occurred in 7 files, and `sessionName` did not occur. Native Claude parsing adopts non-empty `aiTitle` when no `/rename` is present; this target leaves - `custom-title` and `sessionName` to compatible producer parsing. - A title appended after the session is stored is persisted by one escalating - full parse while the stored name is still empty, and repeated records stay + `custom-title` and `sessionName` to compatible producer parsing. A title + appended after the session is stored is persisted by one escalating full + parse while the stored name is still empty, and repeated records stay incremental after that parse. A transcript that is no longer being written is not re-read, so it re-titles on its next full parse. @@ -2081,7 +2081,11 @@ preservation of archived messages for OpenCode, Kilo, MiMoCode, and Icodemate. - **Agentsview:** `internal/parser/kiro.go`, `internal/parser/kiro_sqlite.go`, and `internal/parser/kiro_provider.go`; both generations must remain - discoverable. + discoverable. SQLite project attribution uses `conversations_v2.key`, with + recorded environment metadata as the fallback when the key is empty. Bulk + and single-session parsing honor the caller's filesystem-discovery policy; + `TestKiroProviderSQLiteProjectDiscoveryPolicy` verifies project names and + filesystem probes with discovery enabled and disabled. ## Kiro IDE (`kiro-ide`) @@ -2987,6 +2991,57 @@ preservation of archived messages for OpenCode, Kilo, MiMoCode, and Icodemate. distinguish machines with identical root paths. No legacy Tau v1 conversion, native transfer, or index metadata synchronization is included. +## Charm Crush (`crush`) + +- **Format:** One SQLite `crush.db` per project under the project's `.crush` + data directory (configurable with `options.data_directory` or `--data-dir`). + Session metadata, cumulative token totals, and cost live in `sessions`; + ordered role messages with a JSON `parts` array (`text`, `reasoning`, + `tool_call`, `tool_result`, `finish`) live in `messages`. A project registry + at the global data directory maps project paths to data directories. + +- **Evidence:** `source`. + +- **Upstream:** Clone `https://github.com/charmbracelet/crush.git` at + `ce980ada68444b7591d8dfa631af7e94b2aba0b3`; see the pinned + [initial schema](https://github.com/charmbracelet/crush/blob/ce980ada68444b7591d8dfa631af7e94b2aba0b3/internal/db/migrations/20250424200609_initial.sql), + + [project registry](https://github.com/charmbracelet/crush/blob/ce980ada68444b7591d8dfa631af7e94b2aba0b3/internal/projects/projects.go), + and + [data-directory resolution](https://github.com/charmbracelet/crush/blob/ce980ada68444b7591d8dfa631af7e94b2aba0b3/internal/config/load.go). + The registry is `projects.json` next to the global config file: + `~/.local/share/crush/projects.json` (or `$XDG_DATA_HOME/crush/`, + `$CRUSH_GLOBAL_DATA/`) on macOS and Linux and + `%LOCALAPPDATA%\crush\projects.json` on Windows. Timestamps are Unix seconds + despite older schema comments claiming milliseconds; the `updated_at` + triggers write `strftime('%s','now')`. Later migrations add + `summary_message_id`, `todos`, `provider`, `is_summary_message`, + `read_files`, and Prism/Hyper metadata columns. The schema was reverified + against a live 2026-09 Crush store on 2026-09-11. + +- **Usage and cost:** `sessions.prompt_tokens` and `sessions.completion_tokens` + are cumulative session totals and `sessions.cost` is a recorded provider + cost, with no per-request breakdown and no per-message token fields. + Agentsview emits exactly one aggregate `session` usage event per session, + tagged with the most recent assistant message's model, and reports no + per-message token data. + +- **Agentsview:** `internal/parser/crush.go` and + `internal/parser/crush_provider.go` require the `messages.parts` column as + the format marker (the `goose_db_version` table belongs to Crush's vendored + goose migration tool and proves nothing), expand `projects.json` entries + into provider roots, attribute each session to the project directory above + the store, pair `tool_result` parts into system tool-result messages keyed + by call ID, emit `is_summary_message` rows as compact-boundary system + messages, and fingerprint the session and message rows so same-second edits + still invalidate freshness (`FingerprintHashRequiredForFreshness`). Watcher + events use bounded rowid cursors over `sessions` and `messages` so work + stays proportional to inserted rows, with a periodic reconciliation pass + covering metadata-only edits. Source row deletion is not authoritative. + Archived sessions remain active until the user deletes them in AgentsView. A + malformed `parts` value fails that session's parse rather than degrading + silently, matching the goose parser's policy. + [evener-source-1]: https://github.com/prime-radiant-inc/evener/blob/da7c06396c9848abfae362dcffce3861a6a0c95a/agent/transcript/transcript.go [evener-source-2]: https://github.com/prime-radiant-inc/evener/blob/da7c06396c9848abfae362dcffce3861a6a0c95a/agent/schema/turn.go [evener-source-3]: https://github.com/prime-radiant-inc/evener/blob/da7c06396c9848abfae362dcffce3861a6a0c95a/llm/types.go diff --git a/frontend/src/lib/utils/agents.test.ts b/frontend/src/lib/utils/agents.test.ts index 977375267..22292aba0 100644 --- a/frontend/src/lib/utils/agents.test.ts +++ b/frontend/src/lib/utils/agents.test.ts @@ -61,6 +61,7 @@ describe("KNOWN_AGENTS", () => { "omnigent", "codebuff", "freebuff", + "crush", ]); }); @@ -101,6 +102,7 @@ describe("agentColor", () => { expect(agentColor("piebald")).toBe("var(--accent-orange)"); expect(agentColor("roocode")).toBe("var(--accent-rose)"); expect(agentColor("omnigent")).toBe("var(--accent-teal)"); + expect(agentColor("crush")).toBe("var(--accent-coral)"); }); it("falls back to blue for unknown agents", () => { @@ -172,6 +174,7 @@ describe("agentLabel", () => { expect(agentLabel("omnigent")).toBe("Omnigent"); expect(agentLabel("traex")).toBe("TraeX"); expect(agentLabel("opencodereview")).toBe("Open Code Review"); + expect(agentLabel("crush")).toBe("Charm Crush"); }); it("capitalizes simple agent names", () => { diff --git a/frontend/src/lib/utils/agents.ts b/frontend/src/lib/utils/agents.ts index 7a5cdbd04..9d4e8a120 100644 --- a/frontend/src/lib/utils/agents.ts +++ b/frontend/src/lib/utils/agents.ts @@ -103,6 +103,11 @@ export const KNOWN_AGENTS: readonly AgentMeta[] = [ { name: "omnigent", color: "var(--accent-teal)", label: "Omnigent" }, { name: "codebuff", color: "var(--accent-amber)", label: "Codebuff" }, { name: "freebuff", color: "var(--accent-sky)", label: "Freebuff" }, + { + name: "crush", + color: "var(--accent-coral)", + label: "Charm Crush", + }, ]; const agentColorMap = new Map(KNOWN_AGENTS.map((a) => [a.name, a.color])); diff --git a/go.mod b/go.mod index cae8ccb36..24b873dbc 100644 --- a/go.mod +++ b/go.mod @@ -6,21 +6,21 @@ require ( github.com/BurntSushi/toml v1.6.0 github.com/ccoveille/go-safecast/v2 v2.0.1 github.com/danielgtaylor/huma/v2 v2.39.1 - github.com/dlclark/regexp2/v2 v2.7.1 + github.com/dlclark/regexp2/v2 v2.7.2 github.com/dmarkham/enumer v1.6.3 github.com/duckdb/duckdb-go/v2 v2.10505.0 github.com/fsnotify/fsnotify v1.10.1 - github.com/gofrs/flock v0.13.0 + github.com/gofrs/flock v0.13.1 github.com/google/go-cmp v0.7.0 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 github.com/jackc/pgx/v5 v5.10.0 - github.com/klauspost/compress v1.19.2 - github.com/mattn/go-runewidth v0.0.28 - github.com/mattn/go-sqlite3 v1.14.50 + github.com/klauspost/compress v1.20.0 + github.com/mattn/go-runewidth v0.0.29 + github.com/mattn/go-sqlite3 v1.14.52 github.com/minio/minio-go/v7 v7.3.0 github.com/modelcontextprotocol/go-sdk v1.7.0 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 - github.com/shirou/gopsutil/v4 v4.26.7 + github.com/shirou/gopsutil/v4 v4.26.8 github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 github.com/stretchr/testify v1.12.1 @@ -28,15 +28,16 @@ require ( github.com/testcontainers/testcontainers-go/modules/postgres v0.44.0 github.com/thlib/go-timezone-local v0.0.8 github.com/tidwall/gjson v1.19.0 - go.kenn.io/docbank v0.13.0 + go.kenn.io/docbank v0.14.0 go.kenn.io/kit v0.24.2-0.20260908151812-b9d6bb63077d - golang.org/x/mod v0.40.0 + golang.org/x/mod v0.41.0 golang.org/x/net v0.58.0 golang.org/x/perf v0.0.0-20260615155930-9e4b9ddef5b6 - golang.org/x/sync v0.22.0 - golang.org/x/sys v0.47.0 + golang.org/x/sync v0.23.0 + golang.org/x/sys v0.48.0 golang.org/x/term v0.45.0 - modernc.org/sqlite v1.57.0 + modernc.org/libc v1.75.6 + modernc.org/sqlite v1.58.0 ) require ( @@ -188,7 +189,6 @@ require ( google.golang.org/grpc v1.81.1 // indirect google.golang.org/protobuf v1.36.11 // indirect gopkg.in/ini.v1 v1.67.3 // indirect - modernc.org/libc v1.74.4 // indirect modernc.org/mathutil v1.7.1 // indirect - modernc.org/memory v1.11.0 // indirect + modernc.org/memory v1.12.1 // indirect ) diff --git a/go.sum b/go.sum index 7c13ebbd0..ad6b9dca3 100644 --- a/go.sum +++ b/go.sum @@ -85,8 +85,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/dlclark/regexp2/v2 v2.7.1 h1:yqDtwI1ptXXvEUNpYTk2lad4jLtAcKqkzepn4savSk4= -github.com/dlclark/regexp2/v2 v2.7.1/go.mod h1:avUrQvPaLz2DrFNHJF0taWAFFX2C1GMSSoeiqFjcBmU= +github.com/dlclark/regexp2/v2 v2.7.2 h1:nBhTfGMNPCDloto6XZRFJ5wpi+lym2eUMHRBY0diX30= +github.com/dlclark/regexp2/v2 v2.7.2/go.mod h1:avUrQvPaLz2DrFNHJF0taWAFFX2C1GMSSoeiqFjcBmU= github.com/dmarkham/enumer v1.6.3 h1:B4aV4OsfzbrS5rvjILt4mMjiWBA//cKxJUMsvHZ8mEI= github.com/dmarkham/enumer v1.6.3/go.mod h1:DyjXaqCglj4GhELF73oWiparNkYkXvmOBLza/o4kO74= github.com/docker/go-connections v0.7.0 h1:6SsRfJddP22WMrCkj19x9WKjEDTB+ahsdiGYf0mN39c= @@ -131,8 +131,8 @@ github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPE github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU= github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= -github.com/gofrs/flock v0.13.0 h1:95JolYOvGMqeH31+FC7D2+uULf6mG61mEZ/A8dRYMzw= -github.com/gofrs/flock v0.13.0/go.mod h1:jxeyy9R1auM5S6JYDBhDt+E2TCo7DkratH4Pgi8P+Z0= +github.com/gofrs/flock v0.13.1 h1:jjREztyBeSKBZYAC+mgc1laB+xsgy4kYMf3FbKF2UBo= +github.com/gofrs/flock v0.13.1/go.mod h1:sf4BFiHwnvgxa25DlQoDqXQnwRMEOwqxRq37P6MzzmE= github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= @@ -167,8 +167,8 @@ github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= -github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= -github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= @@ -186,10 +186,10 @@ github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8S github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= -github.com/mattn/go-runewidth v0.0.28 h1:rPyg2ybwEKPebvpzVWe1gKBkH8EQFkxO4Y0hjBeLaBU= -github.com/mattn/go-runewidth v0.0.28/go.mod h1:3qAiGCV4Koz/yuveO58qUefmUTRm8r0IGEXZ9jeHp/8= -github.com/mattn/go-sqlite3 v1.14.50 h1:dmdFvo1XG4MPzA4IkAmE9upVz/Nj31uRoM5+jC8hYbY= -github.com/mattn/go-sqlite3 v1.14.50/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= +github.com/mattn/go-runewidth v0.0.29 h1:3oGF3R/S2N9DQ3ptftzVIvg2eicmojCzlwBEmqEPDfQ= +github.com/mattn/go-runewidth v0.0.29/go.mod h1:3qAiGCV4Koz/yuveO58qUefmUTRm8r0IGEXZ9jeHp/8= +github.com/mattn/go-sqlite3 v1.14.52 h1:wVbm2Qnf4OXkqhBTSPuCRZDRnxfbVrrmiCEroVdog8U= +github.com/mattn/go-sqlite3 v1.14.52/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= github.com/mdelapenya/tlscert v0.2.0 h1:7H81W6Z/4weDvZBNOfQte5GpIMo0lGYEeWbkGp5LJHI= github.com/mdelapenya/tlscert v0.2.0/go.mod h1:O4njj3ELLnJjGdkN7M/vIVCpZ+Cf0L6muqOG4tLSl8o= github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs= @@ -265,8 +265,8 @@ github.com/segmentio/asm v1.1.3 h1:WM03sfUOENvvKexOLp+pCqgb/WDjsi7EK8gIsICtzhc= github.com/segmentio/asm v1.1.3/go.mod h1:Ld3L4ZXGNcSLRg4JBsZ3//1+f/TjYl0Mzen/DQy1EJg= github.com/segmentio/encoding v0.5.4 h1:OW1VRern8Nw6ITAtwSZ7Idrl3MXCFwXHPgqESYfvNt0= github.com/segmentio/encoding v0.5.4/go.mod h1:HS1ZKa3kSN32ZHVZ7ZLPLXWvOVIiZtyJnO1gPH1sKt0= -github.com/shirou/gopsutil/v4 v4.26.7 h1:IXzpHz/dkMRYAhKkOXr1HB6SuzWU3eoyyeWe7g3bNZc= -github.com/shirou/gopsutil/v4 v4.26.7/go.mod h1:5O9FjBiXoTDFatIWjZZosqj4pV0DRtLx598xGbBehzM= +github.com/shirou/gopsutil/v4 v4.26.8 h1:YQMTF/1J50B5+Y0vlo1eDRf5DoR7Gk69hY+8wjYkQeo= +github.com/shirou/gopsutil/v4 v4.26.8/go.mod h1:5O9FjBiXoTDFatIWjZZosqj4pV0DRtLx598xGbBehzM= github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= @@ -316,8 +316,8 @@ github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= -go.kenn.io/docbank v0.13.0 h1:sO4GsPNk2U/c7vAuIXLTVpYliZ59oVWzEA6+QsG61bE= -go.kenn.io/docbank v0.13.0/go.mod h1:FhKONSj2UYowwhgK2Lq3pIkYiIF00fSQt7PDBqqUbNc= +go.kenn.io/docbank v0.14.0 h1:qzkCnFIAKucpZIvblMouEV948MyjX+p49F+pqPtB5bk= +go.kenn.io/docbank v0.14.0/go.mod h1:sd4cATphlFCUU/Ea6VlB+LnFKFt18mbPvkDijhG42h4= go.kenn.io/kit v0.24.2-0.20260908151812-b9d6bb63077d h1:KJr8spxLUMutd0vGst6Fhnxo88Oogy2H4DTCD90PUVw= go.kenn.io/kit v0.24.2-0.20260908151812-b9d6bb63077d/go.mod h1:dO7SImegZe4P9PdnxuANdSQFbnrUvF3eE6/RDOfOMqo= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= @@ -393,23 +393,23 @@ golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= golang.org/x/exp v0.0.0-20260603202125-055de637280b h1:v1uXiEBHo8QA0LiGCo7UgHMzHT4Kdfpl2zmtH5vaP1Q= golang.org/x/exp v0.0.0-20260603202125-055de637280b/go.mod h1:d2fgXJLVs4dYDHUk5lwMIfzRzSrWCfGZb0ZqeLa/Vcw= -golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs= -golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE= +golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= +golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o= golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/perf v0.0.0-20260615155930-9e4b9ddef5b6 h1:YTGOochus7nQXtT5KnsVfJ3/1yslyOyP3m9vMG1qMuQ= golang.org/x/perf v0.0.0-20260615155930-9e4b9ddef5b6/go.mod h1:FisaKCtzcRx02gCop3DILSnoD7CMnqwmde2yVxo4meM= -golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= -golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= +golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= -golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/telemetry v0.0.0-20260811182544-a038080d80e5 h1:ZUSxONxc981v7AW7QUg+I9WwZzSTTJ019ENBYr5pV/Q= golang.org/x/telemetry v0.0.0-20260811182544-a038080d80e5/go.mod h1:LVehoXe41cL5SCVQilsV7Gg6BNG+Js6P9PhSbYTIUkQ= golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= @@ -439,30 +439,30 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= -modernc.org/cc/v4 v4.29.1 h1:MKgdCV3WykTSPqpVrnxdEDS0HEd2FHpKZDzxzU5LyeI= -modernc.org/cc/v4 v4.29.1/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= -modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU= -modernc.org/ccgo/v4 v4.34.6/go.mod h1:SZ8YcN9NG7XVsQYdm6jYBvi8PQP1qi+kqB6OhjqI3Fk= +modernc.org/cc/v4 v4.29.2 h1:h6+9ciCnPKutf4I03CvheAvDLX7+IHlqR6Iy6J+cgd8= +modernc.org/cc/v4 v4.29.2/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.35.0 h1:F+TUsmw09QxLzmi3aeYYGxjAXarmZaKgj3mKQHNaA8w= +modernc.org/ccgo/v4 v4.35.0/go.mod h1:qrVGs9S3Sr2Ztcg9ve+kTAYMp5a3YvWjo+SoN06kJ5I= modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= -modernc.org/gc/v3 v3.1.4 h1:2g65LGVSmFQrXeITAw97x7hCRvZFcyE1uDP+7Vng7JI= -modernc.org/gc/v3 v3.1.4/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/gc/v3 v3.1.5 h1:21ldfPfRYE31Tb7B3mwAK8gy1AxP4+dKjrOQPfqakoc= +modernc.org/gc/v3 v3.1.5/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= -modernc.org/libc v1.74.4 h1:fX1Omw4o2/1C2iRkkIsrQTasJQldLhRmuPreXLoWs9k= -modernc.org/libc v1.74.4/go.mod h1:eeQAS9W3sZeKYMFubydxJpII9ybHWshk+7or7bLG9co= +modernc.org/libc v1.75.6 h1:yKk8qo+Di4gkmvRboK8ocCqH22FiUCR6jRy2OwtCRus= +modernc.org/libc v1.75.6/go.mod h1:bO5o2ztHxBb2rjz0PgdHN0sSMw57CgxGFLZ3Qd/QpVQ= modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= -modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= -modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g= +modernc.org/memory v1.12.1/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= -modernc.org/sqlite v1.57.0 h1:qNQP6xnx5M0ISNtlnxoOX0+cD5bJ0/gr9aMmndFczzg= -modernc.org/sqlite v1.57.0/go.mod h1:yCJ2cmAaIkHQ25oXWrF8H4O1lIfPYPR26yCEDj2P3pQ= +modernc.org/sqlite v1.58.0 h1:38u40/bwkfM7f0Myhosl+SEMltSDxnGdQf8o6Kjmys0= +modernc.org/sqlite v1.58.0/go.mod h1:rsD2CckafgObKC4DhBlGBf+RiHxkc3hINGt1Xw32tVY= modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= diff --git a/internal/config/config.go b/internal/config/config.go index f8c135b8c..6136cede4 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1062,6 +1062,15 @@ func Default() (Config, error) { dirs = []string{filepath.Join(stateHome, "evener")} } } + // CRUSH_GLOBAL_DATA and XDG_DATA_HOME override Crush's default + // data directory following the upstream XDG conventions. + if def.Type == parser.AgentCrush && root == "" { + if crushGlobal := os.Getenv("CRUSH_GLOBAL_DATA"); crushGlobal != "" && filepath.IsAbs(crushGlobal) { + dirs = []string{crushGlobal} + } else if xdgData := os.Getenv("XDG_DATA_HOME"); xdgData != "" && filepath.IsAbs(xdgData) { + dirs = []string{filepath.Join(xdgData, "crush")} + } + } // Keep the Hermes profiles container as a stable provider root. The // provider enumerates its children on every discovery pass, so profiles // created after startup become visible without rebuilding Config or the diff --git a/internal/parser/capabilities.go b/internal/parser/capabilities.go index 78fb307a1..e888008f8 100644 --- a/internal/parser/capabilities.go +++ b/internal/parser/capabilities.go @@ -73,6 +73,10 @@ type SourceCapabilities struct { // its stored virtual members. A still-present container remains // authoritative for member deletion. PersistentArchive CapabilitySupport + // ExplicitDeletionOnly means source discovery and reconciliation may + // refresh sessions but must never mark an archived session missing. The + // user deletes these sessions explicitly from AgentsView. + ExplicitDeletionOnly CapabilitySupport // MultiFileStatHash declares that a provider's on-disk source layout // spans multiple sibling files (Codebuff's chat-messages.json plus // run-state.json and chat-meta.json, for example) and that the engine diff --git a/internal/parser/capabilities_sync_test.go b/internal/parser/capabilities_sync_test.go index 29f6b8e61..b1d749b4f 100644 --- a/internal/parser/capabilities_sync_test.go +++ b/internal/parser/capabilities_sync_test.go @@ -50,6 +50,10 @@ func TestProviderSyncSemanticsDeclarations(t *testing.T) { FingerprintHashInCacheKey: true, FingerprintHashRequiredForFreshness: true, }, + AgentCrush: { + FingerprintHashInCacheKey: true, + FingerprintHashRequiredForFreshness: true, + }, AgentZed: { UnchangedResults: UnchangedResultMTime, }, diff --git a/internal/parser/commandcode.go b/internal/parser/commandcode.go index 13eed0327..299e10645 100644 --- a/internal/parser/commandcode.go +++ b/internal/parser/commandcode.go @@ -1,6 +1,7 @@ package parser import ( + "context" "encoding/json/v2" "fmt" "os" @@ -19,9 +20,9 @@ type commandCodeMeta struct { Cwd string `json:"cwd"` } -// parseSession parses a Command Code JSONL transcript. -func (p *commandCodeProvider) parseSession( - path, machine string, +// parseSessionContext parses a Command Code JSONL transcript. +func (p *commandCodeProvider) parseSessionContext( + ctx context.Context, path, machine string, ) (*ParsedSession, []ParsedMessage, error) { info, err := os.Stat(path) if err != nil { @@ -168,7 +169,7 @@ func (p *commandCodeProvider) parseSession( sessionID = strings.TrimSuffix(filepath.Base(path), ".jsonl") } - project := ExtractProjectFromCwd(cwd) + project := ExtractProjectFromCwdWithBranchContext(ctx, cwd, "") if project == "" { project = NormalizeName(filepath.Base(filepath.Dir(path))) } diff --git a/internal/parser/commandcode_provider.go b/internal/parser/commandcode_provider.go index 59e08185d..3b2937fb3 100644 --- a/internal/parser/commandcode_provider.go +++ b/internal/parser/commandcode_provider.go @@ -98,7 +98,7 @@ func (p *commandCodeProvider) Parse( return ParseOutcome{}, fmt.Errorf("commandcode source path unavailable") } machine := firstNonEmptyJSONLString(req.Machine, p.Config.Machine) - sess, msgs, err := p.parseSession(path, machine) + sess, msgs, err := p.parseSessionContext(ctx, path, machine) if err != nil { return ParseOutcome{}, err } diff --git a/internal/parser/crush.go b/internal/parser/crush.go new file mode 100644 index 000000000..bc049d2d2 --- /dev/null +++ b/internal/parser/crush.go @@ -0,0 +1,761 @@ +package parser + +import ( + "context" + "database/sql" + "encoding/json/v2" + "fmt" + "math" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/tidwall/gjson" + + "go.kenn.io/agentsview/internal/money" +) + +// CrushDBName is the SQLite store filename inside each project's +// /.crush data directory. +const CrushDBName = "crush.db" + +// CrushProjectsFileName is the registry file inside Crush's data directory +// that maps project paths to their per-project data directories. +const CrushProjectsFileName = "projects.json" + +// crushSourceVersion identifies the Crush SQLite session format parsed here. +const crushSourceVersion = "crush-sqlite-v1" + +type crushSessionRow struct { + id string + title string + parentSessionID string + messageCount int64 + promptTokens int64 + completionTokens int64 + cost float64 + createdAt int64 + updatedAt int64 + maxMessageAt sql.NullInt64 +} + +// crushProjectsFile holds the structure of +// ~/.local/share/crush/projects.json. +type crushProjectsFile struct { + Projects []struct { + Path string `json:"path"` + DataDir string `json:"data_dir"` + } `json:"projects"` +} + +// crushProjectsDataDirs reads a Crush projects.json registry and returns +// the per-project data directories it lists. +func crushProjectsDataDirs(registryPath string) []string { + data, err := os.ReadFile(registryPath) + if err != nil { + return nil + } + var pf crushProjectsFile + if err := json.Unmarshal(data, &pf); err != nil { + return nil + } + dirs := make([]string, 0, len(pf.Projects)) + for _, project := range pf.Projects { + dir := strings.TrimSpace(project.DataDir) + if dir == "" { + continue + } + dirs = append(dirs, filepath.Clean(dir)) + } + return dirs +} + +// crushProjectDirsMapping reads a Crush projects.json registry and returns +// a mapping from each data directory to its project path. The project path +// is used for project attribution when the data directory does not follow +// the default /.crush layout. +func crushProjectDirsMapping(registryPath string) map[string]string { + data, err := os.ReadFile(registryPath) + if err != nil { + return nil + } + var pf crushProjectsFile + if err := json.Unmarshal(data, &pf); err != nil { + return nil + } + mapping := make(map[string]string, len(pf.Projects)) + for _, project := range pf.Projects { + dir := strings.TrimSpace(project.DataDir) + if dir == "" { + continue + } + path := strings.TrimSpace(project.Path) + if path == "" { + continue + } + mapping[filepath.Clean(dir)] = filepath.Clean(path) + } + return mapping +} + +func crushProjectDir(dbPath string, projectMapping map[string]string) string { + dataDir := filepath.Dir(dbPath) + if projectDir := projectMapping[dataDir]; projectDir != "" { + return projectDir + } + return filepath.Clean(filepath.Dir(dataDir)) +} + +func openCrushDB(dbPath string, stableSnapshot bool) (*sql.DB, error) { + // Immutable mode is used only for explicit stable snapshots. Live + // reads must not fall back to it: a WAL-backed store opened with + // immutable=1 ignores the WAL and can return stale session contents. + immutable := "0" + if stableSnapshot { + immutable = "1" + } + dsn := "file:" + sqliteURIPath(dbPath) + "?mode=ro&immutable=" + immutable + "&_busy_timeout=3000" + db, err := sql.Open("sqlite3", dsn) + if err != nil { + return nil, fmt.Errorf("opening crush sessions database %s: %w", dbPath, err) + } + if err := db.PingContext(context.Background()); err != nil { + _ = db.Close() + return nil, fmt.Errorf("opening crush sessions database %s: %w (WAL corruption is a possible cause; Crush must repair the store)", dbPath, err) + } + return db, nil +} + +func crushTableExists(ctx context.Context, db *sql.DB, table string) (bool, error) { + var count int + err := db.QueryRowContext(ctx, ` + SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = ? + `, table).Scan(&count) + if err != nil { + return false, fmt.Errorf("checking crush table %s: %w", table, err) + } + return count > 0, nil +} + +func crushTableColumns( + ctx context.Context, db *sql.DB, table string, +) (map[string]bool, error) { + rows, err := db.QueryContext(ctx, "PRAGMA table_info("+table+")") + if err != nil { + return nil, fmt.Errorf("listing crush %s columns: %w", table, err) + } + defer rows.Close() + columns := make(map[string]bool) + for rows.Next() { + var ( + cid int + name string + typeName string + notNull int + defaultV sql.NullString + pk int + ) + if err := rows.Scan(&cid, &name, &typeName, ¬Null, &defaultV, &pk); err != nil { + return nil, fmt.Errorf("scanning crush %s columns: %w", table, err) + } + columns[name] = true + } + if err := rows.Err(); err != nil { + return nil, err + } + return columns, nil +} + +// validateCrushSchema requires the Crush shape: sessions with title +// columns and messages with a parts JSON column. The parts column is the +// agent-specific marker — Crush vendors goose's migration tool, so a +// goose_db_version table proves nothing about the format. +func validateCrushSchema(ctx context.Context, db *sql.DB) error { + hasSessions, err := crushTableExists(ctx, db, "sessions") + if err != nil { + return err + } + hasMessages, err := crushTableExists(ctx, db, "messages") + if err != nil { + return err + } + if !hasSessions || !hasMessages { + return fmt.Errorf("unsupported crush schema: missing sessions or messages table") + } + sessionColumns, err := crushTableColumns(ctx, db, "sessions") + if err != nil { + return err + } + for _, required := range []string{"id", "title", "created_at", "updated_at"} { + if !sessionColumns[required] { + return fmt.Errorf("unsupported crush sessions schema: missing sessions.%s", required) + } + } + messageColumns, err := crushTableColumns(ctx, db, "messages") + if err != nil { + return err + } + for _, required := range []string{"id", "session_id", "role", "parts", "created_at"} { + if !messageColumns[required] { + return fmt.Errorf("unsupported crush messages schema: missing messages.%s", required) + } + } + return nil +} + +// crushSessionSelect joins per-session message activity so discovery and +// fingerprints can stat one snapshot. created_at values are Unix seconds +// despite schema comments claiming milliseconds. +const crushSessionSelect = ` + SELECT id, + COALESCE(title, ''), + COALESCE(parent_session_id, ''), + COALESCE(message_count, 0), + COALESCE(prompt_tokens, 0), + COALESCE(completion_tokens, 0), + COALESCE(cost, 0), + COALESCE(created_at, 0), + COALESCE(updated_at, 0), + (SELECT MAX(m.created_at) FROM messages m WHERE m.session_id = sessions.id) + FROM sessions +` + +func scanCrushSessionRow(scanner interface{ Scan(...any) error }) (crushSessionRow, error) { + var row crushSessionRow + err := scanner.Scan( + &row.id, &row.title, &row.parentSessionID, &row.messageCount, + &row.promptTokens, &row.completionTokens, &row.cost, + &row.createdAt, &row.updatedAt, &row.maxMessageAt, + ) + if err != nil { + return crushSessionRow{}, err + } + return row, nil +} + +func forEachCrushSessionMeta( + ctx context.Context, dbPath string, stableSnapshot bool, + yield func(dbBackedSessionMeta) error, +) error { + if !IsRegularFile(dbPath) { + return nil + } + db, err := openCrushDB(dbPath, stableSnapshot) + if err != nil { + return err + } + defer db.Close() + if err := validateCrushSchema(ctx, db); err != nil { + return err + } + rows, err := db.QueryContext(ctx, crushSessionSelect+" ORDER BY id") + if err != nil { + return fmt.Errorf("listing crush sessions: %w", err) + } + defer rows.Close() + for rows.Next() { + row, err := scanCrushSessionRow(rows) + if err != nil { + return fmt.Errorf("scanning crush session metadata: %w", err) + } + observeStreamingDiscoveryBuffer(ctx, 1) + if err := yield(dbBackedSessionMeta{ + SessionID: row.id, + VirtualPath: VirtualSourcePath(dbPath, row.id), + FileMtime: crushSessionMtime(dbPath, row), + }); err != nil { + return err + } + } + if err := rows.Err(); err != nil { + return err + } + return nil +} + +func crushSessionMeta( + ctx context.Context, dbPath, sessionID string, stableSnapshot bool, +) (dbBackedSessionMeta, bool, error) { + if !IsRegularFile(dbPath) { + return dbBackedSessionMeta{}, false, nil + } + db, err := openCrushDB(dbPath, stableSnapshot) + if err != nil { + return dbBackedSessionMeta{}, false, err + } + defer db.Close() + if err := validateCrushSchema(ctx, db); err != nil { + return dbBackedSessionMeta{}, false, err + } + row, err := scanCrushSessionRow(db.QueryRowContext( + ctx, crushSessionSelect+" WHERE sessions.id = ?", sessionID, + )) + if err == sql.ErrNoRows { + return dbBackedSessionMeta{}, false, nil + } + if err != nil { + return dbBackedSessionMeta{}, false, fmt.Errorf( + "loading crush session %s: %w", sessionID, err, + ) + } + return dbBackedSessionMeta{ + SessionID: row.id, + VirtualPath: VirtualSourcePath(dbPath, row.id), + FileMtime: crushSessionMtime(dbPath, row), + }, true, nil +} + +func crushSessionMtime(dbPath string, row crushSessionRow) int64 { + maxTime := maxCrushTime( + crushUnixTimestamp(row.updatedAt), + crushUnixTimestamp(row.createdAt), + crushUnixTimestamp(row.maxMessageAt.Int64), + ) + if !maxTime.IsZero() { + return maxTime.UnixNano() + } + mtime, _ := sqliteDBCompositeMtime(dbPath, []string{"", "-wal"}) + return mtime +} + +func parseCrushSession( + ctx context.Context, dbPath, sessionID, machine string, + stableSnapshot bool, projectMapping map[string]string, +) (*ParsedSession, []ParsedMessage, error) { + db, err := openCrushDB(dbPath, stableSnapshot) + if err != nil { + return nil, nil, err + } + defer db.Close() + if err := validateCrushSchema(ctx, db); err != nil { + return nil, nil, err + } + row, err := scanCrushSessionRow(db.QueryRowContext( + ctx, crushSessionSelect+" WHERE sessions.id = ?", sessionID, + )) + if err == sql.ErrNoRows { + return nil, nil, sql.ErrNoRows + } + if err != nil { + return nil, nil, fmt.Errorf("loading crush session %s: %w", sessionID, err) + } + messages, err := loadCrushMessages(ctx, db, sessionID) + if err != nil { + return nil, nil, err + } + links, err := crushSubagentLinks(ctx, db, sessionID) + if err != nil { + return nil, nil, err + } + for i := range messages { + for j := range messages[i].ToolCalls { + if child, ok := links[messages[i].ToolCalls[j].ToolUseID]; ok { + messages[i].ToolCalls[j].SubagentSessionID = child + } + } + } + + // Resolve the project directory: first check the registry mapping for + // an explicit project path, then fall back to the two-levels-above + // heuristic for the default /.crush layout. + projectDir := crushProjectDir(dbPath, projectMapping) + project := ExtractProjectFromCwdWithBranchContext(ctx, projectDir, "") + if project == "" { + project = "crush" + } + + sessionName := strings.TrimSpace(row.title) + firstMessage := crushFirstMessage(messages) + if firstMessage == "" && sessionName != "" { + firstMessage = truncate(strings.ReplaceAll(sessionName, "\n", " "), 300) + } + + userMessages := 0 + for _, message := range messages { + if message.Role == RoleUser && len(message.ToolResults) == 0 { + userMessages++ + } + } + + startedAt := crushUnixTimestamp(row.createdAt) + endedAt := crushUnixTimestamp(row.updatedAt) + for _, message := range messages { + if message.Timestamp.After(endedAt) { + endedAt = message.Timestamp + } + if startedAt.IsZero() || + (!message.Timestamp.IsZero() && message.Timestamp.Before(startedAt)) { + startedAt = message.Timestamp + } + } + if startedAt.IsZero() { + startedAt = endedAt + } + if endedAt.IsZero() { + endedAt = startedAt + } + + session := &ParsedSession{ + ID: "crush:" + row.id, + Project: project, + Machine: machine, + Agent: AgentCrush, + Cwd: projectDir, + SourceSessionID: row.id, + SourceVersion: crushSourceVersion, + FirstMessage: firstMessage, + SessionName: sessionName, + StartedAt: startedAt, + EndedAt: endedAt, + MessageCount: len(messages), + UserMessageCount: userMessages, + CountsAuthoritative: true, + File: FileInfo{ + Path: VirtualSourcePath(dbPath, row.id), + Mtime: crushSessionMtime(dbPath, row), + }, + } + if info, err := os.Stat(dbPath); err == nil { + session.File.Size = info.Size() + } + if parentID := strings.TrimSpace(row.parentSessionID); parentID != "" { + session.ParentSessionID = "crush:" + parentID + session.RelationshipType = RelSubagent + } + usageEvents := crushUsageEvents(session, row, messages) + applyUsageEventTokenTotals(session, usageEvents) + session.UsageEvents = usageEvents + return session, messages, nil +} + +// crushSubagentLinks maps a session's prefixed tool-call IDs to the +// crush-prefixed child session they spawned. Crush names subagent sessions +// "$$" under the delegating session. +func crushSubagentLinks( + ctx context.Context, db *sql.DB, sessionID string, +) (map[string]string, error) { + rows, err := db.QueryContext(ctx, ` + SELECT id FROM sessions WHERE parent_session_id = ? + `, sessionID) + if err != nil { + return nil, fmt.Errorf("listing crush child sessions for %s: %w", sessionID, err) + } + defer rows.Close() + links := make(map[string]string) + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + return nil, fmt.Errorf("scanning crush child session: %w", err) + } + if idx := strings.LastIndex(id, "$$"); idx >= 0 && idx+2 < len(id) { + links["crush:"+id[idx+2:]] = "crush:" + id + } + } + if err := rows.Err(); err != nil { + return nil, err + } + return links, nil +} + +func loadCrushMessages( + ctx context.Context, db *sql.DB, sessionID string, +) ([]ParsedMessage, error) { + columns, err := crushTableColumns(ctx, db, "messages") + if err != nil { + return nil, fmt.Errorf("inspecting crush messages columns: %w", err) + } + selectCols := []string{ + "id", + "COALESCE(role, '')", + "COALESCE(parts, '[]')", + "COALESCE(model, '')", + "COALESCE(created_at, 0)", + } + if columns["provider"] { + selectCols = append(selectCols, "COALESCE(provider, '')") + } + if columns["is_summary_message"] { + selectCols = append(selectCols, "COALESCE(is_summary_message, 0)") + } + query := "SELECT " + strings.Join(selectCols, ", ") + ` + FROM messages + WHERE session_id = ? + ORDER BY created_at, rowid` + rows, err := db.QueryContext(ctx, query, sessionID) + if err != nil { + return nil, fmt.Errorf("listing crush messages for %s: %w", sessionID, err) + } + defer rows.Close() + parsed := make([]ParsedMessage, 0) + for rows.Next() { + var ( + rowID string + role string + parts string + model string + createdAt int64 + provider string + isSummary int64 + ) + scanArgs := []any{&rowID, &role, &parts, &model, &createdAt} + if columns["provider"] { + scanArgs = append(scanArgs, &provider) + } + if columns["is_summary_message"] { + scanArgs = append(scanArgs, &isSummary) + } + if err := rows.Scan(scanArgs...); err != nil { + return nil, fmt.Errorf("scanning crush message row: %w", err) + } + message, ok, err := buildCrushMessage( + ctx, len(parsed), rowID, role, parts, model, provider, createdAt, + isSummary != 0, + ) + if err != nil { + return nil, err + } + if ok { + parsed = append(parsed, message) + } + } + if err := rows.Err(); err != nil { + return nil, err + } + return parsed, nil +} + +func buildCrushMessage( + ctx context.Context, ordinal int, rowID, role, parts, model, provider string, + createdAt int64, isSummary bool, +) (ParsedMessage, bool, error) { + contentJSON := gjson.Parse(parts) + if !gjson.Valid(parts) || !contentJSON.IsArray() { + return ParsedMessage{}, false, fmt.Errorf( + "parsing crush message %s parts: expected JSON array", rowID, + ) + } + message := ParsedMessage{ + Ordinal: ordinal, + Timestamp: crushUnixTimestamp(createdAt), + SourceUUID: rowID, + } + switch strings.ToLower(strings.TrimSpace(role)) { + case "user": + message.Role = RoleUser + case "assistant": + message.Role = RoleAssistant + message.Model = strings.TrimSpace(model) + message.ProviderID = strings.TrimSpace(provider) + case "tool": + message.Role = RoleUser + message.IsSystem = true + default: + return ParsedMessage{}, false, nil + } + if isSummary { + // Crush marks condensed conversation summaries with + // is_summary_message=1 (backed by sessions.summary_message_id); + // they are context-management boundaries, not conversation + // content of the row's original role. + message.Role = RoleSystem + message.IsSystem = true + message.IsCompactBoundary = true + message.Model = "" + message.ProviderID = "" + } + + var texts []string + var thinking []string + contentJSON.ForEach(func(_, part gjson.Result) bool { + switch part.Get("type").Str { + case "text": + if text := strings.TrimSpace(part.Get("data.text").Str); text != "" { + texts = append(texts, text) + } + case "reasoning": + message.HasThinking = true + if text := strings.TrimSpace(part.Get("data.thinking").Str); text != "" { + thinking = append(thinking, text) + texts = append(texts, "[Thinking]\n"+text+"\n[/Thinking]") + } + case "tool_call": + if call, ok := crushParseToolCall(ctx, rowID, part); ok { + message.HasToolUse = true + message.ToolCalls = append(message.ToolCalls, call) + } + case "tool_result": + if result, ok := crushParseToolResult(part); ok { + message.ToolResults = append(message.ToolResults, result) + } + case "finish": + if message.Role == RoleAssistant { + if reason := strings.TrimSpace(part.Get("data.reason").Str); reason != "" { + message.StopReason = reason + } + } + default: + // Unknown part types are skipped; Crush adds part types in + // minor releases and unknown data must not fail the session. + } + return true + }) + message.Content = strings.Join(texts, "\n") + message.ThinkingText = strings.Join(thinking, "\n\n") + message.ContentLength = len(message.Content) + return message, true, nil +} + +func crushParseToolCall( + ctx context.Context, rowID string, part gjson.Result, +) (ParsedToolCall, bool) { + data := part.Get("data") + name := strings.TrimSpace(data.Get("name").Str) + if name == "" { + return ParsedToolCall{}, false + } + toolUseID := strings.TrimSpace(data.Get("id").Str) + if toolUseID == "" { + // Payload call IDs are unique per request; fall back to the row ID + // plus part index so repeated calls never share a ToolUseID. + toolUseID = rowID + ":" + strconv.Itoa(part.Index) + } + inputJSON := data.Get("input").Str + if !gjson.Valid(inputJSON) { + inputJSON = "{}" + } + call := ParsedToolCall{ + ToolUseID: "crush:" + toolUseID, + ToolName: name, + Category: NormalizeToolCategory(name), + InputJSON: inputJSON, + SkillName: inferToolSkillName(ctx, name, inputJSON), + } + return call, true +} + +func crushParseToolResult(part gjson.Result) (ParsedToolResult, bool) { + data := part.Get("data") + toolUseID := strings.TrimSpace(data.Get("tool_call_id").Str) + if toolUseID == "" { + return ParsedToolResult{}, false + } + content := data.Get("content") + if !content.Exists() || content.Type == gjson.Null { + return ParsedToolResult{ToolUseID: "crush:" + toolUseID, ContentRaw: "null"}, true + } + var quoted []byte + if content.Type == gjson.String { + quoted, _ = json.Marshal(content.Str) + } else { + quoted = []byte(content.Raw) + } + return ParsedToolResult{ + ToolUseID: "crush:" + toolUseID, + ContentLength: len(content.Str), + ContentRaw: string(quoted), + }, true +} + +func crushFirstMessage(messages []ParsedMessage) string { + for _, message := range messages { + if message.Role != RoleUser || message.IsSystem { + continue + } + text := strings.TrimSpace(message.Content) + if text == "" { + continue + } + return truncate(strings.ReplaceAll(text, "\n", " "), 300) + } + return "" +} + +// crushUsageEvents emits one aggregate usage event per session. Crush +// tracks cumulative session totals (prompt_tokens, completion_tokens, cost) +// with no per-request breakdown; the model is the most recent assistant +// message's model. +func crushUsageEvents( + session *ParsedSession, row crushSessionRow, messages []ParsedMessage, +) []ParsedUsageEvent { + promptTokens := nonnegativeCrushToken(row.promptTokens) + completionTokens := nonnegativeCrushToken(row.completionTokens) + cost := row.cost + if promptTokens <= 0 && completionTokens <= 0 && cost <= 0 { + return nil + } + event := ParsedUsageEvent{ + SessionID: "crush:" + row.id, + Source: "session", + Model: crushLatestModel(messages), + ProviderID: crushLatestProvider(messages), + InputTokens: promptTokens, + OutputTokens: completionTokens, + OccurredAt: timeString(session.EndedAt, session.StartedAt), + DedupKey: "session:crush:" + row.id + "|aggregate", + } + if cost >= 0 && !math.IsNaN(cost) && !math.IsInf(cost, 0) { + if parsed, err := money.FromFloatDollars(cost); err == nil { + event.Cost = &parsed + event.CostStatus = "unknown" + event.CostSource = "crush-session" + } + } + return []ParsedUsageEvent{event} +} + +func crushLatestModel(messages []ParsedMessage) string { + model := "" + for _, message := range messages { + if message.Role == RoleAssistant && message.Model != "" { + model = message.Model + } + } + return model +} + +func crushLatestProvider(messages []ParsedMessage) string { + provider := "" + for _, message := range messages { + if message.Role == RoleAssistant && message.ProviderID != "" { + provider = message.ProviderID + } + } + return provider +} + +func nonnegativeCrushToken(value int64) int { + if value <= 0 { + return 0 + } + maxInt := int64(^uint(0) >> 1) + if value > maxInt { + return int(maxInt) + } + return int(value) +} + +// crushUnixTimestamp decodes Crush timestamps. The schema comments claim +// milliseconds, but Crush writes Unix seconds (its update triggers use +// strftime('%s','now')); values beyond the millisecond threshold are +// decoded as milliseconds for forward compatibility. +func crushUnixTimestamp(value int64) time.Time { + if value <= 0 { + return time.Time{} + } + if value >= 10_000_000_000 { + return time.UnixMilli(value).UTC() + } + return time.Unix(value, 0).UTC() +} + +func maxCrushTime(values ...time.Time) time.Time { + var result time.Time + for _, value := range values { + if value.After(result) { + result = value + } + } + return result +} diff --git a/internal/parser/crush_paths.go b/internal/parser/crush_paths.go new file mode 100644 index 000000000..9cca68d52 --- /dev/null +++ b/internal/parser/crush_paths.go @@ -0,0 +1,15 @@ +package parser + +// crushDefaultDirs returns the platform data directories that contain the +// Crush project registry. Paths are relative to the user home so +// nonexistent entries for other platforms are skipped during discovery. +// Crush resolves its global data dir via XDG on macOS and Linux and +// %LOCALAPPDATA% on Windows (internal/config/load.go upstream). +func crushDefaultDirs() []string { + return []string{ + // macOS and Linux + ".local/share/crush", + // Windows + "AppData/Local/crush", + } +} diff --git a/internal/parser/crush_provider.go b/internal/parser/crush_provider.go new file mode 100644 index 000000000..850d7f622 --- /dev/null +++ b/internal/parser/crush_provider.go @@ -0,0 +1,657 @@ +package parser + +import ( + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "fmt" + "hash" + "maps" + "path/filepath" + "strconv" + "strings" + "sync" +) + +type crushChildRelationshipsCacheEntry struct { + mu sync.Mutex + known bool + state SQLiteContainerState + children map[string][]string +} + +type crushChildRelationshipsCache struct { + mu sync.Mutex + entries map[string]*crushChildRelationshipsCacheEntry +} + +type crushProviderFactory struct { + def AgentDef + tracker *sqliteChangeTracker +} + +func newCrushProviderFactory(def AgentDef) ProviderFactory { + return &crushProviderFactory{ + def: cloneAgentDef(def), + tracker: &sqliteChangeTracker{ + agent: AgentCrush, open: openCrushDB, schema: crushCursorSchema, + }, + } +} + +func (f *crushProviderFactory) Definition() AgentDef { + return cloneAgentDef(f.def) +} + +func (f *crushProviderFactory) Capabilities() Capabilities { + return withDBBackedRawCapture(crushProviderCapabilities()) +} + +func (f *crushProviderFactory) NewProvider(cfg ProviderConfig) Provider { + cfg = cfg.Clone() + originalRoots := make([]string, len(cfg.Roots)) + copy(originalRoots, cfg.Roots) + expandedRoots, registryMapping, projectMapping := normalizeCrushRoots(cfg.Roots) + cfg.Roots = expandedRoots + spec := crushProviderSpec(cfg.StableSourceSnapshots) + base := &dbBackedProvider{ + Def: cloneAgentDef(f.def), + Caps: withDBBackedRawCapture(spec.caps), + Config: cfg, + spec: spec, + sources: newDBBackedSourceSet(spec, cfg.Roots), + } + base.sources.tracker = f.tracker + base.sources.stableSnapshot = cfg.StableSourceSnapshots + p := &crushProvider{ + dbBackedProvider: base, + registryMapping: registryMapping, + projectMapping: projectMapping, + configuredRoot: crushConfiguredRootByExpanded(originalRoots, registryMapping), + } + // Replace the parse closure to capture the project mapping so + // parseCrushSession can attribute sessions to the correct project + // when the data directory does not follow the default layout. + p.spec.parse = func( + ctx context.Context, dbPath, sessionID, machine string, + ) ([]ParseResult, error) { + sess, msgs, err := parseCrushSession( + ctx, dbPath, sessionID, machine, + cfg.StableSourceSnapshots, p.projectMapping, + ) + if err != nil || sess == nil { + return nil, err + } + return []ParseResult{{ + Session: *sess, + Messages: msgs, + UsageEvents: sess.UsageEvents, + }}, nil + } + return p +} + +type crushProvider struct { + *dbBackedProvider + registryMapping map[string][]string + projectMapping map[string]string + childCache crushChildRelationshipsCache + // configuredRoot maps each expanded data directory back onto the + // configured registry, crush.db, or data-directory root that produced + // it, so source-machine mapping stays bound to the user's spelling. + configuredRoot map[string]string +} + +// crushConfiguredRootByExpanded maps every expanded data directory onto the +// configured root that produced it. Later originals do not overwrite earlier +// ones: the first configured spelling wins when roots collapse together. +func crushConfiguredRootByExpanded( + originalRoots []string, registryMapping map[string][]string, +) map[string]string { + mapping := make(map[string]string, len(originalRoots)) + add := func(expanded, original string) { + expanded = filepath.Clean(expanded) + original = filepath.Clean(original) + if expanded == "" || original == "" || expanded == "." || original == "." { + return + } + if _, ok := mapping[expanded]; !ok { + mapping[expanded] = original + } + } + for _, root := range originalRoots { + cleaned := filepath.Clean(root) + if cleaned == "" || cleaned == "." { + continue + } + physical := cleaned + if container, _, ok := ParseVirtualSourcePath(physical); ok { + physical = container + } + if filepath.Base(physical) == CrushDBName { + add(filepath.Dir(physical), cleaned) + continue + } + if dataDirs, ok := registryMapping[cleaned]; ok { + for _, dir := range dataDirs { + add(dir, cleaned) + } + continue + } + add(cleaned, cleaned) + } + return mapping +} + +// withConfiguredRoot stamps the original configured root onto a source whose +// expanded data directory came from a registry or crush.db spelling. +func (p *crushProvider) withConfiguredRoot(source SourceRef) SourceRef { + if source.ConfiguredRoot != "" { + return source + } + src, ok := source.Opaque.(dbBackedSource) + if !ok { + return source + } + if configured, ok := p.configuredRoot[src.Root]; ok { + source.ConfiguredRoot = configured + } + return source +} + +func (p *crushProvider) withConfiguredRoots(sources []SourceRef) []SourceRef { + for i := range sources { + sources[i] = p.withConfiguredRoot(sources[i]) + } + return sources +} + +// ResolveReconciliationScopes expands registry roots to their per-project +// data directories before scope resolution. The configured roots contain +// only expanded data directories, so a request root that is a registry +// directory would otherwise match no scope. A crush.db database-file root +// or virtual member widens through the container topology onto the owning +// data directory so virtual session members stay in the proof scope. +// TraversalRoots are rewritten back onto the original configured spelling +// so a scoped NewProvider reconstruction re-applies registry expansion, +// project mapping, and configured-root machine attribution. +func (p *crushProvider) ResolveReconciliationScopes( + _ context.Context, req ReconciliationScopeRequest, +) (ReconciliationScopePlan, error) { + expanded := make([]string, 0, len(req.Roots)) + for _, root := range req.Roots { + if dataDirs, ok := p.registryMapping[filepath.Clean(root)]; ok { + expanded = append(expanded, dataDirs...) + continue + } + expanded = append(expanded, root) + } + if err := ValidateReconciliationScopeRoots( + p.Def.Type, p.Config.Roots, expanded, + ); err != nil { + return ReconciliationScopePlan{}, err + } + plan := containerAwareReconciliationScopePlan( + p.Config.Roots, expanded, p.reconciliationContainer, + ) + for i := range plan.Scopes { + plan.Scopes[i].TraversalRoots = p.withOriginalTraversalRoots( + plan.Scopes[i].TraversalRoots, + ) + } + return plan, nil +} + +// withOriginalTraversalRoots maps each expanded data directory back onto the +// configured registry, crush.db, or data-directory root that produced it. A +// registry traversal also includes all of its expanded data directories so +// sibling projects discovered through the registry stay inside the declared +// traversal boundary. +func (p *crushProvider) withOriginalTraversalRoots(roots []string) []string { + out := make([]string, 0, len(roots)) + seen := make(map[string]struct{}, len(roots)) + appendRoot := func(root string) { + root = filepath.Clean(root) + if _, ok := seen[root]; ok { + return + } + seen[root] = struct{}{} + out = append(out, root) + } + for _, root := range roots { + if original, ok := p.configuredRoot[filepath.Clean(root)]; ok { + appendRoot(original) + for _, dataDir := range p.registryMapping[filepath.Clean(original)] { + appendRoot(dataDir) + } + continue + } + appendRoot(root) + } + return out +} + +// reconciliationContainer maps a crush.db path or virtual member onto the +// owning data directory, which is the spelling configured roots carry after +// normalizeCrushRoots. Classification must not stat: a deleted database must +// still resolve so its members remain reclaimable. +func (p *crushProvider) reconciliationContainer(requested string) (string, bool) { + physical := requested + if container, _, ok := ParseVirtualSourcePath(physical); ok { + physical = container + } + if filepath.Base(physical) != CrushDBName { + return "", false + } + return filepath.Dir(physical), true +} + +func (p *crushProvider) Discover(ctx context.Context) ([]SourceRef, error) { + sources, err := p.dbBackedProvider.Discover(ctx) + if err != nil { + return nil, err + } + return p.withConfiguredRoots(sources), nil +} + +func (p *crushProvider) DiscoverEach( + ctx context.Context, yield func(SourceRef) error, +) error { + return p.dbBackedProvider.DiscoverEach(ctx, func(source SourceRef) error { + return yield(p.withConfiguredRoot(source)) + }) +} + +func (p *crushProvider) SourcesForChangedPath( + ctx context.Context, req ChangedPathRequest, +) ([]SourceRef, error) { + sources, err := p.dbBackedProvider.SourcesForChangedPath(ctx, req) + if err != nil { + return nil, err + } + return p.withConfiguredRoots(sources), nil +} + +func (p *crushProvider) FindSource( + ctx context.Context, req FindSourceRequest, +) (SourceRef, bool, error) { + source, found, err := p.dbBackedProvider.FindSource(ctx, req) + if err != nil || !found { + return source, found, err + } + return p.withConfiguredRoot(source), true, nil +} + +func (p *crushProvider) Fingerprint( + ctx context.Context, source SourceRef, +) (SourceFingerprint, error) { + fingerprint, err := p.dbBackedProvider.Fingerprint(ctx, source) + if err != nil { + return SourceFingerprint{}, err + } + src, ok := p.sources.sourceFromRef(source) + if !ok || !IsRegularFile(src.DBPath) { + return fingerprint, nil + } + hash, found, err := crushSessionFingerprint( + ctx, src.DBPath, src.SessionID, p.Config.StableSourceSnapshots, + &p.childCache, + ) + if err != nil { + return SourceFingerprint{}, err + } + if found { + hasher := sha256.New() + crushWriteFingerprintField(hasher, hash) + crushWriteFingerprintField( + hasher, crushProjectDir(src.DBPath, p.projectMapping), + ) + fingerprint.Hash = hex.EncodeToString(hasher.Sum(nil)) + } + return fingerprint, nil +} + +func crushProviderCapabilities() Capabilities { + source := dbBackedSourceCapabilities(CapabilityNotApplicable) + // Crush does not consume stored source hints; scheduling them would + // enumerate every session for each WAL event. + source.StoredSourceHints = CapabilityUnsupported + source.ExplicitDeletionOnly = CapabilitySupported + return Capabilities{ + Source: source, + Content: ContentCapabilities{ + FirstMessage: CapabilitySupported, + SessionName: CapabilitySupported, + Cwd: CapabilitySupported, + Relationships: CapabilitySupported, + Thinking: CapabilitySupported, + ToolCalls: CapabilitySupported, + ToolResults: CapabilitySupported, + AggregateUsageEvents: CapabilitySupported, + Model: CapabilitySupported, + StopReason: CapabilitySupported, + }, + Sync: ProviderSyncSemantics{ + FingerprintHashInCacheKey: true, + FingerprintHashRequiredForFreshness: true, + }, + } +} + +func crushProviderSpec(stableSnapshot bool) dbBackedProviderSpec { + return dbBackedProviderSpec{ + agent: AgentCrush, + dbName: CrushDBName, + findDB: crushDBPath, + streamMeta: func( + ctx context.Context, dbPath string, yield func(dbBackedSessionMeta) error, + ) error { + return forEachCrushSessionMeta(ctx, dbPath, stableSnapshot, yield) + }, + metaForID: func( + ctx context.Context, dbPath, sessionID string, + ) (dbBackedSessionMeta, bool, error) { + return crushSessionMeta(ctx, dbPath, sessionID, stableSnapshot) + }, + parse: func( + ctx context.Context, dbPath, sessionID, machine string, + ) ([]ParseResult, error) { + sess, msgs, err := parseCrushSession(ctx, dbPath, sessionID, machine, stableSnapshot, nil) + if err != nil || sess == nil { + return nil, err + } + // The engine writes usage rows only from ParseResult + // .UsageEvents; the ParsedSession field feeds ID validation. + return []ParseResult{{ + Session: *sess, + Messages: msgs, + UsageEvents: sess.UsageEvents, + }}, nil + }, + caps: crushProviderCapabilities(), + } +} + +// normalizeCrushRoots expands configured roots into per-project data +// directories and returns the expanded roots alongside a mapping from +// each original registry root to its expanded data directories. A root +// is one of: +// - a directory directly holding crush.db (a /.crush data dir) +// - the path to a crush.db file itself +// - a Crush data directory holding projects.json, whose listed data +// dirs are each expanded (deduplicated); an unreadable or empty +// registry leaves the root in place rather than failing discovery +func normalizeCrushRoots(roots []string) ([]string, map[string][]string, map[string]string) { + cleaned := cleanJSONLRoots(roots) + out := make([]string, 0, len(cleaned)) + seen := make(map[string]struct{}, len(cleaned)) + registryMapping := make(map[string][]string) + projectMapping := make(map[string]string) + add := func(root string) { + if _, ok := seen[root]; ok { + return + } + seen[root] = struct{}{} + out = append(out, root) + } + for _, root := range cleaned { + root = filepath.Clean(root) + if root == "" || root == "." { + continue + } + if filepath.Base(root) == CrushDBName { + add(filepath.Dir(root)) + continue + } + if IsRegularFile(filepath.Join(root, CrushDBName)) { + add(root) + continue + } + expanded := crushProjectsDataDirs(filepath.Join(root, CrushProjectsFileName)) + if len(expanded) == 0 { + add(root) + continue + } + registryMapping[root] = expanded + mapping := crushProjectDirsMapping(filepath.Join(root, CrushProjectsFileName)) + maps.Copy(projectMapping, mapping) + for _, dir := range expanded { + add(dir) + } + } + return out, registryMapping, projectMapping +} + +func crushDBPath(dir string) string { + if dir == "" { + return "" + } + path := filepath.Join(dir, CrushDBName) + if !IsRegularFile(path) { + return "" + } + return path +} + +// crushSessionFingerprint hashes the session row and every message row so a +// same-second metadata or parts edit produces a fresh fingerprint even +// though the store's second-resolution timestamps did not move. +func crushSessionFingerprint( + ctx context.Context, dbPath, sessionID string, stableSnapshot bool, + childCache *crushChildRelationshipsCache, +) (string, bool, error) { + db, err := openCrushDB(dbPath, stableSnapshot) + if err != nil { + return "", false, err + } + defer db.Close() + row, err := scanCrushSessionRow(db.QueryRowContext( + ctx, crushSessionSelect+" WHERE sessions.id = ?", sessionID, + )) + if err == sql.ErrNoRows { + return "", false, nil + } + if err != nil { + return "", false, fmt.Errorf("fingerprinting crush session %s: %w", sessionID, err) + } + hasher := sha256.New() + for _, value := range []string{ + row.id, row.title, row.parentSessionID, + strconv.FormatInt(row.messageCount, 10), + strconv.FormatInt(row.promptTokens, 10), + strconv.FormatInt(row.completionTokens, 10), + strconv.FormatFloat(row.cost, 'g', -1, 64), + strconv.FormatInt(row.createdAt, 10), + strconv.FormatInt(row.updatedAt, 10), + strconv.FormatInt(row.maxMessageAt.Int64, 10), + } { + crushWriteFingerprintField(hasher, value) + } + messageColumns, err := crushTableColumns(ctx, db, "messages") + if err != nil { + return "", false, fmt.Errorf("fingerprinting crush messages: %w", err) + } + columns := []struct { + name string + exists bool + express string + }{ + {"id", true, "id"}, + {"session_id", true, "session_id"}, + {"role", true, "COALESCE(role, '')"}, + {"parts", true, "COALESCE(parts, '')"}, + {"model", true, "COALESCE(model, '')"}, + {"provider", messageColumns["provider"], "COALESCE(provider, '')"}, + {"created_at", true, "CAST(COALESCE(created_at, 0) AS TEXT)"}, + {"updated_at", messageColumns["updated_at"], "CAST(COALESCE(updated_at, 0) AS TEXT)"}, + {"finished_at", messageColumns["finished_at"], "COALESCE(CAST(finished_at AS TEXT), '')"}, + {"is_summary_message", messageColumns["is_summary_message"], "CAST(COALESCE(is_summary_message, 0) AS TEXT)"}, + } + selectExprs := make([]string, 0, len(columns)) + for _, col := range columns { + if col.exists { + selectExprs = append(selectExprs, col.express) + } + } + selectStmt := "SELECT " + strings.Join(selectExprs, ", ") + + " FROM messages WHERE session_id = ? ORDER BY rowid" + messageRows, err := db.QueryContext(ctx, selectStmt, sessionID) + if err != nil { + return "", false, fmt.Errorf("fingerprinting crush messages: %w", err) + } + defer messageRows.Close() + for messageRows.Next() { + values := make([]string, len(selectExprs)) + destinations := make([]any, len(values)) + for i := range values { + destinations[i] = &values[i] + } + if err := messageRows.Scan(destinations...); err != nil { + return "", false, fmt.Errorf("scanning crush fingerprint message: %w", err) + } + for _, value := range values { + crushWriteFingerprintField(hasher, value) + } + } + if err := messageRows.Err(); err != nil { + return "", false, err + } + children, err := crushChildSessionIDsCached(ctx, db, dbPath, childCache) + if err != nil { + return "", false, fmt.Errorf("fingerprinting crush child sessions: %w", err) + } + for _, childID := range children[sessionID] { + crushWriteFingerprintField(hasher, childID) + } + return hex.EncodeToString(hasher.Sum(nil)), true, nil +} + +// crushChildSessionIDsCached scans the unindexed parent relationship once and +// reuses the grouped result until SQLiteContainerState proves the DB changed. +func crushChildSessionIDsCached( + ctx context.Context, db *sql.DB, dbPath string, + cache *crushChildRelationshipsCache, +) (map[string][]string, error) { + entry := cache.entry(dbPath) + entry.mu.Lock() + defer entry.mu.Unlock() + + state, ok := StatSQLiteContainerState(dbPath) + if !ok { + return loadCrushChildSessionIDs(ctx, db) + } + if entry.known && entry.state == state { + return entry.children, nil + } + children, err := loadCrushChildSessionIDs(ctx, db) + if err != nil { + return nil, err + } + after, unchanged := StatSQLiteContainerState(dbPath) + if unchanged && after == state { + entry.known = true + entry.state = state + entry.children = children + } else { + entry.known = false + entry.children = nil + } + return children, nil +} + +func (c *crushChildRelationshipsCache) entry( + dbPath string, +) *crushChildRelationshipsCacheEntry { + c.mu.Lock() + defer c.mu.Unlock() + key := filepath.Clean(dbPath) + if c.entries == nil { + c.entries = make(map[string]*crushChildRelationshipsCacheEntry) + } + entry := c.entries[key] + if entry == nil { + entry = &crushChildRelationshipsCacheEntry{} + c.entries[key] = entry + } + return entry +} + +func loadCrushChildSessionIDs( + ctx context.Context, db *sql.DB, +) (map[string][]string, error) { + rows, err := db.QueryContext(ctx, ` + SELECT parent_session_id, id + FROM sessions + WHERE parent_session_id IS NOT NULL + ORDER BY parent_session_id, id + `) + if err != nil { + return nil, err + } + defer rows.Close() + children := make(map[string][]string) + for rows.Next() { + var parentID, childID string + if err := rows.Scan(&parentID, &childID); err != nil { + return nil, err + } + if idx := strings.LastIndex(childID, "$$"); idx >= 0 && idx+2 < len(childID) { + children[parentID] = append(children[parentID], childID) + } + } + return children, rows.Err() +} + +func crushWriteFingerprintField(hasher hash.Hash, value string) { + _, _ = hasher.Write([]byte(strconv.Itoa(len(value)))) + _, _ = hasher.Write([]byte{':'}) + _, _ = hasher.Write([]byte(value)) +} + +func crushCursorSchema( + ctx context.Context, db *sql.DB, +) (int, []sqliteCursorTable, error) { + version, err := crushSchemaVersion(ctx, db) + if err != nil { + return 0, nil, err + } + columns, err := crushTableColumns(ctx, db, "messages") + if err != nil { + return 0, nil, fmt.Errorf("inspecting crush messages columns: %w", err) + } + messageIdentity := "session_id || char(31) || COALESCE(role, '') || char(31) || " + + "CAST(COALESCE(created_at, 0) AS TEXT)" + if columns["finished_at"] { + messageIdentity += " || char(31) || COALESCE(CAST(finished_at AS TEXT), '')" + } + return version, []sqliteCursorTable{ + { + name: "sessions", rowID: "rowid", sessionID: "id", + identity: "CAST(id AS TEXT)", + }, + { + name: "messages", rowID: "rowid", sessionID: "session_id", + identity: messageIdentity, + }, + }, nil +} + +// crushSchemaVersion reads the vendored goose migration version so a +// re-written or downgraded database invalidates stored cursors. +func crushSchemaVersion(ctx context.Context, db *sql.DB) (int, error) { + hasVersion, err := crushTableExists(ctx, db, "goose_db_version") + if err != nil { + return 0, err + } + if !hasVersion { + return 0, nil + } + var version int + if err := db.QueryRowContext( + ctx, "SELECT COALESCE(MAX(version_id), 0) FROM goose_db_version", + ).Scan(&version); err != nil { + return 0, fmt.Errorf("reading crush schema version: %w", err) + } + return version, nil +} diff --git a/internal/parser/crush_test.go b/internal/parser/crush_test.go new file mode 100644 index 000000000..e2b7f14eb --- /dev/null +++ b/internal/parser/crush_test.go @@ -0,0 +1,482 @@ +package parser + +import ( + "context" + "database/sql" + "os" + "path/filepath" + "testing" + "time" + + _ "github.com/mattn/go-sqlite3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/agentsview/internal/money" +) + +const crushTestSchema = ` + CREATE TABLE sessions ( + id TEXT PRIMARY KEY, + parent_session_id TEXT, + title TEXT NOT NULL, + message_count INTEGER NOT NULL DEFAULT 0, + prompt_tokens INTEGER NOT NULL DEFAULT 0, + completion_tokens INTEGER NOT NULL DEFAULT 0, + cost REAL NOT NULL DEFAULT 0.0, + updated_at INTEGER NOT NULL, + created_at INTEGER NOT NULL, + summary_message_id TEXT, + todos TEXT + ); + CREATE TABLE messages ( + id TEXT PRIMARY KEY, + session_id TEXT NOT NULL, + role TEXT NOT NULL, + parts TEXT NOT NULL DEFAULT '[]', + model TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + finished_at INTEGER, + provider TEXT, + is_summary_message INTEGER DEFAULT 0 NOT NULL + ); + CREATE INDEX idx_messages_session_id ON messages (session_id); +` + +type crushTestFixture struct { + projectDir string + dataDir string + dbPath string + database *sql.DB +} + +func newCrushTestFixture(t *testing.T) *crushTestFixture { + t.Helper() + projectDir := t.TempDir() + dataDir := filepath.Join(projectDir, ".crush") + require.NoError(t, os.MkdirAll(dataDir, 0o755)) + dbPath := filepath.Join(dataDir, CrushDBName) + database, err := sql.Open("sqlite3", dbPath) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, database.Close()) }) + _, err = database.Exec(crushTestSchema) + require.NoError(t, err) + return &crushTestFixture{ + projectDir: projectDir, + dataDir: dataDir, + dbPath: dbPath, + database: database, + } +} + +func (f *crushTestFixture) insertSession( + t *testing.T, id, title, parentID string, + created, updated int64, prompt, completion int64, cost float64, +) { + t.Helper() + _, err := f.database.Exec(` + INSERT INTO sessions ( + id, parent_session_id, title, created_at, updated_at, + prompt_tokens, completion_tokens, cost + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + `, id, nullableCrushTestString(parentID), title, created, updated, + prompt, completion, cost) + require.NoError(t, err) +} + +func (f *crushTestFixture) insertMessage( + t *testing.T, id, sessionID, role, parts string, created int64, + model, provider string, +) { + t.Helper() + _, err := f.database.Exec(` + INSERT INTO messages ( + id, session_id, role, parts, model, created_at, updated_at, + provider + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + `, id, sessionID, role, parts, nullableCrushTestString(model), + created, created, nullableCrushTestString(provider)) + require.NoError(t, err) +} + +func nullableCrushTestString(value string) any { + if value == "" { + return nil + } + return value +} + +func TestCrushProviderParsesTranscriptToolsAndUsage(t *testing.T) { + fixture := newCrushTestFixture(t) + // Unix seconds, 2026 era: 1789093626 is 2026-09-10 UTC. + const created = int64(1_789_093_626) + fixture.insertSession(t, "sess-1", "Review auth flow", "", + created, created+120, 43_922, 185, 0.0126) + fixture.insertSession(t, "child$$chatcmpl-tool-1", "Child", "sess-1", + created, created, 0, 0, 0) + fixture.insertMessage(t, "msg-user", "sess-1", "user", `[ + {"type":"text","data":{"text":"Review the auth flow please."}}, + {"type":"finish","data":{"reason":"stop","time":0}} + ]`, created, "", "") + fixture.insertMessage(t, "msg-assistant-1", "sess-1", "assistant", `[ + {"type":"reasoning","data":{"thinking":"Inspect auth.go first.","started_at":1789093629,"finished_at":1789093629}}, + {"type":"reasoning","data":{"thinking":"Then check the middleware.","started_at":1789093630,"finished_at":1789093630}}, + {"type":"tool_call","data":{"id":"chatcmpl-tool-1","name":"view","input":"{\"file_path\":\"auth.go\"}","finished":true,"provider_executed":false}}, + {"type":"tool_call","data":{"id":"chatcmpl-tool-2","name":"bash","input":"{\"command\":\"go test\"}","finished":true,"provider_executed":false}}, + {"type":"finish","data":{"reason":"tool_use","time":1789093630}} + ]`, created+3, "glm-5.3-flash", "hyper") + fixture.insertMessage(t, "msg-tool-1", "sess-1", "tool", `[ + {"type":"tool_result","data":{"tool_call_id":"chatcmpl-tool-1","name":"view","content":"package auth\n"}} + ]`, created+4, "", "") + fixture.insertMessage(t, "msg-tool-2", "sess-1", "tool", `[ + {"type":"tool_result","data":{"tool_call_id":"chatcmpl-tool-2","name":"bash","content":""}} + ]`, created+5, "", "") + fixture.insertMessage(t, "msg-assistant-2", "sess-1", "assistant", `[ + {"type":"text","data":{"text":"The auth flow looks correct."}}, + {"type":"finish","data":{"reason":"stop","time":1789093740}} + ]`, created+30, "glm-5.3-flash", "hyper") + + session, messages, err := parseCrushSession(context.Background(), fixture.dbPath, "sess-1", "workstation", false, nil) + require.NoError(t, err) + + require.NotNil(t, session) + assert.Equal(t, "crush:sess-1", session.ID) + assert.Equal(t, AgentCrush, session.Agent) + assert.Equal(t, "Review auth flow", session.SessionName) + assert.Equal(t, "crush-sqlite-v1", session.SourceVersion) + assert.Equal(t, "workstation", session.Machine) + assert.Equal(t, "Review the auth flow please.", session.FirstMessage) + // Timestamps are Unix seconds; a ms interpretation would land in 1970. + assert.Equal(t, time.Unix(created, 0).UTC(), session.StartedAt) + assert.Equal(t, time.Unix(created+120, 0).UTC(), session.EndedAt) + assert.Equal(t, 5, session.MessageCount) + assert.Equal(t, 1, session.UserMessageCount) + // Project resolves from the /.crush store location. + assert.Equal(t, filepath.Base(fixture.projectDir), session.Project) + assert.Equal(t, fixture.projectDir, session.Cwd) + + require.Len(t, messages, 5) + for i, message := range messages { + assert.Equal(t, i, message.Ordinal) + } + + user := messages[0] + assert.Equal(t, RoleUser, user.Role) + assert.False(t, user.IsSystem) + assert.Equal(t, "Review the auth flow please.", user.Content) + assert.Empty(t, user.StopReason, "finish parts are internal metadata") + + assistant := messages[1] + assert.Equal(t, RoleAssistant, assistant.Role) + assert.Equal(t, "glm-5.3-flash", assistant.Model) + assert.Equal(t, "hyper", assistant.ProviderID) + assert.True(t, assistant.HasThinking) + // Multiple reasoning parts accumulate instead of overwriting. + assert.Contains(t, assistant.ThinkingText, "Inspect auth.go first.") + assert.Contains(t, assistant.ThinkingText, "Then check the middleware.") + assert.Equal(t, "tool_use", assistant.StopReason) + require.Len(t, assistant.ToolCalls, 2) + assert.True(t, assistant.HasToolUse) + + firstCall := assistant.ToolCalls[0] + assert.Equal(t, "crush:chatcmpl-tool-1", firstCall.ToolUseID) + assert.Equal(t, "view", firstCall.ToolName) + assert.Equal(t, "Read", firstCall.Category) + assert.JSONEq(t, `{"file_path":"auth.go"}`, firstCall.InputJSON) + assert.Equal(t, "crush:child$$chatcmpl-tool-1", firstCall.SubagentSessionID) + secondCall := assistant.ToolCalls[1] + assert.Equal(t, "crush:chatcmpl-tool-2", secondCall.ToolUseID) + assert.Equal(t, "bash", secondCall.ToolName) + assert.Equal(t, "Bash", secondCall.Category) + + toolResult := messages[2] + assert.Equal(t, RoleUser, toolResult.Role) + assert.True(t, toolResult.IsSystem, + "role='tool' rows are system messages, not human turns") + require.Len(t, toolResult.ToolResults, 1) + assert.Equal(t, "crush:chatcmpl-tool-1", toolResult.ToolResults[0].ToolUseID) + + // An empty-but-present result still completes its call. + emptyResult := messages[3] + require.Len(t, emptyResult.ToolResults, 1) + assert.Equal(t, "crush:chatcmpl-tool-2", emptyResult.ToolResults[0].ToolUseID) + + final := messages[4] + assert.Equal(t, RoleAssistant, final.Role) + assert.Equal(t, "The auth flow looks correct.", final.Content) + assert.Equal(t, "stop", final.StopReason) + + // Exactly one aggregate usage event; session totals are the sole + // accounting source and per-message tokens are absent. + require.Len(t, session.UsageEvents, 1) + event := session.UsageEvents[0] + assert.Equal(t, "crush:sess-1", event.SessionID) + assert.Equal(t, "glm-5.3-flash", event.Model) + assert.Equal(t, "hyper", event.ProviderID) + assert.Equal(t, 43_922, event.InputTokens) + assert.Equal(t, 185, event.OutputTokens) + require.NotNil(t, event.Cost) + assert.Equal(t, money.Money{Microdollars: 12_600}, *event.Cost) + + assert.True(t, session.HasTotalOutputTokens) + assert.Equal(t, 185, session.TotalOutputTokens) + assert.True(t, session.HasPeakContextTokens) + assert.Equal(t, 43_922, session.PeakContextTokens) + + assert.False(t, hasOrphanedToolCall(messages), + "every tool call has a paired result message") +} + +func TestCrushProviderDiscoveryAndRoots(t *testing.T) { + fixture := newCrushTestFixture(t) + const created = int64(1_789_093_626) + fixture.insertSession(t, "sess-1", "Discovery", "", + created, created, 10, 5, 0.0) + + // A Crush data dir with projects.json expands to per-project roots. + registryDir := t.TempDir() + registry := `{"projects":[{"path":"` + filepath.ToSlash(fixture.projectDir) + + `","data_dir":"` + filepath.ToSlash(fixture.dataDir) + `"}]}` + require.NoError(t, os.WriteFile( + filepath.Join(registryDir, CrushProjectsFileName), + []byte(registry), 0o600, + )) + roots, registryMapping, projectMapping := normalizeCrushRoots([]string{registryDir}) + require.Equal(t, []string{fixture.dataDir}, roots) + require.Len(t, registryMapping, 1) + assert.Equal(t, []string{fixture.dataDir}, registryMapping[filepath.Clean(registryDir)]) + require.Len(t, projectMapping, 1) + assert.Equal(t, filepath.Clean(fixture.projectDir), projectMapping[filepath.Clean(fixture.dataDir)]) + provider := newCrushProviderFactory(AgentDef{ + Type: AgentCrush, IDPrefix: "crush:", + }).NewProvider(ProviderConfig{Roots: []string{registryDir}}) + sources, err := provider.Discover(t.Context()) + require.NoError(t, err) + require.Len(t, sources, 1) + assert.Equal(t, filepath.Clean(registryDir), sources[0].ConfiguredRoot) + + metas := make([]dbBackedSessionMeta, 0) + require.NoError(t, forEachCrushSessionMeta( + context.Background(), crushDBPath(roots[0]), false, + func(meta dbBackedSessionMeta) error { + metas = append(metas, meta) + return nil + }, + )) + require.Len(t, metas, 1) + assert.Equal(t, "sess-1", metas[0].SessionID) + assert.Equal(t, VirtualSourcePath(fixture.dbPath, "sess-1"), metas[0].VirtualPath) + assert.Positive(t, metas[0].FileMtime) + + meta, found, err := crushSessionMeta(context.Background(), fixture.dbPath, "missing", false) + require.NoError(t, err) + assert.False(t, found) + assert.Empty(t, meta.SessionID) + + // A root pointing directly at a .crush data dir is kept as-is. + dataDirs, _, _ := normalizeCrushRoots([]string{fixture.dataDir}) + assert.Equal(t, []string{fixture.dataDir}, dataDirs) + // A root pointing at the db file resolves to its directory. + dataDirs, _, _ = normalizeCrushRoots([]string{fixture.dbPath}) + assert.Equal(t, []string{fixture.dataDir}, dataDirs) + // An unreadable registry leaves the root untouched rather than + // failing discovery. + dataDirs, _, _ = normalizeCrushRoots([]string{registryDir + "-missing"}) + assert.Equal(t, []string{registryDir + "-missing"}, dataDirs) +} + +// A configured database-file root must map onto the data directory that +// holds virtual session members, so reconciliation can prove the whole +// membership rather than the bare crush.db path. +func TestCrushResolveReconciliationScopesMapsDatabaseFileRoot(t *testing.T) { + fixture := newCrushTestFixture(t) + factory := newCrushProviderFactory(AgentDef{ + Type: AgentCrush, IDPrefix: "crush:", + }) + + for name, requested := range map[string]string{ + "database": fixture.dbPath, + "virtual member": VirtualSourcePath(fixture.dbPath, "sess-1"), + "data directory": fixture.dataDir, + } { + t.Run(name, func(t *testing.T) { + provider := factory.NewProvider(ProviderConfig{ + Roots: []string{fixture.dbPath}, + }) + plan, err := provider.ResolveReconciliationScopes( + t.Context(), ReconciliationScopeRequest{Roots: []string{requested}}, + ) + require.NoError(t, err) + require.Len(t, plan.Scopes, 1) + scope := plan.Scopes[0] + assert.Equal(t, []string{filepath.Clean(fixture.dbPath)}, scope.TraversalRoots, + "traversal must keep the original configured database-file root") + assert.Equal(t, + []string{cleanReconciliationScopeRoot(fixture.dataDir)}, + scope.CoverageIdentities, + "the database-file request must cover the configured data directory") + assert.Equal(t, []string{requested}, scope.RetryRoots) + assert.Equal(t, + []string{cleanReconciliationScopeRoot(fixture.dataDir)}, + plan.RequiredCoverageIdentities) + }) + } + + // A configured data directory requested by its crush.db path still + // covers the configured root. + provider := factory.NewProvider(ProviderConfig{ + Roots: []string{fixture.dataDir}, + }) + plan, err := provider.ResolveReconciliationScopes( + t.Context(), ReconciliationScopeRequest{Roots: []string{fixture.dbPath}}, + ) + require.NoError(t, err) + require.Len(t, plan.Scopes, 1) + assert.Equal(t, []string{filepath.Clean(fixture.dataDir)}, plan.Scopes[0].TraversalRoots) + assert.Equal(t, + []string{cleanReconciliationScopeRoot(fixture.dataDir)}, + plan.Scopes[0].CoverageIdentities, + ) + assert.Equal(t, []string{fixture.dbPath}, plan.Scopes[0].RetryRoots) +} + +func TestCrushSchemaValidationRejectsGooseStores(t *testing.T) { + fixture := newCrushTestFixture(t) + // Drop the parts column marker: a messages table without it is not + // a Crush store (e.g. goose's vendored migrations). + _, err := fixture.database.Exec(` + CREATE TABLE messages_like_goose ( + id TEXT PRIMARY KEY, session_id TEXT, role TEXT, + content_json TEXT, created_at INTEGER + ) + `) + require.NoError(t, err) + _, err = fixture.database.Exec(`DROP TABLE messages`) + require.NoError(t, err) + _, err = fixture.database.Exec(`ALTER TABLE messages_like_goose RENAME TO messages`) + require.NoError(t, err) + require.Error(t, validateCrushSchema( + context.Background(), fixture.database, + ), "a messages table without parts must not be treated as Crush") +} + +func TestCrushSummaryMessageIsCompactBoundary(t *testing.T) { + fixture := newCrushTestFixture(t) + const created = int64(1_789_093_626) + fixture.insertSession(t, "sess-summary", "Compacted", "", + created, created, 0, 0, 0) + fixture.insertMessage(t, "msg-sum", "sess-summary", "assistant", `[ + {"type":"text","data":{"text":"Summary of the conversation so far."}} + ]`, created, "glm-5.3-flash", "") + // Mark the row as a condensed summary. + _, err := fixture.database.Exec( + `UPDATE messages SET is_summary_message = 1 WHERE id = 'msg-sum'`, + ) + require.NoError(t, err) + + session, messages, err := parseCrushSession(context.Background(), fixture.dbPath, "sess-summary", "m", false, nil) + require.NoError(t, err) + require.Len(t, messages, 1) + message := messages[0] + assert.Equal(t, RoleSystem, message.Role) + assert.True(t, message.IsSystem) + assert.True(t, message.IsCompactBoundary) + assert.Equal(t, "Summary of the conversation so far.", message.Content) + assert.Empty(t, message.Model, + "summary rows must not attribute the original row's model") + assert.Equal(t, 0, session.UserMessageCount) +} + +func TestCrushFingerprintReflectsMessageContent(t *testing.T) { + fixture := newCrushTestFixture(t) + const created = int64(1_789_093_626) + fixture.insertSession(t, "sess-fp", "Fingerprint", "", + created, created, 0, 0, 0) + fixture.insertMessage(t, "msg-fp", "sess-fp", "user", `[ + {"type":"text","data":{"text":"before"}} + ]`, created, "", "") + + var childCache crushChildRelationshipsCache + first, found, err := crushSessionFingerprint( + context.Background(), fixture.dbPath, "sess-fp", false, &childCache, + ) + require.NoError(t, err) + require.True(t, found) + + // An edit within the same second leaves every timestamp unchanged; + // the content hash must still move. + _, err = fixture.database.Exec(` + UPDATE messages SET parts = '[{"type":"text","data":{"text":"after"}}]' + WHERE id = 'msg-fp' + `) + require.NoError(t, err) + second, found, err := crushSessionFingerprint( + context.Background(), fixture.dbPath, "sess-fp", false, &childCache, + ) + require.NoError(t, err) + require.True(t, found) + assert.NotEqual(t, first, second, + "a same-second parts edit must change the fingerprint") + + // A vanished session reports not-found rather than a stale hash. + _, found, err = crushSessionFingerprint( + context.Background(), fixture.dbPath, "sess-missing", false, &childCache, + ) + require.NoError(t, err) + assert.False(t, found) +} + +func TestCrushParseSessionWithoutOptionalColumns(t *testing.T) { + t.Parallel() + projectDir := t.TempDir() + dataDir := filepath.Join(projectDir, ".crush") + require.NoError(t, os.MkdirAll(dataDir, 0o755)) + dbPath := filepath.Join(dataDir, CrushDBName) + db, err := sql.Open("sqlite3", dbPath) + require.NoError(t, err) + defer db.Close() + _, err = db.Exec(` + CREATE TABLE sessions ( + id TEXT PRIMARY KEY, parent_session_id TEXT, title TEXT NOT NULL, + message_count INTEGER NOT NULL DEFAULT 0, + prompt_tokens INTEGER NOT NULL DEFAULT 0, + completion_tokens INTEGER NOT NULL DEFAULT 0, + cost REAL NOT NULL DEFAULT 0.0, + updated_at INTEGER NOT NULL, created_at INTEGER NOT NULL + ); + CREATE TABLE messages ( + id TEXT PRIMARY KEY, session_id TEXT NOT NULL, + role TEXT NOT NULL, parts TEXT NOT NULL DEFAULT '[]', + model TEXT, created_at INTEGER NOT NULL + ); + INSERT INTO sessions (id, title, updated_at, created_at) + VALUES ('sess-min', 'Minimal', 1000, 1000); + INSERT INTO messages (id, session_id, role, parts, model, created_at) + VALUES ('msg-1', 'sess-min', 'user', '[]', '', 1000); + `) + require.NoError(t, err) + require.NoError(t, db.Close()) + + session, messages, err := parseCrushSession( + context.Background(), dbPath, "sess-min", "m", false, nil, + ) + + require.NoError(t, err) + require.NotNil(t, session) + require.Len(t, messages, 1) + assert.Equal(t, RoleUser, messages[0].Role) + assert.Empty(t, messages[0].ProviderID) + assert.False(t, messages[0].IsSystem) + + // Fingerprint freshness is mandatory for Crush sync; a minimal accepted + // schema without messages.updated_at must still produce a hash. + var childCache crushChildRelationshipsCache + hash, found, err := crushSessionFingerprint( + context.Background(), dbPath, "sess-min", false, &childCache, + ) + require.NoError(t, err) + require.True(t, found, + "fingerprint must work without the optional messages.updated_at column") + assert.NotEmpty(t, hash) +} diff --git a/internal/parser/db_backed_provider.go b/internal/parser/db_backed_provider.go index 83299ef8a..49bc70842 100644 --- a/internal/parser/db_backed_provider.go +++ b/internal/parser/db_backed_provider.go @@ -309,8 +309,8 @@ func (p *dbBackedProvider) Parse( // persistent archive: sessions must be preserved even when their // source file no longer exists on disk. Skip without ForceReplace // so the engine keeps the stored sessions instead of deleting them. - // The sql.ErrNoRows / empty-results cases below keep ForceReplace - // because the DB is still present and the row was genuinely removed. + // A present DB is normally authoritative for missing members below; + // ExplicitDeletionOnly providers preserve those members as well. return ParseOutcome{ ResultSetComplete: true, SkipReason: SkipNoSession, @@ -330,21 +330,13 @@ func (p *dbBackedProvider) Parse( } results, err := p.spec.parse(ctx, src.DBPath, src.SessionID, machine) if errors.Is(err, sql.ErrNoRows) { - return ParseOutcome{ - ResultSetComplete: true, - ForceReplace: true, - SkipReason: SkipNoSession, - }, nil + return p.missingMemberOutcome(), nil } if err != nil { return ParseOutcome{}, err } if len(results) == 0 { - return ParseOutcome{ - ResultSetComplete: true, - ForceReplace: true, - SkipReason: SkipNoSession, - }, nil + return p.missingMemberOutcome(), nil } out := make([]ParseResultOutcome, 0, len(results)) for _, result := range results { @@ -363,6 +355,15 @@ func (p *dbBackedProvider) Parse( }, nil } +func (p *dbBackedProvider) missingMemberOutcome() ParseOutcome { + return ParseOutcome{ + ResultSetComplete: true, + ForceReplace: p.Caps.Source.ExplicitDeletionOnly != + CapabilitySupported, + SkipReason: SkipNoSession, + } +} + type dbBackedSource struct { Root string DBPath string diff --git a/internal/parser/kiro.go b/internal/parser/kiro.go index c3599b2ca..9ec69129e 100644 --- a/internal/parser/kiro.go +++ b/internal/parser/kiro.go @@ -1,6 +1,7 @@ package parser import ( + "context" "encoding/json/v2" "fmt" "os" @@ -114,11 +115,11 @@ func loadKiroMetaStrict(jsonlPath string) (*kiroMeta, error) { return &m, nil } -// parseLegacySession parses a Kiro CLI session from its JSONL file. +// parseLegacySessionContext parses a Kiro CLI session from its JSONL file. // Returns (nil, nil, nil) if the file doesn't exist or contains // no user/assistant messages. -func (p *kiroProvider) parseLegacySession( - path, machine string, +func (p *kiroProvider) parseLegacySessionContext( + ctx context.Context, path, machine string, ) (*ParsedSession, []ParsedMessage, error) { info, err := os.Stat(path) if err != nil { @@ -243,7 +244,7 @@ func (p *kiroProvider) parseLegacySession( } cwd = meta.Cwd if cwd != "" { - project = ExtractProjectFromCwd(cwd) + project = ExtractProjectFromCwdWithBranchContext(ctx, cwd, "") } if meta.Title != "" && firstMessage == "" { firstMessage = meta.Title @@ -293,7 +294,9 @@ type kiroCurrentMeta struct { WorkspacePaths []string `json:"workspacePaths"` } -func (p *kiroProvider) parseCurrentSession(path, sessionID, machine string) (*ParsedSession, []ParsedMessage, error) { +func (p *kiroProvider) parseCurrentSessionContext( + ctx context.Context, path, sessionID, machine string, +) (*ParsedSession, []ParsedMessage, error) { info, err := os.Stat(path) if err != nil { if os.IsNotExist(err) { @@ -389,7 +392,7 @@ func (p *kiroProvider) parseCurrentSession(path, sessionID, machine string) (*Pa if len(meta.WorkspacePaths) > 0 { cwd = meta.WorkspacePaths[0] } - project := ExtractProjectFromCwd(cwd) + project := ExtractProjectFromCwdWithBranchContext(ctx, cwd, "") if project == "" { project = "unknown" } diff --git a/internal/parser/kiro_provider.go b/internal/parser/kiro_provider.go index 1e9a25b29..f63930854 100644 --- a/internal/parser/kiro_provider.go +++ b/internal/parser/kiro_provider.go @@ -134,18 +134,19 @@ func (p *kiroProvider) Parse( case kiroSourceSQLiteDB: return p.parseSQLiteDB(ctx, src, machine) case kiroSourceSQLiteSession: - return p.parseSQLiteSession(src, machine, req.Fingerprint) + return p.parseSQLiteSession(ctx, src, machine, req.Fingerprint) case kiroSourceCurrentJSONL: - return p.parseCurrentJSONL(src, machine, req.Fingerprint) + return p.parseCurrentJSONL(ctx, src, machine, req.Fingerprint) default: - return p.parseLegacyJSONL(src, machine, req.Fingerprint) + return p.parseLegacyJSONL(ctx, src, machine, req.Fingerprint) } } func (p *kiroProvider) parseCurrentJSONL( - src kiroSource, machine string, fingerprint SourceFingerprint, + ctx context.Context, src kiroSource, machine string, + fingerprint SourceFingerprint, ) (ParseOutcome, error) { - sess, msgs, err := p.parseCurrentSession(src.Path, src.SessionID, machine) + sess, msgs, err := p.parseCurrentSessionContext(ctx, src.Path, src.SessionID, machine) if err != nil { return ParseOutcome{}, err } @@ -205,7 +206,7 @@ func (p *kiroProvider) parseSQLiteDB( if err := ctx.Err(); err != nil { return ParseOutcome{}, err } - sess, msgs, err := store.ParseSession(meta.SessionID, machine) + sess, msgs, err := store.ParseSession(ctx, meta.SessionID, machine) if err != nil { sourceErrs = append(sourceErrs, SourceError{ SourceKey: meta.VirtualPath, @@ -243,6 +244,7 @@ func (p *kiroProvider) parseSQLiteDB( } func (p *kiroProvider) parseSQLiteSession( + ctx context.Context, src kiroSource, machine string, fingerprint SourceFingerprint, @@ -261,7 +263,7 @@ func (p *kiroProvider) parseSQLiteSession( } return ParseOutcome{}, fmt.Errorf("stat %s: %w", src.DBPath, err) } - sess, msgs, err := parseKiroSQLiteSession(src.DBPath, src.SessionID, machine) + sess, msgs, err := parseKiroSQLiteSession(ctx, src.DBPath, src.SessionID, machine) if errors.Is(err, sql.ErrNoRows) { return ParseOutcome{ ResultSetComplete: true, @@ -296,11 +298,12 @@ func (p *kiroProvider) parseSQLiteSession( } func (p *kiroProvider) parseLegacyJSONL( + ctx context.Context, src kiroSource, machine string, fingerprint SourceFingerprint, ) (ParseOutcome, error) { - sess, msgs, err := p.parseLegacySession(src.Path, machine) + sess, msgs, err := p.parseLegacySessionContext(ctx, src.Path, machine) if err != nil { return ParseOutcome{}, err } diff --git a/internal/parser/kiro_provider_test.go b/internal/parser/kiro_provider_test.go index a74e44b00..b5087551d 100644 --- a/internal/parser/kiro_provider_test.go +++ b/internal/parser/kiro_provider_test.go @@ -149,6 +149,67 @@ func TestKiroProviderParsePhysicalVirtualAndLegacySources(t *testing.T) { assert.Equal(t, SkipNoSession, missingOutcome.SkipReason) } +func TestKiroProviderSQLiteProjectDiscoveryPolicy(t *testing.T) { + root := t.TempDir() + _, db := newKiroProviderSQLiteDBAt(t, root) + repo := filepath.Join(t.TempDir(), "local-repository") + cwd := filepath.Join(repo, "recorded-project") + require.NoError(t, os.MkdirAll(filepath.Join(repo, ".git"), 0o755)) + require.NoError(t, os.MkdirAll(cwd, 0o755)) + seedKiroSQLiteSession(t, db, cwd, "sqlite-session", + readKiroFixture(t, "standard_payload.json"), 1779012000000, 1779012030000) + + provider, ok := NewProvider(AgentKiro, ProviderConfig{Roots: []string{root}}) + require.True(t, ok) + sources, err := provider.Discover(t.Context()) + require.NoError(t, err) + require.Len(t, sources, 1) + member, ok, err := provider.FindSource(t.Context(), FindSourceRequest{ + RawSessionID: "sqlite-session", + }) + require.NoError(t, err) + require.True(t, ok) + + origStat, origLstat := osStat, osLstat + t.Cleanup(func() { osStat, osLstat = origStat, origLstat }) + var probes int + osStat = func(path string) (os.FileInfo, error) { + probes++ + return origStat(path) + } + osLstat = func(path string) (os.FileInfo, error) { + probes++ + return origLstat(path) + } + for _, route := range []struct { + name string + source SourceRef + }{ + {"bulk", sources[0]}, + {"session", member}, + } { + for _, disabled := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/disabled=%t", route.name, disabled), func(t *testing.T) { + ctx := WithProjectRootMemo(t.Context()) + if disabled { + ctx = WithoutFilesystemProjectDiscovery(ctx) + } + probes = 0 + outcome, err := provider.Parse(ctx, ParseRequest{Source: route.source, Machine: "remote"}) + require.NoError(t, err) + require.Len(t, outcome.Results, 1) + if disabled { + assert.Equal(t, "recorded_project", outcome.Results[0].Result.Session.Project) + assert.Zero(t, probes) + } else { + assert.Equal(t, "local_repository", outcome.Results[0].Result.Session.Project) + assert.Positive(t, probes) + } + }) + } + } +} + func TestKiroProviderSkipsShadowedLegacySource(t *testing.T) { root := t.TempDir() dbPath, db := newKiroProviderSQLiteDBAt(t, root) diff --git a/internal/parser/kiro_sqlite.go b/internal/parser/kiro_sqlite.go index 5a0f9e4eb..521d91e6c 100644 --- a/internal/parser/kiro_sqlite.go +++ b/internal/parser/kiro_sqlite.go @@ -314,20 +314,20 @@ func KiroSQLiteSourceMtime(path string) (int64, error) { // parseKiroSQLiteSession parses one current-store Kiro CLI conversation // into normal AgentsView session/message records. func parseKiroSQLiteSession( - dbPath, sessionID, machine string, + ctx context.Context, dbPath, sessionID, machine string, ) (*ParsedSession, []ParsedMessage, error) { store, err := OpenKiroSQLiteStore(dbPath) if err != nil { return nil, nil, err } defer store.Close() - return store.ParseSession(sessionID, machine) + return store.ParseSession(ctx, sessionID, machine) } // ParseSession parses one current-store Kiro CLI conversation using // the store's existing SQLite handle. func (s *KiroSQLiteStore) ParseSession( - sessionID, machine string, + ctx context.Context, sessionID, machine string, ) (*ParsedSession, []ParsedMessage, error) { row, err := s.loadRow(sessionID) if err != nil { @@ -400,7 +400,7 @@ func (s *KiroSQLiteStore) ParseSession( if row.key != "" { cwd = row.key } - project := ExtractProjectFromCwd(cwd) + project := ExtractProjectFromCwdWithBranchContext(ctx, cwd, "") if project == "" { project = "unknown" } diff --git a/internal/parser/kiro_sqlite_test.go b/internal/parser/kiro_sqlite_test.go index 2ad08d534..2dd8fa4bf 100644 --- a/internal/parser/kiro_sqlite_test.go +++ b/internal/parser/kiro_sqlite_test.go @@ -65,7 +65,7 @@ func TestParseKiroSQLiteSession(t *testing.T) { ) sess, msgs, err := parseKiroSQLiteSession( - dbPath, "sqlite-session", "test-machine", + t.Context(), dbPath, "sqlite-session", "test-machine", ) require.NoError(t, err, "parseKiroSQLiteSession") require.NotNil(t, sess, "expected session") @@ -160,7 +160,7 @@ func TestParseKiroSQLiteSessionRejectsMalformedPayload(t *testing.T) { 1, 2, ) _, _, err := parseKiroSQLiteSession( - dbPath, "broken-session", "test-machine", + t.Context(), dbPath, "broken-session", "test-machine", ) require.Error(t, err, "expected malformed payload error") } diff --git a/internal/parser/opencode.go b/internal/parser/opencode.go index e150cc133..f14c39640 100644 --- a/internal/parser/opencode.go +++ b/internal/parser/opencode.go @@ -526,6 +526,14 @@ func OpenCodeChildDigestMetadataWatermarkNS(hash string) (int64, bool) { // path; Kilo and MiMoCode reuse it and relabel the result. func parseOpenCodeDBSession( dbPath, sessionID, machine string, +) (*ParsedSession, []ParsedMessage, error) { + return parseOpenCodeDBSessionContext( + context.Background(), dbPath, sessionID, machine, + ) +} + +func parseOpenCodeDBSessionContext( + ctx context.Context, dbPath, sessionID, machine string, ) (*ParsedSession, []ParsedMessage, error) { if _, err := os.Stat(dbPath); os.IsNotExist(err) { return nil, nil, fmt.Errorf( @@ -581,8 +589,8 @@ func parseOpenCodeDBSession( if !openCodeUsableWorktree(projectWorktree) { projectWorktree = cwd } - return buildOpenCodeSession( - db, s, cwd, projectWorktree, dbPath, machine, + return buildOpenCodeSessionContext( + ctx, db, s, cwd, projectWorktree, dbPath, machine, ) } @@ -656,12 +664,21 @@ func resolveOpenCodeStorageWorktree( // and relabel the result. func parseOpenCodeStorageFile( sessionPath, machine string, +) (*ParsedSession, []ParsedMessage, error) { + return parseOpenCodeStorageFileContext( + context.Background(), sessionPath, machine, + ) +} + +func parseOpenCodeStorageFileContext( + ctx context.Context, sessionPath, machine string, ) (*ParsedSession, []ParsedMessage, error) { snapshot, err := loadOpenCodeStorageSnapshot(sessionPath, true) if err != nil { return nil, nil, err } - sess, parsed, err := buildOpenCodeParsedSession( + sess, parsed, err := buildOpenCodeParsedSessionContext( + ctx, snapshot.session, snapshot.worktree, snapshot.worktree, @@ -1235,7 +1252,8 @@ func loadOpenCodeParts( return parts, rows.Err() } -func buildOpenCodeSession( +func buildOpenCodeSessionContext( + ctx context.Context, db *sql.DB, s openCodeSessionRow, cwd, projectWorktree, dbPath, machine string, @@ -1289,7 +1307,7 @@ func buildOpenCodeSession( if err != nil { return nil, nil, fmt.Errorf("loading parts for %s: %w", s.id, err) } - _, legacy, err := buildOpenCodeParsedSession(s, cwd, projectWorktree, dbPath+"#"+s.id, fileMtime*1_000_000, machine, msgs, parts) + _, legacy, err := buildOpenCodeParsedSessionContext(ctx, s, cwd, projectWorktree, dbPath+"#"+s.id, fileMtime*1_000_000, machine, msgs, parts) if err != nil { return nil, nil, err } @@ -1308,7 +1326,7 @@ func buildOpenCodeSession( for i := range parsed { parsed[i].Ordinal = i } - sess, parsed, err := finishOpenCodeSession(s, cwd, projectWorktree, dbPath+"#"+s.id, fileMtime*1_000_000, machine, parsed) + sess, parsed, err := finishOpenCodeSessionContext(ctx, s, cwd, projectWorktree, dbPath+"#"+s.id, fileMtime*1_000_000, machine, parsed) if err != nil || sess == nil { return sess, parsed, err } @@ -1320,7 +1338,8 @@ func buildOpenCodeSession( return sess, parsed, nil } -func buildOpenCodeParsedSession( +func buildOpenCodeParsedSessionContext( + ctx context.Context, s openCodeSessionRow, cwd, projectWorktree, filePath string, fileMtime int64, @@ -1373,10 +1392,11 @@ func buildOpenCodeParsedSession( return nil, nil, nil } - return finishOpenCodeSession(s, cwd, projectWorktree, filePath, fileMtime, machine, parsed) + return finishOpenCodeSessionContext(ctx, s, cwd, projectWorktree, filePath, fileMtime, machine, parsed) } -func finishOpenCodeSession( +func finishOpenCodeSessionContext( + ctx context.Context, s openCodeSessionRow, cwd, projectWorktree, filePath string, fileMtime int64, machine string, parsed []ParsedMessage, ) (*ParsedSession, []ParsedMessage, error) { @@ -1395,7 +1415,7 @@ func finishOpenCodeSession( } } - project := ExtractProjectFromCwd(projectWorktree) + project := ExtractProjectFromCwdWithBranchContext(ctx, projectWorktree, "") if project == "" { project = "unknown" } diff --git a/internal/parser/opencode_provider.go b/internal/parser/opencode_provider.go index 6b99cf871..12562fffa 100644 --- a/internal/parser/opencode_provider.go +++ b/internal/parser/opencode_provider.go @@ -63,9 +63,9 @@ func (s openCodeFormatSourceSet) Parse( ) dbPath, sessionID, sqliteSource := s.spec.parseVirtual(path) if sqliteSource { - sess, msgs, err = s.spec.parseSQLite(dbPath, sessionID, machine) + sess, msgs, err = s.spec.parseSQLite(ctx, dbPath, sessionID, machine) } else { - sess, msgs, err = s.spec.parseFile(path, machine) + sess, msgs, err = s.spec.parseFile(ctx, path, machine) } if err != nil { return ParseOutcome{}, err @@ -255,9 +255,9 @@ func (spec openCodeProviderSpec) containerGlobs() []string { // parseFile parses a file-backed storage session and relabels it onto // this agent's ID prefix when the agent is a fork of OpenCode. func (spec openCodeProviderSpec) parseFile( - sessionPath, machine string, + ctx context.Context, sessionPath, machine string, ) (*ParsedSession, []ParsedMessage, error) { - sess, msgs, err := parseOpenCodeStorageFile(sessionPath, machine) + sess, msgs, err := parseOpenCodeStorageFileContext(ctx, sessionPath, machine) if err != nil || sess == nil { return sess, msgs, err } @@ -270,9 +270,9 @@ func (spec openCodeProviderSpec) parseFile( // parseSQLite parses a single SQLite-backed session and relabels it // onto this agent's ID prefix when the agent is a fork of OpenCode. func (spec openCodeProviderSpec) parseSQLite( - dbPath, sessionID, machine string, + ctx context.Context, dbPath, sessionID, machine string, ) (*ParsedSession, []ParsedMessage, error) { - sess, msgs, err := parseOpenCodeDBSession(dbPath, sessionID, machine) + sess, msgs, err := parseOpenCodeDBSessionContext(ctx, dbPath, sessionID, machine) if err != nil || sess == nil { return sess, msgs, err } diff --git a/internal/parser/project.go b/internal/parser/project.go index 0227bc4e5..e2e4c1cab 100644 --- a/internal/parser/project.go +++ b/internal/parser/project.go @@ -246,18 +246,20 @@ func ExtractProjectFromCwdWithBranchContext( ctx context.Context, cwd, gitBranch string, ) string { return extractProjectFromCwdWithBranchPolicy( - cwd, gitBranch, !filesystemProjectDiscoveryDisabled(ctx), + ctx, cwd, gitBranch, !filesystemProjectDiscoveryDisabled(ctx), ) } func extractProjectFromCwdWithBranch( cwd, gitBranch string, ) string { - return extractProjectFromCwdWithBranchPolicy(cwd, gitBranch, true) + return extractProjectFromCwdWithBranchPolicy( + context.Background(), cwd, gitBranch, true, + ) } func extractProjectFromCwdWithBranchPolicy( - cwd, gitBranch string, discoverFilesystem bool, + ctx context.Context, cwd, gitBranch string, discoverFilesystem bool, ) string { if cwd == "" { return "" @@ -278,7 +280,11 @@ func extractProjectFromCwdWithBranchPolicy( if discoverFilesystem && filepath.IsAbs(cleaned) && !isForeignOSPath(cwd, cleaned, winPath) && probeGitRootForCwd(cleaned) { - if root, linkedToRecordedPath := findGitRepoRoot(cleaned); root != "" && + rootResult := projectRootMemoFrom(ctx).root(cleaned, func() gitRootResult { + root, linked := findGitRepoRootCtx(ctx, cleaned) + return gitRootResult{root: root, linked: linked} + }) + if root, linkedToRecordedPath := rootResult.root, rootResult.linked; root != "" && (linkedToRecordedPath || anchoredProject == "") { name := filepath.Base(root) if isInvalidPathBase(name) { @@ -644,12 +650,12 @@ func isInvalidPathBase(name string) bool { return false } -// findGitRepoRoot walks upward from cwd to find the enclosing git +// findGitRepoRootCtx walks upward from cwd to find the enclosing git // repository root. Supports both standard repos (.git directory) // and linked worktrees/submodules (.git file). When cwd no longer // exists on disk, sibling directories are checked for worktree // .git files that can reveal the true repo root. -func findGitRepoRoot(cwd string) (string, bool) { +func findGitRepoRootCtx(ctx context.Context, cwd string) (string, bool) { if cwd == "" { return "", false } @@ -686,7 +692,7 @@ func findGitRepoRoot(cwd string) (string, bool) { } sibDir = parent } - if root := repoRootFromSiblings(sibDir, cwd); root != "" { + if root := repoRootFromSiblingsCtx(ctx, sibDir, cwd); root != "" { return root, deletedChildIsWorktree(sibDir, cwd, root) } } @@ -736,22 +742,43 @@ func findGitRepoRootLocal( } } -// repoRootFromSiblings checks child directories of dir for +// repoRootFromSiblingsCtx checks child directories of dir for // linked-worktree .git files and uses them to discover the // true repo root. Submodule .git files are skipped, and all // candidates must agree on the same root to avoid // misattributing unrelated paths. -func repoRootFromSiblings(dir, cwd string) string { +func repoRootFromSiblingsCtx( + ctx context.Context, dir, cwd string, +) string { + scan := projectRootMemoFrom(ctx).scan(dir, func() siblingScanResult { + return scanSiblingRepoCandidates(dir) + }) + if scan.selfIsRepo { + return "" + } + if scan.worktreeCount == 0 { + if scan.dirCount != 1 || + !deletedChildIsWorktree(dir, cwd, scan.singleDirRoot) { + return "" + } + return scan.singleDirRoot + } + return scan.agreedRoot +} + +func scanSiblingRepoCandidates(dir string) siblingScanResult { + var result siblingScanResult // If dir is itself a repo or worktree, let the normal upward walk // handle it. A refused .git symlink counts too: it is a boundary the // walk stops at, and typing it must not follow the link. if _, err := statGitEntry(filepath.Join(dir, ".git")); err == nil || errors.Is(err, errRefusedGitEntry) { - return "" + result.selfIsRepo = true + return result } entries, err := os.ReadDir(dir) if err != nil { - return "" + return result } worktreeMarker := string(filepath.Separator) + ".git" + string(filepath.Separator) + "worktrees" + @@ -822,14 +849,12 @@ func repoRootFromSiblings(dir, cwd string) string { } // Count worktree and directory siblings. - var worktreeCount, dirCount int - var singleDirRoot string for _, s := range siblings { if s.isDir { - dirCount++ - singleDirRoot = s.root + result.dirCount++ + result.singleDirRoot = s.root } else { - worktreeCount++ + result.worktreeCount++ } } @@ -838,16 +863,8 @@ func repoRootFromSiblings(dir, cwd string) string { // accept a single main checkout only if its // .git/worktrees/ exists, proving it has (or had) // linked worktrees. - if worktreeCount == 0 { - if dirCount != 1 { - return "" - } - // Verify the deleted child matches a known worktree - // entry under .git/worktrees/. - if !deletedChildIsWorktree(dir, cwd, singleDirRoot) { - return "" - } - return singleDirRoot + if result.worktreeCount == 0 { + return result } var found string @@ -855,10 +872,11 @@ func repoRootFromSiblings(dir, cwd string) string { if found == "" { found = s.root } else if found != s.root { - return "" + return result } } - return found + result.agreedRoot = found + return result } // deletedChildIsWorktree checks whether the first missing diff --git a/internal/parser/project_pass_memo.go b/internal/parser/project_pass_memo.go new file mode 100644 index 000000000..bc3ea3d7b --- /dev/null +++ b/internal/parser/project_pass_memo.go @@ -0,0 +1,64 @@ +package parser + +import ( + "context" + "sync" +) + +type projectRootMemo struct { + roots sync.Map + scans sync.Map +} + +type gitRootResult struct { + root string + linked bool +} + +type siblingScanResult struct { + selfIsRepo bool + agreedRoot string + singleDirRoot string + worktreeCount int + dirCount int +} + +type projectRootMemoContextKey struct{} + +func WithProjectRootMemo(ctx context.Context) context.Context { + if projectRootMemoFrom(ctx) != nil { + return ctx + } + return context.WithValue(ctx, projectRootMemoContextKey{}, &projectRootMemo{}) +} + +func projectRootMemoFrom(ctx context.Context) *projectRootMemo { + memo, _ := ctx.Value(projectRootMemoContextKey{}).(*projectRootMemo) + return memo +} + +func (memo *projectRootMemo) root( + key string, fn func() gitRootResult, +) gitRootResult { + if memo == nil { + return fn() + } + return memoizedProjectResult(&memo.roots, key, fn) +} + +func (memo *projectRootMemo) scan( + key string, fn func() siblingScanResult, +) siblingScanResult { + if memo == nil { + return fn() + } + return memoizedProjectResult(&memo.scans, key, fn) +} + +func memoizedProjectResult[T any](cache *sync.Map, key string, fn func() T) T { + value, ok := cache.Load(key) + if !ok { + value, _ = cache.LoadOrStore(key, sync.OnceValue(fn)) + } + return value.(func() T)() +} diff --git a/internal/parser/project_pass_memo_test.go b/internal/parser/project_pass_memo_test.go new file mode 100644 index 000000000..e1d3bbd8e --- /dev/null +++ b/internal/parser/project_pass_memo_test.go @@ -0,0 +1,338 @@ +package parser + +import ( + "context" + "fmt" + "os" + "path/filepath" + "runtime" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestProjectRootMemoRepeatedCwdCost(t *testing.T) { + _, cwd, _ := projectRootMemoFixture(t) + origStat, origLstat := osStat, osLstat + defer func() { osStat, osLstat = origStat, origLstat }() + var calls atomic.Int64 + osStat = func(path string) (os.FileInfo, error) { + calls.Add(1) + return origStat(path) + } + osLstat = func(path string) (os.FileInfo, error) { + calls.Add(1) + return origLstat(path) + } + + ctx := WithProjectRootMemo(context.Background()) + first := ExtractProjectFromCwdWithBranchContext(ctx, cwd, "") + firstCalls := calls.Load() + for range 7 { + assert.Equal(t, first, + ExtractProjectFromCwdWithBranchContext(ctx, cwd, "")) + } + assert.Equal(t, "main_repo", first) + assert.Equal(t, firstCalls, calls.Load(), + "repeated resolutions must not repeat the root fill") + t.Logf("root fills=1; underlying calls after first=%d final=%d; project=%s", + firstCalls, calls.Load(), first) +} + +func TestProjectRootMemoAncestorScanKeepsCwdAssociation(t *testing.T) { + container, knownCwd, _ := projectRootMemoFixture(t) + otherCwd := filepath.Join(container, "unrelated-child") + ctx := WithProjectRootMemo(context.Background()) + + assert.Equal(t, "main_repo", + ExtractProjectFromCwdWithBranchContext(ctx, knownCwd, "")) + require.NoError(t, os.MkdirAll(filepath.Join( + container, "other_repo", ".git", + ), 0o755)) + assert.Equal(t, "unrelated_child", + ExtractProjectFromCwdWithBranchContext(ctx, otherCwd, "")) +} + +func TestProjectRootMemoConcurrentFill(t *testing.T) { + _, cwd, _ := projectRootMemoFixture(t) + origStat, origLstat := osStat, osLstat + defer func() { osStat, osLstat = origStat, origLstat }() + var calls atomic.Int64 + osStat = func(path string) (os.FileInfo, error) { + calls.Add(1) + return origStat(path) + } + osLstat = func(path string) (os.FileInfo, error) { + calls.Add(1) + return origLstat(path) + } + + baselineMemo := WithProjectRootMemo(context.Background()) + _ = ExtractProjectFromCwdWithBranchContext(baselineMemo, cwd, "") + baselineCalls := calls.Load() + + calls.Store(0) + started := make(chan struct{}) + release := make(chan struct{}) + var blocked atomic.Bool + osStat = func(path string) (os.FileInfo, error) { + calls.Add(1) + if blocked.CompareAndSwap(false, true) { + close(started) + <-release + } + return origStat(path) + } + ctx := WithProjectRootMemo(context.Background()) + results := make([]string, 16) + var wg sync.WaitGroup + ready := make(chan struct{}, len(results)) + goStart := make(chan struct{}) + for i := range results { + wg.Add(1) + go func(i int) { + defer wg.Done() + ready <- struct{}{} + <-goStart + results[i] = ExtractProjectFromCwdWithBranchContext(ctx, cwd, "") + }(i) + } + for range results { + <-ready + } + close(goStart) + <-started + time.Sleep(50 * time.Millisecond) + close(release) + wg.Wait() + + assert.Equal(t, baselineCalls, calls.Load()) + for _, result := range results[1:] { + assert.Equal(t, results[0], result) + } + t.Logf("concurrent fills=1; underlying calls baseline=%d concurrent=%d; project=%s", + baselineCalls, calls.Load(), results[0]) +} + +func TestProjectRootMemoAbsentContextValue(t *testing.T) { + cwd := filepath.Join(t.TempDir(), "missing-project", "src") + assert.Equal(t, "src", ExtractProjectFromCwd(cwd)) + assert.Equal(t, "src", + ExtractProjectFromCwdWithBranchContext(context.Background(), cwd, "")) +} + +func TestProjectRootMemoPolicyGates(t *testing.T) { + cwd := filepath.Join(t.TempDir(), "protected-project", "src") + origProbe, origStat, origLstat := probeGitRootForCwd, osStat, osLstat + t.Cleanup(func() { + probeGitRootForCwd, osStat, osLstat = origProbe, origStat, origLstat + }) + osStat = func(path string) (os.FileInfo, error) { + assert.Fail(t, "policy gates must skip filesystem discovery", path) + return origStat(path) + } + osLstat = func(path string) (os.FileInfo, error) { + assert.Fail(t, "policy gates must skip filesystem discovery", path) + return origLstat(path) + } + + disabled := WithProjectRootMemo( + WithoutFilesystemProjectDiscovery(context.Background()), + ) + assert.Equal(t, "src", + ExtractProjectFromCwdWithBranchContext(disabled, cwd, "")) + + probeGitRootForCwd = func(string) bool { return false } + refused := WithProjectRootMemo(context.Background()) + assert.Equal(t, "src", + ExtractProjectFromCwdWithBranchContext(refused, cwd, "")) + + foreign := `C:\Users\agent\project\src` + if runtime.GOOS == "windows" { + foreign = "/Users/agent/project/src" + } + winPath := looksLikeWindowsPath(foreign) + norm := foreign + if winPath { + norm = filepath.ToSlash(foreign) + } + cleaned := filepath.Clean(norm) + if filepath.IsAbs(cleaned) && isForeignOSPath(foreign, cleaned, winPath) { + foreignCtx := WithProjectRootMemo(context.Background()) + assert.Equal(t, "src", + ExtractProjectFromCwdWithBranchContext(foreignCtx, foreign, "")) + } +} + +func TestProjectRootMemoStandaloneRefreshes(t *testing.T) { + root := t.TempDir() + container := filepath.Join(root, "container") + cwd := filepath.Join(container, "deleted-child", "src") + repoA := filepath.Join(container, "repo-a") + makeProjectMemoRepo(t, repoA, "deleted-child") + assert.Equal(t, "repo_a", ExtractProjectFromCwd(cwd)) + + require.NoError(t, os.RemoveAll(repoA)) + repoB := filepath.Join(container, "repo-b") + makeProjectMemoRepo(t, repoB, "deleted-child") + assert.Equal(t, "repo_b", ExtractProjectFromCwd(cwd)) +} + +func TestProjectRootMemoProviderRoutes(t *testing.T) { + t.Run("OpenCode storage", func(t *testing.T) { + root, cwd := projectRootMemoProviderFixture(t) + sessionPath := filepath.Join( + root, "storage", "session", "global", "session.json", + ) + writeOpenCodeStorageFile(t, sessionPath, map[string]any{ + "id": "session", "directory": cwd, "title": "Storage", + "time": map[string]any{ + "created": int64(1700000000000), + "updated": int64(1700000060000), + }, + }) + writeOpenCodeStorageFile(t, filepath.Join( + root, "storage", "message", "session", "msg.json", + ), map[string]any{ + "id": "msg", "sessionID": "session", "role": "user", + "time": map[string]any{"created": int64(1700000000000)}, + }) + writeOpenCodeStorageFile(t, filepath.Join( + root, "storage", "part", "msg", "part.json", + ), map[string]any{ + "id": "part", "sessionID": "session", "messageID": "msg", + "type": "text", "text": "storage message", + "time": map[string]any{"created": int64(1700000000000)}, + }) + + provider, ok := NewProvider(AgentOpenCode, ProviderConfig{Roots: []string{root}}) + require.True(t, ok) + sources, err := provider.Discover(context.Background()) + require.NoError(t, err) + require.Len(t, sources, 1) + fingerprint, err := provider.Fingerprint(context.Background(), sources[0]) + require.NoError(t, err) + assertProjectRootMemoProviderRoute(t, provider, ParseRequest{ + Source: sources[0], Fingerprint: fingerprint, + }) + }) + + t.Run("Command Code JSONL", func(t *testing.T) { + root, cwd := projectRootMemoProviderFixture(t) + path := filepath.Join(root, "project", "command-session.jsonl") + writeSourceFile(t, path, fmt.Sprintf( + `{"id":"m1","timestamp":"2026-06-01T10:00:00Z","sessionId":"command-session","role":"user","content":[{"type":"text","text":"Question"}],"metadata":{"cwd":%q}} +{"id":"m2","timestamp":"2026-06-01T10:00:03Z","sessionId":"command-session","role":"assistant","content":[{"type":"text","text":"Answer"}]}`, + cwd, + )) + provider, ok := NewProvider(AgentCommandCode, ProviderConfig{Roots: []string{root}}) + require.True(t, ok) + sources, err := provider.Discover(context.Background()) + require.NoError(t, err) + require.Len(t, sources, 1) + assertProjectRootMemoProviderRoute(t, provider, ParseRequest{Source: sources[0]}) + }) + + t.Run("Kiro legacy JSONL", func(t *testing.T) { + root, cwd := projectRootMemoProviderFixture(t) + path := filepath.Join(root, "kiro-session.jsonl") + writeSourceFile(t, path, kiroProviderJSONLFixture("Question")) + writeSourceFile(t, filepath.Join(root, "kiro-session.json"), fmt.Sprintf( + `{"session_id":"kiro-session","cwd":%q,"title":"kiro-session","created_at":"2026-06-01T10:00:00Z","updated_at":"2026-06-01T10:01:00Z"}`+"\n", + cwd, + )) + provider, ok := NewProvider(AgentKiro, ProviderConfig{Roots: []string{root}}) + require.True(t, ok) + sources, err := provider.Discover(context.Background()) + require.NoError(t, err) + require.Len(t, sources, 1) + assertProjectRootMemoProviderRoute(t, provider, ParseRequest{Source: sources[0]}) + }) + + t.Run("Kiro current JSONL", func(t *testing.T) { + root, cwd := projectRootMemoProviderFixture(t) + rawID := "sess_0123456789abcdef" + path := filepath.Join(root, "workspace", rawID, "messages.jsonl") + writeSourceFile(t, path, + `{"payload":{"type":"user","content":"Question"}}`+"\n"+ + `{"payload":{"type":"assistant","content":"Answer"}}`+"\n") + writeSourceFile(t, filepath.Join(filepath.Dir(path), "session.json"), + fmt.Sprintf(`{"workspacePaths":[%q],"createdAt":"2026-06-01T10:00:00Z","lastModifiedAt":"2026-06-01T10:01:00Z"}`+"\n", cwd)) + provider, ok := NewProvider(AgentKiro, ProviderConfig{Roots: []string{root}}) + require.True(t, ok) + sources, err := provider.Discover(context.Background()) + require.NoError(t, err) + require.Len(t, sources, 1) + assertProjectRootMemoProviderRoute(t, provider, ParseRequest{Source: sources[0]}) + }) +} + +func assertProjectRootMemoProviderRoute( + t *testing.T, provider Provider, req ParseRequest, +) { + t.Helper() + origStat, origLstat := osStat, osLstat + defer func() { osStat, osLstat = origStat, origLstat }() + var calls atomic.Int64 + osStat = func(path string) (os.FileInfo, error) { + calls.Add(1) + return origStat(path) + } + osLstat = func(path string) (os.FileInfo, error) { + calls.Add(1) + return origLstat(path) + } + + parse := func(ctx context.Context) string { + outcome, err := provider.Parse(ctx, req) + require.NoError(t, err) + require.Len(t, outcome.Results, 1) + return outcome.Results[0].Result.Session.Project + } + assert.Equal(t, "src", parse(WithoutFilesystemProjectDiscovery(t.Context()))) + assert.Zero(t, calls.Load(), "disabled discovery must not inspect local repositories") + + uncachedProject := parse(context.Background()) + uncachedProjectAgain := parse(context.Background()) + uncachedCalls := calls.Load() + + calls.Store(0) + memoCtx := WithProjectRootMemo(context.Background()) + memoProject := parse(memoCtx) + memoProjectAgain := parse(memoCtx) + memoCalls := calls.Load() + + assert.Equal(t, uncachedProject, uncachedProjectAgain) + assert.Equal(t, memoProject, memoProjectAgain) + assert.Equal(t, uncachedProject, memoProject) + assert.Greater(t, uncachedCalls, memoCalls) + assert.Greater(t, memoCalls, int64(0)) + t.Logf("provider=%s uncached_calls=%d memo_calls=%d project=%s", + provider.Definition().Type, uncachedCalls, memoCalls, memoProject) +} + +func projectRootMemoProviderFixture(t *testing.T) (string, string) { + t.Helper() + root := t.TempDir() + container := filepath.Join(root, "worktrees") + mainRepo := filepath.Join(container, "main_repo") + mustMkdirAll(t, filepath.Join(mainRepo, ".git", "worktrees", "deleted-child")) + return root, filepath.Join(container, "deleted-child", "src") +} + +func projectRootMemoFixture(t *testing.T) (string, string, string) { + t.Helper() + container := filepath.Join(t.TempDir(), "worktrees") + mainRepo := filepath.Join(container, "main_repo") + makeProjectMemoRepo(t, mainRepo, "known-child") + return container, filepath.Join(container, "known-child", "src"), mainRepo +} + +func makeProjectMemoRepo(t *testing.T, repo, child string) { + t.Helper() + mustMkdirAll(t, filepath.Join(repo, ".git", "worktrees", child)) +} diff --git a/internal/parser/provider.go b/internal/parser/provider.go index 1674fc133..c56946cd5 100644 --- a/internal/parser/provider.go +++ b/internal/parser/provider.go @@ -1160,6 +1160,8 @@ func providerFactoryForDef(def AgentDef) ProviderFactory { return newImportOnlyProviderFactory(def) case AgentCommandCode: return newCommandCodeProviderFactory(def) + case AgentCrush: + return newCrushProviderFactory(def) case AgentCodex: return newCodexProviderFactory(def) case AgentTraeX: diff --git a/internal/parser/provider_migration.go b/internal/parser/provider_migration.go index 47ea03939..21c4fba7e 100644 --- a/internal/parser/provider_migration.go +++ b/internal/parser/provider_migration.go @@ -81,6 +81,7 @@ var providerMigrationModes = map[AgentType]ProviderMigrationMode{ AgentPoolside: ProviderMigrationProviderAuthoritative, AgentOmnigent: ProviderMigrationProviderAuthoritative, AgentCodebuff: ProviderMigrationProviderAuthoritative, + AgentCrush: ProviderMigrationProviderAuthoritative, AgentDeepSeekHarness: ProviderMigrationProviderAuthoritative, } diff --git a/internal/parser/taxonomy.go b/internal/parser/taxonomy.go index 85e857380..5bf2ffe90 100644 --- a/internal/parser/taxonomy.go +++ b/internal/parser/taxonomy.go @@ -295,6 +295,11 @@ func NormalizeToolCategory(rawName string) string { case "shell_kill", "shell_status", "shell_tail": return "Bash" + // Charm Crush tools (only tools not already covered above: + // bash→Bash, view→Read, edit→Edit, write→Write) + case "todos": + return "Tool" + default: // MCP tools may carry a server prefix (e.g. // "Zencoder_subagent__ZencoderSubagent") or use diff --git a/internal/parser/taxonomy_test.go b/internal/parser/taxonomy_test.go index f33fdb531..cdc98b0db 100644 --- a/internal/parser/taxonomy_test.go +++ b/internal/parser/taxonomy_test.go @@ -132,6 +132,10 @@ func TestNormalizeToolCategory(t *testing.T) { {"skill", "Tool"}, {"search", "Tool"}, + // Charm Crush tools + // bash, view, edit, and write are covered in earlier sections. + {"todos", "Tool"}, + // Unknown {"view_image", "Other"}, {"update_plan", "Other"}, diff --git a/internal/parser/types.go b/internal/parser/types.go index 34769b751..fd3b5119d 100644 --- a/internal/parser/types.go +++ b/internal/parser/types.go @@ -81,6 +81,7 @@ const ( AgentOmnigent AgentType = "omnigent" AgentCodebuff AgentType = "codebuff" AgentFreebuff AgentType = "freebuff" + AgentCrush AgentType = "crush" ) const AgentDeepSeekHarness AgentType = "deepseek-harness" @@ -1034,6 +1035,26 @@ var Registry = []AgentDef{ NoPerMessageTokenData: true, }, }, + { + // Charm Crush keeps one SQLite store per project at + // /.crush/crush.db and records the project list in + // ~/.local/share/crush/projects.json; the provider expands that + // registry into roots at configuration time. + Type: AgentCrush, + DisplayName: "Charm Crush", + EnvVar: "CRUSH_DIR", + ConfigKey: "crush_dirs", + DefaultDirs: crushDefaultDirs(), + IDPrefix: "crush:", + FileBased: false, + Usage: UsageCapabilities{ + NoPerMessageTokenData: true, + }, + // Session rows live in a WAL-mode SQLite store whose change + // events are not authoritative; periodic reconcile covers + // registry and schema churn. + PeriodicReconcile: true, + }, } // AgentByType returns the AgentDef for the given type. diff --git a/internal/parser/types_test.go b/internal/parser/types_test.go index da610cb07..769a26c4e 100644 --- a/internal/parser/types_test.go +++ b/internal/parser/types_test.go @@ -504,6 +504,7 @@ func TestRegistryCompleteness(t *testing.T) { AgentPoolside, AgentOmnigent, AgentCodebuff, + AgentCrush, } expected := make(map[AgentType]bool, len(allTypes)) diff --git a/internal/remotesync/project_discovery_test.go b/internal/remotesync/project_discovery_test.go index 019483cd5..c66fb1ba8 100644 --- a/internal/remotesync/project_discovery_test.go +++ b/internal/remotesync/project_discovery_test.go @@ -31,6 +31,11 @@ func TestImporterUsesRecordedProjectWithoutLocalGitDiscovery(t *testing.T) { agent: parser.AgentCodex, filename: "rollout-2026-09-01T10-00-00-11111111-2222-4333-8444-555555555555.jsonl", id: "codex:11111111-2222-4333-8444-555555555555", body: `{"type":"session_meta","payload":{"id":"11111111-2222-4333-8444-555555555555","cwd":%s,"timestamp":"2026-09-01T10:00:00Z"}} {"type":"response_item","timestamp":"2026-09-01T10:01:00Z","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"Remote session content"}]}} +`, + }, + { + agent: parser.AgentCommandCode, filename: "project/demo.jsonl", id: "commandcode:demo", + body: `{"id":"m1","timestamp":"2026-09-01T10:00:00Z","sessionId":"demo","role":"user","content":[{"type":"text","text":"Remote session content"}],"metadata":{"cwd":%s}} `, }, { @@ -43,7 +48,6 @@ func TestImporterUsesRecordedProjectWithoutLocalGitDiscovery(t *testing.T) { }, } { t.Run(string(tc.agent), func(t *testing.T) { - database := dbtest.OpenTestDB(t) repo := filepath.Join(t.TempDir(), "local-repository") cwd := filepath.Join(repo, "recorded-project") // Plain directories exercise project discovery without invoking Git. @@ -58,20 +62,43 @@ func TestImporterUsesRecordedProjectWithoutLocalGitDiscovery(t *testing.T) { require.NoError(t, os.WriteFile(filepath.Join(sessions, tc.filename), []byte(fmt.Sprintf(tc.body, cwdJSON)), 0o600)) - stats, err := (Importer{Host: "source-host", DB: database}).ImportExtracted( - t.Context(), TargetSet{Dirs: map[parser.AgentType][]string{tc.agent: {remoteDir}}}, extracted, - ) - require.NoError(t, err) - require.Zero(t, stats.Failed) - require.Equal(t, 1, stats.SessionsSynced) - session, err := database.GetSessionFull(t.Context(), "source-host~"+tc.id) - require.NoError(t, err) - require.NotNil(t, session) - assert.Equal(t, "recorded_project", session.Project) - messages, err := database.GetMessages(t.Context(), session.ID, 0, 100, true) - require.NoError(t, err) - require.Len(t, messages, 1) - assert.Equal(t, "Remote session content", messages[0].Content) + for _, mode := range []string{"full", "delta"} { + t.Run(mode, func(t *testing.T) { + database := dbtest.OpenTestDB(t) + importer := Importer{ + Host: "source-host", DB: database, Root: extracted, + Targets: TargetSet{Dirs: map[parser.AgentType][]string{tc.agent: {remoteDir}}}, + } + var stats SyncStats + var err error + if mode == "full" { + stats, err = importer.ImportExtracted(t.Context(), importer.Targets, extracted) + } else { + journalPath, pathErr := mirrorRelativeLocalChangePath(extracted, filepath.Join(sessions, tc.filename)) + require.NoError(t, pathErr) + pending, prepareErr := importer.PreparePending(t.Context(), DeltaImportRequest{ + Journal: MirrorChangeJournal{ + Version: mirrorJournalVersion, + Entries: []MirrorChangeEntry{{Path: journalPath}}, + }, + }) + require.NoError(t, prepareErr) + stats, err = pending.Execute(t.Context()) + } + + require.NoError(t, err) + require.Zero(t, stats.Failed) + require.Equal(t, 1, stats.SessionsSynced) + session, err := database.GetSessionFull(t.Context(), "source-host~"+tc.id) + require.NoError(t, err) + require.NotNil(t, session) + assert.Equal(t, "recorded_project", session.Project) + messages, err := database.GetMessages(t.Context(), session.ID, 0, 100, true) + require.NoError(t, err) + require.Len(t, messages, 1) + assert.Equal(t, "Remote session content", messages[0].Content) + }) + } }) } } diff --git a/internal/sync/changed_path_sync.go b/internal/sync/changed_path_sync.go index ed4d8d4c0..cf7aa4384 100644 --- a/internal/sync/changed_path_sync.go +++ b/internal/sync/changed_path_sync.go @@ -52,6 +52,7 @@ func (e *Engine) SyncChangedPathPlanWithOptionsContext( options ChangedPathSyncOptions, onProgress ProgressFunc, ) (ChangedPathSyncResult, error) { + ctx = e.parsePolicyContext(ctx) result := ChangedPathSyncResult{ CachedSourceKeys: make(map[string]struct{}), CachedFallbackProviders: make(map[parser.AgentType]int), @@ -132,6 +133,7 @@ func (e *Engine) SyncChangedPathPlanWithOptionsContext( preContainerStates := e.captureSQLiteContainerStates(physicalPaths) e.beginSQLiteContainerPass(files, preContainerStates) processingCtx := context.WithValue(ctx, deferGlobalLinkContextKey{}, true) + processingCtx = parser.WithProjectRootMemo(processingCtx) results := e.startWorkers(processingCtx, files) affectedSessionIDs := make(map[string]struct{}) stats = e.collectAndBatchWithOptions( diff --git a/internal/sync/crush_integration_test.go b/internal/sync/crush_integration_test.go new file mode 100644 index 000000000..afd585bfe --- /dev/null +++ b/internal/sync/crush_integration_test.go @@ -0,0 +1,168 @@ +package sync + +import ( + "context" + "database/sql" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/agentsview/internal/parser" +) + +const crushSyncTestSchema = ` + CREATE TABLE sessions ( + id TEXT PRIMARY KEY, + parent_session_id TEXT, + title TEXT NOT NULL, + message_count INTEGER NOT NULL DEFAULT 0, + prompt_tokens INTEGER NOT NULL DEFAULT 0, + completion_tokens INTEGER NOT NULL DEFAULT 0, + cost REAL NOT NULL DEFAULT 0.0, + updated_at INTEGER NOT NULL, + created_at INTEGER NOT NULL, + summary_message_id TEXT, + todos TEXT + ); + CREATE TABLE messages ( + id TEXT PRIMARY KEY, + session_id TEXT NOT NULL, + role TEXT NOT NULL, + parts TEXT NOT NULL DEFAULT '[]', + model TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + finished_at INTEGER, + provider TEXT, + is_summary_message INTEGER DEFAULT 0 NOT NULL + ); + CREATE INDEX idx_messages_session_id ON messages (session_id); +` + +func TestSyncCrushReparsesWhenRegistryProjectPathChanges(t *testing.T) { + dataDir, _, _ := writeSyncCrushDB(t) + oldProjectDir := filepath.Join(t.TempDir(), "old-project") + newProjectDir := filepath.Join(t.TempDir(), "new-project") + require.NoError(t, os.MkdirAll(oldProjectDir, 0o755)) + require.NoError(t, os.MkdirAll(newProjectDir, 0o755)) + + registryDir := t.TempDir() + registryPath := filepath.Join(registryDir, parser.CrushProjectsFileName) + registry := `{"projects":[{"path":"` + filepath.ToSlash(oldProjectDir) + + `","data_dir":"` + filepath.ToSlash(dataDir) + `"}]}` + require.NoError(t, os.WriteFile(registryPath, []byte(registry), 0o600)) + + database := openTestDB(t) + engine := NewEngine(database, EngineConfig{ + AgentDirs: map[parser.AgentType][]string{ + parser.AgentCrush: {registryDir}, + }, + Machine: "devbox", + }) + t.Cleanup(engine.Close) + runSyncAndAssert(t, engine, SyncStats{TotalSessions: 1, Synced: 1}) + + session, err := database.GetSession(context.Background(), "crush:sess-001") + require.NoError(t, err) + require.NotNil(t, session) + assert.Equal(t, oldProjectDir, session.Cwd) + assert.Equal(t, "old_project", session.Project) + + registry = `{"projects":[{"path":"` + filepath.ToSlash(newProjectDir) + + `","data_dir":"` + filepath.ToSlash(dataDir) + `"}]}` + require.NoError(t, os.WriteFile(registryPath, []byte(registry), 0o600)) + runSyncAndAssert(t, engine, SyncStats{TotalSessions: 1, Synced: 1}) + + session, err = database.GetSession(context.Background(), "crush:sess-001") + require.NoError(t, err) + require.NotNil(t, session) + assert.Equal(t, newProjectDir, session.Cwd) + assert.Equal(t, "new_project", session.Project) +} + +func TestReconcileProviderRootsCrushDBFileRootPreservesDeletedSourceSession(t *testing.T) { + _, dbPath, sourceDB := writeSyncCrushDB(t) + database := openTestDB(t) + // Configure the database-file root that normalizeCrushRoots accepts and + // ResolveReconciliationScopes must map onto the data directory. + engine := NewEngine(database, EngineConfig{ + AgentDirs: map[parser.AgentType][]string{ + parser.AgentCrush: {dbPath}, + }, + Machine: "devbox", + }) + t.Cleanup(engine.Close) + runSyncAndAssert(t, engine, SyncStats{TotalSessions: 1, Synced: 1}) + + _, err := sourceDB.Exec(` + DELETE FROM messages WHERE session_id = 'sess-001'; + DELETE FROM sessions WHERE id = 'sess-001'; + `) + require.NoError(t, err) + require.NoError(t, sourceDB.Close()) + + // Reconcile with the original crush.db request root: the provider must + // expand it to the data directory so virtual members are in scope. + require.NoError(t, engine.ReconcileProviderRoots( + context.Background(), parser.AgentCrush, []string{dbPath}, + )) + + active, err := database.GetSession(context.Background(), "crush:sess-001") + require.NoError(t, err) + assert.NotNil(t, active) + archived, err := database.GetSessionFull(context.Background(), "crush:sess-001") + require.NoError(t, err) + require.NotNil(t, archived) + assert.Nil(t, archived.SourceMissingAt, + "source deletion must not hide a session from the persistent archive") +} + +func writeSyncCrushDB(t *testing.T) (string, string, *sql.DB) { + t.Helper() + projectDir := t.TempDir() + dataDir := filepath.Join(projectDir, ".crush") + require.NoError(t, os.MkdirAll(dataDir, 0o755)) + dbPath := filepath.Join(dataDir, parser.CrushDBName) + database, err := sql.Open("sqlite3", dbPath) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, database.Close()) }) + _, err = database.Exec(crushSyncTestSchema) + require.NoError(t, err) + _, err = database.Exec(` + INSERT INTO sessions ( + id, title, created_at, updated_at, + prompt_tokens, completion_tokens, cost + ) VALUES ( + 'sess-001', 'Auth review', 1789093626, 1789093746, 100, 20, 0.0125 + ) + `) + require.NoError(t, err) + insertSyncCrushMessage(t, database, "msg-user", "user", `[ + {"type":"text","data":{"text":"Inspect the auth flow."}} + ]`, 1_789_093_626, "") + insertSyncCrushMessage(t, database, "msg-assistant", "assistant", `[ + {"type":"text","data":{"text":"I will inspect the file."}} + ]`, 1_789_093_629, "glm-5.3-flash") + return dataDir, dbPath, database +} + +func insertSyncCrushMessage( + t *testing.T, database *sql.DB, id, role, parts string, created int64, model string, +) { + t.Helper() + _, err := database.Exec(` + INSERT INTO messages ( + id, session_id, role, parts, model, created_at, updated_at + ) VALUES (?, 'sess-001', ?, ?, ?, ?, ?) + `, id, role, parts, model, created, created) + require.NoError(t, err) + _, err = database.Exec(` + UPDATE sessions SET message_count = ( + SELECT COUNT(*) FROM messages WHERE session_id = 'sess-001' + ) WHERE id = 'sess-001' + `) + require.NoError(t, err) +} diff --git a/internal/sync/engine.go b/internal/sync/engine.go index fa4c86f99..bf6038690 100644 --- a/internal/sync/engine.go +++ b/internal/sync/engine.go @@ -1752,6 +1752,7 @@ func (e *Engine) syncChangedPathsLocked( Phase: PhaseDiscovering, Detail: "Preparing changed session paths", }) + ctx = parser.WithProjectRootMemo(ctx) return e.applyChangedPathSyncLocked( ctx, e.prepareChangedPathSync(ctx, paths), ) @@ -5140,6 +5141,7 @@ func (e *Engine) reconcileWatchRootsStreamedLocked( stats.Aborted = true return stats, metrics, 0, eligibility, err } + ctx = parser.WithProjectRootMemo(ctx) defer func() { if cleanupErr := closeProviderCache(); cleanupErr != nil { stats.Aborted = true @@ -6795,6 +6797,11 @@ func (e *Engine) tombstoneMissingWatchSourceScopesLocked( PathRewriter: e.pathRewriter, }) } + if provider != nil && + provider.Capabilities().Source.ExplicitDeletionOnly == + parser.CapabilitySupported { + continue + } for _, scope := range agentScopes { ownershipScopes := storedSourceDBHintScopes(scope.proofScopes) if len(ownershipScopes) == 0 { @@ -7561,6 +7568,7 @@ func (e *Engine) syncAllLocked( return SyncStats{Aborted: true} } ctx = e.parsePolicyContext(ctx) + ctx = parser.WithProjectRootMemo(ctx) if recordSyncState { e.recordSyncStarted() @@ -7771,7 +7779,7 @@ func (e *Engine) syncAllLocked( // through the provider facade in the file-sync phase above, so no // dedicated DB-backed sync pass is needed here. - // Sync Warp, Forge, Piebald, ZCode, and Goose sessions. These are + // Sync Warp, Forge, Piebald, ZCode, Goose, and Crush sessions. These are // provider-authoritative DB-backed providers: a shared SQLite DB hosts every // session, so the provider facade enumerates sources and parses only the // changed ones. @@ -7820,9 +7828,18 @@ func (e *Engine) syncAllLocked( return stats } } + if scope.includesAny(e.agentDirs[parser.AgentCrush]) { + if e.syncProviderDBBackedAgent( + ctx, parser.AgentCrush, "crush", + writeMode, verbose, scope, &stats, advanceDBProgress, + ) { + stats.Aborted = true + return stats + } + } // Link subagent child sessions to their parents after all DB-backed - // agent writes (including provider-authoritative Forge, Goose, Piebald, - // and ZCode). + // agent writes (including provider-authoritative Crush, Forge, Goose, + // Piebald, and ZCode). // LinkSubagentSessions is idempotent — its WHERE filter and partial index // make it a cheap no-op when nothing new was written — so no guard is // needed. @@ -9435,7 +9452,7 @@ func (e *Engine) providerDBBackedSourceFresh( } // syncProviderDBBackedAgent runs the full-sync phase for a provider-authoritative -// DB-backed agent (Forge, Goose, Piebald, Warp, ZCode). It mirrors +// DB-backed agent (Crush, Forge, Goose, Piebald, Warp, ZCode). It mirrors // syncOpenCodeFormatAgent: // only changed sessions are parsed (so the second sync of unchanged data is a // no-op), and the per-session write semantics match the legacy DB sync. @@ -13575,7 +13592,7 @@ func (e *Engine) providerSkipCacheEntryFreshInDB( func processFileUsesProvider(agent parser.AgentType) bool { switch agent { case parser.AgentForge, parser.AgentGoose, parser.AgentPiebald, - parser.AgentWarp, parser.AgentZCode: + parser.AgentWarp, parser.AgentZCode, parser.AgentCrush: return true default: return false @@ -13628,7 +13645,8 @@ func (e *Engine) shouldSkipProviderSource( func providerSourceSupportsPersistedFreshness(agent parser.AgentType) bool { switch agent { - case parser.AgentForge, parser.AgentGoose, parser.AgentWarp, parser.AgentZCode: + case parser.AgentForge, parser.AgentGoose, parser.AgentWarp, parser.AgentZCode, + parser.AgentCrush: return true default: return false @@ -19972,8 +19990,8 @@ func (e *Engine) FindSourceFile(sessionID string) string { } rawSessionID := strings.TrimPrefix(rawID, def.IDPrefix) if !def.FileBased { - // Forge, Piebald, Warp, and ZCode are DB-backed providers that own - // discovery and source lookup through the provider facade. Their + // Crush, Forge, Piebald, Warp, and ZCode are DB-backed providers that + // own discovery and source lookup through the provider facade. Their // virtual # path is resolved by findProviderSourceFile // below. Non-provider, non-file-based agents (e.g. remote imports) // have no local source file. @@ -20261,8 +20279,8 @@ func (e *Engine) SourceMtime(sessionID string) int64 { } rawSessionID := strings.TrimPrefix(rawID, def.IDPrefix) if !def.FileBased { - // Forge, Piebald, Warp, and ZCode are DB-backed providers: their - // per-session source mtime comes from the provider fingerprint + // Crush, Forge, Piebald, Warp, and ZCode are DB-backed providers: + // their per-session source mtime comes from the provider fingerprint // (which mirrors the legacy List*SessionMeta last-modified value). // Non-provider, non-file-based agents have no local source. if e.isProviderAuthoritative(def.Type) { @@ -20555,11 +20573,11 @@ func (e *Engine) SyncSingleSessionContext( return fmt.Errorf("unknown agent for session %s", sessionID) } if !def.FileBased { - // Forge, Piebald, Warp, and ZCode are DB-backed providers: re-sync routes - // through FindSourceFile (resolving the virtual # - // path) plus the provider-aware processFile path below, mirroring - // the file-based agents. Other non-file-based agents use the - // OpenCode-format storage path. + // Crush, Forge, Piebald, Warp, and ZCode are DB-backed providers: + // re-sync routes through FindSourceFile (resolving the virtual + // # path) plus the provider-aware processFile path + // below, mirroring the file-based agents. Other non-file-based + // agents use the OpenCode-format storage path. if !e.isProviderAuthoritative(def.Type) { return fmt.Errorf( "cannot resync non-file-based session %s for agent %s", diff --git a/internal/sync/engine_integration_test.go b/internal/sync/engine_integration_test.go index 781c63220..a84e46990 100644 --- a/internal/sync/engine_integration_test.go +++ b/internal/sync/engine_integration_test.go @@ -1284,6 +1284,127 @@ func (f openCodeStorageParseCountingFactory) NewProvider( return f.provider } +type crushParseCountingProvider struct { + parser.Provider + parseCalls atomic.Int64 +} + +func (p *crushParseCountingProvider) Parse( + ctx context.Context, req parser.ParseRequest, +) (parser.ParseOutcome, error) { + p.parseCalls.Add(1) + return p.Provider.Parse(ctx, req) +} + +func (p *crushParseCountingProvider) DiscoverEach( + ctx context.Context, yield func(parser.SourceRef) error, +) error { + discoverer, ok := p.Provider.(parser.StreamingDiscoverer) + if !ok { + return fmt.Errorf("Crush provider does not support streaming discovery") + } + return discoverer.DiscoverEach(ctx, yield) +} + +type crushParseCountingFactory struct { + provider *crushParseCountingProvider +} + +func (f crushParseCountingFactory) Definition() parser.AgentDef { + return f.provider.Definition() +} + +func (f crushParseCountingFactory) Capabilities() parser.Capabilities { + return f.provider.Capabilities() +} + +func (f crushParseCountingFactory) NewProvider( + parser.ProviderConfig, +) parser.Provider { + return f.provider +} + +func TestSyncAllCrushSkipsUnchangedSessionBeforeParse(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test") + } + root := t.TempDir() + crushPath := filepath.Join(root, parser.CrushDBName) + source, err := sql.Open("sqlite3", crushPath) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, source.Close()) }) + _, err = source.Exec(` + CREATE TABLE sessions ( + id TEXT PRIMARY KEY, + parent_session_id TEXT, + title TEXT NOT NULL, + message_count INTEGER NOT NULL DEFAULT 0, + prompt_tokens INTEGER NOT NULL DEFAULT 0, + completion_tokens INTEGER NOT NULL DEFAULT 0, + cost REAL NOT NULL DEFAULT 0, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ); + CREATE TABLE messages ( + id TEXT PRIMARY KEY, + session_id TEXT NOT NULL, + role TEXT NOT NULL, + parts TEXT NOT NULL, + model TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ); + INSERT INTO sessions ( + id, title, message_count, created_at, updated_at + ) VALUES ('stable', 'Stable session', 1, 1789093626, 1789093626); + INSERT INTO messages ( + id, session_id, role, parts, created_at, updated_at + ) VALUES ( + 'message-1', 'stable', 'user', + '[{"type":"text","data":{"text":"Keep this session."}}]', + 1789093626, 1789093626 + ); + `) + require.NoError(t, err) + + innerFactory, ok := parser.ProviderFactoryByType(parser.AgentCrush) + require.True(t, ok, "Crush provider factory registered") + inner := innerFactory.NewProvider(parser.ProviderConfig{ + Roots: []string{root}, + Machine: "local", + }) + counting := &crushParseCountingProvider{Provider: inner} + database := dbtest.OpenTestDB(t) + engine := sync.NewEngine(database, sync.EngineConfig{ + AgentDirs: map[parser.AgentType][]string{ + parser.AgentCrush: {root}, + }, + Machine: "local", + ProviderFactories: []parser.ProviderFactory{ + crushParseCountingFactory{provider: counting}, + }, + ProviderMigrationModes: map[parser.AgentType]parser.ProviderMigrationMode{ + parser.AgentCrush: parser.ProviderMigrationProviderAuthoritative, + }, + }) + t.Cleanup(engine.Close) + + first := engine.SyncAll(t.Context(), nil) + require.False(t, first.Aborted, "first sync aborted: %+v", first) + require.Equal(t, 1, first.Synced, "first sync writes the session") + require.Equal(t, int64(1), counting.parseCalls.Load()) + + second := engine.SyncAll(t.Context(), nil) + require.False(t, second.Aborted, "second sync aborted: %+v", second) + assert.Zero(t, second.Synced, "unchanged session must not be rewritten") + assert.Equal(t, int64(1), counting.parseCalls.Load(), + "unchanged session must be skipped before parsing") + + engine.SyncPaths([]string{crushPath + "#stable"}) + assert.Equal(t, int64(1), counting.parseCalls.Load(), + "an unchanged session path must use the stored fingerprint") +} + // A fresh engine has no in-memory storage-tree trust, so it fingerprints each // legacy OpenCode session once. The persisted source metadata must then prove // an unchanged session fresh without parsing the storage tree a second time. diff --git a/internal/sync/issue1717_pass_memo_test.go b/internal/sync/issue1717_pass_memo_test.go new file mode 100644 index 000000000..effab58a4 --- /dev/null +++ b/internal/sync/issue1717_pass_memo_test.go @@ -0,0 +1,304 @@ +package sync + +import ( + "context" + "fmt" + "os" + "path/filepath" + stdsync "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/agentsview/internal/parser" +) + +func TestIssue1717PassMemoReusesAcrossPages(t *testing.T) { + const sourceCount = reconciliationPageSize + 1 + provider, root := newIssue1717Provider(t, sourceCount, reconciliationPageSize-1) + engine := newIssue1717Engine(t, provider, root) + + require.NoError(t, engine.ReconcileWatchRoots( + t.Context(), []string{root}, false, + )) + for i := range sourceCount { + session, err := engine.db.GetSession( + t.Context(), issue1717SessionID(i), + ) + require.NoError(t, err) + require.NotNil(t, session, "session %d must be archived", i) + assert.Equal(t, "main_repo", session.Project, + "page boundary must keep the memoized project") + } + t.Logf("sources=%d page_size=%d stable_project=%s", + sourceCount, reconciliationPageSize, "main_repo") +} + +func TestIssue1717ChangedPathPlanMemoLifetime(t *testing.T) { + provider, root := newIssue1717Provider(t, 2, 0) + engine := newIssue1717Engine(t, provider, root) + files := make([]parser.DiscoveredFile, len(provider.sources)) + for i, source := range provider.sources { + files[i] = parser.DiscoveredFile{ + Path: source.Key, + Agent: source.Provider, + ProviderSource: &provider.sources[i], + ProviderProcess: true, + ForceParse: true, + } + } + + result, err := engine.SyncChangedPathPlanContext(t.Context(), ChangedPathPlan{ + Files: files, + }, nil) + require.NoError(t, err) + assert.Equal(t, len(files), result.FilesProcessed) + for i := range files { + session, err := engine.db.GetSession(t.Context(), issue1717SessionID(i)) + require.NoError(t, err) + require.NotNil(t, session) + assert.Equal(t, "main_repo", session.Project) + } +} + +func TestIssue1717NextOperationRefresh(t *testing.T) { + provider, root := newIssue1717Provider(t, 1, -1) + engine := newIssue1717Engine(t, provider, root) + + require.NoError(t, engine.ReconcileWatchRootsAfterLostEvents( + t.Context(), []string{root}, false, + )) + require.NoError(t, os.RemoveAll(provider.mainRepo)) + makeIssue1717Repo(t, provider.newRepo) + provider.resetForOperation() + + require.NoError(t, engine.ReconcileWatchRootsAfterLostEvents( + t.Context(), []string{root}, false, + )) + session, err := engine.db.GetSession(t.Context(), issue1717SessionID(0)) + require.NoError(t, err) + require.NotNil(t, session) + assert.Equal(t, "new_repo", session.Project) +} + +type issue1717Provider struct { + parser.ProviderBase + sources []parser.SourceRef + indices map[string]int + cwd string + mutate func() + mutateAt int + mainRepo string + newRepo string + + mu stdsync.Mutex + cond *stdsync.Cond + next int +} + +func (p *issue1717Provider) Discover( + context.Context, +) ([]parser.SourceRef, error) { + return append([]parser.SourceRef(nil), p.sources...), nil +} + +func (p *issue1717Provider) DiscoverEach( + ctx context.Context, yield func(parser.SourceRef) error, +) error { + for _, source := range p.sources { + if err := ctx.Err(); err != nil { + return err + } + if err := yield(source); err != nil { + return err + } + } + return nil +} + +func (p *issue1717Provider) SourceForReconciliation( + ctx context.Context, path, project string, +) (parser.SourceRef, bool, error) { + if err := ctx.Err(); err != nil { + return parser.SourceRef{}, false, err + } + for _, source := range p.sources { + if source.DisplayPath == path { + source.ProjectHint = project + return source, true, nil + } + } + return parser.SourceRef{}, false, nil +} + +func (p *issue1717Provider) WatchPlan( + context.Context, +) (parser.WatchPlan, error) { + return parser.WatchPlan{}, nil +} + +func (p *issue1717Provider) Fingerprint( + context.Context, parser.SourceRef, +) (parser.SourceFingerprint, error) { + return parser.SourceFingerprint{Hash: "issue1717"}, nil +} + +func (p *issue1717Provider) Parse( + ctx context.Context, req parser.ParseRequest, +) (parser.ParseOutcome, error) { + index := p.indices[req.Source.Key] + p.mu.Lock() + for index != p.next { + p.cond.Wait() + } + p.mu.Unlock() + + project := parser.ExtractProjectFromCwdWithBranchContext(ctx, p.cwd, "") + if index == p.mutateAt && p.mutate != nil { + p.mutate() + } + + p.mu.Lock() + p.next++ + p.cond.Broadcast() + p.mu.Unlock() + + started := time.Unix(1704067200, 0) + return parser.ParseOutcome{ + Results: []parser.ParseResultOutcome{{ + Result: parser.ParseResult{Session: parser.ParsedSession{ + ID: issue1717SessionID(index), Agent: req.Source.Provider, + Project: project, Machine: "local", Cwd: p.cwd, + StartedAt: started, EndedAt: started, + File: parser.FileInfo{Path: req.Source.FingerprintKey}, + }}, + DataVersion: parser.DataVersionCurrent, + }}, + ResultSetComplete: true, + }, nil +} + +func (p *issue1717Provider) resetForOperation() { + p.mu.Lock() + p.next = 0 + p.mu.Unlock() +} + +type issue1717Factory struct { + provider *issue1717Provider +} + +func (f issue1717Factory) Definition() parser.AgentDef { + return f.provider.Definition() +} + +func (f issue1717Factory) Capabilities() parser.Capabilities { + return f.provider.Capabilities() +} + +func (f issue1717Factory) NewProvider( + cfg parser.ProviderConfig, +) parser.Provider { + return issue1717ScopedProvider{ + issue1717Provider: f.provider, + scopes: parser.ProviderBase{ + Def: f.provider.Def, Caps: f.provider.Caps, Config: cfg.Clone(), + }, + } +} + +type issue1717ScopedProvider struct { + *issue1717Provider + scopes parser.ProviderBase +} + +func (p issue1717ScopedProvider) ResolveReconciliationScopes( + ctx context.Context, req parser.ReconciliationScopeRequest, +) (parser.ReconciliationScopePlan, error) { + return p.scopes.ResolveReconciliationScopes(ctx, req) +} + +func newIssue1717Provider( + t *testing.T, sourceCount, mutateAt int, +) (*issue1717Provider, string) { + t.Helper() + root := t.TempDir() + worktreeRoot := filepath.Join(root, "worktrees") + mainRepo := filepath.Join(worktreeRoot, "main_repo") + makeIssue1717Repo(t, mainRepo) + newRepo := filepath.Join(worktreeRoot, "new_repo") + cwd := filepath.Join(worktreeRoot, "deleted-child", "src") + sourceRoot := filepath.Join(root, "sources") + require.NoError(t, os.MkdirAll(sourceRoot, 0o755)) + + sources := make([]parser.SourceRef, sourceCount) + indices := make(map[string]int, sourceCount) + for i := range sources { + path := filepath.Join(sourceRoot, fmt.Sprintf("session-%03d.jsonl", i)) + require.NoError(t, os.WriteFile(path, []byte("source\n"), 0o600)) + sources[i] = parser.SourceRef{ + Provider: parser.AgentType("issue1717"), Key: path, + DisplayPath: path, FingerprintKey: path, + } + indices[path] = i + } + provider := &issue1717Provider{ + sources: sources, indices: indices, cwd: cwd, mutateAt: mutateAt, + mainRepo: mainRepo, newRepo: newRepo, + } + provider.ProviderBase = parser.ProviderBase{ + Def: parser.AgentDef{Type: parser.AgentType("issue1717"), FileBased: true}, + Caps: parser.Capabilities{Source: parser.SourceCapabilities{ + DiscoverSources: parser.CapabilitySupported, + StreamingDiscovery: parser.CapabilitySupported, + WatchSources: parser.CapabilitySupported, + }}, + } + provider.cond = stdsync.NewCond(&provider.mu) + provider.mutate = func() { + if err := os.RemoveAll(mainRepo); err != nil { + t.Errorf("remove initial repository: %v", err) + } + if err := os.MkdirAll( + filepath.Join(newRepo, ".git", "worktrees", "deleted-child"), + 0o755, + ); err != nil { + t.Errorf("create replacement repository: %v", err) + } + } + return provider, root +} + +func newIssue1717Engine( + t *testing.T, provider *issue1717Provider, root string, +) *Engine { + t.Helper() + engine := NewEngine(openTestDB(t), EngineConfig{ + AgentDirs: map[parser.AgentType][]string{ + provider.Def.Type: {root}, + }, + Machine: "local", + ProviderFactories: []parser.ProviderFactory{ + issue1717Factory{provider: provider}, + }, + ProviderMigrationModes: map[parser.AgentType]parser.ProviderMigrationMode{ + provider.Def.Type: parser.ProviderMigrationProviderAuthoritative, + }, + }) + t.Cleanup(engine.Close) + return engine +} + +func makeIssue1717Repo(t *testing.T, repo string) { + t.Helper() + require.NoError(t, os.MkdirAll( + filepath.Join(repo, ".git", "worktrees", "deleted-child"), + 0o755, + )) +} + +func issue1717SessionID(index int) string { + return fmt.Sprintf("issue1717:%03d", index) +} diff --git a/internal/sync/parsediff.go b/internal/sync/parsediff.go index 939496bf2..d4e373afd 100644 --- a/internal/sync/parsediff.go +++ b/internal/sync/parsediff.go @@ -135,6 +135,7 @@ func (e *Engine) ParseDiff(ctx context.Context, opts ParseDiffOptions) (*ParseDi // instead of parsing every remaining file just to drain it. runCtx, cancel := context.WithCancel(ctx) defer cancel() + runCtx = parser.WithProjectRootMemo(runCtx) results := e.startWorkers(runCtx, files) for i := range total { var r syncJob diff --git a/internal/sync/provider_process_test.go b/internal/sync/provider_process_test.go index 47062f6f8..4a036da6f 100644 --- a/internal/sync/provider_process_test.go +++ b/internal/sync/provider_process_test.go @@ -459,6 +459,7 @@ func TestProcessFileProviderZCodeVirtualSource(t *testing.T) { func TestProcessFileUsesProviderDBBackedFamily(t *testing.T) { for _, agent := range []parser.AgentType{ + parser.AgentCrush, parser.AgentForge, parser.AgentGoose, parser.AgentPiebald,