diff --git a/cmd/agentsview/sync_worker.go b/cmd/agentsview/sync_worker.go index b8504408b..41e7bc7d5 100644 --- a/cmd/agentsview/sync_worker.go +++ b/cmd/agentsview/sync_worker.go @@ -196,11 +196,17 @@ func runSyncWorkerStartup( var stats sync.SyncStats var tombstoned int var auditErr error + // The audit is also the periodic content-verification pass for + // checkpointed sources: bypass the stat-trust gate so the provider's + // full-source fingerprint detects and repairs same-stat in-place + // rewrites that append-trust would otherwise keep stale. + engine.SetCheckpointAudit(true) if auditRoots := reconcileRootPaths(cfg); len(auditRoots) > 0 { stats, tombstoned, auditErr = engine.ReconcileWatchRootsWithStats( ctx, auditRoots, false, onProgress, ) } + engine.SetCheckpointAudit(false) result = workerResultFromStats(ctx, stats) result.Tombstoned = tombstoned if auditErr != nil && result.Status == "ok" { diff --git a/docs/internal/background-sync-efficiency.md b/docs/internal/background-sync-efficiency.md index a533ec2da..9fbae602b 100644 --- a/docs/internal/background-sync-efficiency.md +++ b/docs/internal/background-sync-efficiency.md @@ -63,32 +63,79 @@ they are never published as safe cursor boundaries. Truncation, known file-identity replacement, manual or project refreshes, `session_index.jsonl` title changes, and records that retroactively update -stored messages all fall back to an authoritative full replacement. Safe -incremental writes preserve the index-folded mtime and lifecycle-derived -termination status alongside message and token aggregates. +stored messages all fall back to an authoritative full replacement. Late tool +results are the exception: the cursor tracks pending tool calls (bounded), so +a `function_call_output` / `custom_tool_call_output` that refers to a call +committed in an earlier batch is applied as an idempotent point update +(`ToolCallResultUpdates`) instead of a full reparse. Agent-scoped and unknown +calls still fall back. Safe incremental writes preserve the index-folded mtime +and lifecycle-derived termination status alongside message and token +aggregates. ## Append-only limitation Cursor correctness assumes that growth is append-only. A same-inode file can grow after bytes inside its already-committed prefix have been rewritten. Size, -identity, and boundary checks do not detect that case, and the current -full-source fingerprint is not compared with a separately verified stored prefix -before incremental parsing. Closing this gap would require rolling hash state or -explicit prefix verification and remains deferred. +identity, and boundary checks cannot prove the prefix was never modified. + +Persisted checkpoints close most of the gap under the documented append-trust +mode: + +- The checkpoint stores a 128 KiB tail anchor of the committed prefix, the + file identity, the committed offset, the parser cursor, and a resumable + SHA-256 state over the committed prefix. +- An append is only resumed when the identity matches, the size only grew, + and the current bytes at the anchor region match the stored anchor; the + full-file fingerprint is then derived by hashing only the appended bytes. +- An unchanged checkpointed source is skipped on stat alone (no transcript + read). An anchor mismatch, identity change, truncation, or undecodable + checkpoint forces an authoritative full parse and checkpoint rebuild. +- A same-size, same-mtime in-place rewrite that preserves the anchor region is + trusted (append-trust). Periodic full audits (`ResyncAll`, `--full`, + force-reverification passes) still hash the whole source and repair such + rewrites. Strict verification remains available by bypassing the checkpoint + gate. ## Cost model and regression evidence A warm Codex cursor makes continuation-state parsing scale with appended records -rather than transcript history. End-to-end append sync is still O(file): the -provider's `Fingerprint` hashes the complete source and the engine's -`ComputeFileHashPrefix` hashes through the newly committed offset. +rather than transcript history. With a persisted checkpoint, end-to-end append +sync is O(d): the engine resumes the SHA-256 state over only the appended +bytes, verifies the 128 KiB tail anchor by digest, and parses only the new +tail. The append path reads the source roughly three times the delta +(fingerprint resume, parser tail, final checkpoint resume) plus the 128 KiB +anchor — for a 1 KiB append that is well inside the 256 KiB source-read gate. +An unchanged checkpointed source costs a stat plus the small checkpoint +metadata row read (the cursor and hash-state blobs live in a separate table +and are never loaded on the stat-only path) and reads 0 transcript bytes. +A full parse captures the resumable hash state and anchor digest on its own +read pass, so persisting the checkpoint adds no second source read. Without +a checkpoint (legacy sessions, first sync after upgrade) the previous +O(file) fingerprint and prefix-hash reads still apply until the next full +parse persists one. + +The daily archive audit (`sync_worker` audit mode) bypasses the checkpoint +stat-trust gate so the provider's full-source fingerprint verifies content and +repairs same-stat in-place rewrites that append-trust would otherwise keep +stale. - `BenchmarkCodexIncrementalCursor` in `internal/parser` compares cold prefix reconstruction with the exact warm cursor. It is diagnostic because `internal/parser` is not in `BENCH_GATE_PACKAGES`. -- `BenchmarkCodexIncrementalSyncReads` in `internal/sync` measures the warm tail - between the two remaining linear reads. It is PR-gated because - `internal/sync` is in `BENCH_GATE_PACKAGES`. +- `BenchmarkCodexCheckpointAppendResume` in `internal/sync` measures the + checkpoint-resumed append's source pipeline (checkpoint gate, seeded tail + parse, resume hash, next anchor digest, checkpoint assembly), bounded by + the anchor window plus the tail. It is PR-gated because `internal/sync` is + in `BENCH_GATE_PACKAGES`. +- `BenchmarkCodexQuietAppendSignals500/5000/15000` in `internal/sync` measure + the non-amortized quiet-session append (call + late output) with full + inline signal/secret maintenance; the three sizes gate the + history-independent latency slope. +- `BenchmarkCodexLateToolOutputDebouncedBurst` in `internal/sync` measures a + debounced stream where every appended batch carries the output for the + previous batch's call: only the first iteration pays the O(history) + signal recompute, so it guards per-append cost without pretending to be + the quiet-session gate. The maintained behavioral gate inventory is in [Performance Gates](performance-gates.md). diff --git a/docs/internal/codex-incremental-streaming-design.md b/docs/internal/codex-incremental-streaming-design.md new file mode 100644 index 000000000..1d575a01e --- /dev/null +++ b/docs/internal/codex-incremental-streaming-design.md @@ -0,0 +1,84 @@ +# Codex incremental checkpoints and streamed full imports + +## Problem + +A busy Codex rollout grows to hundreds of megabytes across thousands of +tool-result events. Three operations that should be cheap are not: + +- an unchanged file costs a full transcript read on every sweep; +- a small appended tail re-reads and re-sums the whole transcript; +- a cold import holds the full message slice and every tool-result body + in memory at once (a 945 MB archive peaked above 1 GB). + +## What this branch does + +1. **Persistent safe-resume checkpoints.** A full parse captures a + resumable SHA-256 state, the trailing 128 KiB anchor digest, and the + file identity in one pass. The checkpoint row commits in the same + transaction as the session content. An unchanged file — inode, device, + size, mtime, and change-time all matching — is skipped without reading + the transcript. Appends resume from the stored offset and are proven + against the tail anchor before the rows commit. + +2. **Cross-sync tool results as transactional deltas.** A tool output + appended after its call was persisted no longer forces a full re-parse. + The incremental tail yields deferred result updates that the writer + applies with targeted probes: event deduplication against stored rows, + a per-call agent-state table that resolves the latest content per agent + by event coordinates (no content copies), and signals/findings folded + incrementally. The checkpoint advances atomically with the write. + +3. **Streamed cold imports.** Decoding emits through a session sink; a + staging sink writes event rows into a scratch SQLite database while the + in-memory model keeps only placeholders. The publish transaction + attaches the scratch database, copies event rows and per-call summaries + into the archive, and commits messages, events, summaries, signals, and + findings atomically. Files above 128 MB take this path. + +## Correctness boundaries + +- The checkpoint is trusted only when the stored hash state matches the + committed prefix hash and the file identity (including change-time) + matches. Truncation, replacement, and same-size same-mtime rewrites + rebuild authoritatively. A periodic audit (`ResyncAll`) remains the + backstop for anything the stat gate cannot see. +- Fork and subagent replays match the parent transcript's turn ids as + opaque membership keys; an unresolved explicit parent keeps the child + visible but marks its data version for retry. +- A scratch write failure is sticky: the parse and the publish fail and + the archive keeps its prior content. The staging ATTACH is torn down + after every transaction, so consecutive publishes share one writer + connection safely. + +## Costs + +- Disk: tool-result content is still stored twice by the existing + archive layout (`tool_result_events.content` and + `tool_calls.result_content`). This branch does not change that. The + agent-state table adds only integer coordinate rows. +- Runtime: a staged cold import holds the process GC target lower for the + duration of the parse and returns parse-phase arenas before the + publish, trading CPU for a bounded RSS. + +## Suggested PR split + +The branch is intentionally one working line of history, but the +mergeable sequence is: + +1. persistent safe-resume checkpoints (0-byte no-op, O(delta) appends); +2. cross-sync tool-result deltas on top of it; +3. byte-bounded bulk admission; +4. the behavior-preserving session-sink parser refactor; +5. the scratch-staging streamed import with its runtime policies. + +## Where to look + +- `internal/parser/codex.go`, `codex_cursor.go`, `codex_provider.go` — + single-pass hash/anchor, cursor codec, fork replay gate. +- `internal/sync/checkpoint.go`, `internal/db/checkpoint.go` — checkpoint + persistence and the append/no-op decision. +- `internal/db/messages.go` — transactional late-result updates and the + agent-state table. +- `internal/sync/codex_staging.go`, `internal/db/staged_content.go` — + scratch staging sink and the staged publish transaction. +- `internal/signals/incremental.go` — the typed incremental reducer. diff --git a/docs/internal/performance-gates.md b/docs/internal/performance-gates.md index 258882f86..42b1b64b7 100644 --- a/docs/internal/performance-gates.md +++ b/docs/internal/performance-gates.md @@ -17,6 +17,7 @@ contracts are documented in | Discovery O(sources) root work | Gemini rebuilt its project map per session; positron/vscode-copilot re-read `workspace.json` per session. A large store spent 2m47s in discovery. | #912 | | Unchanged sources reparsed | The provider migration dropped pre-parse DB-freshness skips; every full sync reparsed and rewrote untouched sessions. | `providerSourceUnchangedInDB` (#883 follow-up) | | O(history) incremental appends | Every streamed line ran a full signal recompute (reload all messages, secret regex scan) and chunk merges delete+reinserted every message row. ~4,700 session updates/day each paid O(session history). | #954 | +| O(file) warm Codex appends | A warm append still hashed the full source for the fingerprint and re-hashed the committed prefix after every write; late tool outputs forced a full transcript reparse, and unchanged startup hashed every source before the DB freshness skip. | persisted parser checkpoints (this PR) | | Bulk ingest throughput | Full resync ran per-row inserts and rebuilt FTS incrementally; 26.7k sessions took 1m17s. | #411 | | Event storms | One SSE emit per watcher flush drove ~1/s dashboard refetch; SQLite WAL sidecar events fanned out to every session in a shared DB. | #367, #956 | | Per-row query shape | `GetDailyUsage` ran 1.2M `json_extract` calls per scan and had no date pushdown. | #309 | @@ -38,6 +39,26 @@ runner noise and fail loudly: - `TestWriteIncrementalDebouncesSignalRecompute` and the rest of `internal/sync/signal_schedule_test.go` — streaming appends must debounce the O(history) signal recompute. +- `TestCodexCheckpoint*` in `internal/sync/checkpoint_test.go` — a full Codex + parse persists a checkpoint, an append resumes from it and advances it + atomically with the delta, truncation/anchor mismatch force an + authoritative rebuild, and an unchanged checkpointed source is trusted on + stat alone (append-trust; the audit path still catches rewrites). +- `TestCodexCheckpointStaleCannotResumeFromNewerDBOffset`, + `TestCodexCheckpointHashStateBoundedToCommittedOffset`, + `TestCodexCheckpointColdRestartResumeParity`, and + `TestCodexCheckpointAuditRepairsSameStatRewrite` + (`internal/sync/checkpoint_review_test.go`) — a surviving checkpoint must + agree with the committed DB offset/ordinal/hash, hash state covers exactly + the committed prefix, cold restarts resume with parity, and the audit + repairs same-stat rewrites. +- `TestParserCheckpointRoundTrip` and + `TestWriteSessionIncrementalPersistsCheckpointInSameTx` (`internal/db`) — + checkpoint rows round-trip and are committed in the same transaction as the + incremental delta. +- `TestFullSyncPassIsByteBudgeted`, `TestBulkParseRetentionBudgetUsesWeightedAdmission`, + and `TestCollectAndBatchFlushesOnByteCap` (`internal/sync`) — bulk passes + and write batches are bounded by estimated bytes, not session count alone. - The count-based seam tests in `internal/parser` (`discovery_workspace_manifest_test.go`, gemini/antigravity provider tests) — root-derived project info is built once per root, not once per source. @@ -55,10 +76,13 @@ runner noise and fail loudly: warm/cold parsing remains equivalent at safe offsets. - `TestIncrementalSync_CodexAppend`, `TestIncrementalSync_CodexLifecycleTailUpdatesTermination`, the partial-tail - tests, and the late-update/title tests in + tests, the late tool-result append test + (`TestIncrementalSync_CodexExecAppendRetainsEvents`), and the + late-update/title tests in `internal/sync/engine_integration_test.go` — safe Codex growth appends only - new rows while lifecycle metadata, incomplete records, title changes, and - retroactive updates preserve full-parse behavior. + new rows, late tool results update stored calls in place, and lifecycle + metadata, incomplete records, title changes, and other retroactive updates + preserve full-parse behavior. - `TestCountDuplicatePromptsAllocationGrowthStaysNearLinear` (`internal/signals/heuristics_test.go`) — session-quality analysis must not rebuild token sets for every pair of user prompts. @@ -79,10 +103,29 @@ with `cmd/benchgate`: skip work only; also self-asserts nothing is re-synced or bulk-rewritten). - `BenchmarkSyncPathsIncrementalAppend` — absorb one appended line into a 1,000-message session. -- `BenchmarkCodexIncrementalSyncReads` — a warm Codex cursor append plus the - remaining full-source fingerprint and committed-prefix hash reads. See +- `BenchmarkCodexCheckpointAppendResume` — the source-reading pipeline of a + checkpoint-resumed Codex append: checkpoint gate (stat + anchor digest + + fingerprint resume), seeded tail parse, committed-prefix resume hash, next + anchor digest, and next checkpoint assembly. Bounded by the anchor window + plus the tail — see [Background Sync Efficiency](background-sync-efficiency.md) for the cost-model boundary. +- `BenchmarkCodexLateToolOutputDebouncedBurst` — a checkpoint-resumed Codex + append stream with the signal debounce stretched, so only the first + iteration pays the O(history) recompute and the rest are amortized. It + guards the late-result update path's per-append cost; it is deliberately + NOT the quiet-session gate (see the `BenchmarkCodexQuietAppendSignals*` + trio below). +- `BenchmarkCodexQuietAppendSignals500` / `...5000` / `...15000` — the + non-amortized quiet-session append gate: every iteration appends a new + function_call plus the previous call's late output and pays the full + inline signal/secret maintenance (debounce disabled). The three sizes + prove per-append latency does not scale with stored history; each + self-asserts zero `GetAllMessages` calls in the timed loop. +- `BenchmarkCodexColdFullSync` — a fresh database and engine ingesting the + transcript from scratch, including the single-pass checkpoint capture. + Per-op cost deliberately exceeds the micro-benchmark band; it exists to + catch a regression that adds a source read pass to the cold pipeline. - `BenchmarkSyncAllColdArchive` — first-sync ingest throughput through the default per-session write path. - `BenchmarkResyncBulkIngest` — the same archive through the resync bulk-write @@ -204,6 +247,23 @@ prefix reconstruction with an exact warm cursor. It is diagnostic rather than PR-gated: `BENCH_GATE_PACKAGES` currently contains `./internal/sync`, `./internal/db`, `./internal/secrets`, and `./internal/signals`. +### 3. Macro ratio gate (run manually, build tag `macrobench`) + +The 10MB-vs-1GB same-append p95 ratio (`< 2x`) is not a CI benchmark: the +1GB fixture takes minutes per run. `internal/sync/codex_macro_bench_test.go` +is excluded from the PR gate by the `macrobench` build tag: + +```bash +cd internal/sync +go test -tags 'fts5,macrobench' -run '^$' \ + -bench 'BenchmarkMacroCodexQuietAppend' -benchmem -count=6 -benchtime=5x +``` + +The gate: the p95 `sec/op` of `BenchmarkMacroCodexQuietAppend1GB` must stay +within 2x of `BenchmarkMacroCodexQuietAppend10MB` for the same +quiet-append shape (call + late output, full inline signal/secret +maintenance). + ## Adding a benchmark to the gate Every benchmark in a gated package is gated — there is no per-name allowlist to diff --git a/internal/db/checkpoint.go b/internal/db/checkpoint.go new file mode 100644 index 000000000..a7477b82f --- /dev/null +++ b/internal/db/checkpoint.go @@ -0,0 +1,228 @@ +package db + +import ( + "database/sql" + "errors" + "fmt" + "time" +) + +// ParserCheckpointVersion is the codec version for parser_checkpoints rows. +// Bump it when the cursor, hash-state, or anchor encoding changes; a version +// mismatch makes the engine fall back to a full parse instead of resuming. +const ParserCheckpointVersion = 1 + +// ParserCheckpoint is the machine-local continuation metadata needed to +// resume parsing an append-only transcript without re-reading the committed +// prefix: the committed byte offset, the digest of the tail anchor window +// proving the boundary region is unchanged, and the committed-prefix hash. +// The heavier payload (cursor and resumable hash state) lives in +// ParserCheckpointBlobs so the stat-only freshness gate never reads it. +type ParserCheckpoint struct { + SessionID string + Agent string + FilePath string + FileInode uint64 + FileDevice uint64 + FileMTime int64 + FileChangeTime int64 + Offset int64 + TailAnchorDigest string + Hash string + NextOrdinal int + Version int + UpdatedAt string +} + +// ParserCheckpointBlobs is the lazy-loaded checkpoint payload. +type ParserCheckpointBlobs struct { + SessionID string + Cursor []byte + HashState []byte +} + +// GetParserCheckpoint loads the checkpoint metadata for a session. ok=false +// means no row exists (legacy session or never checkpointed), which is not +// an error. The blob payload is deliberately not loaded here. +func (db *DB) GetParserCheckpoint( + sessionID string, +) (*ParserCheckpoint, bool, error) { + var cp ParserCheckpoint + var inode, device, nextOrdinal int64 + err := db.getReader().QueryRow( + `SELECT agent, file_path, file_inode, file_device, file_mtime, + file_change_time, + offset, tail_anchor_digest, hash, + next_ordinal, checkpoint_version, updated_at + FROM parser_checkpoints + WHERE session_id = ?`, + sessionID, + ).Scan( + &cp.Agent, &cp.FilePath, &inode, &device, &cp.FileMTime, + &cp.FileChangeTime, + &cp.Offset, &cp.TailAnchorDigest, &cp.Hash, + &nextOrdinal, &cp.Version, &cp.UpdatedAt, + ) + if errors.Is(err, sql.ErrNoRows) { + return nil, false, nil + } + if err != nil { + return nil, false, fmt.Errorf( + "reading parser checkpoint %s: %w", sessionID, err, + ) + } + cp.SessionID = sessionID + cp.FileInode = uint64(inode) + cp.FileDevice = uint64(device) + cp.NextOrdinal = int(nextOrdinal) + return &cp, true, nil +} + +// GetParserCheckpointBlobs loads the lazy checkpoint payload. ok=false when +// no blobs row exists. +func (db *DB) GetParserCheckpointBlobs( + sessionID string, +) (ParserCheckpointBlobs, bool, error) { + var b ParserCheckpointBlobs + err := db.getReader().QueryRow( + `SELECT cursor, hash_state + FROM parser_checkpoint_blobs + WHERE session_id = ?`, + sessionID, + ).Scan(&b.Cursor, &b.HashState) + if errors.Is(err, sql.ErrNoRows) { + return ParserCheckpointBlobs{}, false, nil + } + if err != nil { + return ParserCheckpointBlobs{}, false, fmt.Errorf( + "reading parser checkpoint blobs %s: %w", sessionID, err, + ) + } + b.SessionID = sessionID + return b, true, nil +} + +// DeleteParserCheckpoint removes a session's checkpoint rows. Used when a +// source is replaced or deleted so a stale checkpoint can never be resumed. +func (db *DB) DeleteParserCheckpoint(sessionID string) error { + db.mu.Lock() + defer db.mu.Unlock() + tx, err := db.getWriter().Begin() + if err != nil { + return fmt.Errorf("beginning checkpoint delete tx: %w", err) + } + defer func() { _ = tx.Rollback() }() + if err := deleteParserCheckpointTx(tx, sessionID); err != nil { + return err + } + return tx.Commit() +} + +func deleteParserCheckpointTx(tx *sql.Tx, sessionID string) error { + if _, err := tx.Exec( + `DELETE FROM parser_checkpoints WHERE session_id = ?`, + sessionID, + ); err != nil { + return fmt.Errorf("deleting parser checkpoint %s: %w", sessionID, err) + } + if _, err := tx.Exec( + `DELETE FROM parser_checkpoint_blobs WHERE session_id = ?`, + sessionID, + ); err != nil { + return fmt.Errorf( + "deleting parser checkpoint blobs %s: %w", sessionID, err, + ) + } + return nil +} + +// UpsertParserCheckpoint stores or replaces a session checkpoint (metadata +// plus blobs) atomically. The full parse path calls this after its session +// rows commit; the incremental path writes the checkpoint inside the same +// transaction as the delta (see WriteSessionIncremental). +func (db *DB) UpsertParserCheckpoint( + cp ParserCheckpoint, blobs ParserCheckpointBlobs, +) error { + if cp.Version == 0 { + cp.Version = ParserCheckpointVersion + } + if cp.UpdatedAt == "" { + cp.UpdatedAt = time.Now().UTC().Format(time.RFC3339) + } + blobs.SessionID = cp.SessionID + db.mu.Lock() + defer db.mu.Unlock() + tx, err := db.getWriter().Begin() + if err != nil { + return fmt.Errorf("beginning checkpoint upsert tx: %w", err) + } + defer func() { _ = tx.Rollback() }() + if err := upsertParserCheckpointExec(tx, cp, blobs); err != nil { + return err + } + return tx.Commit() +} + +func upsertParserCheckpointTx( + tx *sql.Tx, cp ParserCheckpoint, blobs ParserCheckpointBlobs, +) error { + return upsertParserCheckpointExec(tx, cp, blobs) +} + +type sqlExecer interface { + Exec(query string, args ...any) (sql.Result, error) +} + +func upsertParserCheckpointExec( + exec sqlExecer, cp ParserCheckpoint, blobs ParserCheckpointBlobs, +) error { + if cp.Version == 0 { + cp.Version = ParserCheckpointVersion + } + if cp.UpdatedAt == "" { + cp.UpdatedAt = time.Now().UTC().Format(time.RFC3339) + } + if _, err := exec.Exec( + `INSERT INTO parser_checkpoints ( + session_id, agent, file_path, file_inode, file_device, file_mtime, + file_change_time, + offset, tail_anchor_digest, hash, + next_ordinal, checkpoint_version, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(session_id) DO UPDATE SET + agent = excluded.agent, + file_path = excluded.file_path, + file_inode = excluded.file_inode, + file_device = excluded.file_device, + file_mtime = excluded.file_mtime, + file_change_time = excluded.file_change_time, + offset = excluded.offset, + tail_anchor_digest = excluded.tail_anchor_digest, + hash = excluded.hash, + next_ordinal = excluded.next_ordinal, + checkpoint_version = excluded.checkpoint_version, + updated_at = excluded.updated_at`, + cp.SessionID, cp.Agent, cp.FilePath, + int64(cp.FileInode), int64(cp.FileDevice), cp.FileMTime, + cp.FileChangeTime, + cp.Offset, cp.TailAnchorDigest, cp.Hash, + cp.NextOrdinal, cp.Version, cp.UpdatedAt, + ); err != nil { + return fmt.Errorf( + "upserting parser checkpoint %s: %w", cp.SessionID, err, + ) + } + if _, err := exec.Exec( + `INSERT INTO parser_checkpoint_blobs (session_id, cursor, hash_state) + VALUES (?, ?, ?) + ON CONFLICT(session_id) DO UPDATE SET + cursor = excluded.cursor, + hash_state = excluded.hash_state`, + cp.SessionID, blobs.Cursor, blobs.HashState, + ); err != nil { + return fmt.Errorf( + "upserting parser checkpoint blobs %s: %w", cp.SessionID, err, + ) + } + return nil +} diff --git a/internal/db/checkpoint_test.go b/internal/db/checkpoint_test.go new file mode 100644 index 000000000..06fdc0088 --- /dev/null +++ b/internal/db/checkpoint_test.go @@ -0,0 +1,222 @@ +package db + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParserCheckpointRoundTrip(t *testing.T) { + d := testDB(t) + cp := ParserCheckpoint{ + SessionID: "codex:019eb791-cf7d-75c1-8439-9ed74c122b02", + Agent: "codex", + FilePath: "/sessions/rollout-x.jsonl", + FileInode: 42, + FileDevice: 7, + FileMTime: 1234567890, + Offset: 4096, + TailAnchorDigest: "anchor-digest", + Hash: "deadbeef", + NextOrdinal: 10, + Version: ParserCheckpointVersion, + } + blobs := ParserCheckpointBlobs{ + SessionID: cp.SessionID, + Cursor: []byte("cursor-bytes"), + HashState: []byte("hash-state"), + } + require.NoError(t, d.UpsertParserCheckpoint(cp, blobs)) + + got, ok, err := d.GetParserCheckpoint(cp.SessionID) + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, cp.SessionID, got.SessionID) + assert.Equal(t, cp.Agent, got.Agent) + assert.Equal(t, cp.FilePath, got.FilePath) + assert.Equal(t, cp.FileInode, got.FileInode) + assert.Equal(t, cp.FileDevice, got.FileDevice) + assert.Equal(t, cp.FileMTime, got.FileMTime) + assert.Equal(t, cp.Offset, got.Offset) + assert.Equal(t, cp.TailAnchorDigest, got.TailAnchorDigest) + assert.Equal(t, cp.Hash, got.Hash) + assert.Equal(t, cp.NextOrdinal, got.NextOrdinal) + assert.Equal(t, cp.Version, got.Version) + assert.NotEmpty(t, got.UpdatedAt) + + gotBlobs, ok, err := d.GetParserCheckpointBlobs(cp.SessionID) + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, blobs.Cursor, gotBlobs.Cursor) + assert.Equal(t, blobs.HashState, gotBlobs.HashState) + + require.NoError(t, d.DeleteParserCheckpoint(cp.SessionID)) + _, ok, err = d.GetParserCheckpoint(cp.SessionID) + require.NoError(t, err) + assert.False(t, ok) + _, ok, err = d.GetParserCheckpointBlobs(cp.SessionID) + require.NoError(t, err) + assert.False(t, ok, "delete must remove the blob payload too") +} + +func TestReplaceSessionContentWithCheckpointUsesPrefixedSessionID(t *testing.T) { + d := testDB(t) + const storedID = "host:codex:native" + insertSession(t, d, storedID, "proj") + msgs := []Message{{ + SessionID: storedID, + Ordinal: 0, + Role: "assistant", + Content: "running", + HasToolUse: true, + ToolCalls: []ToolCall{{ + SessionID: storedID, + ToolName: "exec_command", + Category: "Bash", + ToolUseID: "call_1", + }}, + }} + cp := &ParserCheckpoint{ + SessionID: "codex:native", + Agent: "codex", + FilePath: "/sessions/rollout.jsonl", + FileInode: 1, + FileDevice: 1, + FileMTime: 1, + FileChangeTime: 1, + Offset: 8, + TailAnchorDigest: "anchor", + Hash: "hash", + NextOrdinal: 0, + Version: ParserCheckpointVersion, + } + blobs := &ParserCheckpointBlobs{ + SessionID: "codex:native", + Cursor: []byte("cursor"), + HashState: []byte("state"), + } + err := d.ReplaceSessionContentWithCheckpoint( + storedID, msgs, SessionSignalUpdate{}, nil, cp, blobs, + ) + require.NoError(t, err) + + var nativeCount, prefixedCount int + require.NoError(t, d.Reader().QueryRow( + `SELECT COUNT(*) FROM parser_checkpoints WHERE session_id = ?`, + "codex:native", + ).Scan(&nativeCount)) + require.NoError(t, d.Reader().QueryRow( + `SELECT COUNT(*) FROM parser_checkpoints WHERE session_id = ?`, + storedID, + ).Scan(&prefixedCount)) + assert.Zero(t, nativeCount, + "the checkpoint must not be stored under the parser-native id") + assert.Equal(t, 1, prefixedCount, + "the checkpoint must be stored under the rewritten session id") + + var blobCount int + require.NoError(t, d.Reader().QueryRow( + `SELECT COUNT(*) FROM parser_checkpoint_blobs WHERE session_id = ?`, + storedID, + ).Scan(&blobCount)) + assert.Equal(t, 1, blobCount) +} + +func TestWriteSessionIncrementalPersistsCheckpointInSameTx(t *testing.T) { + d := testDB(t) + insertSession(t, d, "s1", "proj") + insertMessages(t, d, Message{ + SessionID: "s1", + Ordinal: 0, + Role: "assistant", + HasToolUse: true, + ToolCalls: []ToolCall{{ + SessionID: "s1", + ToolName: "exec_command", + Category: "Bash", + ToolUseID: "call_cmd", + }}, + }) + cp := ParserCheckpoint{ + SessionID: "s1", + Agent: "codex", + FilePath: "/sessions/rollout-s1.jsonl", + FileInode: 1, + FileDevice: 2, + FileMTime: 100, + Offset: 1024, + TailAnchorDigest: "anchor-a", + Hash: "abc", + NextOrdinal: 2, + Version: ParserCheckpointVersion, + } + blobs := ParserCheckpointBlobs{ + SessionID: "s1", + Cursor: []byte("c"), + HashState: []byte("h"), + } + _, werr := d.WriteSessionIncremental("s1", nil, IncrementalSessionUpdate{ + MsgCount: 1, + NextOrdinal: 1, + Checkpoint: &cp, + CheckpointBlobs: &blobs, + }) + require.NoError(t, werr) + + got, ok, err := d.GetParserCheckpoint("s1") + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, int64(1024), got.Offset) + gotBlobs, ok, err := d.GetParserCheckpointBlobs("s1") + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, []byte("c"), gotBlobs.Cursor) + + // A delta without a checkpoint must not disturb the stored one. + _, werr = d.WriteSessionIncremental("s1", nil, IncrementalSessionUpdate{ + MsgCount: 1, + NextOrdinal: 2, + }) + require.NoError(t, werr) + got, ok, err = d.GetParserCheckpoint("s1") + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, int64(1024), got.Offset) +} + +func TestParserCheckpointRollsBackWithTransaction(t *testing.T) { + d := testDB(t) + tx, err := d.getWriter().Begin() + require.NoError(t, err) + require.NoError(t, upsertParserCheckpointTx(tx, ParserCheckpoint{ + SessionID: "s-rollback", + Agent: "codex", + FilePath: "/sessions/rollout-s-rollback.jsonl", + Offset: 512, + TailAnchorDigest: "anchor-a", + Hash: "h", + NextOrdinal: 1, + Version: ParserCheckpointVersion, + }, ParserCheckpointBlobs{ + SessionID: "s-rollback", + Cursor: []byte("c"), + HashState: []byte("h"), + })) + require.NoError(t, tx.Rollback()) + + _, ok, err := d.GetParserCheckpoint("s-rollback") + require.NoError(t, err) + assert.False(t, ok, + "an aborted transaction must not leave a checkpoint behind") + _, ok, err = d.GetParserCheckpointBlobs("s-rollback") + require.NoError(t, err) + assert.False(t, ok, + "an aborted transaction must not leave checkpoint blobs behind") +} + +// TestParserCheckpointSchemaMigratesFromPreSplitShape simulates an archive +// written by the pre-split schema (raw tail_anchor/cursor/hash_state +// columns): reopening must add the digest column, drop the dead columns, +// and leave a version-1 row readable (the engine rebuilds it +// authoritatively) while new version-2 upserts succeed. diff --git a/internal/db/db.go b/internal/db/db.go index e73734dda..9a323d6d2 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -684,6 +684,17 @@ type DB struct { vectorMu sync.RWMutex vectorSearcher VectorSearcher recallSearcher RecallVectorSearcher + + // messagesLoadCount counts GetAllMessages calls. Tests use it to gate + // the incremental signal path: a maintained delta must not load + // session history. + messagesLoadCount atomic.Int64 +} + +// MessagesLoadCount returns the total number of GetAllMessages calls the +// database has served. Monotonic; used by the incremental-path gates. +func (db *DB) MessagesLoadCount() int64 { + return db.messagesLoadCount.Load() } // Reader exposes guarded read-only query operations. It intentionally does @@ -2726,6 +2737,15 @@ func (db *DB) migrateColumns(ctx context.Context) error { "creating idx_tool_calls_file_path: %w", err, ) } + if _, err := w.Exec( + `CREATE INDEX IF NOT EXISTS idx_tool_calls_session_tool_use + ON tool_calls(session_id, tool_use_id) + WHERE tool_use_id IS NOT NULL`, + ); err != nil { + return fmt.Errorf( + "creating idx_tool_calls_session_tool_use: %w", err, + ) + } if _, err := w.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_sessions_termination_status diff --git a/internal/db/db_test.go b/internal/db/db_test.go index a750656bb..263b219d5 100644 --- a/internal/db/db_test.go +++ b/internal/db/db_test.go @@ -3978,9 +3978,8 @@ func TestWriteSessionIncrementalBlocksLinkedResultContent(t *testing.T) { }}, BlockedResultCategories: map[string]bool{"Task": true}, } - require.NoError(t, d.WriteSessionIncremental( - "s1", nil, update, - ), "incremental write") + _, werr := d.WriteSessionIncremental("s1", nil, update) + require.NoError(t, werr, "incremental write") var subagent, content string var contentLen int @@ -4000,8 +3999,8 @@ func TestWriteSessionIncrementalBlocksLinkedResultContent(t *testing.T) { require.NotNil(t, after.TranscriptRevision) assert.Equal(t, "2", *after.TranscriptRevision) - require.NoError(t, d.WriteSessionIncremental("s1", nil, update), - "idempotent incremental write") + _, werr = d.WriteSessionIncremental("s1", nil, update) + require.NoError(t, werr, "idempotent incremental write") idempotent, err := d.GetSession(context.Background(), "s1") require.NoError(t, err) require.NotNil(t, idempotent) @@ -4050,7 +4049,8 @@ func TestWriteSessionIncrementalResultOnlyLink(t *testing.T) { HasResult: true, }}, } - require.NoError(t, d.WriteSessionIncremental("s1", nil, update)) + _, werr := d.WriteSessionIncremental("s1", nil, update) + require.NoError(t, werr) var subagent, content string var contentLen int @@ -4075,6 +4075,613 @@ func TestWriteSessionIncrementalResultOnlyLink(t *testing.T) { "result-only link must not disturb other calls") } +func TestWriteSessionIncrementalToolCallResultUpdate(t *testing.T) { + d := testDB(t) + insertSession(t, d, "s1", "proj") + insertMessages(t, d, Message{ + SessionID: "s1", + Ordinal: 0, + Role: "assistant", + HasToolUse: true, + ToolCalls: []ToolCall{{ + SessionID: "s1", + ToolName: "exec_command", + Category: "Bash", + ToolUseID: "call_cmd", + }}, + }) + + update := IncrementalSessionUpdate{ + MsgCount: 1, + NextOrdinal: 1, + ToolCallResultUpdates: []ToolCallResultUpdate{{ + ToolUseID: "call_cmd", + Position: ToolCallPosition{MessageOrdinal: 0, CallIndex: 0}, + Events: []ToolResultEvent{{ + ToolUseID: "call_cmd", + Source: "function_call_output", + Content: "command finished", + ContentLength: len("command finished"), + Timestamp: "2026-08-02T09:00:00Z", + }}, + }}, + } + _, werr := d.WriteSessionIncremental("s1", nil, update) + require.NoError(t, werr) + + var result string + var resultLen int + require.NoError(t, d.Reader().QueryRow(` + SELECT COALESCE(result_content, ''), result_content_length + FROM tool_calls + WHERE session_id = ? AND tool_use_id = ?`, + "s1", "call_cmd", + ).Scan(&result, &resultLen)) + assert.Equal(t, "command finished", result) + assert.Equal(t, len("command finished"), resultLen) + + var source, content, timestamp string + var eventIndex, eventCount int + require.NoError(t, d.Reader().QueryRow(` + SELECT source, content, COALESCE(timestamp, ''), event_index + FROM tool_result_events + WHERE session_id = ? AND tool_use_id = ?`, + "s1", "call_cmd", + ).Scan(&source, &content, ×tamp, &eventIndex)) + assert.Equal(t, "function_call_output", source) + assert.Equal(t, "command finished", content) + assert.Equal(t, "2026-08-02T09:00:00Z", timestamp) + assert.Zero(t, eventIndex) + + _, werr = d.WriteSessionIncremental("s1", nil, update) + require.NoError(t, werr, + "replaying an identical output must be idempotent") + require.NoError(t, d.Reader().QueryRow(` + SELECT COUNT(*) FROM tool_result_events + WHERE session_id = ? AND tool_use_id = ?`, + "s1", "call_cmd", + ).Scan(&eventCount)) + assert.Equal(t, 1, eventCount) + + sess, err := d.GetSession(context.Background(), "s1") + require.NoError(t, err) + require.NotNil(t, sess) + require.NotNil(t, sess.TranscriptRevision) + assert.Equal(t, "2", *sess.TranscriptRevision, + "idempotent replay must not bump the transcript revision") +} + +func TestWriteSessionIncrementalTargetsDuplicateCallIDOccurrence(t *testing.T) { + d := testDB(t) + insertSession(t, d, "s1", "proj") + for ordinal := range 2 { + insertMessages(t, d, Message{ + SessionID: "s1", + Ordinal: ordinal, + Role: "assistant", + HasToolUse: true, + ToolCalls: []ToolCall{{ + SessionID: "s1", + ToolName: "exec_command", + Category: "Bash", + ToolUseID: "reused-call", + }}, + }) + } + + write := func(position ToolCallPosition, content string) { + t.Helper() + _, err := d.WriteSessionIncremental("s1", nil, IncrementalSessionUpdate{ + MsgCount: 2, + NextOrdinal: 2, + ToolCallResultUpdates: []ToolCallResultUpdate{{ + ToolUseID: "reused-call", + Position: position, + Events: []ToolResultEvent{{ + ToolUseID: "reused-call", + Source: "function_call_output", + Content: content, + ContentLength: len(content), + }}, + }}, + }) + require.NoError(t, err) + } + + write(ToolCallPosition{MessageOrdinal: 1, CallIndex: 0}, "second") + + rows, err := d.Reader().Query(` + SELECT m.ordinal, COALESCE(tc.result_content, '') + FROM tool_calls tc JOIN messages m ON m.id = tc.message_id + WHERE tc.session_id = ? AND tc.tool_use_id = ? + ORDER BY m.ordinal, tc.call_index`, "s1", "reused-call") + require.NoError(t, err) + defer rows.Close() + got := make(map[int]string) + for rows.Next() { + var ordinal int + var content string + require.NoError(t, rows.Scan(&ordinal, &content)) + got[ordinal] = content + } + require.NoError(t, rows.Err()) + assert.Equal(t, map[int]string{0: "", 1: "second"}, got) + + var eventCount int + require.NoError(t, d.Reader().QueryRow(` + SELECT COUNT(*) FROM tool_result_events + WHERE session_id = ? AND tool_call_message_ordinal = ? + AND call_index = ?`, "s1", 1, 0).Scan(&eventCount)) + assert.Equal(t, 1, eventCount) + + write(ToolCallPosition{MessageOrdinal: 0, CallIndex: 0}, "first") + var stateOccurrences int + require.NoError(t, d.Reader().QueryRow(` + SELECT COUNT(DISTINCT printf('%d/%d', message_ordinal, call_index)) + FROM tool_call_occurrence_agent_state + WHERE session_id = ?`, "s1").Scan(&stateOccurrences)) + assert.Equal(t, 2, stateOccurrences, + "reused provider IDs must keep independent per-occurrence state") +} + +func TestWriteSessionIncrementalLateResultAndCommittedUsageAreAtomic(t *testing.T) { + d := testDB(t) + insertSession(t, d, "s1", "proj") + insertMessages(t, d, Message{ + SessionID: "s1", + Ordinal: 0, + Role: "assistant", + HasToolUse: true, + ToolCalls: []ToolCall{{ + SessionID: "s1", + ToolName: "exec_command", + Category: "Bash", + ToolUseID: "call_cmd", + }}, + }) + + update := IncrementalSessionUpdate{ + MsgCount: 1, + NextOrdinal: 1, + ToolCallResultUpdates: []ToolCallResultUpdate{{ + ToolUseID: "call_cmd", + Position: ToolCallPosition{MessageOrdinal: 0, CallIndex: 0}, + Events: []ToolResultEvent{{ + ToolUseID: "call_cmd", + Source: "function_call_output", + Content: "command finished", + ContentLength: len("command finished"), + }}, + }}, + MessageTokenUsageUpdates: []MessageTokenUsageUpdate{{ + Ordinal: 0, + TokenUsage: jsontext.Value(`{"input_tokens":100000,"output_tokens":250}`), + ContextTokens: 100000, + OutputTokens: 250, + HasContextTokens: true, + HasOutputTokens: true, + }}, + } + _, werr := d.WriteSessionIncremental("s1", nil, update) + require.NoError(t, werr) + + var tokenUsage, result string + var contextTokens, outputTokens int + require.NoError(t, d.Reader().QueryRow(` + SELECT token_usage, context_tokens, output_tokens + FROM messages WHERE session_id = ? AND ordinal = ?`, + "s1", 0, + ).Scan(&tokenUsage, &contextTokens, &outputTokens)) + assert.JSONEq(t, `{"input_tokens":100000,"output_tokens":250}`, tokenUsage) + assert.Equal(t, 100000, contextTokens) + assert.Equal(t, 250, outputTokens) + require.NoError(t, d.Reader().QueryRow(` + SELECT COALESCE(result_content, '') FROM tool_calls + WHERE session_id = ? AND tool_use_id = ?`, + "s1", "call_cmd", + ).Scan(&result)) + assert.Equal(t, "command finished", result) + + sess, err := d.GetSession(context.Background(), "s1") + require.NoError(t, err) + require.NotNil(t, sess) + require.NotNil(t, sess.TranscriptRevision) + assert.Equal(t, "2", *sess.TranscriptRevision) + + _, werr = d.WriteSessionIncremental("s1", nil, update) + require.NoError(t, werr, "identical late-result usage replay must be idempotent") + sess, err = d.GetSession(context.Background(), "s1") + require.NoError(t, err) + require.NotNil(t, sess) + require.NotNil(t, sess.TranscriptRevision) + assert.Equal(t, "2", *sess.TranscriptRevision) +} + +func TestWriteSessionIncrementalResultEventIndexesAreMonotonic(t *testing.T) { + d := testDB(t) + insertSession(t, d, "s1", "proj") + insertMessages(t, d, Message{ + SessionID: "s1", + Ordinal: 0, + Role: "assistant", + HasToolUse: true, + ToolCalls: []ToolCall{{ + SessionID: "s1", + ToolName: "exec_command", + Category: "Bash", + ToolUseID: "call_cmd", + }}, + }) + + for _, content := range []string{"first output", "second output"} { + _, werr := d.WriteSessionIncremental("s1", nil, IncrementalSessionUpdate{ + MsgCount: 1, + NextOrdinal: 1, + ToolCallResultUpdates: []ToolCallResultUpdate{{ + ToolUseID: "call_cmd", + Position: ToolCallPosition{MessageOrdinal: 0, CallIndex: 0}, + Events: []ToolResultEvent{{ + ToolUseID: "call_cmd", + Source: "function_call_output", + Content: content, + ContentLength: len(content), + }}, + }}, + }) + require.NoError(t, werr) + } + + var secondRows int + require.NoError(t, d.Reader().QueryRow(` + SELECT COUNT(*) FROM tool_result_events + WHERE session_id = ? AND tool_use_id = ? AND event_index = 1`, + "s1", "call_cmd", + ).Scan(&secondRows)) + assert.Equal(t, 1, secondRows, + "each late result event must get the next per-call event index") + + var latestIndex int + require.NoError(t, d.Reader().QueryRow(` + SELECT latest_event_index FROM tool_call_occurrence_agent_state + WHERE session_id = ? AND message_ordinal = ? AND call_index = ? + AND agent_id = ''`, + "s1", 0, 0, + ).Scan(&latestIndex)) + assert.Equal(t, 1, latestIndex, + "the agent state must point at the newest event") + + var result string + require.NoError(t, d.Reader().QueryRow(` + SELECT COALESCE(result_content, '') FROM tool_calls + WHERE session_id = ? AND tool_use_id = ?`, + "s1", "call_cmd", + ).Scan(&result)) + assert.Equal(t, "second output", result) +} + +func TestWriteSessionIncrementalBlockedResultKeepsLength(t *testing.T) { + d := testDB(t) + insertSession(t, d, "s1", "proj") + insertMessages(t, d, Message{ + SessionID: "s1", + Ordinal: 0, + Role: "assistant", + HasToolUse: true, + ToolCalls: []ToolCall{{ + SessionID: "s1", + ToolName: "exec_command", + Category: "Bash", + ToolUseID: "call_cmd", + }}, + }) + + _, werr := d.WriteSessionIncremental("s1", nil, IncrementalSessionUpdate{ + MsgCount: 1, + NextOrdinal: 1, + BlockedResultCategories: map[string]bool{"Bash": true}, + ToolCallResultUpdates: []ToolCallResultUpdate{{ + ToolUseID: "call_cmd", + Position: ToolCallPosition{MessageOrdinal: 0, CallIndex: 0}, + Events: []ToolResultEvent{ + { + AgentID: "a", + ToolUseID: "call_cmd", + Source: "function_call_output", + Content: "x", + ContentLength: 1, + }, + { + AgentID: "b", + ToolUseID: "call_cmd", + Source: "function_call_output", + Content: "yy", + ContentLength: 2, + }, + }, + }}, + }) + require.NoError(t, werr) + + var storedContent string + var storedLen int + require.NoError(t, d.Reader().QueryRow(` + SELECT COALESCE(result_content, ''), result_content_length + FROM tool_calls + WHERE session_id = ? AND tool_use_id = ?`, + "s1", "call_cmd", + ).Scan(&storedContent, &storedLen)) + assert.Empty(t, storedContent, "blocked result content stays blank") + assert.Equal(t, 11, storedLen, + "blocked result length keeps agent labels and separators: "+ + "\"a:\\nx\\n\\nb:\\nyy\"") +} + +// TestWriteSessionIncrementalBlockedResultKeepsRawLengthWithControlChars +// pins the length invariant blocked categories keep on the full-parse and +// staged paths: sanitizing (which strips control/NUL bytes) must never run +// against content this path is about to blank, or the stored length would +// come up short by however many bytes sanitize removed instead of +// reflecting the original raw output size. +func TestWriteSessionIncrementalBlockedResultKeepsRawLengthWithControlChars(t *testing.T) { + d := testDB(t) + insertSession(t, d, "s1", "proj") + insertMessages(t, d, Message{ + SessionID: "s1", + Ordinal: 0, + Role: "assistant", + HasToolUse: true, + ToolCalls: []ToolCall{{ + SessionID: "s1", + ToolName: "exec_command", + Category: "Bash", + ToolUseID: "call_cmd", + }}, + }) + + const raw = "before\x00after\x1b[31mred\u0085done" + _, werr := d.WriteSessionIncremental("s1", nil, IncrementalSessionUpdate{ + MsgCount: 1, + NextOrdinal: 1, + BlockedResultCategories: map[string]bool{"Bash": true}, + ToolCallResultUpdates: []ToolCallResultUpdate{{ + ToolUseID: "call_cmd", + Position: ToolCallPosition{MessageOrdinal: 0, CallIndex: 0}, + Events: []ToolResultEvent{{ + ToolUseID: "call_cmd", + Source: "function_call_output", + Content: raw, + ContentLength: len(raw), + }}, + }}, + }) + require.NoError(t, werr) + + var storedContent string + var storedLen int + require.NoError(t, d.Reader().QueryRow(` + SELECT COALESCE(result_content, ''), result_content_length + FROM tool_calls + WHERE session_id = ? AND tool_use_id = ?`, + "s1", "call_cmd", + ).Scan(&storedContent, &storedLen)) + assert.Empty(t, storedContent, "blocked result content stays blank") + assert.Equal(t, len(raw), storedLen, + "blocked result length must be the original raw byte count, "+ + "not the sanitized (control-stripped) count") + + var eventLen int + require.NoError(t, d.Reader().QueryRow(` + SELECT content_length FROM tool_result_events + WHERE session_id = ? AND tool_use_id = ?`, + "s1", "call_cmd", + ).Scan(&eventLen)) + assert.Equal(t, len(raw), eventLen, + "the stored event's content_length must also be the raw byte count") +} + +// TestWriteSessionIncrementalBlockedResultsDedupByLengthAcrossBatches pins +// the blocked-category equivalence rule for late results arriving in +// separate incremental writes. Every stored blocked row has an empty +// content column, so equivalence must be decided by original length: two +// outputs of different length for the same agent and status are distinct +// events (the full parse keeps both), while a replay of the same length is +// the same event and must not be stored twice. +func TestWriteSessionIncrementalBlockedResultsDedupByLengthAcrossBatches(t *testing.T) { + d := testDB(t) + insertSession(t, d, "s1", "proj") + insertMessages(t, d, Message{ + SessionID: "s1", + Ordinal: 0, + Role: "assistant", + HasToolUse: true, + ToolCalls: []ToolCall{{ + SessionID: "s1", + ToolName: "exec_command", + Category: "Bash", + ToolUseID: "call_cmd", + }}, + }) + write := func(content string) { + t.Helper() + _, err := d.WriteSessionIncremental("s1", nil, IncrementalSessionUpdate{ + MsgCount: 1, + NextOrdinal: 1, + BlockedResultCategories: map[string]bool{"Bash": true}, + ToolCallResultUpdates: []ToolCallResultUpdate{{ + ToolUseID: "call_cmd", + Position: ToolCallPosition{MessageOrdinal: 0, CallIndex: 0}, + Events: []ToolResultEvent{{ + ToolUseID: "call_cmd", + Source: "function_call_output", + Content: content, + ContentLength: len(content), + }}, + }}, + }) + require.NoError(t, err) + } + countEvents := func() int { + t.Helper() + var n int + require.NoError(t, d.Reader().QueryRow(` + SELECT COUNT(*) FROM tool_result_events + WHERE session_id = ? AND tool_use_id = ?`, + "s1", "call_cmd", + ).Scan(&n)) + return n + } + + write("x") + write("yy") + assert.Equal(t, 2, countEvents(), + "blocked events of different length are distinct and must both be stored") + + write("zz") + assert.Equal(t, 2, countEvents(), + "a blocked replay of an already-stored length is the same event") + + var lengths []int + rows, err := d.Reader().Query(` + SELECT content_length FROM tool_result_events + WHERE session_id = ? AND tool_use_id = ? ORDER BY event_index`, + "s1", "call_cmd") + require.NoError(t, err) + defer rows.Close() + for rows.Next() { + var n int + require.NoError(t, rows.Scan(&n)) + lengths = append(lengths, n) + } + require.NoError(t, rows.Err()) + assert.Equal(t, []int{1, 2}, lengths) +} + +func TestBackfillToolCallAgentStateTracksFirstAndLatest(t *testing.T) { + d := testDB(t) + insertSession(t, d, "s1", "proj") + insertMessages(t, d, Message{ + SessionID: "s1", + Ordinal: 0, + Role: "assistant", + HasToolUse: true, + ToolCalls: []ToolCall{{ + SessionID: "s1", + ToolName: "exec_command", + Category: "Bash", + ToolUseID: "call_cmd", + }}, + }) + + tx, err := d.getWriter().BeginTx(context.Background(), nil) + require.NoError(t, err) + defer func() { _ = tx.Rollback() }() + for i, content := range []string{"old", "mid", "new"} { + _, err := tx.Exec(` + INSERT INTO tool_result_events + (session_id, tool_call_message_ordinal, call_index, tool_use_id, + agent_id, source, status, content, content_length, event_index) + VALUES (?, 0, 0, ?, 'agent-a', 'function_call_output', + 'completed', ?, ?, ?)`, + "s1", "call_cmd", content, len(content), i, + ) + require.NoError(t, err) + } + require.NoError(t, backfillToolCallAgentStateTx( + tx, "s1", ToolCallPosition{MessageOrdinal: 0, CallIndex: 0}, + )) + require.NoError(t, tx.Commit()) + + var first, latest int + require.NoError(t, d.Reader().QueryRow(` + SELECT first_event_index, latest_event_index + FROM tool_call_occurrence_agent_state + WHERE session_id = ? AND message_ordinal = ? AND call_index = ? + AND agent_id = 'agent-a'`, + "s1", 0, 0, + ).Scan(&first, &latest)) + assert.Equal(t, 0, first) + assert.Equal(t, 2, latest, + "backfill keeps the newest event index per agent") +} + +func TestReplaceSessionContentDiffClearsStaleAgentState(t *testing.T) { + d := testDB(t) + insertSession(t, d, "s1", "proj") + + msgs := func(content string, events []ToolResultEvent) []Message { + return []Message{{ + SessionID: "s1", + Ordinal: 0, + Role: "assistant", + Content: content, + HasToolUse: true, + ToolCalls: []ToolCall{{ + SessionID: "s1", + ToolName: "exec_command", + Category: "Bash", + ToolUseID: "call_1", + ResultEvents: events, + }}, + }} + } + first := msgs("running", []ToolResultEvent{ + { + ToolUseID: "call_1", + AgentID: "a", + Source: "function_call_output", + Content: "a1", + ContentLength: 2, + }, + { + ToolUseID: "call_1", + AgentID: "b", + Source: "function_call_output", + Content: "b1", + ContentLength: 2, + }, + }) + require.NoError(t, d.ReplaceSessionContent( + "s1", first, SessionSignalUpdate{}, nil, + )) + + var before int + require.NoError(t, d.Reader().QueryRow( + `SELECT COUNT(*) FROM tool_call_occurrence_agent_state + WHERE session_id = ? AND message_ordinal = ? AND call_index = ?`, + "s1", 0, 0, + ).Scan(&before)) + require.Equal(t, 2, before) + + second := msgs("done", []ToolResultEvent{{ + ToolUseID: "call_1", + AgentID: "a", + Source: "function_call_output", + Content: "a2", + ContentLength: 2, + }}) + require.NoError(t, d.ReplaceSessionContent( + "s1", second, SessionSignalUpdate{}, nil, + )) + + rows, err := d.Reader().Query( + `SELECT agent_id FROM tool_call_occurrence_agent_state + WHERE session_id = ? AND message_ordinal = ? AND call_index = ?`, + "s1", 0, 0, + ) + require.NoError(t, err) + agents := make(map[string]bool) + for rows.Next() { + var agent string + require.NoError(t, rows.Scan(&agent)) + agents[agent] = true + } + require.NoError(t, rows.Err()) + require.NoError(t, rows.Close()) + require.Len(t, agents, 1, + "a removed agent must not survive the diff rewrite") + require.True(t, agents["a"], "the surviving agent must be rebuilt") +} + // claude_linear_parse round-trips through upsert and the incremental // lookup, stays NULL for legacy rows, and survives an upsert that // carries no verdict. @@ -7871,7 +8478,7 @@ func TestLastWriteIncrementalMarker(t *testing.T) { assert.False(t, got.LastWriteIncremental, "full write path must leave last_write_incremental false") - requireNoError(t, d.WriteSessionIncremental( + _, werr := d.WriteSessionIncremental( "inc-marker", []Message{asstMsg("inc-marker", 1, "appended reply")}, IncrementalSessionUpdate{ @@ -7881,7 +8488,8 @@ func TestLastWriteIncrementalMarker(t *testing.T) { FileMtime: 200, NextOrdinal: 2, }, - ), "incremental write") + ) + requireNoError(t, werr, "incremental write") got, err = d.GetSessionFull(context.Background(), "inc-marker") requireNoError(t, err, "get after incremental write") @@ -7937,11 +8545,12 @@ func TestBatchWriteIncrementalMarkerReplaceMode(t *testing.T) { UserMessageCount: 1, } requireNoError(t, d.UpsertSession(base), "initial upsert") - requireNoError(t, d.WriteSessionIncremental( + _, werr := d.WriteSessionIncremental( "batch-marker", []Message{asstMsg("batch-marker", 1, "appended reply")}, IncrementalSessionUpdate{MsgCount: 2, UserMsgCount: 1, NextOrdinal: 2}, - ), "incremental write") + ) + requireNoError(t, werr, "incremental write") got, err := d.GetSessionFull(context.Background(), "batch-marker") requireNoError(t, err, "get after incremental write") @@ -8008,8 +8617,8 @@ func TestIncrementalWriteAtomicityRollsBackMessages(t *testing.T) { reflect.ValueOf(msgsToWrite), update, }) - if !results[0].IsNil() { - err = results[0].Interface().(error) + if !results[1].IsNil() { + err = results[1].Interface().(error) } else { err = nil } diff --git a/internal/db/messages.go b/internal/db/messages.go index 187eb588f..d1d32c227 100644 --- a/internal/db/messages.go +++ b/internal/db/messages.go @@ -94,6 +94,328 @@ type ToolResultEvent struct { EventIndex int `json:"event_index"` } +func formatToolCallPosition(position ToolCallPosition) string { + return fmt.Sprintf("%d/%d", position.MessageOrdinal, position.CallIndex) +} + +// SummarizeToolResultEvents derives the display result stored on a tool call +// from its chronological result events. Anonymous events use the latest +// content; agent-scoped events keep the latest content for each agent. +// summarizeToolCallFromStateTx assembles the result summary for one tool +// call from the per-call agent state table, mirroring +// SummarizeToolResultEvents over the call's stored events. Reading only +// the distinct agents keeps a late result update O(delta) instead of +// rescanning the call's event history. +func summarizeToolCallFromStateTx( + tx *sql.Tx, sessionID string, position ToolCallPosition, +) (string, error) { + rows, err := tx.Query( + `SELECT s.agent_id, e.content + FROM tool_call_occurrence_agent_state s + JOIN tool_result_events e + ON e.session_id = s.session_id + AND e.tool_call_message_ordinal = s.message_ordinal + AND e.call_index = s.call_index + AND e.event_index = s.latest_event_index + WHERE s.session_id = ? AND s.message_ordinal = ? AND s.call_index = ? + ORDER BY s.first_event_index`, + sessionID, position.MessageOrdinal, position.CallIndex, + ) + if err != nil { + return "", fmt.Errorf( + "loading agent state for %s/%s: %w", + sessionID, formatToolCallPosition(position), err, + ) + } + var orderedAgents []string + latest := make(map[string]string) + lastAnon := "" + allHaveAgentID := true + for rows.Next() { + var agentID, content string + if err := rows.Scan(&agentID, &content); err != nil { + _ = rows.Close() + return "", fmt.Errorf( + "scanning agent state for %s/%s: %w", + sessionID, formatToolCallPosition(position), err, + ) + } + if agentID == "" { + allHaveAgentID = false + lastAnon = content + continue + } + if _, ok := latest[agentID]; !ok { + orderedAgents = append(orderedAgents, agentID) + } + latest[agentID] = content + } + if err := rows.Err(); err != nil { + _ = rows.Close() + return "", fmt.Errorf( + "reading agent state for %s/%s: %w", + sessionID, formatToolCallPosition(position), err, + ) + } + if err := rows.Close(); err != nil { + return "", fmt.Errorf( + "closing agent state for %s/%s: %w", + sessionID, formatToolCallPosition(position), err, + ) + } + if len(latest) <= 1 { + if len(latest) == 1 { + var summary string + for _, content := range latest { + summary = content + } + if lastAnon != "" { + return summary + "\n\n" + lastAnon, nil + } + return summary, nil + } + return lastAnon, nil + } + parts := make([]string, 0, len(orderedAgents)) + for _, agentID := range orderedAgents { + parts = append(parts, agentID+":\n"+latest[agentID]) + } + if !allHaveAgentID && lastAnon != "" { + parts = append(parts, lastAnon) + } + return strings.Join(parts, "\n\n"), nil +} + +// summarizeToolCallLengthFromStateTx returns the byte length of the summary +// summarizeToolCallFromStateTx would assemble, without materializing the +// contents. Blocked-category rows store blank content but keep their +// content_length, so this reconstructs the original summary length — agent +// labels and separators included — instead of the zero the blanked display +// summary would report. It must stay structurally identical to +// summarizeToolCallFromStateTx: a change to either assembly rule must update +// both. +func summarizeToolCallLengthFromStateTx( + tx *sql.Tx, sessionID string, position ToolCallPosition, +) (int, error) { + rows, err := tx.Query( + `SELECT s.agent_id, e.content_length + FROM tool_call_occurrence_agent_state s + JOIN tool_result_events e + ON e.session_id = s.session_id + AND e.tool_call_message_ordinal = s.message_ordinal + AND e.call_index = s.call_index + AND e.event_index = s.latest_event_index + WHERE s.session_id = ? AND s.message_ordinal = ? AND s.call_index = ? + ORDER BY s.first_event_index`, + sessionID, position.MessageOrdinal, position.CallIndex, + ) + if err != nil { + return 0, fmt.Errorf( + "loading agent state lengths for %s/%s: %w", + sessionID, formatToolCallPosition(position), err, + ) + } + var orderedAgents []string + latest := make(map[string]int) + lastAnonLen := 0 + for rows.Next() { + var agentID string + var contentLength int + if err := rows.Scan(&agentID, &contentLength); err != nil { + _ = rows.Close() + return 0, fmt.Errorf( + "scanning agent state lengths for %s/%s: %w", + sessionID, formatToolCallPosition(position), err, + ) + } + if agentID == "" { + lastAnonLen = contentLength + continue + } + if _, ok := latest[agentID]; !ok { + orderedAgents = append(orderedAgents, agentID) + } + latest[agentID] = contentLength + } + if err := rows.Err(); err != nil { + _ = rows.Close() + return 0, fmt.Errorf( + "reading agent state lengths for %s/%s: %w", + sessionID, formatToolCallPosition(position), err, + ) + } + if err := rows.Close(); err != nil { + return 0, fmt.Errorf( + "closing agent state lengths for %s/%s: %w", + sessionID, formatToolCallPosition(position), err, + ) + } + switch { + case len(latest) == 0: + return lastAnonLen, nil + case len(latest) == 1: + var total int + for _, length := range latest { + total = length + } + if lastAnonLen > 0 { + total += 2 + lastAnonLen + } + return total, nil + default: + total := 0 + for _, agentID := range orderedAgents { + total += len(agentID) + 2 + latest[agentID] + } + total += 2 * (len(orderedAgents) - 1) + if lastAnonLen > 0 { + total += 2 + lastAnonLen + } + return total, nil + } +} + +// backfillToolCallAgentStateTx rebuilds the per-call agent state rows +// from the stored events. It runs once per call for sessions written +// before the state table existed or through the staged publish; after +// that, every late result update reads only the state table. +func backfillToolCallAgentStateTx( + tx *sql.Tx, sessionID string, position ToolCallPosition, +) error { + messageOrdinal := position.MessageOrdinal + callIndex := position.CallIndex + rows, err := tx.Query( + `SELECT COALESCE(agent_id, ''), content, content_length, event_index + FROM tool_result_events + WHERE session_id = ? AND tool_call_message_ordinal = ? + AND call_index = ? + ORDER BY event_index, id`, + sessionID, messageOrdinal, callIndex, + ) + if err != nil { + return fmt.Errorf( + "backfilling agent state for %s/%s: %w", + sessionID, formatToolCallPosition(position), err, + ) + } + type stateRow struct { + firstIndex int + latestIndex int + } + latest := make(map[string]stateRow) + for rows.Next() { + var agentID, content string + var length, eventIndex int + if err := rows.Scan( + &agentID, &content, &length, &eventIndex, + ); err != nil { + _ = rows.Close() + return fmt.Errorf( + "backfill scan for %s/%s: %w", + sessionID, formatToolCallPosition(position), err, + ) + } + if strings.TrimSpace(content) == "" && length == 0 { + continue + } + key := strings.TrimSpace(agentID) + entry, ok := latest[key] + if !ok { + entry.firstIndex = eventIndex + } + entry.latestIndex = eventIndex + latest[key] = entry + } + if err := rows.Err(); err != nil { + _ = rows.Close() + return fmt.Errorf( + "backfill read for %s/%s: %w", + sessionID, formatToolCallPosition(position), err, + ) + } + if err := rows.Close(); err != nil { + return fmt.Errorf( + "backfill close for %s/%s: %w", + sessionID, formatToolCallPosition(position), err, + ) + } + args := make([]any, 0, len(latest)*6) + for key, entry := range latest { + args = append(args, + sessionID, position.MessageOrdinal, position.CallIndex, key, + entry.firstIndex, entry.latestIndex, + ) + } + if len(args) == 0 { + return nil + } + if _, err := tx.Exec( + `INSERT INTO tool_call_occurrence_agent_state + (session_id, message_ordinal, call_index, agent_id, + first_event_index, latest_event_index) + VALUES `+multiRowPlaceholders(len(args)/6, 6)+` + ON CONFLICT(session_id, message_ordinal, call_index, agent_id) + DO UPDATE SET latest_event_index = excluded.latest_event_index`, + args..., + ); err != nil { + return fmt.Errorf( + "backfilling tool_call_agent_state (%d rows): %w", + len(args)/6, err, + ) + } + return nil +} + +func SummarizeToolResultEvents(events []ToolResultEvent) string { + if len(events) == 0 { + return "" + } + type agentSummary struct { + content string + } + latestByAgent := map[string]agentSummary{} + orderedAgents := make([]string, 0, len(events)) + lastAnon := "" + allHaveAgentID := true + for _, ev := range events { + if strings.TrimSpace(ev.Content) == "" { + continue + } + agentID := strings.TrimSpace(ev.AgentID) + if agentID == "" { + allHaveAgentID = false + lastAnon = ev.Content + continue + } + if _, ok := latestByAgent[agentID]; !ok { + latestByAgent[agentID] = agentSummary{content: ev.Content} + orderedAgents = append(orderedAgents, agentID) + continue + } + entry := latestByAgent[agentID] + entry.content = ev.Content + latestByAgent[agentID] = entry + } + if len(latestByAgent) <= 1 { + if len(latestByAgent) == 1 { + summary := latestByAgent[orderedAgents[0]].content + if lastAnon != "" { + return summary + "\n\n" + lastAnon + } + return summary + } + return lastAnon + } + parts := make([]string, 0, len(orderedAgents)) + for _, agentID := range orderedAgents { + parts = append(parts, agentID+":\n"+latestByAgent[agentID].content) + } + if !allHaveAgentID && lastAnon != "" { + parts = append(parts, lastAnon) + } + return strings.Join(parts, "\n\n") +} + // Message represents a row in the messages table. type Message struct { ID int64 `json:"id"` @@ -351,6 +673,7 @@ func roleFilterClause(roles []string) (string, []any) { func (db *DB) GetAllMessages( ctx context.Context, sessionID string, ) ([]Message, error) { + db.messagesLoadCount.Add(1) rows, err := db.getReader().QueryContext(ctx, fmt.Sprintf(` SELECT %s FROM messages @@ -885,6 +1208,52 @@ func insertToolResultEventsChunkTx( len(rows), err, ) } + if err := upsertToolCallAgentStateRows(tx, rows); err != nil { + return err + } + return nil +} + +// upsertToolCallAgentStateRows mirrors the inserted events into the +// per-call agent state table as event coordinates: the latest event per +// trimmed agent key in first-write order, so incremental summary +// recomputation never rescans a call's full event history and never +// duplicates event content. Empty-content events contribute nothing, +// matching SummarizeToolResultEvents. +func upsertToolCallAgentStateRows( + tx transactionQueries, rows []toolResultEventRow, +) error { + args := make([]any, 0, len(rows)*6) + for _, r := range rows { + if strings.TrimSpace(r.Event.Content) == "" && + r.Event.ContentLength == 0 { + continue + } + args = append(args, + r.SessionID, + r.MessageOrdinal, + r.CallIndex, + strings.TrimSpace(r.Event.AgentID), + r.Event.EventIndex, + r.Event.EventIndex, + ) + } + if len(args) == 0 { + return nil + } + query := ` + INSERT INTO tool_call_occurrence_agent_state + (session_id, message_ordinal, call_index, agent_id, + first_event_index, latest_event_index) + VALUES ` + multiRowPlaceholders(len(args)/6, 6) + ` + ON CONFLICT(session_id, message_ordinal, call_index, agent_id) + DO UPDATE SET latest_event_index = excluded.latest_event_index` + if _, err := tx.Exec(query, args...); err != nil { + return fmt.Errorf( + "upserting tool_call_agent_state (%d rows): %w", + len(args)/6, err, + ) + } return nil } @@ -1030,9 +1399,100 @@ func writeMessagesTx(tx *sql.Tx, msgs []Message) error { return nil } +// WriteSessionIncremental applies an incremental delta in one transaction: +// appended messages, tool-result updates, session metadata, the parser +// checkpoint, and — when update.SignalMaintainer is set and accepts the +// delta — the incremental signal/secret maintenance. The returned bool +// reports whether signals were maintained inside the transaction; when +// false the session's signal version was invalidated and the caller must +// schedule the debounced full recompute. +func applyMessageTokenUsageUpdateTx( + tx *sql.Tx, sessionID string, update MessageTokenUsageUpdate, +) (bool, error) { + var role, tokenUsage string + var contextTokens, outputTokens int + var hasContextTokens, hasOutputTokens bool + if err := tx.QueryRow( + `SELECT role, token_usage, context_tokens, output_tokens, + has_context_tokens, has_output_tokens + FROM messages + WHERE session_id = ? AND ordinal = ?`, + sessionID, update.Ordinal, + ).Scan( + &role, &tokenUsage, &contextTokens, &outputTokens, + &hasContextTokens, &hasOutputTokens, + ); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return false, fmt.Errorf( + "message usage target %s/%d does not exist", + sessionID, update.Ordinal, + ) + } + return false, fmt.Errorf( + "loading message usage target %s/%d: %w", + sessionID, update.Ordinal, err, + ) + } + if role != string(parser.RoleAssistant) { + return false, fmt.Errorf( + "message usage target %s/%d has role %q", + sessionID, update.Ordinal, role, + ) + } + + incomingUsage := string(update.TokenUsage) + if tokenUsage == incomingUsage && + contextTokens == update.ContextTokens && + outputTokens == update.OutputTokens && + hasContextTokens == update.HasContextTokens && + hasOutputTokens == update.HasOutputTokens { + return false, nil + } + if tokenUsage != "" { + return false, fmt.Errorf( + "message usage target %s/%d already has different usage", + sessionID, update.Ordinal, + ) + } + + result, err := tx.Exec( + `UPDATE messages + SET token_usage = ?, context_tokens = ?, output_tokens = ?, + has_context_tokens = ?, has_output_tokens = ? + WHERE session_id = ? AND ordinal = ? AND token_usage = ''`, + incomingUsage, + update.ContextTokens, + update.OutputTokens, + update.HasContextTokens, + update.HasOutputTokens, + sessionID, + update.Ordinal, + ) + if err != nil { + return false, fmt.Errorf( + "updating message usage target %s/%d: %w", + sessionID, update.Ordinal, err, + ) + } + rows, err := result.RowsAffected() + if err != nil { + return false, fmt.Errorf( + "reading message usage update result %s/%d: %w", + sessionID, update.Ordinal, err, + ) + } + if rows != 1 { + return false, fmt.Errorf( + "message usage target %s/%d changed concurrently", + sessionID, update.Ordinal, + ) + } + return true, nil +} + func (db *DB) WriteSessionIncremental( sessionID string, msgs []Message, update IncrementalSessionUpdate, -) error { +) (bool, error) { t := time.Now() defer func() { if d := time.Since(t); d > slowOpThreshold { @@ -1048,42 +1508,107 @@ func (db *DB) WriteSessionIncremental( tx, err := db.getWriter().Begin() if err != nil { - return fmt.Errorf("beginning incremental write tx: %w", err) + return false, fmt.Errorf("beginning incremental write tx: %w", err) } defer func() { _ = tx.Rollback() }() if err := writeMessagesTx(tx, msgs); err != nil { - return err + return false, err } transcriptChanged := len(msgs) > 0 + var updatedMessageUsageOrdinals map[int]struct{} + for _, usageUpdate := range update.MessageTokenUsageUpdates { + changed, err := applyMessageTokenUsageUpdateTx( + tx, sessionID, usageUpdate, + ) + if err != nil { + return false, err + } + if changed { + if updatedMessageUsageOrdinals == nil { + updatedMessageUsageOrdinals = make(map[int]struct{}) + } + updatedMessageUsageOrdinals[usageUpdate.Ordinal] = struct{}{} + } + transcriptChanged = transcriptChanged || changed + } for _, link := range update.SubagentLinks { changed, err := applyToolCallSubagentLinkTx( tx, sessionID, link, update.BlockedResultCategories, ) if err != nil { - return err + return false, err } transcriptChanged = transcriptChanged || changed } + var insertedResultEvents map[ToolCallPosition][]ToolResultEvent + for _, resultUpdate := range update.ToolCallResultUpdates { + changed, inserted, err := applyToolCallResultUpdateTx( + tx, sessionID, resultUpdate, + update.BlockedResultCategories, + ) + if err != nil { + return false, err + } + transcriptChanged = transcriptChanged || changed + if len(inserted) > 0 { + if insertedResultEvents == nil { + insertedResultEvents = make(map[ToolCallPosition][]ToolResultEvent) + } + key := resultUpdate.Position + insertedResultEvents[key] = append( + insertedResultEvents[key], inserted..., + ) + } + } if transcriptChanged { if err := bumpTranscriptRevisionTx(tx, sessionID); err != nil { - return err + return false, err } } if err := updateSessionIncrementalTx(tx, sessionID, update); err != nil { - return err + return false, err + } + if update.Checkpoint != nil && update.CheckpointBlobs != nil { + if err := upsertParserCheckpointTx( + tx, *update.Checkpoint, *update.CheckpointBlobs, + ); err != nil { + return false, err + } } if err := updateSessionAutomationFromMessagesTx(tx, sessionID); err != nil { - return err + return false, err + } + signalsMaintained := false + if update.SignalMaintainer != nil { + delta, err := update.SignalMaintainer.MaintainTx( + context.Background(), signalTxQuery{ + tx: tx, + sessionID: sessionID, + insertedResultEvents: insertedResultEvents, + updatedMessageUsageOrdinals: updatedMessageUsageOrdinals, + }, + ) + if err != nil { + return false, err + } + if delta != nil { + if err := applySignalDeltaTx(tx, sessionID, *delta); err != nil { + return false, err + } + signalsMaintained = true + } } - if err := invalidateSessionSignalsTx(tx, sessionID); err != nil { - return err + if !signalsMaintained { + if err := invalidateSessionSignalsTx(tx, sessionID); err != nil { + return false, err + } } if err := tx.Commit(); err != nil { - return fmt.Errorf("committing incremental write tx: %w", err) + return false, fmt.Errorf("committing incremental write tx: %w", err) } db.notifyUsageSessions([]string{sessionID}) - return nil + return signalsMaintained, nil } func messageSessionIDs(msgs []Message) []string { @@ -1430,7 +1955,23 @@ func deleteSessionMessagesTx(tx transactionQueries, sessionID string) error { "deleting old tool_result_events: %w", err, ) } - return deleteSessionMessageRowsTx(tx, sessionID) + if err := deleteSessionMessageRowsTx(tx, sessionID); err != nil { + return err + } + // Machine-local side tables are keyed by session id without foreign + // keys; hard deletes must not leave checkpoint, blob, or signal-state + // orphans behind. + for _, stmt := range []string{ + "DELETE FROM parser_checkpoints WHERE session_id = ?", + "DELETE FROM parser_checkpoint_blobs WHERE session_id = ?", + "DELETE FROM session_signal_state WHERE session_id = ?", + "DELETE FROM tool_call_occurrence_agent_state WHERE session_id = ?", + } { + if _, err := tx.Exec(stmt, sessionID); err != nil { + return fmt.Errorf("deleting session side rows: %w", err) + } + } + return nil } // ReplaceSessionContent atomically replaces a session's messages, signal @@ -1439,6 +1980,26 @@ func deleteSessionMessagesTx(tx transactionQueries, sessionID string) error { func (db *DB) ReplaceSessionContent( sessionID string, msgs []Message, signals SessionSignalUpdate, findings []SecretFinding, +) error { + return db.replaceSessionContent(sessionID, msgs, signals, findings, nil, nil) +} + +// ReplaceSessionContentWithCheckpoint replaces a session's content and +// persists the parser checkpoint in the same transaction, so the archive +// can never contain committed content whose resume state is missing. The +// checkpoint params are optional (nil skips the upsert). +func (db *DB) ReplaceSessionContentWithCheckpoint( + sessionID string, msgs []Message, + signals SessionSignalUpdate, findings []SecretFinding, + cp *ParserCheckpoint, blobs *ParserCheckpointBlobs, +) error { + return db.replaceSessionContent(sessionID, msgs, signals, findings, cp, blobs) +} + +func (db *DB) replaceSessionContent( + sessionID string, msgs []Message, + signals SessionSignalUpdate, findings []SecretFinding, + cp *ParserCheckpoint, blobs *ParserCheckpointBlobs, ) error { db.mu.Lock() defer db.mu.Unlock() @@ -1515,6 +2076,28 @@ func (db *DB) ReplaceSessionContent( ); err != nil { return err } + if cp == nil || blobs == nil { + // A full replacement without safe resume state invalidates any + // checkpoint for the previous projection. Leaving it behind would + // pair a newer committed transcript with an older cursor and hash. + if err := deleteParserCheckpointTx(tx, sessionID); err != nil { + return err + } + } else { + c := *cp + b := *blobs + // The checkpoint row is keyed by session id just like the blobs. + // The engine may hand in a parser-native id while the write lands + // under an idPrefix-rewritten session id; storing the two tables + // under different ids would strand the resume state and let a + // prefixed session overwrite (or borrow) a local checkpoint that + // shares the same native id. + c.SessionID = sessionID + b.SessionID = sessionID + if err := upsertParserCheckpointTx(tx, c, b); err != nil { + return err + } + } if err := tx.Commit(); err != nil { return err } @@ -2520,6 +3103,184 @@ func applyToolCallSubagentLinkTx( return err == nil, err } +func applyToolCallResultUpdateTx( + tx *sql.Tx, sessionID string, update ToolCallResultUpdate, + blockedResultCategories map[string]bool, +) (bool, []ToolResultEvent, error) { + if strings.TrimSpace(update.ToolUseID) == "" || len(update.Events) == 0 { + return false, nil, nil + } + + position := update.Position + var toolCallID int64 + var category string + if err := tx.QueryRow( + `SELECT tc.id, tc.category + FROM tool_calls tc + JOIN messages m ON m.id = tc.message_id + WHERE tc.session_id = ? AND m.ordinal = ? + AND COALESCE(tc.call_index, 0) = ? + AND COALESCE(tc.tool_use_id, '') = ?`, + sessionID, position.MessageOrdinal, position.CallIndex, + update.ToolUseID, + ).Scan(&toolCallID, &category); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return false, nil, nil + } + return false, nil, fmt.Errorf( + "checking tool result target for %s/%s: %w", + sessionID, update.ToolUseID, err, + ) + } + + // The per-call agent state table makes the late-result update O(delta): + // deduplication probes the stored rows directly and the summary reads + // only the call's distinct agents, never the call's full event + // history. Sessions written before the state table existed (or through + // the staged publish) get a one-time backfill on their first late + // result; every later update stays O(delta). + var stateExists int + err := tx.QueryRow( + `SELECT 1 FROM tool_call_occurrence_agent_state + WHERE session_id = ? AND message_ordinal = ? AND call_index = ? + LIMIT 1`, + sessionID, position.MessageOrdinal, position.CallIndex, + ).Scan(&stateExists) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return false, nil, fmt.Errorf( + "checking agent state for %s/%s: %w", + sessionID, update.ToolUseID, err, + ) + } + if err != nil { + if err := backfillToolCallAgentStateTx( + tx, sessionID, position, + ); err != nil { + return false, nil, err + } + } + var nextEventIndex int + if err := tx.QueryRow( + `SELECT COALESCE(MAX(event_index), -1) + 1 + FROM tool_result_events + WHERE session_id = ? AND tool_call_message_ordinal = ? + AND call_index = ?`, + sessionID, position.MessageOrdinal, position.CallIndex, + ).Scan(&nextEventIndex); err != nil { + return false, nil, fmt.Errorf( + "reading next event index for %s/%s: %w", + sessionID, update.ToolUseID, err, + ) + } + + incoming := append([]ToolResultEvent(nil), update.Events...) + for i := range incoming { + if incoming[i].ToolUseID == "" { + incoming[i].ToolUseID = update.ToolUseID + } + if incoming[i].ContentLength == 0 { + incoming[i].ContentLength = len(incoming[i].Content) + } + } + + blocked := blockedResultCategories[category] + // Blocked-category content is blanked below and never stored, so it + // must skip sanitization: sanitizing first (like the full-parse + // pairToolResultEventSummaries path avoids by blanking before its + // central validation pass runs) would shrink ContentLength by the + // stripped-byte count before the blank overwrites Content, losing the + // original result length the full and staged paths both preserve. + if !blocked { + toolCall := ToolCall{ResultEvents: incoming} + _ = SanitizeToolCall(&toolCall) + incoming = toolCall.ResultEvents + } + + insertRows := make([]toolResultEventRow, 0, len(incoming)) + var inserted []ToolResultEvent + for _, candidate := range incoming { + stored := candidate + if blocked { + stored.Content = "" + } + // Equivalence mirrors the parser's raw dedup as closely as the + // stored columns allow: same agent and status, plus identical + // (sanitized) content for storable categories, or identical + // original length for blocked categories. The two arms are + // exclusive on purpose: every stored blocked row has an empty + // content column, so a content comparison there would match any + // earlier blocked event of the same agent/status and collapse + // distinct outputs; and the raw blocked content is never bound. + var exists int + err := tx.QueryRow( + `SELECT 1 FROM tool_result_events + WHERE session_id = ? AND tool_call_message_ordinal = ? + AND call_index = ? AND COALESCE(agent_id, '') IS ? + AND status IS ? + AND ((? = 0 AND content IS ?) + OR (? = 1 AND content_length = ?)) + LIMIT 1`, + sessionID, position.MessageOrdinal, position.CallIndex, + stored.AgentID, stored.Status, + blocked, stored.Content, + blocked, stored.ContentLength, + ).Scan(&exists) + if err == nil { + continue // equivalent event already stored + } + if !errors.Is(err, sql.ErrNoRows) { + return false, nil, fmt.Errorf( + "checking tool result equivalence for %s/%s: %w", + sessionID, update.ToolUseID, err, + ) + } + stored.EventIndex = nextEventIndex + nextEventIndex++ + inserted = append(inserted, stored) + insertRows = append(insertRows, toolResultEventRow{ + SessionID: sessionID, + MessageOrdinal: position.MessageOrdinal, + CallIndex: position.CallIndex, + Event: stored, + }) + } + if len(insertRows) == 0 { + return false, nil, nil + } + if err := insertToolResultEventsTx(tx, insertRows); err != nil { + return false, nil, err + } + + summary, err := summarizeToolCallFromStateTx( + tx, sessionID, position, + ) + if err != nil { + return false, nil, err + } + resultLength, err := summarizeToolCallLengthFromStateTx( + tx, sessionID, position, + ) + if err != nil { + return false, nil, err + } + storedSummary := summary + if blocked { + storedSummary = "" + } + if _, err := tx.Exec( + `UPDATE tool_calls + SET result_content_length = ?, result_content = ? + WHERE id = ?`, + resultLength, storedSummary, toolCallID, + ); err != nil { + return false, nil, fmt.Errorf( + "updating tool result summary for %s/%s: %w", + sessionID, update.ToolUseID, err, + ) + } + return true, inserted, nil +} + // SystemMessageFingerprint returns the ordered, comma-separated list of // ordinals for system messages in a session (e.g. "0,2,5"). This is an // exact fingerprint of the system-message ordinal set: any reclassification diff --git a/internal/db/messages_diff.go b/internal/db/messages_diff.go index b534995ec..fd57ad59a 100644 --- a/internal/db/messages_diff.go +++ b/internal/db/messages_diff.go @@ -393,6 +393,20 @@ func deleteToolRowsForMessagesTx( for _, id := range ids[start:end] { idArgs = append(idArgs, id) } + // Agent-state rows use stable message/call coordinates. Clear every + // occurrence owned by the messages being rebuilt so removed agents and + // reused provider IDs cannot leave stale summary state. + if _, err := tx.Exec( + "DELETE FROM tool_call_occurrence_agent_state WHERE session_id = ?"+ + " AND message_ordinal IN ("+ + "SELECT ordinal FROM messages WHERE id IN ("+ + placeholderList(len(idArgs))+"))", + append([]any{sessionID}, idArgs...)..., + ); err != nil { + return fmt.Errorf( + "deleting stale tool-call occurrence state: %w", err, + ) + } if _, err := tx.Exec( "DELETE FROM tool_calls WHERE message_id IN ("+ placeholderList(len(idArgs))+")", diff --git a/internal/db/schema.sql b/internal/db/schema.sql index 9d13157ca..feb8e0ec2 100644 --- a/internal/db/schema.sql +++ b/internal/db/schema.sql @@ -331,6 +331,9 @@ CREATE TABLE IF NOT EXISTS tool_calls ( CREATE INDEX IF NOT EXISTS idx_tool_calls_session ON tool_calls(session_id); +CREATE INDEX IF NOT EXISTS idx_tool_calls_session_tool_use + ON tool_calls(session_id, tool_use_id) + WHERE tool_use_id IS NOT NULL; CREATE INDEX IF NOT EXISTS idx_tool_calls_session_category ON tool_calls(session_id, category); -- idx_tool_calls_message backs the ON DELETE CASCADE from @@ -1342,3 +1345,61 @@ CREATE TABLE IF NOT EXISTS artifact_imported_sessions ( ), PRIMARY KEY (origin, gid) ); + +-- Machine-local per-call-occurrence agent content state for incremental +-- summary recomputation: the latest raw content per agent in first-write +-- order. Provider call IDs are not unique, so the natural stored coordinates +-- (message ordinal, call index) are the authoritative key. A late result +-- update reads only this table (O(distinct agents)) instead of rescanning the +-- call's full event history. Never mirrored to PostgreSQL or DuckDB. +CREATE TABLE IF NOT EXISTS tool_call_occurrence_agent_state ( + session_id TEXT NOT NULL, + message_ordinal INTEGER NOT NULL, + call_index INTEGER NOT NULL, + agent_id TEXT NOT NULL, + first_event_index INTEGER NOT NULL, + latest_event_index INTEGER NOT NULL, + PRIMARY KEY (session_id, message_ordinal, call_index, agent_id) +); + +-- Machine-local parse checkpoints. SQLite-only, never mirrored to +-- PostgreSQL or DuckDB: parsers never run against those read-side stores, +-- and a copy that drops this table degrades to the conservative no-checkpoint +-- behavior (full parse / prefix rescan), never to a wrong resume. +CREATE TABLE IF NOT EXISTS parser_checkpoints ( + session_id TEXT PRIMARY KEY, + agent TEXT NOT NULL, + file_path TEXT NOT NULL, + file_inode INTEGER NOT NULL, + file_device INTEGER NOT NULL, + file_mtime INTEGER NOT NULL, + file_change_time INTEGER NOT NULL DEFAULT 0, + offset INTEGER NOT NULL, + tail_anchor_digest TEXT NOT NULL, + hash TEXT NOT NULL, + next_ordinal INTEGER NOT NULL, + checkpoint_version INTEGER NOT NULL, + updated_at TEXT NOT NULL +); + +-- Lazy-loaded checkpoint payload: the provider cursor and the resumable +-- hash state. Kept out of parser_checkpoints so the stat-only freshness +-- gate reads the small metadata row without touching the blobs. +CREATE TABLE IF NOT EXISTS parser_checkpoint_blobs ( + session_id TEXT PRIMARY KEY, + cursor BLOB NOT NULL, + hash_state BLOB +); + +-- Compact per-session signal/secret maintenance state. SQLite-only: the +-- state is machine-local sync bookkeeping and is never mirrored to +-- PostgreSQL or DuckDB. The state row carries a verification token +-- (transcript revision + signal version); a row whose token disagrees with +-- the stored session must never be folded into an incremental delta. +CREATE TABLE IF NOT EXISTS session_signal_state ( + session_id TEXT PRIMARY KEY, + state BLOB NOT NULL, + transcript_revision TEXT NOT NULL, + signal_version INTEGER NOT NULL, + updated_at TEXT NOT NULL +); diff --git a/internal/db/session_batch.go b/internal/db/session_batch.go index ab314f598..43725020e 100644 --- a/internal/db/session_batch.go +++ b/internal/db/session_batch.go @@ -30,6 +30,8 @@ type SessionBatchWrite struct { ReplaceMessages bool // RejectMessageCountDecrease prevents full replacement with fewer messages. RejectMessageCountDecrease bool + Checkpoint *ParserCheckpoint + CheckpointBlobs *ParserCheckpointBlobs } // SessionWouldShortenError reports a rejected message-count decrease. @@ -615,6 +617,21 @@ func writeOneSessionBatchTx( return 0, err } } + if write.ReplaceMessages { + if write.Checkpoint == nil || write.CheckpointBlobs == nil { + if err := deleteParserCheckpointTx(tx, write.Session.ID); err != nil { + return 0, err + } + } else { + checkpoint := *write.Checkpoint + blobs := *write.CheckpointBlobs + checkpoint.SessionID = write.Session.ID + blobs.SessionID = write.Session.ID + if err := upsertParserCheckpointTx(tx, checkpoint, blobs); err != nil { + return 0, err + } + } + } if err := enqueueArtifactExportIfGenerationUnchangedTx( queries, write.Session.ID, queueGenerationBefore, queueExistedBefore, ); err != nil { diff --git a/internal/db/sessions.go b/internal/db/sessions.go index 9a33db9fb..01b0b936a 100644 --- a/internal/db/sessions.go +++ b/internal/db/sessions.go @@ -6,6 +6,7 @@ import ( "crypto/sha256" "database/sql" "encoding/base64" + "encoding/json/jsontext" "encoding/json/v2" "errors" "fmt" @@ -2672,24 +2673,52 @@ type IncrementalInfo struct { PeakContextTokens int HasTotalOutputTokens bool HasPeakContextTokens bool + // PendingUsageOrdinal identifies the last committed assistant message in + // the current turn whose token usage has not yet been persisted. Codex + // incremental tails use it to apply the token_count that follows a late + // tool result without reparsing the committed prefix. + PendingUsageOrdinal *int +} + +// MessageTokenUsageUpdate applies token metadata to an assistant message +// that was committed in an earlier incremental batch. +type MessageTokenUsageUpdate struct { + Ordinal int + TokenUsage jsontext.Value + ContextTokens int + OutputTokens int + HasContextTokens bool + HasOutputTokens bool } type IncrementalSessionUpdate struct { - EndedAt *string - TerminationStatus *string - MsgCount int - UserMsgCount int - FileSize int64 - FileMtime int64 - FileHash *string - NextOrdinal int - LastEntryUUID string - TotalOutputTokens int - PeakContextTokens int - HasTotalOutputTokens bool - HasPeakContextTokens bool - SubagentLinks []ToolCallSubagentLink + EndedAt *string + TerminationStatus *string + MsgCount int + UserMsgCount int + FileSize int64 + FileMtime int64 + FileHash *string + NextOrdinal int + LastEntryUUID string + TotalOutputTokens int + PeakContextTokens int + HasTotalOutputTokens bool + HasPeakContextTokens bool + SubagentLinks []ToolCallSubagentLink + ToolCallResultUpdates []ToolCallResultUpdate + MessageTokenUsageUpdates []MessageTokenUsageUpdate + // Checkpoint/CheckpointBlobs are the machine-local parser checkpoint + // metadata and lazy payload to persist in the same transaction as this + // delta. nil keeps any existing checkpoint. + Checkpoint *ParserCheckpoint + CheckpointBlobs *ParserCheckpointBlobs BlockedResultCategories map[string]bool + // SignalMaintainer, when set, computes the incremental signal/secret + // delta inside the write transaction (after messages and result + // updates are applied). nil keeps the legacy behavior: signals are + // invalidated and recomputed by the debounced full path. + SignalMaintainer SignalMaintainer } type ToolCallSubagentLink struct { @@ -2700,6 +2729,22 @@ type ToolCallSubagentLink struct { HasResult bool } +// ToolCallPosition identifies one stored tool-call occurrence by its stable +// normalized message ordinal and call index. +type ToolCallPosition struct { + MessageOrdinal int + CallIndex int +} + +// ToolCallResultUpdate carries result events for one exact tool-call occurrence +// that already exists in the database. ToolUseID remains a provider identity +// check; Position is the authoritative target when providers reuse call IDs. +type ToolCallResultUpdate struct { + ToolUseID string + Position ToolCallPosition + Events []ToolResultEvent +} + // GetSessionForIncremental returns session state needed for incremental // parsing, looked up by agent and file_path. Returns false when the scoped path // is unknown or maps to multiple sessions (e.g. Claude DAG forks), since @@ -2723,7 +2768,7 @@ func (db *DB) GetSessionForIncremental( } var info IncrementalInfo - var fs, fm, fi, fd sql.NullInt64 + var fs, fm, fi, fd, pendingUsageOrdinal sql.NullInt64 var firstMsg, lastEntryUUID sql.NullString var linearParse sql.NullBool err = db.getReader().QueryRow( @@ -2735,7 +2780,19 @@ func (db *DB) GetSessionForIncremental( message_count, user_message_count, first_message, total_output_tokens, peak_context_tokens, - has_total_output_tokens, has_peak_context_tokens + has_total_output_tokens, has_peak_context_tokens, + (SELECT m.ordinal + FROM messages m + WHERE m.session_id = s.id + AND m.role = 'assistant' + AND (m.token_usage IS NULL OR length(m.token_usage) = 0) + AND m.ordinal > COALESCE(( + SELECT MAX(u.ordinal) + FROM messages u + WHERE u.session_id = s.id AND u.role = 'user' + ), -1) + ORDER BY m.ordinal DESC + LIMIT 1) FROM sessions s LEFT JOIN session_project_identity_snapshots snap ON snap.session_id = s.id @@ -2754,6 +2811,7 @@ func (db *DB) GetSessionForIncremental( &firstMsg, &info.TotalOutputTokens, &info.PeakContextTokens, &info.HasTotalOutputTokens, &info.HasPeakContextTokens, + &pendingUsageOrdinal, ) if err != nil { return nil, false @@ -2779,6 +2837,10 @@ func (db *DB) GetSessionForIncremental( if fd.Valid { info.FileDevice = fd.Int64 } + if pendingUsageOrdinal.Valid { + ordinal := int(pendingUsageOrdinal.Int64) + info.PendingUsageOrdinal = &ordinal + } info.HasTotalOutputTokens = info.HasTotalOutputTokens || info.TotalOutputTokens != 0 info.HasPeakContextTokens = @@ -2972,6 +3034,26 @@ func (db *DB) GetFileInfoByAgentPath( return s.Int64, m.Int64, true } +// GetFileIdentityByAgentPath extends GetFileInfoByAgentPath with the stored +// file identity (inode/device), used by the stat-only freshness gate that +// skips a checkpointed Codex source without hashing its transcript. +func (db *DB) GetFileIdentityByAgentPath( + path, agent string, +) (size int64, mtime int64, inode, device uint64, ok bool) { + var s, m, i, d sql.NullInt64 + err := db.getReader().QueryRow( + "SELECT file_size, file_mtime, file_inode, file_device FROM sessions"+ + " WHERE file_path = ? AND agent = ?"+ + " AND source_missing_at IS NULL"+ + " ORDER BY file_mtime DESC LIMIT 1", + path, agent, + ).Scan(&s, &m, &i, &d) + if err != nil || !s.Valid || !m.Valid || !i.Valid || !d.Valid { + return 0, 0, 0, 0, false + } + return s.Int64, m.Int64, uint64(i.Int64), uint64(d.Int64), true +} + // GetCwdByAgentPath returns the stored Cwd for the source owned by agent. A // source-missing row remains eligible because its positive Cwd is the // preservation authority when the source is parsed again. diff --git a/internal/db/sessions_sync_marker_test.go b/internal/db/sessions_sync_marker_test.go index 5ad1761b0..03c0eab59 100644 --- a/internal/db/sessions_sync_marker_test.go +++ b/internal/db/sessions_sync_marker_test.go @@ -67,9 +67,10 @@ func TestMessageMutationAdvancesUsageVersion(t *testing.T) { insertSession(t, d, id, "proj") }, write: func(t *testing.T, d *DB, id string, changed Message) { - require.NoError(t, d.WriteSessionIncremental( + _, err := d.WriteSessionIncremental( id, []Message{changed}, IncrementalSessionUpdate{}, - )) + ) + require.NoError(t, err) }, }, { diff --git a/internal/db/signal_maintenance.go b/internal/db/signal_maintenance.go new file mode 100644 index 000000000..1d146d995 --- /dev/null +++ b/internal/db/signal_maintenance.go @@ -0,0 +1,795 @@ +package db + +import ( + "context" + "database/sql" + "errors" + "fmt" + "strings" + "time" + + "go.kenn.io/agentsview/internal/secrets" +) + +// ToolCallSignalFact is the bounded per-call fact set the incremental signal +// maintainer reads inside the incremental write transaction. It mirrors the +// columns extractToolCallRows reads from stored rows, so facts the maintainer +// sees match a full recompute over GetAllMessages. +type ToolCallSignalFact struct { + MessageOrdinal int + CallIndex int + ToolName string + Category string + InputJSON string + ResultContent string + EventStatus string + ToolUseID string +} + +// FindingDeleteKey addresses secret findings by their natural coordinates: +// every finding produced from a call's summarized result_content is removed +// when that call gains its first result event. +type FindingDeleteKey struct { + MessageOrdinal int + CallIndex int + LocationKind string +} + +// SignalDelta is the incremental signal/findings/state change applied inside +// the incremental write transaction, atomically with the message rows and +// the parser checkpoint. +type SignalDelta struct { + Update SessionSignalUpdate + InsertFindings []SecretFinding + DeleteFindingKeys []FindingDeleteKey + // State is the post-delta compact state row to persist (SQLite-only). + State *SessionSignalState +} + +// SessionSignalState is one session's persisted compact signal state row. +// TranscriptRevision and SignalVersion form the verification token: a state +// whose token disagrees with the stored rows must never be folded. +type SessionSignalState struct { + SessionID string + State []byte + TranscriptRevision string + SignalVersion int + UpdatedAt string +} + +// SessionSignalInputSnapshot is the complete session-row input set consumed +// by the full signal recompute. TranscriptRevision protects message and tool +// rows; the remaining fields protect metadata that can change without a +// transcript rewrite. A full recompute may publish only while every field +// still matches the snapshot it computed from. +type SessionSignalInputSnapshot struct { + TranscriptRevision string + MessageCount int + IsAutomated bool + EndedAt string + HasEndedAt bool + PeakContextTokens int + HasPeakContextTokens bool +} + +// SignalInputSnapshot returns the signal-driving session-row inputs from s. +func SignalInputSnapshot(s Session) (SessionSignalInputSnapshot, error) { + if s.TranscriptRevision == nil { + return SessionSignalInputSnapshot{}, fmt.Errorf( + "session %s has no transcript revision", s.ID, + ) + } + snapshot := SessionSignalInputSnapshot{ + TranscriptRevision: *s.TranscriptRevision, + MessageCount: s.MessageCount, + IsAutomated: s.IsAutomated, + PeakContextTokens: s.PeakContextTokens, + HasPeakContextTokens: s.HasPeakContextTokens, + } + if s.EndedAt != nil { + snapshot.EndedAt = *s.EndedAt + snapshot.HasEndedAt = true + } + return snapshot, nil +} + +// SignalMaintainer computes the incremental signal delta inside the +// incremental write transaction, after messages and result updates are +// applied but before the transaction commits. It may return a nil delta to +// decline maintenance: the write then invalidates the signal version as +// before and the caller debounces a full recompute. +type SignalMaintainer interface { + MaintainTx(ctx context.Context, q SignalQuery) (*SignalDelta, error) +} + +// SignalQuery is the read-only in-transaction view the maintainer uses. +// Every read reflects the transaction's uncommitted writes. +type SignalQuery interface { + // Session returns the session row snapshot (signal columns plus the + // outcome inputs). nil means the session row is gone. + Session(ctx context.Context) (*Session, error) + TranscriptRevision(ctx context.Context) (string, error) + SignalState(ctx context.Context) (SessionSignalState, bool, error) + // TrailingToolCalls returns the last n tool-call facts in + // (message ordinal, call index) order, oldest first. + TrailingToolCalls(ctx context.Context, n int) ([]ToolCallSignalFact, error) + // ToolCallsByPosition returns the facts of the exact call occurrences + // after the transaction's result updates. + ToolCallsByPosition( + ctx context.Context, positions []ToolCallPosition, + ) ([]ToolCallSignalFact, error) + // CallResultEvents returns the stored result events of one call in + // event_index order, after the transaction's updates. + CallResultEvents( + ctx context.Context, messageOrdinal, callIndex int, + ) ([]ToolResultEvent, error) + // InsertedResultEvents returns only the result events this transaction + // just inserted for the exact call occurrence, with their assigned event + // indexes. Previously stored events already carry findings. + InsertedResultEvents(position ToolCallPosition) []ToolResultEvent + // MessageTokenUsageUpdated reports whether this transaction actually + // changed the token metadata of the named committed message. Identical + // replays must not fold the same token-drop compaction twice. + MessageTokenUsageUpdated(ordinal int) bool +} + +// signalTxQuery implements SignalQuery over the incremental write +// transaction. +type signalTxQuery struct { + tx *sql.Tx + sessionID string + insertedResultEvents map[ToolCallPosition][]ToolResultEvent + updatedMessageUsageOrdinals map[int]struct{} +} + +func (q signalTxQuery) Session( + ctx context.Context, +) (*Session, error) { + var s Session + var isAutomated, hasPeak, hasToolCalls, hasContextData int + var unstructuredStart int + var cpMax sql.NullFloat64 + var healthScore sql.NullInt64 + var healthGrade, endedAt, signalsPending sql.NullString + err := q.tx.QueryRowContext(ctx, ` + SELECT message_count, is_automated, ended_at, + peak_context_tokens, has_peak_context_tokens, + tool_failure_signal_count, tool_retry_count, + edit_churn_count, consecutive_failure_max, + outcome, outcome_confidence, ended_with_role, + final_failure_streak, signals_pending_since, + compaction_count, mid_task_compaction_count, + context_pressure_max, health_score, health_grade, + has_tool_calls, has_context_data, + secret_leak_count, secrets_rules_version, + quality_signal_version, short_prompt_count, + unstructured_start, missing_success_criteria_count, + missing_verification_count, duplicate_prompt_count, + no_code_context_count, runaway_tool_loop_count + FROM sessions WHERE id = ?`, + q.sessionID, + ).Scan( + &s.MessageCount, &isAutomated, &endedAt, + &s.PeakContextTokens, &hasPeak, + &s.ToolFailureSignalCount, &s.ToolRetryCount, + &s.EditChurnCount, &s.ConsecutiveFailureMax, + &s.Outcome, &s.OutcomeConfidence, &s.EndedWithRole, + &s.FinalFailureStreak, &signalsPending, + &s.CompactionCount, &s.MidTaskCompactionCount, + &cpMax, &healthScore, &healthGrade, + &hasToolCalls, &hasContextData, + &s.SecretLeakCount, &s.SecretsRulesVersion, + &s.QualitySignalVersion, &s.ShortPromptCount, + &unstructuredStart, &s.MissingSuccessCriteriaCount, + &s.MissingVerificationCount, &s.DuplicatePromptCount, + &s.NoCodeContextCount, &s.RunawayToolLoopCount, + ) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf( + "loading session signal snapshot %s: %w", q.sessionID, err, + ) + } + s.ID = q.sessionID + s.IsAutomated = isAutomated != 0 + s.HasPeakContextTokens = hasPeak != 0 + s.HasToolCalls = hasToolCalls != 0 + s.HasContextData = hasContextData != 0 + s.UnstructuredStart = unstructuredStart != 0 + if endedAt.Valid { + s.EndedAt = &endedAt.String + } + if signalsPending.Valid { + s.SignalsPendingSince = &signalsPending.String + } + if cpMax.Valid { + v := cpMax.Float64 + s.ContextPressureMax = &v + } + if healthScore.Valid { + v := int(healthScore.Int64) + s.HealthScore = &v + } + if healthGrade.Valid { + s.HealthGrade = &healthGrade.String + } + return &s, nil +} + +func (q signalTxQuery) TranscriptRevision( + ctx context.Context, +) (string, error) { + var rev string + if err := q.tx.QueryRowContext(ctx, + `SELECT transcript_revision FROM sessions WHERE id = ?`, + q.sessionID, + ).Scan(&rev); err != nil { + return "", fmt.Errorf( + "loading transcript revision %s: %w", q.sessionID, err, + ) + } + return rev, nil +} + +func (q signalTxQuery) SignalState( + ctx context.Context, +) (SessionSignalState, bool, error) { + var st SessionSignalState + err := q.tx.QueryRowContext(ctx, ` + SELECT state, transcript_revision, signal_version + FROM session_signal_state WHERE session_id = ?`, + q.sessionID, + ).Scan(&st.State, &st.TranscriptRevision, &st.SignalVersion) + if errors.Is(err, sql.ErrNoRows) { + return SessionSignalState{}, false, nil + } + if err != nil { + return SessionSignalState{}, false, fmt.Errorf( + "loading session signal state %s: %w", q.sessionID, err, + ) + } + st.SessionID = q.sessionID + return st, true, nil +} + +func (q signalTxQuery) TrailingToolCalls( + ctx context.Context, n int, +) ([]ToolCallSignalFact, error) { + rows, err := q.tx.QueryContext(ctx, ` + SELECT m.ordinal, COALESCE(tc.call_index, 0), + tc.tool_name, tc.category, COALESCE(tc.input_json, ''), + COALESCE(tc.result_content, ''), + COALESCE(tc.tool_use_id, ''), + COALESCE(( + SELECT tre.status FROM tool_result_events tre + WHERE tre.session_id = tc.session_id + AND tre.tool_call_message_ordinal = m.ordinal + AND tre.call_index = COALESCE(tc.call_index, 0) + ORDER BY tre.event_index DESC, tre.id DESC + LIMIT 1 + ), '') + FROM tool_calls tc + JOIN messages m ON m.id = tc.message_id + WHERE tc.session_id = ? + ORDER BY m.ordinal DESC, tc.call_index DESC + LIMIT ?`, + q.sessionID, n, + ) + if err != nil { + return nil, fmt.Errorf( + "loading trailing tool calls %s: %w", q.sessionID, err, + ) + } + defer rows.Close() + var facts = make([]ToolCallSignalFact, 0) + for rows.Next() { + var f ToolCallSignalFact + if err := rows.Scan( + &f.MessageOrdinal, &f.CallIndex, &f.ToolName, &f.Category, + &f.InputJSON, &f.ResultContent, &f.ToolUseID, &f.EventStatus, + ); err != nil { + return nil, fmt.Errorf( + "scanning trailing tool call %s: %w", q.sessionID, err, + ) + } + facts = append(facts, f) + } + // The query orders descending for the LIMIT; reverse to chronological. + for i, j := 0, len(facts)-1; i < j; i, j = i+1, j-1 { + facts[i], facts[j] = facts[j], facts[i] + } + return facts, rows.Err() +} + +func (q signalTxQuery) ToolCallsByPosition( + ctx context.Context, positions []ToolCallPosition, +) ([]ToolCallSignalFact, error) { + if len(positions) == 0 { + return nil, nil + } + clauses := make([]string, len(positions)) + args := make([]any, 0, 1+2*len(positions)) + args = append(args, q.sessionID) + for i, position := range positions { + clauses[i] = "(m.ordinal = ? AND COALESCE(tc.call_index, 0) = ?)" + args = append(args, position.MessageOrdinal, position.CallIndex) + } + rows, err := q.tx.QueryContext(ctx, ` + SELECT m.ordinal, COALESCE(tc.call_index, 0), + tc.tool_name, tc.category, COALESCE(tc.input_json, ''), + COALESCE(tc.result_content, ''), + COALESCE(tc.tool_use_id, ''), + COALESCE(( + SELECT tre.status FROM tool_result_events tre + WHERE tre.session_id = tc.session_id + AND tre.tool_call_message_ordinal = m.ordinal + AND tre.call_index = COALESCE(tc.call_index, 0) + ORDER BY tre.event_index DESC, tre.id DESC + LIMIT 1 + ), '') + FROM tool_calls tc + JOIN messages m ON m.id = tc.message_id + WHERE tc.session_id = ? AND (`+strings.Join(clauses, " OR ")+")", + args..., + ) + if err != nil { + return nil, fmt.Errorf( + "loading tool call facts %s: %w", q.sessionID, err, + ) + } + defer rows.Close() + var facts []ToolCallSignalFact + for rows.Next() { + var f ToolCallSignalFact + if err := rows.Scan( + &f.MessageOrdinal, &f.CallIndex, &f.ToolName, &f.Category, + &f.InputJSON, &f.ResultContent, &f.ToolUseID, &f.EventStatus, + ); err != nil { + return nil, fmt.Errorf( + "scanning tool call fact %s: %w", q.sessionID, err, + ) + } + facts = append(facts, f) + } + return facts, rows.Err() +} + +func (q signalTxQuery) CallResultEvents( + ctx context.Context, messageOrdinal, callIndex int, +) ([]ToolResultEvent, error) { + rows, err := q.tx.QueryContext(ctx, ` + SELECT COALESCE(tool_use_id, ''), COALESCE(agent_id, ''), + COALESCE(subagent_session_id, ''), source, status, + content, content_length, COALESCE(timestamp, ''), event_index + FROM tool_result_events + WHERE session_id = ? AND tool_call_message_ordinal = ? + AND call_index = ? + ORDER BY event_index, id`, + q.sessionID, messageOrdinal, callIndex, + ) + if err != nil { + return nil, fmt.Errorf( + "loading result events %s/%d/%d: %w", + q.sessionID, messageOrdinal, callIndex, err, + ) + } + defer rows.Close() + var events []ToolResultEvent + for rows.Next() { + var ev ToolResultEvent + if err := rows.Scan( + &ev.ToolUseID, &ev.AgentID, &ev.SubagentSessionID, + &ev.Source, &ev.Status, &ev.Content, &ev.ContentLength, + &ev.Timestamp, &ev.EventIndex, + ); err != nil { + return nil, fmt.Errorf( + "scanning result event %s: %w", q.sessionID, err, + ) + } + events = append(events, ev) + } + return events, rows.Err() +} + +// InsertedResultEvents returns the result events this transaction inserted +// for one exact call occurrence. +func (q signalTxQuery) InsertedResultEvents( + position ToolCallPosition, +) []ToolResultEvent { + return q.insertedResultEvents[position] +} + +func (q signalTxQuery) MessageTokenUsageUpdated(ordinal int) bool { + _, ok := q.updatedMessageUsageOrdinals[ordinal] + return ok +} + +// TranscriptRevision returns a session's stored transcript revision. The +// incremental signal maintainer uses the pre-write value to verify the +// persisted state token before folding a delta. +func (db *DB) TranscriptRevision(sessionID string) (string, error) { + var rev string + if err := db.getReader().QueryRow( + `SELECT transcript_revision FROM sessions WHERE id = ?`, + sessionID, + ).Scan(&rev); err != nil { + return "", fmt.Errorf( + "loading transcript revision %s: %w", sessionID, err, + ) + } + return rev, nil +} + +// SessionSecretsRulesVersion returns a session's stored secrets rules +// version. The incremental signal maintainer uses the pre-write value to +// verify the session was scanned at the current definite rules version +// before folding a delta: the write transaction blanks the column when it +// bumps the transcript revision, so only the pre-write value is meaningful. +func (db *DB) SessionSecretsRulesVersion(sessionID string) (string, error) { + var ver string + if err := db.getReader().QueryRow( + `SELECT secrets_rules_version FROM sessions WHERE id = ?`, + sessionID, + ).Scan(&ver); err != nil { + return "", fmt.Errorf( + "loading secrets rules version %s: %w", sessionID, err, + ) + } + return ver, nil +} + +// GetSessionSignalState loads a session's compact signal state row. +// ok=false means no row exists. +func (db *DB) GetSessionSignalState( + sessionID string, +) (SessionSignalState, bool, error) { + var st SessionSignalState + err := db.getReader().QueryRow(` + SELECT state, transcript_revision, signal_version + FROM session_signal_state WHERE session_id = ?`, + sessionID, + ).Scan(&st.State, &st.TranscriptRevision, &st.SignalVersion) + if errors.Is(err, sql.ErrNoRows) { + return SessionSignalState{}, false, nil + } + if err != nil { + return SessionSignalState{}, false, fmt.Errorf( + "loading session signal state %s: %w", sessionID, err, + ) + } + st.SessionID = sessionID + return st, true, nil +} + +// UpsertSessionSignalState stores or replaces a session's compact signal +// state row. Full recompute paths call it after their signal columns +// commit so later incremental deltas can fold. +func (db *DB) UpsertSessionSignalState(st SessionSignalState) error { + db.mu.Lock() + defer db.mu.Unlock() + tx, err := db.getWriter().Begin() + if err != nil { + return fmt.Errorf("beginning signal state tx: %w", err) + } + defer func() { _ = tx.Rollback() }() + if err := upsertSessionSignalStateTx(tx, st); err != nil { + return err + } + return tx.Commit() +} + +// UpsertSessionSignalStateIfRevision stores compact signal state only while +// the session still has the transcript revision represented by st.State. +// The boolean is false when the snapshot became stale before publication. +func (db *DB) UpsertSessionSignalStateIfRevision( + st SessionSignalState, +) (bool, error) { + db.mu.Lock() + defer db.mu.Unlock() + tx, err := db.getWriter().Begin() + if err != nil { + return false, fmt.Errorf("beginning conditional signal state tx: %w", err) + } + defer func() { _ = tx.Rollback() }() + matches, err := sessionTranscriptRevisionMatchesTx( + tx, st.SessionID, st.TranscriptRevision, + ) + if err != nil || !matches { + return false, err + } + if err := upsertSessionSignalStateTx(tx, st); err != nil { + return false, err + } + if err := tx.Commit(); err != nil { + return false, fmt.Errorf( + "committing conditional signal state %s: %w", st.SessionID, err, + ) + } + return true, nil +} + +// ReplaceSessionSignalsIfInputsMatch atomically publishes findings, aggregate +// columns, and compact incremental state for one coherent signal-input +// snapshot. It returns false without modifying any rows when a concurrent +// writer changes either the transcript or metadata consumed by the recompute. +func (db *DB) ReplaceSessionSignalsIfInputsMatch( + sessionID string, + expected SessionSignalInputSnapshot, + findings []SecretFinding, + update SessionSignalUpdate, + state SessionSignalState, +) (bool, error) { + db.mu.Lock() + defer db.mu.Unlock() + tx, err := db.getWriter().Begin() + if err != nil { + return false, fmt.Errorf("beginning conditional signal recompute tx: %w", err) + } + defer func() { _ = tx.Rollback() }() + matches, err := sessionSignalInputSnapshotMatchesTx( + tx, sessionID, expected, + ) + if err != nil || !matches { + return false, err + } + if err := replaceSecretFindingsTx( + tx, sessionID, findings, update.SecretLeakCount, + update.SecretsRulesVersion, + ); err != nil { + return false, err + } + if err := updateSessionSignalsTx(tx, sessionID, update); err != nil { + return false, err + } + state.SessionID = sessionID + state.TranscriptRevision = expected.TranscriptRevision + if err := upsertSessionSignalStateTx(tx, state); err != nil { + return false, err + } + if err := tx.Commit(); err != nil { + return false, fmt.Errorf( + "committing conditional signal recompute %s: %w", sessionID, err, + ) + } + return true, nil +} + +// ReplaceSessionSignalsIfRevision is retained for callers that only have a +// transcript token. New full-recompute code should use +// ReplaceSessionSignalsIfInputsMatch so metadata-only races are rejected too. +func (db *DB) ReplaceSessionSignalsIfRevision( + sessionID, expectedRevision string, + findings []SecretFinding, + update SessionSignalUpdate, + state SessionSignalState, +) (bool, error) { + sess, err := db.GetSessionFull(context.Background(), sessionID) + if err != nil { + return false, err + } + if sess == nil || sess.TranscriptRevision == nil || + *sess.TranscriptRevision != expectedRevision { + return false, nil + } + snapshot, err := SignalInputSnapshot(*sess) + if err != nil { + return false, err + } + return db.ReplaceSessionSignalsIfInputsMatch( + sessionID, snapshot, findings, update, state, + ) +} + +func sessionSignalInputSnapshotMatchesTx( + tx *sql.Tx, sessionID string, + expected SessionSignalInputSnapshot, +) (bool, error) { + var ( + currentRevision string + messageCount int + isAutomated int + endedAt sql.NullString + peakTokens int + hasPeak int + ) + err := tx.QueryRow(` + SELECT transcript_revision, message_count, is_automated, ended_at, + peak_context_tokens, has_peak_context_tokens + FROM sessions WHERE id = ?`, sessionID, + ).Scan( + ¤tRevision, &messageCount, &isAutomated, &endedAt, + &peakTokens, &hasPeak, + ) + if errors.Is(err, sql.ErrNoRows) { + return false, nil + } + if err != nil { + return false, fmt.Errorf( + "loading signal input snapshot %s: %w", sessionID, err, + ) + } + current := SessionSignalInputSnapshot{ + TranscriptRevision: currentRevision, + MessageCount: messageCount, + IsAutomated: isAutomated != 0, + EndedAt: endedAt.String, + HasEndedAt: endedAt.Valid, + PeakContextTokens: peakTokens, + HasPeakContextTokens: hasPeak != 0, + } + return current == expected, nil +} + +func sessionTranscriptRevisionMatchesTx( + tx *sql.Tx, sessionID, expectedRevision string, +) (bool, error) { + var current string + err := tx.QueryRow( + `SELECT transcript_revision FROM sessions WHERE id = ?`, sessionID, + ).Scan(¤t) + if errors.Is(err, sql.ErrNoRows) { + return false, nil + } + if err != nil { + return false, fmt.Errorf( + "loading transcript revision %s: %w", sessionID, err, + ) + } + return current == expectedRevision, nil +} + +func applySignalDeltaTx( + tx *sql.Tx, sessionID string, d SignalDelta, +) error { + // Remove stale findings by natural coordinates, counting definite + // removals toward the leak-count adjustment. + deletedDefinite := 0 + for _, key := range d.DeleteFindingKeys { + var n int + if err := tx.QueryRow(` + SELECT COUNT(*) FROM secret_findings + WHERE session_id = ? AND message_ordinal = ? + AND COALESCE(call_index, -1) = ? + AND location_kind = ? AND confidence = ?`, + sessionID, key.MessageOrdinal, key.CallIndex, + key.LocationKind, secrets.ConfidenceDefinite, + ).Scan(&n); err != nil { + return fmt.Errorf( + "counting removed findings %s: %w", sessionID, err, + ) + } + deletedDefinite += n + if _, err := tx.Exec(` + DELETE FROM secret_findings + WHERE session_id = ? AND message_ordinal = ? + AND COALESCE(call_index, -1) = ? + AND location_kind = ?`, + sessionID, key.MessageOrdinal, key.CallIndex, + key.LocationKind, + ); err != nil { + return fmt.Errorf( + "deleting findings %s: %w", sessionID, err, + ) + } + } + + // Insert new findings, skipping rows an equivalent finding already + // covers (idempotent replays of a result update). RowsAffected tells + // which inserts actually landed, so the definite-leak adjustment + // counts each finding once. + addedDefinite := 0 + for i := range d.InsertFindings { + f := &d.InsertFindings[i] + // Unlike replaceSecretFindingsTx (which overrides RulesVersion + // with the caller's stamp), this path inserts f.RulesVersion + // verbatim. Default an un-stamped finding to the current definite + // version so it stays visible to current-version listings. + if f.RulesVersion == "" { + f.RulesVersion = secrets.DefiniteRulesVersion() + } + res, err := tx.Exec(` + INSERT INTO secret_findings ( + session_id, rule_name, confidence, + location_kind, message_ordinal, call_index, event_index, + match_start, match_end, match_index, + redacted_match, rules_version + ) + SELECT ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? + WHERE NOT EXISTS ( + SELECT 1 FROM secret_findings + WHERE session_id = ? AND rule_name = ? + AND location_kind = ? AND message_ordinal = ? + AND COALESCE(call_index, -1) = ? + AND COALESCE(event_index, -1) = ? + AND match_start = ? AND match_end = ? + )`, + sessionID, f.RuleName, f.Confidence, + f.LocationKind, f.MessageOrdinal, f.CallIndex, f.EventIndex, + f.MatchStart, f.MatchEnd, f.MatchIndex, + f.RedactedMatch, f.RulesVersion, + sessionID, f.RuleName, + f.LocationKind, f.MessageOrdinal, + coalesceInt(f.CallIndex, -1), + coalesceInt(f.EventIndex, -1), + f.MatchStart, f.MatchEnd, + ) + if err != nil { + return fmt.Errorf("inserting secret finding: %w", err) + } + n, err := res.RowsAffected() + if err != nil { + return fmt.Errorf("counting inserted finding: %w", err) + } + if n > 0 && f.Confidence == secrets.ConfidenceDefinite { + addedDefinite++ + } + } + + // Adjust the leak count from the stored value. + var currentLeak int + if err := tx.QueryRow( + `SELECT secret_leak_count FROM sessions WHERE id = ?`, + sessionID, + ).Scan(¤tLeak); err != nil { + return fmt.Errorf( + "reading leak count %s: %w", sessionID, err, + ) + } + newLeak := currentLeak - deletedDefinite + addedDefinite + newLeak = max(newLeak, 0) + + if err := updateSessionSignalsTx(tx, sessionID, d.Update); err != nil { + return err + } + if _, err := tx.Exec(` + UPDATE sessions + SET secret_leak_count = ?, secrets_rules_version = ? + WHERE id = ?`, + newLeak, d.Update.SecretsRulesVersion, sessionID, + ); err != nil { + return fmt.Errorf( + "updating session secret columns %s: %w", sessionID, err, + ) + } + if d.State != nil { + if err := upsertSessionSignalStateTx(tx, *d.State); err != nil { + return err + } + } + return nil +} + +func coalesceInt(p *int, fallback int) int { + if p == nil { + return fallback + } + return *p +} + +func upsertSessionSignalStateTx( + tx *sql.Tx, st SessionSignalState, +) error { + if st.UpdatedAt == "" { + st.UpdatedAt = time.Now().UTC().Format(time.RFC3339) + } + if _, err := tx.Exec(` + INSERT INTO session_signal_state ( + session_id, state, transcript_revision, signal_version, + updated_at + ) VALUES (?, ?, ?, ?, ?) + ON CONFLICT(session_id) DO UPDATE SET + state = excluded.state, + transcript_revision = excluded.transcript_revision, + signal_version = excluded.signal_version, + updated_at = excluded.updated_at`, + st.SessionID, st.State, st.TranscriptRevision, + st.SignalVersion, st.UpdatedAt, + ); err != nil { + return fmt.Errorf( + "upserting session signal state %s: %w", st.SessionID, err, + ) + } + return nil +} diff --git a/internal/db/signal_snapshot_test.go b/internal/db/signal_snapshot_test.go new file mode 100644 index 000000000..a0a7a037d --- /dev/null +++ b/internal/db/signal_snapshot_test.go @@ -0,0 +1,139 @@ +package db + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestReplaceSessionSignalsIfRevisionRejectsStaleSnapshot(t *testing.T) { + d := testDB(t) + insertSession(t, d, "signal-race", "proj") + + sess, err := d.GetSessionFull(context.Background(), "signal-race") + require.NoError(t, err) + require.NotNil(t, sess) + require.NotNil(t, sess.TranscriptRevision) + currentRevision := *sess.TranscriptRevision + initialOutcome := sess.Outcome + + update := SessionSignalUpdate{ + Outcome: "completed", + OutcomeConfidence: "high", + SecretLeakCount: 1, + SecretsRulesVersion: "rules-v1", + QualitySignals: QualitySignals{ + Version: CurrentQualitySignalVersion, + }, + } + finding := SecretFinding{ + RuleName: "test-secret", + Confidence: "definite", + LocationKind: "message", + MessageOrdinal: 0, + MatchEnd: 4, + RedactedMatch: "****", + RulesVersion: "rules-v1", + } + state := SessionSignalState{ + State: []byte("stale-state"), + SignalVersion: CurrentQualitySignalVersion, + } + + applied, err := d.ReplaceSessionSignalsIfRevision( + "signal-race", currentRevision+"-stale", []SecretFinding{finding}, + update, state, + ) + require.NoError(t, err) + require.False(t, applied) + + afterReject, err := d.GetSessionFull(context.Background(), "signal-race") + require.NoError(t, err) + require.Equal(t, initialOutcome, afterReject.Outcome) + _, ok, err := d.GetSessionSignalState("signal-race") + require.NoError(t, err) + require.False(t, ok) + findings, err := d.SessionSecretFindings(context.Background(), "signal-race") + require.NoError(t, err) + require.Empty(t, findings) + + applied, err = d.ReplaceSessionSignalsIfRevision( + "signal-race", currentRevision, []SecretFinding{finding}, update, state, + ) + require.NoError(t, err) + require.True(t, applied) + + afterApply, err := d.GetSessionFull(context.Background(), "signal-race") + require.NoError(t, err) + require.Equal(t, "completed", afterApply.Outcome) + storedState, ok, err := d.GetSessionSignalState("signal-race") + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, currentRevision, storedState.TranscriptRevision) + require.Equal(t, []byte("stale-state"), storedState.State) + findings, err = d.SessionSecretFindings(context.Background(), "signal-race") + require.NoError(t, err) + require.Len(t, findings, 1) +} + +func TestReplaceSessionSignalsIfInputsMatchRejectsMetadataOnlyRace(t *testing.T) { + d := testDB(t) + insertSession(t, d, "signal-metadata-race", "proj") + + sess, err := d.GetSessionFull(context.Background(), "signal-metadata-race") + require.NoError(t, err) + require.NotNil(t, sess) + expected, err := SignalInputSnapshot(*sess) + require.NoError(t, err) + initialOutcome := sess.Outcome + + endedAt := "2026-08-18T12:00:00Z" + _, err = d.getWriter().Exec(` + UPDATE sessions + SET ended_at = ?, is_automated = 1, message_count = 7, + peak_context_tokens = 12345, has_peak_context_tokens = 1 + WHERE id = ?`, endedAt, "signal-metadata-race") + require.NoError(t, err) + + update := SessionSignalUpdate{ + Outcome: "stale-result", OutcomeConfidence: "high", + SecretsRulesVersion: "rules-v1", + QualitySignals: QualitySignals{Version: CurrentQualitySignalVersion}, + } + state := SessionSignalState{ + State: []byte("stale-state"), SignalVersion: CurrentQualitySignalVersion, + } + applied, err := d.ReplaceSessionSignalsIfInputsMatch( + "signal-metadata-race", expected, nil, update, state, + ) + require.NoError(t, err) + require.False(t, applied, + "metadata-only changes must invalidate a full signal snapshot") + + afterReject, err := d.GetSessionFull(context.Background(), "signal-metadata-race") + require.NoError(t, err) + require.Equal(t, initialOutcome, afterReject.Outcome) + _, ok, err := d.GetSessionSignalState("signal-metadata-race") + require.NoError(t, err) + require.False(t, ok) + + fresh, err := SignalInputSnapshot(*afterReject) + require.NoError(t, err) + update.Outcome = "fresh-result" + state.State = []byte("fresh-state") + applied, err = d.ReplaceSessionSignalsIfInputsMatch( + "signal-metadata-race", fresh, nil, update, state, + ) + require.NoError(t, err) + require.True(t, applied) + + afterApply, err := d.GetSessionFull(context.Background(), "signal-metadata-race") + require.NoError(t, err) + require.Equal(t, "fresh-result", afterApply.Outcome) + stored, ok, err := d.GetSessionSignalState("signal-metadata-race") + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, fresh.TranscriptRevision, stored.TranscriptRevision) + require.Equal(t, []byte("fresh-state"), stored.State) +} diff --git a/internal/db/signals_test.go b/internal/db/signals_test.go index fffc696ef..d6f5edf25 100644 --- a/internal/db/signals_test.go +++ b/internal/db/signals_test.go @@ -277,13 +277,14 @@ func TestMessageWritesInvalidateQualitySignalVersion(t *testing.T) { { name: "WriteSessionIncremental", write: func(t *testing.T, d *DB, id string) { - require.NoError(t, d.WriteSessionIncremental(id, + _, werr := d.WriteSessionIncremental(id, []Message{{SessionID: id, Ordinal: 1, Role: "user", Content: "appended"}}, IncrementalSessionUpdate{ MsgCount: 2, UserMsgCount: 2, NextOrdinal: 2, }, - ), "WriteSessionIncremental") + ) + require.NoError(t, werr, "WriteSessionIncremental") }, }, { diff --git a/internal/db/staged_content.go b/internal/db/staged_content.go new file mode 100644 index 000000000..111998e25 --- /dev/null +++ b/internal/db/staged_content.go @@ -0,0 +1,630 @@ +package db + +import ( + "context" + "crypto/sha256" + "database/sql" + "database/sql/driver" + "encoding/binary" + "errors" + "fmt" + "log" + "strconv" +) + +// StagedToolResults is the publish-side handle for tool-result rows staged +// during a streaming parse. The staged write inserts event rows straight +// from the handle and resolves per-call result summaries with transient +// memory, so neither the event contents nor the summaries ever live as one +// full in-memory session slice. +type StagedToolResults interface { + // ResolveSummary returns the stored result summary and its length for + // one tool call, mirroring SummarizeToolResultEvents over the staged + // events. + ResolveSummary( + ctx context.Context, toolUseID string, + ) (string, int, error) + // InsertEventsTx inserts the staged rows for the whole session into + // tool_result_events within tx, keyed by tool_use_id through the + // final message coordinates in positions. The scratch database is + // already attached to the tx's connection by the caller. + InsertEventsTx( + ctx context.Context, + tx *sql.Tx, + sessionID string, + positions map[string]StagedToolCallPosition, + ) error + // Path returns the staging storage path for ATTACH-based publishing. + Path() string + // Close releases the staging storage. + Close() error +} + +// StagedSignalsFunc computes the final signal update and secret findings +// for a staged session once every per-call summary has been resolved in +// the publish transaction. verdicts carries the per-call content-failure +// verdicts the summary resolution recorded, so the returned values can be +// persisted atomically with the message, tool-call, and event rows instead +// of being recomputed after commit. +type StagedSignalsFunc func( + verdicts map[string]bool, +) (SessionSignalUpdate, []SecretFinding, error) + +// StagedToolCallPosition identifies one tool call's final coordinates. +type StagedToolCallPosition struct { + ToolUseID string + Ordinal int + CallIndex int +} + +// StagedToolCallKey returns the internal staging key for one occurrence of a +// provider call ID. Provider call IDs are not guaranteed unique within a +// transcript, so staged rows use occurrence identity while the published +// tool_use_id remains unchanged. +func StagedToolCallKey(toolUseID string, occurrence int) string { + // Use a printable, length-prefixed key. Embedded NUL bytes are legal in + // Go strings but are not a safe SQLite TEXT identity across every + // go-sqlite3 binding path: a bound value may be observed only up to the + // first NUL on a later connection. The byte length keeps the encoding + // unambiguous even when the provider call ID itself contains colons. + return strconv.Itoa(len(toolUseID)) + ":" + toolUseID + ":" + + strconv.Itoa(occurrence) +} + +// stagedAttachName is the schema name the scratch database is attached +// under for the publish transaction. +const stagedAttachName = "codex_staging" + +// ReplaceSessionContentStaged replaces a session's content from a +// streaming parse: messages and tool-call metadata arrive as the (small) +// in-memory slice whose result events carry placeholders, while the event +// rows and per-call summaries come from the staged handle. Blocked +// categories are blanked exactly like the legacy write path. This is the +// atomic publish of the staged cold-import path; it mirrors +// ReplaceSessionContent's transaction sequence with the event/summary +// inserts replaced by staged sources. +// +// The scratch database is attached to the writer connection before the +// transaction begins and detached after it settles, so consecutive staged +// publishes on the same single-connection writer pool never collide with +// a leftover attachment. signalsFn runs inside the transaction after all +// summaries are resolved, so the persisted signals and findings carry the +// real content-failure verdicts in the same atomic commit as the rows +// they describe. +func (db *DB) ReplaceSessionContentStaged( + ctx context.Context, + sessionID string, msgs []Message, + staged StagedToolResults, blocked map[string]bool, + signalsFn StagedSignalsFunc, +) error { + return db.replaceSessionContentStaged( + ctx, sessionID, msgs, staged, blocked, signalsFn, nil, nil, + ) +} + +// ReplaceSessionContentStagedWithCheckpoint is the staged publish plus an +// in-transaction parser checkpoint upsert, so content and resume state +// commit atomically. +func (db *DB) ReplaceSessionContentStagedWithCheckpoint( + ctx context.Context, + sessionID string, msgs []Message, + staged StagedToolResults, blocked map[string]bool, + signalsFn StagedSignalsFunc, + cp *ParserCheckpoint, blobs *ParserCheckpointBlobs, +) error { + return db.replaceSessionContentStaged( + ctx, sessionID, msgs, staged, blocked, signalsFn, cp, blobs, + ) +} + +func writeStagedDigestString(h interface{ Write([]byte) (int, error) }, value string) { + var size [8]byte + binary.LittleEndian.PutUint64(size[:], uint64(len(value))) + _, _ = h.Write(size[:]) + _, _ = h.Write([]byte(value)) +} + +func writeStagedDigestInt(h interface{ Write([]byte) (int, error) }, value int64) { + var encoded [8]byte + binary.LittleEndian.PutUint64(encoded[:], uint64(value)) + _, _ = h.Write(encoded[:]) +} + +// stagedSessionHasStoredMessagesTx reports whether sessionID already has any +// stored message rows. tool_calls and tool_result_events are always inserted +// alongside the message they belong to, so a session with no message rows +// can never have any content stagedSessionContentDigestTx would find either. +// A cold staged import can use this to skip both full-table digest scans +// entirely instead of proving byte-for-byte equality against nothing. +func stagedSessionHasStoredMessagesTx( + tx *sql.Tx, sessionID string, +) (bool, error) { + var exists int + err := tx.QueryRow( + `SELECT 1 FROM messages WHERE session_id = ? LIMIT 1`, sessionID, + ).Scan(&exists) + if errors.Is(err, sql.ErrNoRows) { + return false, nil + } + if err != nil { + return false, fmt.Errorf( + "checking existing staged content for %s: %w", sessionID, err, + ) + } + return true, nil +} + +// stagedSessionContentDigestTx hashes every parser-owned transcript field while +// ignoring SQLite row IDs. It lets a forced staged verification prove that the +// normalized content is unchanged and avoid delete/reinsert revision churn. +func stagedSessionContentDigestTx( + tx *sql.Tx, sessionID string, +) ([sha256.Size]byte, error) { + h := sha256.New() + rows, err := tx.Query(` + SELECT ordinal, role, content, thinking_text, COALESCE(timestamp, ''), + has_thinking, has_tool_use, content_length, is_system, model, + token_usage, context_tokens, output_tokens, + has_context_tokens, has_output_tokens, + claude_message_id, claude_request_id, source_type, + source_subtype, prompt_source, source_uuid, + source_parent_uuid, is_sidechain, is_compact_boundary + FROM messages WHERE session_id = ? ORDER BY ordinal`, sessionID) + if err != nil { + return [sha256.Size]byte{}, err + } + for rows.Next() { + var ordinal, hasThinking, hasToolUse, contentLength, isSystem int64 + var contextTokens, outputTokens, hasContext, hasOutput int64 + var isSidechain, isCompact int64 + var role, content, thinking, timestamp, model, tokenUsage string + var claudeMessageID, claudeRequestID, sourceType, sourceSubtype string + var promptSource, sourceUUID, sourceParentUUID string + if err := rows.Scan( + &ordinal, &role, &content, &thinking, ×tamp, + &hasThinking, &hasToolUse, &contentLength, &isSystem, &model, + &tokenUsage, &contextTokens, &outputTokens, &hasContext, &hasOutput, + &claudeMessageID, &claudeRequestID, &sourceType, &sourceSubtype, + &promptSource, &sourceUUID, &sourceParentUUID, &isSidechain, &isCompact, + ); err != nil { + rows.Close() + return [sha256.Size]byte{}, err + } + writeStagedDigestString(h, "message") + for _, value := range []int64{ + ordinal, hasThinking, hasToolUse, contentLength, isSystem, + contextTokens, outputTokens, hasContext, hasOutput, + isSidechain, isCompact, + } { + writeStagedDigestInt(h, value) + } + for _, value := range []string{ + role, content, thinking, timestamp, model, tokenUsage, + claudeMessageID, claudeRequestID, sourceType, sourceSubtype, + promptSource, sourceUUID, sourceParentUUID, + } { + writeStagedDigestString(h, value) + } + } + if err := rows.Err(); err != nil { + rows.Close() + return [sha256.Size]byte{}, err + } + if err := rows.Close(); err != nil { + return [sha256.Size]byte{}, err + } + + rows, err = tx.Query(` + SELECT m.ordinal, COALESCE(tc.call_index, 0), tc.tool_name, tc.category, + COALESCE(tc.tool_use_id, ''), COALESCE(tc.input_json, ''), + COALESCE(tc.skill_name, ''), + COALESCE(tc.result_content_length, 0), + COALESCE(tc.result_content, ''), + COALESCE(tc.subagent_session_id, ''), COALESCE(tc.file_path, '') + FROM tool_calls tc JOIN messages m ON m.id = tc.message_id + WHERE tc.session_id = ? + ORDER BY m.ordinal, COALESCE(tc.call_index, 0), tc.id`, sessionID) + if err != nil { + return [sha256.Size]byte{}, err + } + for rows.Next() { + var ordinal, callIndex, resultLength int64 + var toolName, category, toolUseID, inputJSON, skillName string + var resultContent, subagentSessionID, filePath string + if err := rows.Scan( + &ordinal, &callIndex, &toolName, &category, &toolUseID, + &inputJSON, &skillName, &resultLength, &resultContent, + &subagentSessionID, &filePath, + ); err != nil { + rows.Close() + return [sha256.Size]byte{}, err + } + writeStagedDigestString(h, "tool_call") + writeStagedDigestInt(h, ordinal) + writeStagedDigestInt(h, callIndex) + writeStagedDigestInt(h, resultLength) + for _, value := range []string{ + toolName, category, toolUseID, inputJSON, skillName, + resultContent, subagentSessionID, filePath, + } { + writeStagedDigestString(h, value) + } + } + if err := rows.Err(); err != nil { + rows.Close() + return [sha256.Size]byte{}, err + } + if err := rows.Close(); err != nil { + return [sha256.Size]byte{}, err + } + + rows, err = tx.Query(` + SELECT tool_call_message_ordinal, call_index, + COALESCE(tool_use_id, ''), COALESCE(agent_id, ''), + COALESCE(subagent_session_id, ''), source, status, content, + content_length, COALESCE(timestamp, ''), event_index + FROM tool_result_events WHERE session_id = ? + ORDER BY tool_call_message_ordinal, call_index, event_index, id`, sessionID) + if err != nil { + return [sha256.Size]byte{}, err + } + for rows.Next() { + var ordinal, callIndex, contentLength, eventIndex int64 + var toolUseID, agentID, subagentID, source, status, content, timestamp string + if err := rows.Scan( + &ordinal, &callIndex, &toolUseID, &agentID, &subagentID, + &source, &status, &content, &contentLength, ×tamp, &eventIndex, + ); err != nil { + rows.Close() + return [sha256.Size]byte{}, err + } + writeStagedDigestString(h, "event") + writeStagedDigestInt(h, ordinal) + writeStagedDigestInt(h, callIndex) + writeStagedDigestInt(h, contentLength) + writeStagedDigestInt(h, eventIndex) + for _, value := range []string{ + toolUseID, agentID, subagentID, source, status, content, timestamp, + } { + writeStagedDigestString(h, value) + } + } + if err := rows.Err(); err != nil { + rows.Close() + return [sha256.Size]byte{}, err + } + if err := rows.Close(); err != nil { + return [sha256.Size]byte{}, err + } + + var digest [sha256.Size]byte + copy(digest[:], h.Sum(nil)) + return digest, nil +} + +func commitStagedDerivedStateAndCheckpoint( + ctx context.Context, conn *sql.Conn, sessionID string, + signals SessionSignalUpdate, findings []SecretFinding, + cp *ParserCheckpoint, blobs *ParserCheckpointBlobs, +) error { + tx, err := conn.BeginTx(ctx, nil) + if err != nil { + return err + } + defer func() { _ = tx.Rollback() }() + // The staged transcript is byte-for-byte identical, so preserve every + // message/tool row and the transcript revision. Metadata, detector rules, + // and prior failed post-processing may still have changed; refresh every + // derived projection in one transaction with the checkpoint. + if err := resetIncrementalMarkerTx(tx, sessionID); err != nil { + return err + } + if err := updateSessionAutomationFromMessagesTx(tx, sessionID); err != nil { + return err + } + if err := updateSessionSignalsTx(tx, sessionID, signals); err != nil { + return err + } + if err := replaceSecretFindingsTx( + tx, sessionID, findings, + signals.SecretLeakCount, signals.SecretsRulesVersion, + ); err != nil { + return err + } + if cp == nil || blobs == nil { + if err := deleteParserCheckpointTx(tx, sessionID); err != nil { + return err + } + } else { + c := *cp + b := *blobs + c.SessionID = sessionID + b.SessionID = sessionID + if err := upsertParserCheckpointTx(tx, c, b); err != nil { + return err + } + } + return tx.Commit() +} + +func (db *DB) replaceSessionContentStaged( + ctx context.Context, + sessionID string, msgs []Message, + staged StagedToolResults, blocked map[string]bool, + signalsFn StagedSignalsFunc, + cp *ParserCheckpoint, blobs *ParserCheckpointBlobs, +) error { + db.mu.Lock() + defer db.mu.Unlock() + + conn, err := db.getWriter().Conn(ctx) + if err != nil { + return fmt.Errorf("pinning writer connection: %w", err) + } + defer conn.Close() + if _, err := conn.ExecContext( + ctx, + "ATTACH DATABASE ? AS "+stagedAttachName, + staged.Path(), + ); err != nil { + return fmt.Errorf("attaching codex staging db: %w", err) + } + defer detachStagedConn(ctx, conn) + + tx, err := conn.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("beginning tx: %w", err) + } + defer func() { _ = tx.Rollback() }() + + queueGenerationBefore, queueExistedBefore, err := artifactExportGenerationTx( + tx, sessionID, + ) + if err != nil { + return err + } + // A cold import has no stored rows for stagedSessionContentDigestTx to + // find, so contentBefore can only ever prove "changed" against + // contentAfter -- skip both full-table scans and go straight to the + // changed-content path below instead of paying to read back every byte + // this same transaction is about to write. + hadStoredContent, err := stagedSessionHasStoredMessagesTx(tx, sessionID) + if err != nil { + return err + } + var contentBefore [sha256.Size]byte + if hadStoredContent { + contentBefore, err = stagedSessionContentDigestTx(tx, sessionID) + if err != nil { + return fmt.Errorf("fingerprinting stored staged content: %w", err) + } + } + var pendingRecallRevocations recallEvidenceRevocationEvents + + if err := replaceSessionMessagesTxStaged( + tx, sessionID, msgs, staged, blocked, + ); err != nil { + return err + } + contentUnchanged := false + if hadStoredContent { + contentAfter, err := stagedSessionContentDigestTx(tx, sessionID) + if err != nil { + return fmt.Errorf("fingerprinting proposed staged content: %w", err) + } + contentUnchanged = contentBefore == contentAfter + } + if contentUnchanged { + // Keep the existing row identities, transcript revision, and recall + // evidence. The session row may have received metadata-only updates + // before this publish, and detector rules may have advanced, so derived + // state still has to be refreshed from the verified staged snapshot. + signals, findings, err := signalsFn(contentFailureVerdicts(staged)) + if err != nil { + return fmt.Errorf( + "computing unchanged staged signals for %s: %w", + sessionID, err, + ) + } + if err := tx.Rollback(); err != nil { + return err + } + return commitStagedDerivedStateAndCheckpoint( + ctx, conn, sessionID, signals, findings, cp, blobs, + ) + } + // Summary resolution above recorded per-call content-failure + // verdicts; fold them into the final signal update and findings now, + // inside the same transaction as the rows they describe. + signals, findings, err := signalsFn(contentFailureVerdicts(staged)) + if err != nil { + return fmt.Errorf( + "computing staged signals for %s: %w", sessionID, err, + ) + } + if err := bumpTranscriptRevisionTx(tx, sessionID); err != nil { + return err + } + if err := reconcileRecallEvidenceForSessionTx( + ctx, tx, sessionID, &pendingRecallRevocations, + ); err != nil { + return err + } + if err := resetIncrementalMarkerTx(tx, sessionID); err != nil { + return err + } + if err := updateSessionAutomationFromMessagesTx(tx, sessionID); err != nil { + return err + } + if err := updateSessionSignalsTx(tx, sessionID, signals); err != nil { + return err + } + if err := replaceSecretFindingsTx(tx, sessionID, findings, + signals.SecretLeakCount, signals.SecretsRulesVersion); err != nil { + return err + } + if err := enqueueArtifactExportIfGenerationUnchangedTx( + tx, sessionID, queueGenerationBefore, queueExistedBefore, + ); err != nil { + return err + } + if cp == nil || blobs == nil { + if err := deleteParserCheckpointTx(tx, sessionID); err != nil { + return err + } + } else { + c := *cp + b := *blobs + // The checkpoint row is keyed by session id just like the blobs; + // an idPrefix-rewritten publish must not strand the row under the + // parser-native id or collide with a local session sharing it. + c.SessionID = sessionID + b.SessionID = sessionID + if err := upsertParserCheckpointTx(tx, c, b); err != nil { + return err + } + } + if err := tx.Commit(); err != nil { + return err + } + pendingRecallRevocations.flush() + return nil +} + +// contentFailureVerdicts reads the per-call verdicts a staging handle +// captured during summary resolution. Handles that do not expose them +// yield nil, which signalsFn treats as an empty verdict set. +func contentFailureVerdicts(staged StagedToolResults) map[string]bool { + if v, ok := staged.(interface { + ContentFailures() map[string]bool + }); ok { + return v.ContentFailures() + } + return nil +} + +// detachStagedConn detaches the scratch database from the writer +// connection. It runs after the transaction settles (deferred), so the +// connection returns to the pool clean. A failed detach would poison the +// single-connection writer pool for every later publish, so the +// connection is discarded instead. +func detachStagedConn(ctx context.Context, conn *sql.Conn) { + if _, err := conn.ExecContext( + ctx, "DETACH DATABASE "+stagedAttachName, + ); err != nil { + log.Printf("detaching codex staging db: %v", err) + _ = conn.Raw(func(any) error { return driver.ErrBadConn }) + } +} + +// replaceSessionMessagesTxStaged mirrors replaceSessionMessagesTx with the +// tool-result event insert and summary pairing replaced by staged sources: +// tool_calls insert in bounded chunks with per-call summaries resolved on +// the fly, and the event rows come from the staged handle. +func replaceSessionMessagesTxStaged( + tx *sql.Tx, sessionID string, msgs []Message, + staged StagedToolResults, blocked map[string]bool, +) error { + pins, err := savePinsTx(tx, sessionID) + if err != nil { + return err + } + if err := deleteSessionMessagesTx(tx, sessionID); err != nil { + return err + } + if len(msgs) == 0 { + return restorePinsTx(tx, sessionID, pins) + } + + ids, err := insertMessagesTx(tx, msgs) + if err != nil { + return err + } + + positions := make(map[string]StagedToolCallPosition) + callOccurrences := make(map[string]int) + chunk := make([]ToolCall, 0, toolCallStagedChunkSize) + var chunkBytes int64 + flush := func() error { + if len(chunk) == 0 { + return nil + } + if err := insertToolCallsChunkTx(tx, chunk); err != nil { + return err + } + chunk = chunk[:0] + chunkBytes = 0 + return nil + } + for i, m := range msgs { + for callIdx := range m.ToolCalls { + tc := ToolCall{ + MessageID: ids[i], + SessionID: m.SessionID, + ToolName: m.ToolCalls[callIdx].ToolName, + Category: m.ToolCalls[callIdx].Category, + ToolUseID: m.ToolCalls[callIdx].ToolUseID, + InputJSON: m.ToolCalls[callIdx].InputJSON, + SkillName: m.ToolCalls[callIdx].SkillName, + SubagentSessionID: m.ToolCalls[callIdx].SubagentSessionID, + FilePath: m.ToolCalls[callIdx].FilePath, + CallIndex: callIdx, + } + if tc.ToolUseID != "" { + occurrence := callOccurrences[tc.ToolUseID] + callOccurrences[tc.ToolUseID] = occurrence + 1 + stageKey := StagedToolCallKey(tc.ToolUseID, occurrence) + positions[stageKey] = StagedToolCallPosition{ + ToolUseID: tc.ToolUseID, + Ordinal: m.Ordinal, + CallIndex: callIdx, + } + summary, length, err := staged.ResolveSummary( + context.Background(), stageKey, + ) + if err != nil { + return fmt.Errorf( + "resolving staged summary for %s/%s: %w", + sessionID, tc.ToolUseID, err, + ) + } + tc.ResultContentLength = length + if !blocked[tc.Category] { + tc.ResultContent = summary + } + chunkBytes += int64(length) + } + chunk = append(chunk, tc) + // Flush by byte budget as well as count: resolved summaries + // are the largest per-call strings, and a count-only bound + // (toolCallStagedChunkSize) would still accumulate up to + // count * max-summary-size bytes of resolved content before + // the insert. + if len(chunk) >= toolCallStagedChunkSize || + chunkBytes >= toolCallStagedChunkBytes { + if err := flush(); err != nil { + return err + } + } + } + } + if err := flush(); err != nil { + return err + } + + if err := staged.InsertEventsTx( + context.Background(), tx, sessionID, positions, + ); err != nil { + return err + } + return restorePinsTx(tx, sessionID, pins) +} + +// toolCallStagedChunkSize bounds the tool-call insert chunks so the +// transient per-chunk summary memory stays fixed. toolCallStagedChunkBytes +// bounds the same chunks by resolved summary content bytes, since one +// call's summary can dwarf hundreds of ordinary rows. +const ( + toolCallStagedChunkSize = 500 + toolCallStagedChunkBytes = 16 << 20 +) diff --git a/internal/db/staged_content_test.go b/internal/db/staged_content_test.go new file mode 100644 index 000000000..3a0187c10 --- /dev/null +++ b/internal/db/staged_content_test.go @@ -0,0 +1,550 @@ +package db + +import ( + "context" + "database/sql" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestStagedToolCallKeyIsSQLiteTextSafeAndUnambiguous(t *testing.T) { + key := StagedToolCallKey("call_a", 0) + require.NotContains(t, key, "\x00") + require.True(t, strings.HasPrefix(key, "6:call_a:")) + + keys := map[string]struct{}{ + StagedToolCallKey("a", 12): {}, + StagedToolCallKey("a1", 2): {}, + StagedToolCallKey("a:1", 2): {}, + StagedToolCallKey("a", 1): {}, + StagedToolCallKey("a", 2): {}, + StagedToolCallKey("a:1", 20): {}, + } + require.Len(t, keys, 6) +} + +// TestStagedSessionHasStoredMessagesTx pins the fast-path check a cold +// staged import uses to skip both of stagedSessionContentDigestTx's +// full-table scans: a session with no message rows yet must report false, +// and gains true as soon as any message row exists, regardless of whether +// that row carries a tool call. +func TestStagedSessionHasStoredMessagesTx(t *testing.T) { + d := testDB(t) + insertSession(t, d, "s1", "proj") + + tx, err := d.getWriter().BeginTx(context.Background(), nil) + require.NoError(t, err) + has, err := stagedSessionHasStoredMessagesTx(tx, "s1") + require.NoError(t, err) + require.False(t, has, "a session with no stored messages must report false") + require.NoError(t, tx.Rollback()) + + insertMessages(t, d, Message{ + SessionID: "s1", Ordinal: 0, Role: "user", Content: "hi", + }) + + tx, err = d.getWriter().BeginTx(context.Background(), nil) + require.NoError(t, err) + has, err = stagedSessionHasStoredMessagesTx(tx, "s1") + require.NoError(t, err) + require.True(t, has, "a session with a stored message must report true") + + has, err = stagedSessionHasStoredMessagesTx(tx, "codex:never-synced") + require.NoError(t, err) + require.False(t, has, "an unrelated session id must not report true") + require.NoError(t, tx.Rollback()) +} + +// scratchStagedResults is a minimal StagedToolResults backed by a real +// scratch SQLite file, so the publish transaction's ATTACH and +// INSERT..SELECT run against genuine cross-database SQL. +type scratchStagedResults struct { + path string + db *sql.DB + seq int64 + closed bool +} + +func newScratchStagedResults(t *testing.T) *scratchStagedResults { + t.Helper() + f, err := os.CreateTemp(t.TempDir(), "staged-*.sqlite") + require.NoError(t, err) + path := f.Name() + require.NoError(t, f.Close()) + db, err := sql.Open("sqlite3", path) + require.NoError(t, err) + t.Cleanup(func() { _ = db.Close() }) + _, err = db.Exec(` + CREATE TABLE stage_events ( + seq INTEGER PRIMARY KEY, + tool_use_id TEXT NOT NULL, + agent_id TEXT NOT NULL DEFAULT '', + subagent_session_id TEXT NOT NULL DEFAULT '', + source TEXT NOT NULL, + status TEXT NOT NULL, + content TEXT NOT NULL, + content_length INTEGER NOT NULL, + timestamp TEXT NOT NULL DEFAULT '', + blanked INTEGER NOT NULL DEFAULT 0 + )`) + require.NoError(t, err) + return &scratchStagedResults{path: path, db: db} +} + +func (s *scratchStagedResults) AddEvent( + t *testing.T, toolUseID, content string, +) { + t.Helper() + s.seq++ + _, err := s.db.Exec( + `INSERT INTO stage_events ( + seq, tool_use_id, agent_id, subagent_session_id, + source, status, content, content_length, timestamp, blanked + ) VALUES (?, ?, '', '', 'function_call_output', '', + ?, ?, '', 0)`, + s.seq, toolUseID, content, len(content), + ) + require.NoError(t, err) +} + +func (s *scratchStagedResults) ResolveSummary( + context.Context, string, +) (string, int, error) { + return "", 0, nil +} + +func (s *scratchStagedResults) InsertEventsTx( + ctx context.Context, tx *sql.Tx, sessionID string, + positions map[string]StagedToolCallPosition, +) error { + for _, pos := range positions { + if _, err := tx.ExecContext(ctx, ` + INSERT INTO tool_result_events ( + session_id, tool_call_message_ordinal, call_index, + tool_use_id, agent_id, subagent_session_id, + source, status, content, content_length, + timestamp, event_index + ) + SELECT ?, ?, ?, tool_use_id, + CASE WHEN agent_id = '' THEN NULL ELSE agent_id END, + CASE WHEN subagent_session_id = '' + THEN NULL ELSE subagent_session_id END, + source, status, content, content_length, + CASE WHEN timestamp = '' THEN NULL ELSE timestamp END, + row_number() OVER (ORDER BY seq) - 1 + FROM codex_staging.stage_events + WHERE tool_use_id = ? + ORDER BY seq`, + sessionID, pos.Ordinal, pos.CallIndex, pos.ToolUseID, + ); err != nil { + return err + } + } + return nil +} + +func (s *scratchStagedResults) Path() string { return s.path } + +func (s *scratchStagedResults) Close() error { + s.closed = true + return nil +} + +// TestReplaceSessionContentStagedAttachLifecycle pins the ATTACH/DETACH +// contract: two consecutive staged publishes on the single-connection +// writer pool must both succeed, and after each one the writer connection +// must be free of the codex_staging schema. +func TestReplaceSessionContentStagedAttachLifecycle(t *testing.T) { + database, err := Open(filepath.Join(t.TempDir(), "staged.db")) + require.NoError(t, err) + t.Cleanup(func() { _ = database.Close() }) + + sessionID := "codex:test-session" + require.NoError(t, database.UpsertSession(Session{ + ID: sessionID, + Agent: "codex", + Project: "project", + Machine: "local", + MessageCount: 1, + UserMessageCount: 1, + })) + + publish := func(toolUseID, content string) { + t.Helper() + staged := newScratchStagedResults(t) + staged.AddEvent(t, toolUseID, content) + msgs := []Message{{ + SessionID: sessionID, + Ordinal: 0, + Role: "assistant", + Content: "running", + HasToolUse: true, + ToolCalls: []ToolCall{{ + ToolUseID: toolUseID, + ToolName: "exec_command", + Category: "Bash", + CallIndex: 0, + }}, + }} + require.NoError(t, database.ReplaceSessionContentStaged( + context.Background(), sessionID, msgs, staged, + map[string]bool{}, + func(map[string]bool) (SessionSignalUpdate, []SecretFinding, error) { + return SessionSignalUpdate{}, nil, nil + }, + )) + } + + // Two consecutive publishes on the same single-connection writer. + publish("call_a", "first output") + publish("call_b", "second output") + + // The writer connection must be clean after each publish: a leftover + // codex_staging attachment is what made the second publish fail. + rows, err := database.getWriter().Query("PRAGMA database_list") + require.NoError(t, err) + defer rows.Close() + for rows.Next() { + var seq int + var name, file string + require.NoError(t, rows.Scan(&seq, &name, &file)) + require.NotEqual(t, "codex_staging", name, + "staging schema must be detached after publish") + } + require.NoError(t, rows.Err()) + + msgs, err := database.GetAllMessages(context.Background(), sessionID) + require.NoError(t, err) + require.Len(t, msgs, 1) + require.Equal(t, "call_b", msgs[0].ToolCalls[0].ToolUseID) + require.Len(t, msgs[0].ToolCalls[0].ResultEvents, 1) + require.Equal(t, "second output", + msgs[0].ToolCalls[0].ResultEvents[0].Content) +} + +func TestReplaceSessionContentStagedIdenticalPublishKeepsRevision(t *testing.T) { + database, err := Open(filepath.Join(t.TempDir(), "staged-idempotent.db")) + require.NoError(t, err) + t.Cleanup(func() { _ = database.Close() }) + + const sessionID = "codex:idempotent" + require.NoError(t, database.UpsertSession(Session{ + ID: sessionID, Agent: "codex", Project: "project", Machine: "local", + MessageCount: 1, + })) + publish := func() { + t.Helper() + staged := newScratchStagedResults(t) + staged.AddEvent(t, "call_1", "same output") + require.NoError(t, database.ReplaceSessionContentStaged( + context.Background(), sessionID, []Message{{ + SessionID: sessionID, Ordinal: 0, Role: "assistant", + Content: "running", HasToolUse: true, + ToolCalls: []ToolCall{{ + SessionID: sessionID, ToolUseID: "call_1", + ToolName: "exec_command", Category: "Bash", CallIndex: 0, + }}, + }}, staged, map[string]bool{}, + func(map[string]bool) (SessionSignalUpdate, []SecretFinding, error) { + return SessionSignalUpdate{}, nil, nil + }, + )) + } + + publish() + first, err := database.GetSession(context.Background(), sessionID) + require.NoError(t, err) + require.NotNil(t, first) + require.NotNil(t, first.TranscriptRevision) + firstRevision := *first.TranscriptRevision + + publish() + second, err := database.GetSession(context.Background(), sessionID) + require.NoError(t, err) + require.NotNil(t, second) + require.NotNil(t, second.TranscriptRevision) + require.Equal(t, firstRevision, *second.TranscriptRevision, + "an identical staged verification must not bump transcript revision") +} + +func TestReplaceSessionContentStagedIdenticalPublishRefreshesDerivedState( + t *testing.T, +) { + database, err := Open(filepath.Join(t.TempDir(), "staged-derived.db")) + require.NoError(t, err) + t.Cleanup(func() { _ = database.Close() }) + + const sessionID = "codex:derived" + firstMessage := "Warmup" + require.NoError(t, database.UpsertSession(Session{ + ID: sessionID, Agent: "codex", Project: "project", Machine: "local", + FirstMessage: &firstMessage, MessageCount: 1, UserMessageCount: 1, + })) + + publish := func(outcome, rules string, finding bool) { + t.Helper() + staged := newScratchStagedResults(t) + staged.AddEvent(t, "call_1", "same output") + require.NoError(t, database.ReplaceSessionContentStaged( + context.Background(), sessionID, []Message{{ + SessionID: sessionID, Ordinal: 0, Role: "assistant", + Content: "running", HasToolUse: true, + ToolCalls: []ToolCall{{ + SessionID: sessionID, ToolUseID: "call_1", + ToolName: "exec_command", Category: "Bash", CallIndex: 0, + }}, + }}, staged, map[string]bool{}, + func(map[string]bool) (SessionSignalUpdate, []SecretFinding, error) { + update := SessionSignalUpdate{ + Outcome: outcome, OutcomeConfidence: "high", + SecretsRulesVersion: rules, + QualitySignals: QualitySignals{Version: CurrentQualitySignalVersion}, + } + if !finding { + return update, nil, nil + } + update.SecretLeakCount = 1 + return update, []SecretFinding{{ + SessionID: sessionID, RuleName: "refreshed-secret", + Confidence: "definite", LocationKind: "message", + MessageOrdinal: 0, MatchEnd: 4, + RedactedMatch: "****", RulesVersion: rules, + }}, nil + }, + )) + } + + publish("old", "old-rules", false) + first, err := database.GetSessionFull(context.Background(), sessionID) + require.NoError(t, err) + require.NotNil(t, first) + require.NotNil(t, first.TranscriptRevision) + firstRevision := *first.TranscriptRevision + + var firstMessageID, firstCallID, firstEventID int64 + require.NoError(t, database.getReader().QueryRow(` + SELECT m.id, tc.id, tre.id + FROM messages m + JOIN tool_calls tc ON tc.message_id = m.id + JOIN tool_result_events tre + ON tre.session_id = tc.session_id + AND tre.tool_call_message_ordinal = m.ordinal + AND tre.call_index = tc.call_index + WHERE m.session_id = ?`, sessionID, + ).Scan(&firstMessageID, &firstCallID, &firstEventID)) + + // Simulate metadata-only drift and stale post-processing while preserving + // the normalized transcript and its revision. + _, err = database.getWriter().Exec(` + UPDATE sessions + SET outcome = 'stale', quality_signal_version = 0, + secrets_rules_version = 'stale-rules', + last_write_incremental = 1, is_automated = 0 + WHERE id = ?`, sessionID) + require.NoError(t, err) + _, err = database.getWriter().Exec(` + INSERT INTO secret_findings ( + session_id, rule_name, confidence, location_kind, + message_ordinal, match_start, match_end, match_index, + redacted_match, rules_version + ) VALUES (?, 'stale-secret', 'definite', 'message', 0, 0, 1, 0, + '*', 'stale-rules')`, sessionID) + require.NoError(t, err) + + publish("completed", "fresh-rules", true) + + after, err := database.GetSessionFull(context.Background(), sessionID) + require.NoError(t, err) + require.NotNil(t, after) + require.NotNil(t, after.TranscriptRevision) + require.Equal(t, firstRevision, *after.TranscriptRevision, + "metadata-only staged repair must preserve transcript revision") + require.Equal(t, "completed", after.Outcome) + require.Equal(t, CurrentQualitySignalVersion, after.QualitySignalVersion) + require.Equal(t, "fresh-rules", after.SecretsRulesVersion) + require.Equal(t, 1, after.SecretLeakCount) + require.True(t, after.IsAutomated, + "unchanged staged publish must refresh automation from stored messages") + require.False(t, after.LastWriteIncremental, + "unchanged staged publish must clear the incremental marker") + + var messageID, callID, eventID int64 + require.NoError(t, database.getReader().QueryRow(` + SELECT m.id, tc.id, tre.id + FROM messages m + JOIN tool_calls tc ON tc.message_id = m.id + JOIN tool_result_events tre + ON tre.session_id = tc.session_id + AND tre.tool_call_message_ordinal = m.ordinal + AND tre.call_index = tc.call_index + WHERE m.session_id = ?`, sessionID, + ).Scan(&messageID, &callID, &eventID)) + require.Equal(t, firstMessageID, messageID) + require.Equal(t, firstCallID, callID) + require.Equal(t, firstEventID, eventID) + + var findingName, findingRules string + require.NoError(t, database.getReader().QueryRow(` + SELECT rule_name, rules_version FROM secret_findings + WHERE session_id = ?`, sessionID, + ).Scan(&findingName, &findingRules)) + require.Equal(t, "refreshed-secret", findingName) + require.Equal(t, "fresh-rules", findingRules) +} + +func TestReplaceSessionContentStagedWithCheckpointUsesPrefixedSessionID(t *testing.T) { + database, err := Open(filepath.Join(t.TempDir(), "staged-prefix.db")) + require.NoError(t, err) + t.Cleanup(func() { _ = database.Close() }) + + const storedID = "host:codex:native" + require.NoError(t, database.UpsertSession(Session{ + ID: storedID, + Agent: "codex", + Project: "project", + Machine: "local", + MessageCount: 1, + UserMessageCount: 1, + })) + + staged := newScratchStagedResults(t) + staged.AddEvent(t, "call_1", "output") + msgs := []Message{{ + SessionID: storedID, + Ordinal: 0, + Role: "assistant", + Content: "running", + HasToolUse: true, + ToolCalls: []ToolCall{{ + SessionID: storedID, + ToolUseID: "call_1", + ToolName: "exec_command", + Category: "Bash", + CallIndex: 0, + }}, + }} + cp := &ParserCheckpoint{ + SessionID: "codex:native", + Agent: "codex", + FilePath: "/sessions/rollout.jsonl", + FileInode: 1, + FileDevice: 1, + FileMTime: 1, + FileChangeTime: 1, + Offset: 8, + TailAnchorDigest: "anchor", + Hash: "hash", + NextOrdinal: 0, + Version: ParserCheckpointVersion, + } + blobs := &ParserCheckpointBlobs{ + SessionID: "codex:native", + Cursor: []byte("cursor"), + HashState: []byte("state"), + } + + err = database.ReplaceSessionContentStagedWithCheckpoint( + context.Background(), storedID, msgs, staged, + map[string]bool{}, + func(map[string]bool) (SessionSignalUpdate, []SecretFinding, error) { + return SessionSignalUpdate{}, nil, nil + }, + cp, blobs, + ) + require.NoError(t, err) + + var nativeCount, prefixedCount int + require.NoError(t, database.Reader().QueryRow( + `SELECT COUNT(*) FROM parser_checkpoints WHERE session_id = ?`, + "codex:native", + ).Scan(&nativeCount)) + require.NoError(t, database.Reader().QueryRow( + `SELECT COUNT(*) FROM parser_checkpoints WHERE session_id = ?`, + storedID, + ).Scan(&prefixedCount)) + require.Zero(t, nativeCount, + "the checkpoint must not be stored under the parser-native id") + require.Equal(t, 1, prefixedCount, + "the checkpoint must be stored under the rewritten session id") +} + +// TestReplaceSessionContentStagedRollbackDetaches pins the failure path: +// an aborted publish must still detach the scratch schema so the next +// publish on the same writer connection can attach again. +func TestReplaceSessionContentStagedRollbackDetaches(t *testing.T) { + database, err := Open(filepath.Join(t.TempDir(), "staged.db")) + require.NoError(t, err) + t.Cleanup(func() { _ = database.Close() }) + + sessionID := "codex:test-session" + require.NoError(t, database.UpsertSession(Session{ + ID: sessionID, + Agent: "codex", + Project: "project", + Machine: "local", + MessageCount: 1, + UserMessageCount: 1, + })) + + failing := &failInsertStagedResults{scratchStagedResults: newScratchStagedResults(t)} + failing.AddEvent(t, "call_a", "content") + msgs := []Message{{ + SessionID: sessionID, + Ordinal: 0, + Role: "assistant", + Content: "running", + HasToolUse: true, + ToolCalls: []ToolCall{{ + ToolUseID: "call_a", + ToolName: "exec_command", + Category: "Bash", + CallIndex: 0, + }}, + }} + err = database.ReplaceSessionContentStaged( + context.Background(), sessionID, msgs, failing, + map[string]bool{}, + func(map[string]bool) (SessionSignalUpdate, []SecretFinding, error) { + return SessionSignalUpdate{}, nil, nil + }, + ) + require.Error(t, err) + + // The aborted publish must have detached the staging schema. + rows, err := database.getWriter().Query("PRAGMA database_list") + require.NoError(t, err) + defer rows.Close() + for rows.Next() { + var seq int + var name, file string + require.NoError(t, rows.Scan(&seq, &name, &file)) + require.NotEqual(t, "codex_staging", name) + } + require.NoError(t, rows.Err()) + + // And a follow-up publish must succeed on the same writer pool. + staged := newScratchStagedResults(t) + staged.AddEvent(t, "call_b", "second") + require.NoError(t, database.ReplaceSessionContentStaged( + context.Background(), sessionID, msgs, staged, + map[string]bool{}, + func(map[string]bool) (SessionSignalUpdate, []SecretFinding, error) { + return SessionSignalUpdate{}, nil, nil + }, + )) +} + +type failInsertStagedResults struct { + *scratchStagedResults +} + +func (f *failInsertStagedResults) InsertEventsTx( + context.Context, *sql.Tx, string, map[string]StagedToolCallPosition, +) error { + return errors.New("injected staged failure") +} diff --git a/internal/parser/codex.go b/internal/parser/codex.go index b6fe19b7a..1aaf09de7 100644 --- a/internal/parser/codex.go +++ b/internal/parser/codex.go @@ -4,14 +4,13 @@ import ( "bufio" "context" "encoding/base64" - "encoding/json/v2" + "errors" "fmt" "io" "os" "path/filepath" "regexp" - "slices" "sort" "strconv" "strings" @@ -58,25 +57,29 @@ type codexSessionIndexEntry struct { // JSONL session file line by line. type codexSessionBuilder struct { codexCursorState - projectContext context.Context - resolveParentTurns codexParentTurnResolver - parentTurnIDs map[string]struct{} - messages []ParsedMessage - firstMessage string - startedAt time.Time - endedAt time.Time - sessionID string - parentSessionID string - relationshipType RelationshipType - project string - ordinal int - callNames map[string]string - callRefs map[string]codexToolCallRef - agentSpawnCalls map[string]string - agentWaitCalls map[string]string - pendingAgentEvents map[string][]codexPendingEvent - orphanNotificationIx map[string]int - unattachedTokenUsage bool + // sink receives every normalized operation; the collecting + // implementation keeps the slice-based behavior, the streaming one + // batches into a scratch store. + sink CodexSessionSink + projectContext context.Context + resolveParentTurns codexParentTurnResolver + parentTurnIDs map[string]struct{} + firstMessage string + startedAt time.Time + endedAt time.Time + sessionID string + parentSessionID string + relationshipType RelationshipType + project string + callNames map[string]string + agentSpawnCalls map[string]string + agentWaitCalls map[string]string + pendingAgentEvents map[string][]codexPendingEvent + unattachedTokenUsage bool + committedUsageTarget *int + committedUsageBlockedByUser bool + messageUsageUpdates []ParsedMessageTokenUsageUpdate + checkpointUnsafe bool } // codexForkGate drops replayed parent history at the top of a Codex @@ -95,6 +98,41 @@ type codexForkGate struct { active bool parentSessionID string parentResolved bool + // lineagePositive marks an explicit replay-prefix signal seen in the + // transcript itself: forked_from_id, or a copied parent session_meta + // after subagent lineage metadata. Only a positive signal can mark a + // session for retry; child-only subagents stay current. + lineagePositive bool + // resolvedOnce latches whether the parent was ever resolved during + // this parse. The in-parse gate resets parentResolved when the first + // child turn opens it, so the retry verdict reads this sticky flag + // instead. + resolvedOnce bool + // parentTurnless records that the resolved parent transcript held no + // turn ids. Codex Desktop writes a rollout when a thread opens, so a + // fork taken before the first prompt names a parent like this and has + // replayed nothing; the child is then current. Only when the child + // also carries the copied parent session_meta (replayObserved) did the + // parent write turns the fork copied but its own file has not flushed + // yet, and the child waits for the parent to advance. + parentTurnless bool + // replayObserved marks that the copied parent session_meta was seen, + // the positive proof that a replay prefix exists in this rollout. + replayObserved bool +} + +// retryReason reports the unresolved-parent retry condition captured +// during the single scan: an explicit replay parent that could not +// provide turn ids keeps the child visible but marks its data version +// for retry. Empty when the parse is current. +func (g *codexForkGate) retryReason() string { + if !g.lineagePositive || g.parentSessionID == "" { + return "" + } + if g.resolvedOnce && !(g.parentTurnless && g.replayObserved) { + return "" + } + return "codex parent turns unresolved for " + g.parentSessionID } type codexParentTurnResolver func(string) (map[string]struct{}, bool) @@ -113,8 +151,10 @@ func (g *codexForkGate) armFromMeta(payload gjson.Result, parentResolved bool) { } g.parentSessionID = parentID g.parentResolved = parentResolved + g.resolvedOnce = g.resolvedOnce || parentResolved if forkedFromID != "" { g.active = parentResolved + g.lineagePositive = true return } // Parent metadata alone also appears in child-only transcripts. Wait @@ -126,14 +166,18 @@ func (g *codexForkGate) armFromMeta(payload gjson.Result, parentResolved bool) { // transcript. Explicit forks are already active when their copied meta // arrives. func (g *codexForkGate) suppressesSessionMeta(payload gjson.Result) bool { + parentID := payload.Get("id").Str + if parentID != "" && parentID == g.parentSessionID { + g.replayObserved = true + } if g.active { return true } - parentID := payload.Get("id").Str if parentID == "" || parentID != g.parentSessionID { return false } g.active = g.parentResolved + g.lineagePositive = true return true } @@ -188,17 +232,17 @@ func newCodexSessionBuilder( ctx context.Context, _ bool, resolveParentTurns codexParentTurnResolver, + sink CodexSessionSink, ) *codexSessionBuilder { return &codexSessionBuilder{ - projectContext: ctx, - resolveParentTurns: resolveParentTurns, - project: "unknown", - callNames: make(map[string]string), - callRefs: make(map[string]codexToolCallRef), - agentSpawnCalls: make(map[string]string), - agentWaitCalls: make(map[string]string), - pendingAgentEvents: make(map[string][]codexPendingEvent), - orphanNotificationIx: make(map[string]int), + sink: sink, + projectContext: ctx, + resolveParentTurns: resolveParentTurns, + project: "unknown", + callNames: make(map[string]string), + agentSpawnCalls: make(map[string]string), + agentWaitCalls: make(map[string]string), + pendingAgentEvents: make(map[string][]codexPendingEvent), } } @@ -212,6 +256,7 @@ func (b *codexSessionBuilder) armForkGate(payload gjson.Result) { b.parentTurnIDs, resolved = b.resolveParentTurns(parentID) } b.forkGate.armFromMeta(payload, resolved) + b.forkGate.parentTurnless = resolved && len(b.parentTurnIDs) == 0 } func (b *codexSessionBuilder) suppresses( @@ -230,6 +275,46 @@ func (b *codexSessionBuilder) incrementalSeed() codexIncrementalSeed { return b.codexCursorState } +func (b *codexSessionBuilder) refreshPendingCallPositions() { + positionsByID := make(map[string][]ParsedToolCallPosition) + for _, msg := range b.sink.Messages() { + for callIndex, call := range msg.ToolCalls { + if call.ToolUseID == "" { + continue + } + positionsByID[call.ToolUseID] = append( + positionsByID[call.ToolUseID], + ParsedToolCallPosition{ + MessageOrdinal: msg.Ordinal, + CallIndex: callIndex, + }, + ) + } + } + + pendingByID := make(map[string][]int) + for i := 0; i < int(b.pendingCallCount); i++ { + b.pendingCalls[i].positionKnown = false + pendingByID[b.pendingCalls[i].id] = append( + pendingByID[b.pendingCalls[i].id], i, + ) + } + for id, pendingIndexes := range pendingByID { + positions, ok := positionsByID[id] + if !ok || len(positions) < len(pendingIndexes) { + b.checkpointUnsafe = true + return + } + positions = positions[len(positions)-len(pendingIndexes):] + for i, pendingIndex := range pendingIndexes { + b.pendingCalls[pendingIndex].messageOrdinal = + positions[i].MessageOrdinal + b.pendingCalls[pendingIndex].callIndex = positions[i].CallIndex + b.pendingCalls[pendingIndex].positionKnown = true + } + } +} + // processLine handles a single non-empty, valid JSON line. func (b *codexSessionBuilder) processLine( line string, @@ -365,17 +450,17 @@ func (b *codexSessionBuilder) handleResponseItem( // not the truncated first-message preview. return } + b.committedUsageTarget = nil + b.committedUsageBlockedByUser = true } - b.messages = append(b.messages, ParsedMessage{ - Ordinal: b.ordinal, + b.sink.AppendMessage(ParsedMessage{ Role: RoleType(role), Content: content, Timestamp: ts, ContentLength: len(content), Model: b.model, }) - b.ordinal++ } func (b *codexSessionBuilder) handleAgentMessage( @@ -394,15 +479,15 @@ func (b *codexSessionBuilder) handleAgentMessage( if replay { return } - b.messages = append(b.messages, ParsedMessage{ - Ordinal: b.ordinal, + b.committedUsageTarget = nil + b.committedUsageBlockedByUser = true + b.sink.AppendMessage(ParsedMessage{ Role: RoleUser, Content: content, Timestamp: ts, ContentLength: len(content), Model: b.model, }) - b.ordinal++ } func (b *codexSessionBuilder) handleEventMsg(payload gjson.Result) { @@ -431,20 +516,33 @@ func (b *codexSessionBuilder) handleTokenCountEvent( return } - // Find last assistant message without usage in the current - // turn. Stop at user message boundary so we don't cross - // turns. - for i, v := range slices.Backward(b.messages) { - if v.Role == RoleUser { - break - } - if v.Role == RoleAssistant && - v.TokenUsage == nil { - b.applyCodexTokenUsage(&b.messages[i], raw) - return - } + // Attach usage to the last assistant message without usage in the + // current turn; the sink stops at the user boundary so turns are + // never crossed. + if b.sink.ApplyTokenUsageToLastAssistant(raw) { + return + } + if b.committedUsageBlockedByUser { + return } - b.unattachedTokenUsage = true + if b.committedUsageTarget == nil { + b.unattachedTokenUsage = true + return + } + msg := ParsedMessage{} + applyCodexTokenUsage(&msg, raw) + b.messageUsageUpdates = append( + b.messageUsageUpdates, + ParsedMessageTokenUsageUpdate{ + Ordinal: *b.committedUsageTarget, + TokenUsage: append([]byte(nil), msg.TokenUsage...), + ContextTokens: msg.ContextTokens, + OutputTokens: msg.OutputTokens, + HasContextTokens: msg.HasContextTokens, + HasOutputTokens: msg.HasOutputTokens, + }, + ) + b.committedUsageTarget = nil } func (b *codexSessionBuilder) handleCollabAgentSpawnEnd( @@ -456,7 +554,10 @@ func (b *codexSessionBuilder) handleCollabAgentSpawnEnd( return } b.agentSpawnCalls[agentID] = callID - b.setCallSubagentSessionID(callID, codexSubagentSessionID(agentID)) + position, _ := b.toolCallPosition(callID) + b.sink.SetCallSubagentSessionID( + callID, position, codexSubagentSessionID(agentID), + ) } func (b *codexSessionBuilder) handleSubagentActivity( @@ -471,45 +572,10 @@ func (b *codexSessionBuilder) handleSubagentActivity( return } b.agentSpawnCalls[agentID] = callID - b.setCallSubagentSessionID(callID, codexSubagentSessionID(agentID)) -} - -// applyCodexTokenUsage normalizes Codex token usage fields -// into the Anthropic-style shape expected by the usage and cost -// queries. Codex reports input_tokens as the full input count -// (cached portion included), while the downstream cost formula -// treats input_tokens as the uncached remainder and bills -// cache_read_input_tokens separately. Subtracting cached here -// prevents double-counting the cached portion at the full input -// rate. -// -// input_tokens - cached_input_tokens → input_tokens (uncached) -// output_tokens → output_tokens -// cached_input_tokens → cache_read_input_tokens -func (b *codexSessionBuilder) applyCodexTokenUsage( - msg *ParsedMessage, raw string, -) { - usage := gjson.Parse(raw) - totalInput := int(usage.Get("input_tokens").Int()) - cached := int(usage.Get("cached_input_tokens").Int()) - output := int(usage.Get("output_tokens").Int()) - - uncached := max(totalInput-cached, 0) - - normalized := map[string]int{ - "input_tokens": uncached, - "output_tokens": output, - "cache_read_input_tokens": cached, - } - j, err := json.Marshal(normalized, json.Deterministic(true)) - if err != nil { - return - } - msg.TokenUsage = j - msg.OutputTokens = output - msg.HasOutputTokens = output > 0 - msg.ContextTokens = uncached + cached - msg.HasContextTokens = totalInput > 0 || cached > 0 + position, _ := b.toolCallPosition(callID) + b.sink.SetCallSubagentSessionID( + callID, position, codexSubagentSessionID(agentID), + ) } func (b *codexSessionBuilder) handleFunctionCall( @@ -533,8 +599,7 @@ func (b *codexSessionBuilder) handleFunctionCall( waitAgentIDs = codexWaitAgentIDs(args) } - b.messages = append(b.messages, ParsedMessage{ - Ordinal: b.ordinal, + messageOrdinal := b.sink.AppendMessage(ParsedMessage{ Role: RoleAssistant, Content: content, Timestamp: ts, @@ -550,12 +615,14 @@ func (b *codexSessionBuilder) handleFunctionCall( }}, }) if callID != "" { - b.callRefs[callID] = codexToolCallRef{ - messageIndex: len(b.messages) - 1, - callIndex: 0, + position := &ParsedToolCallPosition{ + MessageOrdinal: messageOrdinal, + CallIndex: 0, + } + if !b.rememberToolCall(callID, name, position) { + b.checkpointUnsafe = true } } - b.ordinal++ if isCodexWaitAgentCall(name) && callID != "" { for _, agentID := range waitAgentIDs { @@ -572,6 +639,7 @@ func (b *codexSessionBuilder) handleFunctionCallOutput( if callID == "" { return } + defer b.forgetToolCall(callID) output, raw := parseCodexFunctionOutput(payload) if !output.Exists() { @@ -580,14 +648,17 @@ func (b *codexSessionBuilder) handleFunctionCallOutput( } } - switch b.callNames[callID] { + switch b.toolCallNameForOutput(callID) { case "spawn_agent": agentID := strings.TrimSpace(output.Get("agent_id").Str) if agentID == "" { return } b.agentSpawnCalls[agentID] = callID - b.setCallSubagentSessionID(callID, codexSubagentSessionID(agentID)) + position, _ := b.toolCallPosition(callID) + b.sink.SetCallSubagentSessionID( + callID, position, codexSubagentSessionID(agentID), + ) case "wait", "wait_agent": status := output.Get("status") if !status.Exists() || !status.IsObject() { @@ -599,7 +670,7 @@ func (b *codexSessionBuilder) handleFunctionCallOutput( if text == "" { return true } - b.appendCallResultEvent(callID, ParsedToolResultEvent{ + b.appendToolResultEvent(callID, ParsedToolResultEvent{ ToolUseID: callID, AgentID: agentID, SubagentSessionID: codexSubagentSessionID(agentID), @@ -621,7 +692,7 @@ func (b *codexSessionBuilder) handleFunctionCallOutput( status = "completed" } } - b.appendCallResultEvent(callID, ParsedToolResultEvent{ + b.appendToolResultEvent(callID, ParsedToolResultEvent{ ToolUseID: callID, Source: source, Status: status, @@ -632,29 +703,24 @@ func (b *codexSessionBuilder) handleFunctionCallOutput( } } -// setCallSubagentSessionID links a tool call to the session of -// the subagent it spawned. Callers must invoke this only after -// the originating function_call has been processed (which -// populates b.callRefs[callID]); otherwise the link is silently -// dropped. In real codex session files the spawn function_call -// always precedes both its function_call_output and the -// collab_agent_spawn_end event_msg. -func (b *codexSessionBuilder) setCallSubagentSessionID( - callID, sessionID string, +func (b *codexSessionBuilder) appendToolResultEvent( + callID string, ev ParsedToolResultEvent, ) { - if callID == "" || sessionID == "" { - return - } - ref, ok := b.callRefs[callID] - if !ok || ref.messageIndex < 0 || ref.messageIndex >= len(b.messages) { - return - } - if ref.callIndex < 0 || ref.callIndex >= len(b.messages[ref.messageIndex].ToolCalls) { - return + position, _ := b.toolCallPosition(callID) + b.sink.AppendToolResultEvent(callID, position, ev) +} + +func (b *codexSessionBuilder) toolCallNameForOutput(callID string) string { + if name, ok := b.toolCallName(callID); ok { + return name } - b.messages[ref.messageIndex].ToolCalls[ref.callIndex].SubagentSessionID = sessionID + return b.callNames[callID] } +// handleSubagentNotification attributes a subagent notification either +// to a known wait call (a result event) or to a pending slot that holds +// its ordinal position until the wait call shows up or EOF flushes it as +// an orphan message. func (b *codexSessionBuilder) handleSubagentNotification( content string, ts time.Time, ) bool { @@ -663,7 +729,7 @@ func (b *codexSessionBuilder) handleSubagentNotification( return false } if callID := b.agentWaitCalls[agentID]; callID != "" { - b.appendCallResultEvent(callID, ParsedToolResultEvent{ + b.appendToolResultEvent(callID, ParsedToolResultEvent{ AgentID: agentID, SubagentSessionID: codexSubagentSessionID(agentID), Source: "subagent_notification", @@ -681,52 +747,12 @@ func (b *codexSessionBuilder) handleSubagentNotification( status: statusName, text: text, timestamp: ts, - ordinal: b.ordinal, + ordinal: b.sink.ReserveOrdinal(), }, ) - b.ordinal++ return true } -func (b *codexSessionBuilder) appendCallResultEvent( - callID string, ev ParsedToolResultEvent, -) { - if callID == "" { - return - } - ref, ok := b.callRefs[callID] - if !ok || ref.messageIndex < 0 || ref.messageIndex >= len(b.messages) { - return - } - if ref.callIndex < 0 || ref.callIndex >= len(b.messages[ref.messageIndex].ToolCalls) { - return - } - tc := &b.messages[ref.messageIndex].ToolCalls[ref.callIndex] - if ev.ToolUseID == "" { - ev.ToolUseID = tc.ToolUseID - } - if ev.SubagentSessionID == "" && ev.AgentID != "" { - ev.SubagentSessionID = codexSubagentSessionID(ev.AgentID) - } - if b.hasEquivalentCallResultEvent(tc.ResultEvents, ev) { - return - } - tc.ResultEvents = append(tc.ResultEvents, ev) -} - -func (b *codexSessionBuilder) hasEquivalentCallResultEvent( - events []ParsedToolResultEvent, candidate ParsedToolResultEvent, -) bool { - for _, existing := range events { - if existing.AgentID == candidate.AgentID && - existing.Status == candidate.Status && - existing.Content == candidate.Content { - return true - } - } - return false -} - func (b *codexSessionBuilder) claimPendingAgentEvents( callID, agentID string, ) { @@ -746,7 +772,7 @@ func (b *codexSessionBuilder) claimPendingAgentEventsContext( if err := contextErrEvery(ctx, i); err != nil { return err } - b.appendCallResultEvent(callID, ParsedToolResultEvent{ + b.appendToolResultEvent(callID, ParsedToolResultEvent{ AgentID: ev.agentID, SubagentSessionID: codexSubagentSessionID(ev.agentID), Source: ev.source, @@ -801,10 +827,7 @@ func (b *codexSessionBuilder) flushPendingAgentResultsContext( return err } key := agentID + "\x00" + ev.status + "\x00" + ev.text - if _, ok := b.orphanNotificationIx[key]; ok { - continue - } - idx, err := b.insertMessageContext(ctx, ParsedMessage{ + b.sink.InsertOrphanMessage(key, ParsedMessage{ Ordinal: ev.ordinal, Role: RoleUser, Content: ev.text, @@ -812,10 +835,7 @@ func (b *codexSessionBuilder) flushPendingAgentResultsContext( Model: b.model, ContentLength: len(ev.text), }) - if err != nil { - return err - } - b.orphanNotificationIx[key] = idx + } delete(b.pendingAgentEvents, agentID) } @@ -834,112 +854,6 @@ func codexSubagentSessionID(agentID string) string { return "codex:" + agentID } -func (b *codexSessionBuilder) normalizeOrdinals() { - _ = b.normalizeOrdinalsContext(context.Background()) -} - -func (b *codexSessionBuilder) normalizeOrdinalsContext( - ctx context.Context, -) error { - if err := ctx.Err(); err != nil { - return err - } - if len(b.messages) > 1 { - if err := stableSortCodexMessagesContext(ctx, b.messages); err != nil { - return err - } - } - for i := range b.messages { - if err := contextErrEvery(ctx, i); err != nil { - return err - } - b.messages[i].Ordinal = i - } - return ctx.Err() -} - -func stableSortCodexMessagesContext( - ctx context.Context, messages []ParsedMessage, -) error { - scratch := make([]ParsedMessage, len(messages)) - source, destination := messages, scratch - sourceIsMessages := true - steps := 0 - for width := 1; width < len(messages); width *= 2 { - for left := 0; left < len(messages); left += 2 * width { - if err := contextErrEvery(ctx, steps); err != nil { - return err - } - steps++ - middle := min(left+width, len(messages)) - right := min(left+2*width, len(messages)) - i, j, out := left, middle, left - for i < middle && j < right { - if err := contextErrEvery(ctx, steps); err != nil { - return err - } - steps++ - if source[i].Ordinal <= source[j].Ordinal { - destination[out] = source[i] - i++ - } else { - destination[out] = source[j] - j++ - } - out++ - } - out += copy(destination[out:right], source[i:middle]) - copy(destination[out:right], source[j:right]) - } - source, destination = destination, source - sourceIsMessages = !sourceIsMessages - if width > len(messages)/2 { - break - } - } - if !sourceIsMessages { - copy(messages, source) - } - return ctx.Err() -} - -func (b *codexSessionBuilder) insertMessage(msg ParsedMessage) int { - idx, _ := b.insertMessageContext(context.Background(), msg) - return idx -} - -func (b *codexSessionBuilder) insertMessageContext( - ctx context.Context, msg ParsedMessage, -) (int, error) { - idx := len(b.messages) - for i, existing := range b.messages { - if err := contextErrEvery(ctx, i); err != nil { - return 0, err - } - if existing.Ordinal > msg.Ordinal || - (existing.Ordinal == msg.Ordinal && - !msg.Timestamp.IsZero() && - (existing.Timestamp.IsZero() || - msg.Timestamp.Before(existing.Timestamp))) { - idx = i - break - } - } - b.messages = append(b.messages, ParsedMessage{}) - copy(b.messages[idx+1:], b.messages[idx:]) - b.messages[idx] = msg - for callID, ref := range b.callRefs { - if err := ctx.Err(); err != nil { - return 0, err - } - if ref.messageIndex >= idx { - ref.messageIndex++ - b.callRefs[callID] = ref - } - } - return idx, ctx.Err() -} - func formatCodexFunctionCall( name string, payload gjson.Result, ) string { @@ -1612,6 +1526,30 @@ func (p *codexProvider) parseSessionContext( ) } +// parseSessionWithCursor is parseSession plus the end-of-snapshot +// continuation cursor (and whether the end is a safe resume boundary). +func (p *codexProvider) parseSessionWithCursor( + ctx context.Context, path, machine string, includeExec bool, +) (*ParsedSession, []ParsedMessage, codexCursorState, bool, []byte, string, string, error) { + if err := ctx.Err(); err != nil { + return nil, nil, codexCursorState{}, false, nil, "", "", err + } + f, err := os.Open(path) + if err != nil { + return nil, nil, codexCursorState{}, false, nil, "", "", + fmt.Errorf("open %s: %w", path, err) + } + defer f.Close() + info, err := f.Stat() + if err != nil { + return nil, nil, codexCursorState{}, false, nil, "", "", + fmt.Errorf("stat %s: %w", path, err) + } + return p.parseSessionSnapshotWithCursor( + ctx, path, machine, includeExec, f, info, + ) +} + func (p *codexProvider) parentTurnResolver( ctx context.Context, childPath string, ) codexParentTurnResolver { @@ -1662,17 +1600,6 @@ func (p *codexProvider) parentTurnResolver( } } -func (p *codexProvider) codexParentResolution( - ctx context.Context, childPath string, -) (string, bool) { - parentID, resolutionNeeded := codexReplayParentIDContext(ctx, childPath) - if parentID == "" || !resolutionNeeded { - return "", false - } - turnIDs, resolved := p.parentTurnResolver(ctx, childPath)(parentID) - return parentID, resolved && len(turnIDs) > 0 -} - // CodexReplayParentID returns the explicit parent only when the rollout has a // positive replay-prefix signal: forked_from_id, or a copied parent // session_meta after subagent lineage metadata. Child-only subagents return no @@ -1746,17 +1673,55 @@ func (p *codexProvider) parseSessionSnapshotContext( f *os.File, info os.FileInfo, ) (*ParsedSession, []ParsedMessage, error) { + sess, msgs, _, _, _, _, _, err := p.parseSessionSnapshotWithCursor( + ctx, path, machine, includeExec, f, info, + ) + return sess, msgs, err +} + +// parseSessionSnapshotWithCursor is parseSessionSnapshot plus the +// continuation state at the end of the parsed snapshot. safe reports whether +// the file's end is a safe resume boundary; a false safe means the returned +// cursor must not be persisted. hashState and anchorDigest cover the whole +// snapshot [0, info.Size()) from the same read pass the parser performed; +// they are meaningful only when safe is true. +func (p *codexProvider) parseSessionSnapshotWithCursor( + ctx context.Context, path, machine string, + includeExec bool, + f *os.File, + info os.FileInfo, +) (*ParsedSession, []ParsedMessage, codexCursorState, bool, []byte, string, string, error) { + return p.parseCodexSessionSnapshotStreaming( + ctx, path, machine, includeExec, f, info, NewCodexCollectingSink(0), + ) +} + +// parseCodexSessionSnapshotStreaming decodes one snapshot, emitting every +// normalized operation into the caller's sink instead of accumulating a +// full message slice inside the parser. The returned message slice comes +// from the sink (for a collecting sink it is the complete transcript; for +// a staging sink it omits result-event content). +func (p *codexProvider) parseCodexSessionSnapshotStreaming( + ctx context.Context, path, machine string, + includeExec bool, + f *os.File, + info os.FileInfo, + sink CodexSessionSink, +) (*ParsedSession, []ParsedMessage, codexCursorState, bool, []byte, string, string, error) { if err := ctx.Err(); err != nil { - return nil, nil, err + return nil, nil, codexCursorState{}, false, nil, "", "", err } - lr := newLineReaderContext(ctx, io.LimitReader(f, info.Size()), maxLineSize) + tee := newCodexHashAnchorTee(io.LimitReader(f, info.Size())) + lr := newLineReaderContext(ctx, tee, maxLineSize) defer releaseLineReader(lr) - b := newCodexSessionBuilder(ctx, includeExec, p.parentTurnResolver(ctx, path)) + b := newCodexSessionBuilder( + ctx, includeExec, p.parentTurnResolver(ctx, path), sink, + ) malformedLines := 0 for { if err := ctx.Err(); err != nil { - return nil, nil, err + return nil, nil, codexCursorState{}, false, nil, "", "", err } line, ok := lr.next() if !ok { @@ -1765,7 +1730,8 @@ func (p *codexProvider) parseSessionSnapshotContext( if !gjson.Valid(line) { var terminator [1]byte if _, err := f.ReadAt(terminator[:], lr.bytesRead-1); err != nil { - return nil, nil, fmt.Errorf("checking codex line terminator: %w", err) + return nil, nil, codexCursorState{}, false, nil, "", "", + fmt.Errorf("checking codex line terminator: %w", err) } // A writer may leave its final JSON record incomplete while the // session is live. Count terminated malformed records, and count an @@ -1776,30 +1742,45 @@ func (p *codexProvider) parseSessionSnapshotContext( continue } if b.processLine(line) { - return nil, nil, nil + return nil, nil, codexCursorState{}, false, nil, "", "", nil } } if err := lr.Err(); err != nil { return nil, nil, + codexCursorState{}, false, nil, "", "", fmt.Errorf("reading codex %s: %w", path, err) } if err := b.flushPendingAgentResultsContext(ctx); err != nil { - return nil, nil, err - } - if err := b.normalizeOrdinalsContext(ctx); err != nil { - return nil, nil, err - } - inode, device := sourceFileIdentity(info) - if safe, safeErr := codexSafeResumeOffsetFile(f, info.Size()); safeErr == nil && safe { - p.cursorCache.Put( - path, - info.Size(), - inode, - device, - b.incrementalSeed(), - ) + return nil, nil, codexCursorState{}, false, nil, "", "", err + } + b.sink.Finalize() + b.refreshPendingCallPositions() + msgs := b.sink.Messages() + seed := b.incrementalSeed() + inode, device := sourceFileIdentityForFile(f, info) + changeTime, _ := codexIndexChangeTimeForFile(f, info) + safe := false + // A snapshot that ends while the fork replay gate is still active + // contains only replayed parent history so far. Persisting a cursor + // here would resume past session_meta with the gate dropped, and the + // remaining replayed parent records would be imported as child + // content. Refuse the checkpoint until a genuine child turn opens the + // gate; the file then reparses authoritatively and earns its cursor. + if !b.forkGate.active && !b.pendingCallsOverflow && !b.checkpointUnsafe { + if safeCheck, safeErr := codexSafeResumeOffsetFile( + f, info.Size(), + ); safeErr == nil && safeCheck { + safe = true + p.cursorCache.Put( + path, + info.Size(), + inode, + device, + seed, + ) + } } sessionID := b.sessionID @@ -1811,9 +1792,9 @@ func (p *codexProvider) parseSessionSnapshotContext( sessionID = "codex:" + sessionID userCount := 0 - for i, m := range b.messages { + for i, m := range msgs { if err := contextErrEvery(ctx, i); err != nil { - return nil, nil, err + return nil, nil, codexCursorState{}, false, nil, "", "", err } if m.Role == RoleUser && m.Content != "" { userCount++ @@ -1852,23 +1833,37 @@ func (p *codexProvider) parseSessionSnapshotContext( MalformedLines: malformedLines, StartedAt: b.startedAt, EndedAt: b.endedAt, - MessageCount: len(b.messages), + MessageCount: len(msgs), UserMessageCount: userCount, TerminationStatus: classifyCodexTermination(b.lastTaskEvent), File: FileInfo{ - Path: path, - Size: info.Size(), - Mtime: mtime, - Inode: int64(inode), - Device: int64(device), + Path: path, + Size: info.Size(), + Mtime: mtime, + Inode: int64(inode), + Device: int64(device), + ChangeTime: changeTime, }, } - if err := accumulateMessageTokenUsageContext(ctx, sess, b.messages); err != nil { - return nil, nil, err + if err := accumulateMessageTokenUsageContext(ctx, sess, msgs); err != nil { + return nil, nil, codexCursorState{}, false, nil, "", "", err + } + + var hashState []byte + var anchorDigest string + if safe { + state, err := tee.HashState() + if err != nil { + return nil, nil, codexCursorState{}, false, nil, "", "", + fmt.Errorf("capturing codex hash state %s: %w", path, err) + } + hashState = state + anchorDigest = tee.AnchorDigest() } - return sess, b.messages, ctx.Err() + return sess, msgs, seed, safe, hashState, anchorDigest, + b.forkGate.retryReason(), nil } // CodexSessionIndexFilename is the name of the Codex index file that maps @@ -2143,7 +2138,10 @@ func seedCodexIncrementalStateFromReader( r io.Reader, resolveParentTurns codexParentTurnResolver, ) (codexIncrementalSeed, error) { - b := newCodexSessionBuilder(context.Background(), false, resolveParentTurns) + sink := newCodexSeedSink() + b := newCodexSessionBuilder( + context.Background(), false, resolveParentTurns, sink, + ) lr := newLineReader(r, maxLineSize) defer releaseLineReader(lr) for { @@ -2154,84 +2152,23 @@ func seedCodexIncrementalStateFromReader( if !gjson.Valid(line) { continue } - lineType := gjson.Get(line, "type").Str - payload := gjson.Get(line, "payload") - if lineType == codexTypeSessionMeta { - // Mirror processLine: the fork's own meta arms the - // gate and supplies cwd, and replayed parent metas are - // dropped while it is active. - if !b.forkGate.suppressesSessionMeta(payload) { - if cwd := payload.Get("cwd").Str; cwd != "" { - b.cwd = cwd - } - b.agentPath = strings.TrimSpace( - payload.Get("agent_path").Str, - ) - if b.agentPath == "" { - b.agentPath = strings.TrimSpace(payload.Get( - "source.subagent.thread_spawn.agent_path", - ).Str) - } - b.armForkGate(payload) - } - continue - } - if b.suppresses(lineType, payload) { - continue - } - switch lineType { - case codexTypeTurnContext: - b.model = payload.Get("model").Str - case codexTypeEventMsg: - eventType := payload.Get("type").Str - b.observeTaskEvent(eventType) - if eventType == "token_count" { - raw := payload.Get("info.last_token_usage").Raw - if raw != "" { - b.observeTokenUsage(raw) - } - } - case codexTypeResponseItem: - observeCodexIncrementalUserMessage(&b.codexCursorState, payload) - } + b.processLine(line) } if err := lr.Err(); err != nil { return codexIncrementalSeed{}, err } - return b.codexCursorState, nil -} - -// observeUserMessage feeds one response_item into the -// re-emitted-prompt dedup state, mirroring handleResponseItem's -// user-message filtering and full-content matching. -func observeCodexIncrementalUserMessage( - s *codexIncrementalSeed, - payload gjson.Result, -) { - if payload.Get("type").Str == "agent_message" { - content := extractCodexInboundAgentMessage(payload, s.agentPath) - if strings.TrimSpace(content) != "" { - s.observeUserPrompt(content) - } - return - } - if payload.Get("role").Str != "user" { - return - } - content := extractCodexContent(payload) - if !s.firstUserSeen { - content = extractCodexInitialUserContent(payload) - } - if strings.TrimSpace(content) == "" { - return - } - if isCodexTurnAbortedMessage(content) { - s.markFirstUserReplayPossible() + b.flushPendingAgentResults() + if b.checkpointUnsafe || b.pendingCallsOverflow { + return codexIncrementalSeed{}, errCodexIncrementalNeedsFullParse } - if isCodexSystemMessage(content) { - return + if sink.hadReservation { + // ReserveOrdinal may later be claimed without materializing a message, + // so a constant-memory prefix scan cannot prove final coordinates. + // Keep the cursor usable for other continuation state while forcing a + // full parse if a later tail needs one of these pending targets. + b.clearPendingCallPositions() } - s.observeUserPrompt(content) + return b.codexCursorState, nil } // CodexTranscriptConsumedSize returns the byte offset after the last complete, @@ -2385,13 +2322,15 @@ func codexSafeResumeOffsetFile(f *os.File, offset int64) (bool, error) { } type codexIncrementalParseResult struct { - messages []ParsedMessage - endedAt time.Time - consumedBytes int64 - initialCursor codexCursorState - cursor codexCursorState - inode uint64 - device uint64 + messages []ParsedMessage + toolCallUpdates []ParsedToolCallUpdate + messageUsageUpdates []ParsedMessageTokenUsageUpdate + endedAt time.Time + consumedBytes int64 + initialCursor codexCursorState + cursor codexCursorState + inode uint64 + device uint64 } // parseSessionFromDetailed parses only new lines from a Codex JSONL file. It @@ -2419,7 +2358,7 @@ func (p *codexProvider) parseSessionFromDetailed( ) } return p.parseSessionFromSnapshot( - path, offset, startOrdinal, includeExec, f, info, info.Size(), + path, offset, startOrdinal, includeExec, f, info, info.Size(), nil, ) } @@ -2431,6 +2370,7 @@ func (p *codexProvider) parseSessionFromSnapshot( f *os.File, info os.FileInfo, limit int64, + committedUsageTarget *int, ) (codexIncrementalParseResult, error) { return p.parseSessionFromWithSources( path, @@ -2447,6 +2387,37 @@ func (p *codexProvider) parseSessionFromSnapshot( p.parentTurnResolver(context.Background(), path), ) }, + committedUsageTarget, + ) +} + +// parseSessionFromCheckpoint is parseSessionFromSnapshot with the committed +// prefix's continuation state supplied from a persisted checkpoint instead +// of a prefix rescan. +func (p *codexProvider) parseSessionFromCheckpoint( + path string, + offset int64, + startOrdinal int, + includeExec bool, + f *os.File, + info os.FileInfo, + limit int64, + seed codexCursorState, + committedUsageTarget *int, +) (codexIncrementalParseResult, error) { + return p.parseSessionFromWithSources( + path, + offset, + startOrdinal, + includeExec, + info, + func(fn func(string)) (int64, error) { + return readCodexJSONLSection(f, offset, limit, fn) + }, + func() (codexIncrementalSeed, error) { + return seed, nil + }, + committedUsageTarget, ) } @@ -2493,6 +2464,7 @@ func (p *codexProvider) parseSessionFromWithReaders( func() (codexIncrementalSeed, error) { return readSeed(path, offset) }, + nil, ) } @@ -2504,8 +2476,9 @@ func (p *codexProvider) parseSessionFromWithSources( info os.FileInfo, readLines func(func(string)) (int64, error), readSeed func() (codexIncrementalSeed, error), + committedUsageTarget *int, ) (codexIncrementalParseResult, error) { - inode, device := sourceFileIdentity(info) + inode, device := sourceFileIdentityForPath(path, info) seed, cacheHit := p.cursorCache.Get(path, offset, inode, device) if !cacheHit { var err error @@ -2521,9 +2494,16 @@ func (p *codexProvider) parseSessionFromWithSources( b := newCodexSessionBuilder( context.Background(), includeExec, p.parentTurnResolver(context.Background(), path), + NewCodexCollectingSink(startOrdinal), ) - b.ordinal = startOrdinal b.codexCursorState = seed + if seed.pendingCallsOverflow { + return codexIncrementalParseResult{}, errCodexIncrementalNeedsFullParse + } + if committedUsageTarget != nil { + ordinal := *committedUsageTarget + b.committedUsageTarget = &ordinal + } var fallbackErr error consumed, err := readLines( @@ -2539,21 +2519,10 @@ func (p *codexProvider) parseSessionFromWithSources( fallbackErr = errCodexIncrementalNeedsFullParse return } - if codexIncrementalNeedsFullParse(line) { + if b.codexIncrementalNeedsFullParse(line) { fallbackErr = errCodexIncrementalNeedsFullParse return } - if lineType == codexTypeResponseItem { - payload := gjson.Get(line, "payload") - switch payload.Get("type").Str { - case "function_call_output", - "custom_tool_call_output": - if b.incrementalOutputNeedsFullParse(payload) { - fallbackErr = errCodexIncrementalNeedsFullParse - return - } - } - } b.processLine(line) if b.unattachedTokenUsage { fallbackErr = errCodexIncrementalNeedsFullParse @@ -2572,8 +2541,20 @@ func (p *codexProvider) parseSessionFromWithSources( } b.flushPendingAgentResults() + if b.checkpointUnsafe { + return codexIncrementalParseResult{}, errCodexIncrementalNeedsFullParse + } + for _, update := range b.sink.ToolCallUpdates() { + if !update.TargetKnown { + return codexIncrementalParseResult{}, errCodexIncrementalNeedsFullParse + } + } result := codexIncrementalParseResult{ - messages: b.messages, + messages: b.sink.Messages(), + toolCallUpdates: b.sink.ToolCallUpdates(), + messageUsageUpdates: append( + []ParsedMessageTokenUsageUpdate(nil), b.messageUsageUpdates..., + ), endedAt: b.endedAt, consumedBytes: consumed, initialCursor: seed, @@ -2609,8 +2590,8 @@ func (p *codexProvider) parseSessionFrom( // parse error requires the caller to fall back to a full parse. func IsIncrementalFullParseFallback(err error) bool { return errors.Is(err, errCodexIncrementalNeedsFullParse) || - errors.Is(err, ErrClaudeIncrementalNeedsFullParse) || - errors.Is(err, ErrIncrementalNeedsFullParse) + errors.Is(err, ErrIncrementalNeedsFullParse) || + errors.Is(err, ErrClaudeIncrementalNeedsFullParse) } func isCodexSystemMessage(content string) bool { @@ -2714,6 +2695,15 @@ func isCodexSubagentNotification(content string) bool { } func codexIncrementalNeedsFullParse(line string) bool { + b := newCodexSessionBuilder( + context.Background(), false, nil, NewCodexCollectingSink(0), + ) + return b.codexIncrementalNeedsFullParse(line) +} + +func (b *codexSessionBuilder) codexIncrementalNeedsFullParse( + line string, +) bool { switch gjson.Get(line, "type").Str { case codexTypeEventMsg: payload := gjson.Get(line, "payload") @@ -2734,6 +2724,17 @@ func codexIncrementalNeedsFullParse(line string) bool { switch payload.Get("type").Str { case "function_call", "custom_tool_call": return isCodexWaitAgentCall(payload.Get("name").Str) + case "function_call_output", "custom_tool_call_output": + output, raw := parseCodexFunctionOutput(payload) + if isCodexSubagentFunctionOutput(output) { + return true + } + if strings.TrimSpace(raw) == "" { + return false + } + name := b.toolCallNameForOutput(payload.Get("call_id").Str) + return name == "" || name == "spawn_agent" || + isCodexWaitAgentCall(name) default: role := payload.Get("role").Str if role != "user" { @@ -2745,34 +2746,3 @@ func codexIncrementalNeedsFullParse(line string) bool { return agentID != "" && text != "" } } - -// incrementalOutputNeedsFullParse reports whether an appended -// function_call_output / custom_tool_call_output line cannot be -// represented by an append-only incremental write. Subagent outputs -// repair lineage on rows outside the append, and an output whose call -// is not part of this appended chunk belongs to an already-stored tool -// call; both need a replacing full parse. An output paired with its -// call in the same chunk attaches in memory exactly as a full parse -// would, so it stays on the incremental path. -func (b *codexSessionBuilder) incrementalOutputNeedsFullParse( - payload gjson.Result, -) bool { - output, raw := parseCodexFunctionOutput(payload) - if isCodexSubagentFunctionOutput(output) { - return true - } - if strings.TrimSpace(raw) == "" { - return false - } - callID := payload.Get("call_id").Str - if callID == "" { - // A full parse drops outputs without a call id too. - return false - } - switch b.callNames[callID] { - case "spawn_agent", "wait", "wait_agent": - return true - } - _, attachable := b.callRefs[callID] - return !attachable -} diff --git a/internal/parser/codex_checkpoint_test.go b/internal/parser/codex_checkpoint_test.go new file mode 100644 index 000000000..f41768c80 --- /dev/null +++ b/internal/parser/codex_checkpoint_test.go @@ -0,0 +1,290 @@ +package parser + +import ( + "context" + "crypto/sha256" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/testjsonl" +) + +func TestCodexCursorStateCheckpointRoundTrip(t *testing.T) { + seed := codexCursorState{ + model: "gpt-5.6-luna", + cwd: "/workspace/project-a", + agentPath: "codex/agents/a", + firstUserSeen: true, + sawUserTurnAfterFirst: true, + mayReplayFirstUserPrompt: false, + lastTokenUsageSeen: true, + lastTokenUsageDigest: [sha256.Size]byte{1, 2, 3}, + forkGate: codexForkGate{ + active: true, + parentSessionID: "019f0000-0000-7000-8000-000000000000", + parentResolved: true, + }, + lastTaskEvent: "task_complete", + } + seed.rememberToolCall("call_1", "exec_command", &ParsedToolCallPosition{MessageOrdinal: 1, CallIndex: 0}) + seed.rememberToolCall("call_2", "apply_patch", &ParsedToolCallPosition{MessageOrdinal: 2, CallIndex: 0}) + + blob, err := seed.MarshalBinary() + require.NoError(t, err) + + var got codexCursorState + require.NoError(t, got.UnmarshalBinary(blob)) + // The fork replay gate is process-only state: it is re-armed from the + // transcript on every parse and is not part of the persisted cursor. + got.forkGate = seed.forkGate + assert.Equal(t, seed, got) +} + +func TestCodexCursorStateCheckpointRejectsBadPayloads(t *testing.T) { + var state codexCursorState + + // Wrong version. + blob, err := state.MarshalBinary() + require.NoError(t, err) + blob[0] = 99 + require.Error(t, state.UnmarshalBinary(blob)) + + // Truncated payload. + blob, err = state.MarshalBinary() + require.NoError(t, err) + require.Error(t, state.UnmarshalBinary(blob[:len(blob)-3])) + + // Oversized pending-call count. + blob, err = state.MarshalBinary() + require.NoError(t, err) + blob[len(blob)-1] = 200 + require.Error(t, state.UnmarshalBinary(blob)) +} + +func TestCodexProviderIncrementalResumesFromCheckpointSeed(t *testing.T) { + const ( + uuid = "019eb791-cf7d-75c1-8439-9ed74c122a01" + callID = "call_checkpoint" + ) + prefix := testjsonl.JoinJSONL( + testjsonl.CodexSessionMetaJSON( + uuid, "/workspace/project-a", "codex_cli_rs", tsEarly, + ), + testjsonl.CodexMsgJSON("user", "run the command", tsEarlyS1), + testjsonl.CodexFunctionCallWithCallIDJSON( + "exec_command", callID, nil, tsEarlyS5, + ), + ) + root := t.TempDir() + path := writeCodexProviderSessionContent( + t, root, uuid, prefix, + ) + provider, ok := NewProvider( + AgentCodex, ProviderConfig{Roots: []string{root}}, + ) + require.True(t, ok) + source := requireCodexProviderSource(t, provider, uuid) + + fingerprint, err := provider.Fingerprint(context.Background(), source) + require.NoError(t, err) + outcome, err := provider.Parse(context.Background(), ParseRequest{ + Source: source, Fingerprint: fingerprint, + }) + require.NoError(t, err) + require.Len(t, outcome.Results, 1) + checkpoint := outcome.Results[0].Result.Checkpoint + require.NotEmpty(t, checkpoint, + "a full parse of a safe-offset transcript must produce a checkpoint") + + tail := testjsonl.JoinJSONL(testjsonl.CodexFunctionCallOutputJSON( + callID, "done", "2026-08-02T09:00:03Z", + )) + appendCodexProviderContent(t, path, tail) + + fingerprint, err = provider.Fingerprint(context.Background(), source) + require.NoError(t, err) + incOutcome, status, err := provider.ParseIncremental( + context.Background(), IncrementalRequest{ + Source: source, + Fingerprint: fingerprint, + SessionID: "codex:" + uuid, + Offset: int64(len(prefix)), + StartOrdinal: 2, + Seed: checkpoint, + }, + ) + require.NoError(t, err) + assert.Equal(t, IncrementalApplied, status) + assert.Empty(t, incOutcome.Messages) + require.Len(t, incOutcome.ToolCallUpdates, 1) + assert.Equal(t, callID, incOutcome.ToolCallUpdates[0].ToolUseID) + assert.NotEmpty(t, incOutcome.NextCursor, + "an applied incremental parse must advance the cursor") +} + +// TestCodexParseCarriesSinglePassHashState verifies the full parse captures +// the resumable SHA-256 state and tail-anchor digest on its own read pass: +// the state digest must equal the snapshot hash and the anchor digest must +// equal the hash of the trailing window, so checkpoint persistence never +// needs a second source read. +func TestCodexParseCarriesSinglePassHashState(t *testing.T) { + const uuid = "019eb791-cf7d-75c1-8439-9ed74c122a02" + prefix := testjsonl.JoinJSONL( + testjsonl.CodexSessionMetaJSON( + uuid, "/workspace/project-a", "codex_cli_rs", tsEarly, + ), + testjsonl.CodexMsgJSON("user", "run the command", tsEarlyS1), + testjsonl.CodexFunctionCallWithCallIDJSON( + "exec_command", "call_tee", nil, tsEarlyS5, + ), + ) + root := t.TempDir() + path := writeCodexProviderSessionContent(t, root, uuid, prefix) + provider, ok := NewProvider( + AgentCodex, ProviderConfig{Roots: []string{root}}, + ) + require.True(t, ok) + source := requireCodexProviderSource(t, provider, uuid) + + fingerprint, err := provider.Fingerprint(context.Background(), source) + require.NoError(t, err) + outcome, err := provider.Parse(context.Background(), ParseRequest{ + Source: source, Fingerprint: fingerprint, + }) + require.NoError(t, err) + require.Len(t, outcome.Results, 1) + result := outcome.Results[0].Result + require.NotEmpty(t, result.CheckpointHashState, + "the parse must capture the resumable hash state") + + // The state digest must equal the snapshot's full hash. + stateHash := sha256.New() + require.NoError(t, stateHash.(interface{ UnmarshalBinary([]byte) error }). + UnmarshalBinary(result.CheckpointHashState)) + assert.Equal(t, fingerprint.Hash, result.Session.File.Hash) + wantDigest := sha256.Sum256([]byte(prefix)) + assert.Equal(t, fingerprint.Hash, + fmt.Sprintf("%x", wantDigest[:]), + "sanity: the provider fingerprint is the snapshot hash") + stateSum := stateHash.Sum(nil) + assert.Equal(t, wantDigest[:], stateSum, + "the captured state must hash exactly the parsed snapshot") + + // The anchor digest must equal the trailing window's hash. + window := prefix[max(0, len(prefix)-codexCheckpointAnchorSize):] + wantAnchor := sha256.Sum256([]byte(window)) + assert.Equal(t, fmt.Sprintf("%x", wantAnchor[:]), + result.CheckpointAnchorDigest) + + // Resuming the state over an appended tail must reproduce the real + // full-file hash — the same property the engine's resume path relies + // on. + tail := testjsonl.JoinJSONL(testjsonl.CodexFunctionCallOutputJSON( + "call_tee", "done", "2026-08-02T09:00:03Z", + )) + appendCodexProviderContent(t, path, tail) + resumed := sha256.New() + require.NoError(t, resumed.(interface{ UnmarshalBinary([]byte) error }). + UnmarshalBinary(result.CheckpointHashState)) + _, err = resumed.Write([]byte(tail)) + require.NoError(t, err) + full := append([]byte(prefix), []byte(tail)...) + wantFull := sha256.Sum256(full) + assert.Equal(t, wantFull[:], resumed.Sum(nil), + "resuming the captured state must reproduce the full-file hash") +} + +func TestCodexCursorPendingDuplicateIDsAreFIFO(t *testing.T) { + var state codexCursorState + first := &ParsedToolCallPosition{MessageOrdinal: 1, CallIndex: 0} + second := &ParsedToolCallPosition{MessageOrdinal: 2, CallIndex: 0} + require.True(t, state.rememberToolCall("reused", "exec_command", first)) + require.True(t, state.rememberToolCall("reused", "apply_patch", second)) + + name, ok := state.toolCallName("reused") + require.True(t, ok) + assert.Equal(t, "exec_command", name) + position, ok := state.toolCallPosition("reused") + require.True(t, ok) + assert.Equal(t, first, position) + + blob, err := state.MarshalBinary() + require.NoError(t, err) + var restored codexCursorState + require.NoError(t, restored.UnmarshalBinary(blob)) + + restored.forgetToolCall("reused") + name, ok = restored.toolCallName("reused") + require.True(t, ok) + assert.Equal(t, "apply_patch", name) + position, ok = restored.toolCallPosition("reused") + require.True(t, ok) + assert.Equal(t, second, position) +} + +func TestCodexProviderIncrementalTargetsPendingDuplicateCallIDOccurrence(t *testing.T) { + const ( + uuid = "019eb791-cf7d-75c1-8439-9ed74c122d01" + callID = "reused-call" + ) + prefix := testjsonl.JoinJSONL( + testjsonl.CodexSessionMetaJSON( + uuid, "/workspace/project-a", "codex_cli_rs", tsEarly, + ), + testjsonl.CodexMsgJSON("user", "run twice", tsEarlyS1), + testjsonl.CodexFunctionCallWithCallIDJSON( + "exec_command", callID, nil, tsEarlyS5, + ), + testjsonl.CodexFunctionCallOutputJSON( + callID, "first result", tsLate, + ), + testjsonl.CodexFunctionCallWithCallIDJSON( + "exec_command", callID, nil, tsLateS5, + ), + ) + root := t.TempDir() + path := writeCodexProviderSessionContent(t, root, uuid, prefix) + provider, ok := NewProvider( + AgentCodex, ProviderConfig{Roots: []string{root}}, + ) + require.True(t, ok) + source := requireCodexProviderSource(t, provider, uuid) + fingerprint, err := provider.Fingerprint(context.Background(), source) + require.NoError(t, err) + outcome, err := provider.Parse(context.Background(), ParseRequest{ + Source: source, Fingerprint: fingerprint, + }) + require.NoError(t, err) + require.Len(t, outcome.Results, 1) + checkpoint := outcome.Results[0].Result.Checkpoint + require.NotEmpty(t, checkpoint) + + tail := testjsonl.JoinJSONL(testjsonl.CodexFunctionCallOutputJSON( + callID, "second result", "2026-08-02T09:00:06Z", + )) + appendCodexProviderContent(t, path, tail) + fingerprint, err = provider.Fingerprint(context.Background(), source) + require.NoError(t, err) + incOutcome, status, err := provider.ParseIncremental( + context.Background(), IncrementalRequest{ + Source: source, + Fingerprint: fingerprint, + SessionID: "codex:" + uuid, + Offset: int64(len(prefix)), + StartOrdinal: 3, + Seed: checkpoint, + }, + ) + require.NoError(t, err) + assert.Equal(t, IncrementalApplied, status) + require.Len(t, incOutcome.ToolCallUpdates, 1) + update := incOutcome.ToolCallUpdates[0] + assert.Equal(t, callID, update.ToolUseID) + assert.True(t, update.TargetKnown) + assert.Equal(t, 2, update.MessageOrdinal) + assert.Equal(t, 0, update.CallIndex) + require.Len(t, update.ResultEvents, 1) + assert.Equal(t, "second result", update.ResultEvents[0].Content) +} diff --git a/internal/parser/codex_collecting_sink.go b/internal/parser/codex_collecting_sink.go new file mode 100644 index 000000000..f783bd13e --- /dev/null +++ b/internal/parser/codex_collecting_sink.go @@ -0,0 +1,292 @@ +package parser + +import ( + "encoding/json/v2" + "sort" + + "github.com/tidwall/gjson" +) + +// CodexCollectingSink is the in-memory CodexSessionSink implementation +// that reproduces the pre-streaming decoder behavior byte for byte: all +// messages stay in a slice, call ids index into it, and deferred updates +// accumulate in memory. It exists so the slice-based parser remains +// available to tests and other providers while the streaming sink lands. +type CodexCollectingSink struct { + messages []ParsedMessage + callRefs map[string]codexToolCallRef + callRefsByPosition map[ParsedToolCallPosition]codexToolCallRef + toolCallUpdates []ParsedToolCallUpdate + orphanNotificationIx map[string]int + nextOrdinal int +} + +func NewCodexCollectingSink(startOrdinal int) *CodexCollectingSink { + return &CodexCollectingSink{ + callRefs: make(map[string]codexToolCallRef), + callRefsByPosition: make(map[ParsedToolCallPosition]codexToolCallRef), + orphanNotificationIx: make(map[string]int), + nextOrdinal: startOrdinal, + } +} + +func (s *CodexCollectingSink) AppendMessage(m ParsedMessage) int { + m.Ordinal = s.nextOrdinal + s.nextOrdinal++ + s.messages = append(s.messages, m) + for callIdx := range m.ToolCalls { + callID := m.ToolCalls[callIdx].ToolUseID + if callID == "" { + continue + } + ref := codexToolCallRef{ + messageIndex: len(s.messages) - 1, + callIndex: callIdx, + } + s.callRefs[callID] = ref + s.callRefsByPosition[ParsedToolCallPosition{ + MessageOrdinal: m.Ordinal, + CallIndex: callIdx, + }] = ref + } + return m.Ordinal +} + +func (s *CodexCollectingSink) ReserveOrdinal() int { + ord := s.nextOrdinal + s.nextOrdinal++ + return ord +} + +func (s *CodexCollectingSink) InsertMessage(m ParsedMessage) int { + idx := len(s.messages) + for i, existing := range s.messages { + if existing.Ordinal > m.Ordinal || + (existing.Ordinal == m.Ordinal && + !m.Timestamp.IsZero() && + (existing.Timestamp.IsZero() || + m.Timestamp.Before(existing.Timestamp))) { + idx = i + break + } + } + s.messages = append(s.messages, ParsedMessage{}) + copy(s.messages[idx+1:], s.messages[idx:]) + s.messages[idx] = m + for callID, ref := range s.callRefs { + if ref.messageIndex >= idx { + ref.messageIndex++ + s.callRefs[callID] = ref + } + } + for position, ref := range s.callRefsByPosition { + if ref.messageIndex >= idx { + ref.messageIndex++ + s.callRefsByPosition[position] = ref + } + } + return idx +} + +func (s *CodexCollectingSink) AppendToolResultEvent( + callID string, target *ParsedToolCallPosition, ev ParsedToolResultEvent, +) { + if callID == "" { + return + } + ref, ok := s.callRef(callID, target) + if !ok { + s.appendToolCallUpdate(callID, target, ev) + return + } + if ref.messageIndex < 0 || ref.messageIndex >= len(s.messages) || + ref.callIndex < 0 || + ref.callIndex >= len(s.messages[ref.messageIndex].ToolCalls) { + return + } + tc := &s.messages[ref.messageIndex].ToolCalls[ref.callIndex] + if ev.ToolUseID == "" { + ev.ToolUseID = tc.ToolUseID + } + if ev.SubagentSessionID == "" && ev.AgentID != "" { + ev.SubagentSessionID = codexSubagentSessionID(ev.AgentID) + } + if hasEquivalentCallResultEvent(tc.ResultEvents, ev) { + return + } + tc.ResultEvents = append(tc.ResultEvents, ev) +} + +func (s *CodexCollectingSink) callRef( + callID string, target *ParsedToolCallPosition, +) (codexToolCallRef, bool) { + if target == nil { + ref, ok := s.callRefs[callID] + return ref, ok + } + ref, ok := s.callRefsByPosition[*target] + if !ok || ref.messageIndex < 0 || ref.messageIndex >= len(s.messages) || + ref.callIndex < 0 || ref.callIndex >= len(s.messages[ref.messageIndex].ToolCalls) { + return codexToolCallRef{}, false + } + if s.messages[ref.messageIndex].ToolCalls[ref.callIndex].ToolUseID != callID { + return codexToolCallRef{}, false + } + return ref, true +} + +func (s *CodexCollectingSink) appendToolCallUpdate( + callID string, target *ParsedToolCallPosition, ev ParsedToolResultEvent, +) { + if ev.ToolUseID == "" { + ev.ToolUseID = callID + } + for i := range s.toolCallUpdates { + update := &s.toolCallUpdates[i] + if update.ToolUseID != callID || + !sameParsedToolCallTarget(update, target) { + continue + } + if hasEquivalentCallResultEvent(update.ResultEvents, ev) { + return + } + update.ResultEvents = append(update.ResultEvents, ev) + return + } + update := ParsedToolCallUpdate{ + ToolUseID: callID, + ResultEvents: []ParsedToolResultEvent{ev}, + } + if target != nil { + update.MessageOrdinal = target.MessageOrdinal + update.CallIndex = target.CallIndex + update.TargetKnown = true + } + s.toolCallUpdates = append(s.toolCallUpdates, update) +} + +func sameParsedToolCallTarget( + update *ParsedToolCallUpdate, target *ParsedToolCallPosition, +) bool { + if target == nil { + return !update.TargetKnown + } + return update.TargetKnown && + update.MessageOrdinal == target.MessageOrdinal && + update.CallIndex == target.CallIndex +} + +func (s *CodexCollectingSink) SetCallSubagentSessionID( + callID string, target *ParsedToolCallPosition, sessionID string, +) { + if callID == "" || sessionID == "" { + return + } + ref, ok := s.callRef(callID, target) + if !ok || ref.messageIndex < 0 || ref.messageIndex >= len(s.messages) || + ref.callIndex < 0 || + ref.callIndex >= len(s.messages[ref.messageIndex].ToolCalls) { + return + } + s.messages[ref.messageIndex].ToolCalls[ref.callIndex]. + SubagentSessionID = sessionID +} + +// ApplyTokenUsageToLastAssistant applies normalized token usage to the +// last assistant message without usage, scanning back to the current +// turn's user boundary. Returns false when no target exists. +func (s *CodexCollectingSink) ApplyTokenUsageToLastAssistant( + raw string, +) bool { + for i := len(s.messages) - 1; i >= 0; i-- { + if s.messages[i].Role == RoleUser { + break + } + if s.messages[i].Role == RoleAssistant && + s.messages[i].TokenUsage == nil { + applyCodexTokenUsage(&s.messages[i], raw) + return true + } + } + return false +} + +func (s *CodexCollectingSink) InsertOrphanMessage( + key string, m ParsedMessage, +) bool { + if _, ok := s.orphanNotificationIx[key]; ok { + return false + } + s.orphanNotificationIx[key] = s.InsertMessage(m) + return true +} + +func (s *CodexCollectingSink) Finalize() { + sort.SliceStable(s.messages, func(i, j int) bool { + if s.messages[i].Ordinal == s.messages[j].Ordinal { + return i < j + } + return s.messages[i].Ordinal < s.messages[j].Ordinal + }) + for i := range s.messages { + s.messages[i].Ordinal = i + } +} + +func (s *CodexCollectingSink) Messages() []ParsedMessage { + return s.messages +} + +func (s *CodexCollectingSink) ToolCallUpdates() []ParsedToolCallUpdate { + return s.toolCallUpdates +} + +// hasEquivalentCallResultEvent reports whether events already contains a +// result equivalent to candidate: same agent, status, and content. +func hasEquivalentCallResultEvent( + events []ParsedToolResultEvent, candidate ParsedToolResultEvent, +) bool { + for _, existing := range events { + if existing.AgentID == candidate.AgentID && + existing.Status == candidate.Status && + existing.Content == candidate.Content { + return true + } + } + return false +} + +// applyCodexTokenUsage normalizes Codex token usage fields into the +// Anthropic-style shape expected by the usage and cost queries. Codex +// reports input_tokens as the full input count (cached portion included), +// while the downstream cost formula treats input_tokens as the uncached +// remainder and bills cache_read_input_tokens separately. Subtracting +// cached here prevents double-counting the cached portion at the full +// input rate. +// +// input_tokens - cached_input_tokens -> input_tokens (uncached) +// output_tokens -> output_tokens +// cached_input_tokens -> cache_read_input_tokens +func applyCodexTokenUsage(msg *ParsedMessage, raw string) { + usage := gjson.Parse(raw) + totalInput := int(usage.Get("input_tokens").Int()) + cached := int(usage.Get("cached_input_tokens").Int()) + output := int(usage.Get("output_tokens").Int()) + + uncached := max(totalInput-cached, 0) + + normalized := map[string]int{ + "input_tokens": uncached, + "output_tokens": output, + "cache_read_input_tokens": cached, + } + j, err := json.Marshal(normalized, json.Deterministic(true)) + if err != nil { + return + } + msg.TokenUsage = j + msg.OutputTokens = output + msg.HasOutputTokens = output > 0 + msg.ContextTokens = uncached + cached + msg.HasContextTokens = totalInput > 0 || cached > 0 +} diff --git a/internal/parser/codex_cursor.go b/internal/parser/codex_cursor.go index a8ea6305e..55576f6fc 100644 --- a/internal/parser/codex_cursor.go +++ b/internal/parser/codex_cursor.go @@ -1,9 +1,13 @@ package parser import ( + "bytes" "container/list" "crypto/sha256" + "encoding/binary" "encoding/json/jsontext" + "fmt" + "io" "path/filepath" "strings" "sync" @@ -12,15 +16,33 @@ import ( const ( codexCursorCacheMaxEntries = 256 codexCursorCacheMaxBytes = 2 << 20 + codexCursorMaxPendingCalls = 8 + // codexCursorCheckpointVersion is the wire version for the persisted + // cursor encoding. Bump when the encoding changes; decode failures fall + // back to a full parse. + // The fork replay gate is process-only state: it is re-armed from the + // transcript on every parse and is not part of the persisted cursor. + codexCursorCheckpointVersion = 2 + codexCursorCheckpointMaxString = 1 << 20 // Account for the map bucket, list element, pointers, string headers, and // allocator overhead that are not represented by the variable-length path - // and cursor strings below. The cache is intentionally an estimate rather + // and cursor strings below. The fixed pending-call array contributes two + // string headers, occurrence coordinates, and flags per slot. The cache is + // intentionally an estimate rather // than a heap profiler, but this conservative allowance keeps its retained // memory bounded near the configured byte limit. - codexCursorEntryOverheadBytes = 256 + codexCursorEntryOverheadBytes = 256 + codexCursorMaxPendingCalls*64 ) +type codexPendingToolCall struct { + id string + name string + messageOrdinal int + callIndex int + positionKnown bool +} + // codexCursorState is the compact state needed to make a tail parse behave as // though the already-persisted prefix had just been scanned. It deliberately // excludes parsed messages, raw transcript data, tool maps, and open files. @@ -36,6 +58,269 @@ type codexCursorState struct { lastTokenUsageSeen bool forkGate codexForkGate lastTaskEvent string + pendingCalls [codexCursorMaxPendingCalls]codexPendingToolCall + pendingCallCount uint8 + pendingCallsOverflow bool +} + +// MarshalBinary encodes the compact continuation state for persistence. +// The encoding is versioned and intentionally bounded: strings are +// length-prefixed and capped on decode, and the pending-call array is +// fixed-size. +func (s *codexCursorState) MarshalBinary() ([]byte, error) { + if s.pendingCallsOverflow { + return nil, fmt.Errorf( + "codex cursor has more than %d unresolved tool calls", + codexCursorMaxPendingCalls, + ) + } + var buf bytes.Buffer + write := func(v any) error { + return binary.Write(&buf, binary.LittleEndian, v) + } + writeStr := func(str string) error { + if err := write(uint32(len(str))); err != nil { + return err + } + _, err := buf.WriteString(str) + return err + } + if err := write(uint8(codexCursorCheckpointVersion)); err != nil { + return nil, err + } + for _, str := range []string{s.model, s.cwd, s.agentPath} { + if err := writeStr(str); err != nil { + return nil, err + } + } + if err := write(s.firstUserDigest); err != nil { + return nil, err + } + flags := uint8(0) + if s.firstUserSeen { + flags |= 1 << 0 + } + if s.sawUserTurnAfterFirst { + flags |= 1 << 1 + } + if s.mayReplayFirstUserPrompt { + flags |= 1 << 2 + } + if s.lastTokenUsageSeen { + flags |= 1 << 3 + } + if err := write(flags); err != nil { + return nil, err + } + if err := write(s.lastTokenUsageDigest); err != nil { + return nil, err + } + if err := writeStr(s.lastTaskEvent); err != nil { + return nil, err + } + if err := write(uint8(s.pendingCallCount)); err != nil { + return nil, err + } + for i := 0; i < int(s.pendingCallCount); i++ { + pending := s.pendingCalls[i] + if err := writeStr(pending.id); err != nil { + return nil, err + } + if err := writeStr(pending.name); err != nil { + return nil, err + } + positionKnown := uint8(0) + if pending.positionKnown { + positionKnown = 1 + } + if err := write(positionKnown); err != nil { + return nil, err + } + if err := write(int64(pending.messageOrdinal)); err != nil { + return nil, err + } + if err := write(int32(pending.callIndex)); err != nil { + return nil, err + } + } + return buf.Bytes(), nil +} + +// UnmarshalBinary restores the state written by MarshalBinary. Any version, +// length, or structure mismatch returns an error so the caller falls back +// to an authoritative full parse. +func (s *codexCursorState) UnmarshalBinary(data []byte) error { + r := bytes.NewReader(data) + read := func(v any) error { + return binary.Read(r, binary.LittleEndian, v) + } + readStr := func() (string, error) { + var n uint32 + if err := read(&n); err != nil { + return "", err + } + if n > codexCursorCheckpointMaxString { + return "", fmt.Errorf( + "codex cursor string length %d exceeds bound %d", + n, codexCursorCheckpointMaxString, + ) + } + b := make([]byte, n) + if _, err := io.ReadFull(r, b); err != nil { + return "", err + } + return string(b), nil + } + + var version uint8 + if err := read(&version); err != nil { + return err + } + if version != codexCursorCheckpointVersion { + return fmt.Errorf( + "unsupported codex cursor version %d", version, + ) + } + *s = codexCursorState{} + var err error + if s.model, err = readStr(); err != nil { + return err + } + if s.cwd, err = readStr(); err != nil { + return err + } + if s.agentPath, err = readStr(); err != nil { + return err + } + if err := read(&s.firstUserDigest); err != nil { + return err + } + var flags uint8 + if err := read(&flags); err != nil { + return err + } + s.firstUserSeen = flags&(1<<0) != 0 + s.sawUserTurnAfterFirst = flags&(1<<1) != 0 + s.mayReplayFirstUserPrompt = flags&(1<<2) != 0 + s.lastTokenUsageSeen = flags&(1<<3) != 0 + if err := read(&s.lastTokenUsageDigest); err != nil { + return err + } + if s.lastTaskEvent, err = readStr(); err != nil { + return err + } + var count uint8 + if err := read(&count); err != nil { + return err + } + if count > codexCursorMaxPendingCalls { + return fmt.Errorf( + "codex cursor pending call count %d exceeds bound %d", + count, codexCursorMaxPendingCalls, + ) + } + s.pendingCallCount = count + for i := 0; i < int(count); i++ { + pending := &s.pendingCalls[i] + if pending.id, err = readStr(); err != nil { + return err + } + if pending.name, err = readStr(); err != nil { + return err + } + var positionKnown uint8 + if err := read(&positionKnown); err != nil { + return err + } + if positionKnown > 1 { + return fmt.Errorf("invalid codex cursor position flag %d", positionKnown) + } + var messageOrdinal int64 + if err := read(&messageOrdinal); err != nil { + return err + } + var callIndex int32 + if err := read(&callIndex); err != nil { + return err + } + pending.positionKnown = positionKnown == 1 + pending.messageOrdinal = int(messageOrdinal) + pending.callIndex = int(callIndex) + } + if r.Len() != 0 { + return fmt.Errorf("codex cursor trailing bytes: %d", r.Len()) + } + return nil +} + +func (s *codexCursorState) rememberToolCall( + id, name string, position *ParsedToolCallPosition, +) bool { + id = strings.TrimSpace(id) + name = strings.TrimSpace(name) + if id == "" || name == "" { + return false + } + if int(s.pendingCallCount) >= len(s.pendingCalls) { + s.pendingCallsOverflow = true + return false + } + pending := codexPendingToolCall{id: id, name: name} + if position != nil { + pending.messageOrdinal = position.MessageOrdinal + pending.callIndex = position.CallIndex + pending.positionKnown = true + } + s.pendingCalls[s.pendingCallCount] = pending + s.pendingCallCount++ + return true +} + +func (s *codexCursorState) toolCallName(id string) (string, bool) { + for i := 0; i < int(s.pendingCallCount); i++ { + if s.pendingCalls[i].id == id { + return s.pendingCalls[i].name, true + } + } + return "", false +} + +func (s *codexCursorState) toolCallPosition( + id string, +) (*ParsedToolCallPosition, bool) { + for i := 0; i < int(s.pendingCallCount); i++ { + pending := s.pendingCalls[i] + if pending.id != id { + continue + } + if !pending.positionKnown { + return nil, false + } + return &ParsedToolCallPosition{ + MessageOrdinal: pending.messageOrdinal, + CallIndex: pending.callIndex, + }, true + } + return nil, false +} + +func (s *codexCursorState) clearPendingCallPositions() { + for i := 0; i < int(s.pendingCallCount); i++ { + s.pendingCalls[i].positionKnown = false + } +} + +func (s *codexCursorState) forgetToolCall(id string) { + for i := 0; i < int(s.pendingCallCount); i++ { + if s.pendingCalls[i].id != id { + continue + } + last := int(s.pendingCallCount) - 1 + copy(s.pendingCalls[i:last], s.pendingCalls[i+1:last+1]) + s.pendingCalls[last] = codexPendingToolCall{} + s.pendingCallCount-- + return + } } // observeUserPrompt advances the first-user replay state using only a digest @@ -237,6 +522,10 @@ func cloneCodexCursorState(state codexCursorState) codexCursorState { state.cwd = strings.Clone(state.cwd) state.agentPath = strings.Clone(state.agentPath) state.lastTaskEvent = strings.Clone(state.lastTaskEvent) + for i := 0; i < int(state.pendingCallCount); i++ { + state.pendingCalls[i].id = strings.Clone(state.pendingCalls[i].id) + state.pendingCalls[i].name = strings.Clone(state.pendingCalls[i].name) + } state.forkGate.parentSessionID = strings.Clone( state.forkGate.parentSessionID, ) @@ -253,6 +542,15 @@ func estimateCodexCursorEntryBytes( len(state.cwd)+ len(state.agentPath)+ len(state.lastTaskEvent)+ + codexPendingCallStringBytes(state)+ len(state.forkGate.parentSessionID), ) } + +func codexPendingCallStringBytes(state codexCursorState) int { + total := 0 + for i := 0; i < int(state.pendingCallCount); i++ { + total += len(state.pendingCalls[i].id) + len(state.pendingCalls[i].name) + } + return total +} diff --git a/internal/parser/codex_cursor_test.go b/internal/parser/codex_cursor_test.go index e0f727000..721d38f8a 100644 --- a/internal/parser/codex_cursor_test.go +++ b/internal/parser/codex_cursor_test.go @@ -141,12 +141,16 @@ func TestCodexCursorCache(t *testing.T) { }) t.Run("least recently used bytes are evicted", func(t *testing.T) { - cache := newCodexCursorCache(10, 900) path := filepath.Join(t.TempDir(), "rollout.jsonl") first := state first.cwd = strings.Repeat("a", 200) second := state second.cwd = strings.Repeat("b", 200) + firstBytes := estimateCodexCursorEntryBytes(newCodexCursorKey(path, 10, 1, 2), first) + secondBytes := estimateCodexCursorEntryBytes(newCodexCursorKey(path, 20, 1, 2), second) + cache := newCodexCursorCache( + 10, max(firstBytes, secondBytes)+min(firstBytes, secondBytes)-1, + ) require.True(t, cache.Put(path, 10, 1, 2, first)) require.True(t, cache.Put(path, 20, 1, 2, second)) @@ -159,10 +163,11 @@ func TestCodexCursorCache(t *testing.T) { }) t.Run("oversized entry is rejected without disturbing cache", func(t *testing.T) { - cache := newCodexCursorCache(4, 900) path := filepath.Join(t.TempDir(), "rollout.jsonl") + maxBytes := estimateCodexCursorEntryBytes(newCodexCursorKey(path, 10, 1, 2), state) + 32 + cache := newCodexCursorCache(4, maxBytes) oversized := state - oversized.cwd = strings.Repeat("x", 901) + oversized.cwd = strings.Repeat("x", int(maxBytes)+1) require.True(t, cache.Put(path, 10, 1, 2, state)) assert.False(t, cache.Put(path, 20, 1, 2, oversized)) @@ -233,7 +238,7 @@ func TestCodexCursorFullParseSeedBoundaries(t *testing.T) { assert.Equal(t, int64(0), sess.File.Size) info, err := os.Stat(path) require.NoError(t, err) - inode, device := sourceFileIdentity(info) + inode, device := sourceFileIdentityForPath(path, info) _, ok := provider.cursorCache.Get(path, 0, inode, device) assert.True(t, ok) }) @@ -256,7 +261,7 @@ func TestCodexCursorFullParseSeedBoundaries(t *testing.T) { assert.Equal(t, int64(len(content)), sess.File.Size) info, err := os.Stat(path) require.NoError(t, err) - inode, device := sourceFileIdentity(info) + inode, device := sourceFileIdentityForPath(path, info) _, ok := provider.cursorCache.Get( path, int64(len(content)), inode, device, ) @@ -277,7 +282,7 @@ func TestCodexCursorFullParseSeedBoundaries(t *testing.T) { assert.Equal(t, int64(len(content)), sess.File.Size) info, err := os.Stat(path) require.NoError(t, err) - inode, device := sourceFileIdentity(info) + inode, device := sourceFileIdentityForPath(path, info) _, ok := provider.cursorCache.Get( path, int64(len(content)), inode, device, ) diff --git a/internal/parser/codex_hash_tee.go b/internal/parser/codex_hash_tee.go new file mode 100644 index 000000000..8ffe4ae8c --- /dev/null +++ b/internal/parser/codex_hash_tee.go @@ -0,0 +1,96 @@ +package parser + +import ( + "crypto/sha256" + "encoding" + "encoding/hex" + "fmt" + "hash" + "io" +) + +// codexCheckpointAnchorSize is the trailing window whose digest the +// checkpoint stores instead of the raw anchor bytes. +const codexCheckpointAnchorSize = 128 << 10 + +// codexHashAnchorTee wraps a snapshot reader so one pass produces both the +// resumable SHA-256 state covering every byte read and the digest of the +// trailing anchor window. The Codex full parse threads it under the line +// reader, which eliminates the second full-file read the checkpoint +// persistence used to perform. +type codexHashAnchorTee struct { + r io.Reader + h hash.Hash + ring []byte + pos int + total int64 +} + +func newCodexHashAnchorTee(r io.Reader) *codexHashAnchorTee { + return &codexHashAnchorTee{ + r: r, + h: sha256.New(), + ring: make([]byte, codexCheckpointAnchorSize), + } +} + +func (t *codexHashAnchorTee) Read(p []byte) (int, error) { + n, err := t.r.Read(p) + if n > 0 { + _, _ = t.h.Write(p[:n]) + t.total += int64(n) + // Keep the trailing anchor window in chronological order with + // bulk copies: this runs on the full-parse hot path for every + // byte of the snapshot, so per-byte loops are not acceptable. + chunk := p[:n] + for len(chunk) > 0 { + space := len(t.ring) - t.pos + if len(chunk) <= space { + copy(t.ring[t.pos:], chunk) + t.pos += len(chunk) + break + } + copy(t.ring[t.pos:], chunk[:space]) + chunk = chunk[space:] + t.pos = 0 + } + } + return n, err +} + +// HashState returns the resumable SHA-256 state covering all bytes read so +// far. +func (t *codexHashAnchorTee) HashState() ([]byte, error) { + m, ok := t.h.(encoding.BinaryMarshaler) + if !ok { + return nil, fmt.Errorf("sha256 does not support state capture") + } + state, err := m.MarshalBinary() + if err != nil { + return nil, fmt.Errorf("marshaling codex hash state: %w", err) + } + return state, nil +} + +// HashDigest finalizes the current state into the full digest. +func (t *codexHashAnchorTee) HashDigest() (string, error) { + return hex.EncodeToString(t.h.Sum(nil)), nil +} + +// AnchorDigest returns the SHA-256 digest of the last +// min(codexCheckpointAnchorSize, total) bytes read, in order. +func (t *codexHashAnchorTee) AnchorDigest() string { + var anchor []byte + switch { + case t.total <= int64(len(t.ring)): + anchor = t.ring[:t.total] + case t.pos == 0: + anchor = t.ring + default: + anchor = make([]byte, len(t.ring)) + copy(anchor, t.ring[t.pos:]) + copy(anchor[len(t.ring)-t.pos:], t.ring[:t.pos]) + } + sum := sha256.Sum256(anchor) + return hex.EncodeToString(sum[:]) +} diff --git a/internal/parser/codex_hash_tee_test.go b/internal/parser/codex_hash_tee_test.go new file mode 100644 index 000000000..b54e9134f --- /dev/null +++ b/internal/parser/codex_hash_tee_test.go @@ -0,0 +1,53 @@ +package parser + +import ( + "bytes" + "crypto/rand" + "crypto/sha256" + "encoding" + "encoding/hex" + "io" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestCodexHashAnchorTeeWrapAndDigest feeds a payload larger than the +// anchor window through the tee and verifies the state digest equals the +// whole payload's hash while the anchor digest equals the trailing +// window's hash — including the ring wrap boundary. +func TestCodexHashAnchorTeeWrapAndDigest(t *testing.T) { + payload := make([]byte, 300<<10) + _, err := rand.Read(payload) + require.NoError(t, err) + + tee := newCodexHashAnchorTee(bytes.NewReader(payload)) + buf := make([]byte, 64<<10) + for { + _, err := tee.Read(buf) + if err == io.EOF { + break + } + require.NoError(t, err) + } + + state, err := tee.HashState() + require.NoError(t, err) + h := sha256.New() + require.NoError(t, h.(encoding.BinaryUnmarshaler).UnmarshalBinary(state)) + require.Equal(t, sha256.Sum256(payload), *(*[32]byte)(h.Sum(nil))) + + wantAnchor := sha256.Sum256(payload[len(payload)-codexCheckpointAnchorSize:]) + require.Equal(t, hex.EncodeToString(wantAnchor[:]), tee.AnchorDigest()) +} + +// TestCodexHashAnchorTeeSmallPayload pins the sub-window behavior: a +// payload shorter than the anchor window digests the whole prefix. +func TestCodexHashAnchorTeeSmallPayload(t *testing.T) { + payload := []byte("short codex snapshot") + tee := newCodexHashAnchorTee(bytes.NewReader(payload)) + _, err := io.Copy(io.Discard, tee) + require.NoError(t, err) + wantAnchor := sha256.Sum256(payload) + require.Equal(t, hex.EncodeToString(wantAnchor[:]), tee.AnchorDigest()) +} diff --git a/internal/parser/codex_index_change_time_darwin.go b/internal/parser/codex_index_change_time_darwin.go index 04bf579b2..343455341 100644 --- a/internal/parser/codex_index_change_time_darwin.go +++ b/internal/parser/codex_index_change_time_darwin.go @@ -7,10 +7,14 @@ import ( "syscall" ) -func codexIndexChangeTime(_ string, info os.FileInfo) (int64, bool) { +func codexIndexChangeTimeForFile(_ *os.File, info os.FileInfo) (int64, bool) { stat, ok := info.Sys().(*syscall.Stat_t) if !ok { return 0, false } return stat.Ctimespec.Sec*1_000_000_000 + stat.Ctimespec.Nsec, true } + +func codexIndexChangeTime(_ string, info os.FileInfo) (int64, bool) { + return codexIndexChangeTimeForFile(nil, info) +} diff --git a/internal/parser/codex_index_change_time_linux.go b/internal/parser/codex_index_change_time_linux.go index 7f8dce600..065e5a59e 100644 --- a/internal/parser/codex_index_change_time_linux.go +++ b/internal/parser/codex_index_change_time_linux.go @@ -7,10 +7,14 @@ import ( "syscall" ) -func codexIndexChangeTime(_ string, info os.FileInfo) (int64, bool) { +func codexIndexChangeTimeForFile(_ *os.File, info os.FileInfo) (int64, bool) { stat, ok := info.Sys().(*syscall.Stat_t) if !ok { return 0, false } return stat.Ctim.Sec*1_000_000_000 + stat.Ctim.Nsec, true } + +func codexIndexChangeTime(_ string, info os.FileInfo) (int64, bool) { + return codexIndexChangeTimeForFile(nil, info) +} diff --git a/internal/parser/codex_index_change_time_other.go b/internal/parser/codex_index_change_time_other.go index fdb71eb4d..63144bdf3 100644 --- a/internal/parser/codex_index_change_time_other.go +++ b/internal/parser/codex_index_change_time_other.go @@ -4,6 +4,10 @@ package parser import "os" -func codexIndexChangeTime(_ string, _ os.FileInfo) (int64, bool) { +func codexIndexChangeTimeForFile(_ *os.File, _ os.FileInfo) (int64, bool) { return 0, false } + +func codexIndexChangeTime(_ string, info os.FileInfo) (int64, bool) { + return codexIndexChangeTimeForFile(nil, info) +} diff --git a/internal/parser/codex_index_change_time_windows.go b/internal/parser/codex_index_change_time_windows.go index e61255e47..92be82609 100644 --- a/internal/parser/codex_index_change_time_windows.go +++ b/internal/parser/codex_index_change_time_windows.go @@ -9,6 +9,8 @@ import ( "golang.org/x/sys/windows" ) +const codexNTFSEpochOffset = int64(116444736000000000) + type codexIndexWindowsFileBasicInfo struct { creationTime int64 lastAccessTime int64 @@ -18,15 +20,12 @@ type codexIndexWindowsFileBasicInfo struct { _ uint32 } -func codexIndexChangeTime(path string, _ os.FileInfo) (int64, bool) { - file, err := os.Open(path) - if err != nil { +func codexIndexChangeTimeForFile(file *os.File, _ os.FileInfo) (int64, bool) { + if file == nil { return 0, false } - defer file.Close() - var info codexIndexWindowsFileBasicInfo - err = windows.GetFileInformationByHandleEx( + err := windows.GetFileInformationByHandleEx( windows.Handle(file.Fd()), windows.FileBasicInfo, (*byte)(unsafe.Pointer(&info)), @@ -35,5 +34,14 @@ func codexIndexChangeTime(path string, _ os.FileInfo) (int64, bool) { if err != nil || info.changeTime == 0 { return 0, false } - return info.changeTime, true + return (info.changeTime - codexNTFSEpochOffset) * 100, true +} + +func codexIndexChangeTime(path string, info os.FileInfo) (int64, bool) { + file, err := os.Open(path) + if err != nil { + return 0, false + } + defer file.Close() + return codexIndexChangeTimeForFile(file, info) } diff --git a/internal/parser/codex_parser_test.go b/internal/parser/codex_parser_test.go index 3d05cd657..e991d61f2 100644 --- a/internal/parser/codex_parser_test.go +++ b/internal/parser/codex_parser_test.go @@ -533,7 +533,9 @@ func TestCodexBuilderCanUseLexicalProjectDiscovery(t *testing.T) { } ctx := WithoutFilesystemProjectDiscovery(t.Context()) - builder := newCodexSessionBuilder(ctx, false, nil) + builder := newCodexSessionBuilder( + ctx, false, nil, NewCodexCollectingSink(0), + ) builder.handleSessionMeta(gjson.Parse(`{"id":"abc","cwd":`+ fmt.Sprintf("%q", cwd)+`}`), time.Time{}) @@ -541,15 +543,16 @@ func TestCodexBuilderCanUseLexicalProjectDiscovery(t *testing.T) { } func TestCodexInsertMessage_PreservesChronologyOnSameOrdinal(t *testing.T) { - b := newCodexSessionBuilder(context.Background(), false, nil) - b.messages = []ParsedMessage{{ + s := NewCodexCollectingSink(0) + s.messages = []ParsedMessage{{ Ordinal: 2, Role: RoleAssistant, Content: "later assistant message", Timestamp: parseTimestamp("2024-01-01T10:01:06Z"), }} + s.nextOrdinal = 3 - idx := b.insertMessage(ParsedMessage{ + idx := s.InsertMessage(ParsedMessage{ Ordinal: 2, Role: RoleUser, Content: "earlier orphan notification", @@ -557,12 +560,12 @@ func TestCodexInsertMessage_PreservesChronologyOnSameOrdinal(t *testing.T) { }) assert.Equal(t, 0, idx) - b.normalizeOrdinals() - require.Len(t, b.messages, 2) - assert.Equal(t, "earlier orphan notification", b.messages[0].Content) - assert.Equal(t, "later assistant message", b.messages[1].Content) - assert.Equal(t, 0, b.messages[0].Ordinal) - assert.Equal(t, 1, b.messages[1].Ordinal) + s.Finalize() + require.Len(t, s.messages, 2) + assert.Equal(t, "earlier orphan notification", s.messages[0].Content) + assert.Equal(t, "later assistant message", s.messages[1].Content) + assert.Equal(t, 0, s.messages[0].Ordinal) + assert.Equal(t, 1, s.messages[1].Ordinal) } func TestParseCodexSession_FunctionCalls(t *testing.T) { @@ -668,12 +671,17 @@ func TestParseCodexSession_FunctionCalls(t *testing.T) { assert.False(t, msgs[0].HasToolUse) }) - t.Run("custom_tool_call_output for a stored call requests full parse", func(t *testing.T) { + t.Run("custom_tool_call_output for a seeded call stays incremental", func(t *testing.T) { + // The late-output path merges outputs for calls recorded in the + // persisted prefix through toolCallUpdates, so the incremental + // gate must not request a full parse for them (P2 contract). line := `{"timestamp":"2026-07-08T03:20:43.376Z","type":"response_item","payload":{"type":"custom_tool_call_output","call_id":"call_abc","output":"Exit code: 0\nWall time: 0 seconds\nOutput:\nSuccess."}}` - b := newCodexSessionBuilder(context.Background(), false, nil) - assert.True(t, - b.incrementalOutputNeedsFullParse(gjson.Get(line, "payload"))) + b := newCodexSessionBuilder( + context.Background(), false, nil, NewCodexCollectingSink(0), + ) + b.rememberToolCall("call_abc", "exec_command", &ParsedToolCallPosition{MessageOrdinal: 1, CallIndex: 0}) + assert.False(t, b.codexIncrementalNeedsFullParse(line)) }) t.Run("write_stdin formats with session and chars", func(t *testing.T) { @@ -1737,6 +1745,53 @@ func TestParseCodexSession_ForkedSessionSkipsReplayedHistory(t *testing.T) { }) } +func TestParseCodexSessionWithCursorActiveForkGateNeverPersistsCheckpoint( + t *testing.T, +) { + forkCreatedMs := time.Date(2024, 1, 1, 10, 0, 0, 0, time.UTC).UnixMilli() + forkID := testUUIDv7(forkCreatedMs, 1) + parentID := testUUIDv7(forkCreatedMs-7200_000, 0) + parentTurnID := testUUIDv7(forkCreatedMs-3600_000, 2) + root := t.TempDir() + parent := testjsonl.JoinJSONL( + testjsonl.CodexSessionMetaJSON(parentID, "/tmp", "user", tsEarly), + testjsonl.CodexTurnContextWithIDJSON("gpt-5.4", parentTurnID, tsEarly), + ) + require.NoError(t, os.WriteFile( + filepath.Join(root, "rollout-2024-01-01T08-00-00-"+parentID+".jsonl"), + []byte(parent), 0o600, + )) + + // The fork ends while the replay gate is still open: only the copied + // parent meta and a replayed parent turn have arrived so far. Such a + // snapshot must not persist a resume cursor, or a later append would + // import the remaining replayed parent records as child content. + content := testjsonl.JoinJSONL( + testjsonl.CodexForkedSessionMetaJSON( + forkID, parentID, "/tmp", "user", tsEarly, + ), + testjsonl.CodexSessionMetaJSON(parentID, "/tmp", "user", tsEarly), + testjsonl.CodexTurnContextWithIDJSON( + "gpt-5.4", parentTurnID, tsEarly, + ), + ) + path := filepath.Join( + root, "rollout-2024-01-01T10-00-00-"+forkID+".jsonl", + ) + require.NoError(t, os.WriteFile(path, []byte(content), 0o600)) + + provider := newCodexTestProvider(t, root) + source := requireCodexProviderSource(t, provider, forkID) + outcome, err := provider.Parse(context.Background(), ParseRequest{ + Source: source, + }) + require.NoError(t, err) + require.Len(t, outcome.Results, 1) + assert.Empty(t, outcome.Results[0].Result.Checkpoint, + "a snapshot ending inside replayed parent history must not "+ + "persist a checkpoint") +} + func TestParseCodexSession_ForkBoundaryTreatsTurnIDsAsOpaque(t *testing.T) { t.Parallel() @@ -2561,7 +2616,7 @@ func TestCodexCursorWarmColdParity(t *testing.T) { prefixInfo, err := os.Stat(path) require.NoError(t, err) prefixOffset := prefixInfo.Size() - inode, device := sourceFileIdentity(prefixInfo) + inode, device := sourceFileIdentityForPath(path, prefixInfo) _, cursorHit := warmProvider.cursorCache.Get( path, prefixOffset, inode, device, ) @@ -2630,7 +2685,7 @@ func TestCodexPromptReplayDigestParity(t *testing.T) { prefixInfo, err := os.Stat(path) require.NoError(t, err) prefixOffset := prefixInfo.Size() - inode, device := sourceFileIdentity(prefixInfo) + inode, device := sourceFileIdentityForPath(path, prefixInfo) seed, cursorHit := warmProvider.cursorCache.Get( path, prefixOffset, inode, device, ) @@ -2772,7 +2827,7 @@ func TestParseCodexSessionFrom_LateTokenCountRequiresFullParse(t *testing.T) { assert.True(t, IsIncrementalFullParseFallback(err)) } -func TestParseCodexSessionFrom_FunctionCallOutputRequiresFullParse(t *testing.T) { +func TestParseCodexSessionFrom_FunctionCallOutputUpdatesStoredCall(t *testing.T) { t.Parallel() initial := testjsonl.JoinJSONL( @@ -2802,9 +2857,21 @@ func TestParseCodexSessionFrom_FunctionCallOutputRequiresFullParse(t *testing.T) require.NoError(t, err) require.NoError(t, f.Close()) - _, _, _, err = parseCodexTestSessionFrom(t, path, offset, 2, false) - require.Error(t, err) - assert.True(t, IsIncrementalFullParseFallback(err)) + result, err := newCodexTestProvider(t).parseSessionFromDetailed( + path, offset, 2, false, + ) + require.NoError(t, err) + assert.Empty(t, result.messages) + require.Len(t, result.toolCallUpdates, 1) + assert.Equal(t, "call_cmd", result.toolCallUpdates[0].ToolUseID) + assertToolResultEvents(t, + result.toolCallUpdates[0].ResultEvents, + []ParsedToolResultEvent{{ + ToolUseID: "call_cmd", + Source: "function_call_output", + Content: "done", + }}, + ) } // A tool call and its output that both arrive in the same appended @@ -3525,3 +3592,34 @@ func TestParseCodexSession_TurnAbortedNotCountedAsUser(t *testing.T) { " synthetic must be filtered from message list") } } + +func TestCodexDuplicateCallIDsAttachOutputsByOccurrence(t *testing.T) { + const callID = "reused-call" + content := testjsonl.JoinJSONL( + testjsonl.CodexSessionMetaJSON( + "duplicate-call-ids", "/tmp", "user", tsEarly, + ), + testjsonl.CodexMsgJSON("user", "run both", tsEarlyS1), + testjsonl.CodexFunctionCallWithCallIDJSON( + "exec_command", callID, nil, tsEarlyS5, + ), + testjsonl.CodexFunctionCallWithCallIDJSON( + "apply_patch", callID, nil, tsLate, + ), + testjsonl.CodexFunctionCallOutputJSON( + callID, "first result", tsLateS5, + ), + testjsonl.CodexFunctionCallOutputJSON( + callID, "second result", "2024-01-01T10:01:06Z", + ), + ) + + _, msgs := runCodexParserTest(t, "duplicate-call-ids.jsonl", content, false) + require.Len(t, msgs, 3) + require.Len(t, msgs[1].ToolCalls, 1) + require.Len(t, msgs[2].ToolCalls, 1) + require.Len(t, msgs[1].ToolCalls[0].ResultEvents, 1) + require.Len(t, msgs[2].ToolCalls[0].ResultEvents, 1) + assert.Equal(t, "first result", msgs[1].ToolCalls[0].ResultEvents[0].Content) + assert.Equal(t, "second result", msgs[2].ToolCalls[0].ResultEvents[0].Content) +} diff --git a/internal/parser/codex_provider.go b/internal/parser/codex_provider.go index f84030484..aaf7bc901 100644 --- a/internal/parser/codex_provider.go +++ b/internal/parser/codex_provider.go @@ -396,8 +396,8 @@ func (p *codexProvider) Parse( EvictCodexSessionIndexForSession(path) } machine := firstNonEmptyJSONLString(req.Machine, p.Config.Machine) - parentID, parentResolved := p.codexParentResolution(ctx, path) - sess, msgs, err := p.parseSessionContext(ctx, path, machine, false) + sess, msgs, cursor, safe, hashState, anchorDigest, retryReason, err := + p.parseSessionWithCursor(ctx, path, machine, false) if err != nil { return ParseOutcome{}, err } @@ -413,16 +413,28 @@ func (p *codexProvider) Parse( if req.Fingerprint.Hash != "" { sess.File.Hash = req.Fingerprint.Hash } + var checkpoint []byte + if safe { + checkpoint, err = cursor.MarshalBinary() + if err != nil { + return ParseOutcome{}, fmt.Errorf( + "encoding codex checkpoint %s: %w", path, err, + ) + } + } result := ParseResultOutcome{ Result: ParseResult{ - Session: *sess, - Messages: msgs, + Session: *sess, + Messages: msgs, + Checkpoint: checkpoint, + CheckpointHashState: hashState, + CheckpointAnchorDigest: anchorDigest, }, DataVersion: DataVersionCurrent, } - if parentID != "" && !parentResolved { + if retryReason != "" { result.DataVersion = DataVersionNeedsRetry - result.RetryReason = "codex parent turns unresolved for " + parentID + result.RetryReason = retryReason } return ParseOutcome{ Results: []ParseResultOutcome{result}, @@ -437,6 +449,70 @@ func (p *codexProvider) Parse( }, nil } +// ParseCodexSessionStreaming decodes one Codex snapshot, emitting every +// normalized operation into sink instead of accumulating the message slice +// inside the parser. It returns the assembled session, the finalized +// message slice (result-event content omitted for staging sinks), the +// marshaled continuation cursor, the single-pass hash state and anchor +// digest covering the snapshot, and a retry reason when an explicit fork +// parent could not be resolved. The cursor, hash state, and anchor digest +// are empty when the snapshot does not end at a safe resume boundary. +func ParseCodexSessionStreaming( + cfg ProviderConfig, + source SourceRef, + sink CodexSessionSink, +) (*ParsedSession, []ParsedMessage, []byte, []byte, string, string, error) { + provider, ok := NewProvider(AgentCodex, cfg) + if !ok { + return nil, nil, nil, nil, "", "", + fmt.Errorf("constructing codex provider") + } + cp, ok := provider.(*codexProvider) + if !ok { + return nil, nil, nil, nil, "", "", + fmt.Errorf("unexpected codex provider type %T", provider) + } + path, ok := cp.sources.pathFromSource(source) + if !ok { + return nil, nil, nil, nil, "", "", + fmt.Errorf("codex source path unavailable") + } + f, err := os.Open(path) + if err != nil { + return nil, nil, nil, nil, "", "", fmt.Errorf("open %s: %w", path, err) + } + defer f.Close() + info, err := f.Stat() + if err != nil { + return nil, nil, nil, nil, "", "", fmt.Errorf("stat %s: %w", path, err) + } + sess, msgs, cursor, safe, hashState, anchorDigest, retryReason, err := + cp.parseCodexSessionSnapshotStreaming( + context.Background(), path, + firstNonEmptyJSONLString("", cfg.Machine), + false, f, info, sink, + ) + if err != nil { + return nil, nil, nil, nil, "", "", err + } + if sess == nil { + return nil, nil, nil, nil, "", "", fmt.Errorf( + "codex session unavailable in %s", path, + ) + } + var cursorBlob []byte + if safe { + cursorBlob, err = cursor.MarshalBinary() + if err != nil { + return nil, nil, nil, nil, "", "", fmt.Errorf( + "encoding codex checkpoint %s: %w", path, err, + ) + } + } + return sess, msgs, cursorBlob, hashState, anchorDigest, + retryReason, nil +} + func (p *codexProvider) ParseIncremental( ctx context.Context, req IncrementalRequest, @@ -462,7 +538,7 @@ func (p *codexProvider) ParseIncremental( if err != nil { return IncrementalOutcome{}, IncrementalNeedsFullParse, err } - inode, device := sourceFileIdentity(info) + inode, device := sourceFileIdentityForFile(f, info) if (req.Fingerprint.Inode != 0 && req.Fingerprint.Inode != inode) || (req.Fingerprint.Device != 0 && req.Fingerprint.Device != device) || info.Size() < req.Fingerprint.Size { @@ -481,15 +557,38 @@ func (p *codexProvider) ParseIncremental( return IncrementalOutcome{}, IncrementalNoNewData, nil } - result, err := p.parseSessionFromSnapshot( - path, - req.Offset, - req.StartOrdinal, - false, - f, - info, - req.Fingerprint.Size, - ) + var result codexIncrementalParseResult + if len(req.Seed) > 0 { + var seed codexCursorState + if err := seed.UnmarshalBinary(req.Seed); err != nil { + // A persisted cursor the current binary cannot decode must not + // resume; rebuild the transcript authoritatively. + return IncrementalOutcome{ForceReplace: true}, + IncrementalNeedsFullParse, nil + } + result, err = p.parseSessionFromCheckpoint( + path, + req.Offset, + req.StartOrdinal, + false, + f, + info, + req.Fingerprint.Size, + seed, + req.StoredPendingUsageOrdinal, + ) + } else { + result, err = p.parseSessionFromSnapshot( + path, + req.Offset, + req.StartOrdinal, + false, + f, + info, + req.Fingerprint.Size, + req.StoredPendingUsageOrdinal, + ) + } if err != nil { if IsIncrementalFullParseFallback(err) { return IncrementalOutcome{ForceReplace: true}, @@ -529,18 +628,27 @@ func (p *codexProvider) ParseIncremental( totalOut, peakCtx, hasTotalOut, hasPeakCtx := codexProviderTokenTotals(result.messages) termination := codexIncrementalTermination(result.cursor.lastTaskEvent) + nextCursor, err := result.cursor.MarshalBinary() + if err != nil { + return IncrementalOutcome{}, IncrementalNeedsFullParse, fmt.Errorf( + "encoding codex cursor %s: %w", path, err, + ) + } return IncrementalOutcome{ - SessionID: req.SessionID, - Messages: result.messages, - EndedAt: result.endedAt, - ConsumedBytes: result.consumedBytes, - MessageCount: len(result.messages), - UserMessageCount: codexProviderUserMessageCount(result.messages), - TotalOutputTokens: totalOut, - PeakContextTokens: peakCtx, - HasTotalOutputTokens: hasTotalOut, - HasPeakContextTokens: hasPeakCtx, - TerminationStatus: termination, + SessionID: req.SessionID, + Messages: result.messages, + ToolCallUpdates: result.toolCallUpdates, + MessageTokenUsageUpdates: result.messageUsageUpdates, + NextCursor: nextCursor, + EndedAt: result.endedAt, + ConsumedBytes: result.consumedBytes, + MessageCount: len(result.messages), + UserMessageCount: codexProviderUserMessageCount(result.messages), + TotalOutputTokens: totalOut, + PeakContextTokens: peakCtx, + HasTotalOutputTokens: hasTotalOut, + HasPeakContextTokens: hasPeakCtx, + TerminationStatus: termination, }, IncrementalApplied, nil } @@ -911,7 +1019,7 @@ func (s codexSourceSet) Fingerprint( if err != nil { return SourceFingerprint{}, err } - inode, device := sourceFileIdentity(info) + inode, device := sourceFileIdentityForPath(path, info) mtime := info.ModTime().UnixNano() if s.agent == AgentCodex { mtime = CodexEffectiveMtime(path, mtime) diff --git a/internal/parser/codex_provider_test.go b/internal/parser/codex_provider_test.go index 06d64adab..58ddbbed9 100644 --- a/internal/parser/codex_provider_test.go +++ b/internal/parser/codex_provider_test.go @@ -75,7 +75,7 @@ func TestCodexProviderSourceMethods(t *testing.T) { assert.Equal(t, sourcePath, fingerprint.Key) assert.Equal(t, info.Size(), fingerprint.Size) assert.Equal(t, newer.UnixNano(), fingerprint.MTimeNS) - wantInode, wantDevice := sourceFileIdentity(info) + wantInode, wantDevice := sourceFileIdentityForPath(sourcePath, info) assert.Equal(t, wantInode, fingerprint.Inode) assert.Equal(t, wantDevice, fingerprint.Device) assert.NotEmpty(t, fingerprint.Hash) @@ -127,6 +127,109 @@ func TestCodexProviderUnresolvedParentNeedsRetry(t *testing.T) { assert.Equal(t, "child answer", result.Result.Messages[1].Content) } +func TestCodexProviderReadableParentWithoutTurnsRetriesUntilParentAdvances( + t *testing.T, +) { + root := t.TempDir() + const childID = "22222222-2222-4222-8222-222222222223" + const parentID = "11111111-1111-4111-8111-111111111112" + const parentTurnID = "parent-turn-later" + const childTurnID = "child-turn" + + parentPath := writeCodexProviderSessionContent(t, root, parentID, + testjsonl.JoinJSONL(testjsonl.CodexSessionMetaJSON( + parentID, "/workspace/project", "codex_cli_rs", tsEarly, + )), + ) + childContent := testjsonl.JoinJSONL( + testjsonl.CodexForkedSessionMetaJSON( + childID, parentID, "/workspace/project", "codex_cli_rs", tsEarly, + ), + testjsonl.CodexSessionMetaJSON( + parentID, "/workspace/project", "codex_cli_rs", tsEarly, + ), + testjsonl.CodexTurnContextWithIDJSON( + "gpt-5.4", parentTurnID, tsEarlyS1, + ), + testjsonl.CodexMsgJSON("user", "replayed task", tsEarlyS1), + testjsonl.CodexMsgJSON("assistant", "replayed answer", "2024-01-01T10:00:02Z"), + testjsonl.CodexTurnContextWithIDJSON( + "gpt-5.4", childTurnID, "2024-01-01T10:00:03Z", + ), + testjsonl.CodexMsgJSON("user", "child task", "2024-01-01T10:00:03Z"), + testjsonl.CodexMsgJSON("assistant", "child answer", tsEarlyS5), + ) + writeCodexProviderSessionContent(t, root, childID, childContent) + + provider, ok := NewProvider(AgentCodex, ProviderConfig{Roots: []string{root}}) + require.True(t, ok) + source := requireCodexProviderSource(t, provider, childID) + + first, err := provider.Parse(t.Context(), ParseRequest{Source: source}) + require.NoError(t, err) + require.Len(t, first.Results, 1) + require.Equal(t, DataVersionNeedsRetry, first.Results[0].DataVersion, + "a readable parent with no turn IDs is still unresolved") + require.Contains(t, first.Results[0].RetryReason, "parent turns") + require.Len(t, first.Results[0].Result.Messages, 4, + "unresolved parsing fails open until the parent becomes usable") + + f, err := os.OpenFile(parentPath, os.O_APPEND|os.O_WRONLY, 0o644) + require.NoError(t, err) + _, err = f.WriteString(testjsonl.JoinJSONL( + testjsonl.CodexTurnContextWithIDJSON( + "gpt-5.4", parentTurnID, tsEarlyS1, + ), + )) + require.NoError(t, err) + require.NoError(t, f.Close()) + + second, err := provider.Parse(t.Context(), ParseRequest{Source: source}) + require.NoError(t, err) + require.Len(t, second.Results, 1) + require.Equal(t, DataVersionCurrent, second.Results[0].DataVersion) + require.Empty(t, second.Results[0].RetryReason) + require.Len(t, second.Results[0].Result.Messages, 2) + require.Equal(t, "child task", second.Results[0].Result.Messages[0].Content) + require.Equal(t, "child answer", second.Results[0].Result.Messages[1].Content) +} + +func TestCodexProviderTurnlessParentResolvesWithoutRetry(t *testing.T) { + // Codex Desktop writes a rollout when a thread opens; forking before the + // first prompt produces a parent holding only session_meta and settings + // events. Such a parent is present and fully readable, so the fork must + // parse as current instead of being marked for a retry that can never + // succeed and blocking every later reconciliation page behind it. + root := t.TempDir() + const childID = "22222222-2222-4222-8222-222222222222" + const parentID = "11111111-1111-4111-8111-111111111111" + writeCodexProviderSessionContent(t, root, parentID, testjsonl.JoinJSONL( + testjsonl.CodexSessionMetaJSON(parentID, "/workspace/project", "codex_cli_rs", tsEarly), + )) + writeCodexProviderSessionContent(t, root, childID, testjsonl.JoinJSONL( + testjsonl.CodexForkedSessionMetaJSON( + childID, parentID, "/workspace/project", "codex_cli_rs", tsEarly, + ), + testjsonl.CodexTurnContextWithIDJSON("gpt-5.4", "child-turn", tsEarlyS1), + testjsonl.CodexMsgJSON("user", "child task", tsEarlyS1), + testjsonl.CodexMsgJSON("assistant", "child answer", tsEarlyS5), + )) + provider, ok := NewProvider(AgentCodex, ProviderConfig{Roots: []string{root}}) + require.True(t, ok) + source := requireCodexProviderSource(t, provider, childID) + + outcome, err := provider.Parse(t.Context(), ParseRequest{Source: source}) + + require.NoError(t, err) + require.Len(t, outcome.Results, 1) + result := outcome.Results[0] + assert.Equal(t, DataVersionCurrent, result.DataVersion) + assert.Empty(t, result.RetryReason) + require.Len(t, result.Result.Messages, 2) + assert.Equal(t, "child task", result.Result.Messages[0].Content) + assert.Equal(t, "child answer", result.Result.Messages[1].Content) +} + func TestCodexProviderUnresolvedParentWithoutFinalNewlineNeedsRetry(t *testing.T) { root := t.TempDir() const childID = "22222222-2222-4222-8222-222222222222" @@ -310,7 +413,7 @@ func TestCodexProviderFactoryScopesCursorCache(t *testing.T) { require.NotNil(t, sess) info, err := os.Stat(path) require.NoError(t, err) - inode, device := sourceFileIdentity(info) + inode, device := sourceFileIdentityForPath(path, info) _, siblingHit := siblingProvider.cursorCache.Get( path, info.Size(), inode, device, @@ -362,7 +465,7 @@ func TestCodexProviderFullParseSnapshotExcludesLaterGrowth(t *testing.T) { require.Len(t, messages, 1) assert.Equal(t, RoleUser, messages[0].Role) assert.Equal(t, "captured request", messages[0].Content) - inode, device := sourceFileIdentity(capturedInfo) + inode, device := sourceFileIdentityForFile(snapshot, capturedInfo) seed, seeded := concrete.cursorCache.Get( path, int64(len(initial)), inode, device, ) @@ -446,14 +549,14 @@ func TestCodexProviderFullParseSnapshotKeepsDescriptorIdentityAfterReplacement( defer snapshot.Close() snapshotInfo, err := snapshot.Stat() require.NoError(t, err) - oldInode, oldDevice := sourceFileIdentity(snapshotInfo) + oldInode, oldDevice := sourceFileIdentityForFile(snapshot, snapshotInfo) replacementPath := path + ".replacement" require.NoError(t, os.WriteFile(replacementPath, []byte(replacement), 0o644)) require.NoError(t, os.Rename(replacementPath, path)) currentInfo, err := os.Stat(path) require.NoError(t, err) - newInode, newDevice := sourceFileIdentity(currentInfo) + newInode, newDevice := sourceFileIdentityForPath(path, currentInfo) sess, messages, err := concrete.parseSessionSnapshot( path, "local", false, snapshot, snapshotInfo, @@ -821,6 +924,142 @@ func TestCodexProviderIncrementalFirstGenuinePromptNeedsFullParse(t *testing.T) } } +func TestCodexProviderIncrementalCustomToolOutputUpdatesStoredCall(t *testing.T) { + const ( + uuid = "019eb791-cf7d-75c1-8439-9ed74c1229f7" + call = `{"timestamp":"2026-08-02T09:00:02Z","type":"response_item","payload":{"type":"custom_tool_call","name":"apply_patch","call_id":"call_patch","input":"*** Begin Patch\n*** End Patch"}}` + output = `{"timestamp":"2026-08-02T09:00:03Z","type":"response_item","payload":{"type":"custom_tool_call_output","call_id":"call_patch","output":"Success. Updated one file.","status":"completed"}}` + ) + prefix := testjsonl.JoinJSONL( + testjsonl.CodexSessionMetaJSON( + uuid, "/workspace/project-a", "codex_cli_rs", tsEarly, + ), + testjsonl.CodexMsgJSON("user", "apply the patch", tsEarlyS1), + call, + ) + tail := testjsonl.JoinJSONL(output) + + for _, mode := range []string{"warm", "cold"} { + t.Run(mode, func(t *testing.T) { + root := t.TempDir() + content := prefix + if mode == "cold" { + content += tail + } + path := writeCodexProviderSessionContent( + t, root, uuid, content, + ) + provider, ok := NewProvider( + AgentCodex, ProviderConfig{Roots: []string{root}}, + ) + require.True(t, ok) + source := requireCodexProviderSource(t, provider, uuid) + + if mode == "warm" { + fingerprint, err := provider.Fingerprint( + context.Background(), source, + ) + require.NoError(t, err) + _, err = provider.Parse(context.Background(), ParseRequest{ + Source: source, Fingerprint: fingerprint, + }) + require.NoError(t, err) + appendCodexProviderContent(t, path, tail) + } + + fingerprint, err := provider.Fingerprint( + context.Background(), source, + ) + require.NoError(t, err) + outcome, status, err := provider.ParseIncremental( + context.Background(), IncrementalRequest{ + Source: source, + Fingerprint: fingerprint, + SessionID: "codex:" + uuid, + Offset: int64(len(prefix)), + StartOrdinal: 2, + }, + ) + + require.NoError(t, err) + assert.Equal(t, IncrementalApplied, status) + assert.Empty(t, outcome.Messages) + require.Len(t, outcome.ToolCallUpdates, 1) + assert.Equal(t, "call_patch", outcome.ToolCallUpdates[0].ToolUseID) + require.Len(t, outcome.ToolCallUpdates[0].ResultEvents, 1) + event := outcome.ToolCallUpdates[0].ResultEvents[0] + assert.Equal(t, "custom_tool_call_output", event.Source) + assert.Equal(t, "completed", event.Status) + assert.Equal(t, "Success. Updated one file.", event.Content) + }) + } +} + +func TestCodexProviderIncrementalLateOutputAttachesCommittedUsage(t *testing.T) { + const uuid = "019eb791-cf7d-75c1-8439-9ed74c1229f8" + prefix := testjsonl.JoinJSONL( + testjsonl.CodexSessionMetaJSON( + uuid, "/workspace/project-a", "codex_cli_rs", tsEarly, + ), + testjsonl.CodexTurnContextJSON("gpt-5.4", tsEarlyS1), + testjsonl.CodexMsgJSON("user", "inspect the repository", tsEarlyS1), + testjsonl.CodexFunctionCallWithCallIDJSON( + "shell", "call_late", map[string]any{"cmd": "git status"}, tsEarlyS5, + ), + ) + tail := testjsonl.JoinJSONL( + testjsonl.CodexFunctionCallOutputJSON( + "call_late", "working tree clean", tsLate, + ), + testjsonl.CodexTokenCountJSON(tsLateS5, 100_000, 250, 64_000), + ) + + root := t.TempDir() + path := writeCodexProviderSessionContent(t, root, uuid, prefix) + provider, ok := NewProvider(AgentCodex, ProviderConfig{Roots: []string{root}}) + require.True(t, ok) + source := requireCodexProviderSource(t, provider, uuid) + prefixFingerprint, err := provider.Fingerprint(context.Background(), source) + require.NoError(t, err) + full, err := provider.Parse(context.Background(), ParseRequest{ + Source: source, Fingerprint: prefixFingerprint, + }) + require.NoError(t, err) + require.Len(t, full.Results, 1) + require.Len(t, full.Results[0].Result.Messages, 2) + assert.Equal(t, RoleAssistant, full.Results[0].Result.Messages[1].Role) + + appendCodexProviderContent(t, path, tail) + fingerprint, err := provider.Fingerprint(context.Background(), source) + require.NoError(t, err) + pendingUsageOrdinal := 1 + outcome, status, err := provider.ParseIncremental( + context.Background(), IncrementalRequest{ + Source: source, + Fingerprint: fingerprint, + SessionID: "codex:" + uuid, + Offset: prefixFingerprint.Size, + StartOrdinal: 2, + StoredPendingUsageOrdinal: &pendingUsageOrdinal, + }, + ) + + require.NoError(t, err) + assert.Equal(t, IncrementalApplied, status) + assert.False(t, outcome.ForceReplace) + assert.Empty(t, outcome.Messages) + require.Len(t, outcome.ToolCallUpdates, 1) + assert.Equal(t, "call_late", outcome.ToolCallUpdates[0].ToolUseID) + require.Len(t, outcome.MessageTokenUsageUpdates, 1) + usage := outcome.MessageTokenUsageUpdates[0] + assert.Equal(t, 1, usage.Ordinal) + assert.Equal(t, 100_000, usage.ContextTokens) + assert.Equal(t, 250, usage.OutputTokens) + assert.True(t, usage.HasContextTokens) + assert.True(t, usage.HasOutputTokens) + assert.NotEmpty(t, usage.TokenUsage) +} + func TestCodexProviderColdIncrementalStagesRetryCursorVersions(t *testing.T) { root := t.TempDir() uuid := "019eb791-cf7d-75c1-8439-9ed74c1229ed" @@ -2088,3 +2327,86 @@ func appendCodexProviderContent(t *testing.T, path, content string) { require.NoError(t, err) require.NoError(t, f.Close()) } + +func TestCodexProviderIncrementalUserBoundaryDoesNotBackfillCommittedUsage(t *testing.T) { + const uuid = "019eb791-cf7d-75c1-8439-9ed74c1229f9" + prefix := testjsonl.JoinJSONL( + testjsonl.CodexSessionMetaJSON( + uuid, "/workspace/project-a", "codex_cli_rs", tsEarly, + ), + testjsonl.CodexTurnContextJSON("gpt-5.4", tsEarlyS1), + testjsonl.CodexMsgJSON("user", "first request", tsEarlyS1), + testjsonl.CodexMsgJSON("assistant", "first response", tsEarlyS5), + ) + + t.Run("user without a new assistant blocks committed target", func(t *testing.T) { + root := t.TempDir() + path := writeCodexProviderSessionContent(t, root, uuid, prefix) + provider, ok := NewProvider(AgentCodex, ProviderConfig{Roots: []string{root}}) + require.True(t, ok) + source := requireCodexProviderSource(t, provider, uuid) + prefixFingerprint, err := provider.Fingerprint(context.Background(), source) + require.NoError(t, err) + + tail := testjsonl.JoinJSONL( + testjsonl.CodexMsgJSON("user", "second request", tsLate), + testjsonl.CodexTokenCountJSON(tsLateS5, 100_000, 250, 64_000), + ) + appendCodexProviderContent(t, path, tail) + fingerprint, err := provider.Fingerprint(context.Background(), source) + require.NoError(t, err) + pendingUsageOrdinal := 1 + outcome, status, err := provider.ParseIncremental( + context.Background(), IncrementalRequest{ + Source: source, + Fingerprint: fingerprint, + SessionID: "codex:" + uuid, + Offset: prefixFingerprint.Size, + StartOrdinal: 2, + StoredPendingUsageOrdinal: &pendingUsageOrdinal, + }, + ) + require.NoError(t, err) + assert.Equal(t, IncrementalApplied, status) + require.Len(t, outcome.Messages, 1) + assert.Equal(t, RoleUser, outcome.Messages[0].Role) + assert.Empty(t, outcome.MessageTokenUsageUpdates, + "usage after a real user boundary must not update the prior turn") + }) + + t.Run("new assistant after the user receives usage", func(t *testing.T) { + root := t.TempDir() + path := writeCodexProviderSessionContent(t, root, uuid, prefix) + provider, ok := NewProvider(AgentCodex, ProviderConfig{Roots: []string{root}}) + require.True(t, ok) + source := requireCodexProviderSource(t, provider, uuid) + prefixFingerprint, err := provider.Fingerprint(context.Background(), source) + require.NoError(t, err) + + tail := testjsonl.JoinJSONL( + testjsonl.CodexMsgJSON("user", "second request", tsLate), + testjsonl.CodexMsgJSON("assistant", "second response", tsLateS5), + testjsonl.CodexTokenCountJSON("2026-08-02T09:00:06Z", 100_000, 250, 64_000), + ) + appendCodexProviderContent(t, path, tail) + fingerprint, err := provider.Fingerprint(context.Background(), source) + require.NoError(t, err) + pendingUsageOrdinal := 1 + outcome, status, err := provider.ParseIncremental( + context.Background(), IncrementalRequest{ + Source: source, + Fingerprint: fingerprint, + SessionID: "codex:" + uuid, + Offset: prefixFingerprint.Size, + StartOrdinal: 2, + StoredPendingUsageOrdinal: &pendingUsageOrdinal, + }, + ) + require.NoError(t, err) + assert.Equal(t, IncrementalApplied, status) + require.Len(t, outcome.Messages, 2) + assert.Equal(t, RoleAssistant, outcome.Messages[1].Role) + assert.NotEmpty(t, outcome.Messages[1].TokenUsage) + assert.Empty(t, outcome.MessageTokenUsageUpdates) + }) +} diff --git a/internal/parser/codex_seed_sink.go b/internal/parser/codex_seed_sink.go new file mode 100644 index 000000000..2bfc9b832 --- /dev/null +++ b/internal/parser/codex_seed_sink.go @@ -0,0 +1,45 @@ +package parser + +// codexSeedSink assigns message ordinals while discarding normalized content. +// Prefix reconstruction uses it to recover occurrence-qualified pending-call +// coordinates without retaining the transcript in memory. +type codexSeedSink struct { + nextOrdinal int + hadReservation bool +} + +func newCodexSeedSink() *codexSeedSink { + return &codexSeedSink{} +} + +func (s *codexSeedSink) AppendMessage(ParsedMessage) int { + ordinal := s.nextOrdinal + s.nextOrdinal++ + return ordinal +} + +func (s *codexSeedSink) ReserveOrdinal() int { + ordinal := s.nextOrdinal + s.nextOrdinal++ + s.hadReservation = true + return ordinal +} + +func (*codexSeedSink) InsertMessage(ParsedMessage) int { return 0 } + +func (*codexSeedSink) AppendToolResultEvent( + string, *ParsedToolCallPosition, ParsedToolResultEvent, +) { +} + +func (*codexSeedSink) SetCallSubagentSessionID(string, *ParsedToolCallPosition, string) {} + +func (*codexSeedSink) ApplyTokenUsageToLastAssistant(string) bool { return false } + +func (*codexSeedSink) InsertOrphanMessage(string, ParsedMessage) bool { return true } + +func (*codexSeedSink) Finalize() {} + +func (*codexSeedSink) Messages() []ParsedMessage { return nil } + +func (*codexSeedSink) ToolCallUpdates() []ParsedToolCallUpdate { return nil } diff --git a/internal/parser/codex_sink.go b/internal/parser/codex_sink.go new file mode 100644 index 000000000..f33bf311a --- /dev/null +++ b/internal/parser/codex_sink.go @@ -0,0 +1,46 @@ +package parser + +// CodexSessionSink receives the normalized operations of one Codex +// transcript decode. The decoder owns parse-time state (cursor, fork +// gate, prompt replay observation, pending-agent attribution) and emits +// through this interface; the sink owns the message stream, its +// call_id -> (message, call) index, deferred tool-result updates, and +// final ordinal normalization. +// +// The collecting implementation keeps the whole session in memory and +// reproduces the pre-streaming behavior exactly; the streaming +// implementation batches operations into a scratch store so memory is +// O(batch + unresolved state) instead of O(file size). Every semantic +// the legacy builder performed in-place maps to exactly one operation: +// +// - AppendMessage — append a message at the tail (sink assigns +// the next ordinal; function-call messages register their call ids) +// - ReserveOrdinal — claim an ordinal slot without a message +// (pending subagent notifications hold their position until they +// are materialized or claimed by a late wait/spawn call) +// - InsertMessage — insert a message before the first message +// with a greater ordinal (same-ordinal ties broken by timestamp, +// then emission order), preserving the reserved slot's position +// - AppendToolResultEvent— attach a result event to an emitted call, +// or record a deferred update when the call id was never emitted +// (equivalent events are deduplicated) +// - SetCallSubagentSessionID — link an emitted call to its subagent +// - ApplyTokenUsageToLastAssistant — attach token usage to the last +// assistant message without usage, scanning back to the current +// turn's user boundary +// - InsertOrphanMessage — materialize a pending notification at its +// reserved ordinal, deduplicated by key +// - Finalize — stable-sort by ordinal (ties keep emission +// order) and renumber 0..n-1 +type CodexSessionSink interface { + AppendMessage(m ParsedMessage) int + ReserveOrdinal() int + InsertMessage(m ParsedMessage) int + AppendToolResultEvent(callID string, target *ParsedToolCallPosition, ev ParsedToolResultEvent) + SetCallSubagentSessionID(callID string, target *ParsedToolCallPosition, sessionID string) + ApplyTokenUsageToLastAssistant(raw string) bool + InsertOrphanMessage(key string, m ParsedMessage) bool + Finalize() + Messages() []ParsedMessage + ToolCallUpdates() []ParsedToolCallUpdate +} diff --git a/internal/parser/context_io_test.go b/internal/parser/context_io_test.go index f6448e996..9c06f146c 100644 --- a/internal/parser/context_io_test.go +++ b/internal/parser/context_io_test.go @@ -104,14 +104,16 @@ func TestClaudeDAGPostProcessingStopsAfterContextCancellation(t *testing.T) { require.ErrorIs(t, err, context.Canceled) } -func TestCodexPostReadNormalizationStopsAfterContextCancellation(t *testing.T) { - builder := newCodexSessionBuilder(context.Background(), false, nil) - for i := 1024; i > 0; i-- { - builder.messages = append(builder.messages, ParsedMessage{Ordinal: i}) +func TestCodexPostReadUsageAccumulationStopsAfterContextCancellation(t *testing.T) { + // Ordinal normalization now lives in the sink's Finalize; the post-read + // work the parser still owns is the per-message usage accumulation. + messages := make([]ParsedMessage, 1024) + for i := range messages { + messages[i] = ParsedMessage{Ordinal: i, HasOutputTokens: true, OutputTokens: 1} } ctx := newCancelOnErrCheckContext(t, 3) - err := builder.normalizeOrdinalsContext(ctx) + err := accumulateMessageTokenUsageContext(ctx, &ParsedSession{}, messages) require.ErrorIs(t, err, context.Canceled) } diff --git a/internal/parser/db_backed_provider.go b/internal/parser/db_backed_provider.go index fca7f19ef..009b889a4 100644 --- a/internal/parser/db_backed_provider.go +++ b/internal/parser/db_backed_provider.go @@ -587,9 +587,12 @@ func (s dbBackedSourceSet) dbPathForEvent(root, path string) (string, bool) { if strings.Contains(rel, string(filepath.Separator)) { return "", false } - if rel == s.spec.dbName || - rel == s.spec.dbName+"-wal" || - rel == s.spec.dbName+"-shm" { + // A bare "-shm" event never resolves to the container. Every write to a + // WAL-mode database lands in the main file or its -wal sibling; the -shm + // index is also rewritten by readers, including this process's own scan, + // so honoring it would make each scan schedule the next one. Omnigent and + // Cursor IDE apply the same rule through classifySQLiteContainerPath. + if rel == s.spec.dbName || rel == s.spec.dbName+"-wal" { dbPath := filepath.Join(root, s.spec.dbName) return dbPath, true } diff --git a/internal/parser/db_backed_provider_test.go b/internal/parser/db_backed_provider_test.go index 723fc9777..9416b4a7b 100644 --- a/internal/parser/db_backed_provider_test.go +++ b/internal/parser/db_backed_provider_test.go @@ -360,6 +360,33 @@ func TestDBBackedProviderFingerprintIgnoresUnrelatedRows(t *testing.T) { assert.Equal(t, before, after) } +func TestDBBackedProviderIgnoresBareShmSiblingEvents(t *testing.T) { + // Opening a WAL-mode database as a reader rewrites its -shm index. If that + // event resolved to the container, every scan would schedule the next + // one and rewrite every member session in between. + dbPath, seeder, db := newForgeTestDB(t) + defer db.Close() + seedForgeConversation(t, seeder) + root := filepath.Dir(dbPath) + + provider, ok := NewProvider(AgentForge, ProviderConfig{Roots: []string{root}}) + require.True(t, ok) + + changed, err := provider.SourcesForChangedPath( + context.Background(), + ChangedPathRequest{Path: dbPath + "-shm", EventKind: "write", WatchRoot: root}, + ) + require.NoError(t, err) + assert.Empty(t, changed) + + changed, err = provider.SourcesForChangedPath( + context.Background(), + ChangedPathRequest{Path: dbPath + "-wal", EventKind: "write", WatchRoot: root}, + ) + require.NoError(t, err) + assert.NotEmpty(t, changed, "-wal writes still resolve to the container") +} + func TestDBBackedProviderDeletedRowFingerprintsTombstoneAndSkips(t *testing.T) { dbPath, seeder, db := newForgeTestDB(t) defer db.Close() diff --git a/internal/parser/file_identity_path_unix.go b/internal/parser/file_identity_path_unix.go new file mode 100644 index 000000000..95044bf08 --- /dev/null +++ b/internal/parser/file_identity_path_unix.go @@ -0,0 +1,13 @@ +//go:build !windows + +package parser + +import "os" + +func sourceFileIdentityForFile(_ *os.File, info os.FileInfo) (inode, device uint64) { + return sourceFileIdentity(info) +} + +func sourceFileIdentityForPath(_ string, info os.FileInfo) (inode, device uint64) { + return sourceFileIdentity(info) +} diff --git a/internal/parser/file_identity_path_windows.go b/internal/parser/file_identity_path_windows.go new file mode 100644 index 000000000..9ca05bd82 --- /dev/null +++ b/internal/parser/file_identity_path_windows.go @@ -0,0 +1,37 @@ +//go:build windows + +package parser + +import ( + "os" + + "golang.org/x/sys/windows" +) + +// sourceFileIdentityForFile returns the stable Windows identity of the exact +// descriptor being parsed: file index plus volume serial number. +func sourceFileIdentityForFile(file *os.File, _ os.FileInfo) (inode, device uint64) { + if file == nil { + return 0, 0 + } + var info windows.ByHandleFileInformation + if err := windows.GetFileInformationByHandle( + windows.Handle(file.Fd()), &info, + ); err != nil { + return 0, 0 + } + fileIndex := uint64(info.FileIndexHigh)<<32 | uint64(info.FileIndexLow) + return fileIndex, uint64(info.VolumeSerialNumber) +} + +// sourceFileIdentityForPath opens path and returns the identity of that path's +// current file. Snapshot parsers must use sourceFileIdentityForFile instead so +// a concurrent path replacement cannot relabel an already-open descriptor. +func sourceFileIdentityForPath(path string, info os.FileInfo) (inode, device uint64) { + file, err := os.Open(path) + if err != nil { + return 0, 0 + } + defer file.Close() + return sourceFileIdentityForFile(file, info) +} diff --git a/internal/parser/provider.go b/internal/parser/provider.go index 6d1e95a8a..70c91f5b5 100644 --- a/internal/parser/provider.go +++ b/internal/parser/provider.go @@ -1026,6 +1026,12 @@ type IncrementalRequest struct { Offset int64 StartOrdinal int Machine string + // Seed is opaque provider continuation state persisted by the sync + // engine (a parser checkpoint). A provider that recognizes the format + // resumes from it instead of rescanning the committed prefix; empty + // means cold-start reconstruction. A seed the provider cannot decode + // must be treated as IncrementalNeedsFullParse. + Seed []byte // LastEntryUUID is the UUID of the last entry stored for this // session, used by DAG-aware parsers (Claude) to detect when an // appended tail forks away from the stored tip and must trigger a @@ -1056,13 +1062,23 @@ type IncrementalRequest struct { // the appended assistant head continues exactly this message id. // nil keeps the conservative fallback. StoredLastClaudeMessageID *string + // StoredPendingUsageOrdinal is the last assistant message without token + // usage in the current turn, as resolved from the committed transcript. + // Codex uses it to attach a token_count that follows a late tool result + // without re-reading or rebuilding the committed prefix. + StoredPendingUsageOrdinal *int } // IncrementalOutcome is the append-only parse output. type IncrementalOutcome struct { - SessionID string - Messages []ParsedMessage - SubagentLinks []ClaudeSubagentLink + SessionID string + Messages []ParsedMessage + SubagentLinks []ClaudeSubagentLink + ToolCallUpdates []ParsedToolCallUpdate + MessageTokenUsageUpdates []ParsedMessageTokenUsageUpdate + // NextCursor is the provider's continuation state after consuming the + // appended tail, for persistence alongside the committed offset. + NextCursor []byte EndedAt time.Time ConsumedBytes int64 MessageCount int diff --git a/internal/parser/types.go b/internal/parser/types.go index 1930c4cdd..63dfea6ac 100644 --- a/internal/parser/types.go +++ b/internal/parser/types.go @@ -1167,6 +1167,10 @@ type FileInfo struct { Inode int64 Device int64 Hash string + // ChangeTime is the change time captured from the same descriptor the + // parser read its snapshot from. Zero means the platform could not + // provide one; checkpoint consumers then rebuild conservatively. + ChangeTime int64 } // ParsedSession holds session metadata extracted from a JSONL file. @@ -1260,6 +1264,38 @@ type ParsedToolCall struct { ResultEvents []ParsedToolResultEvent } +// ParsedToolCallPosition identifies one emitted tool-call occurrence by its +// stable normalized message ordinal and call index. +type ParsedToolCallPosition struct { + MessageOrdinal int + CallIndex int +} + +// ParsedToolCallUpdate carries result events for a tool call that was parsed +// before the current append-only chunk. TargetKnown is required for safe +// incremental application when a provider reuses call IDs. +type ParsedToolCallUpdate struct { + ToolUseID string + MessageOrdinal int + CallIndex int + TargetKnown bool + ResultEvents []ParsedToolResultEvent +} + +// ParsedMessageTokenUsageUpdate carries token metadata for an assistant +// message that was committed before the current append-only chunk. Codex +// emits a token_count record immediately after a late tool result, so the +// target assistant message may not be present in the incremental message +// slice even though its ordinal is known from the stored transcript. +type ParsedMessageTokenUsageUpdate struct { + Ordinal int + TokenUsage jsontext.Value + ContextTokens int + OutputTokens int + HasContextTokens bool + HasOutputTokens bool +} + // ParsedToolResult holds metadata about a tool result block in a // user message (the response to a prior tool_use). type ParsedToolResult struct { @@ -1563,6 +1599,19 @@ type ParseResult struct { Session ParsedSession Messages []ParsedMessage UsageEvents []ParsedUsageEvent + // Checkpoint is opaque provider continuation state (a parser + // checkpoint) that the sync engine persists after this result's + // session rows commit, so later appends can resume without rescanning + // the transcript prefix. Empty for providers without checkpoints. + Checkpoint []byte + // CheckpointHashState is the resumable SHA-256 state covering the + // parsed snapshot [0, Session.File.Size), captured on the same read + // pass as the parse. CheckpointAnchorDigest is the digest of the + // snapshot's trailing anchor window. Both are empty for providers + // without single-pass hashing; the engine persists them with the + // checkpoint so it never re-reads the source after a full parse. + CheckpointHashState []byte + CheckpointAnchorDigest string } // InferRelationshipTypes sets RelationshipType on results that have diff --git a/internal/parser/zcode.go b/internal/parser/zcode.go index 0487c3fc7..1e9e595d1 100644 --- a/internal/parser/zcode.go +++ b/internal/parser/zcode.go @@ -912,7 +912,11 @@ func zcodeSessionFileMtime(dbPath string, db *sql.DB, row zcodeSessionRow) int64 if usageMtime, err := zcodeMaxUsageMtime(db, row.id); err == nil { maxMtime = max(maxMtime, usageMtime) } - for _, path := range []string{dbPath, dbPath + "-wal", dbPath + "-shm"} { + // The -shm index is excluded on purpose: readers rewrite it, this + // provider's own read connection included, so folding its mtime into the + // fingerprint made every scan report the whole container as changed. + // Content changes always touch the main file or the -wal sibling. + for _, path := range []string{dbPath, dbPath + "-wal"} { if info, err := os.Stat(path); err == nil { maxMtime = max(maxMtime, info.ModTime().UnixNano()) } diff --git a/internal/parser/zcode_test.go b/internal/parser/zcode_test.go index 470e9d653..8dfaeff76 100644 --- a/internal/parser/zcode_test.go +++ b/internal/parser/zcode_test.go @@ -825,6 +825,42 @@ func TestZCodeFingerprintTracksDBMtimeForUsageOnlyChanges(t *testing.T) { assert.Equal(t, dbMtime.UnixNano(), fingerprint.MTimeNS) } +func TestZCodeFingerprintIgnoresShmIndexMtime(t *testing.T) { + // Readers rewrite the -shm index, this provider's own connection + // included, so its mtime must not move the fingerprint or every scan + // would report the whole container as changed. + fixture := newZCodeTestFixture(t) + fixture.insertSession( + t, + "session-shm", + "/Users/alice/code/acme-app", + "SHM", + "2026-07-06T13:00:01Z", + "2026-07-06T13:10:00Z", + "", + "", + ) + dbMtime := time.Date(2026, 7, 6, 13, 20, 0, 0, time.UTC) + require.NoError(t, os.Chtimes(fixture.DBPath, dbMtime, dbMtime)) + shmPath := fixture.DBPath + "-shm" + require.NoError(t, os.WriteFile(shmPath, make([]byte, 32), 0o644)) + shmMtime := dbMtime.Add(time.Hour) + require.NoError(t, os.Chtimes(shmPath, shmMtime, shmMtime)) + + provider, ok := NewProvider(AgentZCode, ProviderConfig{ + Roots: []string{fixture.CLIRoot}, + Machine: "devbox", + }) + require.True(t, ok) + sources, err := provider.Discover(context.Background()) + require.NoError(t, err) + require.Len(t, sources, 1) + + fingerprint, err := provider.Fingerprint(context.Background(), sources[0]) + require.NoError(t, err) + assert.Equal(t, dbMtime.UnixNano(), fingerprint.MTimeNS) +} + func TestZCodeFallsBackToDBMtimeWhenTimestampsAreMissing(t *testing.T) { fixture := newZCodeTestFixture(t) fixture.insertSession( diff --git a/internal/signals/heuristics.go b/internal/signals/heuristics.go index 57e2a5591..7125abc51 100644 --- a/internal/signals/heuristics.go +++ b/internal/signals/heuristics.go @@ -539,15 +539,18 @@ func jaccardFromOverlap(currentUnique, previousTotal, intersections int) float64 } func hasContextToolActivity(calls []ToolCallRow) bool { - for _, c := range calls { - switch c.Category { - case "Read", "Grep", "Glob": - return true - case "Bash": - if isContextCommand(commandText(c.InputJSON)) { - return true - } - } + return slices.ContainsFunc(calls, IsContextToolCall) +} + +// IsContextToolCall reports whether a tool call counts as context-gathering +// activity for the no-code-context heuristic (Read/Grep/Glob or a Bash +// context command). +func IsContextToolCall(c ToolCallRow) bool { + switch c.Category { + case "Read", "Grep", "Glob": + return true + case "Bash": + return isContextCommand(commandText(c.InputJSON)) } return false } diff --git a/internal/signals/incremental.go b/internal/signals/incremental.go new file mode 100644 index 000000000..7a9c03d3a --- /dev/null +++ b/internal/signals/incremental.go @@ -0,0 +1,883 @@ +package signals + +// Incremental signal maintenance: compact per-session aggregate state that +// lets the sync engine fold an incremental delta (appended tool calls plus +// late tool-result updates) into the same signal values a full recompute +// would produce, without loading session history. +// +// Correctness contract: every detector whose input can be changed by a +// delta is maintained exactly, as long as +// +// - appended calls arrive in chronological order, and +// - every modified (late-updated) call sits within the trailing +// ModifiedWindowSize calls of the pre-delta session. +// +// The caller must fall back to a full recompute (and reseed) when a delta +// touches a call older than ModifiedWindowSize, when the delta contains +// user prompts or compact boundaries (not maintained here — see the sync +// maintainer), or when the persisted state is missing/stale. + +import ( + "encoding/json" + "fmt" + "maps" + "slices" +) + +const ( + // IncrementalStateCodecVersion is the wire version for IncrementalState. + // Bump when the struct or any detector semantics change; a mismatch + // makes the caller fall back to a full recompute. + // v3 added LastValidTokensOrdinal. + IncrementalStateCodecVersion = 3 + + // TrailingFactCount is the size of the trailing facts window. It must + // cover every window any delta can affect: a modified call in the last + // ModifiedWindowSize calls can touch runaway windows that start up to + // 11 calls earlier, so the window must exceed ModifiedWindowSize + 11. + TrailingFactCount = 35 + + // ModifiedWindowSize is the number of trailing calls whose facts a + // late tool-result update may change on the incremental path. Updates + // to older calls require a full recompute. + ModifiedWindowSize = 12 +) + +// CallPos identifies one tool call by its natural message/call coordinates. +type CallPos struct { + MessageOrdinal int `json:"message_ordinal"` + CallIndex int `json:"call_index"` +} + +// ToolFact is the per-call fact set the incremental machinery needs: the +// position plus the failure bit, exact tool signature, and command class. +type ToolFact struct { + CallPos + Failure bool `json:"failure"` + ExactSignature string `json:"exact_signature,omitempty"` + CommandClass string `json:"command_class,omitempty"` +} + +// EditChurnState tracks one file path's churn detection: the last two edit +// ordinals plus a counted latch (churn counts once per file). HasLast1 and +// HasLast2 distinguish "no prior edit" from "prior edit at ordinal 0". +type EditChurnState struct { + Last1 int `json:"last1"` + Last2 int `json:"last2"` + HasLast1 bool `json:"has_last1"` + HasLast2 bool `json:"has_last2"` + Counted bool `json:"counted"` +} + +// PendingBoundary tracks a compact boundary whose after-window is still +// open: fewer than midTaskWindowAfter calls have arrived since it, and the +// overlap threshold has not been met. BeforeNames is frozen at seed time +// (appends are chronological, so the pre-boundary window never changes). +type PendingBoundary struct { + Ordinal int `json:"ordinal"` + BeforeNames []string `json:"before_names,omitempty"` + AfterNames []string `json:"after_names,omitempty"` +} + +// IncrementalState is the SQLite-only compact aggregate state for one +// session. It is JSON-encoded and versioned; the sync layer additionally +// stamps a verification token (transcript revision + signal version) next +// to it so a state that fell behind the rows is never folded. +type IncrementalState struct { + CodecVersion int `json:"codec_version"` + + // Failure runs. PrefixFailureMax is the longest failure run ending + // before the trailing window; TailFailureRun is the length of the + // failure run ending at the last call before the window (the run that + // crosses into the window when window[0] is a failure). + PrefixFailureMax int `json:"prefix_failure_max"` + TailFailureRun int `json:"tail_failure_run"` + + // Retry run tail: the trailing run of calls sharing (ToolName, + // InputJSON). RetryCount itself lives on the sessions row. + RetryRunName string `json:"retry_run_name,omitempty"` + RetryRunInput string `json:"retry_run_input,omitempty"` + RetryRunLen int `json:"retry_run_len"` + + // Edit churn: the last two edit ordinals per file path plus the + // counted latch. EditChurnCount itself lives on the sessions row. + EditLast map[string]EditChurnState `json:"edit_last,omitempty"` + + // Runaway loop: RunawayHistorical latches hasRunawayToolWindow over + // every 12-window that has fully left the mutable late-result region. + RunawayHistorical bool `json:"runaway_historical"` + + // Exact failing run crossing into the trailing window. ExactRunSig is + // empty when the run containing window[0] starts inside the window. + // ExactHistorical latches qualifying runs fully before the window. + ExactRunSig string `json:"exact_run_sig,omitempty"` + ExactRunLen int `json:"exact_run_len"` + ExactRunFailures int `json:"exact_run_failures"` + ExactHistorical bool `json:"exact_historical"` + + // Message-derived aggregates not stored on the sessions row. + LastRole string `json:"last_role,omitempty"` + LastContent string `json:"last_content,omitempty"` + MsgIndex int `json:"msg_index"` + // LastValidTokens is the most recent assistant context-token + // measurement folded so far; LastValidTokensOrdinal is the ordinal of + // the message it was measured from (meaningless when LastValidTokens + // is 0). A late usage update targeting an ordinal at or before this one + // arrived chronologically out of order -- a later assistant already + // contributed a newer measurement -- and must not be folded forward. + LastValidTokens int `json:"last_valid_tokens"` + LastValidTokensOrdinal int `json:"last_valid_tokens_ordinal"` + ModelCounts map[string]int `json:"model_counts,omitempty"` + ModelFirstSeen map[string]int `json:"model_first_seen,omitempty"` + + // TotalCalls counts calls folded so far; used for window arithmetic. + TotalCalls int `json:"total_calls"` + + // HasExplicitBoundaries records whether the session has explicit + // compact-boundary messages. When true, the full compute derives the + // compaction count from the boundary count and ignores token-drop + // compactions; the incremental fold must do the same. Appends never + // carry boundaries (the maintainer declines them), so this is fixed at + // seed time. + HasExplicitBoundaries bool `json:"has_explicit_boundaries"` + + // PendingBoundaries are compact boundaries with an open after-window. + PendingBoundaries []PendingBoundary `json:"pending_boundaries,omitempty"` + + // Trailing holds the facts of the last TrailingFactCount calls in + // chronological order. + Trailing []ToolFact `json:"trailing,omitempty"` +} + +// ToolHealthRow is the subset of the sessions row the tool-health fold +// combines with deltas: the current stored aggregates. +type ToolHealthRow struct { + FailureCount int + RetryCount int + EditChurnCount int +} + +// ToolHealthResult carries the absolute tool-health signal values after a +// fold. MidTaskCompactions is the number of pending boundaries the delta +// newly classified as mid-task (a delta against the stored row value). +type ToolHealthResult struct { + FailureCount int + ConsecutiveFailureMax int + RetryCount int + EditChurnCount int + RunawayToolLoopCount int + FinalFailureStreak int + MidTaskCompactions int +} + +// SeedIncrementalState builds the initial state from a full compute's +// inputs. calls must be ordered by (MessageOrdinal, CallIndex) — the same +// order extractToolCallRows produces. boundaries must be ascending +// compact-boundary ordinals. modelCounts/modelFirstSeen/msgIndex mirror +// extractMostCommonModel's inputs; lastValidTokens is the last assistant +// context-token measurement (0 when none) and lastValidTokensOrdinal is +// the ordinal of the message it came from (meaningless when +// lastValidTokens is 0). +func SeedIncrementalState( + calls []ToolCallRow, + boundaries []int, + lastRole, lastContent string, + modelCounts, modelFirstSeen map[string]int, + msgIndex int, + lastValidTokens, lastValidTokensOrdinal int, +) IncrementalState { + modelCounts = writableIntMap(modelCounts) + modelFirstSeen = writableIntMap(modelFirstSeen) + s := IncrementalState{ + CodecVersion: IncrementalStateCodecVersion, + EditLast: map[string]EditChurnState{}, + LastRole: lastRole, + LastContent: lastContent, + LastValidTokens: lastValidTokens, + LastValidTokensOrdinal: lastValidTokensOrdinal, + MsgIndex: msgIndex, + ModelCounts: modelCounts, + ModelFirstSeen: modelFirstSeen, + TotalCalls: len(calls), + HasExplicitBoundaries: len(boundaries) > 0, + } + cut := max(0, len(calls)-TrailingFactCount) + s.Trailing = factsFor(calls[cut:]) + + // Failure runs: latch runs ending before the cut, crossing run at it. + // TailFailureRun holds the portion of the boundary run before the + // window (positions < cut), so a fold can replay it as leading trues. + for i := 0; i < len(calls); { + if !IsFailure(calls[i]) { + i++ + continue + } + j := i + for j+1 < len(calls) && IsFailure(calls[j+1]) { + j++ + } + if j < cut { + s.PrefixFailureMax = max(s.PrefixFailureMax, j-i+1) + } + if cut > 0 && i < cut && j >= cut-1 { + s.TailFailureRun = cut - i + } + i = j + 1 + } + + // Retry tail: the run containing the last call. + if n := len(calls); n > 0 { + s.RetryRunName = calls[n-1].ToolName + s.RetryRunInput = calls[n-1].InputJSON + s.RetryRunLen = 1 + for i := n - 2; i >= 0; i-- { + if calls[i].ToolName == s.RetryRunName && + calls[i].InputJSON == s.RetryRunInput { + s.RetryRunLen++ + } else { + break + } + } + } + + // Edit churn: last two edit ordinals per file plus the counted latch. + fileOrdinals := map[string][]int{} + for _, c := range calls { + if c.Category != "Edit" && c.Category != "Write" { + continue + } + path := extractFilePath(c.InputJSON) + if path == "" { + continue + } + fileOrdinals[path] = append(fileOrdinals[path], c.MessageOrdinal) + } + for path, ords := range fileOrdinals { + st := EditChurnState{Counted: hasChurnWindow(ords, 3, 10)} + if n := len(ords); n >= 1 { + st.Last2 = ords[n-1] + st.HasLast2 = true + } + if n := len(ords); n >= 2 { + st.Last1 = ords[n-2] + st.HasLast1 = true + } + s.EditLast[path] = st + } + + // Exact-run detector: seed via the same unified fold the incremental + // path uses, over a deep prefix of nothing and the full call list. + deep := deepRun{} + historical, crossing, _ := foldExactRuns( + deep, factsFor(calls), cut, false, + ) + s.ExactHistorical = historical + s.ExactRunSig = crossing.sig + s.ExactRunLen = crossing.len + s.ExactRunFailures = crossing.failures + + // Runaway window detector: latch every qualifying 12-call window that + // can no longer be changed by a late result. Only the final + // ModifiedWindowSize calls remain mutable, which can be substantially + // smaller than the retained facts window. + immutableEnd := max(0, len(calls)-ModifiedWindowSize) + for i := 0; i+12 <= immutableEnd; i++ { + if windowFactsQualify(factsFor(calls[i : i+12])) { + s.RunawayHistorical = true + } + } + + // Pending compaction boundaries: after-window still open and the + // pre-boundary window non-empty (an empty before-window can never + // reach the overlap threshold — mirror CountMidTaskCompactions). + ords := callOrdinals(calls) + for _, b := range boundaries { + before := toolWindowBefore(ords, b, midTaskWindowBefore) + if len(before) == 0 { + continue + } + after := toolWindowAfter(ords, b, midTaskWindowAfter) + if len(after) >= midTaskWindowAfter { + continue // decided: below threshold forever + } + if overlapCount(before, after) >= midTaskOverlapThreshold { + continue // already counted by the full compute + } + s.PendingBoundaries = append(s.PendingBoundaries, PendingBoundary{ + Ordinal: b, + BeforeNames: before, + AfterNames: after, + }) + } + return s +} + +func writableIntMap(in map[string]int) map[string]int { + out := maps.Clone(in) + if out == nil { + out = make(map[string]int) + } + return out +} + +func factsFor(calls []ToolCallRow) []ToolFact { + facts := make([]ToolFact, 0, len(calls)) + for _, c := range calls { + facts = append(facts, ToolFact{ + CallPos: CallPos{ + MessageOrdinal: c.MessageOrdinal, + CallIndex: c.CallIndex, + }, + Failure: IsFailure(c), + ExactSignature: ExactToolSignature(c), + CommandClass: CommandClass(c), + }) + } + return facts +} + +// ExactToolSignature returns the exact signature the runaway exact-run +// detector uses for a call. +func ExactToolSignature(c ToolCallRow) string { return toolSignature(c) } + +// CommandClass returns the command class the runaway window detector uses. +func CommandClass(c ToolCallRow) string { return commandClass(c) } + +func callOrdinals(calls []ToolCallRow) []ToolCallOrdinal { + ords := make([]ToolCallOrdinal, 0, len(calls)) + for _, c := range calls { + ords = append(ords, ToolCallOrdinal{ + MessageOrdinal: c.MessageOrdinal, + ToolName: c.ToolName, + }) + } + return ords +} + +func overlapCount(a, b []string) int { + set := make(map[string]struct{}, len(a)) + for _, name := range a { + set[name] = struct{}{} + } + matched := 0 + seen := make(map[string]struct{}) + for _, name := range b { + if _, ok := set[name]; ok { + if _, dup := seen[name]; !dup { + seen[name] = struct{}{} + matched++ + } + } + } + return matched +} + +// FoldToolHealth folds one incremental delta into the state. appended are +// the newly appended calls in chronological order; modified carries the +// post-delta facts of calls whose facts changed via late results (their +// pre-delta facts must be in s.Trailing); row supplies the stored +// aggregates the deltas combine with. It returns the next state and the +// absolute tool-health values, or ok=false when a modified position falls +// outside the maintenance window and the caller must fall back to a full +// recompute. +func (s *IncrementalState) FoldToolHealth( + appended []ToolCallRow, + modified map[CallPos]ToolFact, + row ToolHealthRow, +) (IncrementalState, ToolHealthResult, bool) { + if s.CodecVersion != IncrementalStateCodecVersion { + return *s, ToolHealthResult{}, false + } + next := *s + next.Trailing = nil // rebuilt below + next.TotalCalls = s.TotalCalls + len(appended) + + // Every modified position must sit within the last ModifiedWindowSize + // calls of the pre-delta session, and its old fact must be known. + oldLimit := max(0, len(s.Trailing)-ModifiedWindowSize) + oldByPos := make(map[CallPos]ToolFact, len(s.Trailing)) + for _, f := range s.Trailing { + oldByPos[f.CallPos] = f + } + for pos := range modified { + oldFact, inOld := oldByPos[pos] + if !inOld { + return *s, ToolHealthResult{}, false + } + idx := slices.IndexFunc(s.Trailing, func(f ToolFact) bool { + return f.CallPos == pos + }) + if idx < 0 || idx < oldLimit { + return *s, ToolHealthResult{}, false + } + _ = oldFact + } + + // Full tail facts: old window + appended, then overlay modifications. + fullTail := mergeFacts(s.Trailing, factsFor(appended)) + if fullTail == nil { + fullTail = []ToolFact{} + } + for pos, newFact := range modified { + idx := slices.IndexFunc(fullTail, func(f ToolFact) bool { + return f.CallPos == pos + }) + if idx < 0 { + return *s, ToolHealthResult{}, false + } + fullTail[idx] = newFact + } + newCutAbs := max(0, next.TotalCalls-TrailingFactCount) + oldCut := max(0, s.TotalCalls-TrailingFactCount) + // fullTail[0] sits at absolute position oldCut, so the window offset + // is the difference between the two cuts. + newCut := newCutAbs - oldCut + newCut = max(newCut, 0) + newCut = min(newCut, len(fullTail)) + next.Trailing = append([]ToolFact(nil), fullTail[newCut:]...) + + // Failure count: flips plus appended failures. + failureDelta := 0 + for _, f := range appended { + if IsFailure(f) { + failureDelta++ + } + } + for pos, newFact := range modified { + oldFact := oldByPos[pos] + if newFact.Failure && !oldFact.Failure { + failureDelta++ + } else if !newFact.Failure && oldFact.Failure { + failureDelta-- + } + } + out := ToolHealthResult{ + FailureCount: row.FailureCount + failureDelta, + } + + // Failure run structure: deep tail run + facts before the new cut + + // new window bits. + seqBefore := make([]bool, 0, s.TailFailureRun+newCut) + for range s.TailFailureRun { + seqBefore = append(seqBefore, true) + } + for _, f := range fullTail[:newCut] { + seqBefore = append(seqBefore, f.Failure) + } + newBits := make([]bool, len(next.Trailing)) + for i, f := range next.Trailing { + newBits[i] = f.Failure + } + prefixMax, tailRun, crossingRun := foldFailureRuns( + s.PrefixFailureMax, seqBefore, newBits, + ) + next.PrefixFailureMax = prefixMax + next.TailFailureRun = tailRun + // The boundary run itself is a real run: it counts toward the global + // max even when the window starts with a non-failure (in which case + // crossingRun is 0 and the run's full length is tailRun). + out.ConsecutiveFailureMax = max( + prefixMax, tailRun, crossingRun, maxRunWithin(newBits), + ) + finalStreak := trailingRun(newBits) + // A failure run that fills the whole trailing window continues before + // it; tailRun is the pre-window length of exactly that crossing run. + // Carry it forward so a run longer than TrailingFactCount reports its + // full length instead of capping at the window size. + if finalStreak == len(newBits) && tailRun > 0 { + finalStreak += tailRun + } + out.FinalFailureStreak = finalStreak + + // Retry runs: only appends change name/input runs. + out.RetryCount = row.RetryCount + for _, c := range appended { + if next.RetryRunLen > 0 && + c.ToolName == next.RetryRunName && + c.InputJSON == next.RetryRunInput { + next.RetryRunLen++ + switch { + case next.RetryRunLen == 3: + out.RetryCount += 2 + case next.RetryRunLen > 3: + out.RetryCount++ + } + } else { + next.RetryRunName = c.ToolName + next.RetryRunInput = c.InputJSON + next.RetryRunLen = 1 + } + } + + // Edit churn: only appends add edit calls; a file counts once. + out.EditChurnCount = row.EditChurnCount + editLast := maps.Clone(s.EditLast) + if editLast == nil { + editLast = make(map[string]EditChurnState) + } + for _, c := range appended { + if c.Category != "Edit" && c.Category != "Write" { + continue + } + path := extractFilePath(c.InputJSON) + if path == "" { + continue + } + st := editLast[path] + if !st.Counted && st.HasLast1 && st.HasLast2 && + c.MessageOrdinal-st.Last1 < 10 { + out.EditChurnCount++ + st.Counted = true + } + st.Last1, st.HasLast1 = st.Last2, st.HasLast2 + st.Last2, st.HasLast2 = c.MessageOrdinal, true + editLast[path] = st + } + next.EditLast = editLast + + // Mid-task compactions: consume appended calls into open boundaries. + var kept []PendingBoundary + for _, b := range next.PendingBoundaries { + after := append([]string(nil), b.AfterNames...) + decided, counted := false, false + for _, c := range appended { + if c.MessageOrdinal <= b.Ordinal { + continue + } + after = append(after, c.ToolName) + if overlapCount(b.BeforeNames, after) >= + midTaskOverlapThreshold { + counted, decided = true, true + break + } + if len(after) >= midTaskWindowAfter { + decided = true + break + } + } + if counted { + out.MidTaskCompactions++ + } + if !decided { + b.AfterNames = after + kept = append(kept, b) + } + } + next.PendingBoundaries = kept + + // Runaway loop: latch windows once every call in the 12-call window is + // older than the mutable late-result region. The retained facts window is + // intentionally wider than that region, so boundary-crossing qualifying + // windows must be preserved even while some of their facts remain retained. + runaway := s.RunawayHistorical + currentRunaway := false + windowStart := max(0, s.TotalCalls-TrailingFactCount) + for i := windowStart; i+12 <= next.TotalCalls; i++ { + if windowAt(fullTail, windowStart, i) { + if i+12 <= next.TotalCalls-ModifiedWindowSize { + runaway = true + } else { + currentRunaway = true + } + } + } + next.RunawayHistorical = runaway + + // Exact failing runs: unified fold over deep crossing state + full + // tail facts, latching runs that left the window. + deep := deepRun{ + sig: s.ExactRunSig, + len: s.ExactRunLen, + failures: s.ExactRunFailures, + } + if deep.sig != "" { + // The deep run's length includes the old window's leading prefix; + // strip it so the concatenated fold sees each fact once. + for _, f := range s.Trailing { + if f.ExactSignature != deep.sig { + break + } + if f.Failure { + deep.failures-- + } + deep.len-- + } + if deep.len < 0 { + deep.len = 0 + } + } + historical, crossingState, exactQualifies := foldExactRuns( + deep, fullTail, newCut, s.ExactHistorical, + ) + next.ExactHistorical = historical + next.ExactRunSig = crossingState.sig + next.ExactRunLen = crossingState.len + next.ExactRunFailures = crossingState.failures + + out.RunawayToolLoopCount = 0 + if exactQualifies || runaway || currentRunaway { + out.RunawayToolLoopCount = 1 + } + + return next, out, true +} + +// deepRun is a same-signature run ending at the facts window's left edge, +// summarized as (sig, len, failures). +type deepRun struct { + sig string + len int + failures int +} + +// foldExactRuns evaluates the exact-failing-run detector over a sequence +// made of an optional deep run followed by post-delta facts. newCut is the +// index into facts where the trailing window starts. It returns the new +// historical latch (qualifying runs that end before the window), the run +// crossing into the window (zero when none), and whether any run — +// historical, crossing, or window-internal — qualifies. +func foldExactRuns( + deep deepRun, facts []ToolFact, newCut int, historical bool, +) (latch bool, crossing deepRun, qualifies bool) { + type run struct { + start int // index into facts; -1 = deep + end int // index into facts; -1 = deep-only + sig string + len int + failures int + } + var runs []run + cur := run{ + start: -1, end: -1, + sig: deep.sig, len: deep.len, failures: deep.failures, + } + hasCur := deep.len > 0 + for i, f := range facts { + if hasCur && f.ExactSignature == cur.sig { + if cur.start < 0 { + cur.start = i + } + cur.end = i + cur.len++ + if f.Failure { + cur.failures++ + } + continue + } + if hasCur { + runs = append(runs, cur) + } + cur = run{ + start: i, end: i, + sig: f.ExactSignature, len: 1, + } + if f.Failure { + cur.failures = 1 + } + hasCur = true + } + if hasCur { + runs = append(runs, cur) + } + + latch = historical + for _, r := range runs { + qual := r.len >= 5 && r.failures >= 3 + switch { + case r.start < 0 && r.end < newCut: + // Deep run that ended before the window. + if qual { + latch = true + } + case r.start < 0: + // Deep run reaching into the window. + if qual { + qualifies = true + } + crossing = deepRun{sig: r.sig, len: r.len, failures: r.failures} + case r.end < newCut: + // Facts run fully before the window. + if qual { + latch = true + } + case r.start < newCut: + // Facts run crossing into the window. + if qual { + qualifies = true + } + crossing = deepRun{sig: r.sig, len: r.len, failures: r.failures} + default: + // Window-internal run. + if qual { + qualifies = true + } + } + } + return latch, crossing, qualifies +} + +// foldFailureRuns computes the new prefix latch, boundary run, and +// crossing run after the trailing window becomes newBits. seqBefore holds +// the failure bits of every call before the new window: the deep-history +// tail run (crossing from before the old window, as TailFailureRun leading +// trues) followed by the old window facts that slid out. prefixMax is the +// previous longest run fully before the old window. It returns the new +// prefix max, the run ending at the last call before the new window, and +// the full length of the run crossing into the window (0 when the window +// starts with a non-failure). +func foldFailureRuns( + prefixMax int, seqBefore, newBits []bool, +) (newPrefixMax, tailRun, crossingRun int) { + newPrefixMax = prefixMax + i := 0 + for i < len(seqBefore) { + if !seqBefore[i] { + i++ + continue + } + j := i + for j+1 < len(seqBefore) && seqBefore[j+1] { + j++ + } + if j == len(seqBefore)-1 { + // The run touches the window boundary. Its length j-i+1 is + // the full run: seqBefore's head carries the deep portion as + // leading trues, so no extra accounting is needed. + tailRun = j - i + 1 + } else { + // Fully before the window: prefix candidate. + newPrefixMax = max(newPrefixMax, j-i+1) + } + i = j + 1 + } + if len(newBits) > 0 && newBits[0] { + head := 0 + for head < len(newBits) && newBits[head] { + head++ + } + crossingRun = tailRun + head + } + return newPrefixMax, tailRun, crossingRun +} + +// maxRunWithin returns the longest true-run fully inside bits. +func maxRunWithin(bits []bool) int { + best := 0 + i := 0 + for i < len(bits) { + if !bits[i] { + i++ + continue + } + j := i + for j+1 < len(bits) && bits[j+1] { + j++ + } + best = max(best, j-i+1) + i = j + 1 + } + return best +} + +func trailingRun(bits []bool) int { + n := 0 + for i := len(bits) - 1; i >= 0 && bits[i]; i-- { + n++ + } + return n +} + +// windowAt reports whether the 12-window starting at absolute call index +// start qualifies. facts holds the call facts starting at absolute index +// windowStart. +func windowAt(facts []ToolFact, windowStart, start int) bool { + first := start - windowStart + if first < 0 || first+12 > len(facts) { + return false + } + return windowFactsQualify(facts[first : first+12]) +} + +// windowFactsQualify mirrors hasRunawayToolWindow's single-window test. +func windowFactsQualify(facts []ToolFact) bool { + if len(facts) < 12 { + return false + } + failures := 0 + classCounts := make(map[string]int, len(facts)) + for _, f := range facts { + if f.Failure { + failures++ + } + classCounts[f.CommandClass]++ + } + if failures >= 6 { + return true + } + return failures >= 3 && dominantCount(classCounts) >= 10 +} + +// mergeFacts merges two position-ordered fact windows, preferring newer +// facts when positions overlap, sorted by position. +func mergeFacts(old, newer []ToolFact) []ToolFact { + byPos := make(map[CallPos]ToolFact, len(old)+len(newer)) + seen := make(map[CallPos]bool, len(old)+len(newer)) + var order []CallPos + for _, f := range old { + if !seen[f.CallPos] { + seen[f.CallPos] = true + order = append(order, f.CallPos) + } + byPos[f.CallPos] = f + } + for _, f := range newer { + if !seen[f.CallPos] { + seen[f.CallPos] = true + order = append(order, f.CallPos) + } + byPos[f.CallPos] = f + } + slices.SortFunc(order, func(a, b CallPos) int { + if a.MessageOrdinal != b.MessageOrdinal { + return a.MessageOrdinal - b.MessageOrdinal + } + return a.CallIndex - b.CallIndex + }) + merged := make([]ToolFact, 0, len(order)) + for _, pos := range order { + merged = append(merged, byPos[pos]) + } + return merged +} + +// MarshalBinary implements encoding.BinaryMarshaler for the state row. +func (s *IncrementalState) MarshalBinary() ([]byte, error) { + return json.Marshal(s) +} + +// UnmarshalBinary implements encoding.BinaryUnmarshaler. A malformed or +// version-mismatched payload is rejected so the caller falls back. +func (s *IncrementalState) UnmarshalBinary(data []byte) error { + if err := json.Unmarshal(data, s); err != nil { + return fmt.Errorf("decoding incremental signal state: %w", err) + } + if s.CodecVersion != IncrementalStateCodecVersion { + return fmt.Errorf( + "incremental signal state codec %d != %d", + s.CodecVersion, IncrementalStateCodecVersion, + ) + } + // Empty mutable maps are omitted by the JSON codec. Normalize decoded + // state before any fold mutates it so the first edit or model append cannot + // panic on assignment to a nil map. + if s.EditLast == nil { + s.EditLast = make(map[string]EditChurnState) + } + if s.ModelCounts == nil { + s.ModelCounts = make(map[string]int) + } + if s.ModelFirstSeen == nil { + s.ModelFirstSeen = make(map[string]int) + } + return nil +} diff --git a/internal/signals/incremental_test.go b/internal/signals/incremental_test.go new file mode 100644 index 000000000..7ed3e02f9 --- /dev/null +++ b/internal/signals/incremental_test.go @@ -0,0 +1,677 @@ +package signals + +import ( + "fmt" + "math/rand" + "slices" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// newTestRand returns a deterministic RNG so failures reproduce. +func newTestRand(t *testing.T, seed int64) *rand.Rand { + t.Helper() + return rand.New(rand.NewSource(seed)) +} + +var parityToolNames = []string{ + "exec_command", "edit_file", "write_file", "read_file", "grep_search", +} + +var parityCommands = []string{ + "ls -la", "npm test", "go build ./...", "cat main.go", "git status", +} + +var parityResultStatuses = []string{ + "", "", "", "completed", "completed", "completed", "errored", "cancelled", +} + +func parityRow(rng *rand.Rand, ordinal int) ToolCallRow { + name := parityToolNames[rng.Intn(len(parityToolNames))] + category := "Other" + switch name { + case "exec_command": + category = "Bash" + case "edit_file": + category = "Edit" + case "write_file": + category = "Write" + case "read_file": + category = "Read" + case "grep_search": + category = "Grep" + } + input := `{"command":"` + parityCommands[rng.Intn(len(parityCommands))] + `"}` + if category == "Edit" || category == "Write" { + input = fmt.Sprintf( + `{"file_path":"/src/file_%d.go"}`, rng.Intn(4), + ) + } + row := ToolCallRow{ + ToolName: name, + Category: category, + InputJSON: input, + MessageOrdinal: ordinal, + EventStatus: parityResultStatuses[rng.Intn(len(parityResultStatuses))], + } + if row.EventStatus == "" && rng.Intn(8) == 0 { + // Content-based failure for pending Bash calls. + row.ResultContent = "command not found" + } + return row +} + +// fullToolHealth computes the authoritative tool-health values the fold +// must reproduce, from the complete post-delta call list. +func fullToolHealth(calls []ToolCallRow) ToolHealthResult { + h := ComputeToolHealth(calls) + out := ToolHealthResult{ + FailureCount: h.FailureSignalCount, + ConsecutiveFailureMax: h.ConsecutiveFailureMax, + RetryCount: h.RetryCount, + EditChurnCount: h.EditChurnCount, + } + for i := len(calls) - 1; i >= 0; i-- { + if IsFailure(calls[i]) { + out.FinalFailureStreak++ + } else { + break + } + } + heuristics := AnalyzeHeuristics(HeuristicInput{ToolRows: calls}) + out.RunawayToolLoopCount = heuristics.RunawayToolLoopCount + return out +} + +func fullMidTask(calls []ToolCallRow, boundaries []int) int { + ords := make([]ToolCallOrdinal, 0, len(calls)) + for _, c := range calls { + ords = append(ords, ToolCallOrdinal{ + MessageOrdinal: c.MessageOrdinal, + ToolName: c.ToolName, + }) + } + return CountMidTaskCompactions(boundaries, ords) +} + +// TestIncrementalFoldParityRandomized drives random append/modify deltas +// through the fold and compares every maintained value against the full +// recompute over the complete history. +func TestIncrementalFoldParityRandomized(t *testing.T) { + rng := newTestRand(t, 20260813) + initialCalls := 20 + rng.Intn(300) + calls := make([]ToolCallRow, 0, initialCalls+300) + boundaries := []int{} + for i := range initialCalls { + calls = append(calls, parityRow(rng, i)) + if rng.Intn(40) == 0 { + boundaries = append(boundaries, i) + } + } + + state := SeedIncrementalState( + calls, boundaries, "", "", nil, nil, 0, 0, 0, + ) + full := fullToolHealth(calls) + row := ToolHealthRow{ + FailureCount: full.FailureCount, + RetryCount: full.RetryCount, + EditChurnCount: full.EditChurnCount, + } + midTask := fullMidTask(calls, boundaries) + + nextOrdinal := initialCalls + callIndexByOrdinal := map[int]int{} + for step := range 300 { + appended := []ToolCallRow{} + modified := map[CallPos]ToolFact{} + if rng.Intn(3) == 0 && len(calls) >= ModifiedWindowSize { + // Modify a trailing call: flip its result fact. + idx := len(calls) - 1 - rng.Intn(ModifiedWindowSize) + calls[idx].EventStatus = parityResultStatuses[rng.Intn(len(parityResultStatuses))] + calls[idx].ResultContent = "" + modified[CallPos{ + MessageOrdinal: calls[idx].MessageOrdinal, + CallIndex: calls[idx].CallIndex, + }] = factsFor(calls[idx : idx+1])[0] + } else { + // Append 1-3 calls sharing one new message ordinal (the + // realistic shape: several tool calls in one message). + n := 1 + rng.Intn(3) + addedBoundary := false + ordinal := nextOrdinal + nextOrdinal++ + for range n { + row := parityRow(rng, ordinal) + row.CallIndex = callIndexByOrdinal[ordinal] + callIndexByOrdinal[ordinal]++ + if rng.Intn(30) == 0 { + boundaries = append(boundaries, ordinal) + addedBoundary = true + } + calls = append(calls, row) + appended = append(appended, row) + } + if addedBoundary { + // A delta containing a compact boundary falls back to a + // full recompute, which reseeds the state. + full = fullToolHealth(calls) + state = SeedIncrementalState( + calls, boundaries, "", "", nil, nil, 0, 0, 0, + ) + row = ToolHealthRow{ + FailureCount: full.FailureCount, + RetryCount: full.RetryCount, + EditChurnCount: full.EditChurnCount, + } + midTask = fullMidTask(calls, boundaries) + continue + } + } + + nextState, got, ok := state.FoldToolHealth(appended, modified, row) + require.True(t, ok, "step %d: fold must accept in-window delta", step) + + full = fullToolHealth(calls) + want := ToolHealthResult{ + FailureCount: full.FailureCount, + ConsecutiveFailureMax: full.ConsecutiveFailureMax, + RetryCount: full.RetryCount, + EditChurnCount: full.EditChurnCount, + RunawayToolLoopCount: full.RunawayToolLoopCount, + FinalFailureStreak: full.FinalFailureStreak, + } + midTaskDelta := got.MidTaskCompactions + got.MidTaskCompactions = 0 + want.MidTaskCompactions = 0 + assert.Equal(t, want, got, "step %d: tool health diverged", step) + + midTask += midTaskDelta + wantMidTask := fullMidTask(calls, boundaries) + assert.Equal(t, wantMidTask, midTask, "step %d: mid-task diverged", step) + + // The next round's row values are the maintained ones. + row = ToolHealthRow{ + FailureCount: got.FailureCount, + RetryCount: got.RetryCount, + EditChurnCount: got.EditChurnCount, + } + state = nextState + + // State invariants. + assert.LessOrEqual(t, len(state.Trailing), TrailingFactCount) + assert.Equal(t, len(calls), state.TotalCalls) + if len(state.Trailing) > 0 { + last := state.Trailing[len(state.Trailing)-1].CallPos + wantLast := CallPos{ + MessageOrdinal: calls[len(calls)-1].MessageOrdinal, + CallIndex: calls[len(calls)-1].CallIndex, + } + assert.Equal(t, wantLast, last, "step %d: window tail", step) + } + } +} + +// TestIncrementalFoldRejectsOutOfWindowModification verifies the fallback +// contract: modifying a call older than the maintenance window returns +// ok=false and leaves the state untouched. +func TestIncrementalFoldRejectsOutOfWindowModification(t *testing.T) { + rng := newTestRand(t, 7) + calls := make([]ToolCallRow, 0, 60) + for i := range 60 { + calls = append(calls, parityRow(rng, i)) + } + state := SeedIncrementalState( + calls, nil, "", "", nil, nil, 0, 0, 0, + ) + full := fullToolHealth(calls) + row := ToolHealthRow{ + FailureCount: full.FailureCount, + RetryCount: full.RetryCount, + EditChurnCount: full.EditChurnCount, + } + old := calls[2] + calls[2].EventStatus = "errored" + _, _, ok := state.FoldToolHealth(nil, map[CallPos]ToolFact{ + {MessageOrdinal: old.MessageOrdinal}: factsFor(calls[2:3])[0], + }, row) + assert.False(t, ok, "out-of-window modification must fall back") +} + +// TestIncrementalStateRoundTrip checks JSON roundtrip and codec rejection. +func TestIncrementalStateRoundTrip(t *testing.T) { + rng := newTestRand(t, 11) + calls := make([]ToolCallRow, 0, 40) + for i := range 40 { + calls = append(calls, parityRow(rng, i)) + } + state := SeedIncrementalState( + calls, []int{5, 20}, "assistant", "done", nil, nil, 12, 4000, 11, + ) + blob, err := state.MarshalBinary() + require.NoError(t, err) + var restored IncrementalState + require.NoError(t, restored.UnmarshalBinary(blob)) + assert.Equal(t, state, restored) + + bad := append([]byte(nil), blob...) + require.Error(t, (&IncrementalState{}).UnmarshalBinary(bad[:len(bad)/2])) + state.CodecVersion++ + blob, err = state.MarshalBinary() + require.NoError(t, err) + require.Error(t, (&IncrementalState{}).UnmarshalBinary(blob)) +} + +// TestIncrementalFoldLongCrossingRun pins the failure-run latch across a +// run far longer than the trailing window. +func TestIncrementalFoldLongCrossingRun(t *testing.T) { + rng := newTestRand(t, 3) + // 100 identical failing Bash calls: one 100-long failure run. + calls := make([]ToolCallRow, 0, 100) + for i := range 100 { + calls = append(calls, ToolCallRow{ + ToolName: "exec_command", + Category: "Bash", + InputJSON: `{"command":"npm test"}`, + MessageOrdinal: i, + EventStatus: "errored", + }) + } + state := SeedIncrementalState( + calls, nil, "", "", nil, nil, 0, 0, 0, + ) + row := ToolHealthRow{ + FailureCount: 100, + RetryCount: ComputeToolHealth(calls).RetryCount, + EditChurnCount: 0, + } + // Append non-failing calls and flip nothing: the max must stay 100. + for step := range 50 { + appended := []ToolCallRow{{ + ToolName: "exec_command", + Category: "Bash", + InputJSON: `{"command":"ls"}`, + MessageOrdinal: 100 + step, + EventStatus: "completed", + }} + next, got, ok := state.FoldToolHealth(appended, nil, row) + require.True(t, ok) + assert.Equal(t, 100, got.ConsecutiveFailureMax, "step %d", step) + assert.Equal(t, 100, got.FailureCount) + assert.Equal(t, 0, got.FinalFailureStreak) + row = ToolHealthRow{ + FailureCount: got.FailureCount, + RetryCount: got.RetryCount, + EditChurnCount: got.EditChurnCount, + } + state = next + } + // Flip the very last call to failure: final streak 1, max still 100. + pos := CallPos{MessageOrdinal: 149} + appended := []ToolCallRow{} + modified := map[CallPos]ToolFact{pos: { + CallPos: pos, Failure: true, + ExactSignature: "exec_command\x00Bash\x00" + `{"command":"ls"}`, + CommandClass: "Bash:ls", + }} + next, got, ok := state.FoldToolHealth(appended, modified, row) + require.True(t, ok) + assert.Equal(t, 100, got.ConsecutiveFailureMax) + assert.Equal(t, 1, got.FinalFailureStreak) + _ = next + _ = rng +} + +// TestIncrementalFoldRetryAcrossSeed pins retry-run maintenance across the +// seed boundary. +func TestIncrementalFoldRetryAcrossSeed(t *testing.T) { + mk := func(ordinal int) ToolCallRow { + return ToolCallRow{ + ToolName: "edit_file", + Category: "Edit", + InputJSON: `{"file_path":"/src/a.go"}`, + MessageOrdinal: ordinal, + EventStatus: "completed", + } + } + calls := []ToolCallRow{mk(0), mk(1)} + state := SeedIncrementalState( + calls, nil, "", "", nil, nil, 0, 0, 0, + ) + row := ToolHealthRow{} + for step := range 6 { + appended := []ToolCallRow{mk(2 + step)} + next, got, ok := state.FoldToolHealth(appended, nil, row) + require.True(t, ok) + // Calls 0..(2+step) share name+input: a run of step+3 calls + // yields runLen-1 = step+2 retries. + assert.Equal(t, step+2, got.RetryCount, "step %d", step) + row = ToolHealthRow{RetryCount: got.RetryCount} + state = next + } +} + +// TestIncrementalFinalFailureStreakAcrossWindow pins the final-failure +// streak for a run longer than the trailing window: the fold must carry +// the pre-window run length forward instead of reporting the window size. +func TestIncrementalFinalFailureStreakAcrossWindow(t *testing.T) { + mk := func(ordinal int, fail bool) ToolCallRow { + status := "completed" + if fail { + status = "errored" + } + return ToolCallRow{ + ToolName: "exec_command", + Category: "Bash", + InputJSON: `{"command":"npm test"}`, + MessageOrdinal: ordinal, + EventStatus: status, + } + } + calls := make([]ToolCallRow, 0, 41) + for i := range 40 { + calls = append(calls, mk(i, true)) + } + state := SeedIncrementalState( + calls, nil, "", "", nil, nil, 0, 0, 0, + ) + row := ToolHealthRow{FailureCount: 40} + + // The 40-failure list itself: the final streak must be 40, not the + // 35-fact trailing window size. + var next IncrementalState + _, got, ok := state.FoldToolHealth(nil, nil, row) + require.True(t, ok) + full := fullToolHealth(calls) + assert.Equal(t, full.FinalFailureStreak, got.FinalFailureStreak, + "final failure streak across the trailing window") + assert.Equal(t, full.ConsecutiveFailureMax, got.ConsecutiveFailureMax) + + // Append a success and fold one call at a time: every fold must match + // the full recompute over the grown list. + appended := []ToolCallRow{mk(40, false), mk(41, false)} + for i := range appended { + full = fullToolHealth(append(slices.Clone(calls), appended[:i+1]...)) + next, got, ok = state.FoldToolHealth(appended[i:i+1], nil, row) + require.True(t, ok, "append step %d", i) + assert.Equal(t, full.FinalFailureStreak, got.FinalFailureStreak, + "append step %d: final streak", i) + assert.Equal(t, full.ConsecutiveFailureMax, got.ConsecutiveFailureMax, + "append step %d: max streak", i) + assert.Equal(t, full.FailureCount, got.FailureCount, + "append step %d: failure count", i) + row = ToolHealthRow{FailureCount: got.FailureCount} + state = next + } +} + +// TestIncrementalEditChurnOrdinalZero pins churn detection when the first +// edit sits at ordinal 0: the sentinel must distinguish "no prior edit" +// from "prior edit at ordinal 0" so three edits at 0,1,2 count one churn. +func TestIncrementalEditChurnOrdinalZero(t *testing.T) { + mk := func(ordinal int) ToolCallRow { + return ToolCallRow{ + ToolName: "edit_file", + Category: "Edit", + InputJSON: `{"file_path":"/src/a.go"}`, + MessageOrdinal: ordinal, + EventStatus: "completed", + } + } + full := ComputeToolHealth([]ToolCallRow{mk(0), mk(1), mk(2)}) + require.Equal(t, 1, full.EditChurnCount, + "full compute must count one churn for edits at 0,1,2") + + state := SeedIncrementalState( + []ToolCallRow{mk(0), mk(1)}, nil, "", "", nil, nil, 0, 0, 0, + ) + next, got, ok := state.FoldToolHealth( + []ToolCallRow{mk(2)}, nil, ToolHealthRow{}, + ) + require.True(t, ok) + assert.Equal(t, full.EditChurnCount, got.EditChurnCount, + "incremental fold must count one churn for edits at 0,1,2") + _ = next +} + +// TestIncrementalFoldMidTaskAcrossSeed pins mid-task counting for a +// boundary seeded with an open after-window. +func TestIncrementalFoldMidTaskAcrossSeed(t *testing.T) { + // Boundary at ordinal 10 with no calls after it; before-window has + // exec_command and edit_file names. + calls := make([]ToolCallRow, 0, 10) + for i := range 10 { + name := "exec_command" + if i%2 == 1 { + name = "edit_file" + } + calls = append(calls, ToolCallRow{ + ToolName: name, + Category: "Bash", + InputJSON: `{"command":"ls"}`, + MessageOrdinal: i, + }) + } + state := SeedIncrementalState( + calls, []int{10}, "", "", nil, nil, 0, 0, 0, + ) + require.Len(t, state.PendingBoundaries, 1) + row := ToolHealthRow{} + // Appends after the boundary: exec_command then edit_file → overlap 2. + appended := []ToolCallRow{ + {ToolName: "exec_command", Category: "Bash", + InputJSON: `{"command":"ls"}`, MessageOrdinal: 11}, + {ToolName: "edit_file", Category: "Edit", + InputJSON: `{"file_path":"/src/b.go"}`, MessageOrdinal: 12}, + } + next, got, ok := state.FoldToolHealth(appended, nil, row) + require.True(t, ok) + assert.Equal(t, 1, got.MidTaskCompactions) + assert.Empty(t, next.PendingBoundaries) +} + +func TestFoldToolHealthRunawayMutableWindowHeals(t *testing.T) { + calls := make([]ToolCallRow, 12) + for i := range calls { + calls[i] = ToolCallRow{ + ToolName: "exec_command", + Category: "Bash", + InputJSON: `{"command":"run"}`, + ResultContent: "failed", + EventStatus: "errored", + MessageOrdinal: i, + CallIndex: 0, + } + } + state := SeedIncrementalState(calls, nil, "", "", nil, nil, 0, 0, 0) + + pos := func(i int) CallPos { + return CallPos{MessageOrdinal: i, CallIndex: 0} + } + healthy := func(i int) ToolFact { + return ToolFact{ + CallPos: pos(i), + Failure: false, + ExactSignature: ExactToolSignature(calls[i]), + CommandClass: CommandClass(calls[i]), + } + } + + // Heal one failure: the trailing window still qualifies, but it is a + // mutable window and must not latch. + next, out, ok := state.FoldToolHealth( + nil, map[CallPos]ToolFact{pos(0): healthy(0)}, ToolHealthRow{}, + ) + require.True(t, ok) + assert.Equal(t, 1, out.RunawayToolLoopCount) + assert.False(t, next.RunawayHistorical, + "a qualifying window inside the mutable trailing region must not latch") + + // Heal all but one more failure: the window no longer qualifies, and + // the previously reported runaway must clear. + modified := make(map[CallPos]ToolFact) + for i := 1; i < 11; i++ { + modified[pos(i)] = healthy(i) + } + next, out, ok = next.FoldToolHealth(nil, modified, ToolHealthRow{}) + require.True(t, ok) + assert.Equal(t, 0, out.RunawayToolLoopCount, + "a healed trailing window must clear the runaway signal") + assert.False(t, next.RunawayHistorical) +} + +func TestFoldToolHealthRunawayHistoricalStaysLatched(t *testing.T) { + calls := make([]ToolCallRow, 47) + for i := range calls { + calls[i] = ToolCallRow{ + ToolName: "exec_command", + Category: "Bash", + InputJSON: `{"command":"run"}`, + ResultContent: "failed", + EventStatus: "errored", + MessageOrdinal: i, + CallIndex: 0, + } + } + state := SeedIncrementalState(calls, nil, "", "", nil, nil, 0, 0, 0) + require.True(t, state.RunawayHistorical, + "the seeded archive already has a fully exited runaway window") + + // Heal the entire trailing window: the mutable windows clear, but the + // window that exited the trailing region stays latched. + modified := make(map[CallPos]ToolFact) + for i := 35; i < 47; i++ { + modified[CallPos{MessageOrdinal: i, CallIndex: 0}] = ToolFact{ + CallPos: CallPos{MessageOrdinal: i, CallIndex: 0}, + Failure: false, + ExactSignature: ExactToolSignature(calls[i]), + CommandClass: CommandClass(calls[i]), + } + } + next, _, ok := state.FoldToolHealth(nil, modified, ToolHealthRow{}) + require.True(t, ok) + assert.True(t, next.RunawayHistorical, + "windows that fully exited the trailing window stay latched") +} + +func TestSeedRunawayWindowCrossingRetainedBoundaryStaysHistorical(t *testing.T) { + calls := make([]ToolCallRow, 47) + for i := range calls { + calls[i] = ToolCallRow{ + ToolName: "read_file", + Category: "Read", + InputJSON: fmt.Sprintf(`{"path":"/tmp/%d"}`, i), + ResultContent: "ok", + MessageOrdinal: i, + } + } + // The qualifying 12-call window starts before the retained-tail cut + // (47-35=12) and ends after it. All six failures are already older than + // the final 12-call mutable region, so the signal must be historical. + for i := 6; i < 18; i++ { + calls[i].ToolName = "exec_command" + calls[i].Category = "Bash" + calls[i].InputJSON = `{"command":"run"}` + if i >= 12 { + calls[i].ResultContent = "failed" + calls[i].EventStatus = "errored" + } + } + + state := SeedIncrementalState(calls, nil, "", "", nil, nil, 0, 0, 0) + require.True(t, state.RunawayHistorical, + "an immutable runaway window crossing the retained boundary must latch") + + appended := make([]ToolCallRow, 36) + for i := range appended { + appended[i] = ToolCallRow{ + ToolName: "read_file", + Category: "Read", + InputJSON: fmt.Sprintf(`{"path":"/healthy/%d"}`, i), + ResultContent: "ok", + MessageOrdinal: len(calls) + i, + } + } + next, out, ok := state.FoldToolHealth(appended, nil, ToolHealthRow{}) + require.True(t, ok) + assert.True(t, next.RunawayHistorical) + assert.Equal(t, 1, out.RunawayToolLoopCount) +} + +func TestFoldRunawayWindowCrossingNewRetainedBoundaryLatches(t *testing.T) { + calls := make([]ToolCallRow, 35) + for i := range calls { + calls[i] = ToolCallRow{ + ToolName: "read_file", + Category: "Read", + InputJSON: fmt.Sprintf(`{"path":"/initial/%d"}`, i), + ResultContent: "ok", + MessageOrdinal: i, + } + } + // This qualifying window is positions [12,24). At seed time the final + // call is still inside the 12-call mutable region, so it must remain + // reevaluable instead of being latched prematurely. + for i := 18; i < 24; i++ { + calls[i].ToolName = "exec_command" + calls[i].Category = "Bash" + calls[i].InputJSON = `{"command":"run"}` + calls[i].ResultContent = "failed" + calls[i].EventStatus = "errored" + } + state := SeedIncrementalState(calls, nil, "", "", nil, nil, 0, 0, 0) + require.False(t, state.RunawayHistorical) + + appendHealthy := func(start, count int) []ToolCallRow { + rows := make([]ToolCallRow, count) + for i := range rows { + rows[i] = ToolCallRow{ + ToolName: "read_file", + Category: "Read", + InputJSON: fmt.Sprintf(`{"path":"/healthy/%d"}`, start+i), + ResultContent: "ok", + MessageOrdinal: start + i, + } + } + return rows + } + + // Eighteen appends move the retained-tail cut to absolute position 18, + // through the middle of the qualifying [12,24) window. The window has + // simultaneously become older than the mutable region and must be latched + // before its left half is discarded. + firstAppend := appendHealthy(len(calls), 18) + next, out, ok := state.FoldToolHealth(firstAppend, nil, ToolHealthRow{}) + require.True(t, ok) + assert.True(t, next.RunawayHistorical) + assert.Equal(t, 1, out.RunawayToolLoopCount) + + // Once the original window has completely left retained facts, later + // healthy appends must not erase the historical signal. + secondAppend := appendHealthy(len(calls)+len(firstAppend), 35) + final, out, ok := next.FoldToolHealth(secondAppend, nil, ToolHealthRow{}) + require.True(t, ok) + assert.True(t, final.RunawayHistorical) + assert.Equal(t, 1, out.RunawayToolLoopCount) +} + +func TestIncrementalStateUnmarshalInitializesMutableMaps(t *testing.T) { + var state IncrementalState + require.NoError(t, state.UnmarshalBinary([]byte( + `{"codec_version":3,"total_calls":0}`, + ))) + require.NotNil(t, state.EditLast) + require.NotNil(t, state.ModelCounts) + require.NotNil(t, state.ModelFirstSeen) + + next, _, ok := state.FoldToolHealth([]ToolCallRow{{ + Category: "Edit", + InputJSON: `{"file_path":"main.go"}`, + MessageOrdinal: 1, + }}, nil, ToolHealthRow{}) + require.True(t, ok) + require.Contains(t, next.EditLast, "main.go", + "the first edit append must not panic on a decoded empty map") +} diff --git a/internal/signals/toolhealth.go b/internal/signals/toolhealth.go index 3b5c32ca2..7346f289d 100644 --- a/internal/signals/toolhealth.go +++ b/internal/signals/toolhealth.go @@ -14,6 +14,13 @@ type ToolCallRow struct { MessageOrdinal int CallIndex int EventStatus string // "", "completed", "errored", "cancelled", "running" + // ContentFailure is a pre-computed content-heuristic verdict used by + // streaming writers whose rows carry placeholder result content (the + // real summary lives in staging). When the last event carries no + // status, IsFailure prefers this verdict over re-scanning + // ResultContent. Legacy rows leave it false and fall through to the + // content scan as before. + ContentFailure bool } // ToolHealthSignals holds computed health metrics for a session's @@ -46,12 +53,16 @@ func ComputeToolHealth(calls []ToolCallRow) ToolHealthSignals { } // IsFailure returns true when a tool call represents a failure, -// either by event status or by content heuristics. +// either by event status, by a pre-computed content verdict, or +// by content heuristics. func IsFailure(c ToolCallRow) bool { if c.EventStatus != "" { return c.EventStatus == "errored" || c.EventStatus == "cancelled" } + if c.ContentFailure { + return true + } return isContentFailure(c.Category, c.ResultContent) } diff --git a/internal/sync/checkpoint.go b/internal/sync/checkpoint.go new file mode 100644 index 000000000..0ebed7b50 --- /dev/null +++ b/internal/sync/checkpoint.go @@ -0,0 +1,423 @@ +package sync + +import ( + "context" + "crypto/sha256" + "encoding" + "encoding/hex" + "fmt" + "io" + "os" + + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/parser" +) + +const ( + codexCheckpointVersion = 1 + codexCheckpointAnchorSize = 128 << 10 +) + +type codexCheckpointDecision int + +const ( + // codexCheckpointFallback means the checkpoint cannot prove a safe + // resume; the caller uses the existing fingerprint/full-parse path. + codexCheckpointFallback codexCheckpointDecision = iota + // codexCheckpointUnchanged means stat + checkpoint prove the transcript + // is byte-identical to the committed prefix; the caller skips without + // hashing anything. + codexCheckpointUnchanged + // codexCheckpointAppend means the checkpoint proves a safe append-only + // growth; the caller parses only the new tail and resumes the hash. + codexCheckpointAppend + // codexCheckpointInvalid means a checkpoint exists but its proof failed + // (identity changed, truncation, anchor mismatch, missing hash state). + // The caller must authoritatively reparse and replace stored rows — + // never resume and never append against the unverified prefix. + codexCheckpointInvalid + // codexCheckpointMissing means a stored Codex session has no usable + // checkpoint (for example, an archive written before checkpoints + // existed). Missing optimization state must not turn an unchanged + // archive into a migration workload. The caller keeps the existing + // freshness gates and creates a checkpoint on the next real source + // change that already requires an authoritative parse. + codexCheckpointMissing +) + +type codexCheckpointResult struct { + fingerprint parser.SourceFingerprint + checkpoint *db.ParserCheckpoint + decision codexCheckpointDecision + // seed is the persisted parser cursor for the append resume; loaded + // lazily from the checkpoint blobs only on the append branch. + seed []byte + // hashState is the resumable SHA-256 state through the committed + // prefix, ready for the incremental resume to continue from. + hashState []byte +} + +// codexCheckpointFingerprint tries to resolve a Codex source fingerprint and +// its persisted checkpoint without reading the transcript prefix: +// +// - unchanged: stat identity + size match the checkpoint offset, so the +// committed prefix is trusted (append-trust mode) and the file is skipped; +// - append: identity matches, the file only grew, the tail anchor digest +// matches, and a resumable SHA-256 state exists, so the full-file +// fingerprint is derived by hashing only [offset, size); +// - otherwise fallback, which keeps every existing conservative path. +func (e *Engine) codexCheckpointFingerprint( + ctx context.Context, + source parser.SourceRef, + file parser.DiscoveredFile, +) (codexCheckpointResult, error) { + res := codexCheckpointResult{decision: codexCheckpointFallback} + if e.forceParse || file.ForceParse || e.checkpointAudit.Load() { + // Audit mode deliberately bypasses the checkpoint gate so the + // provider's full-source fingerprint can verify content and repair + // same-stat in-place rewrites that append-trust would otherwise miss. + return res, nil + } + path := providerDiscoveredPath(source) + if path == "" { + return res, nil + } + lookupPath := path + if e.pathRewriter != nil { + lookupPath = e.pathRewriter(path) + } + // Resolve the session through the DB so codex-format forks (TraeX) + // use their real session id prefix, and use the same row to validate + // the checkpoint against the committed offset/ordinal/hash. A + // checkpoint that disagrees with the DB (e.g. a crash after a full + // replacement committed but before its checkpoint upsert) must never + // seed a resume: mark it invalid so the caller rebuilds + // authoritatively. + inc, ok := e.db.GetSessionForIncremental( + lookupPath, string(file.Agent), + ) + if !ok { + return res, nil + } + cp, hasCP, err := e.db.GetParserCheckpoint(inc.ID) + if err != nil { + return res, fmt.Errorf("loading checkpoint %s: %w", inc.ID, err) + } + if !hasCP || cp.Version != codexCheckpointVersion { + res.decision = codexCheckpointMissing + return res, nil + } + if e.pathRewriter != nil { + // Remote materializations have no trustworthy local identity: + // the checkpoint gate cannot stat-trust a rewritten path, so the + // caller deep-verifies on every pass. + return res, nil + } + if cp.SessionID != inc.ID || + cp.FilePath != e.effectiveSourcePath(path) || + cp.Agent != string(file.Agent) { + return res, nil + } + storedHash, hasStoredHash := e.db.GetFileHashByAgentPath( + lookupPath, string(file.Agent), + ) + if !hasStoredHash || storedHash != cp.Hash || + inc.FileSize != cp.Offset || inc.NextOrdinal != cp.NextOrdinal || + inc.FileMtime != cp.FileMTime { + // The committed DB prefix is newer than (or inconsistent with) the + // surviving checkpoint: resuming from the old seed would silently + // parse against the wrong prefix. Rebuild instead. + res.decision = codexCheckpointInvalid + return res, nil + } + if e.db.GetDataVersionByAgentPath(lookupPath, string(file.Agent)) < + db.CurrentDataVersion() { + return res, nil + } + if e.pathNeedsProjectReparse(file.Agent, path) { + return res, nil + } + if file.Agent == parser.AgentCodex && e.codexIndexSessionNameChanged(path) { + return res, nil + } + + info, err := os.Stat(path) + if err != nil { + return res, nil // missing/raced source: existing path handles it + } + inode, device := getFileIdentity(path, info) + if inode != int64(cp.FileInode) || device != int64(cp.FileDevice) { + res.decision = codexCheckpointInvalid + return res, nil + } + rawMtime := info.ModTime().UnixNano() + effectiveMtime := rawMtime + if file.Agent == parser.AgentCodex { + effectiveMtime = parser.CodexEffectiveMtime(path, rawMtime) + } + + if info.Size() == cp.Offset { + // Change-time guards same-size same-mtime rewrites for free: a + // write restores mtime but cannot restore ctime, so a mismatch + // here proves the bytes changed even when every other identity + // field matches. Rows without a stored change time (0) stay + // conservative and rebuild. + if changeTime, ok := fileChangeTime(path, info); !ok || + cp.FileChangeTime == 0 || changeTime != cp.FileChangeTime { + return res, nil + } + // The mtime gate applies only to the unchanged branch: an append + // naturally moves the mtime, and the append branch is proven by + // identity + tail anchor + monotonic size instead. An index-only + // mtime bump (transcript mtime unchanged) is safe to skip when the + // title did not change, mirroring shouldSkipCodexFingerprint. + indexOnlyBump := effectiveMtime != cp.FileMTime && + rawMtime <= cp.FileMTime + if effectiveMtime != cp.FileMTime && !indexOnlyBump { + return res, nil + } + res.decision = codexCheckpointUnchanged + res.fingerprint = parser.SourceFingerprint{ + Key: codexCheckpointFingerprintKey(source, path), + Size: info.Size(), + MTimeNS: effectiveMtime, + Inode: uint64(inode), + Device: uint64(device), + Hash: cp.Hash, + } + return res, nil + } + if info.Size() < cp.Offset { + res.decision = codexCheckpointInvalid // truncation: never resume + return res, nil + } + if cp.TailAnchorDigest == "" { + res.decision = codexCheckpointInvalid + return res, nil + } + matches, err := codexCheckpointAnchorMatches(path, cp) + if err != nil || !matches { + res.decision = codexCheckpointInvalid + return res, nil + } + // The append branch loads the lazy payload (cursor + hash state); the + // unchanged branch above never touches it. + blobs, hasBlobs, err := e.db.GetParserCheckpointBlobs(inc.ID) + if err != nil { + return res, fmt.Errorf("loading checkpoint blobs %s: %w", inc.ID, err) + } + if !hasBlobs || len(blobs.HashState) == 0 { + res.decision = codexCheckpointInvalid + return res, nil + } + stateDigest, err := codexHashStateDigest(blobs.HashState) + if err != nil || stateDigest != cp.Hash { + res.decision = codexCheckpointInvalid + return res, nil + } + _, hash, err := codexResumeHash( + path, cp.Offset, info.Size(), blobs.HashState, + ) + if err != nil { + res.decision = codexCheckpointInvalid + return res, nil + } + res.decision = codexCheckpointAppend + res.checkpoint = cp + res.seed = blobs.Cursor + // The incremental path resumes from the OLD state through the committed + // safe offset (which may stop before a partial tail at EOF); the + // full-file hash above is only the fingerprint. Advancing the state here + // would double-count the tail when newOffset < info.Size(). + res.hashState = blobs.HashState + res.fingerprint = parser.SourceFingerprint{ + Key: codexCheckpointFingerprintKey(source, path), + Size: info.Size(), + MTimeNS: effectiveMtime, + Inode: uint64(inode), + Device: uint64(device), + Hash: hash, + } + return res, nil +} + +func codexCheckpointFingerprintKey( + source parser.SourceRef, path string, +) string { + for _, candidate := range []string{ + source.FingerprintKey, source.Key, + } { + if candidate != "" { + return candidate + } + } + return path +} + +// codexCheckpointAnchorDigest returns the SHA-256 digest of the last +// min(codexCheckpointAnchorSize, offset) bytes of the committed prefix +// [0, offset). The append gate reads that bounded window once and compares +// the digest, instead of storing the raw anchor bytes in the checkpoint row. +func codexCheckpointAnchorDigest( + path string, offset int64, +) (string, error) { + if offset <= 0 { + return "", nil + } + start := offset - codexCheckpointAnchorSize + start = max(start, 0) + f, err := os.Open(path) + if err != nil { + return "", err + } + defer f.Close() + if _, err := f.Seek(start, io.SeekStart); err != nil { + return "", err + } + h := sha256.New() + if _, err := io.CopyN(h, f, offset-start); err != nil { + return "", fmt.Errorf( + "reading anchor window for %s: %w", path, err, + ) + } + return hex.EncodeToString(h.Sum(nil)), nil +} + +func codexCheckpointAnchorMatches( + path string, cp *db.ParserCheckpoint, +) (bool, error) { + digest, err := codexCheckpointAnchorDigest(path, cp.Offset) + if err != nil { + return false, err + } + return digest == cp.TailAnchorDigest, nil +} + +// codexResumeHashFn is an overridable seam for tests that exercise the +// post-parse reconstruction failure path. Production always resumes through +// codexResumeHash; the checkpoint gate keeps using codexResumeHash directly +// so a test can fail only the engine's later reconstruction. +var codexResumeHashFn = codexResumeHash + +// codexResumeHash continues a persisted SHA-256 state over [offset, size) and +// returns the new state plus the full-file digest. +func codexResumeHash( + path string, offset, size int64, state []byte, +) ([]byte, string, error) { + h := sha256.New() + unmarshaler, ok := h.(encoding.BinaryUnmarshaler) + if !ok { + return nil, "", fmt.Errorf("sha256 does not support state restore") + } + if err := unmarshaler.UnmarshalBinary(state); err != nil { + return nil, "", fmt.Errorf("restoring hash state: %w", err) + } + f, err := os.Open(path) + if err != nil { + return nil, "", err + } + defer f.Close() + if _, err := f.Seek(offset, io.SeekStart); err != nil { + return nil, "", err + } + if _, err := io.CopyN(h, f, size-offset); err != nil { + return nil, "", fmt.Errorf( + "hashing appended bytes %d..%d of %s: %w", + offset, size, path, err, + ) + } + newState, err := h.(encoding.BinaryMarshaler).MarshalBinary() + if err != nil { + return nil, "", fmt.Errorf("marshaling hash state: %w", err) + } + return newState, hex.EncodeToString(h.Sum(nil)), nil +} + +// codexHashStateDigest finalizes a resumable SHA-256 state into its digest. +func codexHashStateDigest(state []byte) (string, error) { + h := sha256.New() + unmarshaler, ok := h.(encoding.BinaryUnmarshaler) + if !ok { + return "", fmt.Errorf("sha256 does not support state restore") + } + if err := unmarshaler.UnmarshalBinary(state); err != nil { + return "", fmt.Errorf("restoring hash state: %w", err) + } + return hex.EncodeToString(h.Sum(nil)), nil +} + +// buildCodexFullParseCheckpoint assembles the checkpoint row and blob +// payload for a just-parsed full snapshot from the pending write's +// captured state. It returns nil when the write carries no usable resume +// payload (the parse did not end at a safe boundary). +func (e *Engine) buildCodexFullParseCheckpoint( + path string, pw pendingWrite, +) (*db.ParserCheckpoint, *db.ParserCheckpointBlobs, error) { + if len(pw.checkpoint) == 0 || len(pw.checkpointHashState) == 0 || + pw.checkpointAnchorDigest == "" { + return nil, nil, nil + } + hash, err := codexHashStateDigest(pw.checkpointHashState) + if err != nil { + return nil, nil, err + } + // Identity comes from the parse snapshot, never from a later path + // stat: the cursor, hash state, and anchor describe the bytes the + // parser read, and pairing them with a fresher stat could bless a + // concurrent rewrite as the parsed content. + cp := &db.ParserCheckpoint{ + SessionID: pw.sess.ID, + Agent: string(pw.sess.Agent), + FilePath: e.effectiveSourcePath(path), + FileInode: uint64(pw.sess.File.Inode), + FileDevice: uint64(pw.sess.File.Device), + FileMTime: pw.sess.File.Mtime, + FileChangeTime: pw.sess.File.ChangeTime, + Offset: pw.sess.File.Size, + TailAnchorDigest: pw.checkpointAnchorDigest, + Hash: hash, + NextOrdinal: pw.sess.MessageCount, + Version: codexCheckpointVersion, + } + return cp, &db.ParserCheckpointBlobs{ + SessionID: pw.sess.ID, + Cursor: pw.checkpoint, + HashState: pw.checkpointHashState, + }, nil +} + +// buildCodexCheckpoint assembles the checkpoint metadata row and the lazy +// blob payload for a committed prefix of size newOffset. anchorDigest is the +// digest of the prefix's trailing anchor window; callers obtain it either +// from the parser's single-pass capture (full parse) or from a bounded +// read of the file tail (incremental path). +func buildCodexCheckpoint( + sessionID, agent, storedPath string, + inode, device uint64, + mtime, changeTime int64, + newOffset int64, + cursor []byte, + hashState []byte, + hash string, + nextOrdinal int, + anchorDigest string, +) (*db.ParserCheckpoint, db.ParserCheckpointBlobs) { + return &db.ParserCheckpoint{ + SessionID: sessionID, + FileChangeTime: changeTime, + Agent: agent, + FilePath: storedPath, + FileInode: uint64(inode), + FileDevice: uint64(device), + FileMTime: mtime, + Offset: newOffset, + TailAnchorDigest: anchorDigest, + Hash: hash, + NextOrdinal: nextOrdinal, + Version: codexCheckpointVersion, + }, db.ParserCheckpointBlobs{ + SessionID: sessionID, + Cursor: cursor, + HashState: hashState, + } +} diff --git a/internal/sync/checkpoint_bootstrap_test.go b/internal/sync/checkpoint_bootstrap_test.go new file mode 100644 index 000000000..cb533b9e7 --- /dev/null +++ b/internal/sync/checkpoint_bootstrap_test.go @@ -0,0 +1,458 @@ +package sync + +import ( + "context" + "errors" + "os" + "path/filepath" + "runtime" + "strconv" + "strings" + "testing" + + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/parser" + "go.kenn.io/agentsview/internal/testjsonl" +) + +// TestCodexCheckpointAdoptionIsLazyForUpgradedArchive pins the upgrade path: +// deleting the machine-local checkpoint from an already stored Codex session +// must leave unchanged archives on the existing stat-digest fast path. The +// checkpoint is adopted on the next real source change, when an authoritative +// parse is already required. +func TestCodexCheckpointAdoptionIsLazyForUpgradedArchive(t *testing.T) { + const uuid = "019eb791-cf7d-75c1-8439-9ed74c122b05" + root := writeCodexParityRoot(t, uuid) + sessionID := "codex:" + uuid + + database, err := db.Open(filepath.Join(t.TempDir(), "bootstrap.db")) + require.NoError(t, err) + t.Cleanup(func() { _ = database.Close() }) + engine := NewEngine(database, EngineConfig{ + AgentDirs: map[parser.AgentType][]string{ + parser.AgentCodex: {root}, + }, + Machine: "local", + }) + t.Cleanup(engine.Close) + + // Initial cold sync commits content and a checkpoint. + require.Equal(t, 1, engine.SyncAll(t.Context(), nil).Synced) + cp, ok, err := database.GetParserCheckpoint(sessionID) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, codexCheckpointVersion, cp.Version) + before, err := database.GetAllMessages(t.Context(), sessionID) + require.NoError(t, err) + require.NotEmpty(t, before) + + // Simulate an archive written before parser checkpoints existed. + require.NoError(t, database.DeleteParserCheckpoint(sessionID)) + + // An unchanged archive stays on the current-main stat-digest path. It + // neither rewrites the session nor eagerly migrates optimization state. + if runtime.GOOS == "linux" { + rcharBefore := processRchar(t) + stats := engine.SyncAll(t.Context(), nil) + require.Zero(t, stats.Synced) + require.Less(t, processRchar(t)-rcharBefore, int64(1<<20)) + } else { + require.Zero(t, engine.SyncAll(t.Context(), nil).Synced) + } + _, ok, err = database.GetParserCheckpoint(sessionID) + require.NoError(t, err) + require.False(t, ok, "unchanged archives must not be eagerly migrated") + afterSkip, err := database.GetAllMessages(t.Context(), sessionID) + require.NoError(t, err) + require.Equal(t, before, afterSkip) + + // A real append already requires parsing. The same authoritative write + // adopts a checkpoint atomically with the updated projection. + path := filepath.Join( + root, "2024", "01", "01", + "rollout-2024-01-01T10-00-00-"+uuid+".jsonl", + ) + appended := testjsonl.JoinJSONL( + testjsonl.CodexFunctionCallOutputJSON( + "call_a", "late output", "2024-01-01T10:00:13Z", + ), + testjsonl.CodexTokenCountJSON( + "2024-01-01T10:00:14Z", 240, 60, 160, + ), + ) + f, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0o644) + require.NoError(t, err) + _, err = f.WriteString(appended) + require.NoError(t, err) + require.NoError(t, f.Close()) + + stats := engine.SyncAll(t.Context(), nil) + require.Zero(t, stats.Failed) + require.Equal(t, 1, stats.Synced) + cp, ok, err = database.GetParserCheckpoint(sessionID) + require.NoError(t, err) + require.True(t, ok, "a real source change must adopt a checkpoint") + require.Equal(t, codexCheckpointVersion, cp.Version) + + // Authoritative full-parse parity for the changed source. + cfg := parser.ProviderConfig{Roots: []string{root}, Machine: "local"} + provider, ok := parser.NewProvider(parser.AgentCodex, cfg) + require.True(t, ok) + source, found, err := provider.FindSource( + context.Background(), parser.FindSourceRequest{FullSessionID: sessionID}, + ) + require.NoError(t, err) + require.True(t, found) + collecting := parser.NewCodexCollectingSink(0) + _, msgs, _, _, _, _, err := parser.ParseCodexSessionStreaming( + cfg, source, collecting, + ) + require.NoError(t, err) + stored, err := database.GetAllMessages(t.Context(), sessionID) + require.NoError(t, err) + for i := range stored { + stored[i].ID = 0 + stored[i].SessionID = "" + for j := range stored[i].ToolCalls { + stored[i].ToolCalls[j].MessageID = 0 + stored[i].ToolCalls[j].SessionID = "" + } + } + require.Equal(t, toDBMessages(pendingWrite{msgs: msgs}, nil), stored) +} + +func TestCodexCheckpointMissingHonorsMatchingSkipEntry(t *testing.T) { + const uuid = "019eb791-cf7d-75c1-8439-9ed74c122c07" + root := writeCodexParityRoot(t, uuid) + sessionID := "codex:" + uuid + + database, err := db.Open(filepath.Join(t.TempDir(), "bootstrap-skip.db")) + require.NoError(t, err) + t.Cleanup(func() { _ = database.Close() }) + engine := NewEngine(database, EngineConfig{ + AgentDirs: map[parser.AgentType][]string{ + parser.AgentCodex: {root}, + }, + Machine: "local", + }) + t.Cleanup(engine.Close) + + require.Equal(t, 1, engine.SyncAll(t.Context(), nil).Synced) + require.NoError(t, database.DeleteParserCheckpoint(sessionID)) + + path := filepath.Join( + root, "2024", "01", "01", + "rollout-2024-01-01T10-00-00-"+uuid+".jsonl", + ) + cfg := parser.ProviderConfig{Roots: []string{root}, Machine: "local"} + provider, ok := parser.NewProvider(parser.AgentCodex, cfg) + require.True(t, ok) + source, found, err := provider.FindSource( + context.Background(), parser.FindSourceRequest{FullSessionID: sessionID}, + ) + require.NoError(t, err) + require.True(t, found) + fingerprint, err := provider.Fingerprint(context.Background(), source) + require.NoError(t, err) + file := parser.DiscoveredFile{ + Path: path, Agent: parser.AgentCodex, + ProviderSource: &source, ProviderProcess: true, + } + cacheKey := providerProcessCacheKey( + file, source, fingerprint, provider.Capabilities().Sync, + ) + engine.InjectSkipCache(map[string]int64{cacheKey: fingerprint.MTimeNS}) + + require.Zero(t, engine.SyncAll(t.Context(), nil).Synced, + "missing optimization state must not defeat a valid skip entry") + _, ok, err = database.GetParserCheckpoint(sessionID) + require.NoError(t, err) + require.False(t, ok) +} + +func TestCodexCheckpointInvalidIsDiscardedOnNextSourceChange(t *testing.T) { + const uuid = "019eb791-cf7d-75c1-8439-9ed74c122c08" + root := writeCodexParityRoot(t, uuid) + sessionID := "codex:" + uuid + path := filepath.Join( + root, "2024", "01", "01", + "rollout-2024-01-01T10-00-00-"+uuid+".jsonl", + ) + + database, err := db.Open(filepath.Join(t.TempDir(), "invalid-cp.db")) + require.NoError(t, err) + t.Cleanup(func() { _ = database.Close() }) + engine := NewEngine(database, EngineConfig{ + AgentDirs: map[parser.AgentType][]string{ + parser.AgentCodex: {root}, + }, + Machine: "local", + }) + t.Cleanup(engine.Close) + + require.Equal(t, 1, engine.SyncAll(t.Context(), nil).Synced) + cp, ok, err := database.GetParserCheckpoint(sessionID) + require.NoError(t, err) + require.True(t, ok) + blobs, hasBlobs, err := database.GetParserCheckpointBlobs(sessionID) + require.NoError(t, err) + require.True(t, hasBlobs) + corrupted := *cp + corrupted.Hash = "corrupted-proof" + require.NoError(t, database.UpsertParserCheckpoint(corrupted, blobs)) + + // Stat-digest freshness may skip an unchanged source without consulting + // disposable checkpoint state. A real source change reaches checkpoint + // validation, rejects the corrupted proof, and performs a full repair. + f, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0o644) + require.NoError(t, err) + _, err = f.WriteString(testjsonl.CodexMsgJSON( + "user", "changed", "2024-01-01T10:00:13Z", + )) + require.NoError(t, err) + require.NoError(t, f.Close()) + + require.Equal(t, 1, engine.SyncAll(t.Context(), nil).Synced) + _, ok, err = database.GetParserCheckpoint(sessionID) + require.NoError(t, err) + require.False(t, ok, + "an unsafe full-parse boundary must discard the invalid checkpoint") + stored, err := database.GetAllMessages(t.Context(), sessionID) + require.NoError(t, err) + require.Equal(t, "changed", stored[len(stored)-1].Content) +} + +func TestCodexCheckpointAuditDeepVerifiesDespiteWarmGates(t *testing.T) { + const uuid = "019eb791-cf7d-75c1-8439-9ed74c122c09" + root := writeCodexParityRoot(t, uuid) + + database, err := db.Open(filepath.Join(t.TempDir(), "audit-cp.db")) + require.NoError(t, err) + t.Cleanup(func() { _ = database.Close() }) + engine := NewEngine(database, EngineConfig{ + AgentDirs: map[parser.AgentType][]string{ + parser.AgentCodex: {root}, + }, + Machine: "local", + }) + t.Cleanup(engine.Close) + + require.Equal(t, 1, engine.SyncAll(t.Context(), nil).Synced) + require.Zero(t, engine.SyncAll(t.Context(), nil).Synced, + "second pass must be a warm no-op") + + engine.SetCheckpointAudit(true) + t.Cleanup(func() { engine.SetCheckpointAudit(false) }) + require.Zero(t, engine.SyncAll(t.Context(), nil).Synced, + "an unchanged source still skips after the audit's full hash check") +} + +func TestCodexCheckpointAuditRepairsPrefixRewriteBeforeAppend(t *testing.T) { + const uuid = "019eb791-cf7d-75c1-8439-9ed74c122c10" + root := t.TempDir() + day := filepath.Join(root, "2024", "01", "01") + require.NoError(t, os.MkdirAll(day, 0o755)) + path := filepath.Join( + day, "rollout-2024-01-01T10-00-00-"+uuid+".jsonl", + ) + // Keep the rewrite well before the trailing 128KiB anchor so a + // tail-append alone cannot detect it. + body := "OLD-MARKER" + strings.Repeat("p", 200*1024) + initial := testjsonl.JoinJSONL( + testjsonl.CodexSessionMetaJSON( + uuid, "/tmp", "user", "2024-01-01T10:00:00Z", + ), + testjsonl.CodexTurnContextJSON( + "gpt-5.4", "2024-01-01T10:00:01Z", + ), + testjsonl.CodexMsgJSON( + "user", body, "2024-01-01T10:00:02Z", + ), + ) + require.NoError(t, os.WriteFile(path, []byte(initial), 0o644)) + + database, err := db.Open(filepath.Join(t.TempDir(), "audit-rewrite.db")) + require.NoError(t, err) + t.Cleanup(func() { _ = database.Close() }) + engine := NewEngine(database, EngineConfig{ + AgentDirs: map[parser.AgentType][]string{ + parser.AgentCodex: {root}, + }, + Machine: "local", + }) + t.Cleanup(engine.Close) + sessionID := "codex:" + uuid + + require.Equal(t, 1, engine.SyncAll(t.Context(), nil).Synced) + before, err := database.GetAllMessages(t.Context(), sessionID) + require.NoError(t, err) + require.NotEmpty(t, before) + require.Contains(t, before[0].Content, "OLD-MARKER") + + info, err := os.Stat(path) + require.NoError(t, err) + rewritten := strings.Replace(initial, "OLD-MARKER", "NEW-MARKER", 1) + require.NotEqual(t, initial, rewritten) + require.NoError(t, os.WriteFile(path, []byte(rewritten), 0o644)) + require.NoError(t, os.Chtimes(path, info.ModTime(), info.ModTime())) + f, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0o644) + require.NoError(t, err) + _, err = f.WriteString(testjsonl.CodexTokenCountJSON( + "2024-01-01T10:00:03Z", 100, 50, 80, + ) + "\n") + require.NoError(t, err) + require.NoError(t, f.Close()) + + engine.SetCheckpointAudit(true) + t.Cleanup(func() { engine.SetCheckpointAudit(false) }) + require.Equal(t, 1, engine.SyncAll(t.Context(), nil).Synced, + "the audit must repair a rewritten prefix instead of tail-applying") + after, err := database.GetAllMessages(t.Context(), sessionID) + require.NoError(t, err) + require.NotEmpty(t, after) + require.Contains(t, after[0].Content, "NEW-MARKER", + "the repaired prefix must replace the stale stored content") +} + +func TestCodexIncrementalResumeHashFailureRetainsCheckpoint(t *testing.T) { + const uuid = "019eb791-cf7d-75c1-8439-9ed74c122c13" + root := t.TempDir() + day := filepath.Join(root, "2024", "01", "01") + require.NoError(t, os.MkdirAll(day, 0o755)) + path := filepath.Join( + day, "rollout-2024-01-01T10-00-00-"+uuid+".jsonl", + ) + initial := testjsonl.JoinJSONL( + testjsonl.CodexSessionMetaJSON( + uuid, "/tmp", "user", "2024-01-01T10:00:00Z", + ), + testjsonl.CodexTurnContextJSON( + "gpt-5.4", "2024-01-01T10:00:01Z", + ), + testjsonl.CodexMsgJSON( + "user", "run the command", "2024-01-01T10:00:02Z", + ), + testjsonl.CodexMsgJSON( + "assistant", "running", "2024-01-01T10:00:03Z", + ), + testjsonl.CodexFunctionCallWithCallIDJSON( + "exec_command", "call_x", nil, "2024-01-01T10:00:04Z", + ), + ) + require.NoError(t, os.WriteFile(path, []byte(initial), 0o644)) + sessionID := "codex:" + uuid + + database, err := db.Open(filepath.Join(t.TempDir(), "resume-fail.db")) + require.NoError(t, err) + t.Cleanup(func() { _ = database.Close() }) + engine := NewEngine(database, EngineConfig{ + AgentDirs: map[parser.AgentType][]string{ + parser.AgentCodex: {root}, + }, + Machine: "local", + }) + t.Cleanup(engine.Close) + + require.Equal(t, 1, engine.SyncAll(t.Context(), nil).Synced) + before, ok, err := database.GetParserCheckpoint(sessionID) + require.NoError(t, err) + require.True(t, ok) + oldOffset := before.Offset + + appended := testjsonl.JoinJSONL( + testjsonl.CodexFunctionCallOutputJSON( + "call_x", "late output", "2024-01-01T10:00:13Z", + ), + ) + f, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0o644) + require.NoError(t, err) + _, err = f.WriteString(appended) + require.NoError(t, err) + require.NoError(t, f.Close()) + + orig := codexResumeHashFn + codexResumeHashFn = func( + string, int64, int64, []byte, + ) ([]byte, string, error) { + return nil, "", errors.New("injected resume failure") + } + t.Cleanup(func() { codexResumeHashFn = orig }) + + require.Equal(t, 1, engine.SyncAll(t.Context(), nil).Synced) + cp, ok, err := database.GetParserCheckpoint(sessionID) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, oldOffset, cp.Offset, + "a failed reconstruction must not advance the checkpoint") + + // The failed hash-state reconstruction does not invalidate the content + // transaction: the stored projection carries the authoritative full-file + // hash, while the disposable checkpoint remains at its previous offset. + storedHash, hasHash := database.GetFileHashByAgentPath(path, "codex") + require.True(t, hasHash) + actualHash, err := ComputeFileHash(path) + require.NoError(t, err) + require.Equal(t, actualHash, storedHash) + + // Lazy checkpoint adoption leaves an unchanged source on the stat-digest + // fast path. Restoring the hasher alone must not rewrite the session or + // advance the stale checkpoint. + codexResumeHashFn = orig + unchanged := engine.SyncAll(t.Context(), nil) + require.Zero(t, unchanged.Failed) + require.Zero(t, unchanged.Synced) + skipped, ok, err := database.GetParserCheckpoint(sessionID) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, oldOffset, skipped.Offset) + + // The next real source change reaches checkpoint validation and repairs + // the stale optimization state in the same authoritative write. + repairBoundary := testjsonl.JoinJSONL( + testjsonl.CodexTokenCountJSON( + "2024-01-01T10:00:14Z", 240, 60, 160, + ), + ) + f, err = os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0o644) + require.NoError(t, err) + _, err = f.WriteString(repairBoundary) + require.NoError(t, err) + require.NoError(t, f.Close()) + + repaired := engine.SyncAll(t.Context(), nil) + require.Zero(t, repaired.Failed) + require.Equal(t, 1, repaired.Synced) + fixed, ok, err := database.GetParserCheckpoint(sessionID) + require.NoError(t, err) + require.True(t, ok) + require.Greater(t, fixed.Offset, oldOffset) + require.Equal(t, + int64(len(initial)+len(appended)+len(repairBoundary)), + fixed.Offset, + ) + storedHash, hasHash = database.GetFileHashByAgentPath(path, "codex") + require.True(t, hasHash) + actualHash, err = ComputeFileHash(path) + require.NoError(t, err) + require.Equal(t, actualHash, storedHash) + require.Equal(t, storedHash, fixed.Hash) +} + +// processRchar returns the process's cumulative read bytes (Linux). +func processRchar(t *testing.T) int64 { + t.Helper() + data, err := os.ReadFile("/proc/self/io") + require.NoError(t, err) + for line := range strings.SplitSeq(string(data), "\n") { + if rest, ok := strings.CutPrefix(line, "rchar: "); ok { + v, err := strconv.ParseInt( + strings.TrimSpace(rest), 10, 64, + ) + require.NoError(t, err) + return v + } + } + t.Fatal("no rchar in /proc/self/io") + return 0 +} diff --git a/internal/sync/checkpoint_review_test.go b/internal/sync/checkpoint_review_test.go new file mode 100644 index 000000000..1de951983 --- /dev/null +++ b/internal/sync/checkpoint_review_test.go @@ -0,0 +1,451 @@ +package sync + +import ( + "context" + "crypto/sha256" + "encoding" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/parser" + "go.kenn.io/agentsview/internal/testjsonl" +) + +// Regression tests for the review findings on checkpoint consistency: +// 1. a checkpoint persisted for a committed prefix must hash exactly that +// prefix, even when the live file kept growing (append-during-checkpoint); +// 2. TraeX must load the checkpoint it persisted (no codex: prefix hardcode); +// 3. a stale checkpoint (crash after full replacement commit, before its +// checkpoint upsert) must never seed a resume against a newer DB prefix; +// 4. a cold restart (new engine, fresh cursor cache) must resume from the +// persisted checkpoint with full parity; +// 5. the checkpoint-bypassing audit must repair same-stat in-place rewrites. + +func TestCodexCheckpointHashStateBoundedToCommittedOffset(t *testing.T) { + const uuid = "019eb791-cf7d-75c1-8439-9ed74c122c99" + root := t.TempDir() + day := filepath.Join(root, "2024", "01", "01") + require.NoError(t, os.MkdirAll(day, 0o755)) + path := filepath.Join(day, "rollout-2024-01-01T10-00-00-"+uuid+".jsonl") + initial := testjsonl.JoinJSONL( + testjsonl.CodexSessionMetaJSON( + uuid, "/workspace/project", "codex_cli_rs", + "2024-01-01T10:00:00Z", + ), + testjsonl.CodexMsgJSON("user", "run command", "2024-01-01T10:00:01Z"), + testjsonl.CodexFunctionCallWithCallIDJSON( + "exec_command", "call_race", nil, "2024-01-01T10:00:02Z", + ), + ) + require.NoError(t, os.WriteFile(path, []byte(initial), 0o644)) + + database := openTestDB(t) + engine := NewEngine(database, EngineConfig{ + AgentDirs: map[parser.AgentType][]string{ + parser.AgentCodex: {root}, + }, + Machine: "local", + }) + t.Cleanup(engine.Close) + require.Equal(t, 1, engine.SyncAll(context.Background(), nil).Synced) + + before, ok, err := database.GetParserCheckpoint("codex:" + uuid) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, int64(len(initial)), before.Offset) + + // Appending after the atomic content/checkpoint commit leaves the stored + // checkpoint anchored to the original committed prefix. The next sync + // resumes from that state and advances both content and checkpoint in one + // transaction. + tail := testjsonl.JoinJSONL(testjsonl.CodexFunctionCallOutputJSON( + "call_race", "done", "2024-01-01T10:00:03Z", + )) + f, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0o644) + require.NoError(t, err) + _, err = f.WriteString(tail) + require.NoError(t, err) + require.NoError(t, f.Close()) + + afterAppend, ok, err := database.GetParserCheckpoint("codex:" + uuid) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, int64(len(initial)), afterAppend.Offset) + + beforeBlobs, ok, err := database.GetParserCheckpointBlobs("codex:" + uuid) + require.NoError(t, err) + require.True(t, ok) + info, err := os.Stat(path) + require.NoError(t, err) + _, resumedHash, err := codexResumeHash( + path, afterAppend.Offset, info.Size(), beforeBlobs.HashState, + ) + require.NoError(t, err) + actualHash, err := ComputeFileHash(path) + require.NoError(t, err) + require.Equal(t, actualHash, resumedHash, + "resuming the persisted state must reproduce the real source hash") + + require.Equal(t, 1, engine.SyncAll(context.Background(), nil).Synced) + stored, err := database.GetSessionFull( + context.Background(), "codex:"+uuid, + ) + require.NoError(t, err) + require.NotNil(t, stored) + require.NotNil(t, stored.FileHash) + require.Equal(t, actualHash, *stored.FileHash, + "a normal append resume must persist the real source hash") +} + +func TestBuildCodexFullParseCheckpointUsesParseSnapshotIdentity(t *testing.T) { + database := openTestDB(t) + engine := NewEngine(database, EngineConfig{Machine: "local"}) + t.Cleanup(engine.Close) + + path := filepath.Join(t.TempDir(), "rollout.jsonl") + require.NoError(t, os.WriteFile(path, []byte("session\n"), 0o600)) + + h := sha256.New() + h.Write([]byte("parsed prefix")) + hashState, err := h.(encoding.BinaryMarshaler).MarshalBinary() + require.NoError(t, err) + + pw := pendingWrite{ + sess: parser.ParsedSession{ + ID: "codex:snapshot", + Agent: parser.AgentCodex, + File: parser.FileInfo{ + Path: path, + Size: 8, + Mtime: 111, + Inode: 222, + Device: 333, + ChangeTime: 444, + }, + MessageCount: 2, + }, + checkpoint: []byte("cursor"), + checkpointHashState: hashState, + checkpointAnchorDigest: "anchor", + } + cp, blobs, err := engine.buildCodexFullParseCheckpoint(path, pw) + require.NoError(t, err) + require.NotNil(t, cp) + require.NotNil(t, blobs) + require.Equal(t, uint64(222), cp.FileInode, + "identity must come from the parse snapshot, not a later stat") + require.Equal(t, uint64(333), cp.FileDevice) + require.Equal(t, int64(111), cp.FileMTime) + require.Equal(t, int64(444), cp.FileChangeTime) + require.Equal(t, int64(8), cp.Offset) +} + +func TestCodexCheckpointTraeXLoadsPersistedCheckpoint(t *testing.T) { + const uuid = "019eb791-cf7d-75c1-8439-9ed74c122d99" + root := t.TempDir() + day := filepath.Join(root, "2024", "01", "01") + require.NoError(t, os.MkdirAll(day, 0o755)) + path := filepath.Join(day, "rollout-2024-01-01T10-00-00-"+uuid+".jsonl") + content := testjsonl.JoinJSONL( + testjsonl.CodexSessionMetaJSON( + uuid, "/workspace/project", "codex_cli_rs", + "2024-01-01T10:00:00Z", + ), + testjsonl.CodexMsgJSON("user", "hello", "2024-01-01T10:00:01Z"), + ) + require.NoError(t, os.WriteFile(path, []byte(content), 0o644)) + + database := openTestDB(t) + engine := NewEngine(database, EngineConfig{ + AgentDirs: map[parser.AgentType][]string{ + parser.AgentTraeX: {root}, + }, + Machine: "local", + }) + t.Cleanup(engine.Close) + require.Equal(t, 1, engine.SyncAll(context.Background(), nil).Synced) + _, ok, err := database.GetParserCheckpoint("traex:" + uuid) + require.NoError(t, err) + require.True(t, ok, "full TraeX parse persists a traex checkpoint") + + provider, ok := parser.NewProvider(parser.AgentTraeX, parser.ProviderConfig{ + Roots: []string{root}, Machine: "local", + }) + require.True(t, ok) + sources, err := provider.Discover(context.Background()) + require.NoError(t, err) + require.Len(t, sources, 1) + result, err := engine.codexCheckpointFingerprint( + context.Background(), sources[0], parser.DiscoveredFile{ + Agent: parser.AgentTraeX, + Path: path, + ProviderSource: &sources[0], + }, + ) + require.NoError(t, err) + require.Equal(t, codexCheckpointUnchanged, result.decision, + "a cold TraeX worker should reuse the checkpoint it persisted") +} + +func TestCodexCheckpointStaleCannotResumeFromNewerDBOffset(t *testing.T) { + const uuid = "019eb791-cf7d-75c1-8439-9ed74c122e99" + root := t.TempDir() + day := filepath.Join(root, "2024", "01", "01") + require.NoError(t, os.MkdirAll(day, 0o755)) + path := filepath.Join(day, "rollout-2024-01-01T10-00-00-"+uuid+".jsonl") + initial := testjsonl.JoinJSONL( + testjsonl.CodexSessionMetaJSON( + uuid, "/workspace/project", "codex_cli_rs", + "2024-01-01T10:00:00Z", + ), + testjsonl.CodexMsgJSON("user", "hello", "2024-01-01T10:00:01Z"), + ) + require.NoError(t, os.WriteFile(path, []byte(initial), 0o644)) + + database := openTestDB(t) + engine := NewEngine(database, EngineConfig{ + AgentDirs: map[parser.AgentType][]string{ + parser.AgentCodex: {root}, + }, + Machine: "local", + }) + t.Cleanup(engine.Close) + require.Equal(t, 1, engine.SyncAll(context.Background(), nil).Synced) + oldCheckpoint, ok, err := database.GetParserCheckpoint("codex:" + uuid) + require.NoError(t, err) + require.True(t, ok) + oldBlobs, ok, err := database.GetParserCheckpointBlobs("codex:" + uuid) + require.NoError(t, err) + require.True(t, ok) + + appendLine := func(line string) { + t.Helper() + f, openErr := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0o644) + require.NoError(t, openErr) + _, writeErr := f.WriteString(testjsonl.JoinJSONL(line)) + require.NoError(t, writeErr) + require.NoError(t, f.Close()) + } + appendLine(testjsonl.CodexTurnContextJSON( + "gpt-5.5", "2024-01-01T10:00:02Z", + )) + require.Equal(t, 1, engine.SyncAll(context.Background(), nil).Synced) + engine.Close() + engine = NewEngine(database, EngineConfig{ + AgentDirs: map[parser.AgentType][]string{ + parser.AgentCodex: {root}, + }, + Machine: "local", + }) + t.Cleanup(engine.Close) + + // Recreate the state left by a crash after a full replacement commits its + // newer file_size/next_ordinal but before its out-of-transaction checkpoint + // upsert: the DB prefix is newer than the surviving checkpoint seed. + require.NoError(t, database.UpsertParserCheckpoint(*oldCheckpoint, oldBlobs)) + appendLine(testjsonl.CodexMsgJSON( + "assistant", "new reply", "2024-01-01T10:00:03Z", + )) + require.Equal(t, 1, engine.SyncAll(context.Background(), nil).Synced) + + messages, err := database.GetAllMessages( + context.Background(), "codex:"+uuid, + ) + require.NoError(t, err) + require.Len(t, messages, 2) + require.Equal(t, "gpt-5.5", messages[1].Model, + "resume seed must describe the same prefix as the DB byte offset") +} + +func TestCodexCheckpointColdRestartResumeParity(t *testing.T) { + const uuid = "019eb791-cf7d-75c1-8439-9ed74c122f99" + root := t.TempDir() + day := filepath.Join(root, "2024", "01", "01") + require.NoError(t, os.MkdirAll(day, 0o755)) + path := filepath.Join(day, "rollout-2024-01-01T10-00-00-"+uuid+".jsonl") + initial := testjsonl.JoinJSONL( + testjsonl.CodexSessionMetaJSON( + uuid, "/workspace/project", "codex_cli_rs", + "2024-01-01T10:00:00Z", + ), + testjsonl.CodexMsgJSON("user", "hello", "2024-01-01T10:00:01Z"), + testjsonl.CodexFunctionCallWithCallIDJSON( + "exec_command", "call_restart", nil, "2024-01-01T10:00:02Z", + ), + ) + require.NoError(t, os.WriteFile(path, []byte(initial), 0o644)) + + database := openTestDB(t) + engine := NewEngine(database, EngineConfig{ + AgentDirs: map[parser.AgentType][]string{ + parser.AgentCodex: {root}, + }, + Machine: "local", + }) + require.Equal(t, 1, engine.SyncAll(context.Background(), nil).Synced) + engine.Close() + + // Cold restart: a fresh engine has an empty cursor cache and must resume + // entirely from the persisted checkpoint. + engine = NewEngine(database, EngineConfig{ + AgentDirs: map[parser.AgentType][]string{ + parser.AgentCodex: {root}, + }, + Machine: "local", + }) + t.Cleanup(engine.Close) + appended := testjsonl.JoinJSONL(testjsonl.CodexFunctionCallOutputJSON( + "call_restart", "done", "2024-01-01T10:00:03Z", + )) + f, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0o644) + require.NoError(t, err) + _, err = f.WriteString(appended) + require.NoError(t, err) + require.NoError(t, f.Close()) + + require.Equal(t, 1, engine.SyncAll(context.Background(), nil).Synced) + sess, err := database.GetSessionFull(context.Background(), "codex:"+uuid) + require.NoError(t, err) + require.NotNil(t, sess) + require.True(t, sess.LastWriteIncremental, + "a cold restart must resume incrementally from the checkpoint") + msgs, err := database.GetAllMessages(context.Background(), "codex:"+uuid) + require.NoError(t, err) + require.Len(t, msgs, 2) + require.Len(t, msgs[1].ToolCalls, 1) + require.Equal(t, "done", msgs[1].ToolCalls[0].ResultContent) +} + +func TestCodexCheckpointAuditRepairsSameStatRewrite(t *testing.T) { + const uuid = "019eb791-cf7d-75c1-8439-9ed74c122a99" + root := t.TempDir() + day := filepath.Join(root, "2024", "01", "01") + require.NoError(t, os.MkdirAll(day, 0o755)) + path := filepath.Join(day, "rollout-2024-01-01T10-00-00-"+uuid+".jsonl") + original := testjsonl.JoinJSONL( + testjsonl.CodexSessionMetaJSON( + uuid, "/workspace/project", "codex_cli_rs", + "2024-01-01T10:00:00Z", + ), + testjsonl.CodexMsgJSON("user", "alpha request", "2024-01-01T10:00:01Z"), + ) + rewritten := testjsonl.JoinJSONL( + testjsonl.CodexSessionMetaJSON( + uuid, "/workspace/project", "codex_cli_rs", + "2024-01-01T10:00:00Z", + ), + testjsonl.CodexMsgJSON("user", "bravo request", "2024-01-01T10:00:01Z"), + ) + require.Len(t, rewritten, len(original)) + require.NoError(t, os.WriteFile(path, []byte(original), 0o644)) + + database := openTestDB(t) + engine := NewEngine(database, EngineConfig{ + AgentDirs: map[parser.AgentType][]string{ + parser.AgentCodex: {root}, + }, + Machine: "local", + }) + t.Cleanup(engine.Close) + require.Equal(t, 1, engine.SyncAll(context.Background(), nil).Synced) + before, err := os.Stat(path) + require.NoError(t, err) + require.NoError(t, os.WriteFile(path, []byte(rewritten), 0o644)) + require.NoError(t, os.Chtimes(path, before.ModTime(), before.ModTime())) + + engine.SetCheckpointAudit(true) + stats, tombstoned, err := engine.ReconcileWatchRootsWithStats( + context.Background(), []string{root}, false, nil, + ) + require.NoError(t, err) + require.Zero(t, tombstoned) + require.Equal(t, 1, stats.Synced, + "the audit must detect and repair the same-stat rewrite") + engine.SetCheckpointAudit(false) + + msgs, err := database.GetAllMessages(context.Background(), "codex:"+uuid) + require.NoError(t, err) + require.Len(t, msgs, 1) + require.Equal(t, "bravo request", msgs[0].Content) +} + +func TestCodexIncrementalDuplicateCallIDTargetsExactOccurrence(t *testing.T) { + const ( + uuid = "019eb791-cf7d-75c1-8439-9ed74c122daa" + callID = "reused-call" + ) + root := t.TempDir() + day := filepath.Join(root, "2024", "01", "01") + require.NoError(t, os.MkdirAll(day, 0o755)) + path := filepath.Join( + day, "rollout-2024-01-01T10-00-00-"+uuid+".jsonl", + ) + initial := testjsonl.JoinJSONL( + testjsonl.CodexSessionMetaJSON( + uuid, "/workspace/project", "codex_cli_rs", + "2024-01-01T10:00:00Z", + ), + testjsonl.CodexTurnContextJSON( + "gpt-5.4", "2024-01-01T10:00:01Z", + ), + testjsonl.CodexMsgJSON( + "user", "run twice", "2024-01-01T10:00:02Z", + ), + testjsonl.CodexFunctionCallWithCallIDJSON( + "exec_command", callID, nil, "2024-01-01T10:00:03Z", + ), + testjsonl.CodexFunctionCallOutputJSON( + callID, "first result", "2024-01-01T10:00:04Z", + ), + testjsonl.CodexTokenCountJSON( + "2024-01-01T10:00:05Z", 100, 10, 80, + ), + testjsonl.CodexFunctionCallWithCallIDJSON( + "exec_command", callID, nil, "2024-01-01T10:00:06Z", + ), + ) + require.NoError(t, os.WriteFile(path, []byte(initial), 0o644)) + + database := openTestDB(t) + engine := NewEngine(database, EngineConfig{ + AgentDirs: map[parser.AgentType][]string{ + parser.AgentCodex: {root}, + }, + Machine: "local", + }) + t.Cleanup(engine.Close) + require.Equal(t, 1, engine.SyncAll(t.Context(), nil).Synced) + + tail := testjsonl.JoinJSONL( + testjsonl.CodexFunctionCallOutputJSON( + callID, "second result", "2024-01-01T10:00:07Z", + ), + testjsonl.CodexTokenCountJSON( + "2024-01-01T10:00:08Z", 200, 20, 160, + ), + ) + f, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0o644) + require.NoError(t, err) + _, err = f.WriteString(tail) + require.NoError(t, err) + require.NoError(t, f.Close()) + + stats := engine.SyncAll(t.Context(), nil) + require.Zero(t, stats.Failed) + require.Equal(t, 1, stats.Synced) + + sess, err := database.GetSessionFull(t.Context(), "codex:"+uuid) + require.NoError(t, err) + require.NotNil(t, sess) + require.True(t, sess.LastWriteIncremental) + msgs, err := database.GetAllMessages(t.Context(), "codex:"+uuid) + require.NoError(t, err) + require.Len(t, msgs, 3) + require.Len(t, msgs[1].ToolCalls, 1) + require.Len(t, msgs[2].ToolCalls, 1) + require.Equal(t, "first result", msgs[1].ToolCalls[0].ResultContent) + require.Equal(t, "second result", msgs[2].ToolCalls[0].ResultContent) + require.NotEmpty(t, msgs[2].TokenUsage) +} diff --git a/internal/sync/checkpoint_test.go b/internal/sync/checkpoint_test.go new file mode 100644 index 000000000..855203d72 --- /dev/null +++ b/internal/sync/checkpoint_test.go @@ -0,0 +1,229 @@ +package sync_test + +import ( + "bytes" + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/testjsonl" +) + +const checkpointTestUUID = "019eb791-cf7d-75c1-8439-9ed74c122c01" + +func writeCheckpointCodexSession( + t *testing.T, env *testEnv, content string, +) string { + t.Helper() + return env.writeCodexSession( + t, + filepath.Join("2024", "01", "01"), + "rollout-2024-01-01T10-00-00-"+checkpointTestUUID+".jsonl", + content, + ) +} + +func checkpointCodexInitial() string { + return testjsonl.JoinJSONL( + testjsonl.CodexSessionMetaJSON( + checkpointTestUUID, "/tmp/proj", "codex_cli_rs", + "2024-01-01T10:00:00Z", + ), + testjsonl.CodexMsgJSON( + "user", "run command", "2024-01-01T10:00:01Z", + ), + testjsonl.CodexFunctionCallWithCallIDJSON( + "exec_command", "call_cp", nil, "2024-01-01T10:00:02Z", + ), + ) +} + +func TestCodexCheckpointFullParsePersistsCheckpoint(t *testing.T) { + env := setupTestEnv(t) + initial := checkpointCodexInitial() + writeCheckpointCodexSession(t, env, initial) + + env.engine.SyncAll(context.Background(), nil) + + cp, ok, err := env.db.GetParserCheckpoint("codex:" + checkpointTestUUID) + require.NoError(t, err) + require.True(t, ok, "a full Codex parse must persist a checkpoint") + blobs, ok, err := env.db.GetParserCheckpointBlobs("codex:" + checkpointTestUUID) + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, int64(len(initial)), cp.Offset) + assert.NotEmpty(t, blobs.Cursor) + assert.NotEmpty(t, blobs.HashState) + assert.NotEmpty(t, cp.Hash) + assert.NotEmpty(t, cp.TailAnchorDigest) + assert.Equal(t, 2, cp.NextOrdinal) + assert.Equal(t, db.ParserCheckpointVersion, cp.Version) +} + +func TestCodexCheckpointIncrementalResumeAdvancesCheckpoint(t *testing.T) { + env := setupTestEnv(t) + ctx := context.Background() + initial := checkpointCodexInitial() + path := writeCheckpointCodexSession(t, env, initial) + env.engine.SyncAll(ctx, nil) + before, ok, err := env.db.GetParserCheckpoint("codex:" + checkpointTestUUID) + require.NoError(t, err) + require.True(t, ok) + beforeBlobs, ok, err := env.db.GetParserCheckpointBlobs("codex:" + checkpointTestUUID) + require.NoError(t, err) + require.True(t, ok) + + appended := testjsonl.JoinJSONL( + testjsonl.CodexFunctionCallOutputJSON( + "call_cp", "done", "2024-01-01T10:00:03Z", + ), + testjsonl.CodexTokenCountJSON( + "2024-01-01T10:00:04Z", 100_000, 250, 64_000, + ), + ) + f, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0o644) + require.NoError(t, err) + _, err = f.WriteString(appended) + require.NoError(t, err) + require.NoError(t, f.Close()) + + stats := env.engine.SyncAll(ctx, nil) + require.Equal(t, 1, stats.Synced) + + sess, err := env.db.GetSessionFull(ctx, "codex:"+checkpointTestUUID) + require.NoError(t, err) + require.NotNil(t, sess) + assert.True(t, sess.LastWriteIncremental, + "a checkpoint-resumed append must stay incremental") + + msgs := fetchMessages(t, env.db, "codex:"+checkpointTestUUID) + require.Len(t, msgs, 2) + require.Len(t, msgs[1].ToolCalls, 1) + assert.Equal(t, "done", msgs[1].ToolCalls[0].ResultContent) + require.Len(t, msgs[1].ToolCalls[0].ResultEvents, 1) + assert.Equal(t, 100_000, msgs[1].ContextTokens) + assert.Equal(t, 250, msgs[1].OutputTokens) + assert.NotEmpty(t, msgs[1].TokenUsage, + "the token_count following a late result must update the committed assistant") + + after, ok, err := env.db.GetParserCheckpoint("codex:" + checkpointTestUUID) + require.NoError(t, err) + require.True(t, ok) + afterBlobs, ok, err := env.db.GetParserCheckpointBlobs("codex:" + checkpointTestUUID) + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, before.Offset+int64(len(appended)), after.Offset) + assert.NotEqual(t, beforeBlobs.Cursor, afterBlobs.Cursor) + assert.NotEqual(t, beforeBlobs.HashState, afterBlobs.HashState) + assert.NotEqual(t, before.Hash, after.Hash, + "the checkpointed hash must advance to the new full-file hash") + require.NotNil(t, sess.FileHash) + assert.Equal(t, after.Hash, *sess.FileHash, + "stored hash must equal the checkpointed full-file hash") +} + +func TestCodexCheckpointTruncationFallsBackToFullParse(t *testing.T) { + env := setupTestEnv(t) + ctx := context.Background() + initial := checkpointCodexInitial() + path := writeCheckpointCodexSession(t, env, initial) + env.engine.SyncAll(ctx, nil) + _, ok, err := env.db.GetParserCheckpoint("codex:" + checkpointTestUUID) + require.NoError(t, err) + require.True(t, ok) + + // Truncate at a safe line boundary (drop the final function_call line, + // keeping the newline-terminated user message) so the rebuilt checkpoint + // is a legal resume offset. + truncated := int64(strings.LastIndex(initial, "\n") + 1) + require.NoError(t, os.Truncate(path, truncated)) + + stats := env.engine.SyncAll(ctx, nil) + require.Equal(t, 1, stats.Synced, + "a truncated transcript must be authoritatively reparsed") + cp, ok, err := env.db.GetParserCheckpoint("codex:" + checkpointTestUUID) + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, truncated, cp.Offset, + "the checkpoint must be rebuilt for the truncated file") +} + +func TestCodexCheckpointAnchorMismatchFallsBackToFullParse(t *testing.T) { + env := setupTestEnv(t) + ctx := context.Background() + initial := checkpointCodexInitial() + path := writeCheckpointCodexSession(t, env, initial) + env.engine.SyncAll(ctx, nil) + info, err := os.Stat(path) + require.NoError(t, err) + origMtime := info.ModTime() + + // Corrupt one byte inside the anchor region while keeping the JSON valid + // (call_cp -> call_cX), then append a real message so the size grows and + // EOF stays a safe boundary. Restoring the mtime isolates the anchor + // check: stat and identity look safe, the anchor must not. + raw, err := os.ReadFile(path) + require.NoError(t, err) + idx := bytes.Index(raw, []byte(`"call_cp"`)) + require.Positive(t, idx) + raw[idx+7] = 'X' + raw = append(raw, []byte(testjsonl.JoinJSONL(testjsonl.CodexMsgJSON( + "user", "after corruption", "2024-01-01T10:00:04Z", + )))...) + require.NoError(t, os.WriteFile(path, raw, 0o644)) + require.NoError(t, os.Chtimes(path, time.Now(), origMtime)) + + env.engine.SyncPaths([]string{path}) + cp, ok, err := env.db.GetParserCheckpoint("codex:" + checkpointTestUUID) + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, int64(len(raw)), cp.Offset) + msgs := fetchMessages(t, env.db, "codex:"+checkpointTestUUID) + require.Len(t, msgs, 3) + require.Len(t, msgs[1].ToolCalls, 1) + assert.Equal(t, "call_cX", msgs[1].ToolCalls[0].ToolUseID, + "the corrupted call id must come from the authoritative full parse") +} + +func TestCodexCheckpointInPlaceRewriteSameSizeSameMtimeIsRejected(t *testing.T) { + env := setupTestEnv(t) + ctx := context.Background() + initial := checkpointCodexInitial() + path := writeCheckpointCodexSession(t, env, initial) + env.engine.SyncAll(ctx, nil) + info, err := os.Stat(path) + require.NoError(t, err) + origMtime := info.ModTime() + + // In-place rewrite with restored size and mtime: the stored + // change-time no longer matches, so the checkpoint no-op path must + // decline and the engine must re-parse the rewritten bytes instead of + // trusting the stale checkpoint. + raw, err := os.ReadFile(path) + require.NoError(t, err) + // Flip the final character of the user message's text so the JSON + // stays valid and the re-parse must produce different content. + textEnd := bytes.Index(raw, []byte("run command")) + len("run command") + require.Greater(t, textEnd, len("run command")) + raw[textEnd-1] ^= 0x01 + require.NoError(t, os.WriteFile(path, raw, 0o644)) + require.NoError(t, os.Chtimes(path, time.Now(), origMtime)) + + env.engine.SyncPaths([]string{path}) + + msgs := fetchMessages(t, env.db, "codex:"+checkpointTestUUID) + require.Len(t, msgs, 2) + assert.NotEqual(t, "run command", msgs[0].Content, + "a same-stat rewrite must be re-parsed, not trusted") + cp, ok, err := env.db.GetParserCheckpoint("codex:" + checkpointTestUUID) + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, int64(len(initial)), cp.Offset, + "the rewritten bytes are the same length, so the checkpoint offset stays") +} diff --git a/internal/sync/codex_bench_test.go b/internal/sync/codex_bench_test.go index 8789a81ee..6e9161476 100644 --- a/internal/sync/codex_bench_test.go +++ b/internal/sync/codex_bench_test.go @@ -2,14 +2,17 @@ package sync import ( "context" + "fmt" "os" "path/filepath" "strconv" "strings" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/db" "go.kenn.io/agentsview/internal/parser" "go.kenn.io/agentsview/internal/testjsonl" ) @@ -26,13 +29,42 @@ var ( codexSyncBenchmarkHashSink string ) -// BenchmarkCodexIncrementalSyncReads measures the source-reading pipeline that -// remains around a warm cursor append: Fingerprint hashes the full source and -// ComputeFileHashPrefix hashes through the proposed committed offset. -func BenchmarkCodexIncrementalSyncReads(b *testing.B) { - b.StopTimer() +// BenchmarkCodexCheckpointAppendResume measures the source-reading pipeline a +// checkpoint-resumed Codex append performs: the checkpoint gate (stat + +// anchor digest + fingerprint resume), the seeded tail parse, the committed +// prefix resume hash, the next anchor digest, and the next checkpoint +// assembly. It replaces the pre-checkpoint benchmark that timed the old +// full-source fingerprint plus prefix re-hash pipeline. +func BenchmarkCodexCheckpointAppendResume(b *testing.B) { + silenceBenchLogs(b) ctx := context.Background() root, path, prefix, tail, startOrdinal := writeCodexSyncBenchmarkTranscript(b) + + database, err := db.Open(filepath.Join(b.TempDir(), "bench.db")) + require.NoError(b, err) + engine := NewEngine(database, EngineConfig{ + AgentDirs: map[parser.AgentType][]string{ + parser.AgentCodex: {root}, + }, + Machine: "benchmark-host", + }) + b.Cleanup(func() { + engine.Close() + if err := database.Close(); err != nil { + b.Errorf("close bench db: %v", err) + } + }) + first := engine.SyncAll(ctx, nil) + require.Equal(b, 1, first.Synced) + require.Zero(b, first.Failed) + sessionID := "codex:" + codexSyncBenchmarkUUID + cp, ok, err := database.GetParserCheckpoint(sessionID) + require.NoError(b, err) + require.True(b, ok, "the full sync must persist a checkpoint") + blobs, ok, err := database.GetParserCheckpointBlobs(sessionID) + require.NoError(b, err) + require.True(b, ok) + cfg := parser.ProviderConfig{ Roots: []string{root}, Machine: "benchmark-host", @@ -40,113 +72,118 @@ func BenchmarkCodexIncrementalSyncReads(b *testing.B) { provider, ok := parser.NewProvider(parser.AgentCodex, cfg) require.True(b, ok) source, found, err := provider.FindSource(ctx, parser.FindSourceRequest{ - FullSessionID: "codex:" + codexSyncBenchmarkUUID, + FullSessionID: sessionID, }) require.NoError(b, err) require.True(b, found) - prefixFingerprint, err := provider.Fingerprint(ctx, source) - require.NoError(b, err) - assert.Equal(b, int64(len(prefix)), prefixFingerprint.Size) - full, err := provider.Parse(ctx, parser.ParseRequest{ - Source: source, - Fingerprint: prefixFingerprint, - }) - require.NoError(b, err) - require.Len(b, full.Results, 1) - assert.Len(b, full.Results[0].Result.Messages, startOrdinal) - - // Keep the timed provider untouched after its prefix-only full parse. A - // separately seeded provider handles output validation after the append. - validationProvider, ok := parser.NewProvider(parser.AgentCodex, cfg) - require.True(b, ok) - _, err = validationProvider.Parse(ctx, parser.ParseRequest{ - Source: source, - Fingerprint: prefixFingerprint, - }) - require.NoError(b, err) - appendCodexSyncBenchmarkTail(b, path, tail) - req := parser.IncrementalRequest{ - Source: source, - SessionID: "codex:" + codexSyncBenchmarkUUID, - Offset: int64(len(prefix)), - StartOrdinal: startOrdinal, - } - currentFingerprint, outcome, status, committedHash, err := - runCodexIncrementalSyncReads(ctx, validationProvider, source, path, req) - require.NoError(b, err) - requireCodexSyncBenchmarkOutcome( - b, outcome, status, startOrdinal, int64(len(tail)), + // Warm the timed pipeline once so the per-op loop measures the + // checkpoint resume work itself. + _, err = runCodexCheckpointAppendReads( + ctx, engine, provider, source, path, cp, blobs, + startOrdinal, len(prefix), len(tail), ) - require.Equal(b, int64(len(prefix)+len(tail)), currentFingerprint.Size) - require.Equal(b, currentFingerprint.Size, req.Offset+outcome.ConsumedBytes) - require.NotEmpty(b, currentFingerprint.Hash) - require.Len(b, committedHash, 64) - require.Equal(b, currentFingerprint.Hash, committedHash) + require.NoError(b, err) - // Two full-length linear reads dominate this pipeline: the provider source - // fingerprint and the engine's committed-prefix hash. The warm tail parse is - // intentionally left in the same measurement between them. - b.SetBytes(2 * int64(len(prefix)+len(tail))) + // Per append the source reads are bounded: the anchor window (twice) + // plus the tail (fingerprint resume, parse, committed-prefix resume). + b.SetBytes(int64(len(tail)) + 2*codexCheckpointAnchorSize) b.ReportAllocs() b.ResetTimer() b.StartTimer() for b.Loop() { - fingerprint, outcome, status, hash, err := runCodexIncrementalSyncReads( - ctx, provider, source, path, req, + outcome, err := runCodexCheckpointAppendReads( + ctx, engine, provider, source, path, cp, blobs, + startOrdinal, len(prefix), len(tail), ) - if err != nil || !codexSyncBenchmarkOutcomeValid( - outcome, status, startOrdinal, int64(len(tail)), - ) || fingerprint.Size != int64(len(prefix)+len(tail)) || - fingerprint.Hash == "" || len(hash) != 64 || fingerprint.Hash != hash { + if err != nil { b.StopTimer() require.NoError(b, err) - requireCodexSyncBenchmarkOutcome( - b, outcome, status, startOrdinal, int64(len(tail)), - ) - assert.Equal(b, int64(len(prefix)+len(tail)), fingerprint.Size) - require.NotEmpty(b, fingerprint.Hash) - assert.Len(b, hash, 64) - assert.Equal(b, fingerprint.Hash, hash) b.StartTimer() } codexSyncBenchmarkOutcomeSink = outcome - codexSyncBenchmarkHashSink = hash } } -func runCodexIncrementalSyncReads( +func runCodexCheckpointAppendReads( ctx context.Context, + engine *Engine, provider parser.Provider, source parser.SourceRef, path string, - req parser.IncrementalRequest, -) ( - parser.SourceFingerprint, - parser.IncrementalOutcome, - parser.IncrementalStatus, - string, - error, -) { - // Fingerprint retains the existing full-source content hash. - fingerprint, err := provider.Fingerprint(ctx, source) + cp *db.ParserCheckpoint, + blobs db.ParserCheckpointBlobs, + startOrdinal int, + prefixLen, tailLen int, +) (parser.IncrementalOutcome, error) { + file := parser.DiscoveredFile{ + Path: path, + Agent: parser.AgentCodex, + } + res, err := engine.codexCheckpointFingerprint(ctx, source, file) if err != nil { - return parser.SourceFingerprint{}, parser.IncrementalOutcome{}, - parser.IncrementalUnsupported, "", err + return parser.IncrementalOutcome{}, err + } + if res.decision != codexCheckpointAppend { + return parser.IncrementalOutcome{}, fmt.Errorf( + "checkpoint gate decided %d, want append", res.decision, + ) } - req.Fingerprint = fingerprint - outcome, status, err := provider.ParseIncremental(ctx, req) + outcome, status, err := provider.ParseIncremental(ctx, parser.IncrementalRequest{ + Source: source, + Fingerprint: res.fingerprint, + SessionID: "codex:" + codexSyncBenchmarkUUID, + Offset: cp.Offset, + StartOrdinal: startOrdinal, + Seed: blobs.Cursor, + }) if err != nil { - return fingerprint, outcome, status, "", err + return outcome, err } - // This is the engine's second remaining linear read, through the offset - // that would be committed after the incremental database write succeeds. - committedHash, err := ComputeFileHashPrefix( - path, req.Offset+outcome.ConsumedBytes, + if status != parser.IncrementalApplied { + return outcome, fmt.Errorf("incremental status %v, want applied", status) + } + if outcome.ConsumedBytes != int64(tailLen) { + return outcome, fmt.Errorf( + "consumed %d, want %d", outcome.ConsumedBytes, tailLen, + ) + } + // The engine's remaining checkpoint work for the append. + state, hash, err := codexResumeHash(path, cp.Offset, cp.Offset+outcome.ConsumedBytes, blobs.HashState) + if err != nil { + return outcome, err + } + anchorDigest, err := codexCheckpointAnchorDigest( + path, cp.Offset+outcome.ConsumedBytes, + ) + if err != nil { + return outcome, err + } + info, err := os.Stat(path) + if err != nil { + return outcome, err + } + inode, device := getFileIdentity(path, info) + changeTime, _ := fileChangeTime(path, info) + built, _ := buildCodexCheckpoint( + "codex:"+codexSyncBenchmarkUUID, + "codex", + path, + uint64(inode), + uint64(device), + info.ModTime().UnixNano(), + changeTime, + cp.Offset+outcome.ConsumedBytes, + outcome.NextCursor, + state, + hash, + startOrdinal+2, + anchorDigest, ) - return fingerprint, outcome, status, committedHash, err + codexSyncBenchmarkHashSink = built.Hash + return outcome, nil } func writeCodexSyncBenchmarkTranscript( @@ -224,44 +261,190 @@ func appendCodexSyncBenchmarkTail(b *testing.B, path, tail string) { require.NoError(b, f.Close()) } -func requireCodexSyncBenchmarkOutcome( - b *testing.B, - outcome parser.IncrementalOutcome, - status parser.IncrementalStatus, - startOrdinal int, - tailBytes int64, -) { - b.Helper() - require.Equal(b, parser.IncrementalApplied, status) - require.Len(b, outcome.Messages, 2) - assert.Equal(b, "codex:"+codexSyncBenchmarkUUID, outcome.SessionID) - assert.Equal(b, 2, outcome.MessageCount) - assert.Equal(b, 1, outcome.UserMessageCount) - assert.Equal(b, tailBytes, outcome.ConsumedBytes) - assert.Equal(b, parser.RoleUser, outcome.Messages[0].Role) - assert.Equal(b, codexSyncBenchmarkTailUser, outcome.Messages[0].Content) - assert.Equal(b, startOrdinal, outcome.Messages[0].Ordinal) - assert.Equal(b, parser.RoleAssistant, outcome.Messages[1].Role) - assert.Equal(b, codexSyncBenchmarkTailAgent, outcome.Messages[1].Content) - assert.Equal(b, startOrdinal+1, outcome.Messages[1].Ordinal) +const ( + codexLateToolBenchmarkUUID = "019eb791-cf7d-75c1-8439-9ed74c122b02" + codexLateToolBenchmarkTurns = 250 +) + +// BenchmarkCodexLateToolOutputDebouncedBurst measures absorbing a stream in +// which each batch appends a new function_call plus the function_call_output +// for the previous batch's call. Output records therefore always refer to a +// call committed in an earlier sync batch. +// +// This is a debounced-burst benchmark, not a single-quiet-append gate: the +// engine's signal scheduler is stretched to an hour so the O(history) +// full recompute runs only on the first iteration and the remaining +// iterations are amortized. It guards the late-result update path's +// per-append cost, but it deliberately does not measure the quiet-session +// first append (see BenchmarkCodexQuietAppendSignals* for that gate). The +// session grows by two records per iteration, so per-op cost is only +// comparable between runs with the same iteration count (the bench gate +// always runs with a fixed -benchtime=Nx). +func BenchmarkCodexLateToolOutputDebouncedBurst(b *testing.B) { + silenceBenchLogs(b) + ctx := context.Background() + root, path, _ := writeCodexLateToolBenchmarkTranscript(b) + + database, err := db.Open(filepath.Join(b.TempDir(), "bench.db")) + require.NoError(b, err) + engine := NewEngine(database, EngineConfig{ + AgentDirs: map[parser.AgentType][]string{ + parser.AgentCodex: {root}, + }, + Machine: "benchmark-host", + }) + b.Cleanup(func() { + engine.Close() + if err := database.Close(); err != nil { + b.Errorf("close bench db: %v", err) + } + }) + + first := engine.SyncAll(ctx, nil) + require.Equal(b, 1, first.Synced) + require.Zero(b, first.Failed) + + // Stretch the debounce window so the flush timer cannot fire inside + // the timed loop (same rationale as BenchmarkSyncPathsIncrementalAppend). + engine.signalSched.mu.Lock() + engine.signalSched.interval = time.Hour + engine.signalSched.quiet = time.Hour + engine.signalSched.mu.Unlock() + + f, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0o644) + require.NoError(b, err) + defer f.Close() + + // Pre-build the appended lines: constructing JSONL inside the timed loop + // would allocate and be gated as if it were sync work. Iteration i appends + // call_{i+1} and the output for call_i, so the output is always "late". + lines := make([]string, b.N) + for i := range lines { + lines[i] = testjsonl.JoinJSONL( + testjsonl.CodexFunctionCallWithCallIDJSON( + "exec_command", + codexLateToolBenchmarkCall(i+1), + nil, + codexLateToolBenchmarkTS(2*i+1), + ), + testjsonl.CodexFunctionCallOutputJSON( + codexLateToolBenchmarkCall(i), + "result "+strconv.Itoa(i), + codexLateToolBenchmarkTS(2*i+2), + ), + ) + } + + b.ReportAllocs() + b.ResetTimer() + for i := range b.N { + if _, err := f.WriteString(lines[i]); err != nil { + b.Fatalf("append: %v", err) + } + stats := engine.SyncAll(ctx, nil) + if stats.Failed != 0 { + b.Fatalf("sync failed for appended output: %+v", stats) + } + } + b.StopTimer() + + msgs, err := database.GetAllMessages(ctx, "codex:"+codexLateToolBenchmarkUUID) + require.NoError(b, err) + wantMsgs := 3 + 2*codexLateToolBenchmarkTurns + b.N + require.Len(b, msgs, wantMsgs, + "each appended call adds exactly one message row") + results := make(map[string]db.ToolCall, len(msgs)) + for i := range msgs { + for j := range msgs[i].ToolCalls { + results[msgs[i].ToolCalls[j].ToolUseID] = msgs[i].ToolCalls[j] + } + } + for i := range b.N { + call, ok := results[codexLateToolBenchmarkCall(i)] + require.True(b, ok, "call %d must be stored", i) + assert.Equal(b, "exec_command", call.ToolName) + assert.Equal(b, "result "+strconv.Itoa(i), call.ResultContent) + require.Len(b, call.ResultEvents, 1, + "each call must carry exactly its own output event") + assert.Equal(b, "result "+strconv.Itoa(i), call.ResultEvents[0].Content) + } + newest, ok := results[codexLateToolBenchmarkCall(b.N)] + require.True(b, ok, "the newest call must be stored") + assert.Empty(b, newest.ResultContent) + assert.Empty(b, newest.ResultEvents) } -func codexSyncBenchmarkOutcomeValid( - outcome parser.IncrementalOutcome, - status parser.IncrementalStatus, - startOrdinal int, - tailBytes int64, -) bool { - return status == parser.IncrementalApplied && - outcome.SessionID == "codex:"+codexSyncBenchmarkUUID && - outcome.MessageCount == 2 && - outcome.UserMessageCount == 1 && - outcome.ConsumedBytes == tailBytes && - len(outcome.Messages) == 2 && - outcome.Messages[0].Role == parser.RoleUser && - outcome.Messages[0].Content == codexSyncBenchmarkTailUser && - outcome.Messages[0].Ordinal == startOrdinal && - outcome.Messages[1].Role == parser.RoleAssistant && - outcome.Messages[1].Content == codexSyncBenchmarkTailAgent && - outcome.Messages[1].Ordinal == startOrdinal+1 +func codexLateToolBenchmarkCall(i int) string { + return "call_" + strconv.Itoa(i) +} + +func codexLateToolBenchmarkTS(i int) string { + return time.Date( + 2026, 7, 10, 7, 12, i, 0, time.UTC, + ).Format("2006-01-02T15:04:05Z") +} + +func writeCodexLateToolBenchmarkTranscript( + b testing.TB, +) (root, path, prefix string) { + b.Helper() + root = filepath.Join(b.TempDir(), "sessions") + path = filepath.Join( + root, + "2026", + "07", + "10", + "rollout-2026-07-10T07-12-15-"+codexLateToolBenchmarkUUID+".jsonl", + ) + require.NoError(b, os.MkdirAll(filepath.Dir(path), 0o755)) + + fixture := testjsonl.NewSessionBuilder(). + AddCodexMeta( + "2026-07-10T07:00:00Z", + codexLateToolBenchmarkUUID, + "/workspace/project-a", + "codex_cli_rs", + ). + AddRaw(testjsonl.CodexTurnContextJSON( + "gpt-5.4", "2026-07-10T07:00:01Z", + )). + AddCodexMessage( + "2026-07-10T07:00:02Z", + "user", + "Initial request: inspect the project and make a careful change.", + ). + AddCodexMessage( + "2026-07-10T07:00:03Z", + "assistant", + "Initial response: I will inspect the relevant code and tests.", + ) + contextPayload := strings.Repeat( + "Retain concrete code, test, and validation context. ", 8, + ) + for i := range codexLateToolBenchmarkTurns { + turn := strconv.Itoa(i) + fixture.AddCodexMessage( + "2026-07-10T07:01:00Z", + "user", + "Prior turn "+turn+" request: continue the implementation. "+ + contextPayload, + ) + fixture.AddCodexMessage( + "2026-07-10T07:01:01Z", + "assistant", + "Prior turn "+turn+" response: applied the next bounded change. "+ + contextPayload, + ) + } + // call_0 is committed in the prefix with no output; iteration 0 appends + // call_1 plus the late output for call_0. + fixture.AddRaw(testjsonl.CodexFunctionCallWithCallIDJSON( + "exec_command", + "call_0", + nil, + codexLateToolBenchmarkTS(0), + )) + prefix = fixture.String() + require.NoError(b, os.WriteFile(path, []byte(prefix), 0o644)) + return root, path, prefix } diff --git a/internal/sync/codex_macro_bench_test.go b/internal/sync/codex_macro_bench_test.go new file mode 100644 index 000000000..180be9e15 --- /dev/null +++ b/internal/sync/codex_macro_bench_test.go @@ -0,0 +1,870 @@ +//go:build macrobench + +package sync + +import ( + "context" + "io" + "os" + "path/filepath" + "runtime" + "runtime/metrics" + "strconv" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/parser" + "go.kenn.io/agentsview/internal/testjsonl" +) + +// Macro benchmarks for the 10MB-vs-1GB same-append ratio gate. They are +// excluded from the PR bench gate (build tag macrobench, and the gated +// packages run without it) because the 1GB fixture takes minutes per run. +// Procedure (documented in docs/internal/performance-gates.md): +// +// cd internal/sync +// go test -tags 'fts5,macrobench' -run '^$' -bench 'BenchmarkMacroCodexQuietAppend' \ +// -benchmem -count=6 -benchtime=5x +// +// The gate: the p95 sec/op of the 1GB run must be within 2x of the 10MB +// run's p95 for the same append shape. +func BenchmarkMacroCodexQuietAppend10MB(b *testing.B) { + benchCodexQuietAppendSignals(b, 8000) +} + +func BenchmarkMacroCodexQuietAppend1GB(b *testing.B) { + benchCodexQuietAppendSignals(b, 790000) +} + +// TestMacroCodexRealSession drives the real-session macro measurement on an +// isolated copy of a production-scale Codex transcript: cold full sync, +// one 794B late tool-output append, and a no-op resync, each logged with +// wall time and the process rchar delta. Paths come from environment +// variables so no private absolute path lives in the repository: +// +// MACRO_CODEX_SESSION=/path/to/rollout-*.jsonl \ +// MACRO_CODEX_LATE_OUTPUT=/path/to/late-output.jsonl \ +// MACRO_CODEX_TRUNCATE= \ +// /usr/bin/time -v go test -tags 'fts5,macrobench' \ +// -run TestMacroCodexRealSession -count=1 -v ./internal/sync/ +// +// The source is stream-copied into a temp directory (never fully read into +// memory, so the peak RSS reflects the engine, not the harness) and +// truncated at MACRO_CODEX_TRUNCATE — the byte offset of the late output's +// own line — so the live archive is never touched by the engine and the +// append is a genuine late-result update. +func TestMacroCodexRealSession(t *testing.T) { + src := os.Getenv("MACRO_CODEX_SESSION") + late := os.Getenv("MACRO_CODEX_LATE_OUTPUT") + if src == "" || late == "" || os.Getenv("MACRO_CODEX_TRUNCATE") == "" { + t.Skip("set MACRO_CODEX_SESSION, MACRO_CODEX_LATE_OUTPUT, and MACRO_CODEX_TRUNCATE") + } + lateBytes, err := os.ReadFile(late) + require.NoError(t, err) + trunc, err := strconv.ParseInt(os.Getenv("MACRO_CODEX_TRUNCATE"), 10, 64) + require.NoError(t, err) + + root := t.TempDir() + day := filepath.Join(root, "2026", "08", "09") + require.NoError(t, os.MkdirAll(day, 0o755)) + dst := filepath.Join(day, filepath.Base(src)) + if err := copyFilePrefix(src, dst, trunc); err != nil { + t.Fatalf("copying snapshot prefix: %v", err) + } + + database, err := db.Open(filepath.Join(t.TempDir(), "macro.db")) + require.NoError(t, err) + engine := NewEngine(database, EngineConfig{ + AgentDirs: map[parser.AgentType][]string{ + parser.AgentCodex: {root}, + }, + Machine: "macro-host", + }) + t.Cleanup(func() { + engine.Close() + _ = database.Close() + }) + + rchar := func() int64 { + t.Helper() + data, err := os.ReadFile("/proc/self/io") + require.NoError(t, err) + for _, line := range strings.Split(string(data), "\n") { + if strings.HasPrefix(line, "rchar: ") { + v, err := strconv.ParseInt( + strings.TrimSpace(strings.TrimPrefix(line, "rchar: ")), + 10, 64, + ) + require.NoError(t, err) + return v + } + } + t.Fatal("no rchar in /proc/self/io") + return 0 + } + + // 1. Cold full sync. + r0 := rchar() + start := time.Now() + stats := engine.SyncAll(context.Background(), nil) + fullDur := time.Since(start) + t.Logf( + "MACRO_FULL synced=%d skipped=%d dur=%s rchar=%d", + stats.Synced, stats.Skipped, fullDur, rchar()-r0, + ) + require.Equal(t, 1, stats.Synced) + + sessionID := func() string { + inc, ok := database.GetSessionForIncremental( + dst, string(parser.AgentCodex), + ) + require.True(t, ok, "the snapshot session must be incremental-tracked") + return inc.ID + }() + + // 2. Append the late tool output. + f, err := os.OpenFile(dst, os.O_APPEND|os.O_WRONLY, 0o644) + require.NoError(t, err) + _, err = f.Write(lateBytes) + require.NoError(t, err) + require.NoError(t, f.Close()) + r0 = rchar() + start = time.Now() + stats = engine.SyncAll(context.Background(), nil) + incDur := time.Since(start) + t.Logf( + "MACRO_INC synced=%d skipped=%d dur=%s rchar=%d", + stats.Synced, stats.Skipped, incDur, rchar()-r0, + ) + require.Equal(t, 1, stats.Synced) + sess, err := database.GetSessionFull(context.Background(), sessionID) + require.NoError(t, err) + require.NotNil(t, sess) + t.Logf( + "MACRO_INC_SESSION last_write_incremental=%v msgs=%d file_size=%d", + sess.LastWriteIncremental, sess.MessageCount, *sess.FileSize, + ) + require.True(t, sess.LastWriteIncremental, + "the late-output append must take the incremental path") + require.Equal(t, db.CurrentQualitySignalVersion, + sess.QualitySignalVersion, + "the maintained append must keep signals current") + + // 3. No-op resync. + r0 = rchar() + start = time.Now() + stats = engine.SyncAll(context.Background(), nil) + noopDur := time.Since(start) + t.Logf( + "MACRO_NOOP synced=%d skipped=%d dur=%s rchar=%d", + stats.Synced, stats.Skipped, noopDur, rchar()-r0, + ) + require.Equal(t, 1, stats.Skipped) +} + +// copyFilePrefix stream-copies the first limit bytes of src to dst, so a +// near-gigabyte snapshot never inflates the macro process's RSS. +func copyFilePrefix(src, dst string, limit int64) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + info, err := in.Stat() + if err != nil { + return err + } + if limit > info.Size() { + limit = info.Size() + } + out, err := os.Create(dst) + if err != nil { + return err + } + _, copyErr := io.Copy(out, io.LimitReader(in, limit)) + closeErr := out.Close() + if copyErr != nil { + return copyErr + } + return closeErr +} + +// TestMacroCodexStreamingMemoryGates is the P3 streaming-parse memory gate: +// it cold-syncs synthetic Codex transcripts whose content is dominated by +// tool outputs at 10MB / 100MB / 1GB, recording wall time, rchar delta, +// peak live heap (polled during the sync), forced-GC live heap, and peak +// RSS. The gate fails on the pre-P3 implementation (the parser keeps the +// whole session in the Go heap) and must pass once the single-pass +// staging sink lands: +// +// - 1GB peak live heap < 512MiB (stretch: < 350MiB); +// - 10MB -> 1GB peak growth <= 2x. +// +// Message count is held constant across sizes (500 turns) so the +// measurement isolates content-proportional memory. +func TestMacroCodexStreamingMemoryGates(t *testing.T) { + sizes := []struct { + name string + turns int + outBytes int + }{ + {name: "10MB", turns: 500, outBytes: 20 << 10}, + {name: "100MB", turns: 500, outBytes: 200 << 10}, + {name: "1GB", turns: 500, outBytes: 2 << 20}, + } + var basePeak uint64 + for i, size := range sizes { + t.Run(size.name, func(t *testing.T) { + root, _, _, _ := writeCodexStreamingBenchmarkTranscript( + t, size.turns, size.outBytes, + ) + database, err := db.Open(filepath.Join(t.TempDir(), "macro.db")) + require.NoError(t, err) + engine := NewEngine(database, EngineConfig{ + AgentDirs: map[parser.AgentType][]string{ + parser.AgentCodex: {root}, + }, + Machine: "macro-host", + }) + t.Cleanup(func() { + engine.Close() + _ = database.Close() + }) + + peak := pollPeakLiveHeap() + r0 := macroRchar(t) + start := time.Now() + stats := engine.SyncAll(context.Background(), nil) + dur := time.Since(start) + rcharDelta := macroRchar(t) - r0 + peakLive := peak() + require.Equal(t, 1, stats.Synced) + + t.Logf( + "STREAMING_GATE %s file=%dMB dur=%s rchar=%d "+ + "peak_live=%dMiB forced_gc=%dMiB peak_rss=%dMiB", + size.name, + (size.turns*size.outBytes)/(1<<20), + dur, + rcharDelta, + peakLive/(1<<20), + forcedGCLiveHeap()/(1<<20), + peakProcessRSSBytes()/(1<<20), + ) + if i == 0 { + basePeak = peakLive + } + if size.name == "1GB" { + require.Less(t, peakLive, uint64(512<<20), + "1GB cold sync peak live heap must stay under 512MiB") + require.LessOrEqual(t, peakLive, 2*basePeak, + "10MB -> 1GB peak live heap must grow at most 2x") + } + }) + } +} + +// writeCodexStreamingBenchmarkTranscript writes a synthetic Codex +// transcript with `turns` turns, each ending in a function_call plus a +// function_call_output carrying outBytes of content. Lines are written +// directly to the file so the fixture itself never allocates a +// file-sized string. +func writeCodexStreamingBenchmarkTranscript( + t testing.TB, turns, outBytes int, +) (root, dst, uuid string, sizeBytes int64) { + t.Helper() + return writeCodexStreamingBenchmarkTranscriptUUID( + t, turns, outBytes, codexSignalBenchmarkUUID, + ) +} + +// writeCodexStreamingBenchmarkTranscriptUUID is the fixture writer with an +// explicit session UUID, for tests that need more than one source. +func writeCodexStreamingBenchmarkTranscriptUUID( + t testing.TB, turns, outBytes int, uuid string, +) (root, dst string, sessionUUID string, sizeBytes int64) { + t.Helper() + sessionUUID = uuid + root = filepath.Join(t.TempDir(), "sessions") + day := filepath.Join(root, "2026", "07", "10") + require.NoError(t, os.MkdirAll(day, 0o755)) + dst = filepath.Join(day, "rollout-2026-07-10T07-00-00-"+uuid+".jsonl") + f, err := os.Create(dst) + require.NoError(t, err) + write := func(line string) { + t.Helper() + _, err := f.WriteString(line + "\n") + require.NoError(t, err) + } + write(testjsonl.CodexSessionMetaJSON( + uuid, "/workspace/project-a", "codex_cli_rs", + "2026-07-10T07:00:00Z", + )) + write(testjsonl.CodexTurnContextJSON( + "gpt-5.4", "2026-07-10T07:00:01Z", + )) + seed := "tool output: build log line with realistic command content. " + pad := strings.Repeat( + seed, (outBytes+len(seed)-1)/len(seed), + )[:outBytes] + for i := range turns { + ts := time.Date( + 2026, 7, 10, 7, 1, 0, 0, time.UTC, + ).Add(time.Duration(i) * time.Second) + tsStr := ts.Format("2006-01-02T15:04:05Z") + callID := "call_" + strconv.Itoa(i) + write(testjsonl.CodexMsgJSON( + "user", "run task "+strconv.Itoa(i), tsStr, + )) + write(testjsonl.CodexMsgJSON( + "assistant", "running task "+strconv.Itoa(i), tsStr, + )) + write(testjsonl.CodexFunctionCallWithCallIDJSON( + "exec_command", callID, nil, tsStr, + )) + write(testjsonl.CodexFunctionCallOutputJSON( + callID, pad, tsStr, + )) + } + require.NoError(t, f.Close()) + info, err := os.Stat(dst) + require.NoError(t, err) + return root, dst, uuid, info.Size() +} + +// pollPeakLiveHeap samples the runtime's live-heap metric (the heap +// occupied by live objects at the last GC, excluding uncollected garbage) +// and returns the peak observed until the returned function is called. +func pollPeakLiveHeap() func() uint64 { + var peak atomic.Uint64 + stop := make(chan struct{}) + done := make(chan struct{}) + all := []metrics.Sample{{Name: "/gc/heap/live:bytes"}} + go func() { + defer close(done) + ticker := time.NewTicker(2 * time.Millisecond) + defer ticker.Stop() + for { + select { + case <-stop: + return + case <-ticker.C: + metrics.Read(all) + if v := all[0].Value.Uint64(); v > peak.Load() { + peak.Store(v) + } + } + } + }() + return func() uint64 { + close(stop) + <-done + return peak.Load() + } +} + +func forcedGCLiveHeap() uint64 { + runtime.GC() + runtime.GC() + var ms runtime.MemStats + runtime.ReadMemStats(&ms) + return ms.HeapAlloc +} + +func peakProcessRSSBytes() uint64 { + data, err := os.ReadFile("/proc/self/status") + if err != nil { + return 0 + } + for _, line := range strings.Split(string(data), "\n") { + if strings.HasPrefix(line, "VmHWM:") { + fields := strings.Fields(line) + if len(fields) == 3 { + kb, err := strconv.ParseUint(fields[1], 10, 64) + if err == nil { + return kb << 10 + } + } + } + } + return 0 +} + +func macroRchar(t testing.TB) int64 { + t.Helper() + data, err := os.ReadFile("/proc/self/io") + require.NoError(t, err) + for _, line := range strings.Split(string(data), "\n") { + if strings.HasPrefix(line, "rchar: ") { + v, err := strconv.ParseInt( + strings.TrimSpace(strings.TrimPrefix(line, "rchar: ")), + 10, 64, + ) + require.NoError(t, err) + return v + } + } + t.Fatal("no rchar in /proc/self/io") + return 0 +} + +// TestMacroCodexStagedParseMemoryGates runs the same three sizes through +// the streaming staged path end to end (scratch-backed parse plus the +// staged publish) and applies the same bounds as the legacy gate. It is +// the direct memory gate for the P3 staging sink; the engine wiring +// behind the >128MB cutoff reuses exactly this path. +func TestMacroCodexStagedParseMemoryGates(t *testing.T) { + sizes := []struct { + name string + turns int + outBytes int + }{ + {name: "10MB", turns: 500, outBytes: 20 << 10}, + {name: "100MB", turns: 500, outBytes: 200 << 10}, + {name: "1GB", turns: 500, outBytes: 2 << 20}, + } + var basePeak uint64 + for i, size := range sizes { + t.Run(size.name, func(t *testing.T) { + root, _, uuid, _ := writeCodexStreamingBenchmarkTranscript( + t, size.turns, size.outBytes, + ) + cfg := parser.ProviderConfig{ + Roots: []string{root}, + Machine: "macro-host", + } + provider, ok := parser.NewProvider(parser.AgentCodex, cfg) + require.True(t, ok) + source, found, err := provider.FindSource( + context.Background(), parser.FindSourceRequest{ + FullSessionID: "codex:" + uuid, + }, + ) + require.NoError(t, err) + require.True(t, found) + + peak := pollPeakLiveHeap() + start := time.Now() + staged, err := newCodexStagingSink("", nil) + require.NoError(t, err) + sess, msgs, _, _, _, _, err := parser.ParseCodexSessionStreaming( + cfg, source, staged, + ) + require.NoError(t, err) + require.NotNil(t, sess) + + database, err := db.Open(filepath.Join(t.TempDir(), "macro.db")) + require.NoError(t, err) + t.Cleanup(func() { _ = database.Close() }) + row := db.Session{ + ID: sess.ID, + Project: sess.Project, + Machine: sess.Machine, + Agent: string(sess.Agent), + MessageCount: sess.MessageCount, + UserMessageCount: sess.UserMessageCount, + } + require.NoError(t, database.UpsertSession(row)) + dbMsgs := toDBMessages(pendingWrite{ + sess: *sess, msgs: msgs, + }, nil) + positions := stagedToolCallPositions(dbMsgs) + require.NoError(t, database.ReplaceSessionContentStaged( + context.Background(), row.ID, dbMsgs, staged, + map[string]bool{}, + func(verdicts map[string]bool) ( + db.SessionSignalUpdate, []db.SecretFinding, error, + ) { + update, findings := + computeSignalsAndSecretsWithContentFailures( + row, dbMsgs, verdicts, + ) + combined := append( + append([]db.SecretFinding(nil), findings...), + staged.Findings(row.ID, positions)..., + ) + update.SecretLeakCount = + definiteFindingCount(combined) + return update, combined, nil + }, + )) + require.NoError(t, staged.Close()) + peakLive := peak() + dur := time.Since(start) + + t.Logf( + "STAGED_GATE %s dur=%s peak_live=%dMiB forced_gc=%dMiB peak_rss=%dMiB", + size.name, dur, + peakLive/(1<<20), + forcedGCLiveHeap()/(1<<20), + peakProcessRSSBytes()/(1<<20), + ) + if i == 0 { + basePeak = peakLive + } + if size.name == "1GB" { + require.Less(t, peakLive, uint64(512<<20), + "1GB staged parse peak live heap must stay under 512MiB") + // The streaming path's baseline (SQLite session, test + // scaffolding, scratch connections) is a fixed overhead + // that dwarfs the 10MB tier's parse work, so the growth + // ratio is measured against a 16MiB floor: below that, + // run-to-run noise dominates and a 2x check on a 7MiB + // base is meaningless. The bound still catches any + // O(file) retention, which lands hundreds of MiB above + // the floor. + growthBase := max(basePeak, 16<<20) + require.LessOrEqual(t, peakLive, 2*growthBase, + "10MB -> 1GB staged peak live heap must grow at most 2x") + } + }) + } +} + +// TestMacroCodexEngineStagedFullParse syncs a >128MB Codex transcript +// through the real engine and asserts the staged streaming path (the +// >stagedCodexParseMinBytes cutoff) publishes the message, tool-call, +// event, and summary rows the archive expects: events carry real content, +// summaries carry the per-call aggregated output, and the session counts +// match the fixture shape. +func TestMacroCodexEngineStagedFullParse(t *testing.T) { + const turns = 750 + const outBytes = 200 << 10 // 150MB total, above the 128MB cutoff + root, _, uuid, _ := writeCodexStreamingBenchmarkTranscript( + t, turns, outBytes, + ) + + database := openTestDB(t) + engine := NewEngine(database, EngineConfig{ + AgentDirs: map[parser.AgentType][]string{ + parser.AgentCodex: {root}, + }, + Machine: "macro-host", + }) + t.Cleanup(engine.Close) + + stats := engine.SyncAll(t.Context(), nil) + require.Zero(t, stats.Failed) + require.Equal(t, 1, stats.Synced) + + sessionID := "codex:" + uuid + got, err := database.GetSession(t.Context(), sessionID) + require.NoError(t, err) + require.NotNil(t, got) + require.Equal(t, turns*3, got.MessageCount) + require.Equal(t, turns, got.UserMessageCount) + + msgs, err := database.GetAllMessages(t.Context(), sessionID) + require.NoError(t, err) + require.Len(t, msgs, turns*3) + var eventCount int + var summaryBytes int + for _, m := range msgs { + for _, tc := range m.ToolCalls { + summaryBytes += len(tc.ResultContent) + for _, ev := range tc.ResultEvents { + eventCount++ + require.NotEmpty(t, ev.Content, + "staged publish must store real event content") + require.GreaterOrEqual(t, ev.ContentLength, outBytes, + "event content length must match the staged row") + } + } + } + require.Equal(t, turns, eventCount) + require.Greater(t, summaryBytes, turns*(outBytes-1<<16), + "per-call summaries must carry the staged output content") +} + +// TestMacroCodexStaged64MBLine pins the single-line bound: a transcript +// whose tool output is one line just under the 64MB line limit must parse +// and publish through the staged path with bounded peak memory (the line +// plus its SQL copies, never line count times line size). +func TestMacroCodexStaged64MBLine(t *testing.T) { + // Keep the full JSON record under maxLineSize (64MB) after the + // wrapper bytes. + const outBytes = 64<<20 - 128<<10 + root, _, uuid, _ := writeCodexStreamingBenchmarkTranscript( + t, 1, outBytes, + ) + cfg := parser.ProviderConfig{ + Roots: []string{root}, + Machine: "macro-host", + } + provider, ok := parser.NewProvider(parser.AgentCodex, cfg) + require.True(t, ok) + source, found, err := provider.FindSource( + context.Background(), parser.FindSourceRequest{ + FullSessionID: "codex:" + uuid, + }, + ) + require.NoError(t, err) + require.True(t, found) + + peak := pollPeakLiveHeap() + staged, err := newCodexStagingSink("", nil) + require.NoError(t, err) + defer func() { require.NoError(t, staged.Close()) }() + sess, msgs, _, _, _, _, err := parser.ParseCodexSessionStreaming( + cfg, source, staged, + ) + require.NoError(t, err) + require.NotNil(t, sess) + + database, err := db.Open(filepath.Join(t.TempDir(), "macro.db")) + require.NoError(t, err) + t.Cleanup(func() { _ = database.Close() }) + row := db.Session{ + ID: sess.ID, + Project: sess.Project, + Machine: sess.Machine, + Agent: string(sess.Agent), + MessageCount: sess.MessageCount, + UserMessageCount: sess.UserMessageCount, + } + require.NoError(t, database.UpsertSession(row)) + dbMsgs := toDBMessages(pendingWrite{ + sess: *sess, msgs: msgs, + }, nil) + positions := stagedToolCallPositions(dbMsgs) + require.NoError(t, database.ReplaceSessionContentStaged( + context.Background(), row.ID, dbMsgs, staged, + map[string]bool{}, + func(verdicts map[string]bool) ( + db.SessionSignalUpdate, []db.SecretFinding, error, + ) { + update, findings := + computeSignalsAndSecretsWithContentFailures( + row, dbMsgs, verdicts, + ) + combined := append( + append([]db.SecretFinding(nil), findings...), + staged.Findings(row.ID, positions)..., + ) + update.SecretLeakCount = definiteFindingCount(combined) + return update, combined, nil + }, + )) + peakLive := peak() + t.Logf("BIGLINE peak_live=%dMiB rss=%dMiB", + peakLive/(1<<20), peakProcessRSSBytes()/(1<<20)) + require.Less(t, peakLive, uint64(512<<20), + "single near-limit line must stay bounded") +} + +// TestMacroCodexEngineTwoLargeStagedSources syncs two >threshold Codex +// transcripts in one pass and asserts both staged publishes land on the +// same archive (the consecutive-ATTACH path the single-connection writer +// pool exercises). +func TestMacroCodexEngineTwoLargeStagedSources(t *testing.T) { + const turns = 700 + const outBytes = 200 << 10 // 140MB each, above the 128MB cutoff + rootA, _, uuidA, _ := writeCodexStreamingBenchmarkTranscriptUUID( + t, turns, outBytes, codexSignalBenchmarkUUID, + ) + rootB, _, uuidB, _ := writeCodexStreamingBenchmarkTranscriptUUID( + t, turns, outBytes, codexSignalBenchmarkUUID+"-b", + ) + + database := openTestDB(t) + engine := NewEngine(database, EngineConfig{ + AgentDirs: map[parser.AgentType][]string{ + parser.AgentCodex: {rootA, rootB}, + }, + Machine: "macro-host", + }) + t.Cleanup(engine.Close) + + stats := engine.SyncAll(t.Context(), nil) + require.Zero(t, stats.Failed) + require.Equal(t, 2, stats.Synced) + + for _, uuid := range []string{uuidA, uuidB} { + sessionID := "codex:" + uuid + msgs, err := database.GetAllMessages(t.Context(), sessionID) + require.NoError(t, err) + var eventCount int + for _, m := range msgs { + for _, tc := range m.ToolCalls { + eventCount += len(tc.ResultEvents) + for _, ev := range tc.ResultEvents { + require.NotContains(t, ev.Content, "staged:", + "second publish wrote a staging placeholder") + } + } + } + require.Equal(t, turns, eventCount) + } +} + +// TestMacroCodexRealArchiveResyncColdSync is the ResyncAll acceptance leg +// for the real archive: the bulk rebuild path must publish real tool +// outputs through the staged transaction with the same RSS sanity bound as +// the plain cold sync. Uses its own process-level copy and gate. +func TestMacroCodexRealArchiveResyncColdSync(t *testing.T) { + src := os.Getenv("MACRO_CODEX_945MB") + if src == "" { + t.Skip("set MACRO_CODEX_945MB to a large real Codex transcript") + } + info, err := os.Stat(src) + require.NoError(t, err) + require.Greater(t, info.Size(), int64(500<<20)) + + srcDir := filepath.Dir(src) + rel := filepath.Join( + filepath.Base(filepath.Dir(filepath.Dir(srcDir))), + filepath.Base(filepath.Dir(srcDir)), + filepath.Base(srcDir), + filepath.Base(src), + ) + root := filepath.Join(t.TempDir(), "sessions") + dst := filepath.Join(root, rel) + require.NoError(t, os.MkdirAll(filepath.Dir(dst), 0o755)) + in, err := os.Open(src) + require.NoError(t, err) + out, err := os.Create(dst) + require.NoError(t, err) + _, err = io.Copy(out, in) + require.NoError(t, err) + require.NoError(t, in.Close()) + require.NoError(t, out.Close()) + + database := openTestDB(t) + engine := NewEngine(database, EngineConfig{ + AgentDirs: map[parser.AgentType][]string{ + parser.AgentCodex: {root}, + }, + Machine: "macro-host", + }) + t.Cleanup(engine.Close) + + start := time.Now() + peak := pollPeakLiveHeap() + stats := engine.ResyncAll(t.Context(), nil) + peakLive := peak() + require.False(t, stats.Aborted, "resync aborted: %v", stats.Warnings) + require.Zero(t, stats.Failed) + require.Equal(t, 1, stats.Synced) + t.Logf("REAL945 resync dur=%s peak_live=%dMiB rss=%dMiB", + time.Since(start), peakLive/(1<<20), + peakProcessRSSBytes()/(1<<20)) + require.Less(t, peakLive, uint64(512<<20), + "945MB resync peak live heap must stay under 512MiB") + require.Less(t, peakProcessRSSBytes(), uint64(1024<<20), + "945MB resync RSS must stay under 1GiB") + + // The bulk rebuild must have published real content, never staged + // placeholders. + msgs, err := database.GetAllMessages( + t.Context(), "codex:"+filepath.Base(src), + ) + if err == nil && len(msgs) > 0 { + for _, m := range msgs { + for _, tc := range m.ToolCalls { + for _, ev := range tc.ResultEvents { + require.NotContains(t, ev.Content, "staged:", + "bulk resync wrote a staging placeholder") + } + } + } + } +} + +// TestMacroCodexRealArchiveColdSync is the acceptance run for the staged +// streaming path against a real large Codex transcript. Set +// MACRO_CODEX_945MB to the archive path; the test copies it into the test +// root (never touching the original), syncs it through the real engine, +// and gates peak live heap under 512MiB and RSS under 1GiB. A second +// no-op sync then +// asserts the unchanged transcript is skipped without re-reading it. +func TestMacroCodexRealArchiveColdSync(t *testing.T) { + src := os.Getenv("MACRO_CODEX_945MB") + if src == "" { + t.Skip("set MACRO_CODEX_945MB to a large real Codex transcript") + } + info, err := os.Stat(src) + require.NoError(t, err) + require.Greater(t, info.Size(), int64(500<<20), + "MACRO_CODEX_945MB must point at a >500MiB transcript") + + // Rebuild the dated codex layout under the test root so discovery + // sees a real transcript copy, and keep the original untouched. + srcDir := filepath.Dir(src) + rel := filepath.Join( + filepath.Base(filepath.Dir(filepath.Dir(srcDir))), + filepath.Base(filepath.Dir(srcDir)), + filepath.Base(srcDir), + filepath.Base(src), + ) + root := filepath.Join(t.TempDir(), "sessions") + dst := filepath.Join(root, rel) + require.NoError(t, os.MkdirAll(filepath.Dir(dst), 0o755)) + in, err := os.Open(src) + require.NoError(t, err) + out, err := os.Create(dst) + require.NoError(t, err) + _, err = io.Copy(out, in) + require.NoError(t, err) + require.NoError(t, in.Close()) + require.NoError(t, out.Close()) + + dbPath := filepath.Join(t.TempDir(), "macro.db") + database, err := db.Open(dbPath) + require.NoError(t, err) + t.Cleanup(func() { _ = database.Close() }) + engine := NewEngine(database, EngineConfig{ + AgentDirs: map[parser.AgentType][]string{ + parser.AgentCodex: {root}, + }, + Machine: "macro-host", + }) + t.Cleanup(engine.Close) + + start := time.Now() + peak := pollPeakLiveHeap() + rcharBefore := macroRchar(t) + stats := engine.SyncAll(t.Context(), nil) + peakLive := peak() + rcharDelta := macroRchar(t) - rcharBefore + require.Zero(t, stats.Failed) + require.Equal(t, 1, stats.Synced) + dbInfo, dbErr := os.Stat(dbPath) + var dbSize int64 + if dbErr == nil { + dbSize = dbInfo.Size() + } + t.Logf( + "REAL945 cold dur=%s peak_live=%dMiB rss=%dMiB read=%dMiB size=%dMiB dbsize=%dMiB", + time.Since(start), peakLive/(1<<20), + peakProcessRSSBytes()/(1<<20), rcharDelta/(1<<20), + info.Size()/(1<<20), dbSize/(1<<20), + ) + require.Less(t, peakLive, uint64(512<<20), + "945MB cold sync peak live heap must stay under 512MiB") + require.Less(t, peakProcessRSSBytes(), uint64(1024<<20), + "945MB cold sync RSS must stay under 1GiB") + // The single-pass tee bounds transcript reads to one file pass. The + // remaining rchar is scratch publish I/O: the staged rows and + // summaries are read back once each while the replace transaction + // copies them into the archive. The ceiling excludes any second + // transcript pass (the pre-P3 cold path re-read the source multiple + // times and exceeded this several times over). + require.LessOrEqual(t, rcharDelta, 10*info.Size(), + "cold sync must not re-read the source multiple times") + + // No-op sync: the unchanged transcript must be skipped without + // touching the source bytes again. + noopBefore := macroRchar(t) + stats = engine.SyncAll(t.Context(), nil) + noopDelta := macroRchar(t) - noopBefore + require.Zero(t, stats.Failed) + require.Zero(t, stats.Synced, "unchanged archive must be skipped") + t.Logf("REAL945 noop dur=%s read=%dKiB", + time.Since(start), noopDelta/(1<<10)) + require.LessOrEqual(t, noopDelta, int64(16<<20), + "no-op sync must not re-read the transcript") +} diff --git a/internal/sync/codex_signal_bench_test.go b/internal/sync/codex_signal_bench_test.go new file mode 100644 index 000000000..83e960c55 --- /dev/null +++ b/internal/sync/codex_signal_bench_test.go @@ -0,0 +1,230 @@ +package sync + +import ( + "context" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/parser" + "go.kenn.io/agentsview/internal/testjsonl" +) + +const codexSignalBenchmarkUUID = "019eb791-cf7d-75c1-8439-9ed74c122c01" + +// BenchmarkCodexQuietAppendSignals500/5000/15000 are the non-amortized +// quiet-session append gate: every iteration performs one engine-level +// append (a new function_call plus the previous call's late output) with +// signals and secret findings fully maintained inline — the debounce +// scheduler is disabled, so no iteration amortizes the recompute against +// the others. The three sizes prove per-append latency does not scale with +// stored history. Each benchmark self-asserts zero GetAllMessages calls +// during the timed loop. +func BenchmarkCodexQuietAppendSignals500(b *testing.B) { + benchCodexQuietAppendSignals(b, 500) +} + +func BenchmarkCodexQuietAppendSignals5000(b *testing.B) { + benchCodexQuietAppendSignals(b, 5000) +} + +func BenchmarkCodexQuietAppendSignals15000(b *testing.B) { + benchCodexQuietAppendSignals(b, 15000) +} + +func benchCodexQuietAppendSignals(b *testing.B, turns int) { + silenceBenchLogs(b) + ctx := context.Background() + root, path, uuid := writeCodexSignalBenchmarkTranscript(b, turns) + + database, err := db.Open(filepath.Join(b.TempDir(), "bench.db")) + require.NoError(b, err) + engine := NewEngine(database, EngineConfig{ + AgentDirs: map[parser.AgentType][]string{ + parser.AgentCodex: {root}, + }, + Machine: "benchmark-host", + }) + b.Cleanup(func() { + engine.Close() + if err := database.Close(); err != nil { + b.Errorf("close bench db: %v", err) + } + }) + first := engine.SyncAll(ctx, nil) + require.Equal(b, 1, first.Synced) + require.Zero(b, first.Failed) + + // Disable the debounce so every append pays the full inline + // signals+secrets maintenance; nothing is amortized across iterations. + engine.signalSched.mu.Lock() + engine.signalSched.interval = 0 + engine.signalSched.quiet = 0 + engine.signalSched.mu.Unlock() + + f, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0o644) + require.NoError(b, err) + defer f.Close() + + // Pre-build the appended lines: constructing JSONL inside the timed + // loop would allocate and be gated as if it were sync work. Iteration i + // appends call_{i+1} and the output for call_i, so the output is + // always "late". + lines := make([]string, b.N) + for i := range lines { + lines[i] = testjsonl.JoinJSONL( + testjsonl.CodexFunctionCallWithCallIDJSON( + "exec_command", + codexSignalBenchmarkCall(i+1), + nil, + codexSignalBenchmarkTS(2*i+1), + ), + testjsonl.CodexFunctionCallOutputJSON( + codexSignalBenchmarkCall(i), + "result "+strconv.Itoa(i), + codexSignalBenchmarkTS(2*i+2), + ), + ) + } + + loadsBefore := database.MessagesLoadCount() + b.ReportAllocs() + b.ResetTimer() + for i := range b.N { + if _, err := f.WriteString(lines[i]); err != nil { + b.Fatalf("append: %v", err) + } + stats := engine.SyncAll(ctx, nil) + if stats.Failed != 0 || stats.Synced != 1 { + b.Fatalf("sync failed for appended output: %+v", stats) + } + } + b.StopTimer() + require.Equal(b, loadsBefore, database.MessagesLoadCount(), + "the maintained quiet append must never call GetAllMessages") + + sess, err := database.GetSessionFull( + ctx, "codex:"+uuid, + ) + require.NoError(b, err) + require.NotNil(b, sess) + require.Equal(b, db.CurrentQualitySignalVersion, + sess.QualitySignalVersion, + "every append must leave the signal version current") +} + +// BenchmarkCodexColdFullSync gates the cold full-parse pipeline: a fresh +// database and engine ingest the transcript from scratch, including the +// single-pass checkpoint capture. Per-op cost deliberately exceeds the +// usual micro-benchmark band; it exists to catch a regression that adds a +// source read pass (as the pre-fix checkpoint persistence did). +func BenchmarkCodexColdFullSync(b *testing.B) { + silenceBenchLogs(b) + ctx := context.Background() + root, _, _ := writeCodexSignalBenchmarkTranscript(b, 7500) + + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + database, err := db.Open(filepath.Join(b.TempDir(), "bench.db")) + require.NoError(b, err) + engine := NewEngine(database, EngineConfig{ + AgentDirs: map[parser.AgentType][]string{ + parser.AgentCodex: {root}, + }, + Machine: "benchmark-host", + }) + stats := engine.SyncAll(ctx, nil) + if stats.Failed != 0 || stats.Synced != 1 { + b.Fatalf("cold full sync failed: %+v", stats) + } + engine.Close() + if err := database.Close(); err != nil { + b.Fatalf("close bench db: %v", err) + } + } +} + +func codexSignalBenchmarkCall(i int) string { + return "call_" + strconv.Itoa(i) +} + +func codexSignalBenchmarkTS(i int) string { + return time.Date( + 2026, 7, 10, 7, 12, i, 0, time.UTC, + ).Format("2006-01-02T15:04:05Z") +} + +// writeCodexSignalBenchmarkTranscript builds a Codex transcript with +// `turns` prior user/assistant turns plus an unanswered call_0, so each +// appended iteration can emit a call plus the previous call's late output. +func writeCodexSignalBenchmarkTranscript( + b testing.TB, turns int, +) (root, path, uuid string) { + b.Helper() + uuid = codexSignalBenchmarkUUID + root = filepath.Join(b.TempDir(), "sessions") + path = filepath.Join( + root, + "2026", + "07", + "10", + "rollout-2026-07-10T07-12-15-"+uuid+".jsonl", + ) + require.NoError(b, os.MkdirAll(filepath.Dir(path), 0o755)) + + fixture := testjsonl.NewSessionBuilder(). + AddCodexMeta( + "2026-07-10T07:00:00Z", + uuid, + "/workspace/project-a", + "codex_cli_rs", + ). + AddRaw(testjsonl.CodexTurnContextJSON( + "gpt-5.4", "2026-07-10T07:00:01Z", + )). + AddCodexMessage( + "2026-07-10T07:00:02Z", + "user", + "Initial request: inspect the project and make a careful change.", + ). + AddCodexMessage( + "2026-07-10T07:00:03Z", + "assistant", + "Initial response: I will inspect the relevant code and tests.", + ) + contextPayload := strings.Repeat( + "Retain concrete code, test, and validation context. ", 8, + ) + for i := range turns { + turn := strconv.Itoa(i) + fixture.AddCodexMessage( + "2026-07-10T07:01:00Z", + "user", + "Prior turn "+turn+" request: continue the implementation. "+ + contextPayload, + ) + fixture.AddCodexMessage( + "2026-07-10T07:01:01Z", + "assistant", + "Prior turn "+turn+" response: applied the next bounded change. "+ + contextPayload, + ) + } + // call_0 is committed in the prefix with no output; iteration 0 + // appends call_1 plus the late output for call_0. + fixture.AddRaw(testjsonl.CodexFunctionCallWithCallIDJSON( + "exec_command", + "call_0", + nil, + codexSignalBenchmarkTS(0), + )) + prefix := fixture.String() + require.NoError(b, os.WriteFile(path, []byte(prefix), 0o644)) + return root, path, uuid +} diff --git a/internal/sync/codex_staging.go b/internal/sync/codex_staging.go new file mode 100644 index 000000000..e58eb5563 --- /dev/null +++ b/internal/sync/codex_staging.go @@ -0,0 +1,856 @@ +package sync + +import ( + "context" + "crypto/sha256" + "database/sql" + "fmt" + "log" + "os" + "path/filepath" + "runtime/debug" + "strings" + "sync" + "time" + + _ "github.com/mattn/go-sqlite3" + + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/parser" + "go.kenn.io/agentsview/internal/secrets" + "go.kenn.io/agentsview/internal/signals" + "go.kenn.io/agentsview/internal/timeutil" +) + +// codexStagingSink implements parser.CodexSessionSink for the streaming +// full-parse path: messages and tool-call metadata stay in memory (they are +// small relative to tool outputs), while every tool-result event row and +// the per-call agent summary state are written to a scratch SQLite file as +// they arrive. The in-memory model therefore never holds result-event +// content — events carry a unique placeholder — and peak memory is +// O(messages + batch), not O(file size). The scratch database is also the +// publish source: the staged write inserts tool_result_events straight +// from it and resolves result_content summaries per call with transient +// memory bounded by one call's distinct agents. +type codexStagingSink struct { + *parser.CodexCollectingSink + + scratch *sql.DB + path string + + // idPrefix is applied to subagent_session_id at publish time, mirroring + // applyRemoteRewrites on the collecting path: staged events are inserted + // directly from scratch and never pass through the in-memory rewrite + // that prefixes remote (SSH/S3) session ids. + idPrefix string + + // blocked marks categories whose stored content is blanked. Their raw + // content never enters scratch storage; only digest, original length, + // ordering metadata, and summary participation are retained. + blocked map[string]bool + + // Calls use an occurrence-qualified staging key because provider call IDs + // can repeat within one transcript. callKeyByPosition is authoritative; + // currentCallKey is only the compatibility fallback for events that do not + // carry a parser-resolved occurrence. + currentCallKey map[string]string + callKeyByPosition map[parser.ParsedToolCallPosition]string + callOccurrences map[string]int + categoryByCallKey map[string]string + + // findings collects definite findings from result-event content as + // it streams by; findingPos records the coordinates to patch after + // ordinal finalization. + findings []db.SecretFinding + findingPos []stagedFindingPos + eventByCallKey map[string]int64 + eventSeq int64 + // contentFailures records per-call content-failure verdicts captured + // while the publish transaction resolves summaries, so the engine can + // fold them into the signal pass after the atomic publish. + contentFailures map[string]bool + // stageErr is the sticky first scratch failure. Once set, staging is + // unrecoverable for this parse: events are no longer accepted and the + // publish must fail, so a disk-full or I/O error can never commit a + // "successful" archive missing tool outputs. + stageErr error + validationStats db.ValidationStats +} + +// fail records the sticky staging failure. Later events and the publish +// path consult Err and refuse to proceed. +func (s *codexStagingSink) fail(err error) { + if s.stageErr == nil { + s.stageErr = fmt.Errorf("codex staging: %w", err) + } +} + +// Err returns the sticky staging failure, or nil when the scratch +// database has been healthy for this parse. +func (s *codexStagingSink) Err() error { + return s.stageErr +} + +func (s *codexStagingSink) addValidationStats(stats db.ValidationStats) { + s.validationStats.ControlCharsStripped += stats.ControlCharsStripped + s.validationStats.ModelClamped += stats.ModelClamped + s.validationStats.TokensClamped += stats.TokensClamped + s.validationStats.RoleCoerced += stats.RoleCoerced + s.validationStats.TimestampsBlanked += stats.TimestampsBlanked +} + +// ValidationStats returns fixes applied to real staged result content. The +// ordinary message validation pass sees placeholders, so the engine records +// these additional counts after the staged publish resolves its summaries. +func (s *codexStagingSink) ValidationStats() db.ValidationStats { + return s.validationStats +} + +type stagedFindingPos struct { + stageKey string + eventIndex int +} + +// codexStagingFilePrefix identifies scratch files owned by AgentsView. +const codexStagingFilePrefix = "agentsview-codex-stage-" + +// prepareCodexStagingDir creates the private scratch directory and removes +// abandoned files older than one day. Recent files may belong to another +// running AgentsView process and are left untouched. +func prepareCodexStagingDir(dir string) error { + if dir == "" { + return nil + } + if err := os.MkdirAll(dir, 0o700); err != nil { + return fmt.Errorf("creating codex staging directory: %w", err) + } + if err := os.Chmod(dir, 0o700); err != nil { + return fmt.Errorf("securing codex staging directory: %w", err) + } + entries, err := os.ReadDir(dir) + if err != nil { + return fmt.Errorf("reading codex staging directory: %w", err) + } + cutoff := time.Now().Add(-24 * time.Hour) + for _, entry := range entries { + if entry.IsDir() || !strings.HasPrefix(entry.Name(), codexStagingFilePrefix) { + continue + } + info, infoErr := entry.Info() + if infoErr != nil || !info.ModTime().Before(cutoff) { + continue + } + _ = os.Remove(filepath.Join(dir, entry.Name())) + } + return nil +} + +// checkCodexStagingSpace fails a staged parse before it writes a byte when +// the scratch directory cannot hold a conservative estimate of the staged +// database plus SQLite overhead. The minimum protects small test/configured +// sources; large sources scale the requirement with their actual size. +func checkCodexStagingSpace(dir string, sourceBytes int64) error { + if dir == "" { + dir = os.TempDir() + } + available, ok, err := stagingDirFreeBytes(dir) + if err != nil || !ok { + // Filesystems without a capacity query fail open here; CreateTemp and + // SQLite report concrete write errors without changing the archive. + return nil + } + const ( + stagedScratchMinFree = int64(256 << 20) + stagedScratchOverhead = int64(64 << 20) + ) + required := stagedScratchMinFree + if sourceBytes > 0 { + scaled := sourceBytes + sourceBytes/2 + stagedScratchOverhead + if scaled > required { + required = scaled + } + } + if int64(available) < required { + return fmt.Errorf( + "codex staging: %s has %dMiB free, need at least %dMiB for a %dMiB source", + dir, available/(1<<20), required/(1<<20), sourceBytes/(1<<20), + ) + } + return nil +} + +// stagedCodexParseMinBytes is the full-parse size above which a Codex +// source streams through the scratch staging path instead of the +// collecting parser. Engines may override it (see +// EngineConfig.StagedCodexParseMinBytes) for tests. +const stagedCodexParseMinBytes = 128 << 20 + +// stagedCodexMinBytes resolves a configured override to the default. +func stagedCodexMinBytes(override int64) int64 { + if override > 0 { + return override + } + return stagedCodexParseMinBytes +} + +// stagedColdSyncGCPercent is the GC percent held while a staged Codex cold +// sync is in flight. The streaming path's live set is bounded, so a lower +// target keeps the peak heap (and with it RSS) near the live set instead +// of letting transient per-event garbage double it. Restored on release. +const stagedColdSyncGCPercent = 30 + +// The GC percent is process-global state, so the refcount lives at package +// scope: two engines interleaving staged syncs must share one lowered +// window rather than racing to restore each other's baseline. +var ( + stagedGCMu sync.Mutex + stagedGCRefs int + stagedGCPrev int +) + +// beginStagedColdSync lowers the process GC percent for the duration of one +// staged Codex cold sync and returns the function that restores the prior +// value. Concurrent staged syncs share one lowered window via a refcount. +func beginStagedColdSync() func() { + stagedGCMu.Lock() + defer stagedGCMu.Unlock() + if stagedGCRefs == 0 { + stagedGCPrev = debug.SetGCPercent(stagedColdSyncGCPercent) + } + stagedGCRefs++ + released := false + return func() { + stagedGCMu.Lock() + defer stagedGCMu.Unlock() + if released { + return + } + released = true + stagedGCRefs-- + if stagedGCRefs == 0 { + debug.SetGCPercent(stagedGCPrev) + } + } +} + +const codexStagingSchema = ` +CREATE TABLE stage_events ( + seq INTEGER PRIMARY KEY, + call_key TEXT NOT NULL, + tool_use_id TEXT NOT NULL, + agent_id TEXT NOT NULL DEFAULT '', + subagent_session_id TEXT NOT NULL DEFAULT '', + source TEXT NOT NULL, + status TEXT NOT NULL, + content TEXT NOT NULL, + raw_content_digest BLOB NOT NULL, + content_length INTEGER NOT NULL, + timestamp TEXT NOT NULL DEFAULT '', + blanked INTEGER NOT NULL DEFAULT 0, + summary_participates INTEGER NOT NULL DEFAULT 0 +); +CREATE INDEX idx_stage_events_call ON stage_events(call_key, seq);` + +// newCodexStagingSink opens a scratch SQLite database for one streaming +// parse. The caller must Close it once the staged write has published. +// dir selects the scratch directory; empty means the system temporary +// directory. +func newCodexStagingSink( + dir string, + blocked map[string]bool, + sourceSize ...int64, +) (*codexStagingSink, error) { + if err := prepareCodexStagingDir(dir); err != nil { + return nil, err + } + var sourceBytes int64 + if len(sourceSize) > 0 { + sourceBytes = sourceSize[0] + } + if err := checkCodexStagingSpace(dir, sourceBytes); err != nil { + return nil, err + } + f, err := os.CreateTemp(dir, codexStagingFilePrefix+"*.sqlite") + if err != nil { + return nil, fmt.Errorf("creating codex staging file: %w", err) + } + path := f.Name() + if err := f.Close(); err != nil { + os.Remove(path) + return nil, err + } + scratch, err := sql.Open("sqlite3", path) + if err != nil { + os.Remove(path) + return nil, fmt.Errorf("opening codex staging db: %w", err) + } + for _, pragma := range []string{ + "PRAGMA journal_mode=OFF", + "PRAGMA synchronous=OFF", + "PRAGMA temp_store=FILE", + } { + if _, err := scratch.Exec(pragma); err != nil { + scratch.Close() + os.Remove(path) + return nil, fmt.Errorf("configuring codex staging db: %w", err) + } + } + if _, err := scratch.Exec(codexStagingSchema); err != nil { + scratch.Close() + os.Remove(path) + return nil, fmt.Errorf("creating codex staging schema: %w", err) + } + return &codexStagingSink{ + CodexCollectingSink: parser.NewCodexCollectingSink(0), + scratch: scratch, + path: path, + blocked: blocked, + currentCallKey: make(map[string]string), + callKeyByPosition: make(map[parser.ParsedToolCallPosition]string), + callOccurrences: make(map[string]int), + categoryByCallKey: make(map[string]string), + eventByCallKey: make(map[string]int64), + }, nil +} + +// Close releases the scratch database and removes its file. +func (s *codexStagingSink) Close() error { + err := s.scratch.Close() + if err == nil { + err = os.Remove(s.path) + } + return err +} + +// Path returns the staging file path for ATTACH-based publishing. +func (s *codexStagingSink) Path() string { + return s.path +} + +// stagedCodexParseOutcome runs the streaming Codex parse through sink and +// folds the result into the same ParseOutcome shape the provider's +// collecting parse returns, so the engine's downstream outcome pipeline is +// shared between the two paths. +func stagedCodexParseOutcome( + cfg parser.ProviderConfig, + source parser.SourceRef, + fingerprint parser.SourceFingerprint, + sink *codexStagingSink, +) (parser.ParseOutcome, error) { + sess, msgs, cursor, hashState, anchorDigest, retryReason, err := + parser.ParseCodexSessionStreaming(cfg, source, sink) + if err != nil { + return parser.ParseOutcome{}, err + } + if stageErr := sink.Err(); stageErr != nil { + return parser.ParseOutcome{}, stageErr + } + // The collecting provider copies the fingerprint hash onto the parsed + // session; the streaming entry point has no fingerprint parameter, so + // mirror that here. A staged full parse with a precomputed fingerprint + // must persist the same file_hash the collecting path would, or the + // checkpoint's hash and the stored file_hash disagree and every later + // validation forces another full parse. + if fingerprint.Hash != "" { + sess.File.Hash = fingerprint.Hash + } + // Return the parse phase's transient arenas before the publish builds + // its own transient working set, so the process RSS high-water mark + // reflects the publish rather than the sum of both phases' slack. + debug.FreeOSMemory() + result := parser.ParseResultOutcome{ + Result: parser.ParseResult{ + Session: *sess, + Messages: msgs, + Checkpoint: cursor, + CheckpointHashState: hashState, + CheckpointAnchorDigest: anchorDigest, + }, + DataVersion: parser.DataVersionCurrent, + } + if retryReason != "" { + // An explicit fork parent could not be resolved: keep the child + // visible but mark its stored data version for retry so a later + // unchanged-object sync can replace the temporary overcount. + result.DataVersion = parser.DataVersionNeedsRetry + result.RetryReason = retryReason + } + return parser.ParseOutcome{ + Results: []parser.ParseResultOutcome{result}, + ResultSetComplete: true, + ForceReplace: true, + }, nil +} + +// definiteFindingCount counts the definite-confidence findings in a +// merged findings slice, stamping the session's secret-leak signal. +func definiteFindingCount(findings []db.SecretFinding) int { + n := 0 + for _, f := range findings { + if f.Confidence == "definite" { + n++ + } + } + return n +} + +// closeCodexStagingSinks releases a batch of staging sinks, removing their +// scratch files. It is the staging analog of releaseParseRetentionLeases. +func closeCodexStagingSinks(sinks []*codexStagingSink) { + for _, s := range sinks { + if s == nil { + continue + } + if err := s.Close(); err != nil { + log.Printf("closing codex staging sink: %v", err) + } + } +} + +// releaseStagedGCGuards restores the process GC percent after a batch of +// staged cold syncs finished, in reverse open order. +func releaseStagedGCGuards(guards []func()) { + for i := len(guards) - 1; i >= 0; i-- { + if guards[i] != nil { + guards[i]() + } + } +} + +func (s *codexStagingSink) AppendMessage(m parser.ParsedMessage) int { + ordinal := s.CodexCollectingSink.AppendMessage(m) + for callIndex, tc := range m.ToolCalls { + if tc.ToolUseID == "" { + continue + } + occurrence := s.callOccurrences[tc.ToolUseID] + s.callOccurrences[tc.ToolUseID] = occurrence + 1 + stageKey := db.StagedToolCallKey(tc.ToolUseID, occurrence) + s.currentCallKey[tc.ToolUseID] = stageKey + s.callKeyByPosition[parser.ParsedToolCallPosition{ + MessageOrdinal: ordinal, + CallIndex: callIndex, + }] = stageKey + s.categoryByCallKey[stageKey] = tc.Category + } + return ordinal +} + +// AppendToolResultEvent stages the full event row and the per-call summary +// state, then records a contentless placeholder in the in-memory model so +// downstream conversions stay shape-compatible without retaining content. +func (s *codexStagingSink) AppendToolResultEvent( + callID string, target *parser.ParsedToolCallPosition, + ev parser.ParsedToolResultEvent, +) { + if callID == "" || s.stageErr != nil { + return + } + // The parser extracts event fields as gjson substrings of the source + // line. Storing those small strings in the in-memory model (event + // identity fields, map keys below) would pin the entire line's backing + // buffer — for large tool outputs that keeps the whole transcript's + // line bytes reachable across the parse. Clone the fields the model + // keeps; content is replaced by a placeholder after staging. + callID = strings.Clone(callID) + ev.ToolUseID = strings.Clone(ev.ToolUseID) + ev.AgentID = strings.Clone(ev.AgentID) + ev.SubagentSessionID = strings.Clone(ev.SubagentSessionID) + ev.Status = strings.Clone(ev.Status) + ev.Source = strings.Clone(ev.Source) + // The legacy write path normalizes event timestamps through + // timeutil.Format before storing them; the staged rows must store the + // same normalized form so stored projections match byte for byte. + tsStr := timeutil.Format(ev.Timestamp) + // Events for calls that never registered in the message model are + // unreachable regardless of how they are held: parser.ParseResult + // carries no ToolCallUpdates field, so every full-parse consumer + // (collecting and staged alike) discards them. Drop the event outright + // instead of forwarding it to the embedded collecting sink's orphan + // path, which would retain ev.Content -- an uncloned reference into + // the source line's backing buffer -- purely to be thrown away, + // defeating the staged sink's bounded-memory guarantee on large + // orphan outputs. Late outputs still merge through the incremental + // append path on later syncs, unchanged. + stageKey, ok := "", false + if target != nil { + stageKey, ok = s.callKeyByPosition[*target] + } else { + stageKey, ok = s.currentCallKey[callID] + } + if !ok { + return + } + // The legacy deduplication compares raw parser content before the central + // sanitizer runs. Keep that identity as a digest so the staged row can + // store sanitized content without collapsing events that differed only by + // stripped controls. The digest also avoids retaining a second copy of a + // potentially very large raw result in scratch. + rawContentDigest := sha256.Sum256([]byte(ev.Content)) + var exists int + err := s.scratch.QueryRow( + `SELECT 1 FROM stage_events + WHERE call_key = ? AND agent_id = ? AND status = ? + AND raw_content_digest = ? LIMIT 1`, + stageKey, ev.AgentID, ev.Status, rawContentDigest[:], + ).Scan(&exists) + if err == nil { + return // equivalent event already staged + } + if err != sql.ErrNoRows { + s.fail(err) + return + } + + s.eventSeq++ + seq := s.eventSeq + subagent := ev.SubagentSessionID + if subagent == "" && strings.TrimSpace(ev.AgentID) != "" { + agentID := strings.TrimSpace(ev.AgentID) + if strings.HasPrefix(agentID, "codex:") { + subagent = agentID + } else { + subagent = "codex:" + agentID + } + } + blanked := 0 + if s.blocked[s.categoryByCallKey[stageKey]] { + blanked = 1 + } + contentLength := len(ev.Content) + summaryParticipates := strings.TrimSpace(ev.Content) != "" + if blanked == 0 { + // The collecting path sanitizes result-event content in the central + // db validation pass. Staged events bypass that pass because the + // in-memory message carries only a placeholder, so apply the same + // contract before the real content enters the scratch publish source. + // Keep dedup above this point raw: two provider events that differ + // only by stripped controls remain two events on the collecting path. + toolCall := db.ToolCall{ResultEvents: []db.ToolResultEvent{{ + Content: ev.Content, + ContentLength: contentLength, + }}} + s.addValidationStats(db.SanitizeToolCall(&toolCall)) + ev.Content = toolCall.ResultEvents[0].Content + contentLength = toolCall.ResultEvents[0].ContentLength + } else { + // Blocked content must not be recoverable from an abandoned scratch + // database after a crash. The digest and original length preserve + // deduplication and result_content_length parity without storing bytes. + ev.Content = "" + } + if _, err := s.scratch.Exec( + `INSERT INTO stage_events ( + seq, call_key, tool_use_id, agent_id, subagent_session_id, + source, status, content, raw_content_digest, content_length, + timestamp, blanked, summary_participates + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + seq, stageKey, callID, ev.AgentID, subagent, ev.Source, ev.Status, + ev.Content, rawContentDigest[:], contentLength, tsStr, blanked, + summaryParticipates, + ); err != nil { + s.fail(err) + return + } + + // Definite findings from the stored (blanked) content — the same + // text the legacy scan reads back from the database. + storedContent := ev.Content + if blanked != 0 { + storedContent = "" + } + eventIndex := int(s.eventByCallKey[stageKey]) + s.eventByCallKey[stageKey]++ + s.addEventFindings(stageKey, eventIndex, storedContent) + + // The in-memory model keeps a unique placeholder instead of the + // content: downstream conversions stay shape-compatible and the + // collecting dedup treats every staged event as distinct. + ev.Content = fmt.Sprintf("staged:%d", seq) + s.CodexCollectingSink.AppendToolResultEvent(callID, target, ev) +} + +func (s *codexStagingSink) addEventFindings( + stageKey string, eventIndex int, content string, +) { + if content == "" { + return + } + matches := secrets.ScanDefinite(content) + for _, match := range matches { + s.findings = append(s.findings, db.SecretFinding{ + RuleName: match.Rule, + Confidence: match.Confidence, + LocationKind: "tool_result_event", + MatchStart: match.Start, + MatchEnd: match.End, + MatchIndex: match.Index, + RedactedMatch: match.Redacted, + RulesVersion: secrets.DefiniteRulesVersion(), + }) + s.findingPos = append(s.findingPos, stagedFindingPos{ + stageKey: stageKey, + eventIndex: eventIndex, + }) + } +} + +// Findings returns the staged event findings with session, ordinal, and +// call coordinates stamped from the final message model. +func (s *codexStagingSink) Findings( + sessionID string, + positions map[string]db.StagedToolCallPosition, +) []db.SecretFinding { + out := make([]db.SecretFinding, len(s.findings)) + for i, f := range s.findings { + f.SessionID = sessionID + pos, ok := positions[s.findingPos[i].stageKey] + if ok { + f.MessageOrdinal = pos.Ordinal + callIdx := pos.CallIndex + evIdx := s.findingPos[i].eventIndex + f.CallIndex = &callIdx + f.EventIndex = &evIdx + } + out[i] = f + } + return out +} + +// InsertEventsTx inserts the staged result events into tool_result_events +// within the caller's publish transaction, ordered by emission so +// event_index matches the legacy slice order. The caller attached the +// scratch database as codex_staging on the transaction's connection and +// detaches it after the transaction settles; the transaction itself only +// ever modifies main, so the cross-database crash-atomicity limit for +// WAL-mode attached databases is respected. +func (s *codexStagingSink) InsertEventsTx( + ctx context.Context, tx *sql.Tx, sessionID string, + messageOrdinals map[string]db.StagedToolCallPosition, +) error { + if s.stageErr != nil { + return s.stageErr + } + for stageKey, pos := range messageOrdinals { + if _, err := tx.ExecContext(ctx, ` + INSERT INTO tool_result_events ( + session_id, tool_call_message_ordinal, call_index, + tool_use_id, agent_id, subagent_session_id, + source, status, content, content_length, + timestamp, event_index + ) + SELECT ?, ?, ?, tool_use_id, + CASE WHEN agent_id = '' THEN NULL ELSE agent_id END, + CASE + WHEN subagent_session_id = '' THEN NULL + WHEN ? = '' OR instr(subagent_session_id, ?) = 1 + THEN subagent_session_id + ELSE ? || subagent_session_id + END, + source, status, + CASE WHEN blanked = 1 THEN '' ELSE content END, + content_length, + CASE WHEN timestamp = '' THEN NULL ELSE timestamp END, + row_number() OVER (ORDER BY seq) - 1 + FROM codex_staging.stage_events + WHERE call_key = ? + ORDER BY seq`, + sessionID, pos.Ordinal, pos.CallIndex, + s.idPrefix, s.idPrefix, s.idPrefix, stageKey, + ); err != nil { + return fmt.Errorf( + "publishing staged events for %s/%s: %w", + sessionID, pos.ToolUseID, err, + ) + } + } + return nil +} + +// ResolveSummary computes the stored result summary for one call by +// walking its staged event rows in emission order, mirroring +// db.SummarizeToolResultEvents: the latest raw content per agent in +// first-write order, followed by the trailing anonymous content. Memory is +// transient and bounded by the call's distinct agents: the strict bound +// is one call's aggregate output (the summary string itself), not the +// whole transcript. While the summary is in hand it also records the +// call's content-failure verdict (see ContentFailures), so the engine's +// post-publish signal fold never resolves summaries a second time. +func (s *codexStagingSink) ResolveSummary( + ctx context.Context, stageKey string, +) (summary string, contentLength int, err error) { + if s.stageErr != nil { + return "", 0, s.stageErr + } + blocked := s.blocked[s.categoryByCallKey[stageKey]] + if blocked { + contentLength, err = s.resolveBlockedSummaryLength(ctx, stageKey) + if err != nil { + return "", 0, err + } + if s.contentFailures == nil { + s.contentFailures = make(map[string]bool) + } + s.contentFailures[stageKey] = signals.IsFailure(signals.ToolCallRow{ + Category: s.categoryByCallKey[stageKey], + }) + return "", contentLength, nil + } + rows, err := s.scratch.QueryContext(ctx, ` + SELECT agent_id, content, summary_participates + FROM stage_events + WHERE call_key = ? + ORDER BY seq`, + stageKey, + ) + if err != nil { + return "", 0, err + } + defer rows.Close() + // Emission order makes the first seen row per agent both the + // first-write anchor and the earliest summary part; later rows simply + // overwrite the content. + var order []string + latest := make(map[string]string) + var lastAnon string + hasAnon := false + for rows.Next() { + var agentID, content string + var participates bool + if err := rows.Scan(&agentID, &content, &participates); err != nil { + return "", 0, err + } + if !participates { + continue + } + agent := strings.TrimSpace(agentID) + if agent == "" { + hasAnon = true + lastAnon = content + continue + } + if _, ok := latest[agent]; !ok { + order = append(order, agent) + } + latest[agent] = content + } + if err := rows.Err(); err != nil { + return "", 0, err + } + var parts []string + for _, agent := range order { + parts = append(parts, agent+":\n"+latest[agent]) + } + switch { + case len(parts) == 0: + summary = lastAnon + case len(parts) == 1: + summary = parts[0][strings.IndexByte(parts[0], '\n')+1:] + if hasAnon { + summary += "\n\n" + lastAnon + } + default: + summary = strings.Join(parts, "\n\n") + if hasAnon { + summary += "\n\n" + lastAnon + } + } + contentLength = len(summary) + // Agent labels become part of result_content but are not themselves + // result-event content. Sanitize the assembled summary as the normal + // message validation pass does after SummarizeToolResultEvents. + toolCall := db.ToolCall{ + ResultContent: summary, + ResultContentLength: contentLength, + } + s.addValidationStats(db.SanitizeToolCall(&toolCall)) + summary = toolCall.ResultContent + contentLength = toolCall.ResultContentLength + verdict := signals.IsFailure(signals.ToolCallRow{ + Category: s.categoryByCallKey[stageKey], + ResultContent: summary, + }) + if s.contentFailures == nil { + s.contentFailures = make(map[string]bool) + } + s.contentFailures[stageKey] = verdict + return summary, contentLength, nil +} + +func (s *codexStagingSink) resolveBlockedSummaryLength( + ctx context.Context, stageKey string, +) (int, error) { + rows, err := s.scratch.QueryContext(ctx, ` + SELECT agent_id, content_length, summary_participates + FROM stage_events + WHERE call_key = ? + ORDER BY seq`, + stageKey, + ) + if err != nil { + return 0, err + } + defer rows.Close() + + order := make([]string, 0) + latestByAgent := make(map[string]int) + lastAnonLength := 0 + allHaveAgentID := true + for rows.Next() { + var agentID string + var length int + var participates bool + if err := rows.Scan(&agentID, &length, &participates); err != nil { + return 0, err + } + if !participates { + continue + } + agentID = strings.TrimSpace(agentID) + if agentID == "" { + allHaveAgentID = false + lastAnonLength = length + continue + } + if _, ok := latestByAgent[agentID]; !ok { + order = append(order, agentID) + } + latestByAgent[agentID] = length + } + if err := rows.Err(); err != nil { + return 0, err + } + + if len(latestByAgent) <= 1 { + if len(latestByAgent) == 0 { + return lastAnonLength, nil + } + length := latestByAgent[order[0]] + if lastAnonLength > 0 { + length += 2 + lastAnonLength + } + return length, nil + } + parts := make([]int, 0, len(order)+1) + for _, agentID := range order { + parts = append(parts, len(agentID)+2+latestByAgent[agentID]) + } + if !allHaveAgentID && lastAnonLength > 0 { + parts = append(parts, lastAnonLength) + } + total := 0 + for i, length := range parts { + if i > 0 { + total += 2 + } + total += length + } + return total, nil +} + +// ContentFailures returns the per-call content-failure verdicts captured +// during summary resolution in the publish transaction. Calls the +// transaction never resolved (no registered tool call) are absent. +func (s *codexStagingSink) ContentFailures() map[string]bool { + return s.contentFailures +} diff --git a/internal/sync/codex_staging_fuzz_test.go b/internal/sync/codex_staging_fuzz_test.go new file mode 100644 index 000000000..61192b605 --- /dev/null +++ b/internal/sync/codex_staging_fuzz_test.go @@ -0,0 +1,411 @@ +package sync + +import ( + "context" + "database/sql" + "fmt" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/parser" + "go.kenn.io/agentsview/internal/testjsonl" +) + +// TestCodexStagedParityPerturbations runs the full dual-path DB comparison +// over structurally perturbed transcripts: reordered lines, truncation, +// duplicated and interleaved events, junk and empty lines, and a forked +// session-meta replay. Every variant must produce byte-identical stored +// projections from both paths. +func TestCodexStagedParityPerturbations(t *testing.T) { + const uuid = "019eb791-cf7d-75c1-8439-9ed74c122b05" + base := strings.Split( + strings.TrimSuffix(codexParityTranscript(uuid), "\n"), "\n", + ) + cases := map[string]func() string{ + "reversed lines": func() string { + rev := slices.Clone(base) + slices.Reverse(rev) + return strings.Join(rev, "\n") + }, + "truncated tail": func() string { + // Drop the last output line: call_c has no result event. + return strings.Join(base[:len(base)-1], "\n") + }, + "duplicated output": func() string { + out := slices.Clone(base) + out = slices.Insert(out, 7, base[6]) + return strings.Join(out, "\n") + }, + "interleaved outputs": func() string { + out := slices.Clone(base) + out[6], out[12] = out[12], out[6] + return strings.Join(out, "\n") + }, + "junk lines": func() string { + out := slices.Clone(base) + out = slices.Insert(out, 3, "not json at all", `{"type":"unknown"}`) + return strings.Join(out, "\n") + }, + "empty lines": func() string { + out := slices.Clone(base) + out = slices.Insert(out, 2, "", " ") + return strings.Join(out, "\n") + }, + "forked meta replay": func() string { + meta := testjsonl.CodexSessionMetaJSON( + uuid, "/workspace/project-a", "codex_cli_rs", + "2024-01-01T10:00:00Z", + ) + out := slices.Clone(base) + out = slices.Insert(out, 1, meta) + return strings.Join(out, "\n") + }, + } + for name, build := range cases { + t.Run(name, func(t *testing.T) { + assertCodexStagedParity(t, uuid, build()) + }) + } +} + +// failingStagedResults wraps a real staging sink and fails a chosen +// publish operation, simulating a mid-publish abort. +type failingStagedResults struct { + *codexStagingSink + failResolve bool + failEvents bool + resolveCalls int + eventsCalls int +} + +func (f *failingStagedResults) ResolveSummary( + ctx context.Context, toolUseID string, +) (string, int, error) { + f.resolveCalls++ + if f.failResolve { + return "", 0, fmt.Errorf("injected summary resolution failure") + } + return f.codexStagingSink.ResolveSummary(ctx, toolUseID) +} + +func (f *failingStagedResults) InsertEventsTx( + ctx context.Context, tx *sql.Tx, sessionID string, + positions map[string]db.StagedToolCallPosition, +) error { + f.eventsCalls++ + if f.failEvents { + return fmt.Errorf("injected event insert failure") + } + return f.codexStagingSink.InsertEventsTx( + ctx, tx, sessionID, positions, + ) +} + +// TestCodexStagedPublishFailureKeepsPriorContent pins the atomicity +// guarantee: when the staged publish aborts mid-transaction (summary +// resolution or event insert), the archive keeps the complete prior +// content — never a partial rewrite. +func TestCodexStagedPublishFailureKeepsPriorContent(t *testing.T) { + const uuid = "019eb791-cf7d-75c1-8439-9ed74c122b05" + root := t.TempDir() + day := filepath.Join(root, "2024", "01", "01") + require.NoError(t, os.MkdirAll(day, 0o755)) + path := filepath.Join( + day, "rollout-2024-01-01T10-00-00-"+uuid+".jsonl", + ) + require.NoError(t, os.WriteFile( + path, []byte(codexParityTranscript(uuid)), 0o644, + )) + cfg := parser.ProviderConfig{Roots: []string{root}, Machine: "local"} + provider, ok := parser.NewProvider(parser.AgentCodex, cfg) + require.True(t, ok) + source, found, err := provider.FindSource( + context.Background(), parser.FindSourceRequest{ + FullSessionID: "codex:" + uuid, + }, + ) + require.NoError(t, err) + require.True(t, found) + + // Prior content via the collecting path (the complete old version). + database := openTestDB(t) + legacySink := parser.NewCodexCollectingSink(0) + legacySess, legacyMsgs, _, _, _, _, err := + parser.ParseCodexSessionStreaming(cfg, source, legacySink) + require.NoError(t, err) + require.NotNil(t, legacySess) + row := db.Session{ + ID: legacySess.ID, + Project: legacySess.Project, + Machine: legacySess.Machine, + Agent: string(legacySess.Agent), + MessageCount: legacySess.MessageCount, + UserMessageCount: legacySess.UserMessageCount, + } + require.NoError(t, database.UpsertSession(row)) + dbMsgs := toDBMessages(pendingWrite{ + sess: *legacySess, msgs: legacyMsgs, + }, nil) + update, findings := computeSignalsAndSecrets(row, dbMsgs) + require.NoError(t, database.ReplaceSessionContent( + row.ID, dbMsgs, update, findings, + )) + before, err := database.GetAllMessages(context.Background(), row.ID) + require.NoError(t, err) + require.NotEmpty(t, before) + + // A failed staged publish must leave the prior rows untouched. The + // summary-resolution failure aborts while the transaction resolves + // per-call summaries; the event-insert failure aborts after messages + // and tool calls were staged into the same transaction. Both must + // roll back to the complete prior content. + for _, tc := range []struct { + name string + resolve bool + events bool + }{ + {name: "summary resolution failure", resolve: true}, + {name: "event insert failure", events: true}, + } { + t.Run(tc.name, func(t *testing.T) { + staged, err := newCodexStagingSink("", map[string]bool{}) + require.NoError(t, err) + failing := &failingStagedResults{ + codexStagingSink: staged, + failResolve: tc.resolve, + failEvents: tc.events, + } + t.Cleanup(func() { require.NoError(t, failing.Close()) }) + stagedSess, stagedMsgs, _, _, _, _, err := + parser.ParseCodexSessionStreaming(cfg, source, failing) + require.NoError(t, err) + require.NotNil(t, stagedSess) + stagedDBMsgs := toDBMessages(pendingWrite{ + sess: *stagedSess, msgs: stagedMsgs, + }, nil) + positions := stagedToolCallPositions(stagedDBMsgs) + err = database.ReplaceSessionContentStaged( + context.Background(), row.ID, stagedDBMsgs, failing, + map[string]bool{}, + func(verdicts map[string]bool) ( + db.SessionSignalUpdate, []db.SecretFinding, error, + ) { + update, findings := + computeSignalsAndSecretsWithContentFailures( + row, stagedDBMsgs, verdicts, + ) + combined := append( + append([]db.SecretFinding(nil), findings...), + failing.Findings(row.ID, positions)..., + ) + update.SecretLeakCount = + definiteFindingCount(combined) + return update, combined, nil + }, + ) + require.Error(t, err, + "the injected failure must abort the publish") + after, err := database.GetAllMessages( + context.Background(), row.ID, + ) + require.NoError(t, err) + require.Equal(t, before, after, + "aborted staged publish must keep the prior content") + }) + } + + // After two aborted publishes on the same single-connection writer + // pool, a successful staged publish must still work: the ATTACH is + // torn down after every transaction, so no stale codex_staging schema + // can collide with the next publish. + staged, err := newCodexStagingSink("", map[string]bool{}) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, staged.Close()) }) + stagedSess, stagedMsgs, _, _, _, _, err := + parser.ParseCodexSessionStreaming(cfg, source, staged) + require.NoError(t, err) + require.NotNil(t, stagedSess) + stagedDBMsgs := toDBMessages(pendingWrite{ + sess: *stagedSess, msgs: stagedMsgs, + }, nil) + positions := stagedToolCallPositions(stagedDBMsgs) + require.NoError(t, database.ReplaceSessionContentStaged( + context.Background(), row.ID, stagedDBMsgs, staged, + map[string]bool{}, + func(verdicts map[string]bool) ( + db.SessionSignalUpdate, []db.SecretFinding, error, + ) { + update, findings := + computeSignalsAndSecretsWithContentFailures( + row, stagedDBMsgs, verdicts, + ) + combined := append( + append([]db.SecretFinding(nil), findings...), + staged.Findings(row.ID, positions)..., + ) + update.SecretLeakCount = definiteFindingCount(combined) + return update, combined, nil + }, + )) + after, err := database.GetAllMessages(context.Background(), row.ID) + require.NoError(t, err) + require.Equal(t, before, after, + "successful staged publish after aborts must match the legacy projection") +} + +// FuzzCodexStagedParityWithCollecting asserts parser-level parity between +// the collecting and staging paths over arbitrary transcript bodies. The +// fuzzed bytes follow a fixed valid session meta so both paths always +// face the same session shape. +// TestCodexStagedScratchFailureIsSticky pins the failure contract: a +// scratch write failure must stick to the sink, fail the parse outcome +// and the publish, and never silently commit an archive missing tool +// outputs. The scratch connection is poisoned by closing it under the +// sink, which is the same failure shape a disk-full or I/O error takes. +func TestCodexStagedScratchFailureIsSticky(t *testing.T) { + const uuid = "019eb791-cf7d-75c1-8439-9ed74c122b05" + root := t.TempDir() + day := filepath.Join(root, "2024", "01", "01") + require.NoError(t, os.MkdirAll(day, 0o755)) + path := filepath.Join( + day, "rollout-2024-01-01T10-00-00-"+uuid+".jsonl", + ) + require.NoError(t, os.WriteFile( + path, []byte(codexParityTranscript(uuid)), 0o644, + )) + cfg := parser.ProviderConfig{Roots: []string{root}, Machine: "local"} + provider, ok := parser.NewProvider(parser.AgentCodex, cfg) + require.True(t, ok) + source, found, err := provider.FindSource( + context.Background(), parser.FindSourceRequest{ + FullSessionID: "codex:" + uuid, + }, + ) + require.NoError(t, err) + require.True(t, found) + + staged, err := newCodexStagingSink("", map[string]bool{}) + require.NoError(t, err) + t.Cleanup(func() { _ = staged.Close() }) + // Poison the scratch so every staged write fails. + require.NoError(t, staged.scratch.Close()) + + _, _, _, _, _, _, err = parser.ParseCodexSessionStreaming( + cfg, source, staged, + ) + require.NoError(t, err, "the parser itself completes; the sink fails") + require.Error(t, staged.Err()) + + // The outcome wrapper must surface the sticky error so the engine + // treats the parse as failed and keeps prior archive content. + _, err = stagedCodexParseOutcome(cfg, source, parser.SourceFingerprint{}, staged) + require.Error(t, err) + require.ErrorContains(t, err, "codex staging") +} + +func FuzzCodexStagedParityWithCollecting(f *testing.F) { + const uuid = "019eb791-cf7d-75c1-8439-9ed74c122b05" + f.Add([]byte(testjsonl.JoinJSONL( + testjsonl.CodexMsgJSON( + "user", "hi", "2024-01-01T10:00:02Z", + ), + testjsonl.CodexFunctionCallWithCallIDJSON( + "exec_command", "call_x", nil, "2024-01-01T10:00:03Z", + ), + testjsonl.CodexFunctionCallOutputJSON( + "call_x", "ok", "2024-01-01T10:00:04Z", + ), + ))) + f.Add([]byte(`{"timestamp":"2024-01-01T10:00:01Z","type":"event_msg","payload":{"type":"task_started"}}`)) + f.Fuzz(func(t *testing.T, data []byte) { + if len(data) > 256<<10 { + t.Skip() + } + // Cap the line count: both paths share the collecting sink's + // O(n) insertion per orphan message, so tens of thousands of + // tiny lines stall iterations without distinguishing the two + // paths (the slowdown is identical on both). + if strings.Count(string(data), "\n") > 2000 { + t.Skip() + } + content := testjsonl.JoinJSONL( + testjsonl.CodexSessionMetaJSON( + uuid, "/w", "codex_cli_rs", "2024-01-01T10:00:00Z", + ), + string(data), + ) + root := t.TempDir() + day := filepath.Join(root, "2024", "01", "01") + require.NoError(t, os.MkdirAll(day, 0o755)) + path := filepath.Join( + day, "rollout-2024-01-01T10-00-00-"+uuid+".jsonl", + ) + require.NoError(t, os.WriteFile(path, []byte(content), 0o644)) + cfg := parser.ProviderConfig{Roots: []string{root}, Machine: "local"} + provider, ok := parser.NewProvider(parser.AgentCodex, cfg) + if !ok { + t.Skip() + } + source, found, err := provider.FindSource( + context.Background(), parser.FindSourceRequest{ + FullSessionID: "codex:" + uuid, + }, + ) + if err != nil || !found { + t.Skip() + } + + legacy := parser.NewCodexCollectingSink(0) + sessL, msgsL, curL, hashL, anchorL, retryL, errL := + parser.ParseCodexSessionStreaming(cfg, source, legacy) + staged, err := newCodexStagingSink("", map[string]bool{}) + require.NoError(t, err) + sessS, msgsS, curS, hashS, anchorS, retryS, errS := + parser.ParseCodexSessionStreaming(cfg, source, staged) + require.NoError(t, staged.Close()) + + if (errL == nil) != (errS == nil) { + t.Fatalf("error parity: legacy=%v staged=%v", errL, errS) + } + if errL != nil { + return + } + require.Equal(t, retryL, retryS) + if sessL == nil || sessS == nil { + require.True(t, sessL == nil && sessS == nil) + return + } + require.Equal(t, sessL.ID, sessS.ID) + require.Equal(t, sessL.MessageCount, sessS.MessageCount) + require.Equal(t, curL, curS) + require.Equal(t, hashL, hashS) + require.Equal(t, anchorL, anchorS) + require.Len(t, msgsS, len(msgsL)) + for i := range msgsL { + require.Equal(t, msgsL[i].Role, msgsS[i].Role) + require.Equal(t, msgsL[i].Content, msgsS[i].Content) + require.Len(t, msgsS[i].ToolCalls, len(msgsL[i].ToolCalls)) + for j := range msgsL[i].ToolCalls { + lc, sc := msgsL[i].ToolCalls[j], msgsS[i].ToolCalls[j] + require.Equal(t, lc.ToolUseID, sc.ToolUseID) + require.Equal(t, lc.Category, sc.Category) + require.Len(t, sc.ResultEvents, len(lc.ResultEvents)) + for k := range lc.ResultEvents { + le, se := lc.ResultEvents[k], sc.ResultEvents[k] + require.Equal(t, le.Status, se.Status) + require.Equal(t, le.Source, se.Source) + require.Equal(t, le.AgentID, se.AgentID) + if le.Content != "" { + require.NotEqual(t, le.Content, se.Content, + "staged events carry placeholders") + } + } + } + } + }) +} diff --git a/internal/sync/codex_staging_test.go b/internal/sync/codex_staging_test.go new file mode 100644 index 000000000..7a5784db7 --- /dev/null +++ b/internal/sync/codex_staging_test.go @@ -0,0 +1,932 @@ +package sync + +import ( + "context" + "os" + "path/filepath" + "runtime/debug" + "sort" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/parser" + "go.kenn.io/agentsview/internal/testjsonl" +) + +func TestDrainResultsReleasesStagedScratchAndGCGuard(t *testing.T) { + dir := t.TempDir() + sink, err := newCodexStagingSink(dir, nil) + require.NoError(t, err) + + // SetGCPercent(-1) disables GC rather than querying it on this + // toolchain, so drive the restore check with an explicit baseline and + // read each transition from SetGCPercent's return value. + debug.SetGCPercent(200) + t.Cleanup(func() { debug.SetGCPercent(100) }) + guard := beginStagedColdSync() + t.Cleanup(guard) + require.Equal(t, stagedColdSyncGCPercent, + debug.SetGCPercent(stagedColdSyncGCPercent), + "the staged parse must lower the GC target") + + results := make(chan syncJob, 1) + results <- syncJob{processResult: processResult{ + staged: sink, + stagedGCRelease: guard, + }} + drainResults(results, 1) + + require.Equal(t, 200, debug.SetGCPercent(200), + "draining must restore the process GC target") + entries, err := os.ReadDir(dir) + require.NoError(t, err) + assert.Empty(t, entries, + "draining must remove the staged scratch file") +} + +func TestCodexStagingBlockedContentNeverEntersScratch(t *testing.T) { + dir := t.TempDir() + sink, err := newCodexStagingSink(dir, map[string]bool{"Bash": true}) + require.NoError(t, err) + defer func() { require.NoError(t, sink.Close()) }() + + sink.AppendMessage(parser.ParsedMessage{ToolCalls: []parser.ParsedToolCall{{ + ToolUseID: "call_secret", + ToolName: "exec_command", + Category: "Bash", + }}}) + const secret = "AKIA7QHWN2DKR4FYPLJM blocked payload" + sink.AppendToolResultEvent("call_secret", nil, parser.ParsedToolResultEvent{ + ToolUseID: "call_secret", + Source: "function_call_output", + Content: secret, + }) + require.NoError(t, sink.Err()) + + var content string + var length, blanked int + require.NoError(t, sink.scratch.QueryRow( + `SELECT content, content_length, blanked FROM stage_events LIMIT 1`, + ).Scan(&content, &length, &blanked)) + require.Empty(t, content) + require.Equal(t, len(secret), length) + require.Equal(t, 1, blanked) + + raw, err := os.ReadFile(sink.path) + require.NoError(t, err) + require.NotContains(t, string(raw), secret, + "blocked raw content must not be recoverable from scratch storage") +} + +// TestCodexStagingDropsOrphanResultEvents pins the staged sink's +// bounded-memory contract for tool-result events whose call never +// registered a message-model position. parser.ParseResult carries no +// ToolCallUpdates field, so every full-parse consumer discards such +// events; the staged sink must not retain their content on the way to +// being discarded, since that content is an uncloned reference into the +// source line's backing buffer and can be arbitrarily large. +func TestCodexStagingDropsOrphanResultEvents(t *testing.T) { + dir := t.TempDir() + sink, err := newCodexStagingSink(dir, nil) + require.NoError(t, err) + defer func() { require.NoError(t, sink.Close()) }() + + large := strings.Repeat("orphan output ", 1000) + sink.AppendToolResultEvent("call_never_registered", nil, + parser.ParsedToolResultEvent{ + ToolUseID: "call_never_registered", + Source: "function_call_output", + Content: large, + }, + ) + require.NoError(t, sink.Err()) + + assert.Empty(t, sink.ToolCallUpdates(), + "an orphan event must not be retained in the collecting "+ + "sink's toolCallUpdates, which no full-parse consumer reads") + + var rowCount int + require.NoError(t, sink.scratch.QueryRow( + `SELECT COUNT(*) FROM stage_events`, + ).Scan(&rowCount)) + assert.Zero(t, rowCount, "an orphan event must not be staged either") +} + +func TestStagedCodexParseOutcomeCopiesFingerprintHash(t *testing.T) { + const uuid = "019eb791-cf7d-75c1-8439-9ed74c122b04" + root := t.TempDir() + day := filepath.Join(root, "2024", "01", "01") + require.NoError(t, os.MkdirAll(day, 0o755)) + path := filepath.Join( + day, "rollout-2024-01-01T10-00-00-"+uuid+".jsonl", + ) + require.NoError(t, os.WriteFile(path, []byte(testjsonl.JoinJSONL( + testjsonl.CodexSessionMetaJSON( + uuid, "/tmp", "user", "2024-01-01T10:00:00Z", + ), + testjsonl.CodexMsgJSON("user", "hi", "2024-01-01T10:00:01Z"), + )), 0o644)) + + cfg := parser.ProviderConfig{Roots: []string{root}, Machine: "local"} + provider, ok := parser.NewProvider(parser.AgentCodex, cfg) + require.True(t, ok) + source, found, err := provider.FindSource( + context.Background(), parser.FindSourceRequest{ + FullSessionID: "codex:" + uuid, + }, + ) + require.NoError(t, err) + require.True(t, found) + + staged, err := newCodexStagingSink("", map[string]bool{}) + require.NoError(t, err) + t.Cleanup(func() { _ = staged.Close() }) + + outcome, err := stagedCodexParseOutcome( + cfg, source, + parser.SourceFingerprint{Hash: "sha256:abc"}, + staged, + ) + require.NoError(t, err) + require.Len(t, outcome.Results, 1) + assert.Equal(t, "sha256:abc", + outcome.Results[0].Result.Session.File.Hash, + "the staged path must persist the fingerprint hash the collecting path would") +} + +func TestSyncSingleSessionStagedPublishesRealContent(t *testing.T) { + const uuid = "019eb791-cf7d-75c1-8439-9ed74c122c11" + root := writeCodexParityRoot(t, uuid) + stagingDir := t.TempDir() + database := openTestDB(t) + engine := NewEngine(database, EngineConfig{ + AgentDirs: map[parser.AgentType][]string{ + parser.AgentCodex: {root}, + }, + Machine: "local", + StagedCodexParseMinBytes: 1, + CodexStagingDir: stagingDir, + }) + t.Cleanup(engine.Close) + sessionID := "codex:" + uuid + + require.NoError(t, engine.SyncSingleSession(sessionID)) + msgs, err := database.GetAllMessages( + context.Background(), sessionID, + ) + require.NoError(t, err) + require.NotEmpty(t, msgs) + + found := false + for _, m := range msgs { + for _, tc := range m.ToolCalls { + if tc.ToolUseID != "call_a" || len(tc.ResultEvents) == 0 { + continue + } + found = true + require.NotContains(t, tc.ResultEvents[0].Content, "staged:", + "the single-session path must publish real output, "+ + "not staged placeholders") + require.Contains(t, tc.ResultEvents[0].Content, + "build passed") + } + } + require.True(t, found, "call_a output must be stored") + + entries, err := os.ReadDir(stagingDir) + require.NoError(t, err) + assert.Empty(t, entries, + "the single-session path must release its scratch file") +} + +func TestParseDiffLargeCodexDoesNotStagePlaceholders(t *testing.T) { + const uuid = "019eb791-cf7d-75c1-8439-9ed74c122c12" + root := writeCodexParityRoot(t, uuid) + database := openTestDB(t) + engine := NewEngine(database, EngineConfig{ + AgentDirs: map[parser.AgentType][]string{ + parser.AgentCodex: {root}, + }, + Machine: "local", + StagedCodexParseMinBytes: 1, + }) + t.Cleanup(engine.Close) + require.Equal(t, 1, engine.SyncAll(t.Context(), nil).Synced) + + diff := NewDiffEngine(database, EngineConfig{ + AgentDirs: map[parser.AgentType][]string{ + parser.AgentCodex: {root}, + }, + Machine: "local", + }) + t.Cleanup(diff.Close) + report, err := diff.ParseDiff(t.Context(), ParseDiffOptions{ + Agents: []parser.AgentType{parser.AgentCodex}, + }) + require.NoError(t, err) + require.Zero(t, report.Totals.Changed, + "parse-diff must compare real content, not staged placeholders") +} + +// TestCodexStreamingParseParityWithLegacy drives one Codex transcript +// through both the collecting parse (legacy full write) and the staging +// parse (scratch-backed write), then compares the stored message, tool +// call, and result-event projections field by field, plus findings and +// status- and content-driven signals. +func TestCodexStreamingParseParityWithLegacy(t *testing.T) { + const uuid = "019eb791-cf7d-75c1-8439-9ed74c122b05" + assertCodexStagedParity(t, uuid, codexParityTranscript(uuid)) + + // The base fixture pins the absolute classification: call_b fails by + // status and call_c by content heuristics, so both paths must see + // exactly those two failures. + root := t.TempDir() + day := filepath.Join(root, "2024", "01", "01") + require.NoError(t, os.MkdirAll(day, 0o755)) + path := filepath.Join( + day, "rollout-2024-01-01T10-00-00-"+uuid+".jsonl", + ) + require.NoError(t, os.WriteFile( + path, []byte(codexParityTranscript(uuid)), 0o644, + )) + database := openTestDB(t) + engine := NewEngine(database, EngineConfig{ + AgentDirs: map[parser.AgentType][]string{ + parser.AgentCodex: {root}, + }, + Machine: "local", + }) + t.Cleanup(engine.Close) + stats := engine.SyncAll(t.Context(), nil) + require.Zero(t, stats.Failed) + require.Equal(t, 1, stats.Synced) + sess, err := database.GetSessionFull(t.Context(), "codex:"+uuid) + require.NoError(t, err) + require.NotNil(t, sess) + require.Equal(t, 2, sess.ToolFailureSignalCount) + require.Equal(t, 2, sess.ConsecutiveFailureMax) + require.Equal(t, 2, sess.FinalFailureStreak) + require.Equal(t, 1, sess.SecretLeakCount) +} + +// codexParityTranscript is the deterministic dual-path fixture: a session +// meta, one token-counted turn with a secret-bearing output, a +// status-errored output, and a status-less content-failure output. +// writeCodexParityRoot writes the dual-path fixture into a fresh codex +// session root and returns the root path. +func writeCodexParityRoot(t *testing.T, uuid string) string { + t.Helper() + return writeCodexTranscriptRoot(t, uuid, codexParityTranscript(uuid)) +} + +func writeCodexTranscriptRoot(t *testing.T, uuid, transcript string) string { + t.Helper() + root := t.TempDir() + day := filepath.Join(root, "2024", "01", "01") + require.NoError(t, os.MkdirAll(day, 0o755)) + path := filepath.Join( + day, "rollout-2024-01-01T10-00-00-"+uuid+".jsonl", + ) + require.NoError(t, os.WriteFile( + path, []byte(transcript), 0o644, + )) + return root +} + +// syncCodexParityEngine builds an engine over the fixture root with the +// given staged threshold and runs one cold sync. +func syncCodexParityEngine( + t *testing.T, database *db.DB, root string, stagedMin int64, +) { + t.Helper() + engine := NewEngine(database, EngineConfig{ + AgentDirs: map[parser.AgentType][]string{ + parser.AgentCodex: {root}, + }, + Machine: "local", + StagedCodexParseMinBytes: stagedMin, + }) + t.Cleanup(engine.Close) + stats := engine.SyncAll(t.Context(), nil) + require.Zero(t, stats.Failed) + require.Equal(t, 1, stats.Synced) +} + +func codexParityTranscript(uuid string) string { + return testjsonl.JoinJSONL( + testjsonl.CodexSessionMetaJSON( + uuid, "/workspace/project-a", "codex_cli_rs", + "2024-01-01T10:00:00Z", + ), + testjsonl.CodexTurnContextJSON( + "gpt-5.4", "2024-01-01T10:00:01Z", + ), + testjsonl.CodexMsgJSON("user", "run the suite", "2024-01-01T10:00:02Z"), + testjsonl.CodexMsgJSON( + "assistant", "running", "2024-01-01T10:00:03Z", + ), + testjsonl.CodexFunctionCallWithCallIDJSON( + "exec_command", "call_a", nil, "2024-01-01T10:00:04Z", + ), + testjsonl.CodexFunctionCallOutputJSON( + "call_a", + "build passed AKIA7QHWN2DKR4FYPLJM", + "2024-01-01T10:00:05Z", + ), + testjsonl.CodexTokenCountJSON( + "2024-01-01T10:00:06Z", 120, 40, 80, + ), + testjsonl.CodexMsgJSON("user", "again", "2024-01-01T10:00:07Z"), + testjsonl.CodexFunctionCallWithCallIDJSON( + "exec_command", "call_b", nil, "2024-01-01T10:00:08Z", + ), + // A status-carrying output keeps the failure classification + // status-driven: the staged in-memory model omits event content + // (content heuristics arrive with the streaming signal reducer), + // so parity here pins the status-based path. + `{"timestamp":"2024-01-01T10:00:09Z","type":"response_item","payload":{"type":"custom_tool_call_output","call_id":"call_b","status":"errored","output":[{"type":"input_text","text":"command not found"}]}}`, + // A status-less output whose failure lives only in the content + // heuristics: the streaming reducer must classify it identically. + testjsonl.CodexMsgJSON("user", "third", "2024-01-01T10:00:10Z"), + testjsonl.CodexFunctionCallWithCallIDJSON( + "exec_command", "call_c", nil, "2024-01-01T10:00:11Z", + ), + testjsonl.CodexFunctionCallOutputJSON( + "call_c", + "bash: python3: command not found", + "2024-01-01T10:00:12Z", + ), + ) +} + +// TestCodexEngineStagedSyncParity syncs the same transcript through two +// real engines — one on the collecting path, one on the staged streaming +// path (threshold lowered to a byte) — and asserts the stored projections +// match exactly. This is the default-CI coverage for the staged wiring +// that the macro gates cannot provide. +func TestCodexEngineStagedSyncParity(t *testing.T) { + const uuid = "019eb791-cf7d-75c1-8439-9ed74c122b05" + legacyDB := openTestDB(t) + stagedDB := openTestDB(t) + syncCodexParityEngine( + t, legacyDB, writeCodexParityRoot(t, uuid), 0, + ) + syncCodexParityEngine( + t, stagedDB, writeCodexParityRoot(t, uuid), 1, + ) + sessionID := "codex:" + uuid + + msgsL, err := legacyDB.GetAllMessages(t.Context(), sessionID) + require.NoError(t, err) + msgsS, err := stagedDB.GetAllMessages(t.Context(), sessionID) + require.NoError(t, err) + require.Equal(t, msgsL, msgsS, + "staged engine sync must match the collecting projection") + + sessL, err := legacyDB.GetSessionFull(t.Context(), sessionID) + require.NoError(t, err) + sessS, err := stagedDB.GetSessionFull(t.Context(), sessionID) + require.NoError(t, err) + require.Equal(t, sessL.ToolFailureSignalCount, + sessS.ToolFailureSignalCount) + require.Equal(t, sessL.ConsecutiveFailureMax, + sessS.ConsecutiveFailureMax) + require.Equal(t, sessL.FinalFailureStreak, sessS.FinalFailureStreak) + require.Equal(t, sessL.SecretLeakCount, sessS.SecretLeakCount) + + findingsL, err := legacyDB.SessionSecretFindings( + t.Context(), sessionID, + ) + require.NoError(t, err) + findingsS, err := stagedDB.SessionSecretFindings( + t.Context(), sessionID, + ) + require.NoError(t, err) + sortFindings(findingsL) + sortFindings(findingsS) + require.Equal(t, len(findingsL), len(findingsS)) + for i := range findingsL { + require.Equal(t, findingsL[i].RuleName, findingsS[i].RuleName) + require.Equal(t, findingsL[i].MessageOrdinal, + findingsS[i].MessageOrdinal) + require.Equal(t, findingsL[i].EventIndex, findingsS[i].EventIndex) + } +} + +// TestCodexEngineStagedSanitizesToolResultContent protects the central +// persistence contract at the staged boundary. The expected string is a +// literal oracle: NUL, ESC, and C1 controls are removed while printable bytes +// remain, in both the event row and its denormalized tool-call summary. +func TestCodexEngineStagedSanitizesToolResultContent(t *testing.T) { + const ( + uuid = "019eb791-cf7d-75c1-8439-9ed74c122b11" + rawOutput = "before\x00after\x1b[31mred\u0085done" + wantOutput = "beforeafter[31mreddone" + ) + transcript := testjsonl.JoinJSONL( + testjsonl.CodexSessionMetaJSON( + uuid, "/workspace/project-a", "codex_cli_rs", + "2024-01-01T10:00:00Z", + ), + testjsonl.CodexMsgJSON( + "user", "run it", "2024-01-01T10:00:01Z", + ), + testjsonl.CodexFunctionCallWithCallIDJSON( + "exec_command", "call_a", nil, "2024-01-01T10:00:02Z", + ), + testjsonl.CodexFunctionCallOutputJSON( + "call_a", rawOutput, "2024-01-01T10:00:03Z", + ), + // Exact raw duplicates collapse before sanitization, while an event + // differing only by stripped controls remains distinct. This pins the + // collecting path's existing identity contract at the staged boundary. + testjsonl.CodexFunctionCallOutputJSON( + "call_a", rawOutput, "2024-01-01T10:00:04Z", + ), + testjsonl.CodexFunctionCallOutputJSON( + "call_a", wantOutput, "2024-01-01T10:00:05Z", + ), + ) + + legacyDB := openTestDB(t) + stagedDB := openTestDB(t) + syncCodexParityEngine( + t, legacyDB, writeCodexTranscriptRoot(t, uuid, transcript), 0, + ) + syncCodexParityEngine( + t, stagedDB, writeCodexTranscriptRoot(t, uuid, transcript), 1, + ) + + assertStored := func(t *testing.T, database *db.DB) []db.Message { + t.Helper() + msgs, err := database.GetAllMessages(t.Context(), "codex:"+uuid) + require.NoError(t, err) + var calls []db.ToolCall + for _, msg := range msgs { + calls = append(calls, msg.ToolCalls...) + } + require.Len(t, calls, 1) + assert.Equal(t, wantOutput, calls[0].ResultContent) + assert.Equal(t, len(wantOutput), calls[0].ResultContentLength) + require.Len(t, calls[0].ResultEvents, 2) + for _, event := range calls[0].ResultEvents { + assert.Equal(t, wantOutput, event.Content) + assert.Equal(t, len(wantOutput), event.ContentLength) + } + return msgs + } + + legacyMsgs := assertStored(t, legacyDB) + stagedMsgs := assertStored(t, stagedDB) + assert.Equal(t, legacyMsgs, stagedMsgs) +} + +// TestCodexEngineStagedSubagentEventIDPrefix pins the staged publish path's +// event-level id-prefix contract for a remote (IDPrefix-configured) sync. +// The collecting path prefixes a subagent result event's SubagentSessionID +// in memory through applyRemoteRewrites before the write; the staged path +// inserts straight from scratch storage and must apply the same prefix at +// publish time, or a large remote Codex import would persist an unprefixed +// native id that cannot resolve to the session it actually belongs to. +func TestCodexEngineStagedSubagentEventIDPrefix(t *testing.T) { + const uuid = "019eb791-cf7d-75c1-8439-9ed74c122b13" + transcript := testjsonl.JoinJSONL( + testjsonl.CodexSessionMetaJSON( + uuid, "/workspace/project-a", "codex_cli_rs", + "2024-01-01T10:00:00Z", + ), + testjsonl.CodexMsgJSON( + "user", "wait for the subagent", "2024-01-01T10:00:01Z", + ), + testjsonl.CodexFunctionCallWithCallIDJSON( + "wait", "call_wait", map[string]any{ + "ids": []string{"agent-1"}, + }, "2024-01-01T10:00:02Z", + ), + testjsonl.CodexFunctionCallOutputJSON( + "call_wait", map[string]any{ + "status": map[string]any{ + "agent-1": map[string]any{ + "completed": "subagent finished", + }, + }, + }, "2024-01-01T10:00:03Z", + ), + ) + + syncPrefixed := func(t *testing.T, stagedMin int64) *db.DB { + t.Helper() + database := openTestDB(t) + engine := NewEngine(database, EngineConfig{ + AgentDirs: map[parser.AgentType][]string{ + parser.AgentCodex: {writeCodexTranscriptRoot(t, uuid, transcript)}, + }, + Machine: "remote", + IDPrefix: "remote-host~", + StagedCodexParseMinBytes: stagedMin, + }) + t.Cleanup(engine.Close) + stats := engine.SyncAll(t.Context(), nil) + require.Zero(t, stats.Failed) + require.Equal(t, 1, stats.Synced) + return database + } + + legacyDB := syncPrefixed(t, 0) + stagedDB := syncPrefixed(t, 1) + sessionID := "remote-host~codex:" + uuid + + assertPrefixed := func(t *testing.T, database *db.DB) db.ToolResultEvent { + t.Helper() + msgs, err := database.GetAllMessages(t.Context(), sessionID) + require.NoError(t, err) + var events []db.ToolResultEvent + for _, msg := range msgs { + for _, call := range msg.ToolCalls { + events = append(events, call.ResultEvents...) + } + } + require.Len(t, events, 1) + return events[0] + } + + legacyEvent := assertPrefixed(t, legacyDB) + stagedEvent := assertPrefixed(t, stagedDB) + const wantSubagentID = "remote-host~codex:agent-1" + assert.Equal(t, wantSubagentID, legacyEvent.SubagentSessionID) + assert.Equal(t, wantSubagentID, stagedEvent.SubagentSessionID, + "staged publish must prefix the event's subagent session id "+ + "the same way the collecting path's applyRemoteRewrites does") +} + +// TestCodexEngineResyncBulkStagedParity pins the bulk-write blocker: a +// full rebuild (ResyncAll) with the staged streaming path active must +// publish real tool outputs, never the staged placeholders, and must +// match the plain cold sync projection. +func TestCodexEngineResyncBulkStagedParity(t *testing.T) { + const uuid = "019eb791-cf7d-75c1-8439-9ed74c122b05" + sessionID := "codex:" + uuid + + legacyDB := openTestDB(t) + syncCodexParityEngine( + t, legacyDB, writeCodexParityRoot(t, uuid), 0, + ) + + database := openTestDB(t) + engine := NewEngine(database, EngineConfig{ + AgentDirs: map[parser.AgentType][]string{ + parser.AgentCodex: {writeCodexParityRoot(t, uuid)}, + }, + Machine: "local", + StagedCodexParseMinBytes: 1, + }) + t.Cleanup(engine.Close) + stats := engine.ResyncAll(t.Context(), nil) + require.False(t, stats.Aborted, "resync aborted: %v", stats.Warnings) + require.Zero(t, stats.Failed) + require.Equal(t, 1, stats.Synced) + + msgsL, err := legacyDB.GetAllMessages(t.Context(), sessionID) + require.NoError(t, err) + msgsS, err := database.GetAllMessages(t.Context(), sessionID) + require.NoError(t, err) + require.Equal(t, msgsL, msgsS, + "bulk resync must publish real tool outputs, not placeholders") + for _, m := range msgsS { + for _, tc := range m.ToolCalls { + for _, ev := range tc.ResultEvents { + require.NotContains(t, ev.Content, "staged:", + "bulk resync wrote a staging placeholder") + } + } + } +} + +// assertCodexStagedParity runs one transcript through the collecting and +// staging parse paths and asserts the parser projections and stored DB +// projections agree field by field. It is shared by the deterministic +// parity test and the perturbation table so every variant gets the full +// dual-path comparison. +func assertCodexStagedParity(t *testing.T, uuid, content string) { + t.Helper() + root := t.TempDir() + day := filepath.Join(root, "2024", "01", "01") + require.NoError(t, os.MkdirAll(day, 0o755)) + path := filepath.Join( + day, "rollout-2024-01-01T10-00-00-"+uuid+".jsonl", + ) + require.NoError(t, os.WriteFile(path, []byte(content), 0o644)) + + cfg := parser.ProviderConfig{ + Roots: []string{root}, + Machine: "local", + } + provider, ok := parser.NewProvider(parser.AgentCodex, cfg) + require.True(t, ok) + source, found, err := provider.FindSource( + context.Background(), parser.FindSourceRequest{ + FullSessionID: "codex:" + uuid, + }, + ) + require.NoError(t, err) + require.True(t, found) + + // Legacy collecting parse. + legacySink := parser.NewCodexCollectingSink(0) + legacySess, legacyMsgs, legacyCursor, legacyHash, legacyAnchor, legacyRetry, err := + parser.ParseCodexSessionStreaming(cfg, source, legacySink) + require.NoError(t, err) + require.NotNil(t, legacySess) + + // Staging parse. + stagedSink, err := newCodexStagingSink("", map[string]bool{}) + require.NoError(t, err) + defer func() { require.NoError(t, stagedSink.Close()) }() + stagedSess, stagedMsgs, stagedCursor, stagedHash, stagedAnchor, stagedRetry, err := + parser.ParseCodexSessionStreaming(cfg, source, stagedSink) + require.NoError(t, err) + require.NotNil(t, stagedSess) + + // Parser-level parity: metadata, cursor, hash state, anchor digest. + assert.Equal(t, legacyCursor, stagedCursor) + assert.Equal(t, legacyHash, stagedHash) + assert.Equal(t, legacyAnchor, stagedAnchor) + assert.Equal(t, legacyRetry, stagedRetry) + assert.Equal(t, legacySess.ID, stagedSess.ID) + assert.Equal(t, legacySess.MessageCount, stagedSess.MessageCount) + require.Len(t, stagedMsgs, len(legacyMsgs)) + for i := range legacyMsgs { + assert.Equal(t, legacyMsgs[i].Role, stagedMsgs[i].Role) + assert.Equal(t, legacyMsgs[i].Content, stagedMsgs[i].Content) + assert.Equal(t, legacyMsgs[i].HasToolUse, stagedMsgs[i].HasToolUse) + assert.Equal(t, legacyMsgs[i].ContextTokens, + stagedMsgs[i].ContextTokens) + require.Len(t, stagedMsgs[i].ToolCalls, + len(legacyMsgs[i].ToolCalls)) + for j := range legacyMsgs[i].ToolCalls { + lc, sc := legacyMsgs[i].ToolCalls[j], + stagedMsgs[i].ToolCalls[j] + assert.Equal(t, lc.ToolUseID, sc.ToolUseID) + assert.Equal(t, lc.Category, sc.Category) + require.Len(t, sc.ResultEvents, len(lc.ResultEvents)) + for k := range lc.ResultEvents { + le, se := lc.ResultEvents[k], sc.ResultEvents[k] + assert.Equal(t, le.AgentID, se.AgentID) + assert.Equal(t, le.Status, se.Status) + assert.Equal(t, le.Source, se.Source) + assert.NotEqual(t, le.Content, se.Content, + "staged events carry placeholders, never content") + } + } + } + + // Database projections. + dbLegacy := openTestDB(t) + dbStaged := openTestDB(t) + writeSessionRow := func(database *db.DB, sess *parser.ParsedSession) db.Session { + var started *string + if !sess.StartedAt.IsZero() { + s := sess.StartedAt.Format(time.RFC3339Nano) + started = &s + } + row := db.Session{ + ID: sess.ID, + Project: sess.Project, + Machine: sess.Machine, + Agent: string(sess.Agent), + FirstMessage: &sess.FirstMessage, + StartedAt: started, + MessageCount: sess.MessageCount, + UserMessageCount: sess.UserMessageCount, + IsAutomated: false, + } + require.NoError(t, database.UpsertSession(row)) + return row + } + rowL := writeSessionRow(dbLegacy, legacySess) + rowS := writeSessionRow(dbStaged, stagedSess) + + legacyDBMsgs := toDBMessages(pendingWrite{ + sess: *legacySess, msgs: legacyMsgs, + }, nil) + stagedDBMsgs := toDBMessages(pendingWrite{ + sess: *stagedSess, msgs: stagedMsgs, + }, nil) + + updateL, findingsL := computeSignalsAndSecrets(rowL, legacyDBMsgs) + require.NoError(t, dbLegacy.ReplaceSessionContent( + rowL.ID, legacyDBMsgs, updateL, findingsL, + )) + + positions := stagedToolCallPositions(stagedDBMsgs) + // The publish transaction resolves each summary once and runs this + // closure before commit, so the content-failure-aware signals and + // findings persist atomically with the rows they describe. + require.NoError(t, dbStaged.ReplaceSessionContentStaged( + context.Background(), rowS.ID, stagedDBMsgs, stagedSink, + map[string]bool{}, + func(verdicts map[string]bool) ( + db.SessionSignalUpdate, []db.SecretFinding, error, + ) { + update, findings := + computeSignalsAndSecretsWithContentFailures( + rowS, stagedDBMsgs, verdicts, + ) + combined := append( + append([]db.SecretFinding(nil), findings...), + stagedSink.Findings(rowS.ID, positions)..., + ) + update.SecretLeakCount = definiteFindingCount(combined) + return update, combined, nil + }, + )) + + msgsL, err := dbLegacy.GetAllMessages(context.Background(), rowL.ID) + require.NoError(t, err) + msgsS, err := dbStaged.GetAllMessages(context.Background(), rowS.ID) + require.NoError(t, err) + require.Len(t, msgsS, len(msgsL)) + for i := range msgsL { + require.Equal(t, msgsL[i].Ordinal, msgsS[i].Ordinal) + assert.Equal(t, msgsL[i].Role, msgsS[i].Role) + assert.Equal(t, msgsL[i].Content, msgsS[i].Content) + assert.Equal(t, msgsL[i].HasToolUse, msgsS[i].HasToolUse) + assert.Equal(t, msgsL[i].ContextTokens, msgsS[i].ContextTokens) + assert.Equal(t, msgsL[i].OutputTokens, msgsS[i].OutputTokens) + require.Len(t, msgsS[i].ToolCalls, len(msgsL[i].ToolCalls)) + for j := range msgsL[i].ToolCalls { + lc, sc := msgsL[i].ToolCalls[j], msgsS[i].ToolCalls[j] + assert.Equal(t, lc.ToolUseID, sc.ToolUseID) + assert.Equal(t, lc.Category, sc.Category) + assert.Equal(t, lc.InputJSON, sc.InputJSON) + assert.Equal(t, lc.ResultContent, sc.ResultContent, + "summaries must match byte for byte") + assert.Equal(t, lc.ResultContentLength, sc.ResultContentLength) + require.Len(t, sc.ResultEvents, len(lc.ResultEvents)) + for k := range lc.ResultEvents { + le, se := lc.ResultEvents[k], sc.ResultEvents[k] + assert.Equal(t, le.ToolUseID, se.ToolUseID) + assert.Equal(t, le.AgentID, se.AgentID) + assert.Equal(t, le.Source, se.Source) + assert.Equal(t, le.Status, se.Status) + assert.Equal(t, le.Content, se.Content) + assert.Equal(t, le.ContentLength, se.ContentLength) + assert.Equal(t, le.EventIndex, se.EventIndex) + } + } + } + + // Findings parity. + findingsStored, err := dbStaged.SessionSecretFindings( + context.Background(), rowS.ID, + ) + require.NoError(t, err) + findingsLegacy, err := dbLegacy.SessionSecretFindings( + context.Background(), rowL.ID, + ) + require.NoError(t, err) + sortFindings(findingsLegacy) + sortFindings(findingsStored) + require.Equal(t, len(findingsLegacy), len(findingsStored)) + for i := range findingsLegacy { + assert.Equal(t, findingsLegacy[i].RuleName, + findingsStored[i].RuleName) + assert.Equal(t, findingsLegacy[i].LocationKind, + findingsStored[i].LocationKind) + assert.Equal(t, findingsLegacy[i].MessageOrdinal, + findingsStored[i].MessageOrdinal) + assert.Equal(t, findingsLegacy[i].CallIndex, + findingsStored[i].CallIndex) + assert.Equal(t, findingsLegacy[i].EventIndex, + findingsStored[i].EventIndex) + assert.Equal(t, findingsLegacy[i].MatchStart, + findingsStored[i].MatchStart) + } + + // Signals parity: call_b fails by event status (kept in the staged + // model) and call_c by content heuristics (folded in through the + // streaming reducer), so both classification paths must agree. + sessL, err := dbLegacy.GetSessionFull(context.Background(), rowL.ID) + require.NoError(t, err) + sessS, err := dbStaged.GetSessionFull(context.Background(), rowS.ID) + require.NoError(t, err) + assert.Equal(t, sessL.ToolFailureSignalCount, + sessS.ToolFailureSignalCount) + assert.Equal(t, sessL.ConsecutiveFailureMax, + sessS.ConsecutiveFailureMax) + assert.Equal(t, sessL.FinalFailureStreak, sessS.FinalFailureStreak) + assert.Equal(t, sessL.SecretLeakCount, sessS.SecretLeakCount) +} + +// TestCodexStagedBlockedCategorySignalParity pins the blocked-category +// parity: a blocked Bash call whose status-less output contains a failure +// marker must not be classified as a content failure. The legacy path +// blanks the summary before computing signals, so the staged summary +// resolver must evaluate its verdict against empty content too. +func TestCodexStagedBlockedCategorySignalParity(t *testing.T) { + const uuid = "019eb791-cf7d-75c1-8439-9ed74c122b09" + blocked := map[string]bool{"Bash": true} + content := testjsonl.JoinJSONL( + testjsonl.CodexSessionMetaJSON( + uuid, "/workspace/project-a", "codex_cli_rs", + "2024-01-01T10:00:00Z", + ), + testjsonl.CodexMsgJSON("user", "run it", "2024-01-01T10:00:01Z"), + testjsonl.CodexFunctionCallWithCallIDJSON( + "exec_command", "call_a", nil, "2024-01-01T10:00:02Z", + ), + testjsonl.CodexFunctionCallOutputJSON( + "call_a", "command not found", "2024-01-01T10:00:03Z", + ), + ) + + root := t.TempDir() + day := filepath.Join(root, "2024", "01", "01") + require.NoError(t, os.MkdirAll(day, 0o755)) + path := filepath.Join( + day, "rollout-2024-01-01T10-00-00-"+uuid+".jsonl", + ) + require.NoError(t, os.WriteFile(path, []byte(content), 0o644)) + + cfg := parser.ProviderConfig{Roots: []string{root}, Machine: "local"} + provider, ok := parser.NewProvider(parser.AgentCodex, cfg) + require.True(t, ok) + source, found, err := provider.FindSource( + context.Background(), parser.FindSourceRequest{ + FullSessionID: "codex:" + uuid, + }, + ) + require.NoError(t, err) + require.True(t, found) + + // Legacy collecting parse: the blocked map blanks Bash content, so the + // status-less "command not found" output is not a content failure. + legacySink := parser.NewCodexCollectingSink(0) + legacySess, legacyMsgs, _, _, _, _, err := + parser.ParseCodexSessionStreaming(cfg, source, legacySink) + require.NoError(t, err) + require.NotNil(t, legacySess) + legacyDBMsgs := toDBMessages(pendingWrite{ + sess: *legacySess, msgs: legacyMsgs, + }, blocked) + legacyUpdate, _ := computeSignalsAndSecrets( + db.Session{ID: legacySess.ID}, legacyDBMsgs, + ) + + // Staging parse with the same blocked map; resolving summaries records + // the per-call content-failure verdicts the publish transaction folds. + stagedSink, err := newCodexStagingSink("", blocked) + require.NoError(t, err) + defer func() { require.NoError(t, stagedSink.Close()) }() + stagedSess, stagedMsgs, _, _, _, _, err := + parser.ParseCodexSessionStreaming(cfg, source, stagedSink) + require.NoError(t, err) + require.NotNil(t, stagedSess) + stagedDBMsgs := toDBMessages(pendingWrite{ + sess: *stagedSess, msgs: stagedMsgs, + }, blocked) + for _, m := range stagedDBMsgs { + for _, tc := range m.ToolCalls { + if tc.ToolUseID == "" { + continue + } + _, _, err := stagedSink.ResolveSummary( + context.Background(), tc.ToolUseID, + ) + require.NoError(t, err) + } + } + stagedUpdate, _ := computeSignalsAndSecretsWithContentFailures( + db.Session{ID: stagedSess.ID}, stagedDBMsgs, + stagedSink.ContentFailures(), + ) + + assert.Equal(t, legacyUpdate.ToolFailureSignalCount, + stagedUpdate.ToolFailureSignalCount, + "blocked-category failure signals must match the legacy path") + assert.Zero(t, legacyUpdate.ToolFailureSignalCount, + "a blocked output with a failure marker must never be a content failure") +} + +func sortFindings(findings []db.SecretFinding) { + sort.Slice(findings, func(i, j int) bool { + a, b := findings[i], findings[j] + if a.MessageOrdinal != b.MessageOrdinal { + return a.MessageOrdinal < b.MessageOrdinal + } + if a.LocationKind != b.LocationKind { + return a.LocationKind < b.LocationKind + } + if a.MatchStart != b.MatchStart { + return a.MatchStart < b.MatchStart + } + if a.MatchIndex != b.MatchIndex { + return a.MatchIndex < b.MatchIndex + } + return a.RuleName < b.RuleName + }) +} diff --git a/internal/sync/engine.go b/internal/sync/engine.go index 3f62ce570..3c6b850b5 100644 --- a/internal/sync/engine.go +++ b/internal/sync/engine.go @@ -437,6 +437,14 @@ type EngineConfig struct { DisabledAgents []parser.AgentType Machine string BlockedResultCategories []string + // StagedCodexParseMinBytes overrides the full-parse size above which a + // Codex source streams through the scratch staging path. Zero selects + // the default (stagedCodexParseMinBytes). Tests lower it so the staged + // wiring is covered by small fixtures in the default test build. + StagedCodexParseMinBytes int64 + // CodexStagingDir selects the directory for staged Codex scratch + // databases. Empty means the system temporary directory. + CodexStagingDir string // IncludeCwdPrefixes, when non-empty, restricts ingestion to // sessions whose working directory equals one of the prefixes // or lives underneath one. Sessions without a recorded cwd are @@ -532,7 +540,12 @@ type Engine struct { preserveAgents []parser.AgentType machine string blockedResultCategories map[string]bool - cwdFilter cwdPrefixFilter + // stagedCodexMin is the resolved full-parse size above which Codex + // sources take the staged streaming path. + stagedCodexMin int64 + // stagedCodexDir is the scratch directory for staged Codex parses. + stagedCodexDir string + cwdFilter cwdPrefixFilter // scanProtectedPaths, homeDir, and goos gate passive probing of macOS // TCC-protected locations. homeDir is empty when the home directory // cannot be resolved, which disables the gate rather than guessing. @@ -566,6 +579,11 @@ type Engine struct { skipHashKeys map[string]string s3CodexIndexMu gosync.Mutex s3CodexIndexCache map[string]s3CodexIndexSnapshot + // checkpointAudit, when set, bypasses the checkpoint stat-trust gate so + // the provider's full-source fingerprint verifies content. The periodic + // archive audit sets it to catch same-stat in-place rewrites that + // append-trust would otherwise keep stale. + checkpointAudit atomic.Bool // idPrefix and pathRewriter support remote sync: // prefix all session IDs to avoid collisions, rewrite // temp paths to "host:/remote/path" form. @@ -879,6 +897,21 @@ func NewEngine( maps.Copy(providerModes, cfg.ProviderMigrationModes) } + stagedCodexDir := cfg.CodexStagingDir + if stagedCodexDir == "" { + dbPath := database.Path() + if dbPath != "" && dbPath != ":memory:" { + stagedCodexDir = filepath.Join( + filepath.Dir(dbPath), "scratch", "codex", + ) + } + } + if stagedCodexDir != "" { + if err := prepareCodexStagingDir(stagedCodexDir); err != nil { + log.Printf("preparing codex staging directory: %v", err) + } + } + if cfg.ScanProtectedPaths { // Parsers extract project names by probing recorded cwds for git // roots; that guard is package-level because extraction runs deep @@ -901,6 +934,8 @@ func NewEngine( preserveAgents: disabledAgents, machine: cfg.Machine, blockedResultCategories: blockedCategorySet(cfg.BlockedResultCategories), + stagedCodexMin: stagedCodexMinBytes(cfg.StagedCodexParseMinBytes), + stagedCodexDir: stagedCodexDir, cwdFilter: newCwdPrefixFilter(cfg.IncludeCwdPrefixes), scanProtectedPaths: cfg.ScanProtectedPaths, homeDir: userHomeDirOrEmpty(), @@ -1537,6 +1572,18 @@ func (e *Engine) Machine() string { return e.machine } +// SetCheckpointAudit enables or disables the checkpoint-bypassing content +// audit. When enabled, checkpointed sources are re-hashed with the provider's +// full-source fingerprint instead of being trusted on stat, so same-stat +// in-place rewrites are detected and repaired. The periodic archive audit +// toggles this around its reconciliation pass. +func (e *Engine) SetCheckpointAudit(enabled bool) { + if e == nil { + return + } + e.checkpointAudit.Store(enabled) +} + type syncJob struct { processResult agent parser.AgentType @@ -1566,6 +1613,17 @@ func (j *syncJob) releaseRetention() { j.retentionLease = nil } +// releaseAll drops every parse-owned resource a discarded result holds: the +// retention lease bounds the parsed payload memory, and releaseStaged closes +// the scratch staging sink and restores the process GC target it lowered. +// Every collector discard path — including cancellation draining — must call +// this instead of releaseRetention alone, or a staged Codex result leaks its +// scratch database and pins the process GC percent at the staged value. +func (j *syncJob) releaseAll() { + j.releaseRetention() + j.releaseStaged() +} + func (j syncJob) skipCacheKey() string { return j.processResult.skipCacheKey(j.path) } @@ -9398,13 +9456,13 @@ func (e *Engine) retentionBudget() *parseRetentionBudget { return e.parseRetentionBudget } -// beginBulkRetentionPass installs the unthrottled bulk retention budget for +// beginBulkRetentionPass installs the byte-bounded bulk retention budget for // the duration of an archive-scale pass and returns the restore func the // caller must defer. Bulk passes (full sync, resync rebuild, remote import -// processing) run at full worker parallelism; the memory they retain is -// returned to the OS by the end-of-pass scavenge instead of being bounded -// per source. The caller holds syncMu, so no other pass can observe the -// switched budget. +// processing) use the same weighted byte admission as daemon passes: large +// sources run exclusively, small sources share the capacity, and batches +// flush on count or estimated bytes. The caller holds syncMu, so no other +// pass can observe the switched budget. func (e *Engine) beginBulkRetentionPass() func() { e.bulkRetentionOnce.Do(func() { if e.bulkRetentionBudget == nil { @@ -9463,7 +9521,10 @@ func (e *Engine) collectAndBatchWithOptions( } var pending []pendingWrite + var pendingBytes int64 var pendingLeases []*parseRetentionLease + var pendingStaged []*codexStagingSink + var pendingStagedGC []func() var pendingCacheWrites []skipCacheWrite baselineCacheWrites := make( map[machineSessionSource]map[string]skipCacheWrite, @@ -9676,6 +9737,8 @@ func (e *Engine) collectAndBatchWithOptions( } func() { defer releaseParseRetentionLeases(pendingLeases) + defer closeCodexStagingSinks(pendingStaged) + defer releaseStagedGCGuards(pendingStagedGC) completionCtx := ctx if !e.discardWritesOnCancel { completionCtx = context.WithoutCancel(ctx) @@ -9793,7 +9856,10 @@ func (e *Engine) collectAndBatchWithOptions( stats.messagesIndexed = progress.MessagesIndexed }() pending = pending[:0] + pendingBytes = 0 pendingLeases = pendingLeases[:0] + pendingStaged = pendingStaged[:0] + pendingStagedGC = pendingStagedGC[:0] pendingCacheWrites = pendingCacheWrites[:0] } @@ -9844,7 +9910,7 @@ func (e *Engine) collectAndBatchWithOptions( // ctx.Done() branch above. if ctx.Err() != nil { stats.Aborted = true - r.releaseRetention() + r.releaseAll() drainResults(results, total-i-1) goto flush } @@ -9857,7 +9923,7 @@ func (e *Engine) collectAndBatchWithOptions( e.cacheSkip(r.skipCacheKey(), r.mtime, r.sourceFingerprint) } log.Printf("sync error: %v", r.err) - r.releaseRetention() + r.releaseAll() continue } if len(r.excludedSessionIDs) > 0 || len(r.sourceMissingMembers) > 0 { @@ -9915,7 +9981,7 @@ func (e *Engine) collectAndBatchWithOptions( } progress.SessionsDone++ e.reportProgress(onProgress, progress) - r.releaseRetention() + r.releaseAll() continue } sourceAllowsParserExclusions := e.sourceAllowsParserExclusions( @@ -9941,7 +10007,7 @@ func (e *Engine) collectAndBatchWithOptions( log.Printf("list pre-write subagent children: %v", err) stats.RecordFailed() e.noteSQLiteContainerResult(r.path, false) - r.releaseRetention() + r.releaseAll() continue } // Persist affected IDs before any exclusion or replacement can @@ -9951,7 +10017,7 @@ func (e *Engine) collectAndBatchWithOptions( log.Printf("queue subagent parent repairs: %v", err) stats.RecordFailed() e.noteSQLiteContainerResult(r.path, false) - r.releaseRetention() + r.releaseAll() continue } atomicDAG := sourceRequiresAtomicDAGCompletion( @@ -9970,7 +10036,7 @@ func (e *Engine) collectAndBatchWithOptions( log.Printf("stage DAG source data versions: %v", err) stats.RecordFailed() e.noteSQLiteContainerResult(r.path, false) - r.releaseRetention() + r.releaseAll() continue } } @@ -9981,7 +10047,7 @@ func (e *Engine) collectAndBatchWithOptions( log.Printf("delete parser-excluded sessions: %v", err) stats.RecordFailed() e.noteSQLiteContainerResult(r.path, false) - r.releaseRetention() + r.releaseAll() continue } if len(excludedSessionIDs) > 0 { @@ -10009,7 +10075,7 @@ func (e *Engine) collectAndBatchWithOptions( ) stats.RecordFailed() e.noteSQLiteContainerResult(r.path, false) - r.releaseRetention() + r.releaseAll() continue } stats.sourceMissingArchiveMembers = append( @@ -10063,7 +10129,7 @@ func (e *Engine) collectAndBatchWithOptions( } progress.SessionsDone++ e.reportProgress(onProgress, progress) - r.releaseRetention() + r.releaseAll() continue } if r.cacheSkip { @@ -10121,7 +10187,7 @@ func (e *Engine) collectAndBatchWithOptions( } progress.SessionsDone++ e.reportProgress(onProgress, progress) - r.releaseRetention() + r.releaseAll() continue } @@ -10129,7 +10195,7 @@ func (e *Engine) collectAndBatchWithOptions( if err := e.writeIncremental(r.incremental); err != nil { log.Printf("%v", err) stats.RecordFailed() - r.releaseRetention() + r.releaseAll() continue } stats.RecordSynced(1) @@ -10142,7 +10208,7 @@ func (e *Engine) collectAndBatchWithOptions( r.incremental.msgs, ) stats.messagesIndexed = progress.MessagesIndexed - r.releaseRetention() + r.releaseAll() } else { sourceNeedsRetry := presenceProofWithheld for i, pr := range allowed { @@ -10152,6 +10218,10 @@ func (e *Engine) collectAndBatchWithOptions( sess: pr.Session, msgs: pr.Messages, usageEvents: pr.UsageEvents, + sourceBytes: r.sourceBytes, + checkpoint: pr.Checkpoint, + checkpointHashState: pr.CheckpointHashState, + checkpointAnchorDigest: pr.CheckpointAnchorDigest, needsRetry: sessionNeedsRetry || atomicDAG, forceReplace: r.forceReplace, baselineEligible: !sourceNeedsRetry, @@ -10177,7 +10247,11 @@ func (e *Engine) collectAndBatchWithOptions( pw.sourceCompletionEligible = !sourceNeedsRetry pw.promoteSourceOnComplete = atomicDAG } + if i == 0 { + pw.staged = r.staged + } pending = append(pending, pw) + pendingBytes += pw.sourceBytes if runtimeMetrics != nil { runtimeMetrics.pendingWrites(len(pending)) } @@ -10186,6 +10260,16 @@ func (e *Engine) collectAndBatchWithOptions( pendingLeases = append(pendingLeases, r.retentionLease) r.retentionLease = nil } + if r.staged != nil { + pendingStaged = append(pendingStaged, r.staged) + r.staged = nil + } + if r.stagedGCRelease != nil { + pendingStagedGC = append( + pendingStagedGC, r.stagedGCRelease, + ) + r.stagedGCRelease = nil + } if r.cacheAfterWrite && !sourceNeedsRetry { pendingCacheWrites = append(pendingCacheWrites, skipCacheWrite{ agent: r.agent, @@ -10194,7 +10278,8 @@ func (e *Engine) collectAndBatchWithOptions( sourceFingerprint: r.sourceFingerprint, }) } - if len(pending) >= batchSize || budget.underPressure() { + if len(pending) >= batchSize || budget.underPressure() || + pendingBytes >= parseBatchBytesLimit { flushPending() } // A Kiro SQLite store is discovered as one container source @@ -10482,7 +10567,7 @@ func (e *Engine) linkSubagentSessions(ctx context.Context) error { func drainResults(results <-chan syncJob, remaining int) { for range remaining { job := <-results - job.releaseRetention() + job.releaseAll() } } @@ -10490,13 +10575,20 @@ func drainResults(results <-chan syncJob, remaining int) { // incremental JSONL parse, used to partially update the // session row without overwriting unrelated columns. type incrementalUpdate struct { - sessionID string - project string - sourceProject string - machine string - cwd string - msgs []parser.ParsedMessage - links []parser.ClaudeSubagentLink + sessionID string + project string + sourceProject string + machine string + cwd string + msgs []parser.ParsedMessage + links []parser.ClaudeSubagentLink + toolCallUpdates []parser.ParsedToolCallUpdate + messageUsageUpdates []parser.ParsedMessageTokenUsageUpdate + // checkpoint is the machine-local parser checkpoint to persist in the + // same transaction as this incremental delta. nil keeps the existing + // checkpoint (or leaves none). + checkpoint *db.ParserCheckpoint + checkpointBlobs *db.ParserCheckpointBlobs endedAt time.Time terminationStatus *string msgCount int // total (old + new) @@ -10513,6 +10605,46 @@ type incrementalUpdate struct { providerStatHash *pendingProviderStatHash } +// hasSubstantiveUserMessage reports whether the delta contains a user +// message with non-whitespace content. Such deltas can change prompt-derived +// heuristics, which the incremental signal maintainer does not fold; they +// fall back to the debounced full recompute. +func (inc *incrementalUpdate) hasSubstantiveUserMessage() bool { + for _, m := range inc.msgs { + if m.Role == parser.RoleUser && + strings.TrimSpace(m.Content) != "" { + return true + } + } + return false +} + +// hasCompactBoundary reports whether the delta contains a compact-boundary +// message. Boundaries change the compaction detectors, which the incremental +// signal maintainer does not fold; they fall back to a full recompute. +func (inc *incrementalUpdate) hasCompactBoundary() bool { + for _, m := range inc.msgs { + if m.IsCompactBoundary { + return true + } + } + return false +} + +// hasResultSubagentLink reports whether the delta carries a subagent link +// with result content. Linked results update tool_calls.result_content +// outside ToolCallResultUpdates, so the incremental secret scan never sees +// them; maintenance must decline and let the debounced full recompute +// rescan the affected call instead of stamping it current. +func (inc *incrementalUpdate) hasResultSubagentLink() bool { + for _, link := range inc.links { + if link.HasResult && strings.TrimSpace(link.ResultContentRaw) != "" { + return true + } + } + return false +} + // sessionParseError is a per-session parse failure inside a shared // SQLite store (OpenCode, Zed, Kiro), where one file path fans out to // many sessions and a single bad payload must not fail the whole db. @@ -10534,6 +10666,10 @@ type sourceMissingMember struct { } type processResult struct { + // sourceBytes is the physical source size used to acquire the retention + // lease and to account this result against the pending write batch byte + // cap. Zero on lease-free skips. + sourceBytes int64 results []parser.ParseResult excludedSessionIDs []string // preservedSessionIDs are higher-ranked members omitted by a shared source; @@ -10635,6 +10771,34 @@ type processResult struct { sourceCwdStoredOK bool sourceCwdPath string sourceCwdAgent parser.AgentType + // staged carries the scratch staging sink for a Codex full parse that + // took the streaming path. The collector moves it onto the pending + // write (and into pendingStaged for release after the batch commits); + // every path that drops the result without writing must releaseStaged. + staged *codexStagingSink + // stagedGCRelease restores the process GC percent after this result's + // staged sink is released. nil when the result carries no sink. + stagedGCRelease func() +} + +// releaseStaged closes the result's staging sink, restores the GC percent +// window it opened, and clears the handles. It must be called exactly once +// on every processResult that carries a sink and is not moving the sink +// onto a pending write. +func (r *processResult) releaseStaged() { + if r.staged == nil && r.stagedGCRelease == nil { + return + } + if r.staged != nil { + if err := r.staged.Close(); err != nil { + log.Printf("closing codex staging sink: %v", err) + } + r.staged = nil + } + if r.stagedGCRelease != nil { + r.stagedGCRelease() + r.stagedGCRelease = nil + } } func (r processResult) needsRetryForSession(sessionID string) bool { @@ -10973,9 +11137,26 @@ func (e *Engine) processProviderFile( } forceSourceCwdParse := cwdDecision.forceParse + // Codex checkpoint decision: an invalid proof requires an authoritative + // replacement. A missing checkpoint is merely absent optimization state: + // unchanged upgraded archives keep the existing stat/content freshness + // gates and earn a checkpoint lazily on their next real source change. + var fingerprint parser.SourceFingerprint + var codexCheckpoint *db.ParserCheckpoint + var codexSeed []byte + var codexFullHash string + var codexHashState []byte + codexForceFullParse := false + codexLazyBootstrap := false + codexProvenUnchanged := false + codexAuditDeepVerify := e.checkpointAudit.Load() && + isCodexFormatAgent(file.Agent) + var codexUnchangedMtime int64 verifiedCapture, verifiedMtime, verifiedFresh, verifiedStateOK := e.verifiedProviderSourceState(provider, source, file) - if !forceSourceCwdParse && verifiedStateOK && verifiedFresh { + if !forceSourceCwdParse && !codexForceFullParse && + !codexProvenUnchanged && !codexAuditDeepVerify && + verifiedStateOK && verifiedFresh { if e.verifiedProviderSourceFreshInDB( verifiedCapture.key.agent, source, verifiedCapture.signature.size, verifiedMtime, @@ -10988,6 +11169,11 @@ func (e *Engine) processProviderFile( e.invalidateVerifiedSource( verifiedCapture.key.agent, verifiedCapture.key.path, ) + // The stat snapshot was fresh, but the persisted projection was not. + // Treat the source as unverified for the remaining gates in this call: + // if the row disappeared, the Codex cold-parse path can derive its + // fingerprint from the parse instead of paying for a redundant read. + verifiedFresh = false } // Capture the per-component stat digest from the same pre-parse @@ -11033,8 +11219,9 @@ func (e *Engine) processProviderFile( // change -- including a same-size same-mtime in-place rewrite -- // bumps a ctime and breaks the digest, falling through to the // content-verified gates. - if !forceSourceCwdParse { - if freshMtime, fresh := e.providerSourceFreshBeforeFingerprint( + if !forceSourceCwdParse && !codexForceFullParse && + !codexProvenUnchanged && !codexAuditDeepVerify { + if freshMTime, fresh := e.providerSourceFreshBeforeFingerprint( ctx, source, file, preParseStatHash, ); fresh { if verifiedStateOK { @@ -11042,11 +11229,39 @@ func (e *Engine) processProviderFile( } return processResult{ skip: true, - mtime: freshMtime, + mtime: freshMTime, }, true } } + // Consult the heavier checkpoint tables only after the free in-memory and + // persisted stat-digest gates have declined. This preserves current-main + // warm-sweep behavior while still proving appends before any full content + // fingerprint read. Audit mode deliberately bypasses this optimization and + // takes the authoritative full-parse path below. + if isCodexFormatAgent(file.Agent) && !codexAuditDeepVerify { + cpResult, cpErr := e.codexCheckpointFingerprint(ctx, source, file) + if cpErr != nil { + log.Printf("codex checkpoint %s: %v", file.Path, cpErr) + } else { + switch cpResult.decision { + case codexCheckpointUnchanged: + codexProvenUnchanged = true + codexUnchangedMtime = cpResult.fingerprint.MTimeNS + case codexCheckpointAppend: + fingerprint = cpResult.fingerprint + codexCheckpoint = cpResult.checkpoint + codexSeed = cpResult.seed + codexFullHash = cpResult.fingerprint.Hash + codexHashState = cpResult.hashState + case codexCheckpointMissing: + codexLazyBootstrap = true + case codexCheckpointInvalid: + codexForceFullParse = true + } + } + } + // DB-freshness skip for single-session JSONL providers (Claude): // when the stored session's size, mtime, and data version already // match the source and its project does not need reparse, skip the @@ -11058,7 +11273,8 @@ func (e *Engine) processProviderFile( // companion touch invalidated); without the stamp those rows would // re-hash on every fresh process forever, since a skip never writes. sourceForceReplace := false - if !forceSourceCwdParse { + if !forceSourceCwdParse && !codexForceFullParse && + !codexProvenUnchanged && !codexAuditDeepVerify { if mtime, fresh, forceReplace, contentVerified := e.providerSingleSessionFresh( ctx, provider, source, file, ); fresh { @@ -11089,8 +11305,11 @@ func (e *Engine) processProviderFile( // did not change, so skip before Fingerprint pays the per-session child // lookup; a child-only edit this cannot see is reconciled by the next // full-discovery pass, whose digest comparison still catches it. - if !forceSourceCwdParse { - if freshMtime, fresh := e.watermarkOnlySQLiteSourceFresh(source, file); fresh { + if !forceSourceCwdParse && !codexForceFullParse && + !codexProvenUnchanged && !codexAuditDeepVerify { + if freshMtime, fresh := e.watermarkOnlySQLiteSourceFresh( + source, file, + ); fresh { return processResult{ skip: true, mtime: freshMtime, @@ -11098,33 +11317,108 @@ func (e *Engine) processProviderFile( } } - fingerprint, err := provider.Fingerprint(ctx, source) - if err != nil { - if (file.ForceParse || file.ForceFullParse) && - providerDeletedPhysicalSQLiteSource(file.Agent, file.Path) && - errors.Is(err, os.ErrNotExist) { - excludedSessionIDs, ownershipErr := - e.providerSourceSessionIDsForForceReplace( - ctx, provider, source, - ) - if ownershipErr != nil { + // The checkpoint proved the committed transcript and the current stat + // snapshot agree. Persist that same snapshot so archives created before + // provider freshness digests do not re-enter the content-hash path after + // every restart; this also refreshes a digest after an unrelated + // session-index touch. + if codexProvenUnchanged { + if verifiedStateOK { + e.promoteVerifiedSource(verifiedCapture) + } + e.stampProviderStatHashForConfirmedSource(ctx, preParseStatHash) + return processResult{ + skip: true, + mtime: codexUnchangedMtime, + noCacheSkip: true, + }, true + } + + // codexFingerprintFromParse marks a never-synced Codex-format source + // whose fingerprint is derived from the parser's single-pass hash + // state instead of a separate full-file fingerprint read. + codexFingerprintFromParse := false + if codexCheckpoint == nil { + var err error + if isCodexFormatAgent(file.Agent) && + !verifiedFresh && + !e.forceParseRequested(file) && + !codexAuditDeepVerify { + // A never-synced Codex-format source has no stored hash to + // compare: skip the standalone fingerprint read and derive + // the hash from the parser's single-pass capture after the + // parse, so the cold full sync reads the source once instead + // of twice. Only the identity fields are needed up front; + // every skip gate above this point requires a stored row or + // cache entry, which a new source does not have. A persisted + // skip-cache entry for the same base path (e.g. a parse-error + // entry keyed by the real source hash) keeps the fingerprint + // read so its cache key still matches. + lookupPath := file.Path + if e.pathRewriter != nil { + lookupPath = e.pathRewriter(file.Path) + } + _, hasHash := e.db.GetFileHashByAgentPath( + lookupPath, string(file.Agent), + ) + e.skipMu.RLock() + skipBase := providerAgentSkipCacheKey(file.Path, file.Agent) + _, hasSkipEntry := e.skipHashKeys[skipBase] + e.skipMu.RUnlock() + if !hasHash && !hasSkipEntry { + if info, statErr := os.Stat(file.Path); statErr == nil { + inode, device := getFileIdentity(file.Path, info) + mtime := info.ModTime().UnixNano() + if file.Agent == parser.AgentCodex { + mtime = parser.CodexEffectiveMtime( + file.Path, mtime, + ) + } + fingerprint = parser.SourceFingerprint{ + Key: codexCheckpointFingerprintKey( + source, file.Path, + ), + Size: info.Size(), + MTimeNS: mtime, + Inode: uint64(inode), + Device: uint64(device), + } + codexFingerprintFromParse = true + } + } + } + if !codexFingerprintFromParse { + fingerprint, err = provider.Fingerprint(ctx, source) + } + if err != nil { + if (file.ForceParse || file.ForceFullParse) && + providerDeletedPhysicalSQLiteSource(file.Agent, file.Path) && + errors.Is(err, os.ErrNotExist) { + excludedSessionIDs, ownershipErr := + e.providerSourceSessionIDsForForceReplace( + ctx, provider, source, + ) + if ownershipErr != nil { + return processResult{ + err: ownershipErr, + noCacheSkip: true, + }, true + } return processResult{ - err: ownershipErr, - noCacheSkip: true, + excludedSessionIDs: excludedSessionIDs, + forceReplace: true, }, true } - return processResult{ - excludedSessionIDs: excludedSessionIDs, - forceReplace: true, - }, true + return processResult{err: err}, true } - return processResult{err: err}, true } cacheKey := providerProcessCacheKey( file, source, fingerprint, providerSemantics, ) cacheSkip := e.shouldCacheSkip(file) - if cacheSkip && !forceSourceCwdParse && !e.forceParseBypassesCache(file) { + if cacheSkip && !forceSourceCwdParse && + !e.forceParseBypassesCache(file) && + !codexForceFullParse && !codexAuditDeepVerify { e.skipMu.RLock() cachedMtime, cached := e.skipCache[cacheKey] e.skipMu.RUnlock() @@ -11235,9 +11529,16 @@ func (e *Engine) processProviderFile( // stored messages instead of appending on top of stale rows. var incRes processResult var incOK bool - if !forceSourceCwdParse { + if codexForceFullParse || codexLazyBootstrap { + incRes = processResult{forceReplace: true} + } else if codexAuditDeepVerify { + // The audit content-hashes the full source below and repairs only + // on mismatch; never tail-apply against a prefix it cannot prove. + incRes = processResult{} + } else if !forceSourceCwdParse { incRes, incOK = e.tryProviderIncrementalAppend( ctx, provider, source, file, fingerprint, + codexSeed, codexFullHash, codexHashState, codexCheckpoint, ) } if incOK { @@ -11320,7 +11621,10 @@ func (e *Engine) processProviderFile( // here the provider parses the source, so acquire the retention lease that // bounds the parsed payload and attach it to every result carrying that // data. A result still classified as a skip below releases it immediately. - lease, err := e.retentionBudget().acquire(ctx, parseRetentionSourceBytes(file)) + sourceBytes := parseRetentionSourceBytes(file) + lease, err := e.retentionBudget().acquire( + ctx, parseRetentionAdmissionBytes(file, sourceBytes), + ) if err != nil { return processResult{err: err}, true } @@ -11329,14 +11633,64 @@ func (e *Engine) processProviderFile( runtimeMetrics.openCodeSQLiteParse() } } - outcome, err := provider.Parse(ctx, parser.ParseRequest{ - Source: source, - Fingerprint: fingerprint, - Machine: machine, - ForceParse: e.forceParseRequested(file), - StoredPathResolver: e.storedPathResolver, - }) + // Large Codex full parses stream through the scratch staging sink: the + // in-memory model keeps placeholders instead of tool-result content, so + // peak memory stays bounded by messages + one scratch batch rather than + // the transcript size. Small files keep the collecting path. Report-only + // parse-diff disables staging through e.forceParse; ordinary forced full + // syncs retain the bounded-memory path. + var stagedSink *codexStagingSink + var stagedGCRelease func() + if file.Agent == parser.AgentCodex && + sourceBytes > e.stagedCodexMin && + !e.forceParse { + stagedSink, err = newCodexStagingSink( + e.stagedCodexDir, e.blockedResultCategories, sourceBytes, + ) + if err != nil { + lease.Release() + return processResult{ + err: fmt.Errorf("codex staging sink: %w", err), + mtime: fingerprint.MTimeNS, + cacheSkip: cacheSkip, + cacheKey: cacheKey, + noCacheSkip: true, + }, true + } + stagedSink.idPrefix = e.idPrefix + stagedGCRelease = beginStagedColdSync() + } + var outcome parser.ParseOutcome + if stagedSink != nil { + if e.forceParseRequested(file) { + parser.EvictCodexSessionIndexForSession( + providerDiscoveredPath(source), + ) + } + outcome, err = stagedCodexParseOutcome( + parser.ProviderConfig{ + Roots: e.agentDirs[file.Agent], + Machine: machine, + PathRewriter: e.pathRewriter, + }, + source, fingerprint, stagedSink, + ) + } else { + outcome, err = provider.Parse(ctx, parser.ParseRequest{ + Source: source, + Fingerprint: fingerprint, + Machine: machine, + ForceParse: e.forceParseRequested(file), + StoredPathResolver: e.storedPathResolver, + }) + } if err != nil { + if stagedSink != nil { + stagedSink.Close() + } + if stagedGCRelease != nil { + stagedGCRelease() + } if !e.forceParse { cwdChanged, reconcileErr := e.reconcileSourceCwdByPath( source, cwdDecision, @@ -11369,6 +11723,12 @@ func (e *Engine) processProviderFile( fingerprint, outcome, ); err != nil { + if stagedSink != nil { + stagedSink.Close() + } + if stagedGCRelease != nil { + stagedGCRelease() + } return processResult{ err: err, mtime: fingerprint.MTimeNS, @@ -11379,6 +11739,32 @@ func (e *Engine) processProviderFile( }, true } applyProviderFingerprintFileInfo(file.Agent, fingerprint, outcome.Results) + if codexFingerprintFromParse && fingerprint.Hash == "" { + // Derive the fingerprint hash from the parser's single-pass + // capture so the stored file_hash matches what a fingerprint + // read would have produced, without re-reading the source. + for i := range outcome.Results { + state := outcome.Results[i].Result.CheckpointHashState + if len(state) == 0 { + continue + } + hash, hashErr := codexHashStateDigest(state) + if hashErr != nil { + log.Printf( + "fingerprint from parse %s: %v", + file.Path, hashErr, + ) + continue + } + outcome.Results[i].Result.Session.File.Hash = hash + fingerprint.Hash = hash + } + if fingerprint.Hash != "" { + cacheKey = providerProcessCacheKey( + file, source, fingerprint, providerSemantics, + ) + } + } cleanCache := providerOutcomeAllowsCleanSkipCache(outcome) providerWideFailureCount := len(outcome.SourceErrors) if !outcome.ResultSetComplete { @@ -11461,6 +11847,7 @@ func (e *Engine) processProviderFile( excludedSessionIDs: excludedSessionIDs, preservedSessionIDs: preservedSessionIDs, sourceMissingMembers: missingMembers, + sourceBytes: sourceBytes, mtime: fingerprint.MTimeNS, cacheSkip: cacheSkip, cacheKey: cacheKey, @@ -11550,11 +11937,20 @@ func (e *Engine) processProviderFile( ) filteredResults, truncationVerifyFailed := e.dropShrinkingTruncatedCursorIDEResults(ctx, file, filteredResults) + if stagedSink != nil && len(filteredResults) == 0 { + // Every result was dropped as unchanged; nothing will publish the + // staged rows, so release the scratch sink now. + stagedSink.Close() + stagedSink = nil + stagedGCRelease() + stagedGCRelease = nil + } res := processResult{ results: filteredResults, excludedSessionIDs: excludedSessionIDs, preservedSessionIDs: preservedSessionIDs, sourceMissingMembers: missingMembers, + sourceBytes: sourceBytes, mtime: fingerprint.MTimeNS, cacheSkip: cacheSkip, cacheKey: cacheKey, @@ -11566,6 +11962,8 @@ func (e *Engine) processProviderFile( providerWideFailureCount: providerWideFailureCount, retentionLease: lease, providerStatHash: preParseStatHash, + staged: stagedSink, + stagedGCRelease: stagedGCRelease, sourceCwdResolution: cwdDecision.resolution, sourceCwdStored: cwdDecision.storedCwd, sourceCwdStoredOK: cwdDecision.storedOK, @@ -14241,6 +14639,10 @@ func (e *Engine) tryProviderIncrementalAppend( source parser.SourceRef, file parser.DiscoveredFile, fingerprint parser.SourceFingerprint, + seed []byte, + fullHash string, + hashState []byte, + checkpoint *db.ParserCheckpoint, ) (processResult, bool) { // Match the shared tryIncrementalJSONL gate: parse-diff and an explicit // full import both require a complete replacement rather than an append. @@ -14279,7 +14681,7 @@ func (e *Engine) tryProviderIncrementalAppend( parseFn := func( _ string, inc *db.IncrementalInfo, - ) ([]parser.ParsedMessage, []parser.ClaudeSubagentLink, time.Time, int64, *string, error) { + ) ([]parser.ParsedMessage, []parser.ClaudeSubagentLink, []parser.ParsedToolCallUpdate, []parser.ParsedMessageTokenUsageUpdate, time.Time, int64, *string, []byte, error) { // The Claude parser needs the stored tail's provider message id // so its queued-command masking fallback fires only for a real // same-message.id continuation; without it, every routine queued @@ -14298,23 +14700,25 @@ func (e *Engine) tryProviderIncrementalAppend( Offset: inc.FileSize, StartOrdinal: inc.NextOrdinal, Machine: inc.Machine, + Seed: seed, LastEntryUUID: inc.LastEntryUUID, StoredAgentLabel: inc.AgentLabel, StoredEntrypoint: inc.Entrypoint, StoredSessionKind: inc.SessionKind, StoredClaudeLinearParse: inc.ClaudeLinearParse, StoredLastClaudeMessageID: storedLastClaudeMessageID, + StoredPendingUsageOrdinal: inc.PendingUsageOrdinal, }, ) if perr != nil { - return nil, nil, time.Time{}, 0, nil, perr + return nil, nil, nil, nil, time.Time{}, 0, nil, nil, perr } switch status { case parser.IncrementalNeedsFullParse: if outcome.ForceReplace { // Signal the shared helper to fall back to a // full parse that replaces stored messages. - return nil, nil, time.Time{}, 0, nil, + return nil, nil, nil, nil, time.Time{}, 0, nil, nil, parser.ErrIncrementalNeedsFullParse } // A plain full-parse fallback without a replace request. @@ -14322,9 +14726,9 @@ func (e *Engine) tryProviderIncrementalAppend( // fallbacks (a DAG fork can drop or re-branch stored // rows), so this branch serves providers that only need // an append-preserving full parse. - return nil, nil, time.Time{}, 0, nil, parser.ErrDAGDetected + return nil, nil, nil, nil, time.Time{}, 0, nil, nil, parser.ErrDAGDetected case parser.IncrementalNoNewData: - return nil, nil, time.Time{}, 0, nil, nil + return nil, nil, nil, nil, time.Time{}, 0, nil, nil, nil default: var terminationStatus *string if outcome.TerminationStatus != nil { @@ -14332,11 +14736,17 @@ func (e *Engine) tryProviderIncrementalAppend( terminationStatus = &status } return outcome.Messages, outcome.SubagentLinks, - outcome.EndedAt, outcome.ConsumedBytes, terminationStatus, nil + outcome.ToolCallUpdates, + outcome.MessageTokenUsageUpdates, + outcome.EndedAt, outcome.ConsumedBytes, terminationStatus, + outcome.NextCursor, nil } } - return e.tryIncrementalJSONL(ctx, file, info, file.Agent, parseFn) + return e.tryIncrementalJSONL( + ctx, file, info, file.Agent, parseFn, + checkpoint, fullHash, hashState, + ) } // incrementalParseFunc reads new JSONL lines from a file @@ -14347,7 +14757,7 @@ func (e *Engine) tryProviderIncrementalAppend( // only complete, valid JSON lines so it can be used as a safe resume offset. type incrementalParseFunc func( path string, inc *db.IncrementalInfo, -) ([]parser.ParsedMessage, []parser.ClaudeSubagentLink, time.Time, int64, *string, error) +) ([]parser.ParsedMessage, []parser.ClaudeSubagentLink, []parser.ParsedToolCallUpdate, []parser.ParsedMessageTokenUsageUpdate, time.Time, int64, *string, []byte, error) // tryIncrementalJSONL attempts an incremental parse of an // append-only JSONL file by reading only bytes appended since @@ -14361,6 +14771,9 @@ func (e *Engine) tryIncrementalJSONL( info os.FileInfo, agent parser.AgentType, parseFn incrementalParseFunc, + checkpoint *db.ParserCheckpoint, + fullHash string, + hashState []byte, ) (processResult, bool) { if e.forceParse || e.forceFullParse || file.ForceFullParse { // Parse-diff and explicit full imports never produce append deltas. @@ -14470,14 +14883,15 @@ func (e *Engine) tryIncrementalJSONL( // retention lease that bounds the parsed payload. It is attached to the // incremental results below and released on every decline (fall-through to // a full parse re-acquires at the provider parse seam) or skip return. + sourceBytes := parseRetentionSourceBytes(file) lease, leaseErr := e.retentionBudget().acquire( - ctx, parseRetentionSourceBytes(file), + ctx, parseRetentionAdmissionBytes(file, sourceBytes), ) if leaseErr != nil { return processResult{err: leaseErr}, true } - newMsgs, links, endedAt, consumed, terminationStatus, err := parseFn( + newMsgs, links, toolCallUpdates, messageUsageUpdates, endedAt, consumed, terminationStatus, cursor, err := parseFn( file.Path, inc, ) if err != nil { @@ -14508,17 +14922,112 @@ func (e *Engine) tryIncrementalJSONL( // the next sync. newOffset := inc.FileSize + consumed var incHash string + resumeOK := false // Refresh the stored content fingerprint on the incremental path. Codex // needs it for parse-diff's raced-skew detection; Claude needs it so // providerSingleSessionFresh can compare the stored hash against the // on-disk bytes and catch a same-size, same-mtime, same-inode in-place // rewrite that the size/mtime/identity skip signals cannot see. - if isCodexFormatAgent(agent) || agent == parser.AgentClaude { + if fullHash != "" { + // The stored fingerprint must cover only the committed safe prefix, + // not an unfinished partial tail at EOF. Resume the hash state + // through newOffset (the last complete record) rather than trusting + // the full-file hash computed by the checkpoint gate. + if state, hash, hashErr := codexResumeHashFn( + file.Path, inc.FileSize, newOffset, hashState, + ); hashErr == nil { + incHash = hash + hashState = state + resumeOK = true + } else { + log.Printf( + "resuming codex hash for %s at %d: %v", + file.Path, newOffset, hashErr, + ) + incHash = fullHash + } + } else if isCodexFormatAgent(agent) || agent == parser.AgentClaude { if hash, err := ComputeFileHashPrefix(file.Path, newOffset); err == nil { incHash = hash } } + // Persist the advanced parser checkpoint in the same transaction as the + // delta when this append was resumed from one. + var nextCheckpoint *db.ParserCheckpoint + var nextCheckpointBlobs *db.ParserCheckpointBlobs + // Only persist the advanced checkpoint when the resumable hash state was + // proven to cover the new offset. On reconstruction failure the write + // keeps the previous checkpoint: its offset now disagrees with the + // committed file size, so the next gate rebuilds authoritatively instead + // of resuming from stale state at a newer offset and omitting bytes. + if checkpoint != nil && len(cursor) > 0 && hashState != nil && resumeOK { + cpNextOrdinal := inc.NextOrdinal + if len(newMsgs) > 0 { + cpNextOrdinal = nextParsedOrdinal(inc.NextOrdinal, newMsgs) + } + anchorDigest, anchorErr := codexCheckpointAnchorDigest( + file.Path, newOffset, + ) + if anchorErr != nil { + log.Printf( + "building codex checkpoint %s: %v", file.Path, anchorErr, + ) + } else { + inode, device := getFileIdentity(file.Path, info) + changeTime, _ := fileChangeTime(file.Path, info) + built, blobs := buildCodexCheckpoint( + inc.ID, + string(agent), + e.effectiveSourcePath(file.Path), + uint64(inode), + uint64(device), + incMtime, + changeTime, + newOffset, + cursor, + hashState, + incHash, + cpNextOrdinal, + anchorDigest, + ) + nextCheckpoint = built + nextCheckpointBlobs = &blobs + } + } + + totalOut := inc.TotalOutputTokens + peakCtx := inc.PeakContextTokens + hasTotalOut := inc.HasTotalOutputTokens + hasPeakCtx := inc.HasPeakContextTokens + for _, m := range newMsgs { + msgHasCtx, msgHasOut := m.TokenPresence() + // Accumulate from per-message values already bounded to the + // per-message clamp the central pass applies to the stored rows, so + // a corrupt new message cannot inflate the session aggregates past + // what the persisted rows justify (parity with the full path, which + // re-derives message-derived totals from the clamped rows). + if msgHasOut { + totalOut += clampedTokens(m.OutputTokens) + hasTotalOut = true + } + if ctx := clampedTokens(m.ContextTokens); msgHasCtx && + (!hasPeakCtx || ctx > peakCtx) { + peakCtx = ctx + hasPeakCtx = true + } + } + for _, usageUpdate := range messageUsageUpdates { + if usageUpdate.HasOutputTokens { + totalOut += clampedTokens(usageUpdate.OutputTokens) + hasTotalOut = true + } + if ctx := clampedTokens(usageUpdate.ContextTokens); usageUpdate.HasContextTokens && (!hasPeakCtx || ctx > peakCtx) { + peakCtx = ctx + hasPeakCtx = true + } + } + if len(newMsgs) == 0 { // No new messages, but advance the offset past // non-message lines (progress events, metadata) @@ -14527,6 +15036,7 @@ func (e *Engine) tryIncrementalJSONL( // with non-message timestamps (e.g. progress). if consumed > 0 { return processResult{ + sourceBytes: sourceBytes, incremental: &incrementalUpdate{ sessionID: inc.ID, project: inc.Project, @@ -14534,6 +15044,10 @@ func (e *Engine) tryIncrementalJSONL( machine: inc.Machine, cwd: inc.Cwd, links: links, + toolCallUpdates: toolCallUpdates, + messageUsageUpdates: messageUsageUpdates, + checkpoint: nextCheckpoint, + checkpointBlobs: nextCheckpointBlobs, endedAt: endedAt, terminationStatus: terminationStatus, msgCount: inc.MsgCount, @@ -14543,10 +15057,10 @@ func (e *Engine) tryIncrementalJSONL( fileHash: incHash, nextOrdinal: inc.NextOrdinal, lastEntryUUID: inc.LastEntryUUID, - totalOutputTokens: inc.TotalOutputTokens, - peakContextTokens: inc.PeakContextTokens, - hasTotalOutputTokens: inc.HasTotalOutputTokens, - hasPeakContextTokens: inc.HasPeakContextTokens, + totalOutputTokens: totalOut, + peakContextTokens: peakCtx, + hasTotalOutputTokens: hasTotalOut, + hasPeakContextTokens: hasPeakCtx, }, retentionLease: lease, }, true @@ -14622,29 +15136,8 @@ func (e *Engine) tryIncrementalJSONL( agent, inc.ID, len(newMsgs), inc.FileSize, ) - totalOut := inc.TotalOutputTokens - peakCtx := inc.PeakContextTokens - hasTotalOut := inc.HasTotalOutputTokens - hasPeakCtx := inc.HasPeakContextTokens - for _, m := range newMsgs { - msgHasCtx, msgHasOut := m.TokenPresence() - // Accumulate from per-message values already bounded to the - // per-message clamp the central pass applies to the stored rows, so - // a corrupt new message cannot inflate the session aggregates past - // what the persisted rows justify (parity with the full path, which - // re-derives message-derived totals from the clamped rows). - if msgHasOut { - totalOut += clampedTokens(m.OutputTokens) - hasTotalOut = true - } - if ctx := clampedTokens(m.ContextTokens); msgHasCtx && - (!hasPeakCtx || ctx > peakCtx) { - peakCtx = ctx - hasPeakCtx = true - } - } - return processResult{ + sourceBytes: sourceBytes, incremental: &incrementalUpdate{ sessionID: inc.ID, project: inc.Project, @@ -14653,6 +15146,10 @@ func (e *Engine) tryIncrementalJSONL( cwd: inc.Cwd, msgs: newMsgs, links: links, + toolCallUpdates: toolCallUpdates, + messageUsageUpdates: messageUsageUpdates, + checkpoint: nextCheckpoint, + checkpointBlobs: nextCheckpointBlobs, endedAt: endedAt, terminationStatus: terminationStatus, msgCount: inc.MsgCount + len(newMsgs), @@ -15411,55 +15908,102 @@ func (e *Engine) recomputeSignalsFromDB( if e.disableSignalRecompute { return 0, nil } - sess, err := e.db.GetSessionFull(ctx, sessionID) - if err != nil { - return 0, fmt.Errorf( - "loading session %s: %w", sessionID, err, - ) - } - if sess == nil { - return 0, nil - } - msgs, err := e.db.GetAllMessages(ctx, sessionID) - if err != nil { - log.Printf( - "signals: load messages %s: %v", - sessionID, err, - ) - return 0, fmt.Errorf( - "loading messages %s: %w", sessionID, err, - ) - } - update, findings := computeSignalsAndSecrets(*sess, msgs) - heapBytes := recomputeHeapBytes(msgs, findings) - // Findings persist before the signals update: UpdateSessionSignals - // advances quality_signal_version, which BackfillSignals treats as - // proof the whole compute persisted. Writing it last keeps a - // session whose findings write failed below the current version, - // so the next backfill retries it. - if err := e.db.ReplaceSessionSecretFindings( - sessionID, findings, update.SecretLeakCount, update.SecretsRulesVersion, - ); err != nil { - log.Printf("secrets: persist %s: %v", sessionID, err) - return 0, fmt.Errorf("persisting findings %s: %w", sessionID, err) - } - if err := e.db.UpdateSessionSignals( - sessionID, update, - ); err != nil { - log.Printf( - "signals: update %s: %v", sessionID, err, + return e.recomputeSignalsFromDBWithHook(ctx, sessionID, nil) +} + +// recomputeSignalsFromDBWithHook retries a full recompute when the transcript +// changes after its revision token is captured. beforePublish is a deterministic +// test seam invoked after the snapshot has been computed and before the +// conditional write; production callers pass nil. +func (e *Engine) recomputeSignalsFromDBWithHook( + ctx context.Context, + sessionID string, + beforePublish func(attempt int), +) (int, error) { + const maxSnapshotAttempts = 3 + for attempt := range maxSnapshotAttempts { + if err := ctx.Err(); err != nil { + return 0, err + } + sess, err := e.db.GetSessionFull(ctx, sessionID) + if err != nil { + return 0, fmt.Errorf( + "loading session %s: %w", sessionID, err, + ) + } + if sess == nil { + return 0, nil + } + // Capture every session-row input consumed by the signal compute before + // loading transcript rows. The conditional transaction below rejects + // transcript changes and metadata-only races alike. + expectedInputs, err := db.SignalInputSnapshot(*sess) + if err != nil { + return 0, err + } + msgs, err := e.db.GetAllMessages(ctx, sessionID) + if err != nil { + log.Printf( + "signals: load messages %s: %v", + sessionID, err, + ) + return 0, fmt.Errorf( + "loading messages %s: %w", sessionID, err, + ) + } + update, findings := computeSignalsAndSecrets(*sess, msgs) + heapBytes := recomputeHeapBytes(msgs, findings) + state, err := buildSignalStateFromRows( + sessionID, msgs, extractToolCallRows(msgs), + expectedInputs.TranscriptRevision, ) - return 0, fmt.Errorf( - "updating signals %s: %w", sessionID, err, + if err != nil { + return 0, err + } + if beforePublish != nil { + beforePublish(attempt) + } + applied, err := e.db.ReplaceSessionSignalsIfInputsMatch( + sessionID, expectedInputs, findings, update, state, ) + if err != nil { + return 0, fmt.Errorf( + "publishing signal snapshot %s: %w", sessionID, err, + ) + } + if applied { + return heapBytes, nil + } } - return heapBytes, nil + return 0, fmt.Errorf( + "session %s changed during %d signal recompute attempts", + sessionID, maxSnapshotAttempts, + ) } type pendingWrite struct { - sess parser.ParsedSession - msgs []parser.ParsedMessage - usageEvents []parser.ParsedUsageEvent + sess parser.ParsedSession + msgs []parser.ParsedMessage + usageEvents []parser.ParsedUsageEvent + // sourceBytes is the physical source size carried from the parse result; + // collectAndBatch uses it to flush batches on estimated bytes as well as + // session count. + sourceBytes int64 + // checkpoint is the provider's persisted continuation cursor for a full + // parse. The flush path persists it as a parser_checkpoints row after the + // session rows commit, so later appends can resume without rescanning the + // transcript prefix. Empty for providers without checkpoints. + checkpoint []byte + // checkpointHashState/checkpointAnchorDigest carry the single-pass + // hash state and tail-anchor digest the parser captured while reading + // the snapshot; persisting them avoids any second source read after a + // full parse. Empty when the provider did not supply them. + checkpointHashState []byte + checkpointAnchorDigest string + // staged carries the scratch staging sink when this write came from a + // streaming Codex full parse; the write path publishes tool-result rows + // and summaries from it and the batch flush closes it. + staged *codexStagingSink needsRetry bool forceReplace bool // sourceIdentityUnverified marks a copy that shares a native session ID @@ -16076,22 +16620,48 @@ func (e *Engine) writeBatchWithOutcomeContext( var update db.SessionSignalUpdate var findings []db.SecretFinding - if !e.disableSignalRecompute { + var werr error + if replaceMessages && pw.staged != nil { + // The staged sink owns this parse's tool-result rows, and only the + // staged write publishes them, so it runs even when signal + // recomputation is disabled. + werr = e.writeStagedFullParse(s, msgs, pw) + } else if replaceMessages && !e.disableSignalRecompute { update, findings = computeSignalsAndSecrets(s, msgs) if ctx.Err() != nil { return outcome } - } - - var werr error - if replaceMessages && !e.disableSignalRecompute { - werr = e.db.ReplaceSessionContent(s.ID, msgs, update, findings) + if isCodexFormatAgent(pw.sess.Agent) { + cp, blobs, cpErr := e.buildCodexFullParseCheckpoint( + pw.sess.File.Path, pw, + ) + if cpErr != nil { + log.Printf( + "checkpoint build %s: %v", + pw.sess.File.Path, cpErr, + ) + cp, blobs = nil, nil + } + werr = e.db.ReplaceSessionContentWithCheckpoint( + s.ID, msgs, update, findings, cp, blobs, + ) + } else { + werr = e.db.ReplaceSessionContent( + s.ID, msgs, update, findings, + ) + } } else if replaceMessages { if msgs == nil { msgs = []db.Message{} } werr = e.db.ReplaceSessionMessages(s.ID, msgs) } else { + if !e.disableSignalRecompute { + update, findings = computeSignalsAndSecrets(s, msgs) + if ctx.Err() != nil { + return outcome + } + } werr = e.writeMessages(s.ID, msgs) } if werr != nil { @@ -16106,6 +16676,14 @@ func (e *Engine) writeBatchWithOutcomeContext( outcome.failedSessions++ continue } + if replaceMessages && pw.staged == nil && + !e.disableSignalRecompute { + if err := e.seedSignalStateFromFull(s.ID, msgs); err != nil { + log.Printf( + "signals: seed state %s: %v", s.ID, err, + ) + } + } if ctx.Err() != nil { return outcome } @@ -17107,6 +17685,77 @@ type localGitIdentity struct { worktreeKind export.WorktreeRelationship } +// stagedToolCallPositions maps every occurrence-qualified staging key in the +// message model to its final message/call coordinates. Provider call IDs can +// repeat, so raw tool_use_id alone cannot identify a staged event target. +func stagedToolCallPositions( + msgs []db.Message, +) map[string]db.StagedToolCallPosition { + positions := make(map[string]db.StagedToolCallPosition) + callOccurrences := make(map[string]int) + for _, m := range msgs { + for callIdx, tc := range m.ToolCalls { + if tc.ToolUseID == "" { + continue + } + occurrence := callOccurrences[tc.ToolUseID] + callOccurrences[tc.ToolUseID] = occurrence + 1 + stageKey := db.StagedToolCallKey(tc.ToolUseID, occurrence) + positions[stageKey] = db.StagedToolCallPosition{ + ToolUseID: tc.ToolUseID, + Ordinal: m.Ordinal, + CallIndex: callIdx, + } + } + } + return positions +} + +// writeStagedFullParse publishes one staged streaming result. The publish +// transaction resolves every per-call summary once and runs the signals +// closure before commit, so the content-failure-aware signals and +// findings persist atomically with the message, tool-call, and event +// rows. The incremental signal state is then seeded with the captured +// verdicts. +func (e *Engine) writeStagedFullParse( + s db.Session, msgs []db.Message, pw pendingWrite, +) error { + positions := stagedToolCallPositions(msgs) + closure := func(verdicts map[string]bool) ( + db.SessionSignalUpdate, []db.SecretFinding, error, + ) { + update, findings := computeSignalsAndSecretsWithContentFailures( + s, msgs, verdicts, + ) + combined := append( + append([]db.SecretFinding(nil), findings...), + pw.staged.Findings(s.ID, positions)..., + ) + update.SecretLeakCount = definiteFindingCount(combined) + return update, combined, nil + } + cp, blobs, cpErr := e.buildCodexFullParseCheckpoint( + pw.sess.File.Path, pw, + ) + if cpErr != nil { + log.Printf("checkpoint build %s: %v", pw.sess.File.Path, cpErr) + cp, blobs = nil, nil + } + if err := e.db.ReplaceSessionContentStagedWithCheckpoint( + context.Background(), s.ID, msgs, pw.staged, + e.blockedResultCategories, closure, cp, blobs, + ); err != nil { + return err + } + e.anomalies.recordSanitize(pw.staged.ValidationStats()) + if err := e.seedSignalStateFromFullWithContentFailures( + s.ID, msgs, pw.staged.ContentFailures(), + ); err != nil { + log.Printf("signals: seed state %s: %v", s.ID, err) + } + return nil +} + func (e *Engine) writeBatchBulkWithOutcome( batch []pendingWrite, forceReplace bool, ) writeBatchOutcome { @@ -17147,6 +17796,77 @@ func (e *Engine) writeBatchBulkWithOutcomeContext( } continue } + if pw.staged != nil { + // Staged streaming results bypass the bulk batch: their + // tool-result rows live in the staging scratch database and + // must be published through the staged transaction, which + // also persists the content-failure-aware signals in the + // same commit. A staged full parse always force-replaces. + // The bulk batch would normally create the session row, so + // mirror the standard write path's session upsert and + // post-write sequence here. + revivingSourceMissing, err := + e.upsertSessionPendingContentForWrite(pw, s) + if err != nil { + if isIntentionalSessionSkip(err) { + if pw.sess.File.Path != "" { + e.cacheSkip( + pw.sess.File.Path, + pw.sess.File.Mtime, + pw.sess.File.Hash, + ) + } + continue + } + log.Printf("upsert session %s: %v", s.ID, err) + e.markStaleFailedMemberWrite(pw) + outcome.failedSessions++ + continue + } + tWrite := time.Now() + err = e.writeStagedFullParse(s, msgs, pw) + e.phaseStats.WriteNanos.Add(int64(time.Since(tWrite))) + if err != nil { + log.Printf( + "write staged session %s: %v", s.ID, err, + ) + e.markStaleFailedMemberWrite(pw) + outcome.failedSessions++ + continue + } + if err := e.db.ReplaceSessionUsageEvents( + s.ID, e.usageEventsForWrite(s.ID, pw.usageEvents), + ); err != nil { + log.Printf( + "write usage events for %s: %v", s.ID, err, + ) + e.markStaleFailedMemberWrite(pw) + outcome.failedSessions++ + continue + } + if err := e.db.SetSessionDataVersion( + s.ID, dataVersionForWrite(pw), + ); err != nil { + log.Printf( + "set data_version for %s: %v", s.ID, err, + ) + e.markStaleFailedMemberWrite(pw) + outcome.failedSessions++ + continue + } + if err := e.db.ClearSessionSourceMissing(s.ID); err != nil { + log.Printf( + "clear source-missing state for session %s: %v", s.ID, err, + ) + outcome.failedSessions++ + continue + } + _ = revivingSourceMissing + outcome.written[pendingIndex] = true + outcome.writtenSessions++ + outcome.writtenMessages += len(msgs) + continue + } replaceMessages := shouldReplaceFullParseMessages( pw, forceReplace, false, false, ) @@ -17161,6 +17881,20 @@ func (e *Engine) writeBatchBulkWithOutcomeContext( e.phaseStats.ScanNanos.Add(int64(time.Since(tScan))) } snapshotProject := pw.sess.Project + var checkpoint *db.ParserCheckpoint + var checkpointBlobs *db.ParserCheckpointBlobs + if isCodexFormatAgent(pw.sess.Agent) { + var checkpointErr error + checkpoint, checkpointBlobs, checkpointErr = + e.buildCodexFullParseCheckpoint(pw.sess.File.Path, pw) + if checkpointErr != nil { + log.Printf( + "checkpoint build %s: %v", + pw.sess.File.Path, checkpointErr, + ) + checkpoint, checkpointBlobs = nil, nil + } + } usageEvents, usageErr := e.usageEventsForWriteContext( ctx, s.ID, pw.usageEvents, ) @@ -17180,6 +17914,8 @@ func (e *Engine) writeBatchBulkWithOutcomeContext( SkipSignalUpdates: e.disableSignalRecompute, DataVersion: dataVersionForWrite(pw), ReplaceMessages: replaceMessages, + Checkpoint: checkpoint, + CheckpointBlobs: checkpointBlobs, }) pendingIndexes = append(pendingIndexes, pendingIndex) pendingByID[s.ID] = pw @@ -17219,6 +17955,17 @@ func (e *Engine) writeBatchBulkWithOutcomeContext( outcome.written[pendingIndex] = true outcome.resolved[pendingIndex] = true } + if writtenIndex >= 0 && writtenIndex < len(writes) { + if err := e.seedSignalStateFromFull( + writes[writtenIndex].Session.ID, + writes[writtenIndex].Messages, + ); err != nil { + log.Printf( + "signals: seed state %s: %v", + writes[writtenIndex].Session.ID, err, + ) + } + } } for _, id := range result.FailedIDs { if pw, ok := pendingByID[id]; ok { @@ -17712,6 +18459,7 @@ func shouldReplaceFullParseMessages( ) bool { return forceReplace || pw.forceReplace || pw.needsRetry || stale || revivingSourceMissing || + isCodexFormatAgent(pw.sess.Agent) || // Kiro full parses rebuild the complete accepted message projection; // append semantics would retain rows removed or rewritten by the source. pw.sess.Agent == parser.AgentKiro || @@ -17847,28 +18595,91 @@ func (e *Engine) writeIncremental( HasResult: link.HasResult, } } - - if err := e.db.WriteSessionIncremental( + toolCallResultUpdates := make( + []db.ToolCallResultUpdate, len(inc.toolCallUpdates), + ) + for i, update := range inc.toolCallUpdates { + resultEvents, err := convertToolResultEventsContext( + context.Background(), update.ResultEvents, + ) + if err != nil { + return err + } + toolCall := db.ToolCall{ResultEvents: resultEvents} + for j := range toolCall.ResultEvents { + toolCall.ResultEvents[j].SubagentSessionID = applyIDPrefixToID( + e.idPrefix, toolCall.ResultEvents[j].SubagentSessionID, + ) + } + e.anomalies.recordSanitize(db.SanitizeToolCall(&toolCall)) + toolCallResultUpdates[i] = db.ToolCallResultUpdate{ + ToolUseID: update.ToolUseID, + Position: db.ToolCallPosition{ + MessageOrdinal: update.MessageOrdinal, + CallIndex: update.CallIndex, + }, + Events: toolCall.ResultEvents, + } + } + messageUsageUpdates := make( + []db.MessageTokenUsageUpdate, len(inc.messageUsageUpdates), + ) + for i, update := range inc.messageUsageUpdates { + messageUsageUpdates[i] = db.MessageTokenUsageUpdate{ + Ordinal: update.Ordinal, + TokenUsage: append([]byte(nil), update.TokenUsage...), + ContextTokens: clampedTokens(update.ContextTokens), + OutputTokens: clampedTokens(update.OutputTokens), + HasContextTokens: update.HasContextTokens, + HasOutputTokens: update.HasOutputTokens, + } + } + + signalsMaintained := false + var maintainer db.SignalMaintainer + if !inc.hasSubstantiveUserMessage() && + !inc.hasCompactBoundary() && + !inc.hasResultSubagentLink() { + preRev, err := e.db.TranscriptRevision(inc.sessionID) + if err == nil { + var preSecrets string + if preSecrets, err = e.db.SessionSecretsRulesVersion( + inc.sessionID, + ); err == nil { + maintainer = e.newIncrementalSignalMaintainer( + inc, dbMsgs, toolCallResultUpdates, + messageUsageUpdates, preRev, preSecrets, + ) + } + } + } + signalsMaintained, err := e.db.WriteSessionIncremental( inc.sessionID, dbMsgs, db.IncrementalSessionUpdate{ - EndedAt: endedAt, - TerminationStatus: inc.terminationStatus, - MsgCount: msgCount, - UserMsgCount: userMsgCount, - FileSize: inc.fileSize, - FileMtime: inc.fileMtime, - FileHash: strPtr(inc.fileHash), - NextOrdinal: inc.nextOrdinal, - LastEntryUUID: inc.lastEntryUUID, - TotalOutputTokens: inc.totalOutputTokens, - PeakContextTokens: inc.peakContextTokens, - HasTotalOutputTokens: inc.hasTotalOutputTokens, - HasPeakContextTokens: inc.hasPeakContextTokens, - SubagentLinks: subagentLinks, - BlockedResultCategories: e.blockedResultCategories, + EndedAt: endedAt, + TerminationStatus: inc.terminationStatus, + MsgCount: msgCount, + UserMsgCount: userMsgCount, + FileSize: inc.fileSize, + FileMtime: inc.fileMtime, + FileHash: strPtr(inc.fileHash), + NextOrdinal: inc.nextOrdinal, + LastEntryUUID: inc.lastEntryUUID, + TotalOutputTokens: inc.totalOutputTokens, + PeakContextTokens: inc.peakContextTokens, + HasTotalOutputTokens: inc.hasTotalOutputTokens, + HasPeakContextTokens: inc.hasPeakContextTokens, + SubagentLinks: subagentLinks, + ToolCallResultUpdates: toolCallResultUpdates, + MessageTokenUsageUpdates: messageUsageUpdates, + Checkpoint: inc.checkpoint, + CheckpointBlobs: inc.checkpointBlobs, + BlockedResultCategories: e.blockedResultCategories, + SignalMaintainer: maintainer, }, - ); err != nil { + ) + if err != nil { return fmt.Errorf( "incremental write %s: %w", inc.sessionID, err, @@ -17898,14 +18709,17 @@ func (e *Engine) writeIncremental( ) } - // Signal/secret recompute costs O(session history), so it is - // debounced per session instead of running on every appended - // line: the first write after a quiet period recomputes - // inline, writes during a streaming burst coalesce into one - // recompute per interval plus a trailing flush. Recompute - // errors are logged inside recomputeSignalsFromDB and are - // non-fatal; a later write or flush retries. - e.signalSched.markDirty(inc.sessionID) + // Signal/secret maintenance normally ran inside the incremental write + // transaction. When the maintainer declined (user prompts, compact + // boundaries, stale state, out-of-window updates), fall back to the + // debounced full recompute: the first write after a quiet period + // recomputes inline, writes during a streaming burst coalesce into one + // recompute per interval plus a trailing flush. Recompute errors are + // logged inside recomputeSignalsFromDB and are non-fatal; a later + // write or flush retries. + if !signalsMaintained { + e.signalSched.markDirty(inc.sessionID) + } if inc.providerStatHash != nil { e.recordProviderStatHash( context.Background(), *inc.providerStatHash, @@ -18009,22 +18823,57 @@ func (e *Engine) writeSessionFullWithResolver( log.Printf("upsert session %s: %v", s.ID, err) return err } - var replaceErr error - if e.disableSignalRecompute { + if pw.staged != nil { + // The staged sink owns this parse's tool-result rows, and only the + // staged write publishes them. + if err := e.writeStagedFullParse(s, msgs, pw); err != nil { + log.Printf( + "write staged session %s: %v", + s.ID, err, + ) + return err + } + } else if e.disableSignalRecompute { if msgs == nil { msgs = []db.Message{} } - replaceErr = e.db.ReplaceSessionMessages(s.ID, msgs) + if err := e.db.ReplaceSessionMessages(s.ID, msgs); err != nil { + log.Printf( + "replace messages for %s: %v", + s.ID, err, + ) + return err + } } else { update, findings := computeSignalsAndSecrets(s, msgs) - replaceErr = e.db.ReplaceSessionContent(s.ID, msgs, update, findings) - } - if replaceErr != nil { - log.Printf( - "replace messages for %s: %v", - s.ID, replaceErr, - ) - return replaceErr + var checkpoint *db.ParserCheckpoint + var checkpointBlobs *db.ParserCheckpointBlobs + if isCodexFormatAgent(pw.sess.Agent) { + var checkpointErr error + checkpoint, checkpointBlobs, checkpointErr = + e.buildCodexFullParseCheckpoint(pw.sess.File.Path, pw) + if checkpointErr != nil { + log.Printf( + "checkpoint build %s: %v", + pw.sess.File.Path, checkpointErr, + ) + checkpoint, checkpointBlobs = nil, nil + } + } + if err := e.db.ReplaceSessionContentWithCheckpoint( + s.ID, msgs, update, findings, checkpoint, checkpointBlobs, + ); err != nil { + log.Printf( + "replace messages for %s: %v", + s.ID, err, + ) + return err + } + if err := e.seedSignalStateFromFull(s.ID, msgs); err != nil { + log.Printf( + "signals: seed state %s: %v", s.ID, err, + ) + } } if err := e.db.ReplaceSessionUsageEvents( s.ID, e.usageEventsForWrite(s.ID, pw.usageEvents), @@ -18050,7 +18899,6 @@ func (e *Engine) writeSessionFullWithResolver( log.Printf("clear source-missing state for session %s: %v", s.ID, err) return err } - return nil } @@ -19458,6 +20306,7 @@ func (e *Engine) processAndWriteSessionFile( res := e.processFile(ctx, file) defer e.retentionBudget().scavengeIfNeeded() defer res.retentionLease.Release() + defer res.releaseStaged() if res.err != nil { sessionsChanged = res.sourceCwdChanged if res.cacheSkip && res.mtime != 0 && !res.noCacheSkip { @@ -19700,14 +20549,20 @@ func (e *Engine) processAndWriteSessionFile( sessionNeedsRetry := res.providerWideFailureCount > 0 || res.needsRetryForSession(pr.Session.ID) write := pendingWrite{ - sess: pr.Session, - msgs: pr.Messages, - usageEvents: pr.UsageEvents, - needsRetry: sessionNeedsRetry || atomicDAG, - forceReplace: res.forceReplace, - sourceCwdResolution: res.sourceCwdResolution, - sourceCwdStored: res.sourceCwdStored, - sourceCwdStoredOK: res.sourceCwdStoredOK, + sess: pr.Session, + msgs: pr.Messages, + usageEvents: pr.UsageEvents, + checkpoint: pr.Checkpoint, + checkpointHashState: pr.CheckpointHashState, + checkpointAnchorDigest: pr.CheckpointAnchorDigest, + needsRetry: sessionNeedsRetry || atomicDAG, + forceReplace: res.forceReplace, + sourceCwdResolution: res.sourceCwdResolution, + sourceCwdStored: res.sourceCwdStored, + sourceCwdStoredOK: res.sourceCwdStoredOK, + } + if i == 0 { + write.staged = res.staged } // The session upsert commits parser-derived parent provenance before // the later content, usage, and completion stages. Queue the attempted diff --git a/internal/sync/engine_integration_test.go b/internal/sync/engine_integration_test.go index a24db84e1..d61d3fdd5 100644 --- a/internal/sync/engine_integration_test.go +++ b/internal/sync/engine_integration_test.go @@ -4795,7 +4795,9 @@ func TestSyncEngineProgress(t *testing.T) { "Finalizing sync: saving session source state", "Finalizing sync: linking file-backed subagent sessions", "Finalizing sync: repairing subagent relationships", - "Finalizing sync: releasing parsed-session memory", + // The memory release phase is reported only when a source at or + // above parseRetentionScavengeThreshold was parsed; these fixtures + // are far smaller. "Finalizing sync: checking database-backed sessions", "Finalizing sync: linking all subagent sessions", "Finalizing sync: saving the skip cache", @@ -13740,7 +13742,7 @@ func TestIncrementalSync_CodexAppend(t *testing.T) { assert.Equal(t, 1, sess.UserMessageCount) } -func TestSyncPathsCodexSameStatInPlaceRewriteUsesContentHash(t *testing.T) { +func TestSyncPathsCodexSameStatInPlaceRewriteRejectedByCheckpoint(t *testing.T) { env := setupSingleAgentTestEnv(t, parser.AgentCodex) const uuid = "019eb791-cf7d-75c1-8439-9ed74c1229f5" @@ -13784,19 +13786,24 @@ func TestSyncPathsCodexSameStatInPlaceRewriteUsesContentHash(t *testing.T) { env.engine.SyncPaths([]string{path}) + // The stored change-time no longer matches after the rewrite, so the + // checkpoint no-op path must decline and the engine must re-parse the + // rewritten bytes. msgs := fetchMessages(t, env.db, "codex:"+uuid) require.Len(t, msgs, 1) - assert.Equal(t, "bravo request", msgs[0].Content) + assert.Equal(t, "bravo request", msgs[0].Content, + "a same-stat rewrite must be re-parsed, not trusted") after, err := env.db.GetSessionFull(context.Background(), "codex:"+uuid) require.NoError(t, err) require.NotNil(t, after) require.NotNil(t, after.FileHash) - assert.False(t, after.LastWriteIncremental, - "same-size rewrite must use a full replacement") - assert.NotEqual(t, beforeHash, *after.FileHash) - wantHash, err := sync.ComputeFileHash(path) + assert.NotEqual(t, beforeHash, *after.FileHash, + "the re-parse must refresh the stored hash") + cp, ok, err := env.db.GetParserCheckpoint("codex:" + uuid) require.NoError(t, err) - assert.Equal(t, wantHash, *after.FileHash) + require.True(t, ok) + assert.Equal(t, int64(len(original)), cp.Offset, + "the rewritten bytes keep the same length, so the offset stays") } func TestSyncAllCodexPathRewriterSameStatRewriteUsesContentHash(t *testing.T) { @@ -14536,6 +14543,9 @@ func TestIncrementalSync_CodexExecAppendRetainsEvents(t *testing.T) { "rollout-20240101-inc-cx-exec.jsonl", initial, ) env.engine.SyncAll(context.Background(), nil) + before := fetchMessages(t, env.db, "codex:inc-cx-exec") + require.Len(t, before, 2) + toolMessageID := before[1].ID appended := testjsonl.JoinJSONL( testjsonl.CodexFunctionCallOutputJSON( @@ -14554,9 +14564,31 @@ func TestIncrementalSync_CodexExecAppendRetainsEvents(t *testing.T) { msgs := fetchMessages(t, env.db, "codex:inc-cx-exec") require.Len(t, msgs, 2) + assert.Equal(t, toolMessageID, msgs[1].ID, + "result-only append must preserve the existing tool message") require.Len(t, msgs[1].ToolCalls, 1) - assert.Equal(t, "exec_command", msgs[1].ToolCalls[0].ToolName, "tool name") - assert.Equal(t, "done", msgs[1].ToolCalls[0].ResultContent, "result_content") + call := msgs[1].ToolCalls[0] + assert.Equal(t, "exec_command", call.ToolName, "tool name") + assert.Equal(t, "done", call.ResultContent, "result_content") + require.Len(t, call.ResultEvents, 1) + assert.Equal(t, "function_call_output", call.ResultEvents[0].Source) + assert.Equal(t, "done", call.ResultEvents[0].Content) + afterIncremental, err := env.db.GetSessionFull( + context.Background(), "codex:inc-cx-exec", + ) + require.NoError(t, err) + require.NotNil(t, afterIncremental) + assert.True(t, afterIncremental.LastWriteIncremental) + + env.engine.ResyncAll(context.Background(), nil) + fullMsgs := fetchMessages(t, env.db, "codex:inc-cx-exec") + require.Len(t, fullMsgs, 2) + require.Len(t, fullMsgs[1].ToolCalls, 1) + fullCall := fullMsgs[1].ToolCalls[0] + assert.Equal(t, call.ResultContent, fullCall.ResultContent) + assert.Equal(t, call.ResultContentLength, fullCall.ResultContentLength) + assert.Equal(t, call.ResultEvents, fullCall.ResultEvents, + "incremental and authoritative full parses must store the same events") } func TestIncrementalSync_CodexLateTokenCountRewritesStoredMessage(t *testing.T) { diff --git a/internal/sync/engine_test.go b/internal/sync/engine_test.go index eeeaff928..b6ac7bdda 100644 --- a/internal/sync/engine_test.go +++ b/internal/sync/engine_test.go @@ -33,6 +33,80 @@ func openTestDB(t *testing.T) *db.DB { return dbtest.OpenTestDB(t) } +func TestIncrementalClaudeLateResultLinkScansDefiniteSecret(t *testing.T) { + // Assemble the AWS-shaped fixture key at runtime so push protection + // does not treat the test source itself as a leaked credential. + secret := "AKIA" + "7QHWN2DKR4FYPLJM" + root := t.TempDir() + projectDir := filepath.Join(root, "proj-a") + require.NoError(t, os.MkdirAll(projectDir, 0o755)) + path := filepath.Join(projectDir, "session.jsonl") + initial := testjsonl.JoinJSONL( + testjsonl.ClaudeUserJSON("hello", "2024-01-01T10:00:00Z"), + testjsonl.ClaudeAssistantJSON("hi", "2024-01-01T10:00:01Z"), + `{"type":"assistant","uuid":"a2","parentUuid":"a1",`+ + `"timestamp":"2024-01-01T10:00:02Z",`+ + `"message":{"id":"msg_tool","content":[{"type":"tool_use",`+ + `"id":"toolu_r","name":"Bash","input":{"command":"ls"}}]}}`, + ) + require.NoError(t, os.WriteFile(path, []byte(initial), 0o600)) + + database := openTestDB(t) + engine := NewEngine(database, EngineConfig{ + AgentDirs: map[parser.AgentType][]string{ + parser.AgentClaude: {root}, + }, + Machine: "local", + }) + t.Cleanup(engine.Close) + + require.Equal(t, 1, engine.SyncAll(t.Context(), nil).Synced) + ids, err := database.ListSessionIDsByFilePath(path, "claude") + require.NoError(t, err) + require.Len(t, ids, 1) + sessionID := ids[0] + + appended := `{"type":"user","timestamp":"2024-01-01T10:00:05Z",` + + `"uuid":"u2","parentUuid":"a2","message":{"content":[` + + `{"type":"tool_result","tool_use_id":"toolu_r",` + + `"content":"` + secret + `","is_error":false}]}}` + "\n" + f, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0o600) + require.NoError(t, err) + _, err = f.WriteString(appended) + require.NoError(t, err) + require.NoError(t, f.Close()) + + require.Equal(t, 1, engine.SyncAll(t.Context(), nil).Synced) + sess, err := database.GetSessionFull(t.Context(), sessionID) + require.NoError(t, err) + require.True(t, sess.LastWriteIncremental, + "the late result must take the incremental path") + + var stored string + require.NoError(t, database.Reader().QueryRow( + `SELECT COALESCE(result_content, '') FROM tool_calls + WHERE session_id = ? AND tool_use_id = ?`, + sessionID, "toolu_r", + ).Scan(&stored)) + require.Contains(t, stored, secret, + "the late link must update the stored result content") + + findings, err := database.SessionSecretFindings( + t.Context(), sessionID, + ) + require.NoError(t, err) + found := false + for _, finding := range findings { + if finding.LocationKind == "tool_result" && + strings.Contains(finding.RedactedMatch, "AKIA") { + found = true + break + } + } + assert.True(t, found, + "a definite secret in a late Claude result link must be reported") +} + func requireClassifyPaths( t *testing.T, engine *Engine, paths []string, ) []parser.DiscoveredFile { @@ -6753,12 +6827,13 @@ func TestProjectIdentityIncrementalStatePreservesExplicitSourceProject( func( _ string, inc *db.IncrementalInfo, - ) ([]parser.ParsedMessage, []parser.ClaudeSubagentLink, time.Time, int64, *string, error) { + ) ([]parser.ParsedMessage, []parser.ClaudeSubagentLink, []parser.ParsedToolCallUpdate, []parser.ParsedMessageTokenUsageUpdate, time.Time, int64, *string, []byte, error) { return []parser.ParsedMessage{{ Role: parser.RoleAssistant, Content: "appended", Ordinal: inc.NextOrdinal, - }}, nil, appendedInfo.ModTime(), int64(len(appended)), nil, nil + }}, nil, nil, nil, appendedInfo.ModTime(), int64(len(appended)), nil, nil, nil }, + nil, "", nil, ) require.True(t, ok) require.NotNil(t, result.incremental) @@ -6854,10 +6929,11 @@ func TestProjectIdentityLegacyMappedSnapshotReparsesBeforeIncrementalAppend( func( _ string, _ *db.IncrementalInfo, - ) ([]parser.ParsedMessage, []parser.ClaudeSubagentLink, time.Time, int64, *string, error) { + ) ([]parser.ParsedMessage, []parser.ClaudeSubagentLink, []parser.ParsedToolCallUpdate, []parser.ParsedMessageTokenUsageUpdate, time.Time, int64, *string, []byte, error) { parseCalled = true - return nil, nil, time.Time{}, 0, nil, nil + return nil, nil, nil, nil, time.Time{}, 0, nil, nil, nil }, + nil, "", nil, ) assert.False(t, ok, "legacy snapshots must fall through to a source-aware full parse") @@ -7890,15 +7966,25 @@ func TestProcessFileCodexDBFreshSkipIsNotCached(t *testing.T) { }, } - res := e.processFile(context.Background(), parser.DiscoveredFile{ - Agent: parser.AgentCodex, - Path: path, - Machine: "host", - }) - require.NoError(t, res.err) - require.True(t, res.skip) - assert.True(t, res.noCacheSkip) - assert.Empty(t, e.SnapshotSkipCache()) + _ = parser.AgentCodex + // A checkpointless stored session earns one lazy bootstrap: the first + // sync parses authoritatively and persists the checkpoint atomically + // with the content. The second sync then takes the fresh-session skip + // and must not cache it. + stats := e.SyncAll(context.Background(), nil) + require.Zero(t, stats.Failed) + require.Equal(t, 1, stats.Synced) + _, cpOk, cpErr := database.GetParserCheckpoint("host~codex:abc") + require.NoError(t, cpErr) + require.True(t, cpOk, + "the bootstrap must persist the checkpoint") + + stats = e.SyncAll(context.Background(), nil) + require.Zero(t, stats.Failed) + require.Zero(t, stats.Synced, + "the second sync must skip the fresh session") + assert.Empty(t, e.SnapshotSkipCache(), + "the fresh skip must not be cached") } func TestClassifyCodexIndexPathSkipsMissingTranscript(t *testing.T) { @@ -8322,6 +8408,7 @@ func TestTryProviderIncrementalAppendPassesPersistedSessionID(t *testing.T) { Size: info.Size(), MTimeNS: info.ModTime().UnixNano(), }, + nil, "", nil, nil, ) require.True(t, applied) diff --git a/internal/sync/parse_retention.go b/internal/sync/parse_retention.go index 29a0b9eee..0a0fac5e1 100644 --- a/internal/sync/parse_retention.go +++ b/internal/sync/parse_retention.go @@ -16,6 +16,10 @@ const ( parseRetentionFixedBytes = int64(64 << 10) parseRetentionMultiplier = int64(4) parseRetentionScavengeThreshold = int64(16 << 20) + // parseBatchBytesLimit bounds a pending write batch by estimated source + // bytes as well as by session count: a 5KiB session and a 945MiB session + // must not both count as "1". Batches flush when either limit is reached. + parseBatchBytesLimit = defaultParseRetentionBytes ) type parseRetentionBudget struct { @@ -43,28 +47,20 @@ func newParseRetentionBudget(capacity int64) *parseRetentionBudget { } // newBulkParseRetentionBudget returns the budget archive-scale passes use -// (full sync, resync rebuild, remote import processing). It never throttles -// parse admission: peak memory during a bulk pass is bounded by worker -// parallelism, not by a byte budget. Instead it releases the pass's retained -// memory back to the OS in one scavenge once the pass completes, keeping the -// long-running daemon's settled footprint low without serializing the pass. +// (full sync, resync rebuild, remote import processing). Bulk passes are +// byte-budgeted like every other pass: large sources acquire the whole +// capacity and run exclusively, small sources share it, and batches flush +// when the pending estimated bytes reach the capacity. This keeps a bulk +// pass's peak parse memory bounded by the configured budget plus one +// oversized parse instead of by worker parallelism. The pass still releases +// retained memory back to the OS in one scavenge once it completes. func newBulkParseRetentionBudget() *parseRetentionBudget { - return &parseRetentionBudget{ - pressure: make(chan struct{}, 1), - scavenge: debug.FreeOSMemory, - } + return newParseRetentionBudget(defaultParseRetentionBytes) } func (budget *parseRetentionBudget) acquire( ctx context.Context, sourceBytes int64, ) (*parseRetentionLease, error) { - if budget.weighted == nil { - // Bulk pass: admit immediately and remember that parsed payloads - // were retained so the end-of-pass scavenge runs exactly once. - budget.scavengePending.Store(true) - budget.acquired.Add(1) - return &parseRetentionLease{}, nil - } weight := budget.weight(sourceBytes) if budget.weighted.TryAcquire(weight) { budget.noteKnownLargeSource(sourceBytes) @@ -140,6 +136,19 @@ func releaseParseRetentionLeases(leases []*parseRetentionLease) { } } +// parseRetentionAdmissionBytes applies the byte-weighted memory admission to +// Codex-family sources only. Other providers retain the archive worker +// parallelism they had before Codex checkpointing; their source size still +// remains available separately for metrics and batch accounting. +func parseRetentionAdmissionBytes( + file parser.DiscoveredFile, sourceBytes int64, +) int64 { + if isCodexFormatAgent(file.Agent) { + return sourceBytes + } + return 1 +} + func parseRetentionSourceBytes(file parser.DiscoveredFile) int64 { if file.SourceSize > 0 { return file.SourceSize diff --git a/internal/sync/parse_retention_test.go b/internal/sync/parse_retention_test.go index 17407b697..c0c57a648 100644 --- a/internal/sync/parse_retention_test.go +++ b/internal/sync/parse_retention_test.go @@ -63,7 +63,7 @@ func TestWarmNoopSyncAcquiresNoRetentionLeases(t *testing.T) { "warm no-op pass must not acquire parse-retention leases") } -func TestFullSyncPassIsUnthrottledAndScavengesOnce(t *testing.T) { +func TestFullSyncPassIsByteBudgeted(t *testing.T) { e, ctx := newWarmBenchEngine(t) var scavenges int e.bulkRetentionBudget = newBulkParseRetentionBudget() @@ -73,16 +73,18 @@ func TestFullSyncPassIsUnthrottledAndScavengesOnce(t *testing.T) { acquired := e.bulkRetentionBudget.acquired.Load() require.Positive(t, acquired, "full pass must admit parses through the bulk budget") + require.NotNil(t, e.bulkRetentionBudget.weighted, + "bulk pass must run under the weighted byte budget") if e.parseRetentionBudget != nil { assert.Zero(t, e.parseRetentionBudget.acquired.Load(), "full pass must not consume the bounded daemon budget") } - assert.Equal(t, 1, scavenges, - "a parse-bearing bulk pass must release memory once at the end") + assert.Zero(t, scavenges, + "small sources do not set the scavenge flag (covered by the large-source test)") stats := e.SyncAll(ctx, nil) // warm pass: everything skips require.Equal(t, 0, stats.Synced) - assert.Equal(t, 1, scavenges, + assert.Zero(t, scavenges, "a warm no-op pass must not force another scavenge") assert.Nil(t, e.activeRetention.Load(), "bulk budget must be uninstalled after the pass") @@ -101,20 +103,23 @@ func TestScopedSyncKeepsBoundedRetentionBudget(t *testing.T) { "a cutoff-scoped pass must not create the bulk budget") } -func TestBulkParseRetentionBudgetNeverBlocks(t *testing.T) { +func TestBulkParseRetentionBudgetUsesWeightedAdmission(t *testing.T) { budget := newBulkParseRetentionBudget() first, err := budget.acquire(t.Context(), defaultParseRetentionBytes) require.NoError(t, err) + ctx, cancel := context.WithTimeout(t.Context(), 50*time.Millisecond) + defer cancel() + _, err = budget.acquire(ctx, defaultParseRetentionBytes) + assert.ErrorIs(t, err, context.DeadlineExceeded, + "a second oversized bulk admission must wait for capacity") + first.Release() second, err := budget.acquire(t.Context(), defaultParseRetentionBytes) require.NoError(t, err) - assert.False(t, budget.underPressure(), - "bulk admissions must never report pressure") - first.Release() second.Release() assert.Equal(t, int64(2), budget.acquired.Load()) } -func TestBulkParseRetentionBudgetScavengesOncePerParseBearingPass(t *testing.T) { +func TestBulkParseRetentionBudgetScavengesOnceAfterLargeSource(t *testing.T) { budget := newBulkParseRetentionBudget() var scavenges int budget.scavenge = func() { scavenges++ } @@ -122,7 +127,9 @@ func TestBulkParseRetentionBudgetScavengesOncePerParseBearingPass(t *testing.T) budget.scavengeIfNeeded() assert.Zero(t, scavenges, "a pass with no parses must not scavenge") - lease, err := budget.acquire(t.Context(), 1) + lease, err := budget.acquire( + t.Context(), parseRetentionScavengeThreshold, + ) require.NoError(t, err) lease.Release() budget.scavengeIfNeeded() @@ -131,6 +138,42 @@ func TestBulkParseRetentionBudgetScavengesOncePerParseBearingPass(t *testing.T) "one parse-bearing pass needs exactly one end-of-pass scavenge") } +func TestCollectAndBatchFlushesOnByteCap(t *testing.T) { + engine := NewEngine(openTestDB(t), EngineConfig{Machine: "local"}) + t.Cleanup(engine.Close) + var batchLengths []int + engine.writeBatchOverride = func( + batch []pendingWrite, _ syncWriteMode, _ bool, + ) (int, int, int, int) { + batchLengths = append(batchLengths, len(batch)) + return len(batch), 0, 0, 0 + } + results := make(chan syncJob, 2) + for i := range 2 { + results <- syncJob{ + path: fmt.Sprintf("/sessions/large-%d.jsonl", i), + processResult: processResult{ + sourceBytes: parseBatchBytesLimit, + results: []parser.ParseResult{{ + Session: parser.ParsedSession{ + ID: fmt.Sprintf("byte-cap-%d", i), + Agent: parser.AgentClaude, + }, + }}, + }, + } + } + close(results) + + stats := engine.collectAndBatch( + t.Context(), results, 2, 2, nil, syncWriteDefault, + ) + + assert.Equal(t, []int{1, 1}, batchLengths, + "each oversized pending result must flush its own batch") + assert.Equal(t, 2, stats.Synced) +} + func TestParseRetentionBudgetBoundsConcurrentSourceWeight(t *testing.T) { budget := newParseRetentionBudget(defaultParseRetentionBytes) first, err := budget.acquire(t.Context(), 7<<20) @@ -345,7 +388,10 @@ func TestCollectAndBatchReportsOrderedBulkFinalization(t *testing.T) { restore := engine.beginBulkRetentionPass() defer restore() budget := engine.retentionBudget() - lease, err := budget.acquire(t.Context(), 1) + // The end-of-pass scavenge is armed only once a source at or above the + // scavenge threshold was retained; a bulk pass over small sources leaves + // nothing worth returning to the OS. + lease, err := budget.acquire(t.Context(), parseRetentionScavengeThreshold) require.NoError(t, err) scavengeEntered := make(chan struct{}) @@ -562,13 +608,16 @@ func TestCollectAndBatchKeepsFanoutUnderOneLeaseUntilOneWrite(t *testing.T) { } func TestStartWorkersFlushesBelowBatchUnderAdmissionPressure(t *testing.T) { - const agent parser.AgentType = "retention-test" + // Codex-format providers retain the weighted source size during parse. + // Non-Codex providers intentionally use unit admission; the separate + // TestParseRetentionAdmissionIsCodexOnly pins that design boundary. + const agent = parser.AgentTraeX provider := &directStreamingProvider{ Def: parser.AgentDef{Type: agent}, parseOutcome: parser.ParseOutcome{ Results: []parser.ParseResultOutcome{{ Result: parser.ParseResult{Session: parser.ParsedSession{ - ID: "retention-test:session", Agent: agent, + ID: "traex:retention-test-session", Agent: agent, }}, }}, ResultSetComplete: true, @@ -679,3 +728,19 @@ func TestStartWorkersCancellationReleasesAdmissionWaiters(t *testing.T) { require.NoError(t, err, "canceled waiters must not leak weighted capacity") next.Release() } + +func TestParseRetentionAdmissionIsCodexOnly(t *testing.T) { + const sourceBytes = int64(32 << 20) + assert.Equal(t, sourceBytes, parseRetentionAdmissionBytes( + parser.DiscoveredFile{Agent: parser.AgentCodex}, sourceBytes, + )) + assert.Equal(t, sourceBytes, parseRetentionAdmissionBytes( + parser.DiscoveredFile{Agent: parser.AgentTraeX}, sourceBytes, + )) + assert.Equal(t, int64(1), parseRetentionAdmissionBytes( + parser.DiscoveredFile{Agent: parser.AgentClaude}, sourceBytes, + )) + assert.Equal(t, int64(1), parseRetentionAdmissionBytes( + parser.DiscoveredFile{Agent: parser.AgentGemini}, sourceBytes, + )) +} diff --git a/internal/sync/parsediff.go b/internal/sync/parsediff.go index 988c3f4d0..13533399a 100644 --- a/internal/sync/parsediff.go +++ b/internal/sync/parsediff.go @@ -149,13 +149,13 @@ func (e *Engine) ParseDiff(ctx context.Context, opts ParseDiffOptions) (*ParseDi // Workers emit ctx.Err() for files skipped after // cancellation. cancel() - r.releaseRetention() + r.releaseAll() drainResults(results, total-i-1) return nil, ctx.Err() } if r.incremental != nil { cancel() - r.releaseRetention() + r.releaseAll() drainResults(results, total-i-1) return nil, fmt.Errorf( "parse-diff: internal error: incremental parse of %s "+ @@ -167,11 +167,11 @@ func (e *Engine) ParseDiff(ctx context.Context, opts ParseDiffOptions) (*ParseDi visited, resolver, &presencePaths, ); err != nil { cancel() - r.releaseRetention() + r.releaseAll() drainResults(results, total-i-1) return nil, err } - r.releaseRetention() + r.releaseAll() if opts.Progress != nil { opts.Progress(i+1, total) } diff --git a/internal/sync/s3.go b/internal/sync/s3.go index 2a97f4319..9e825310b 100644 --- a/internal/sync/s3.go +++ b/internal/sync/s3.go @@ -294,7 +294,8 @@ func (e *Engine) processS3Session( // so acquire the retention lease that bounds the materialized-and-parsed // payload just before the object is fetched and parsed. Every result from // here carries the lease; releaseRetention frees it after consumption. - lease, err := e.retentionBudget().acquire(ctx, parseRetentionSourceBytes(file)) + sourceBytes := parseRetentionSourceBytes(file) + lease, err := e.retentionBudget().acquire(ctx, sourceBytes) if err != nil { return processResult{err: err} } @@ -393,6 +394,7 @@ func (e *Engine) processS3Session( res.excludedSessionIDs = applyIDPrefixToIDs( idPrefix, res.excludedSessionIDs, ) + res.sourceBytes = sourceBytes switch file.Agent { case parser.AgentClaude: missing, err := e.claudeSourceMissingSessionOwnershipsForCompleteResult( diff --git a/internal/sync/secret_scan.go b/internal/sync/secret_scan.go index 9a51bee84..aa6cacc37 100644 --- a/internal/sync/secret_scan.go +++ b/internal/sync/secret_scan.go @@ -1,10 +1,22 @@ package sync import ( + "sync/atomic" + "go.kenn.io/agentsview/internal/db" "go.kenn.io/agentsview/internal/secrets" ) +// secretScanBytes counts the content bytes passed to the secret scanner. +// Tests use deltas of it to gate the incremental path: a maintained delta +// must scan no more than the delta's own content. +var secretScanBytes atomic.Int64 + +// SecretScanBytes returns the total scanned byte count so far. Monotonic. +func SecretScanBytes() int64 { + return secretScanBytes.Load() +} + // computeSignalsAndSecrets computes a session's signal update and its secret // findings from the same message slice, returning the update with the // secret-leak count and rules version already populated. Every sync write @@ -25,6 +37,21 @@ func computeSignalsAndSecrets( return update, findings } +// computeSignalsAndSecretsWithContentFailures is the staged streaming +// variant: the signal pass receives pre-computed per-call content-failure +// verdicts in place of the placeholder result content, while the secret +// scan is unchanged (staged event findings arrive through the sink's +// Findings instead). +func computeSignalsAndSecretsWithContentFailures( + s db.Session, msgs []db.Message, failures map[string]bool, +) (db.SessionSignalUpdate, []db.SecretFinding) { + update := computeSignalsFromMessagesWithContentFailures(s, msgs, failures) + findings, leak := scanSecretsFromMessages(s, msgs, secrets.ScanDefinite) + update.SecretLeakCount = leak + update.SecretsRulesVersion = secrets.DefiniteRulesVersion() + return update, findings +} + // scanSecretsFromMessages detects secrets across a session's message content, // tool inputs, and canonical tool output (result events when present, else // result_content) using scan: secrets.ScanDefinite for the fast inline path, @@ -35,7 +62,8 @@ func scanSecretsFromMessages( _ db.Session, msgs []db.Message, scan func(string) []secrets.Match, ) (findings []db.SecretFinding, definiteCount int) { findings = make([]db.SecretFinding, 0) - add := func(sessionID, loc string, ord int, call, event *int, matches []secrets.Match) { + add := func(sessionID, loc string, ord int, call, event *int, content string, matches []secrets.Match) { + secretScanBytes.Add(int64(len(content))) for _, m := range matches { findings = append(findings, db.SecretFinding{ SessionID: sessionID, @@ -49,6 +77,11 @@ func scanSecretsFromMessages( MatchEnd: m.End, MatchIndex: m.Index, RedactedMatch: m.Redacted, + // The incremental persist path (applySignalDeltaTx) inserts + // f.RulesVersion verbatim, unlike replaceSecretFindingsTx + // which overrides it; stamp it here so inline findings are + // visible to current-version listings. + RulesVersion: secrets.DefiniteRulesVersion(), }) if m.Confidence == secrets.ConfidenceDefinite { definiteCount++ @@ -57,12 +90,12 @@ func scanSecretsFromMessages( } for _, msg := range msgs { add(msg.SessionID, "message", msg.Ordinal, nil, nil, - scan(msg.Content)) + msg.Content, scan(msg.Content)) for ci := range msg.ToolCalls { tc := msg.ToolCalls[ci] callIdx := ci add(msg.SessionID, "tool_input", msg.Ordinal, &callIdx, nil, - scan(tc.InputJSON)) + tc.InputJSON, scan(tc.InputJSON)) if len(tc.ResultEvents) > 0 { for ei := range tc.ResultEvents { // Store the slice position, which is what the persistence @@ -71,11 +104,12 @@ func scanSecretsFromMessages( // normalized value, so --reveal can re-locate the source. evIdx := ei add(msg.SessionID, "tool_result_event", msg.Ordinal, - &callIdx, &evIdx, scan(tc.ResultEvents[ei].Content)) + &callIdx, &evIdx, tc.ResultEvents[ei].Content, + scan(tc.ResultEvents[ei].Content)) } } else { add(msg.SessionID, "tool_result", msg.Ordinal, &callIdx, nil, - scan(tc.ResultContent)) + tc.ResultContent, scan(tc.ResultContent)) } } } diff --git a/internal/sync/secret_scan_test.go b/internal/sync/secret_scan_test.go index 5bd5ce68e..fade71d8e 100644 --- a/internal/sync/secret_scan_test.go +++ b/internal/sync/secret_scan_test.go @@ -195,6 +195,34 @@ func TestScanSecretsBreakdown(t *testing.T) { } } +// TestScanSecretsFromMessagesStampsRulesVersion pins that every finding the +// inline scanner builds carries the current definite rules version. The +// incremental persist path (applySignalDeltaTx) inserts f.RulesVersion +// verbatim — unlike replaceSecretFindingsTx, which overrides it — so a +// missing stamp here would persist empty-version findings invisible to +// current-version listings. +func TestScanSecretsFromMessagesStampsRulesVersion(t *testing.T) { + sess := db.Session{ID: "s1"} + msgs := []db.Message{ + {SessionID: "s1", Ordinal: 0, Role: "user", + Content: "AKIA7QHWN2DKR4FYPLJM"}, + {SessionID: "s1", Ordinal: 1, Role: "assistant", + ToolCalls: []db.ToolCall{{ + ToolName: "Bash", ToolUseID: "tu1", + InputJSON: `{"command":"x"}`, + ResultContent: "AKIA7QHWN2DKR4FYPLJM", + }}}, + } + findings, _ := scanSecretsFromMessages( + sess, msgs, secrets.ScanDefinite, + ) + require.NotEmpty(t, findings, "expected definite findings") + for _, f := range findings { + assert.Equal(t, secrets.DefiniteRulesVersion(), f.RulesVersion, + "finding %s/%s has no rules version", f.LocationKind, f.RuleName) + } +} + func countConfidence(findings []db.SecretFinding, confidence string) int { n := 0 for _, f := range findings { diff --git a/internal/sync/signal_compute.go b/internal/sync/signal_compute.go index 0b4577463..4f558cf91 100644 --- a/internal/sync/signal_compute.go +++ b/internal/sync/signal_compute.go @@ -17,8 +17,61 @@ import ( // which reads msgs from the DB once and then calls this). func computeSignalsFromMessages( sess db.Session, msgs []db.Message, +) db.SessionSignalUpdate { + return computeSignalsFromToolRows(sess, msgs, extractToolCallRows(msgs)) +} + +// computeSignalsFromMessagesWithContentFailures is the staged streaming +// variant: tool rows whose placeholder content hides a pre-computed +// content-failure verdict get that verdict stamped before the signal pass, +// so content-driven tool-health signals match the collecting path byte for +// byte. +func computeSignalsFromMessagesWithContentFailures( + sess db.Session, msgs []db.Message, failures map[string]bool, ) db.SessionSignalUpdate { toolRows := extractToolCallRows(msgs) + patchToolCallRowsWithContentFailures(toolRows, msgs, failures) + return computeSignalsFromToolRows(sess, msgs, toolRows) +} + +// patchToolCallRowsWithContentFailures stamps pre-computed content-failure +// verdicts onto rows whose last event status is empty (status-driven +// verdicts win). toolRows must be the output of extractToolCallRows over +// the same msgs slice, so the walk order is identical. +func patchToolCallRowsWithContentFailures( + toolRows []signals.ToolCallRow, + msgs []db.Message, + failures map[string]bool, +) { + if len(failures) == 0 { + return + } + idx := 0 + callOccurrences := make(map[string]int) + for _, m := range msgs { + for _, tc := range m.ToolCalls { + if idx >= len(toolRows) { + return + } + failed := false + if tc.ToolUseID != "" { + occurrence := callOccurrences[tc.ToolUseID] + callOccurrences[tc.ToolUseID] = occurrence + 1 + failed = failures[db.StagedToolCallKey( + tc.ToolUseID, occurrence, + )] || failures[tc.ToolUseID] + } + if failed && toolRows[idx].EventStatus == "" { + toolRows[idx].ContentFailure = true + } + idx++ + } + } +} + +func computeSignalsFromToolRows( + sess db.Session, msgs []db.Message, toolRows []signals.ToolCallRow, +) db.SessionSignalUpdate { heuristics := signals.AnalyzeHeuristics(signals.HeuristicInput{ Messages: extractHeuristicMessages(msgs), ToolRows: toolRows, diff --git a/internal/sync/signal_maintain.go b/internal/sync/signal_maintain.go new file mode 100644 index 000000000..7e7aaea04 --- /dev/null +++ b/internal/sync/signal_maintain.go @@ -0,0 +1,537 @@ +package sync + +import ( + "context" + "fmt" + "maps" + "slices" + "time" + + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/secrets" + "go.kenn.io/agentsview/internal/signals" +) + +// incrementalSignalMaintainer folds one incremental write delta into a +// session's signal columns, secret findings, and compact state inside the +// write transaction. It declines (returns a nil delta) whenever the delta +// cannot be folded exactly — the write then invalidates the signal version +// and the debounced full recompute reseeds the state. +type incrementalSignalMaintainer struct { + engine *Engine + sessionID string + + // appended carries the sanitized, filtered message rows the write + // transaction is inserting. + appended []db.Message + // resultUpdates carries the sanitized late tool-result updates. + resultUpdates []db.ToolCallResultUpdate + // messageUsageUpdates carries token metadata attached to assistant + // messages committed before this incremental batch. These updates are + // ordered before appended messages in the canonical transcript. + messageUsageUpdates []db.MessageTokenUsageUpdate + // preWriteRevision is the transcript revision before this write's + // bump; the persisted state token must match it. + preWriteRevision string + + // preWriteSecretsVersion is the session's secrets rules version + // before this write. The incremental write transaction blanks the + // session's secrets_rules_version when it bumps the transcript + // revision (the recorded scan no longer covers the new rows), so the + // maintainer must compare the pre-write value against the current + // definite version instead of reading the blanked session row. + preWriteSecretsVersion string + + // qualitySignalVersion and secretsRulesVersion are the current + // detector versions. Maintenance is only valid when the persisted + // state and the pre-write session row are both at these versions; + // otherwise a stale-but-self-consistent session would be folded with + // newly-added rules and stamped current without rescanning history. + qualitySignalVersion int + secretsRulesVersion string +} + +func (e *Engine) newIncrementalSignalMaintainer( + inc *incrementalUpdate, + appended []db.Message, + resultUpdates []db.ToolCallResultUpdate, + messageUsageUpdates []db.MessageTokenUsageUpdate, + preWriteRevision, preWriteSecretsVersion string, +) db.SignalMaintainer { + return &incrementalSignalMaintainer{ + engine: e, + sessionID: inc.sessionID, + appended: appended, + resultUpdates: resultUpdates, + messageUsageUpdates: messageUsageUpdates, + preWriteRevision: preWriteRevision, + preWriteSecretsVersion: preWriteSecretsVersion, + qualitySignalVersion: db.CurrentQualitySignalVersion, + secretsRulesVersion: secrets.DefiniteRulesVersion(), + } +} + +func (m *incrementalSignalMaintainer) MaintainTx( + ctx context.Context, q db.SignalQuery, +) (*db.SignalDelta, error) { + sess, err := q.Session(ctx) + if err != nil { + return nil, err + } + if sess == nil { + return nil, nil + } + + stored, hasState, err := q.SignalState(ctx) + if err != nil { + return nil, err + } + if !hasState { + return nil, nil // seed via the debounced full recompute + } + var state signals.IncrementalState + if err := state.UnmarshalBinary(stored.State); err != nil { + return nil, nil + } + // The persisted state and the pre-write session row must both be at + // the current quality and secrets rules versions. Requiring the + // state's signal version to equal the current version (not merely the + // session's) and the pre-write secrets version to equal the current + // definite version prevents a rules-version upgrade from folding a + // stale-but-equal session and stamping it current without rescanning + // history. An empty pre-write secrets version means the row was never + // stamped, which also reseeds. + if stored.TranscriptRevision != m.preWriteRevision || + stored.SignalVersion != m.qualitySignalVersion || + sess.QualitySignalVersion != m.qualitySignalVersion || + m.preWriteSecretsVersion != m.secretsRulesVersion { + return nil, nil // state fell behind the rows: reseed + } + + // Appended tool-call rows in the same shape the full compute uses. + appendedRows := extractToolCallRows(m.appended) + + // Modified facts for late result updates, resolved in-transaction so + // the fold sees the post-update stored facts. + modified := make(map[signals.CallPos]signals.ToolFact) + deleteKeys := make([]db.FindingDeleteKey, 0, len(m.resultUpdates)) + var insertFindings []db.SecretFinding + positions := make([]db.ToolCallPosition, 0, len(m.resultUpdates)) + for _, u := range m.resultUpdates { + if u.ToolUseID != "" { + positions = append(positions, u.Position) + } + } + callFacts, err := q.ToolCallsByPosition(ctx, positions) + if err != nil { + return nil, err + } + factByPosition := make( + map[db.ToolCallPosition]db.ToolCallSignalFact, len(callFacts), + ) + for _, f := range callFacts { + factByPosition[db.ToolCallPosition{ + MessageOrdinal: f.MessageOrdinal, + CallIndex: f.CallIndex, + }] = f + } + for _, u := range m.resultUpdates { + fact, ok := factByPosition[u.Position] + if !ok { + continue // update targeted nothing stored + } + row := signals.ToolCallRow{ + ToolName: fact.ToolName, + Category: fact.Category, + InputJSON: fact.InputJSON, + ResultContent: fact.ResultContent, + MessageOrdinal: fact.MessageOrdinal, + CallIndex: fact.CallIndex, + EventStatus: fact.EventStatus, + } + f := signals.ToolFact{ + CallPos: signals.CallPos{ + MessageOrdinal: fact.MessageOrdinal, + CallIndex: fact.CallIndex, + }, + Failure: signals.IsFailure(row), + ExactSignature: signals.ExactToolSignature(row), + CommandClass: signals.CommandClass(row), + } + modified[f.CallPos] = f + + // Only the events this transaction inserted need scanning: + // previously stored events already carry findings, and rescanning + // the call's whole history makes repeated late outputs quadratic. + // When any event exists the full compute scans events instead of + // the result_content summary, so the summary-derived finding must + // go and the inserted events are scanned with their real indexes. + events := q.InsertedResultEvents(u.Position) + if len(events) == 0 { + continue + } + deleteKeys = append(deleteKeys, db.FindingDeleteKey{ + MessageOrdinal: fact.MessageOrdinal, + CallIndex: fact.CallIndex, + LocationKind: "tool_result", + }) + for _, ev := range events { + secretScanBytes.Add(int64(len(ev.Content))) + matches := secrets.ScanDefinite(ev.Content) + for _, match := range matches { + callIdx := fact.CallIndex + evIdx := ev.EventIndex + insertFindings = append(insertFindings, db.SecretFinding{ + SessionID: m.sessionID, + RuleName: match.Rule, + Confidence: match.Confidence, + LocationKind: "tool_result_event", + MessageOrdinal: fact.MessageOrdinal, + CallIndex: &callIdx, + EventIndex: &evIdx, + MatchStart: match.Start, + MatchEnd: match.End, + MatchIndex: match.Index, + RedactedMatch: match.Redacted, + RulesVersion: secrets.DefiniteRulesVersion(), + }) + } + } + } + + // Appended message content: scan exactly the stored rows. + newFindings, _ := scanSecretsFromMessages( + db.Session{}, m.appended, secrets.ScanDefinite, + ) + insertFindings = append(insertFindings, newFindings...) + + row := signals.ToolHealthRow{ + FailureCount: sess.ToolFailureSignalCount, + RetryCount: sess.ToolRetryCount, + EditChurnCount: sess.EditChurnCount, + } + nextState, toolHealth, ok := state.FoldToolHealth( + appendedRows, modified, row, + ) + if !ok { + return nil, nil // out-of-window modification: reseed + } + + // Message-derived aggregates. + lastRole, lastContent := nextState.LastRole, nextState.LastContent + msgIndex := nextState.MsgIndex + modelCounts := cloneCounts(nextState.ModelCounts) + modelFirstSeen := cloneCounts(nextState.ModelFirstSeen) + compactionDelta := 0 + lastTokens := nextState.LastValidTokens + lastTokensOrdinal := nextState.LastValidTokensOrdinal + appendedHasContextData := false + + // A resumed Codex tail may carry token_count for the final assistant + // message committed before the checkpoint. That message is already + // represented in the compact state, but its token-derived contribution + // is not. Apply the usage update before later appended messages so the + // compaction detector sees canonical message order without loading the + // historical transcript. + for _, usage := range m.messageUsageUpdates { + if !q.MessageTokenUsageUpdated(usage.Ordinal) || + !usage.HasContextTokens { + continue + } + if lastTokens > 0 && usage.Ordinal <= lastTokensOrdinal { + // A later assistant already contributed a newer measurement + // than the one this late update targets: applying it here + // would fold tokens out of chronological order and corrupt + // compaction detection and the tail token value. Decline and + // let the caller fall back to a full recompute. + return nil, nil + } + appendedHasContextData = true + if lastTokens > 0 && + float64(usage.ContextTokens) < 0.7*float64(lastTokens) { + compactionDelta++ + } + lastTokens = usage.ContextTokens + lastTokensOrdinal = usage.Ordinal + } + for _, msg := range m.appended { + msgIndex++ + if msg.IsSystem { + continue + } + lastRole, lastContent = string(msg.Role), msg.Content + if msg.Role == "assistant" && msg.Model != "" { + if _, seen := modelCounts[msg.Model]; !seen { + modelFirstSeen[msg.Model] = msgIndex + } + modelCounts[msg.Model]++ + } + if msg.Role == "assistant" && msg.HasContextTokens { + appendedHasContextData = true + if lastTokens > 0 && + float64(msg.ContextTokens) < 0.7*float64(lastTokens) { + compactionDelta++ + } + lastTokens = msg.ContextTokens + lastTokensOrdinal = msg.Ordinal + } + } + nextState.LastRole = lastRole + nextState.LastContent = lastContent + nextState.MsgIndex = msgIndex + nextState.ModelCounts = modelCounts + nextState.ModelFirstSeen = modelFirstSeen + nextState.LastValidTokens = lastTokens + nextState.LastValidTokensOrdinal = lastTokensOrdinal + + hasToolCalls := sess.HasToolCalls || len(appendedRows) > 0 + hasContextData := sess.HasContextData || appendedHasContextData + noCodeContext := sess.NoCodeContextCount + if noCodeContext > 0 && + slices.ContainsFunc(appendedRows, signals.IsContextToolCall) { + noCodeContext = 0 + } + // When the session has explicit compact boundaries the full compute + // derives the compaction count from the boundary count and ignores + // token-drop compactions; the fold must match by not adding the + // token-drop delta on top. + compactionCount := sess.CompactionCount + if !nextState.HasExplicitBoundaries { + compactionCount += compactionDelta + } + midTaskCount := sess.MidTaskCompactionCount + + toolHealth.MidTaskCompactions + + // Outcome and score from the compact aggregates. + var lastActivity time.Time + if sess.EndedAt != nil { + lastActivity, _ = time.Parse(time.RFC3339Nano, *sess.EndedAt) + } + outcomeResult := signals.ClassifyOutcome(signals.OutcomeInput{ + IsAutomated: sess.IsAutomated, + MessageCount: sess.MessageCount, + EndedWithRole: lastRole, + FinalFailureStreak: toolHealth.FinalFailureStreak, + LastAssistantText: lastContent, + LastActivity: lastActivity, + }) + model := mostCommonModelFromCounts(modelCounts, modelFirstSeen) + pressure := signals.ComputeContextPressure( + nil, sess.PeakContextTokens, model, + ) + scoreResult := signals.ComputeHealthScore(signals.ScoreInput{ + Outcome: outcomeResult.Outcome, + OutcomeConfidence: outcomeResult.Confidence, + HasToolCalls: hasToolCalls, + FailureSignalCount: toolHealth.FailureCount, + RetryCount: toolHealth.RetryCount, + EditChurnCount: toolHealth.EditChurnCount, + ConsecutiveFailMax: toolHealth.ConsecutiveFailureMax, + HasContextData: hasContextData, + CompactionCount: compactionCount, + MidTaskCompactionCount: midTaskCount, + PressureMax: pressure.PressureMax, + Heuristics: signals.HeuristicSignals{ + ShortPromptCount: sess.ShortPromptCount, + UnstructuredStart: sess.UnstructuredStart, + MissingSuccessCriteriaCount: sess.MissingSuccessCriteriaCount, + MissingVerificationCount: sess.MissingVerificationCount, + DuplicatePromptCount: sess.DuplicatePromptCount, + NoCodeContextCount: noCodeContext, + RunawayToolLoopCount: toolHealth.RunawayToolLoopCount, + }, + }) + + var pendingSince *string + if outcomeResult.IsRecent { + now := time.Now().UTC().Format(time.RFC3339) + pendingSince = &now + } + var healthGrade *string + if scoreResult.Grade != "" { + healthGrade = &scoreResult.Grade + } + + update := db.SessionSignalUpdate{ + ToolFailureSignalCount: toolHealth.FailureCount, + ToolRetryCount: toolHealth.RetryCount, + EditChurnCount: toolHealth.EditChurnCount, + ConsecutiveFailureMax: toolHealth.ConsecutiveFailureMax, + Outcome: outcomeResult.Outcome, + OutcomeConfidence: outcomeResult.Confidence, + EndedWithRole: lastRole, + FinalFailureStreak: toolHealth.FinalFailureStreak, + SignalsPendingSince: pendingSince, + CompactionCount: compactionCount, + MidTaskCompactionCount: midTaskCount, + ContextPressureMax: pressure.PressureMax, + HealthScore: scoreResult.Score, + HealthGrade: healthGrade, + HasToolCalls: hasToolCalls, + HasContextData: hasContextData, + SecretsRulesVersion: secrets.DefiniteRulesVersion(), + QualitySignals: db.QualitySignals{ + Version: db.CurrentQualitySignalVersion, + ShortPromptCount: sess.ShortPromptCount, + UnstructuredStart: sess.UnstructuredStart, + MissingSuccessCriteriaCount: sess. + MissingSuccessCriteriaCount, + MissingVerificationCount: sess. + MissingVerificationCount, + DuplicatePromptCount: sess.DuplicatePromptCount, + NoCodeContextCount: noCodeContext, + RunawayToolLoopCount: toolHealth.RunawayToolLoopCount, + }, + } + + revision, err := q.TranscriptRevision(ctx) + if err != nil { + return nil, err + } + blob, err := nextState.MarshalBinary() + if err != nil { + return nil, fmt.Errorf( + "encoding signal state %s: %w", m.sessionID, err, + ) + } + + return &db.SignalDelta{ + Update: update, + InsertFindings: insertFindings, + DeleteFindingKeys: deleteKeys, + State: &db.SessionSignalState{ + SessionID: m.sessionID, + State: blob, + TranscriptRevision: revision, + SignalVersion: db.CurrentQualitySignalVersion, + }, + }, nil +} + +func cloneCounts(in map[string]int) map[string]int { + out := maps.Clone(in) + if out == nil { + out = make(map[string]int) + } + return out +} + +// mostCommonModelFromCounts mirrors extractMostCommonModel's tie-break: +// the model appearing most often, ties broken by first chronological +// appearance. +func mostCommonModelFromCounts( + counts, firstSeen map[string]int, +) string { + var best string + bestCount := -1 + for model, n := range counts { + switch { + case n > bestCount: + best, bestCount = model, n + case n == bestCount && firstSeen[model] < firstSeen[best]: + best = model + } + } + return best +} + +// extractModelCounts mirrors extractMostCommonModel's walk: per-model +// counts, first chronological appearance (by message index), and the total +// message count the maintainer continues indexing from. +func extractModelCounts( + msgs []db.Message, +) (counts, firstSeen map[string]int, msgIndex int) { + counts = map[string]int{} + firstSeen = map[string]int{} + for i, m := range msgs { + if m.Role != "assistant" || m.Model == "" { + continue + } + counts[m.Model]++ + if _, ok := firstSeen[m.Model]; !ok { + firstSeen[m.Model] = i + } + } + return counts, firstSeen, len(msgs) +} + +// seedSignalStateFromFull builds and persists the compact incremental state +// after a synchronous full-content write so later incremental deltas can fold. +// Callers run under the engine's sync serialization; the conditional database +// write still refuses publication if another writer advances the transcript +// after the revision is captured. +func (e *Engine) seedSignalStateFromFull( + sessionID string, msgs []db.Message, +) error { + return e.seedSignalStateFromRows( + sessionID, msgs, extractToolCallRows(msgs), + ) +} + +// seedSignalStateFromFullWithContentFailures is the staged streaming +// variant: the seeded incremental state must see the same pre-computed +// content-failure verdicts the signal pass used, or later deltas fold +// against a different failure history. +func (e *Engine) seedSignalStateFromFullWithContentFailures( + sessionID string, msgs []db.Message, failures map[string]bool, +) error { + toolRows := extractToolCallRows(msgs) + patchToolCallRowsWithContentFailures(toolRows, msgs, failures) + return e.seedSignalStateFromRows(sessionID, msgs, toolRows) +} + +func (e *Engine) seedSignalStateFromRows( + sessionID string, msgs []db.Message, toolRows []signals.ToolCallRow, +) error { + rev, err := e.db.TranscriptRevision(sessionID) + if err != nil { + return err + } + state, err := buildSignalStateFromRows(sessionID, msgs, toolRows, rev) + if err != nil { + return err + } + _, err = e.db.UpsertSessionSignalStateIfRevision(state) + return err +} + +// buildSignalStateFromRows is the pure full-snapshot counterpart of the +// incremental fold. revision must be captured before the rows represented by +// msgs are loaded; callers that read from the database then publish through +// ReplaceSessionSignalsIfRevision so a concurrent transcript change cannot +// make stale aggregates appear current. +func buildSignalStateFromRows( + sessionID string, + msgs []db.Message, + toolRows []signals.ToolCallRow, + revision string, +) (db.SessionSignalState, error) { + lastRole, lastContent := extractLastMessageRole(msgs) + counts, firstSeen, msgIndex := extractModelCounts(msgs) + lastTokens, lastTokensOrdinal := 0, 0 + for _, m := range slices.Backward(msgs) { + if m.Role == "assistant" && m.HasContextTokens { + lastTokens = m.ContextTokens + lastTokensOrdinal = m.Ordinal + break + } + } + state := signals.SeedIncrementalState( + toolRows, + extractCompactBoundaryOrdinals(msgs), + lastRole, lastContent, + counts, firstSeen, msgIndex, lastTokens, lastTokensOrdinal, + ) + blob, err := state.MarshalBinary() + if err != nil { + return db.SessionSignalState{}, fmt.Errorf( + "encoding signal state %s: %w", sessionID, err, + ) + } + return db.SessionSignalState{ + SessionID: sessionID, + State: blob, + TranscriptRevision: revision, + SignalVersion: db.CurrentQualitySignalVersion, + }, nil +} diff --git a/internal/sync/signal_maintain_test.go b/internal/sync/signal_maintain_test.go new file mode 100644 index 000000000..a87be4b3b --- /dev/null +++ b/internal/sync/signal_maintain_test.go @@ -0,0 +1,1095 @@ +package sync + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/parser" + "go.kenn.io/agentsview/internal/secrets" + "go.kenn.io/agentsview/internal/signals" + "go.kenn.io/agentsview/internal/testjsonl" +) + +// fakeSignalQuery is a minimal in-memory db.SignalQuery for maintainer +// unit tests. It lets the maintainer run without a live transaction. +type fakeSignalQuery struct { + sess *db.Session + state db.SessionSignalState + hasState bool + revision string + callFacts []db.ToolCallSignalFact + // callEvents is the call's full stored history, returned by + // CallResultEvents. inserted maps a tool_use_id to the events this + // transaction inserted, returned by InsertedResultEvents. + callEvents []db.ToolResultEvent + inserted map[db.ToolCallPosition][]db.ToolResultEvent + updatedUsageOrdinal map[int]bool + requestedPositions []db.ToolCallPosition +} + +func (f *fakeSignalQuery) Session(context.Context) (*db.Session, error) { + return f.sess, nil +} + +func (f *fakeSignalQuery) TranscriptRevision(context.Context) (string, error) { + return f.revision, nil +} + +func (f *fakeSignalQuery) SignalState( + context.Context, +) (db.SessionSignalState, bool, error) { + return f.state, f.hasState, nil +} + +func (f *fakeSignalQuery) TrailingToolCalls( + context.Context, int, +) ([]db.ToolCallSignalFact, error) { + return nil, nil +} + +func (f *fakeSignalQuery) ToolCallsByPosition( + _ context.Context, positions []db.ToolCallPosition, +) ([]db.ToolCallSignalFact, error) { + f.requestedPositions = append([]db.ToolCallPosition(nil), positions...) + return f.callFacts, nil +} + +func (f *fakeSignalQuery) CallResultEvents( + context.Context, int, int, +) ([]db.ToolResultEvent, error) { + return f.callEvents, nil +} + +func (f *fakeSignalQuery) InsertedResultEvents( + position db.ToolCallPosition, +) []db.ToolResultEvent { + return f.inserted[position] +} + +func (f *fakeSignalQuery) MessageTokenUsageUpdated(ordinal int) bool { + return f.updatedUsageOrdinal[ordinal] +} + +// newTestMaintainer builds a maintainer stamped with the current quality +// and secrets rules versions, mirroring newIncrementalSignalMaintainer. +func newTestMaintainer( + preWriteRevision, preWriteSecretsVersion string, appended []db.Message, +) *incrementalSignalMaintainer { + return &incrementalSignalMaintainer{ + sessionID: "s1", + appended: appended, + preWriteRevision: preWriteRevision, + preWriteSecretsVersion: preWriteSecretsVersion, + qualitySignalVersion: db.CurrentQualitySignalVersion, + secretsRulesVersion: secrets.DefiniteRulesVersion(), + } +} + +func currentStateBlob(t *testing.T) []byte { + t.Helper() + state := signals.SeedIncrementalState( + nil, nil, "", "", nil, nil, 0, 0, 0, + ) + blob, err := state.MarshalBinary() + require.NoError(t, err) + return blob +} + +// TestIncrementalMaintainerDeclinesStaleVersions pins the version gate: a +// state or session whose quality/secret versions are not current must +// decline (nil delta) so the debounced full recompute reseeds — even when +// the stored state and session versions agree with each other but are both +// stale. +func TestIncrementalMaintainerDeclinesStaleVersions(t *testing.T) { + blob := currentStateBlob(t) + cases := []struct { + name string + sessQuality int + preWriteSecrets string + storedSignal int + }{ + { + name: "both versions stale but equal", + sessQuality: db.CurrentQualitySignalVersion - 1, + preWriteSecrets: secrets.DefiniteRulesVersion(), + storedSignal: db.CurrentQualitySignalVersion - 1, + }, + { + name: "stale stored signal version", + sessQuality: db.CurrentQualitySignalVersion, + preWriteSecrets: secrets.DefiniteRulesVersion(), + storedSignal: db.CurrentQualitySignalVersion - 1, + }, + { + name: "stale session quality version", + sessQuality: db.CurrentQualitySignalVersion - 1, + preWriteSecrets: secrets.DefiniteRulesVersion(), + storedSignal: db.CurrentQualitySignalVersion, + }, + { + name: "stale pre-write secrets version", + sessQuality: db.CurrentQualitySignalVersion, + preWriteSecrets: "stale-secrets-version", + storedSignal: db.CurrentQualitySignalVersion, + }, + { + name: "un-stamped pre-write secrets version", + sessQuality: db.CurrentQualitySignalVersion, + preWriteSecrets: "", + storedSignal: db.CurrentQualitySignalVersion, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + m := newTestMaintainer("rev", tc.preWriteSecrets, nil) + q := &fakeSignalQuery{ + sess: &db.Session{ + QualitySignalVersion: tc.sessQuality, + }, + state: db.SessionSignalState{ + State: blob, + TranscriptRevision: "rev", + SignalVersion: tc.storedSignal, + }, + hasState: true, + revision: "rev", + } + delta, err := m.MaintainTx(context.Background(), q) + require.NoError(t, err) + assert.Nil(t, delta, "stale versions must decline") + }) + } +} + +// TestIncrementalMaintainerProceedsCurrentVersions pins the positive case: +// with a current transcript revision, current quality version, and current +// secrets rules version, the maintainer must produce a delta. +func TestIncrementalMaintainerProceedsCurrentVersions(t *testing.T) { + m := newTestMaintainer("rev", secrets.DefiniteRulesVersion(), nil) + q := &fakeSignalQuery{ + sess: &db.Session{ + QualitySignalVersion: db.CurrentQualitySignalVersion, + }, + state: db.SessionSignalState{ + State: currentStateBlob(t), + TranscriptRevision: "rev", + SignalVersion: db.CurrentQualitySignalVersion, + }, + hasState: true, + revision: "rev", + } + delta, err := m.MaintainTx(context.Background(), q) + require.NoError(t, err) + require.NotNil(t, delta, + "current versions with a matching revision must proceed") +} + +func TestIncrementalMaintainerScansOnlyInsertedResultEvents(t *testing.T) { + const newContent = "new AKIA7QHWN2DKR4FYPLJM" + const oldContent = "old output that must not be rescanned" + + m := newTestMaintainer("rev", secrets.DefiniteRulesVersion(), nil) + m.resultUpdates = []db.ToolCallResultUpdate{{ + ToolUseID: "call_0", + Position: db.ToolCallPosition{MessageOrdinal: 0, CallIndex: 0}, + }} + state := signals.SeedIncrementalState( + []signals.ToolCallRow{{ + ToolName: "exec_command", + Category: "Bash", + InputJSON: "{}", + ResultContent: oldContent, + MessageOrdinal: 0, + CallIndex: 0, + EventStatus: "completed", + }}, + nil, "", "", nil, nil, 0, 0, 0, + ) + blob, err := state.MarshalBinary() + require.NoError(t, err) + q := &fakeSignalQuery{ + sess: &db.Session{ + QualitySignalVersion: db.CurrentQualitySignalVersion, + }, + state: db.SessionSignalState{ + State: blob, + TranscriptRevision: "rev", + SignalVersion: db.CurrentQualitySignalVersion, + }, + hasState: true, + revision: "rev", + callFacts: []db.ToolCallSignalFact{{ + MessageOrdinal: 0, + CallIndex: 0, + ToolName: "exec_command", + Category: "Bash", + InputJSON: "{}", + ResultContent: newContent, + ToolUseID: "call_0", + EventStatus: "completed", + }}, + callEvents: []db.ToolResultEvent{{ + ToolUseID: "call_0", + Source: "function_call_output", + Content: oldContent, + ContentLength: len(oldContent), + EventIndex: 0, + }}, + inserted: map[db.ToolCallPosition][]db.ToolResultEvent{ + {MessageOrdinal: 0, CallIndex: 0}: {{ + ToolUseID: "call_0", + Source: "function_call_output", + Content: newContent, + ContentLength: len(newContent), + EventIndex: 1, + }}, + }, + } + + scanBytesBefore := SecretScanBytes() + delta, err := m.MaintainTx(context.Background(), q) + require.NoError(t, err) + require.NotNil(t, delta) + scanned := SecretScanBytes() - scanBytesBefore + assert.LessOrEqual(t, scanned, int64(len(newContent)), + "the maintainer must scan only the newly inserted result events") + + var newEventFindings int + for _, finding := range delta.InsertFindings { + if finding.EventIndex != nil && *finding.EventIndex == 1 { + newEventFindings++ + } + } + assert.Equal(t, 1, newEventFindings, + "the inserted event's finding must land with its real index") +} + +func TestIncrementalMaintainerTargetsDuplicateCallIDOccurrence(t *testing.T) { + first := db.ToolCallPosition{MessageOrdinal: 0, CallIndex: 0} + second := db.ToolCallPosition{MessageOrdinal: 1, CallIndex: 0} + state := signals.SeedIncrementalState( + []signals.ToolCallRow{ + { + ToolName: "exec_command", Category: "Bash", InputJSON: `{}`, + ResultContent: "ok", MessageOrdinal: first.MessageOrdinal, + CallIndex: first.CallIndex, + }, + { + ToolName: "exec_command", Category: "Bash", InputJSON: `{}`, + ResultContent: "ok", MessageOrdinal: second.MessageOrdinal, + CallIndex: second.CallIndex, + }, + }, + nil, "", "", nil, nil, 0, 0, 0, + ) + blob, err := state.MarshalBinary() + require.NoError(t, err) + + m := newTestMaintainer("rev", secrets.DefiniteRulesVersion(), nil) + m.resultUpdates = []db.ToolCallResultUpdate{{ + ToolUseID: "reused-call", + Position: second, + }} + q := &fakeSignalQuery{ + sess: &db.Session{ + QualitySignalVersion: db.CurrentQualitySignalVersion, + }, + state: db.SessionSignalState{ + State: blob, + TranscriptRevision: "rev", + SignalVersion: db.CurrentQualitySignalVersion, + }, + hasState: true, + revision: "rev", + // Return both reused-ID facts in the opposite order. The maintainer + // must select by occurrence coordinates rather than whichever raw ID + // happens to win a map assignment. + callFacts: []db.ToolCallSignalFact{ + { + MessageOrdinal: second.MessageOrdinal, + CallIndex: second.CallIndex, + ToolName: "exec_command", + Category: "Bash", + InputJSON: `{}`, + ResultContent: "failed", + EventStatus: "errored", + ToolUseID: "reused-call", + }, + { + MessageOrdinal: first.MessageOrdinal, + CallIndex: first.CallIndex, + ToolName: "exec_command", + Category: "Bash", + InputJSON: `{}`, + ResultContent: "ok", + ToolUseID: "reused-call", + }, + }, + } + + delta, err := m.MaintainTx(context.Background(), q) + require.NoError(t, err) + require.NotNil(t, delta) + assert.Equal(t, []db.ToolCallPosition{second}, q.requestedPositions) + assert.Equal(t, 1, delta.Update.ToolFailureSignalCount, + "only the targeted reused-ID occurrence should become a failure") +} + +// TestIncrementalMaintainerCompactionExplicitBoundaryParity pins parity +// between the full compute and the incremental fold for compaction count: +// an explicit compact boundary suppresses token-drop compactions, so a +// session with both must count only the boundary. +func TestIncrementalMaintainerCompactionExplicitBoundaryParity(t *testing.T) { + boundary := db.Message{SessionID: "s1", Ordinal: 0, Role: "assistant", + IsCompactBoundary: true} + preCtx := db.Message{SessionID: "s1", Ordinal: 1, Role: "assistant", + HasContextTokens: true, ContextTokens: 1000} + drop := db.Message{SessionID: "s1", Ordinal: 2, Role: "assistant", + HasContextTokens: true, ContextTokens: 500} + + t.Run("explicit boundary suppresses token drop", func(t *testing.T) { + full := computeSignalsFromMessages( + db.Session{}, []db.Message{boundary, preCtx, drop}, + ) + require.Equal(t, 1, full.CompactionCount) + + state := signals.SeedIncrementalState( + nil, []int{0}, "", "", nil, nil, 0, 1000, 1, + ) + blob, err := state.MarshalBinary() + require.NoError(t, err) + + m := newTestMaintainer("rev", secrets.DefiniteRulesVersion(), []db.Message{drop}) + q := &fakeSignalQuery{ + sess: &db.Session{ + CompactionCount: 1, + QualitySignalVersion: db.CurrentQualitySignalVersion, + }, + state: db.SessionSignalState{ + State: blob, + TranscriptRevision: "rev", + SignalVersion: db.CurrentQualitySignalVersion, + }, + hasState: true, + revision: "rev", + } + delta, err := m.MaintainTx(context.Background(), q) + require.NoError(t, err) + require.NotNil(t, delta) + assert.Equal(t, full.CompactionCount, delta.Update.CompactionCount, + "explicit boundaries must suppress token-drop compactions") + }) + + t.Run("no boundary counts token drop", func(t *testing.T) { + full := computeSignalsFromMessages( + db.Session{}, []db.Message{preCtx, drop}, + ) + require.Equal(t, 1, full.CompactionCount) + + state := signals.SeedIncrementalState( + nil, nil, "", "", nil, nil, 0, 1000, 1, + ) + blob, err := state.MarshalBinary() + require.NoError(t, err) + + m := newTestMaintainer("rev", secrets.DefiniteRulesVersion(), []db.Message{drop}) + q := &fakeSignalQuery{ + sess: &db.Session{ + QualitySignalVersion: db.CurrentQualitySignalVersion, + }, + state: db.SessionSignalState{ + State: blob, + TranscriptRevision: "rev", + SignalVersion: db.CurrentQualitySignalVersion, + }, + hasState: true, + revision: "rev", + } + delta, err := m.MaintainTx(context.Background(), q) + require.NoError(t, err) + require.NotNil(t, delta) + assert.Equal(t, full.CompactionCount, delta.Update.CompactionCount, + "without a boundary the token drop must still be counted") + }) +} + +// TestIncrementalMaintainerDeclinesOutOfOrderLateUsage pins the guard for +// a resumed Codex tail whose token_count targets an assistant message +// earlier than one that already carries usage: a later sync can already +// have resolved a chronologically-later assistant's usage (via the +// in-memory backward-walk ApplyTokenUsageToLastAssistant performs when +// multiple token_count events land in one parse) before an earlier +// assistant's own token_count -- appearing later in the file -- is ever +// read. Folding that late update forward would treat an older +// measurement as newer and corrupt compaction detection. The maintainer +// must decline (nil, nil) so the caller falls back to a full recompute. +func TestIncrementalMaintainerDeclinesOutOfOrderLateUsage(t *testing.T) { + state := signals.SeedIncrementalState( + nil, nil, "", "", nil, nil, 0, 1000, 5, + ) + blob, err := state.MarshalBinary() + require.NoError(t, err) + + m := newTestMaintainer("rev", secrets.DefiniteRulesVersion(), nil) + m.messageUsageUpdates = []db.MessageTokenUsageUpdate{{ + Ordinal: 3, ContextTokens: 200, HasContextTokens: true, + }} + q := &fakeSignalQuery{ + sess: &db.Session{ + QualitySignalVersion: db.CurrentQualitySignalVersion, + }, + state: db.SessionSignalState{ + State: blob, + TranscriptRevision: "rev", + SignalVersion: db.CurrentQualitySignalVersion, + }, + hasState: true, + revision: "rev", + updatedUsageOrdinal: map[int]bool{3: true}, + } + delta, err := m.MaintainTx(context.Background(), q) + require.NoError(t, err) + assert.Nil(t, delta, + "a late usage update targeting an ordinal at or before the "+ + "recorded last measurement must decline, not fold out of order") +} + +// TestIncrementalMaintainerAppliesInOrderLateUsage is the companion +// positive case: a late usage update targeting an ordinal after the +// recorded last measurement folds normally and advances both the token +// value and its ordinal in the persisted compact state. +func TestIncrementalMaintainerAppliesInOrderLateUsage(t *testing.T) { + state := signals.SeedIncrementalState( + nil, nil, "", "", nil, nil, 0, 1000, 3, + ) + blob, err := state.MarshalBinary() + require.NoError(t, err) + + m := newTestMaintainer("rev", secrets.DefiniteRulesVersion(), nil) + m.messageUsageUpdates = []db.MessageTokenUsageUpdate{{ + Ordinal: 5, ContextTokens: 200, HasContextTokens: true, + }} + q := &fakeSignalQuery{ + sess: &db.Session{ + QualitySignalVersion: db.CurrentQualitySignalVersion, + }, + state: db.SessionSignalState{ + State: blob, + TranscriptRevision: "rev", + SignalVersion: db.CurrentQualitySignalVersion, + }, + hasState: true, + revision: "rev", + updatedUsageOrdinal: map[int]bool{5: true}, + } + delta, err := m.MaintainTx(context.Background(), q) + require.NoError(t, err) + require.NotNil(t, delta, + "a late usage update after the recorded last measurement must fold") + assert.Equal(t, 1, delta.Update.CompactionCount, + "the 1000->200 drop must be detected as a compaction") + + var next signals.IncrementalState + require.NoError(t, next.UnmarshalBinary(delta.State.State)) + assert.Equal(t, 200, next.LastValidTokens) + assert.Equal(t, 5, next.LastValidTokensOrdinal) +} + +// TestIncrementalMaintainerContextPressureSessionPeakParity pins parity +// for context pressure when the session-level peak exceeds per-message +// maxima: both paths must feed sess.PeakContextTokens into +// ComputeContextPressure and land the same ContextPressureMax. +func TestIncrementalMaintainerContextPressureSessionPeakParity(t *testing.T) { + const model = "claude-sonnet-4-5" + sess := db.Session{ + PeakContextTokens: 200_000, + HasPeakContextTokens: true, + QualitySignalVersion: db.CurrentQualitySignalVersion, + SecretsRulesVersion: secrets.DefiniteRulesVersion(), + } + msgs := []db.Message{{ + SessionID: "s1", Ordinal: 0, Role: "assistant", Model: model, + HasContextTokens: true, ContextTokens: 1000, + }} + full := computeSignalsFromMessages(sess, msgs) + require.NotNil(t, full.ContextPressureMax) + + state := signals.SeedIncrementalState( + nil, nil, "", "", + map[string]int{model: 1}, map[string]int{model: 0}, 1, 0, 0, + ) + blob, err := state.MarshalBinary() + require.NoError(t, err) + + m := newTestMaintainer("rev", secrets.DefiniteRulesVersion(), nil) + q := &fakeSignalQuery{ + sess: &sess, + state: db.SessionSignalState{ + State: blob, + TranscriptRevision: "rev", + SignalVersion: db.CurrentQualitySignalVersion, + }, + hasState: true, + revision: "rev", + } + delta, err := m.MaintainTx(context.Background(), q) + require.NoError(t, err) + require.NotNil(t, delta) + require.NotNil(t, delta.Update.ContextPressureMax) + assert.Equal(t, full.ContextPressureMax, delta.Update.ContextPressureMax, + "session-level peak must feed context pressure in both paths") +} + +// signalSnapshot captures the session signal columns for parity +// comparisons between the incremental maintainer and an authoritative +// full recompute. +type signalSnapshot struct { + ToolFailureSignalCount int + ToolRetryCount int + EditChurnCount int + ConsecutiveFailureMax int + Outcome string + OutcomeConfidence string + EndedWithRole string + FinalFailureStreak int + CompactionCount int + MidTaskCompactionCount int + ContextPressureMax *float64 + HealthScore *int + HealthGrade *string + HasToolCalls bool + HasContextData bool + SecretLeakCount int + QualitySignalVersion int + ShortPromptCount int + UnstructuredStart bool + MissingSuccessCriteria int + MissingVerificationCount int + DuplicatePromptCount int + NoCodeContextCount int + RunawayToolLoopCount int +} + +func snapshotSessionSignals(s *db.Session) signalSnapshot { + return signalSnapshot{ + ToolFailureSignalCount: s.ToolFailureSignalCount, + ToolRetryCount: s.ToolRetryCount, + EditChurnCount: s.EditChurnCount, + ConsecutiveFailureMax: s.ConsecutiveFailureMax, + Outcome: s.Outcome, + OutcomeConfidence: s.OutcomeConfidence, + EndedWithRole: s.EndedWithRole, + FinalFailureStreak: s.FinalFailureStreak, + CompactionCount: s.CompactionCount, + MidTaskCompactionCount: s.MidTaskCompactionCount, + ContextPressureMax: s.ContextPressureMax, + HealthScore: s.HealthScore, + HealthGrade: s.HealthGrade, + HasToolCalls: s.HasToolCalls, + HasContextData: s.HasContextData, + SecretLeakCount: s.SecretLeakCount, + QualitySignalVersion: s.QualitySignalVersion, + ShortPromptCount: s.ShortPromptCount, + UnstructuredStart: s.UnstructuredStart, + MissingSuccessCriteria: s.MissingSuccessCriteriaCount, + MissingVerificationCount: s.MissingVerificationCount, + DuplicatePromptCount: s.DuplicatePromptCount, + NoCodeContextCount: s.NoCodeContextCount, + RunawayToolLoopCount: s.RunawayToolLoopCount, + } +} + +const signalMaintainUUID = "019eb791-cf7d-75c1-8439-9ed74c122a01" + +// TestIncrementalSignalMaintainerParityWithFullResync drives a Codex +// session through a full sync, an incremental late-tool-output append, and +// an authoritative full reparse, then proves the maintained signal columns +// and secret findings equal the full recompute. It also gates the delta +// path: zero GetAllMessages calls and secret-scan bytes bounded by the +// delta content. +func TestIncrementalSignalMaintainerParityWithFullResync(t *testing.T) { + root := t.TempDir() + day := filepath.Join(root, "2024", "01", "01") + require.NoError(t, os.MkdirAll(day, 0o755)) + path := filepath.Join( + day, "rollout-2024-01-01T10-00-00-"+signalMaintainUUID+".jsonl", + ) + initial := testjsonl.JoinJSONL( + testjsonl.CodexSessionMetaJSON( + signalMaintainUUID, "/workspace/project", "codex_cli_rs", + "2024-01-01T10:00:00Z", + ), + testjsonl.CodexMsgJSON("user", "hello", "2024-01-01T10:00:01Z"), + testjsonl.CodexMsgJSON( + "assistant", "I will run the command.", + "2024-01-01T10:00:02Z", + ), + testjsonl.CodexFunctionCallWithCallIDJSON( + "exec_command", "call_0", nil, "2024-01-01T10:00:03Z", + ), + ) + require.NoError(t, os.WriteFile(path, []byte(initial), 0o644)) + + database := openTestDB(t) + engine := NewEngine(database, EngineConfig{ + AgentDirs: map[parser.AgentType][]string{ + parser.AgentCodex: {root}, + }, + Machine: "local", + }) + t.Cleanup(engine.Close) + + require.Equal(t, 1, engine.SyncAll(context.Background(), nil).Synced) + sessionID := "codex:" + signalMaintainUUID + + // Baseline: the full sync seeded the compact state. + state, ok, err := database.GetSessionSignalState(sessionID) + require.NoError(t, err) + require.True(t, ok, "full sync must seed the compact signal state") + + appended := testjsonl.JoinJSONL( + testjsonl.CodexFunctionCallWithCallIDJSON( + "exec_command", "call_1", nil, "2024-01-01T10:00:04Z", + ), + testjsonl.CodexFunctionCallOutputJSON( + "call_0", + "build passed AKIA7QHWN2DKR4FYPLJM", + "2024-01-01T10:00:05Z", + ), + ) + f, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0o644) + require.NoError(t, err) + _, err = f.WriteString(appended) + require.NoError(t, err) + require.NoError(t, f.Close()) + + loadsBefore := database.MessagesLoadCount() + scanBytesBefore := SecretScanBytes() + require.Equal(t, 1, engine.SyncAll(context.Background(), nil).Synced) + + // Delta gates: the maintained append must not load session history, + // and the secret scan must stay within the delta's own content. + assert.Equal(t, loadsBefore, database.MessagesLoadCount(), + "maintained delta must not call GetAllMessages") + assert.LessOrEqual(t, SecretScanBytes()-scanBytesBefore, + int64(len(appended)), + "secret scan bytes must not exceed the delta content") + + sess, err := database.GetSessionFull(context.Background(), sessionID) + require.NoError(t, err) + require.NotNil(t, sess) + require.True(t, sess.LastWriteIncremental, + "the append must take the incremental path") + require.Equal(t, db.CurrentQualitySignalVersion, sess.QualitySignalVersion, + "maintenance must keep the signal version current") + + findings, err := database.SessionSecretFindings( + context.Background(), sessionID, + ) + require.NoError(t, err) + require.Len(t, findings, 1, "the AWS key in the output must be found") + assert.Equal(t, "tool_result_event", findings[0].LocationKind) + + incrementalSignals := snapshotSessionSignals(sess) + + // Authoritative rebuild: rewrite the file in place (same size), + // drop the checkpoint, and bump the mtime so the engine takes the + // full-parse replacement path. + rewritten := testjsonl.JoinJSONL( + testjsonl.CodexSessionMetaJSON( + signalMaintainUUID, "/workspace/project", "codex_cli_rs", + "2024-01-01T10:00:00Z", + ), + testjsonl.CodexMsgJSON("user", "heylo", "2024-01-01T10:00:01Z"), + testjsonl.CodexMsgJSON( + "assistant", "I will run the command.", + "2024-01-01T10:00:02Z", + ), + testjsonl.CodexFunctionCallWithCallIDJSON( + "exec_command", "call_0", nil, "2024-01-01T10:00:03Z", + ), + testjsonl.CodexFunctionCallWithCallIDJSON( + "exec_command", "call_1", nil, "2024-01-01T10:00:04Z", + ), + testjsonl.CodexFunctionCallOutputJSON( + "call_0", + "build passed AKIA7QHWN2DKR4FYPLJM", + "2024-01-01T10:00:05Z", + ), + ) + require.Equal(t, len(initial)+len(appended), len(rewritten)) + require.NoError(t, database.DeleteParserCheckpoint(sessionID)) + require.NoError(t, os.WriteFile(path, []byte(rewritten), 0o644)) + future := time.Now().Add(2 * time.Minute) + require.NoError(t, os.Chtimes(path, future, future)) + + require.Equal(t, 1, engine.SyncAll(context.Background(), nil).Synced) + + sess, err = database.GetSessionFull(context.Background(), sessionID) + require.NoError(t, err) + require.NotNil(t, sess) + require.False(t, sess.LastWriteIncremental, + "the same-size rewrite must take the full replacement path") + + fullFindings, err := database.SessionSecretFindings( + context.Background(), sessionID, + ) + require.NoError(t, err) + assert.Equal(t, findings, fullFindings, + "incremental findings must match the authoritative full resync") + assert.Equal(t, incrementalSignals, snapshotSessionSignals(sess), + "incremental signals must match the authoritative full resync") + + // The full resync reseeds the state; a follow-up append must fold + // again without a decline. + appended2 := testjsonl.JoinJSONL(testjsonl.CodexFunctionCallOutputJSON( + "call_1", "second result", "2024-01-01T10:00:06Z", + )) + f, err = os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0o644) + require.NoError(t, err) + _, err = f.WriteString(appended2) + require.NoError(t, err) + require.NoError(t, f.Close()) + loadsBefore = database.MessagesLoadCount() + require.Equal(t, 1, engine.SyncAll(context.Background(), nil).Synced) + assert.Equal(t, loadsBefore, database.MessagesLoadCount(), + "post-resync append must fold incrementally") + sess, err = database.GetSessionFull(context.Background(), sessionID) + require.NoError(t, err) + require.Equal(t, db.CurrentQualitySignalVersion, sess.QualitySignalVersion) + _ = state +} + +// TestIncrementalSignalMaintainerHandlesCommittedUsageUpdate proves the +// real Codex late-output shape (function_call_output followed by token_count) +// stays on the bounded delta path. The usage belongs to an assistant message +// committed before the checkpoint, so maintenance must fold its token-derived +// state without loading the historical transcript. +func TestIncrementalSignalMaintainerHandlesCommittedUsageUpdate(t *testing.T) { + root := t.TempDir() + day := filepath.Join(root, "2024", "01", "01") + require.NoError(t, os.MkdirAll(day, 0o755)) + path := filepath.Join( + day, "rollout-2024-01-01T10-00-00-"+signalMaintainUUID+".jsonl", + ) + initial := testjsonl.JoinJSONL( + testjsonl.CodexSessionMetaJSON( + signalMaintainUUID, "/workspace/project", "codex_cli_rs", + "2024-01-01T10:00:00Z", + ), + testjsonl.CodexTurnContextJSON( + "gpt-5.4", "2024-01-01T10:00:00Z", + ), + testjsonl.CodexMsgJSON( + "user", "first turn", "2024-01-01T10:00:01Z", + ), + testjsonl.CodexMsgJSON( + "assistant", "first response", "2024-01-01T10:00:02Z", + ), + testjsonl.CodexTokenCountJSON( + "2024-01-01T10:00:03Z", 1000, 20, 100, + ), + testjsonl.CodexMsgJSON( + "user", "run the command", "2024-01-01T10:00:04Z", + ), + testjsonl.CodexFunctionCallWithCallIDJSON( + "exec_command", "call_usage", nil, + "2024-01-01T10:00:05Z", + ), + ) + require.NoError(t, os.WriteFile(path, []byte(initial), 0o644)) + + database := openTestDB(t) + engine := NewEngine(database, EngineConfig{ + AgentDirs: map[parser.AgentType][]string{ + parser.AgentCodex: {root}, + }, + Machine: "local", + }) + t.Cleanup(engine.Close) + require.Equal(t, 1, engine.SyncAll(t.Context(), nil).Synced) + + appended := testjsonl.JoinJSONL( + testjsonl.CodexFunctionCallOutputJSON( + "call_usage", "command complete", "2024-01-01T10:00:06Z", + ), + testjsonl.CodexTokenCountJSON( + "2024-01-01T10:00:07Z", 500, 30, 50, + ), + ) + f, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0o644) + require.NoError(t, err) + _, err = f.WriteString(appended) + require.NoError(t, err) + require.NoError(t, f.Close()) + + loadsBefore := database.MessagesLoadCount() + require.Equal(t, 1, engine.SyncAll(t.Context(), nil).Synced) + require.Equal(t, loadsBefore, database.MessagesLoadCount(), + "late result plus committed usage must not load session history") + + sessionID := "codex:" + signalMaintainUUID + incrementalSession, err := database.GetSessionFull(t.Context(), sessionID) + require.NoError(t, err) + require.NotNil(t, incrementalSession) + require.True(t, incrementalSession.LastWriteIncremental) + require.Equal(t, db.CurrentQualitySignalVersion, + incrementalSession.QualitySignalVersion) + require.Equal(t, 1, incrementalSession.CompactionCount, + "the committed usage update must contribute token-drop compaction state") + + incrementalMessages, err := database.GetAllMessages(t.Context(), sessionID) + require.NoError(t, err) + var usageMessage *db.Message + for i := range incrementalMessages { + if len(incrementalMessages[i].ToolCalls) > 0 && + incrementalMessages[i].ToolCalls[0].ToolUseID == "call_usage" { + usageMessage = &incrementalMessages[i] + break + } + } + require.NotNil(t, usageMessage) + require.True(t, usageMessage.HasContextTokens) + require.Equal(t, 500, usageMessage.ContextTokens) + + // A fresh authoritative import of the complete file must produce the + // same observable signal state and message token metadata. + fullDatabase := openTestDB(t) + fullEngine := NewEngine(fullDatabase, EngineConfig{ + AgentDirs: map[parser.AgentType][]string{ + parser.AgentCodex: {root}, + }, + Machine: "local", + }) + t.Cleanup(fullEngine.Close) + require.Equal(t, 1, fullEngine.SyncAll(t.Context(), nil).Synced) + fullSession, err := fullDatabase.GetSessionFull(t.Context(), sessionID) + require.NoError(t, err) + require.NotNil(t, fullSession) + require.Equal(t, snapshotSessionSignals(fullSession), + snapshotSessionSignals(incrementalSession)) + fullMessages, err := fullDatabase.GetAllMessages(t.Context(), sessionID) + require.NoError(t, err) + require.Equal(t, fullMessages, incrementalMessages) +} + +// TestIncrementalSignalMaintainerDeclinesForUserMessage verifies the +// fallback contract: a delta carrying a user message is not folded — the +// debounced full recompute runs (loading history) and keeps signals +// current. +func TestIncrementalSignalMaintainerDeclinesForUserMessage(t *testing.T) { + root := t.TempDir() + day := filepath.Join(root, "2024", "01", "01") + require.NoError(t, os.MkdirAll(day, 0o755)) + path := filepath.Join( + day, "rollout-2024-01-01T10-00-00-"+signalMaintainUUID+".jsonl", + ) + initial := testjsonl.JoinJSONL( + testjsonl.CodexSessionMetaJSON( + signalMaintainUUID, "/workspace/project", "codex_cli_rs", + "2024-01-01T10:00:00Z", + ), + testjsonl.CodexMsgJSON("user", "hello", "2024-01-01T10:00:01Z"), + ) + require.NoError(t, os.WriteFile(path, []byte(initial), 0o644)) + + database := openTestDB(t) + engine := NewEngine(database, EngineConfig{ + AgentDirs: map[parser.AgentType][]string{ + parser.AgentCodex: {root}, + }, + Machine: "local", + }) + t.Cleanup(engine.Close) + require.Equal(t, 1, engine.SyncAll(context.Background(), nil).Synced) + sessionID := "codex:" + signalMaintainUUID + + appended := testjsonl.JoinJSONL(testjsonl.CodexMsgJSON( + "user", "please fix the failing test", "2024-01-01T10:00:02Z", + )) + f, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0o644) + require.NoError(t, err) + _, err = f.WriteString(appended) + require.NoError(t, err) + require.NoError(t, f.Close()) + + loadsBefore := database.MessagesLoadCount() + require.Equal(t, 1, engine.SyncAll(context.Background(), nil).Synced) + assert.Greater(t, database.MessagesLoadCount(), loadsBefore, + "a user-message delta must fall back to the full recompute") + sess, err := database.GetSessionFull(context.Background(), sessionID) + require.NoError(t, err) + require.NotNil(t, sess) + require.Equal(t, db.CurrentQualitySignalVersion, sess.QualitySignalVersion, + "the fallback recompute must keep the signal version current") + require.Equal(t, 2, sess.MessageCount) +} + +func TestFullSignalRecomputeRetriesWhenTranscriptRevisionChanges(t *testing.T) { + const uuid = "019eb791-cf7d-75c1-8439-9ed74c122d77" + root := t.TempDir() + day := filepath.Join(root, "2024", "01", "01") + require.NoError(t, os.MkdirAll(day, 0o755)) + path := filepath.Join( + day, "rollout-2024-01-01T10-00-00-"+uuid+".jsonl", + ) + initial := testjsonl.JoinJSONL( + testjsonl.CodexSessionMetaJSON( + uuid, "/workspace/project", "codex_cli_rs", + "2024-01-01T10:00:00Z", + ), + testjsonl.CodexMsgJSON( + "user", "start", "2024-01-01T10:00:01Z", + ), + testjsonl.CodexMsgJSON( + "assistant", "old answer", "2024-01-01T10:00:02Z", + ), + ) + require.NoError(t, os.WriteFile(path, []byte(initial), 0o644)) + + database := openTestDB(t) + engine := NewEngine(database, EngineConfig{ + AgentDirs: map[parser.AgentType][]string{ + parser.AgentCodex: {root}, + }, + Machine: "local", + }) + t.Cleanup(engine.Close) + require.Equal(t, 1, engine.SyncAll(t.Context(), nil).Synced) + + sessionID := "codex:" + uuid + hookCalls := 0 + _, err := engine.recomputeSignalsFromDBWithHook( + t.Context(), sessionID, + func(attempt int) { + hookCalls++ + if attempt != 0 { + return + } + appended := testjsonl.JoinJSONL(testjsonl.CodexMsgJSON( + "assistant", "new answer", "2024-01-01T10:00:03Z", + )) + f, openErr := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0o644) + require.NoError(t, openErr) + _, writeErr := f.WriteString(appended) + require.NoError(t, writeErr) + require.NoError(t, f.Close()) + require.Equal(t, 1, engine.SyncAll(t.Context(), nil).Synced) + }, + ) + require.NoError(t, err) + require.GreaterOrEqual(t, hookCalls, 2, + "a changed revision must force a fresh snapshot attempt") + + sess, err := database.GetSessionFull(t.Context(), sessionID) + require.NoError(t, err) + require.NotNil(t, sess) + require.NotNil(t, sess.TranscriptRevision) + stored, ok, err := database.GetSessionSignalState(sessionID) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, *sess.TranscriptRevision, stored.TranscriptRevision) + + var state signals.IncrementalState + require.NoError(t, state.UnmarshalBinary(stored.State)) + require.Equal(t, "new answer", state.LastContent, + "the current revision must never be stamped onto the stale snapshot") +} + +func TestFullSignalRecomputeRetriesWhenMetadataChanges(t *testing.T) { + const uuid = "019eb791-cf7d-75c1-8439-9ed74c122d88" + root := t.TempDir() + day := filepath.Join(root, "2024", "01", "01") + require.NoError(t, os.MkdirAll(day, 0o755)) + path := filepath.Join( + day, "rollout-2024-01-01T10-00-00-"+uuid+".jsonl", + ) + initial := testjsonl.JoinJSONL( + testjsonl.CodexSessionMetaJSON( + uuid, "/workspace/project", "codex_cli_rs", + "2024-01-01T10:00:00Z", + ), + testjsonl.CodexMsgJSON( + "user", "start", "2024-01-01T10:00:01Z", + ), + testjsonl.CodexMsgJSON( + "assistant", "answer", "2024-01-01T10:00:02Z", + ), + ) + require.NoError(t, os.WriteFile(path, []byte(initial), 0o644)) + + database := openTestDB(t) + engine := NewEngine(database, EngineConfig{ + AgentDirs: map[parser.AgentType][]string{ + parser.AgentCodex: {root}, + }, + Machine: "local", + }) + t.Cleanup(engine.Close) + require.Equal(t, 1, engine.SyncAll(t.Context(), nil).Synced) + + sessionID := "codex:" + uuid + before, err := database.GetSessionFull(t.Context(), sessionID) + require.NoError(t, err) + require.NotNil(t, before) + require.NotNil(t, before.TranscriptRevision) + originalRevision := *before.TranscriptRevision + + hookCalls := 0 + _, err = engine.recomputeSignalsFromDBWithHook( + t.Context(), sessionID, + func(attempt int) { + hookCalls++ + if attempt != 0 { + return + } + current, loadErr := database.GetSessionFull(t.Context(), sessionID) + require.NoError(t, loadErr) + require.NotNil(t, current) + endedAt := "2026-08-18T12:00:00Z" + current.EndedAt = &endedAt + current.IsAutomated = true + current.MessageCount = 7 + current.PeakContextTokens = 12345 + current.HasPeakContextTokens = true + require.NoError(t, database.UpsertSession(*current)) + afterMetadata, loadErr := database.GetSessionFull( + t.Context(), sessionID, + ) + require.NoError(t, loadErr) + require.NotNil(t, afterMetadata) + require.NotNil(t, afterMetadata.TranscriptRevision) + require.Equal(t, originalRevision, *afterMetadata.TranscriptRevision, + "metadata-only update must leave transcript revision unchanged") + }, + ) + require.NoError(t, err) + require.GreaterOrEqual(t, hookCalls, 2, + "metadata-only input changes must force a fresh snapshot attempt") + + after, err := database.GetSessionFull(t.Context(), sessionID) + require.NoError(t, err) + require.NotNil(t, after) + require.NotNil(t, after.TranscriptRevision) + require.Equal(t, originalRevision, *after.TranscriptRevision) + require.Equal(t, 7, after.MessageCount) + require.True(t, after.IsAutomated) + require.Equal(t, 12345, after.PeakContextTokens) + stored, ok, err := database.GetSessionSignalState(sessionID) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, originalRevision, stored.TranscriptRevision) +} + +func TestCloneCountsNilReturnsWritableMap(t *testing.T) { + counts := cloneCounts(nil) + require.NotNil(t, counts) + counts["gpt-test"] = 1 + require.Equal(t, 1, counts["gpt-test"]) +} diff --git a/internal/sync/staging_space_other.go b/internal/sync/staging_space_other.go new file mode 100644 index 000000000..e2aa0e2cc --- /dev/null +++ b/internal/sync/staging_space_other.go @@ -0,0 +1,9 @@ +//go:build !linux && !darwin && !windows + +package sync + +// stagingDirFreeBytes fails open on platforms without a capacity query; +// CreateTemp reports real failures. +func stagingDirFreeBytes(string) (uint64, bool, error) { + return 0, false, nil +} diff --git a/internal/sync/staging_space_unix.go b/internal/sync/staging_space_unix.go new file mode 100644 index 000000000..5f8b478e4 --- /dev/null +++ b/internal/sync/staging_space_unix.go @@ -0,0 +1,15 @@ +//go:build linux || darwin + +package sync + +import "golang.org/x/sys/unix" + +// stagingDirFreeBytes returns the free bytes available in dir. A nil +// error with ok=false means the filesystem does not report capacity. +func stagingDirFreeBytes(dir string) (free uint64, ok bool, err error) { + var st unix.Statfs_t + if err := unix.Statfs(dir, &st); err != nil { + return 0, false, err + } + return st.Bavail * uint64(st.Bsize), true, nil +} diff --git a/internal/sync/staging_space_windows.go b/internal/sync/staging_space_windows.go new file mode 100644 index 000000000..387385a4f --- /dev/null +++ b/internal/sync/staging_space_windows.go @@ -0,0 +1,26 @@ +//go:build windows + +package sync + +import ( + "golang.org/x/sys/windows" + "unsafe" +) + +// stagingDirFreeBytes returns the free bytes available in dir. +func stagingDirFreeBytes(dir string) (free uint64, ok bool, err error) { + path, err := windows.UTF16PtrFromString(dir) + if err != nil { + return 0, false, err + } + var available, total, totalFree uint64 + if err := windows.GetDiskFreeSpaceEx( + path, &available, &total, &totalFree, + ); err != nil { + return 0, false, err + } + _ = total + return available, true, nil +} + +var _ = unsafe.Pointer(nil) diff --git a/internal/sync/verified_source_gate_integration_test.go b/internal/sync/verified_source_gate_integration_test.go index 0bfa94000..21d235aaf 100644 --- a/internal/sync/verified_source_gate_integration_test.go +++ b/internal/sync/verified_source_gate_integration_test.go @@ -155,6 +155,11 @@ func newVerifiedSourceArchiveWithRewriter( require.NoError(t, database.SetSessionDataVersion( "codex:"+uuid, db.CurrentDataVersion(), )) + seedVerifiedSourceCheckpoint( + t, database, engine, parser.AgentCodex, + path, "codex:"+uuid, + fileSize, fileMtime, fileHash, + ) source := parser.SourceRef{ Provider: parser.AgentCodex, Key: path, @@ -171,6 +176,42 @@ func newVerifiedSourceArchiveWithRewriter( return engine, provider, files } +// seedVerifiedSourceCheckpoint writes a metadata-only checkpoint for a +// gate-test fixture, so the checkpoint gate's decisions apply exactly as +// they do to a session that already went through the upgrade bootstrap. +func seedVerifiedSourceCheckpoint( + t *testing.T, + database *db.DB, + engine *Engine, + agent parser.AgentType, + path, sessionID string, + size, mtime int64, + fileHash string, +) { + t.Helper() + info, err := os.Stat(path) + require.NoError(t, err) + inode, device := getFileIdentity(path, info) + changeTime, _ := fileChangeTime(path, info) + require.NoError(t, database.UpsertParserCheckpoint(db.ParserCheckpoint{ + SessionID: sessionID, + Agent: string(agent), + FilePath: engine.effectiveSourcePath(path), + FileInode: uint64(inode), + FileDevice: uint64(device), + FileMTime: mtime, + FileChangeTime: changeTime, + Offset: size, + TailAnchorDigest: "seed", + Hash: fileHash, + NextOrdinal: 0, + Version: codexCheckpointVersion, + }, db.ParserCheckpointBlobs{ + Cursor: []byte("seed"), + HashState: []byte("seed"), + })) +} + func runVerifiedSourcePass( t *testing.T, engine *Engine, @@ -193,13 +234,16 @@ func TestVerifiedSourceGateWarmFingerprintWorkIsCardinalityIndependent( t.Run(fmt.Sprintf("sources=%d", count), func(t *testing.T) { engine, provider, files := newVerifiedSourceArchive(t, count) + // Checkpointed Codex sources skip through the checkpoint gate + // on every pass: no content fingerprints, ever, at any + // source count. runVerifiedSourcePass(t, engine, files) - assert.Equal(t, count, provider.fingerprintCalls, - "cold pass must deep-verify every source") + assert.Zero(t, provider.fingerprintCalls, + "checkpointed sources must skip without fingerprinting") runVerifiedSourcePass(t, engine, files) - assert.Equal(t, count, provider.fingerprintCalls, - "warm pass must perform zero content fingerprints") + assert.Zero(t, provider.fingerprintCalls, + "the warm pass must also skip without fingerprinting") assert.Len(t, engine.verifiedSources, count) }) } @@ -208,18 +252,21 @@ func TestVerifiedSourceGateWarmFingerprintWorkIsCardinalityIndependent( func TestVerifiedSourceGateWarmTrustDoesNotMaskDatabaseRepair(t *testing.T) { const sessionID = "codex:00000000-0000-0000-0000-000000000001" tests := []struct { - name string - mutate func(*testing.T, *db.DB) + name string + wantFingerprints int + mutate func(*testing.T, *db.DB) }{ { - name: "missing row", + name: "missing row", + wantFingerprints: 0, mutate: func(t *testing.T, database *db.DB) { t.Helper() require.NoError(t, database.DeleteSession(sessionID)) }, }, { - name: "stale data version", + name: "stale data version", + wantFingerprints: 1, mutate: func(t *testing.T, database *db.DB) { t.Helper() require.NoError(t, database.SetSessionDataVersion( @@ -228,7 +275,8 @@ func TestVerifiedSourceGateWarmTrustDoesNotMaskDatabaseRepair(t *testing.T) { }, }, { - name: "project requires reparse", + name: "project requires reparse", + wantFingerprints: 1, mutate: func(t *testing.T, database *db.DB) { t.Helper() session, err := database.GetSessionFull( @@ -242,6 +290,10 @@ func TestVerifiedSourceGateWarmTrustDoesNotMaskDatabaseRepair(t *testing.T) { }, { name: "file mtimes reset", + // The stored-mtime mismatch declines the checkpoint gate, so + // the caller fingerprints once before the authoritative + // parse. + wantFingerprints: 1, mutate: func(t *testing.T, database *db.DB) { t.Helper() require.NoError(t, database.ResetAllMtimes()) @@ -254,15 +306,16 @@ func TestVerifiedSourceGateWarmTrustDoesNotMaskDatabaseRepair(t *testing.T) { engine, provider, files := newVerifiedSourceArchive(t, 1) runVerifiedSourcePass(t, engine, files) runVerifiedSourcePass(t, engine, files) - require.Equal(t, 1, provider.fingerprintCalls) + require.Zero(t, provider.fingerprintCalls, + "checkpointed sources skip both passes") tt.mutate(t, engine.db) res := engine.processFile(context.Background(), files[0]) require.ErrorContains(t, res.err, "unexpected parse after seeding stored source state") - assert.Equal(t, 2, provider.fingerprintCalls, - "persisted state requiring repair must bypass warm trust") + assert.Equal(t, tt.wantFingerprints, provider.fingerprintCalls, + "persisted state requiring repair must bypass the checkpoint trust") }) } } @@ -335,6 +388,14 @@ func TestVerifiedSourceGateDoesNotBorrowRepairState(t *testing.T) { parser.AgentTraeX: parser.ProviderMigrationProviderAuthoritative, }, }) + for _, agent := range []parser.AgentType{ + parser.AgentCodex, parser.AgentTraeX, + } { + seedVerifiedSourceCheckpoint( + t, database, engine, agent, path, string(agent)+":shared", + fileSize, fileMtime, fileHash, + ) + } fileFor := func(agent parser.AgentType) parser.DiscoveredFile { source := parser.SourceRef{ Provider: agent, Key: path, @@ -358,8 +419,8 @@ func TestVerifiedSourceGateDoesNotBorrowRepairState(t *testing.T) { res := engine.processFile(t.Context(), fileFor(parser.AgentTraeX)) require.ErrorContains(t, res.err, "unexpected parse after seeding stored source state") - assert.Equal(t, 2, traexProvider.fingerprintCalls, - "missing TraeX state must invalidate only TraeX trust and reverify") + assert.Zero(t, traexProvider.fingerprintCalls, + "the checkpointed pass must not fingerprint; the missing TraeX row declines and reverifies") } func TestVerifiedSourceGateRechecksAfterStatAndWatcherInvalidation(t *testing.T) { @@ -367,7 +428,8 @@ func TestVerifiedSourceGateRechecksAfterStatAndWatcherInvalidation(t *testing.T) file := files[0] runVerifiedSourcePass(t, engine, files) runVerifiedSourcePass(t, engine, files) - require.Equal(t, 1, provider.fingerprintCalls) + require.Zero(t, provider.fingerprintCalls, + "checkpointed sources skip both passes") info, err := os.Stat(file.Path) require.NoError(t, err) @@ -389,7 +451,7 @@ func TestVerifiedSourceGateRechecksAfterStatAndWatcherInvalidation(t *testing.T) require.NotEqual(t, baselineChangeTime, changeTime, "fixture must cross a native change-time tick") runVerifiedSourcePass(t, engine, files) - assert.Equal(t, 2, provider.fingerprintCalls, + assert.Equal(t, 1, provider.fingerprintCalls, "same-size rewrite with restored mtime must deep-verify") classified := requireClassifyPaths(t, engine, []string{file.Path}) @@ -397,14 +459,14 @@ func TestVerifiedSourceGateRechecksAfterStatAndWatcherInvalidation(t *testing.T) res := engine.processFile(context.Background(), classified[0]) require.NoError(t, res.err) assert.True(t, res.skip) - assert.Equal(t, 3, provider.fingerprintCalls, + assert.Equal(t, 2, provider.fingerprintCalls, "a watcher-classified source must invalidate warm trust") engine.clearWatcherOverflowCaches() res = engine.processFile(context.Background(), file) require.NoError(t, res.err) assert.True(t, res.skip) - assert.Equal(t, 4, provider.fingerprintCalls, + assert.Equal(t, 3, provider.fingerprintCalls, "watcher overflow must clear every verified-source trust record") }