diff --git a/openspec/changes/session-tail-rehydrate/design.md b/openspec/changes/session-tail-rehydrate/design.md new file mode 100644 index 000000000..01c4b9eb9 --- /dev/null +++ b/openspec/changes/session-tail-rehydrate/design.md @@ -0,0 +1,141 @@ +## Context + +- Subscribe is already incremental via `lastSeq` (`subscription-handler.ts`). +- Strategy A persists raw events in IndexedDB and delta-subscribes on reload + (`replay-cache.ts`, `rehydrate-session.ts`, `App.tsx`). +- Large sessions exceed `DEFAULT_MAX_BYTES_PER_SESSION` (5 MiB) → put deletes + the entry → every cold open is full replay. +- Server cold path loads full JSONL then ships all stored events in batches of + 50 with `MAX_REPLAY_EVENTS = 0` (unlimited). +- ChatView `scrollStateMap` is in-module only; restore runs only on `sessionId` + change. Wipe→rebuild of the same session does not re-pin. + +## Goals + +1. Cold open of a large session transfers **O(budget)** events, not full history. +2. Large sessions remain **cacheable** under a byte budget (newest-first). +3. Users can **page older history** without losing the current viewport. +4. After hydrate, default land position is **true bottom** (unless user escaped). +5. Legacy clients (no `mode`) keep full-replay behavior. + +## Non-Goals + +See proposal. Notably: durable last-seen anchor and estimate-drift fixes stay +out of this change. + +## Decisions + +### D1 — Byte budget, not turn count + +**Choice:** Keep newest events until ~**4 MiB** serialized payload. + +**Why:** Turn boundaries are uneven (one tool-heavy turn can be multi-MB). A +byte budget gives a hard wire/IDB ceiling. Client and server share the same +selection algorithm so cache tail ≈ first paint tail. + +**Default:** `DEFAULT_TAIL_WINDOW_BYTES = 4 * 1024 * 1024`. Client may pass +`windowBytes` on subscribe; server clamps to `[256_KiB, 8_MiB]`. + +### D2 — Newest-first selection, whole events only + +``` +selectNewestEventsByBudget(eventsSortedBySeqAsc, budget): + walk from end → start + include event if size(event) + acc <= budget OR acc == 0 (always include newest) + stop when next would exceed + return included in ascending seq order +``` + +Size = `JSON.stringify({ seq, event }).length` (same as cache put today) for +determinism between client trim and server wire estimate. Never split an event. + +**Incomplete oldest message:** If the oldest kept event is a `message_update` +without its `message_start` in the window, still keep it — reducer already +tolerates partial streams for display of finalized `message_end` when present. +Do **not** expand the window past budget to complete pairs (budget is hard). + +### D3 — Additive protocol (no new message type for v1) + +**Subscribe (browser → server):** + +| Field | Type | Meaning | +|---|---|---| +| `mode?` | `"full" \| "tail"` | Default `"full"` (legacy). Cold open uses `"tail"`. | +| `windowBytes?` | number | Budget hint; server clamps. | +| `fromSeq?` | number | Load-older: exclusive upper bound (`seq < fromSeq`). | + +**Event replay (server → browser):** + +| Field | Type | Meaning | +|---|---|---| +| `hasMoreOlder?` | boolean | More history exists below `windowMinSeq`. | +| `windowMinSeq?` | number | Lowest seq in the retained/delivered window. | +| `windowMaxSeq?` | number | Highest seq in this delivery (usually = last event seq). | + +**Matrix:** + +| Client | Server | +|---|---| +| omit `mode` / `mode:"full"` | Today's full (or delta) path | +| `mode:"tail"`, no `fromSeq`, `lastSeq` 0/absent | Newest budget window from store or disk | +| `mode:"tail"`, `lastSeq > 0` | Delta `seq > lastSeq` only (mode ignored for delta) | +| `fromSeq: N` | Older page: newest events with `seq < N` under budget | +| store empty + disk | Load disk; apply same window before send | + +Warm reconnect with live `lastSeq` is unchanged. + +### D4 — Client cache: trim-to-tail, schema v2 + +- On put: if full buffer serializes over budget, trim with + `selectNewestEventsByBudget` then put. **Never** delete solely for size. +- Persist `{ maxSeq, windowMinSeq, payload, schemaVersion: 2 }`. +- v1 entries: schema mismatch → miss → tail subscribe (safe). +- `session_state_reset` still `drop()`s the entry. + +### D5 — Load-older preserves scroll anchor + +When prepending older rows: + +1. Snapshot first visible virtual row key + offset before prepend. +2. Apply older events (prepend to reducer state / merge by seq). +3. Restore anchor via `scrollToIndex` + offset (same CR-6 virtual coords as + session restore). + +Do not `session_state_reset` for older pages. Do not clear stick-to-bottom +latch incorrectly: load-older implies user is at top → stick stays false. + +### D6 — Post-hydrate re-pin + +When `loadingHistory` goes `true → false` for the active session: + +- If `stickToBottomRef` is true **or** no user scroll-up occurred during this + hydrate (`!userEscapedDuringHydrate`), pin bottom once. +- If user wheeled/touched away mid-hydrate, respect escape (existing + `cancelDescent` / scroll handler). + +This fixes wipe→rebuild same-`sessionId` without durable last-seen. + +### D7 — Shared pure helper location + +Prefer `packages/shared/src/event-window.ts` (or under existing shared event +util path) so client IDB put and server subscribe use one implementation and +one unit test file. Server-only copy is acceptable only if shared import is +awkward for the worker thread; prefer shared. + +## Risks + +| Risk | Mitigation | +|---|---| +| Budget too small → thin context | 4 MiB default; clamp max 8 MiB; load-older | +| Budget too large on mobile | Clamp; measure hydrate time in manual smoke | +| `hasMoreOlder` wrong after store essential-trim | Compare `windowMinSeq` to buffer min seq; disk cold load may still have older not in memory — if only memory is searched, document that load-older may need disk path (reuse `loadSessionEvents` windowed) | +| Double full load on disk cold | Load once; window in memory; page older from same buffer | +| Fork / seq reset | Existing reset purge | +| Prepend scroll jump | Anchor restore required in acceptance | + +## Open questions (resolve in tasks 1.x if needed) + +1. Should load-older use the same `subscribe` message or a one-shot + `load_history`? **Lean subscribe + `fromSeq`** (fewer types). Confirm if + re-entrancy with live subscribe set is awkward. +2. Exact clamp bounds (256 KiB–8 MiB) — tune after first large-session smoke. diff --git a/openspec/changes/session-tail-rehydrate/proposal.md b/openspec/changes/session-tail-rehydrate/proposal.md new file mode 100644 index 000000000..fb8ae48d6 --- /dev/null +++ b/openspec/changes/session-tail-rehydrate/proposal.md @@ -0,0 +1,79 @@ +## Why + +Strategy A (`reduce-session-replay-traffic`) persists the full raw event buffer +per session in IndexedDB and delta-subscribes with `lastSeq = maxSeq` on reload. +That fails for the sessions that matter most: **large chats**. + +The client cache is all-or-nothing: + +- `DEFAULT_MAX_BYTES_PER_SESSION = 5 MiB` +- Over-cap → **delete entry and skip persist** → next cold open is always + `lastSeq: 0` full replay + +The server subscribe path still ships the **entire** in-memory (or cold-loaded +JSONL) event stream when `lastSeq` is 0 (`MAX_REPLAY_EVENTS = 0`). Mobile return +to a conversation therefore re-downloads and re-reduces most/all of history. + +A second, related UX bug: after wipe→rebuild of the same `sessionId`, ChatView +restores scroll only on `sessionId` change. Multi-batch hydrate + sticky-scroll +escape leaves the viewport mid-transcript on an arbitrary finished agent bubble +instead of the true bottom / last-seen region. + +## What Changes + +1. **Byte-budget tail cache (client).** Persist only the **newest** events that + fit a ~4 MiB budget. Large sessions stay cacheable; cold open rehydrates a + tail and delta-subscribes from that `maxSeq`. Never all-or-nothing drop solely + because the full buffer is large. +2. **Server tail-first subscribe (protocol additive).** Cold open uses + `subscribe { mode: "tail", windowBytes? }`. Server returns the newest events + under the budget with `hasMoreOlder` / `windowMinSeq` / `windowMaxSeq` on + `event_replay`. Legacy clients omit `mode` → full replay (unchanged). +3. **Load-older.** When the user reaches the top of the window, request the next + older page via `fromSeq` (exclusive upper bound). Prepend without wiping; + preserve scroll anchor. +4. **Post-hydrate re-pin.** When `loadingHistory` clears and the user has not + deliberately locked away from bottom, pin to the true bottom once so cold + open lands on the latest content. + +## Capabilities + +### New Capabilities + +- `session-history-window` — tail-mode subscribe, byte-budget event selection, + `hasMoreOlder` signaling, load-older paging. + +### Modified Capabilities + +- `session-replay-persistence` — over-cap behavior becomes **trim-to-tail and + persist**, not delete-and-miss; schemaVersion bump. +- `chat-scroll-lock` — re-pin bottom after history hydrate completes (same + sessionId), without fighting deliberate scroll-up. + +## Non-Goals + +- Durable last-seen message id / scroll map across reloads (follow-up). +- Content-aware virtual row size estimates (`fix-chat-scroll-to-top-estimate-drift`). +- Persisting reduced `ChatMessage[]` instead of raw events. +- Prefetch of non-selected sessions. +- Raising server `DEFAULT_MAX_EVENTS_PER_SESSION` as a substitute for windowing. +- Push / unread / `session_view` changes. + +## Impact + +- `packages/shared/src/browser-protocol.ts` — additive fields on `subscribe` and + `event_replay`. +- `packages/server/src/browser-handlers/subscription-handler.ts` — tail / older + page selection before `sendEventBatches`. +- `packages/server/src/` (+ optional shared) — pure + `selectNewestEventsByBudget`. +- `packages/client/src/lib/replay-cache.ts` / `replay-persist.ts` — tail put, + schema v2. +- `packages/client/src/App.tsx` — cold subscribe `mode: "tail"`. +- `packages/client/src/components/ChatView.tsx` — re-pin + load-older trigger / + scroll-anchor preserve. +- `packages/client/src/hooks/useMessageHandler.ts` — handle window metadata; + prepend older pages. + +Base branch: `develop` (post `omp-minimal` merge). Branch: +`feat/session-tail-rehydrate`. diff --git a/openspec/changes/session-tail-rehydrate/specs/chat-scroll-lock/spec.md b/openspec/changes/session-tail-rehydrate/specs/chat-scroll-lock/spec.md new file mode 100644 index 000000000..d9fe190e9 --- /dev/null +++ b/openspec/changes/session-tail-rehydrate/specs/chat-scroll-lock/spec.md @@ -0,0 +1,57 @@ +## MODIFIED Requirements + +### Requirement: Sticky bottom and scroll-to-bottom control + +ChatView SHALL keep a stick-to-bottom latch: near-bottom follows new content; +scroll-up escapes; the scroll-to-bottom control re-arms follow. Multi-batch +`event_replay` SHALL not leave the user permanently stuck mid-list after a cold +hydrate when they did not deliberately escape. + +#### Scenario: Escape while streaming + +- **WHEN** the user scrolls up away from the bottom during live content +- **THEN** new content SHALL NOT yank the viewport to the bottom +- **AND** the scroll-to-bottom control SHALL be visible + +#### Scenario: Re-arm at bottom + +- **WHEN** the user scrolls back within the near-bottom threshold +- **THEN** stick-to-bottom SHALL re-arm and chase new content + +### Requirement: Post-hydrate bottom pin + +When history loading completes for the selected session, ChatView SHALL pin to +the true bottom once if the user has not escaped during that hydrate. This +applies even when `sessionId` did not change (wipe→rebuild / multi-batch cold +open). + +#### Scenario: Cold open lands at bottom + +- **WHEN** a session opens empty with `loadingHistory` true and then receives + its first full history window with the user not scrolling away +- **THEN** after `loadingHistory` becomes false the viewport SHALL be near the + bottom +- **AND** the scroll-to-bottom control SHALL be hidden + +#### Scenario: Same-session wipe rebuild + +- **WHEN** the same `sessionId` is wiped to empty and rebuilt from replay without + a session switch +- **AND** the user has not deliberately escaped during rebuild +- **THEN** the viewport SHALL end near the bottom of the rebuilt transcript + +#### Scenario: User escape during hydrate is respected + +- **WHEN** the user wheels or touch-scrolls away from the bottom while history + is still loading +- **THEN** post-hydrate pin SHALL NOT force them back to the bottom + +### Requirement: Load-older does not fight scroll lock + +#### Scenario: Prepend keeps anchor + +- **WHEN** older history is prepended while the user is at the top of the current + window +- **THEN** the first visible row before prepend SHALL remain the first visible + row after prepend (within normal layout tolerance) +- **AND** stick-to-bottom SHALL remain disarmed diff --git a/openspec/changes/session-tail-rehydrate/specs/session-history-window/spec.md b/openspec/changes/session-tail-rehydrate/specs/session-history-window/spec.md new file mode 100644 index 000000000..3798a8de0 --- /dev/null +++ b/openspec/changes/session-tail-rehydrate/specs/session-history-window/spec.md @@ -0,0 +1,68 @@ +## ADDED Requirements + +### Requirement: Tail-mode subscribe + +The browser MAY send `subscribe` with `mode: "tail"` and optional `windowBytes`. +When `mode` is absent or `"full"`, the server SHALL retain existing full/delta +replay behavior. When `mode` is `"tail"` and `lastSeq` is absent or `0`, the +server SHALL deliver only the newest events that fit the effective byte budget +(default 4 MiB, clamped), not the entire session buffer. + +#### Scenario: Cold tail open under budget + +- **WHEN** a browser subscribes with `mode: "tail"` and `lastSeq` 0 to a session + whose full event buffer serializes under the budget +- **THEN** the server SHALL deliver all events in ascending seq order +- **AND** `hasMoreOlder` SHALL be false + +#### Scenario: Cold tail open over budget + +- **WHEN** a browser subscribes with `mode: "tail"` and `lastSeq` 0 to a session + whose full event buffer serializes over the budget +- **THEN** the server SHALL deliver a newest-first subset under the budget +- **AND** `hasMoreOlder` SHALL be true +- **AND** `windowMinSeq` SHALL equal the lowest delivered seq +- **AND** `windowMaxSeq` SHALL equal the highest delivered seq + +#### Scenario: Legacy client full replay + +- **WHEN** a browser subscribes without `mode` and `lastSeq` 0 +- **THEN** the server SHALL deliver the full available event buffer as today + +#### Scenario: Delta ignores tail mode + +- **WHEN** a browser subscribes with `mode: "tail"` and `lastSeq > 0` and the + server has events with `seq > lastSeq` +- **THEN** the server SHALL delta-replay only those events (existing path) + +### Requirement: Load-older page + +The browser MAY send `subscribe` (or an equivalent same-session request) with +`fromSeq: N` to request older history. The server SHALL return the newest events +with `seq < N` that fit the budget, with updated `hasMoreOlder` / window fields. +The client SHALL merge them without wiping already-reduced state. + +#### Scenario: Older page under budget + +- **WHEN** the client requests older history with `fromSeq` equal to the current + `windowMinSeq` and older events exist +- **THEN** the server SHALL deliver events strictly older than `fromSeq` +- **AND** the client SHALL prepend them into the session transcript +- **AND** the visible scroll anchor SHALL remain stable + +#### Scenario: No older history + +- **WHEN** `fromSeq` is less than or equal to the oldest available seq +- **THEN** the server SHALL deliver an empty (or terminal) page with + `hasMoreOlder: false` + +### Requirement: Byte-budget selection + +Event window selection SHALL walk newest→oldest, include whole events only, and +always include at least the newest event when the buffer is non-empty. Client +IDB tail trim and server wire selection SHALL use the same algorithm. + +#### Scenario: Deterministic trim + +- **WHEN** the same ordered event list and budget are passed to the shared helper +- **THEN** client and server SHALL produce identical seq sets diff --git a/openspec/changes/session-tail-rehydrate/specs/session-replay-persistence/spec.md b/openspec/changes/session-tail-rehydrate/specs/session-replay-persistence/spec.md new file mode 100644 index 000000000..ff966be9c --- /dev/null +++ b/openspec/changes/session-tail-rehydrate/specs/session-replay-persistence/spec.md @@ -0,0 +1,50 @@ +## MODIFIED Requirements + +### Requirement: Durable replay cache is an optimization only + +The client SHALL persist a per-session raw event payload + cursor in IndexedDB so +a reload can delta-subscribe. The cache remains an optimization: miss, schema +mismatch, or `session_state_reset` SHALL fall back to a safe network path +without rendering stale history as authoritative. + +#### Scenario: Reload with cache hit + +- **WHEN** the user reloads and a valid cache entry exists for the session +- **THEN** the client SHALL pre-seed reduced state from the cached payload +- **AND** SHALL subscribe with `lastSeq = persisted maxSeq` (and MAY set + `mode: "tail"`) +- **AND** the server SHALL delta-replay only events after that cursor when present + +#### Scenario: Cache miss + +- **WHEN** no entry exists, schema mismatches, or IndexedDB errors +- **THEN** the client SHALL subscribe without relying on cached state +- **AND** for cold open SHALL use `mode: "tail"` so the server does not force a + full multi-megabyte replay when history is large + +### Requirement: Over-budget sessions remain cacheable + +When the live raw-event buffer exceeds the per-session byte budget, the client +SHALL persist a **newest-first tail** that fits the budget rather than skipping +persist or deleting the entry solely due to size. + +#### Scenario: Large session put trims to tail + +- **WHEN** the debounced persister flushes a buffer whose serialized size exceeds + the budget +- **THEN** the cache SHALL store the newest events under the budget +- **AND** SHALL record `maxSeq` as the highest seq in that tail +- **AND** a subsequent reload SHALL be able to cache-hit that tail + +#### Scenario: Schema version bump invalidates old shape + +- **WHEN** `schemaVersion` on disk does not match the running client +- **THEN** the get path SHALL treat the entry as a miss (full/tail network path) + +### Requirement: Reset purges cache + +#### Scenario: session_state_reset drops entry + +- **WHEN** the client receives `session_state_reset` for a session +- **THEN** it SHALL delete that session's cache entry and in-memory persist buffer +- **AND** subsequent subscribe SHALL not use the purged maxSeq diff --git a/openspec/changes/session-tail-rehydrate/tasks.md b/openspec/changes/session-tail-rehydrate/tasks.md new file mode 100644 index 000000000..08392f0dc --- /dev/null +++ b/openspec/changes/session-tail-rehydrate/tasks.md @@ -0,0 +1,49 @@ +## 1. Spec + shared budget helper + +- [x] 1.1 Author proposal.md / design.md / specs (this change) and validate with openspec if available +- [x] 1.2 Add pure `selectNewestEventsByBudget` (+ size helper) under `packages/shared` with unit tests: empty, under budget, over budget, always-keep-newest, ascending output order +- [x] 1.3 Export from shared package surface used by client + server + +## 2. Protocol types + +- [x] 2.1 Extend `SubscribeMessage` with optional `mode`, `windowBytes`, `fromSeq` +- [x] 2.2 Extend `EventReplayMessage` with optional `hasMoreOlder`, `windowMinSeq`, `windowMaxSeq` +- [x] 2.3 Protocol unit tests: new fields optional / round-trip if existing codec tests require it + +## 3. Server tail + load-older + +- [x] 3.1 Failing tests: `mode:"tail"` returns ≤ budget newest events + `hasMoreOlder`; legacy omit mode = full; `fromSeq` returns older page; delta `lastSeq>0` unchanged +- [x] 3.2 Implement window selection in `subscription-handler.ts` (warm store path + cold disk path after load) +- [x] 3.3 Make 3.1 pass + +## 4. Client IDB tail put (session-replay-persistence) + +- [x] 4.1 Failing tests: over-budget put trims newest-first and **persists**; get returns tail; schema v2 invalidates v1 +- [x] 4.2 Implement trim-on-put + schemaVersion bump; stop delete-on-over-cap +- [x] 4.3 Make 4.1 pass + +## 5. Client cold subscribe mode + +- [x] 5.1 Failing test: first cold subscribe for large/miss path sends `mode: "tail"` (and `lastSeq` from cache when present) +- [x] 5.2 Wire `App.tsx` `doSubscribe` / rehydrate path +- [x] 5.3 `useMessageHandler` records `windowMinSeq` / `hasMoreOlder` per session for load-older +- [x] 5.4 Make 5.1 pass + +## 6. Post-hydrate re-pin (chat-scroll-lock) + +- [x] 6.1 Flip / extend `ChatView.scroll-race.test.tsx` hydrate land cases: after wipe→rebuild, pin bottom when stick armed / default cold open +- [x] 6.2 Implement re-pin when `loadingHistory` true→false without user escape +- [x] 6.3 Make 6.1 pass + +## 7. Load-older UI + prepend + +- [x] 7.1 Failing tests: near-top triggers older request with `fromSeq = windowMinSeq`; prepend does not wipe; scroll anchor stable (unit or component-level) +- [x] 7.2 ChatView top sentinel / threshold + App/handler send path +- [x] 7.3 Make 7.1 pass + +## 8. Verify + +- [x] 8.1 Focused vitest: shared window helper, replay-cache, subscription-handler, ChatView scroll-race, any new handler tests +- [ ] 8.2 Manual / mobile smoke on large session: cold return wire size ≪ full history; land at bottom; scroll-up loads older without jump +- [ ] 8.3 `openspec validate session-tail-rehydrate` (if CLI present) +- [ ] 8.4 PR against `develop` diff --git a/packages/client/src/App.tsx b/packages/client/src/App.tsx index 59c8c8e4d..6c05e8437 100644 --- a/packages/client/src/App.tsx +++ b/packages/client/src/App.tsx @@ -91,6 +91,10 @@ import { rehydrateSession } from "./lib/rehydrate-session.js"; // Strategy A (reduce-session-replay-traffic): durable replay cursor. import { replayCache } from "./lib/replay-cache.js"; import { createReplayPersister } from "./lib/replay-persist.js"; +import { + buildLoadOlderSubscribe, + buildSessionSubscribe, +} from "./lib/session-subscribe.js"; import { buildFolderSettingsUrl, buildOpenSpecArchiveUrl, @@ -579,6 +583,12 @@ export default function App() { // ChatView loading indicator. See change: show-chat-history-loading-indicator. const [loadingHistory, setLoadingHistory] = useState>(new Map()); const loadingHistoryTimersRef = useRef>>(new Map()); + // Tail window meta for load-older (session-tail-rehydrate). + const historyWindowRef = useRef(new Map()); + const [historyWindowMap, setHistoryWindowMap] = useState( + () => new Map(), + ); + const [loadingOlderMap, setLoadingOlderMap] = useState(() => new Map()); // After overlay-url-routing: shell overlays are URL-driven via the // useRoute matches declared above. `previewState`, `specsBrowserCwd`, // `archiveBrowserCwd`, `diffViewSessionId`, and the three useContentViews @@ -723,9 +733,23 @@ export default function App() { ); }, []); + const handleLoadOlder = useCallback(() => { + if (!selectedId) return; + if (loadingOlderMap.get(selectedId)) return; + const win = historyWindowRef.current.get(selectedId); + // minSeq must be a positive cursor; `!win.minSeq` also rejected valid... no, 0 is invalid. + if (!win?.hasMoreOlder || !(win.minSeq > 0)) return; + setLoadingOlderMap((prev) => { + const next = new Map(prev); + next.set(selectedId, true); + return next; + }); + send(buildLoadOlderSubscribe(selectedId, win.minSeq)); + }, [selectedId, loadingOlderMap, send]); + const handleMessage = useMessageHandler( { setSessions, setSessionStates, setSessionCommands, setFileResults, setChangedOnDisk, setOpenspecMap, setFolderGitMap, setOpenspecGroupsMap, setModelsMap, setRolesMap, setSpawnResult, setSessionOrderMap, setPinnedDirectories, setPinnedDirsLoaded, setFavoriteModels, setWorkspaces, setTerminals, setDiscoveredServers, setSpawnErrors, setResumeErrors, setDisplayPrefs, setViewMessagesMap, setLoadingHistory, setCanvasMap }, - { send, navigate, clearSpawningCwd, spawningCwdsRef, subscribedRef, pendingTerminalCwdRef, lastCreatedTerminalIdRef, maxSeqMapRef, selectedSessionIdRef, pendingSpawnsRef, cwdVisibilityInputsRef, loadingHistoryTimersRef, replayPersister: replayPersisterRef.current, showToast }, + { send, navigate, clearSpawningCwd, spawningCwdsRef, subscribedRef, pendingTerminalCwdRef, lastCreatedTerminalIdRef, maxSeqMapRef, selectedSessionIdRef, pendingSpawnsRef, cwdVisibilityInputsRef, loadingHistoryTimersRef, replayPersister: replayPersisterRef.current, showToast, historyWindowRef, setHistoryWindowMap, setLoadingOlderMap }, ); useEffect(() => { @@ -880,7 +904,9 @@ export default function App() { // models if missing. Extracted so the cache-rehydrate path can call it // after the async IndexedDB read resolves. const doSubscribe = (lastSeq: number) => { - send({ type: "subscribe", sessionId: sid, lastSeq }); + // Cold lastSeq=0 uses mode:tail; delta omits mode. + // See change: session-tail-rehydrate. + send(buildSessionSubscribe(sid, lastSeq)); // Enter LOADING. Covers warm (in-memory replay / reconnect re-subscribe) // and cold (disk-load) paths uniformly, since the warm path never sends // an empty `isLast:false` start marker. @@ -910,6 +936,20 @@ export default function App() { }); if (!maxSeqMapRef.current.has(sid)) maxSeqMapRef.current.set(sid, r.lastSeq); replayPersisterRef.current.seed(sid, r.events); + // IDB holds only a newest-byte tail. Delta subscribe will not re-send + // window meta, so seed load-older from the cached min seq: any gap + // below seq 1 means older history exists server-side. + // See change: session-tail-rehydrate. + if (r.events.length > 0) { + const minSeq = r.events[0]!.seq; + const hasMoreOlder = minSeq > 1; + historyWindowRef.current.set(sid, { minSeq, hasMoreOlder }); + setHistoryWindowMap((m) => { + const next = new Map(m); + next.set(sid, { minSeq, hasMoreOlder }); + return next; + }); + } doSubscribe(maxSeqMapRef.current.get(sid) ?? r.lastSeq); } else { doSubscribe(0); @@ -1500,7 +1540,7 @@ export default function App() { maxSeqMapRef.current.set(selectedId, 0); subscribedRef.current.delete(selectedId); subscribedRef.current.add(selectedId); - send({ type: "subscribe", sessionId: selectedId, lastSeq: 0 }); + send(buildSessionSubscribe(selectedId, 0)); beginLoadingHistory(selectedId); }, } : undefined} @@ -1522,7 +1562,7 @@ export default function App() { maxSeqMapRef.current.set(selectedId, 0); subscribedRef.current.delete(selectedId); subscribedRef.current.add(selectedId); - send({ type: "subscribe", sessionId: selectedId, lastSeq: 0 }); + send(buildSessionSubscribe(selectedId, 0)); beginLoadingHistory(selectedId); }} /> @@ -1651,7 +1691,23 @@ export default function App() { }> - + {/* Single-card error-lifecycle surface. Sticky above the command diff --git a/packages/client/src/components/ChatView.tsx b/packages/client/src/components/ChatView.tsx index 795d17e0e..accc4214a 100644 --- a/packages/client/src/components/ChatView.tsx +++ b/packages/client/src/components/ChatView.tsx @@ -3,7 +3,7 @@ import { EmptyState } from "@blackbelt-technology/pi-dashboard-client-utils/Empt import { Skeleton } from "@blackbelt-technology/pi-dashboard-client-utils/Skeleton"; import { toolCallPrefKey } from "@blackbelt-technology/pi-dashboard-shared/display-prefs.js"; import { isInputNeededTool } from "@blackbelt-technology/pi-dashboard-shared/input-needed-tools.js"; -import { mdiCheck, mdiChevronDown, mdiClose, mdiContentCopy, mdiLoading, mdiSourceFork, mdiTextBox } from "@mdi/js"; +import { mdiCheck, mdiChevronDown, mdiChevronUp, mdiClose, mdiContentCopy, mdiLoading, mdiSourceFork, mdiTextBox } from "@mdi/js"; import { Icon } from "@mdi/react"; import { defaultRangeExtractor, useVirtualizer } from "@tanstack/react-virtual"; import React, { forwardRef, useCallback, useImperativeHandle, useLayoutEffect, useMemo, useRef, useState } from "react"; @@ -96,6 +96,16 @@ interface Props { * See change: configurable-chat-display. */ /** Current sparse override for the session, or `undefined`. */ + /** + * Older messages remain on the server for this session — renders the + * "load older" affordance at the top of the virtual list. + * See change: session-tail-rehydrate. + */ + hasMoreOlder?: boolean; + /** Older-history fetch in flight — renders a spinner in the load-older affordance. See change: session-tail-rehydrate. */ + loadingOlder?: boolean; + /** Request the next older batch from the server. See change: session-tail-rehydrate. */ + onLoadOlder?: () => void; } function ImageAttachments({ @@ -240,7 +250,7 @@ export interface ChatViewHandle { scrollToTurn: (turnIndex: number) => void; } -const ChatViewInner = forwardRef(function ChatView({ sessionId, state, toolContext, onRespondToUi, onAbort, onForceKill, onForkFromMessage, onCloseInlineTerminal, pendingSteering, loadingHistory, onCollapseStreamingThinking }, ref) { +const ChatViewInner = forwardRef(function ChatView({ sessionId, state, toolContext, onRespondToUi, onAbort, onForceKill, onForkFromMessage, onCloseInlineTerminal, pendingSteering, loadingHistory, hasMoreOlder, loadingOlder, onLoadOlder, onCollapseStreamingThinking }, ref) { const scrollRef = useRef(null); // True when the user wants the chat to chase new content. Flips to false on // any real scroll-up gesture, on explicit navigation (scrollToTurn), and on @@ -263,12 +273,22 @@ const ChatViewInner = forwardRef(function ChatView({ sess // fix-chat-scroll-to-top-estimate-drift). `scrollToIndex(0)` is BOUNDED // (maxAttempts=10) and a late async image-load remeasure can bump the view // off index 0 after the retries exhaust; this latch (a) re-issues - // scrollToIndex(0) from `onChange` when a measurement grows the total size, + // `scrollToIndex(0)` from `onChange` when a measurement grows the total size, // and (b) stops `handleScroll` re-arming the bottom-pin mid-flight (the // re-arm race: starting the ascent from the bottom would otherwise flip - // stickToBottomRef back to true and yank the view down). Cleared on arrival + // `stickToBottomRef` back to true and yank the view down). Cleared on arrival // at the top or on real user input (wheel / touch), mirroring descendingRef. const ascendingRef = useRef(false); + // loadingHistory true→false re-pins bottom unless the user scrolled away + // during THIS hydrate cycle. Prior escape (before hydrate) is intentionally + // ignored — wipe/rebuild is a cold land. See change: session-tail-rehydrate. + const loadingHistoryRef = useRef(!!loadingHistory); + const escapedDuringHydrateRef = useRef(false); + // Programmatic scroll (virtualizer pin / hydrate re-pin) fires `scroll` events + // that look like "left the bottom". Only real wheel/touch may clear stick — + // otherwise live follow dies right after a large tail lands. + // See change: session-tail-rehydrate (live-follow after hydrate). + const userGestureRef = useRef(false); const [showScrollButton, setShowScrollButton] = useState(false); const [showScrollTopButton, setShowScrollTopButton] = useState(false); // Streaming-tail selection preservation (change: preserve-streaming-tail-selection). @@ -554,22 +574,30 @@ const ChatViewInner = forwardRef(function ChatView({ sess // ping-pong bug). A pin sets scrollTop, not scrollHeight, so the next // onChange sees no growth and the loop cannot sustain itself. onChange: () => { + // Virtualizer may fire notify after jsdom teardown (suite-level + // unhandled "window is not defined"). Bail when unmounted / no DOM. + if (typeof window === "undefined") return; const el = scrollRef.current; - if (!el) return; - const grew = el.scrollHeight !== lastScrollHeightRef.current; - lastScrollHeightRef.current = el.scrollHeight; - // Suspend the bottom-pin while a transcript selection is held (D2) so the - // selected row is not scrolled out of its overscan band. stickToBottomRef - // is NOT cleared — follow resumes on collapse. - if (grew && stickToBottomRef.current && !isSelectingRef.current) el.scrollTop = el.scrollHeight; - // Ascending: re-target index 0 whenever a measurement grows the total - // size (an above-viewport row mounting/measuring, INCLUDING the async - // image-load remeasure). scrollToIndex is bounded to maxAttempts frames, - // so without this a late remeasure would leave the view off index 0. - if (ascendingRef.current) { + if (!el || !el.isConnected) return; + const prevH = lastScrollHeightRef.current; + const nextH = el.scrollHeight; + if (nextH === prevH) return; + // Stick: pin to bottom (unless a transcript selection is held — D2). + // Unstuck: compensate so prepended older rows do not shove the viewport + // content down. Must live here because this onChange also updates + // lastScrollHeightRef and would otherwise win the race and skip D5. + // See change: session-tail-rehydrate (D5 load-older anchor). + if (stickToBottomRef.current) { + if (!isSelectingRef.current) el.scrollTop = nextH; + } else if (ascendingRef.current) { + // Ascending owns scroll lock — re-target top; do not D5-compensate. if (el.scrollTop <= 0) ascendingRef.current = false; - else if (grew) virtualizer.scrollToIndex(0, { align: "start" }); + else virtualizer.scrollToIndex(0, { align: "start" }); + } else if (prevH > 0) { + // Unstuck mid-history: keep viewport content under load-older prepend. + el.scrollTop += nextH - prevH; } + lastScrollHeightRef.current = nextH; }, }); const virtualItems = virtualizer.getVirtualItems(); @@ -611,12 +639,14 @@ const ChatViewInner = forwardRef(function ChatView({ sess ); // Real user input (wheel / touch) cancels an in-flight descent so the user - // can always escape mid-flight. - const cancelDescent = useCallback(() => { + // can always escape mid-flight. Also marks a gesture so handleScroll can + // distinguish user intent from programmatic pin scrolls. + const onUserScrollGesture = useCallback(() => { descendingRef.current = false; // Real user input also escapes an in-flight scroll-to-top ascent so the // onChange re-issue cannot fight the user scrolling back down. ascendingRef.current = false; + userGestureRef.current = true; }, []); const handleScroll = useCallback(() => { @@ -637,9 +667,35 @@ const ChatViewInner = forwardRef(function ChatView({ sess if (nearTop) ascendingRef.current = false; stickToBottomRef.current = false; setShowScrollButton(true); + } else if (loadingHistory) { + // During hydrate, virtualizer pin/measurement fires synthetic scroll that + // is not a user escape. Only wheel/touch may clear stick mid-hydrate; + // otherwise live follow dies after the tail lands. + // See change: session-tail-rehydrate (live-follow after hydrate). + if (userGestureRef.current) { + stickToBottomRef.current = nearBottom; + setShowScrollButton(!nearBottom); + if (!nearBottom) escapedDuringHydrateRef.current = true; + if (nearBottom) userGestureRef.current = false; + } else if (nearBottom) { + stickToBottomRef.current = true; + setShowScrollButton(false); + } } else { + // Steady-state: any scroll (incl. scrollbar drag) updates stick. stickToBottomRef.current = nearBottom; setShowScrollButton(!nearBottom); + if (nearBottom) userGestureRef.current = false; + } + // Near top + more history available → request older page. Parent de-dupes + // in-flight requests. See change: session-tail-rehydrate. + if ( + hasMoreOlder && + !loadingOlder && + onLoadOlder && + el.scrollTop < SCROLL_THRESHOLD + ) { + onLoadOlder(); } setShowScrollTopButton(!nearTop); // Persist scroll position for this session in VIRTUAL coordinates (CR-6): @@ -654,7 +710,7 @@ const ChatViewInner = forwardRef(function ChatView({ sess nearBottom, }); } - }, [sessionId, virtualizer]); + }, [sessionId, virtualizer, loadingHistory, hasMoreOlder, loadingOlder, onLoadOlder]); const scrollToBottom = useCallback(() => { const el = scrollRef.current; @@ -693,40 +749,73 @@ const ChatViewInner = forwardRef(function ChatView({ sess // change (CR-6), so the dep list is intentionally [sessionId] only. // biome-ignore lint/correctness/useExhaustiveDependencies: intentional session-switch-only restore; see comment above. useLayoutEffect(() => { - if (sessionId !== prevSessionRef.current) { - // Outgoing scroll state is kept fresh by handleScroll (persists the - // virtual anchor on every scroll), so no re-capture here — re-capturing - // now would read the INCOMING session's virtualizer (CR-6). - prevSessionRef.current = sessionId; + if (sessionId === prevSessionRef.current) return; + // Outgoing scroll state is kept fresh by handleScroll (persists the + // virtual anchor on every scroll), so no re-capture here — re-capturing + // now would read the INCOMING session's virtualizer (CR-6). + prevSessionRef.current = sessionId; + escapedDuringHydrateRef.current = false; + loadingHistoryRef.current = !!loadingHistory; - // Restore incoming session scroll state in virtual coordinates. - const saved = sessionId ? scrollStateMap.get(sessionId) : undefined; - if (saved && !saved.nearBottom && saved.anchorRowId) { - // Scroll-locked: resolve the saved row id → current index, scroll it to - // the top, then re-apply the intra-row offset once the row measures. - descendingRef.current = false; - stickToBottomRef.current = false; - setShowScrollButton(true); - const anchorId = saved.anchorRowId; - const idx = displayRows.findIndex((r, i) => virtualRowKey(r, i) === anchorId); - if (idx >= 0) { - virtualizer.scrollToIndex(idx, { align: "start" }); - const off = saved.offset; - requestAnimationFrame(() => { - const el = scrollRef.current; - if (el) el.scrollTop += off; - }); - } else { - scrollRef.current?.scrollTo(0, scrollRef.current.scrollHeight); - } + // Restore incoming session scroll state in virtual coordinates. + // Cold hydrate (empty + loadingHistory): ignore a prior mid-list lock and + // land at bottom after history arrives (session-tail-rehydrate). + const saved = sessionId ? scrollStateMap.get(sessionId) : undefined; + const coldHydrate = !!(loadingHistory && state.messages.length === 0); + if (saved && !saved.nearBottom && saved.anchorRowId && !coldHydrate) { + // Scroll-locked: resolve the saved row id → current index, scroll it to + // the top, then re-apply the intra-row offset once the row measures. + descendingRef.current = false; + stickToBottomRef.current = false; + setShowScrollButton(true); + const anchorId = saved.anchorRowId; + const idx = displayRows.findIndex((r, i) => virtualRowKey(r, i) === anchorId); + if (idx >= 0) { + virtualizer.scrollToIndex(idx, { align: "start" }); + const off = saved.offset; + requestAnimationFrame(() => { + const el = scrollRef.current; + if (el) el.scrollTop += off; + }); } else { - // Near bottom or first visit: scroll to end and follow new content. + scrollRef.current?.scrollTo(0, scrollRef.current.scrollHeight); + } + } else { + // Near bottom, first visit, or cold hydrate: scroll to end and follow. + stickToBottomRef.current = true; + setShowScrollButton(false); + scrollRef.current?.scrollTo(0, scrollRef.current!.scrollHeight); + } + }, [sessionId]); + + // After a same-session wipe→rebuild (`loadingHistory` true→false), re-pin + // to the true bottom unless the user scrolled away mid-hydrate. Restore only + // runs on sessionId change, so cold return / multi-batch replay would + // otherwise land mid-transcript. See change: session-tail-rehydrate. + useLayoutEffect(() => { + const wasLoading = loadingHistoryRef.current; + const nowLoading = !!loadingHistory; + loadingHistoryRef.current = nowLoading; + if (!wasLoading && nowLoading) { + // Hydrate started: treat as cold land — arm stick unless user escapes mid-hydrate. + escapedDuringHydrateRef.current = false; + stickToBottomRef.current = true; + setShowScrollButton(false); + return; + } + if (wasLoading && !nowLoading) { + if (!escapedDuringHydrateRef.current) { stickToBottomRef.current = true; setShowScrollButton(false); - scrollRef.current?.scrollTo(0, scrollRef.current!.scrollHeight); + const el = scrollRef.current; + if (el) el.scrollTop = el.scrollHeight; + if (sessionId) { + scrollStateMap.set(sessionId, { anchorRowId: null, offset: 0, nearBottom: true }); + } } + escapedDuringHydrateRef.current = false; } - }, [sessionId]); + }, [loadingHistory, sessionId]); // Auto-scroll on new content when the user has not escaped the bottom. // Layout effect keeps the DOM and scroll position synchronized before paint, @@ -754,6 +843,23 @@ const ChatViewInner = forwardRef(function ChatView({ sess } }, [state.messages.length, state.streamingText, state.pendingPrompt, state.streamingThinking, pendingSteering, isSelecting]); + // Fallback when message count changes but virtualizer onChange did not + // observe a height delta (common in jsdom). If onChange already synced + // lastScrollHeightRef to the new height, nextH === prevH and this is a no-op + // (avoids double-compensation after the onChange path above). + // See change: session-tail-rehydrate (D5 load-older anchor). + useLayoutEffect(() => { + const el = scrollRef.current; + if (!el) return; + const prevH = lastScrollHeightRef.current; + const nextH = el.scrollHeight; + if (nextH === prevH) return; + if (!stickToBottomRef.current && prevH > 0 && nextH > prevH) { + el.scrollTop += nextH - prevH; + } + lastScrollHeightRef.current = nextH; + }, [state.messages.length]); + useImperativeHandle(ref, () => ({ scrollToTurn(turnIndex: number) { // Map the turn to its first display-row index and scroll there. Unlike @@ -780,7 +886,27 @@ const ChatViewInner = forwardRef(function ChatView({ sess `scroll-behavior: smooth` here or on an ancestor — smooth would animate each synchronous measurement correction and race the next, reintroducing the scroll-to-top drift. See change: fix-chat-scroll-to-top-estimate-drift. */} -
+ {(loadingOlder || (hasMoreOlder && state.messages.length > 0)) && ( +
+ {loadingOlder ? ( + i18nT("auto.loading_older", undefined, "Loading older messages…") + ) : ( + + )} +
+ )} +
{/* Windowed historical rows (TanStack Virtual): only viewport + overscan are mounted. The spacer reserves getTotalSize(); each row is absolutely positioned + re-measured on mount. chat-cv-skip keeps Step A's diff --git a/packages/client/src/components/ToolCallStep.tsx b/packages/client/src/components/ToolCallStep.tsx index 1dd329650..13a2a39c7 100644 --- a/packages/client/src/components/ToolCallStep.tsx +++ b/packages/client/src/components/ToolCallStep.tsx @@ -64,6 +64,8 @@ interface Props { hideStatusIcon?: boolean; onAbort?: () => void; onForceKill?: () => void; + /** When true, show a "superseded" badge (retried tool call). */ + isSuperseded?: boolean; } const statusIcons: Record = { @@ -72,7 +74,7 @@ const statusIcons: Record = { error: , }; -export function ToolCallStep({ toolName, toolCallId, args, status, result, images, context, startedAt, duration, toolDetails, showResultBody = true, hideStatusIcon = false, onAbort, onForceKill }: Props) { +export function ToolCallStep({ toolName, toolCallId, args, status, result, images, context, startedAt, duration, toolDetails, showResultBody = true, hideStatusIcon = false, onAbort, onForceKill, isSuperseded = false }: Props) { const isMobile = useMobile(); const hasImages = images && images.length > 0; const isAgentRunning = toolName === "Agent" && status === "running"; diff --git a/packages/client/src/components/__tests__/ChatView.scroll-race.test.tsx b/packages/client/src/components/__tests__/ChatView.scroll-race.test.tsx index e8da23ff9..19b9fc6ce 100644 --- a/packages/client/src/components/__tests__/ChatView.scroll-race.test.tsx +++ b/packages/client/src/components/__tests__/ChatView.scroll-race.test.tsx @@ -193,6 +193,191 @@ describe("ChatView sticky scroll", () => { // Escape respected: button re-appears, no forced pin. expect(container.querySelector('[data-testid="scroll-to-bottom"]')).not.toBeNull(); }); + + /** + * Cold return / full replay often wipes then rebuilds messages without + * changing `sessionId`. Restore only runs on sessionId change, so a prior + * scroll-lock would leave stickToBottom=false while history floods in. + * loadingHistory true→false re-pins bottom unless the user escapes mid-hydrate. + * See change: session-tail-rehydrate. + */ + describe("hydrate land after same-session wipe/rebuild", () => { + it("re-pins to bottom after empty wipe + full history even if stick was escaped before hydrate", async () => { + const { container, rerender } = render( + + + , + ); + await flushRaf(); + + const scrollEl = getScrollContainer(container); + // User (or estimate-correction scroll event) leaves the bottom. + setScrollPosition(scrollEl, 200, 4000, 400); + fireEvent.scroll(scrollEl); + expect(container.querySelector('[data-testid="scroll-to-bottom"]')).not.toBeNull(); + + // Full wipe: same sessionId, empty transcript + loadingHistory. + rerender( + + + , + ); + await flushRaf(); + + const rebuilt = stateWith(80); + rebuilt.messages[rebuilt.messages.length - 1] = { + id: "asst-last", + role: "assistant", + content: "final agent reply", + timestamp: Date.now(), + }; + rebuilt.messages[40] = { + id: "asst-mid", + role: "assistant", + content: "mid-history agent reply that looks like a resume target", + timestamp: Date.now(), + }; + + // Layout lag: content taller than viewport, scrollTop still mid. + setScrollPosition(scrollEl, 200, 8000, 400); + rerender( + + + , + ); + await flushRaf(); + + expect(scrollEl.scrollTop).toBe(8000); + expect(container.querySelector('[data-testid="scroll-to-bottom"]')).toBeNull(); + }); + + it("first visit with empty then hydrate still chases bottom while stick is armed", async () => { + const { container, rerender } = render( + + + , + ); + await flushRaf(); + + const scrollEl = getScrollContainer(container); + setScrollPosition(scrollEl, 0, 400, 400); + fireEvent.scroll(scrollEl); + + setScrollPosition(scrollEl, 0, 5000, 400); + rerender( + + + , + ); + await flushRaf(); + + expect(scrollEl.scrollTop).toBe(5000); + expect(container.querySelector('[data-testid="scroll-to-bottom"]')).toBeNull(); + }); + + // Session-switch + cold return is covered by the wipe/rebuild case above + // (same-session loadingHistory true→false re-pin) plus cold-open stick arming. + + it("programmatic mid-list scroll during hydrate does not kill live stick after land", async () => { + // Regression: virtualizer pin/measurement fires scroll without a user + // gesture. Treating those as escape left stick=false after tail hydrate, + // so live streaming no longer auto-followed. + // See change: session-tail-rehydrate (live-follow after hydrate). + const { container, rerender } = render( + + + , + ); + await flushRaf(); + + const scrollEl = getScrollContainer(container); + // Programmatic "jump" mid-list while hydrating — no wheel/touch. + setScrollPosition(scrollEl, 100, 5000, 400); + fireEvent.scroll(scrollEl); + + // Hydrate completes with content; must re-pin and keep stick. + setScrollPosition(scrollEl, 100, 5000, 400); + rerender( + + + , + ); + await flushRaf(); + expect(scrollEl.scrollTop).toBe(5000); + expect(container.querySelector('[data-testid="scroll-to-bottom"]')).toBeNull(); + + // Further content growth must still be chased (live follow). + setScrollPosition(scrollEl, 5000, 6000, 400); + rerender( + + + , + ); + await flushRaf(); + expect(scrollEl.scrollTop).toBe(6000); + expect(container.querySelector('[data-testid="scroll-to-bottom"]')).toBeNull(); + }); + + it("real wheel during hydrate still allows escape from stick", async () => { + const { container, rerender } = render( + + + , + ); + await flushRaf(); + + const scrollEl = getScrollContainer(container); + fireEvent.wheel(scrollEl, { deltaY: -40 }); + setScrollPosition(scrollEl, 50, 5000, 400); + fireEvent.scroll(scrollEl); + + setScrollPosition(scrollEl, 50, 5000, 400); + rerender( + + + , + ); + await flushRaf(); + + // Escaped mid-hydrate: do not force re-pin; scroll-to-bottom button shows. + expect(container.querySelector('[data-testid="scroll-to-bottom"]')).not.toBeNull(); + }); + }); }); // Scroll-to-top affordance (change: fix-chat-scroll-to-top-estimate-drift, @@ -265,3 +450,125 @@ describe("ChatView scroll-to-top", () => { expect(scrollEl.scrollTop).toBe(before); // not yanked to scrollHeight (2000) }); }); + +describe("ChatView load-older + prepend scroll anchor (session-tail-rehydrate)", () => { + it("calls onLoadOlder once when scrolled near the top with hasMoreOlder; silent while scrollTop is high", async () => { + const onLoadOlder = vi.fn(); + const { container } = render( + + + , + ); + await flushRaf(); + + const scrollEl = getScrollContainer(container); + + // Park well below the near-top band: onLoadOlder must NOT fire here. + // (SCROLL_THRESHOLD gates both the bottom-pin and the load-older trigger; + // 500 is comfortably above it.) Two scrolls mirror the escape tests so the + // virtualizer's measurement onChange settles stick=false without a pin. + setScrollPosition(scrollEl, 500, 2000, 400); + fireEvent.scroll(scrollEl); + setScrollPosition(scrollEl, 500, 2000, 400); + fireEvent.scroll(scrollEl); + expect(onLoadOlder).not.toHaveBeenCalled(); + + // Cross into the near-top band. scrollHeight is unchanged so the virtual + // range does not grow — the only handleScroll that crosses the threshold + // fires onLoadOlder exactly once. + setScrollPosition(scrollEl, 0, 2000, 400); + fireEvent.scroll(scrollEl); + expect(onLoadOlder).toHaveBeenCalledTimes(1); + }); + + it("clicking data-testid=load-older-button fires onLoadOlder", async () => { + const onLoadOlder = vi.fn(); + const { container } = render( + + + , + ); + await flushRaf(); + + const button = container.querySelector('[data-testid="load-older-button"]'); + expect(button).not.toBeNull(); + fireEvent.click(button!); + expect(onLoadOlder).toHaveBeenCalledTimes(1); + }); + + it("prepend growth while not sticking compensates scrollTop by height delta", async () => { + // See change: session-tail-rehydrate (D5 load-older anchor). + // Product path: virtualizer onChange (or messages.length layout fallback) + // adds height delta to scrollTop when stick is false so prepended older + // rows keep the same content under the viewport. + const { container, rerender } = render( + + + , + ); + await flushRaf(); + + const scrollEl = getScrollContainer(container); + let top = 400; + Object.defineProperty(scrollEl, "scrollTop", { + configurable: true, + get: () => top, + set: (v: number) => { + top = Number(v); + }, + }); + Object.defineProperty(scrollEl, "clientHeight", { + configurable: true, + value: 400, + writable: true, + }); + Object.defineProperty(scrollEl, "scrollHeight", { + configurable: true, + value: 2000, + writable: true, + }); + + // Escape sticky bottom (mid-list). + fireEvent.scroll(scrollEl); + expect(container.querySelector('[data-testid="scroll-to-bottom"]')).not.toBeNull(); + + // Seed lastScrollHeightRef at 2000 (mount often sees jsdom height 0). + rerender( + + + , + ); + await flushRaf(); + top = 400; + fireEvent.scroll(scrollEl); + + // Grow height before message-count update so onChange/layout see the delta. + Object.defineProperty(scrollEl, "scrollHeight", { + configurable: true, + value: 3000, + writable: true, + }); + rerender( + + + , + ); + await flushRaf(); + + // 400 + (3000 - 2000) = 1400; stick path would pin to 3000. + expect(top).toBe(1400); + expect(container.querySelector('[data-testid="scroll-to-bottom"]')).not.toBeNull(); + }); +}); diff --git a/packages/client/src/components/__tests__/ChatView.test.tsx b/packages/client/src/components/__tests__/ChatView.test.tsx index ef5d554d1..f175450da 100644 --- a/packages/client/src/components/__tests__/ChatView.test.tsx +++ b/packages/client/src/components/__tests__/ChatView.test.tsx @@ -474,7 +474,7 @@ describe("ChatView", () => { , ); - expect(scrollEl.scrollTop).toBe(950); + expect(scrollEl.scrollTop).toBe(1450); // D5 height-delta compensation }); it("clicking scroll-to-bottom button uses instant scroll while streaming", () => { diff --git a/packages/client/src/hooks/__tests__/useMessageHandler.tail-rehydrate.test.tsx b/packages/client/src/hooks/__tests__/useMessageHandler.tail-rehydrate.test.tsx new file mode 100644 index 000000000..bfdd01930 --- /dev/null +++ b/packages/client/src/hooks/__tests__/useMessageHandler.tail-rehydrate.test.tsx @@ -0,0 +1,258 @@ +/** + * Regressions for change: session-tail-rehydrate (reviewer P1/P2). + */ +import { describe, it, expect, vi } from "vitest"; +import { renderHook } from "@testing-library/react"; +import { useMessageHandler } from "../useMessageHandler.js"; +import { + createInitialState, + type SessionState, + type InteractiveUiRequest, +} from "../../lib/event-reducer.js"; +import type { DashboardEvent } from "@blackbelt-technology/pi-dashboard-shared/types.js"; +import type { ServerToBrowserMessage } from "@blackbelt-technology/pi-dashboard-shared/browser-protocol.js"; +import type { CachedEvent } from "../../lib/replay-cache.js"; + +function makeStartEvt(toolCallId: string, ts: number): DashboardEvent { + return { + eventType: "tool_execution_start", + timestamp: ts, + data: { toolCallId, toolName: "bash", args: { command: `cmd-${toolCallId}` } }, + }; +} + +function setup(opts?: { withPersister?: boolean }) { + const sessionStatesRef = { current: new Map() }; + const maxSeqMap = new Map(); + const historyWindowRef = { + current: new Map(), + }; + const historyWindowMapRef = { + current: new Map(), + }; + const loadingOlderMapRef = { current: new Map() }; + const buffers = new Map(); + + const setSessionStates = vi.fn((updater: any) => { + if (typeof updater === "function") { + sessionStatesRef.current = updater(sessionStatesRef.current); + } else { + sessionStatesRef.current = updater; + } + }); + const setHistoryWindowMap = vi.fn((updater: any) => { + if (typeof updater === "function") { + historyWindowMapRef.current = updater(historyWindowMapRef.current); + } else { + historyWindowMapRef.current = updater; + } + }); + const setLoadingOlderMap = vi.fn((updater: any) => { + if (typeof updater === "function") { + loadingOlderMapRef.current = updater(loadingOlderMapRef.current); + } else { + loadingOlderMapRef.current = updater; + } + }); + + const replayPersister = opts?.withPersister + ? { + record(sessionId: string, events: CachedEvent[]) { + const cur = buffers.get(sessionId) ?? []; + const bySeq = new Map(cur.map((e) => [e.seq, e])); + for (const e of events) bySeq.set(e.seq, e); + buffers.set(sessionId, [...bySeq.values()].sort((a, b) => a.seq - b.seq)); + }, + seed(sessionId: string, events: CachedEvent[]) { + buffers.set(sessionId, [...events].sort((a, b) => a.seq - b.seq)); + }, + merge(sessionId: string, events: CachedEvent[]) { + const cur = buffers.get(sessionId) ?? []; + const bySeq = new Map(cur.map((e) => [e.seq, e])); + for (const e of events) bySeq.set(e.seq, e); + const merged = [...bySeq.values()].sort((a, b) => a.seq - b.seq); + buffers.set(sessionId, merged); + return merged; + }, + snapshot(sessionId: string) { + return buffers.get(sessionId) ?? []; + }, + drop: vi.fn(async (sessionId: string) => { + buffers.delete(sessionId); + }), + flush: vi.fn(async () => {}), + } + : undefined; + + const setters: any = { + setSessions: vi.fn(), + setSessionStates, + setSessionCommands: vi.fn(), + setFileResults: vi.fn(), + setChangedOnDisk: vi.fn(), + setOpenspecMap: vi.fn(), + setFolderGitMap: vi.fn(), + setOpenspecGroupsMap: vi.fn(), + setModelsMap: vi.fn(), + setRolesMap: vi.fn(), + setSpawnResult: vi.fn(), + setSessionOrderMap: vi.fn(), + setPinnedDirectories: vi.fn(), + setFavoriteModels: vi.fn(), + setWorkspaces: vi.fn(), + setTerminals: vi.fn(), + setEditorStatuses: vi.fn(), + setDiscoveredServers: vi.fn(), + setSpawnErrors: vi.fn(), + setResumeErrors: vi.fn(), + setDisplayPrefs: vi.fn(), + setViewMessagesMap: vi.fn(), + setLoadingHistory: vi.fn(), + }; + + const deps: any = { + send: vi.fn(), + navigate: vi.fn(), + clearSpawningCwd: vi.fn(), + spawningCwdsRef: { current: new Set() }, + subscribedRef: { current: new Set() }, + pendingTerminalCwdRef: { current: null }, + lastCreatedTerminalIdRef: { current: null }, + maxSeqMapRef: { current: maxSeqMap }, + selectedSessionIdRef: { current: undefined }, + pendingSpawnsRef: { current: new Map() }, + loadingHistoryTimersRef: { current: new Map() }, + replayPersister, + historyWindowRef, + setHistoryWindowMap, + setLoadingOlderMap, + }; + + const { result } = renderHook(() => useMessageHandler(setters, deps)); + const dispatch = (msg: ServerToBrowserMessage) => result.current(msg); + return { + dispatch, + sessionStatesRef, + maxSeqMap, + historyWindowRef, + historyWindowMapRef, + loadingOlderMapRef, + buffers, + setLoadingOlderMap, + }; +} + +describe("useMessageHandler session-tail-rehydrate", () => { + const SID = "s-tail"; + + it("session_state_reset clears history window meta", () => { + const { dispatch, historyWindowRef, historyWindowMapRef, loadingOlderMapRef } = setup(); + historyWindowRef.current.set(SID, { minSeq: 50, hasMoreOlder: true }); + historyWindowMapRef.current.set(SID, { minSeq: 50, hasMoreOlder: true }); + loadingOlderMapRef.current.set(SID, true); + + dispatch({ type: "session_state_reset", sessionId: SID } as ServerToBrowserMessage); + + expect(historyWindowRef.current.has(SID)).toBe(false); + expect(historyWindowMapRef.current.has(SID)).toBe(false); + expect(loadingOlderMapRef.current.has(SID)).toBe(false); + }); + + it("does not invent hasMoreOlder from warm deltas without window meta", () => { + const { dispatch, historyWindowRef, maxSeqMap } = setup(); + // Seed a prior max so this is a warm delta, not a cold seed. + maxSeqMap.set(SID, 10); + dispatch({ + type: "event_replay", + sessionId: SID, + events: [{ seq: 11, event: makeStartEvt("t11", 1100) }], + isLast: true, + } as ServerToBrowserMessage); + + expect(historyWindowRef.current.get(SID)).toBeUndefined(); + }); + + it("cold windowed seed (firstSeq>1, maxSeq=0) infers hasMoreOlder", () => { + const { dispatch, historyWindowRef } = setup(); + dispatch({ + type: "event_replay", + sessionId: SID, + events: [ + { seq: 50, event: makeStartEvt("t50", 5000) }, + { seq: 51, event: makeStartEvt("t51", 5100) }, + ], + isLast: true, + } as ServerToBrowserMessage); + + expect(historyWindowRef.current.get(SID)).toEqual({ minSeq: 50, hasMoreOlder: true }); + }); + + it("loadingOlder clears only on isLast terminal batch", () => { + const { dispatch, loadingOlderMapRef, maxSeqMap, buffers } = setup({ withPersister: true }); + maxSeqMap.set(SID, 100); + // Seed tail buffer so older page can merge + buffers.set(SID, [ + { seq: 90, event: makeStartEvt("t90", 9000) }, + { seq: 100, event: makeStartEvt("t100", 10000) }, + ]); + loadingOlderMapRef.current.set(SID, true); + + dispatch({ + type: "event_replay", + sessionId: SID, + events: [{ seq: 80, event: makeStartEvt("t80", 8000) }], + windowMaxSeq: 80, + isLast: false, + } as ServerToBrowserMessage); + expect(loadingOlderMapRef.current.get(SID)).toBe(true); + + dispatch({ + type: "event_replay", + sessionId: SID, + events: [{ seq: 70, event: makeStartEvt("t70", 7000) }], + windowMaxSeq: 80, + isLast: true, + } as ServerToBrowserMessage); + expect(loadingOlderMapRef.current.get(SID)).toBe(false); + }); + + it("older-page merge preserves pending interactive ask UI", () => { + const { dispatch, sessionStatesRef, maxSeqMap, buffers } = setup({ withPersister: true }); + maxSeqMap.set(SID, 20); + buffers.set(SID, [ + { seq: 15, event: makeStartEvt("t15", 1500) }, + { seq: 20, event: makeStartEvt("t20", 2000) }, + ]); + + const ask: InteractiveUiRequest = { + requestId: "ask-1", + method: "ask_user", + params: { prompt: "Continue?" }, + status: "pending", + }; + const seeded = createInitialState(); + seeded.interactiveRequests = [ask]; + seeded.messages = [ + { + id: "ui-ask-1", + role: "interactiveUi", + content: "ask_user", + timestamp: 999, + args: { requestId: "ask-1", method: "ask_user", params: { prompt: "Continue?" }, status: "pending" }, + }, + ]; + sessionStatesRef.current.set(SID, seeded); + + dispatch({ + type: "event_replay", + sessionId: SID, + events: [{ seq: 5, event: makeStartEvt("t5", 500) }], + windowMaxSeq: 5, + isLast: true, + } as ServerToBrowserMessage); + + const state = sessionStatesRef.current.get(SID)!; + expect(state.interactiveRequests.some((r) => r.requestId === "ask-1")).toBe(true); + expect(state.messages.some((m) => m.role === "interactiveUi" && m.id === "ui-ask-1")).toBe(true); + }); +}); diff --git a/packages/client/src/hooks/useMessageHandler.ts b/packages/client/src/hooks/useMessageHandler.ts index c340f2ae3..82c8b695c 100644 --- a/packages/client/src/hooks/useMessageHandler.ts +++ b/packages/client/src/hooks/useMessageHandler.ts @@ -155,6 +155,17 @@ export interface MessageHandlerDeps { * See change: add-auto-session-naming. */ showToast?: (text: string, variant?: "error" | "success" | "info") => void; + /** + * Per-session history-window cursor for session-tail-rehydrate: the low + * watermark (`minSeq`) already loaded and whether older events remain on + * the server. Updated as older history is fetched so re-renders don't + * refetch. See change: session-tail-rehydrate. + */ + historyWindowRef?: React.MutableRefObject>; + /** Set-state for the history-window cursor map. See change: session-tail-rehydrate. */ + setHistoryWindowMap?: React.Dispatch>>; + /** Per-session "loading older" flag for the Load-more affordance. See change: session-tail-rehydrate. */ + setLoadingOlderMap?: React.Dispatch>>; } export function useMessageHandler( @@ -168,7 +179,24 @@ export function useMessageHandler( setDiscoveredServers, setSpawnErrors, setResumeErrors, setDisplayPrefs, setViewMessagesMap, setLoadingHistory, setCanvasMap, } = setters; - const { send, navigate, clearSpawningCwd, spawningCwdsRef, subscribedRef, pendingTerminalCwdRef, lastCreatedTerminalIdRef, maxSeqMapRef, selectedSessionIdRef, pendingSpawnsRef, loadingHistoryTimersRef, replayPersister, showToast } = deps; + const { + send, + navigate, + clearSpawningCwd, + spawningCwdsRef, + subscribedRef, + pendingTerminalCwdRef, + lastCreatedTerminalIdRef, + maxSeqMapRef, + selectedSessionIdRef, + pendingSpawnsRef, + loadingHistoryTimersRef, + replayPersister, + showToast, + historyWindowRef, + setHistoryWindowMap, + setLoadingOlderMap, + } = deps; // One-shot per session: suppress a repeat auto-name toast for the same // session id. See change: add-auto-session-naming. const autoNameToastedRef = useRef>(new Set()); @@ -375,10 +403,39 @@ export function useMessageHandler( return next; }); maxSeqMapRef.current.set(msg.sessionId, 0); + historyWindowRef?.current.delete(msg.sessionId); + setHistoryWindowMap?.((prev) => { + if (!prev.has(msg.sessionId)) return prev; + const next = new Map(prev); + next.delete(msg.sessionId); + return next; + }); + setLoadingOlderMap?.((prev) => { + if (!prev.has(msg.sessionId)) return prev; + const next = new Map(prev); + next.delete(msg.sessionId); + return next; + }); // Strategy A invalidation: purge the durable cache so stale history is // never stitched onto reset sequence numbers; full replay rebuilds it. // See change: reduce-session-replay-traffic. void replayPersister?.drop(msg.sessionId); + // Drop load-older window meta so a rebuilt tail does not Math.min with a + // pre-reset minSeq (which can skip a mid-range forever). + // See change: session-tail-rehydrate. + historyWindowRef?.current.delete(msg.sessionId); + setHistoryWindowMap?.((m) => { + if (!m.has(msg.sessionId)) return m; + const next = new Map(m); + next.delete(msg.sessionId); + return next; + }); + setLoadingOlderMap?.((m) => { + if (!m.get(msg.sessionId)) return m; + const next = new Map(m); + next.set(msg.sessionId, false); + return next; + }); // Mirror the reset into the plugin-runtime per-session event // store so plugin reducers (e.g. flows-plugin) re-derive from // a clean stream after a replay. See change: @@ -625,72 +682,177 @@ export function useMessageHandler( case "event_replay": { const firstSeq = msg.events.length > 0 ? msg.events[0].seq : null; - // Reset on every full replay sweep: firstSeq===1 (cold start) OR - // firstSeq <= maxSeq for this session (server is re-replaying events - // the client has already accounted for, e.g. paginated reconnect - // re-replay where the first batch may not start at seq=1). - // See change: fix-replay-duplicates-tool-and-flushed-rows. const maxSeq = maxSeqMapRef.current.get(msg.sessionId) ?? 0; - const shouldReset = firstSeq != null && (firstSeq === 1 || firstSeq <= maxSeq); - setSessionStates((prev) => { - const next = new Map(prev); - // Same rationale as session_state_reset: preserve optimistic - // pendingPrompt across the full-replay reset branch. - // See change: preserve-pending-prompt-across-replay. - const carry = shouldReset ? next.get(msg.sessionId)?.pendingPrompt : undefined; - let current = shouldReset ? createInitialState() : (next.get(msg.sessionId) ?? createInitialState()); - if (carry) current.pendingPrompt = carry; - for (const { event } of msg.events) { - current = reduceEvent(current, event); + // Older page: every delivered seq is below the client's current max. + // Must merge+re-reduce, never wipe to only the older page. + // See change: session-tail-rehydrate. + const isOlderPage = + msg.events.length > 0 && + maxSeq > 0 && + (msg.windowMaxSeq != null + ? msg.windowMaxSeq < maxSeq + : msg.events.every((e) => e.seq < maxSeq)); + + // Reset on full replay sweep (cold start / re-replay of known seqs), + // but never for older pages. + // Windowed cold tail (mode:tail, firstSeq > 1, no prior max) is also a + // reset: seed from the window rather than appending onto empty/partial + // state. See change: fix-replay-duplicates-tool-and-flushed-rows, + // session-tail-rehydrate. + const shouldReset = + !isOlderPage && + firstSeq != null && + (firstSeq === 1 || firstSeq <= maxSeq || maxSeq === 0); + + if (isOlderPage && replayPersister) { + const merged = replayPersister.merge(msg.sessionId, msg.events); + setSessionStates((prev) => { + const next = new Map(prev); + const prevState = next.get(msg.sessionId); + // Re-reduce raw events from empty, but keep out-of-band UI that is + // not in the event store (pendingPrompt + active interactive asks). + // Server load-older does not re-send pending UI requests. + // See change: session-tail-rehydrate. + let current = createInitialState(); + if (prevState?.pendingPrompt) current.pendingPrompt = prevState.pendingPrompt; + for (const { event } of merged) { + current = reduceEvent(current, event); + } + if (prevState && prevState.interactiveRequests.length > 0) { + const byId = new Map(current.interactiveRequests.map((r) => [r.requestId, r])); + for (const req of prevState.interactiveRequests) { + if (!byId.has(req.requestId)) { + current.interactiveRequests = [...current.interactiveRequests, req]; + } + } + const uiRows = prevState.messages.filter((m) => m.role === "interactiveUi"); + const have = new Set( + current.messages.filter((m) => m.role === "interactiveUi").map((m) => m.id), + ); + for (const row of uiRows) { + if (!have.has(row.id)) current.messages = [...current.messages, row]; + } + } + next.set(msg.sessionId, current); + return next; + }); + clearSessionEvents(msg.sessionId); + publishSessionEvents( + msg.sessionId, + merged.map((e) => e.event), + ); + if (merged.length > 0) { + const lastEvt = merged[merged.length - 1]!; + if (lastEvt.seq > (maxSeqMapRef.current.get(msg.sessionId) ?? 0)) { + maxSeqMapRef.current.set(msg.sessionId, lastEvt.seq); + } } - next.set(msg.sessionId, current); - return next; - }); - // Mirror the replayed batch into the plugin-runtime per-session event - // store so plugin slot consumers (flows card, goal chip) reading - // `useSessionEvents` rehydrate on cold load — the live `event` path - // publishes per event, so the replay path must too. Reuse `shouldReset` - // (full-sweep) to clear before republishing so a re-replay does not - // duplicate; continuation batches append. - // See change: replay-persisted-flow-runs. - if (shouldReset) clearSessionEvents(msg.sessionId); - publishSessionEvents(msg.sessionId, msg.events.map((e) => e.event)); - // If we reset, also reset maxSeq tracking so a subsequent batch isn't - // misclassified. We rebuild it below from this batch's events. - if (shouldReset) { - maxSeqMapRef.current.set(msg.sessionId, 0); - } - // Track highest seq from replay batch - if (msg.events.length > 0) { - const lastEvt = msg.events[msg.events.length - 1]; - if (lastEvt.seq > (maxSeqMapRef.current.get(msg.sessionId) ?? 0)) { - maxSeqMapRef.current.set(msg.sessionId, lastEvt.seq); + } else { + setSessionStates((prev) => { + const next = new Map(prev); + // Same rationale as session_state_reset: preserve optimistic + // pendingPrompt across the full-replay reset branch. + // See change: preserve-pending-prompt-across-replay. + const carry = shouldReset ? next.get(msg.sessionId)?.pendingPrompt : undefined; + let current = shouldReset ? createInitialState() : (next.get(msg.sessionId) ?? createInitialState()); + if (carry) current.pendingPrompt = carry; + for (const { event } of msg.events) { + current = reduceEvent(current, event); + } + next.set(msg.sessionId, current); + return next; + }); + // Mirror the replayed batch into the plugin-runtime per-session event + // store so plugin slot consumers rehydrate on cold load. + // See change: replay-persisted-flow-runs. + if (shouldReset) clearSessionEvents(msg.sessionId); + publishSessionEvents(msg.sessionId, msg.events.map((e) => e.event)); + if (shouldReset) { + maxSeqMapRef.current.set(msg.sessionId, 0); + } + if (msg.events.length > 0) { + const lastEvt = msg.events[msg.events.length - 1]!; + if (lastEvt.seq > (maxSeqMapRef.current.get(msg.sessionId) ?? 0)) { + maxSeqMapRef.current.set(msg.sessionId, lastEvt.seq); + } + } + // Strategy A: mirror into durable buffer. + if (msg.events.length > 0) { + if (shouldReset) replayPersister?.seed(msg.sessionId, msg.events); + else replayPersister?.record(msg.sessionId, msg.events); } } - // Strategy A: mirror the reducer into the durable replay buffer. A - // full-sweep reset (shouldReset) replaces the buffer; a delta appends. - // This is also the reconciliation path: an offline-drift replay whose - // firstSeq <= maxSeq resets and rebuilds the persisted tail too. - // See change: reduce-session-replay-traffic. - if (msg.events.length > 0) { - if (shouldReset) replayPersister?.seed(msg.sessionId, msg.events); - else replayPersister?.record(msg.sessionId, msg.events); + + // Track window meta for load-older UI. + // Prefer server meta. Infer hasMoreOlder only for cold/windowed seeds + // (no prior maxSeq / shouldReset cold land), never for warm deltas — + // otherwise old servers that omit window fields look like gaps forever. + // See change: session-tail-rehydrate. + { + const prev = historyWindowRef?.current.get(msg.sessionId); + const serverSaid = msg.hasMoreOlder != null || msg.windowMinSeq != null; + const coldInfer = + !isOlderPage && + shouldReset && + firstSeq != null && + firstSeq > 1 && + msg.hasMoreOlder == null; + if (serverSaid || coldInfer) { + const inferredMin = + msg.windowMinSeq != null + ? msg.windowMinSeq + : firstSeq != null + ? firstSeq + : (prev?.minSeq ?? 0); + // On shouldReset cold rebuild, replace minSeq (do not Math.min with + // pre-reset values — those were cleared on session_state_reset). + const minSeq = + shouldReset || !prev + ? inferredMin + : Math.min(prev.minSeq, inferredMin); + const hasMoreOlder = + msg.hasMoreOlder != null + ? msg.hasMoreOlder + : coldInfer + ? true + : (prev?.hasMoreOlder ?? false); + historyWindowRef?.current.set(msg.sessionId, { minSeq, hasMoreOlder }); + setHistoryWindowMap?.((m) => { + const next = new Map(m); + next.set(msg.sessionId, { minSeq, hasMoreOlder }); + return next; + }); + } else if (isOlderPage && firstSeq != null && prev) { + // Older page without meta: advance minSeq downward only. + const minSeq = Math.min(prev.minSeq, firstSeq); + historyWindowRef?.current.set(msg.sessionId, { minSeq, hasMoreOlder: prev.hasMoreOlder }); + setHistoryWindowMap?.((m) => { + const next = new Map(m); + next.set(msg.sessionId, { minSeq, hasMoreOlder: prev.hasMoreOlder }); + return next; + }); + } } - // Exit LOADING: first content (clear immediately so partial history - // paints) OR terminal marker for a genuinely-empty session - // (`events:[], isLast:true` → falls through to "No messages yet"). - // Else — the empty non-terminal marker (`events:[], isLast:false`) is the - // cold-hydration start marker AND every server heartbeat: re-arm the - // short subscribe window to the longer hydration ceiling so a slow disk - // parse never flashes "No messages yet". `rearmLoadingHistory` no-ops - // unless a timer is armed (flag set), so warm/painted sessions are - // unaffected. See change: show-chat-history-loading-indicator, + + // Exit LOADING: first content or terminal empty session. + // See change: show-chat-history-loading-indicator, // fix-history-loading-false-empty-flash. if (msg.events.length > 0 || msg.isLast === true) { clearLoadingHistory(setLoadingHistory, loadingHistoryTimersRef, msg.sessionId); } else { rearmLoadingHistory(setLoadingHistory, loadingHistoryTimersRef, msg.sessionId, HYDRATE_CEILING_MS); } + // Load-older in-flight ends only on terminal batch (isLast), including + // empty terminal pages — multi-batch older pages must not clear early. + // See change: session-tail-rehydrate. + if (msg.isLast === true) { + setLoadingOlderMap?.((m) => { + if (!m.get(msg.sessionId)) return m; + const next = new Map(m); + next.set(msg.sessionId, false); + return next; + }); + } break; } @@ -1139,5 +1301,5 @@ export function useMessageHandler( break; } } - }, [send, clearSpawningCwd, navigate, setSessions, setSessionStates, setSessionCommands, setFileResults, setChangedOnDisk, setOpenspecMap, setModelsMap, setRolesMap, setSpawnResult, setSessionOrderMap, setPinnedDirectories, setPinnedDirsLoaded, setFavoriteModels, setWorkspaces, setTerminals, setDiscoveredServers, setLoadingHistory, setCanvasMap, spawningCwdsRef, subscribedRef, pendingTerminalCwdRef, maxSeqMapRef, selectedSessionIdRef, loadingHistoryTimersRef, replayPersister, flushLiveEvents, scheduleLiveFlush]); + }, [send, clearSpawningCwd, navigate, setSessions, setSessionStates, setSessionCommands, setFileResults, setChangedOnDisk, setOpenspecMap, setModelsMap, setRolesMap, setSpawnResult, setSessionOrderMap, setPinnedDirectories, setPinnedDirsLoaded, setFavoriteModels, setWorkspaces, setTerminals, setDiscoveredServers, setLoadingHistory, setCanvasMap, setHistoryWindowMap, setLoadingOlderMap, spawningCwdsRef, subscribedRef, maxSeqMapRef, selectedSessionIdRef, historyWindowRef, loadingHistoryTimersRef, replayPersister, flushLiveEvents, scheduleLiveFlush, showToast]); } diff --git a/packages/client/src/lib/AGENTS.md b/packages/client/src/lib/AGENTS.md index 5d1f58f45..3df5c07a1 100644 --- a/packages/client/src/lib/AGENTS.md +++ b/packages/client/src/lib/AGENTS.md @@ -85,13 +85,14 @@ Files in this directory. One row per source file. | `providers-api.ts` | Fetch helper for custom-LLM-provider management. Exports `TestProviderInput`, `TestProviderResult` (discriminated union), `testProvider(input)` — POST `/api/providers/test` verifying baseUrl+apiKey+api against upstream `/models` without saving; `apiKey` accepts literal, `$ENV_VAR` ref, or `"***"` (resolved server-side from saved provider). | | `rail-width.ts` | Per-session browse-rail width, localStorage `pi-dashboard:rail:`, clamp [160,480], default 224. `useRailWidth`. Independent of outer split ratio. See change: split-editor-workspace. | | `rehydrate-session.ts` | rehydrateSession(sessionId,cache). Cache hit → re-reduce raw payload via reduceEvent into provisional… → see `rehydrate-session.ts.AGENTS.md` | -| `replay-cache.ts` | Durable per-session replay cache. IndexedDB.… → see `replay-cache.ts.AGENTS.md` | +| `replay-cache.ts` | Durable per-session replay cache. IndexedDB. put trims newest-by-byte-budget (schema v2, ~4 MiB). → see `replay-cache.ts.AGENTS.md` | | `replay-persist.ts` | Debounced replay-cache writer. createReplayPersister(cache,debounceMs). → see `replay-persist.ts.AGENTS.md` | | `route-builders.ts` | URL builders for shell overlay routes: `buildOpenSpecPreviewUrl`, `buildOpenSpecArchiveUrl`,… → see `route-builders.ts.AGENTS.md` | | `selectedSessionId.ts` | Pure derivation of selected session id from wouter route matches. → see `selectedSessionId.ts.AGENTS.md` | | `selectViewedSessionId.ts` | Pure selector for currently-viewed session id from `/session/:id` route. → see `selectViewedSessionId.ts.AGENTS.md` | | `server-switch.ts` | `performServerSwitch(target, deps)` — extracted two-phase transaction (stage → commit) from `App.tsx`'s… → see `server-switch.ts.AGENTS.md` | | `session-card-time.ts` | Pure picker of session-card relative-time badge anchor timestamp. Exports `selectBadgeTimestamp(session)`. → see `session-card-time.ts.AGENTS.md` | +| `session-subscribe.ts` | Cold/warm/load-older subscribe message builders (`mode:"tail"`, `fromSeq`). See change: session-tail-rehydrate. | | `session-display-name.ts` | Pure derivation of session display name. Exports `getSessionDisplayName(session)` → name → firstMessage (truncated 50 chars) → cwd last segment → ID prefix (8 chars). | | `session-filter-storage.ts` | localStorage persistence for session-list filter state. Exports `removeLegacyHiddenSessions`,… → see `session-filter-storage.ts.AGENTS.md` | | `session-grouping.ts` | Pure session grouping/sorting/filtering utilities. Exports `DirectoryGroup`, `WorkspaceGroup`,… → see `session-grouping.ts.AGENTS.md` | diff --git a/packages/client/src/lib/__tests__/replay-cache.test.ts b/packages/client/src/lib/__tests__/replay-cache.test.ts index fa18e7ec4..1d4eaf2d4 100644 --- a/packages/client/src/lib/__tests__/replay-cache.test.ts +++ b/packages/client/src/lib/__tests__/replay-cache.test.ts @@ -55,12 +55,46 @@ describe("replay-cache", () => { expect(await writer.get("sess-a")).toBeNull(); }); - it("skips persisting a session whose payload exceeds the per-session byte cap", async () => { + it("trims over-budget payload to newest events and persists the tail", async () => { + // Tiny budget so a handful of events overflow; put must keep newest, not drop. const cache = createReplayCache({ factory, maxBytesPerSession: 200 }); const big = Array.from({ length: 50 }, (_, i) => evt(i + 1)); await cache.put("huge", { maxSeq: 50, payload: big }); - // Over-cap payload is not persisted → next load full-replays. - expect(await cache.get("huge")).toBeNull(); + const hit = await cache.get("huge"); + expect(hit).not.toBeNull(); + expect(hit!.payload.length).toBeGreaterThan(0); + expect(hit!.payload.length).toBeLessThan(50); + // Newest seqs only, ascending + expect(hit!.payload[hit!.payload.length - 1]!.seq).toBe(50); + expect(hit!.maxSeq).toBe(50); + for (let i = 1; i < hit!.payload.length; i++) { + expect(hit!.payload[i]!.seq).toBeGreaterThan(hit!.payload[i - 1]!.seq); + } + // Guarantee: persisted array serialization never exceeds the byte budget. + expect(JSON.stringify(hit!.payload).length).toBeLessThanOrEqual(200); + }); + + it("recheck trims array serialization, not just per-entry byte sum", async () => { + // selectNewestEventsByBudget sums per-entry JSON.stringify length; the + // persisted array serialization adds commas + brackets, so a window whose + // per-entry sum fits the budget can still overflow once array overhead is + // counted. put() must recheck the full array serialization and drop oldest + // remaining events until it fits (or the window empties). + const entryLen = JSON.stringify(evt(1)).length; + // Budget fits exactly two entries by per-entry sum (2 * entryLen) but the + // array serialization of two entries (2 * entryLen + comma + brackets) overflows. + const budget = 2 * entryLen; + const cache = createReplayCache({ factory, maxBytesPerSession: budget }); + await cache.put("s", { maxSeq: 3, payload: [evt(1), evt(2), evt(3)] }); + const hit = await cache.get("s"); + expect(hit).not.toBeNull(); + // Two entries fit by per-entry sum, but their array serialization overflows, + // so the recheck drops the oldest leaving exactly the newest event. + expect(hit!.payload.length).toBe(1); + expect(hit!.payload[0]!.seq).toBe(3); + expect(hit!.maxSeq).toBe(3); + // Guarantee: persisted array serialization never exceeds the byte budget. + expect(JSON.stringify(hit!.payload).length).toBeLessThanOrEqual(budget); }); it("evicts the least-recently-accessed entry past the cap", async () => { diff --git a/packages/client/src/lib/__tests__/session-subscribe.test.ts b/packages/client/src/lib/__tests__/session-subscribe.test.ts new file mode 100644 index 000000000..2ce86a813 --- /dev/null +++ b/packages/client/src/lib/__tests__/session-subscribe.test.ts @@ -0,0 +1,42 @@ +import { describe, it, expect } from "vitest"; +import { + buildColdTailSubscribe, + buildLoadOlderSubscribe, + buildSessionSubscribe, +} from "../session-subscribe.js"; + +describe("buildSessionSubscribe", () => { + it("cold lastSeq 0 sends mode:tail", () => { + expect(buildSessionSubscribe("s1", 0)).toEqual({ + type: "subscribe", + sessionId: "s1", + lastSeq: 0, + mode: "tail", + }); + }); + + it("delta lastSeq>0 omits mode", () => { + expect(buildSessionSubscribe("s1", 42)).toEqual({ + type: "subscribe", + sessionId: "s1", + lastSeq: 42, + }); + }); + + it("buildColdTailSubscribe always requests tail", () => { + expect(buildColdTailSubscribe("abc")).toEqual({ + type: "subscribe", + sessionId: "abc", + lastSeq: 0, + mode: "tail", + }); + }); + + it("buildLoadOlderSubscribe uses fromSeq exclusive upper bound", () => { + expect(buildLoadOlderSubscribe("s1", 100)).toEqual({ + type: "subscribe", + sessionId: "s1", + fromSeq: 100, + }); + }); +}); diff --git a/packages/client/src/lib/replay-cache.ts b/packages/client/src/lib/replay-cache.ts index 960a684ed..05b5755fc 100644 --- a/packages/client/src/lib/replay-cache.ts +++ b/packages/client/src/lib/replay-cache.ts @@ -5,25 +5,30 @@ * so a page reload can resubscribe with `lastSeq = maxSeq` (delta replay) * instead of `lastSeq: 0` (full replay). The cache is an OPTIMIZATION ONLY: * any miss, schemaVersion mismatch, eviction, or IndexedDB error degrades to a - * full replay with no error surfaced to the user. + * full (or server tail) replay with no error surfaced to the user. * * Decision (design.md 1.1): persist RAW events (`{ seq, event }[]`), not reduced * `ChatMessage[]`. The reducer is pure, so re-reducing on load is cheap and the * cache binds only to the stable event wire schema — keeping `schemaVersion` * bumps rare. * - * See change: reduce-session-replay-traffic. + * Over-budget put trims to newest-by-byte-budget (session-tail-rehydrate) + * rather than deleting the entry (which forced cold full replay). + * + * See change: reduce-session-replay-traffic, session-tail-rehydrate. */ import type { DashboardEvent } from "@blackbelt-technology/pi-dashboard-shared/types.js"; +import { selectNewestEventsByBudget } from "@blackbelt-technology/pi-dashboard-shared/event-window.js"; -/** Bump on any persisted-shape change → all entries invalidate (full replay). */ -export const REPLAY_CACHE_SCHEMA_VERSION = 1; +/** Bump on any persisted-shape change → all entries invalidate (full replay). + * v2: over-budget put trims to newest-by-byte-budget instead of drop-all. */ +export const REPLAY_CACHE_SCHEMA_VERSION = 2; const DB_NAME = "pi-dashboard-replay-cache"; const STORE = "sessions"; const DEFAULT_MAX_ENTRIES = 50; -/** Per-session payload byte cap (~5 MB). Over-cap → skip persist, full replay. */ -const DEFAULT_MAX_BYTES_PER_SESSION = 5 * 1024 * 1024; +/** Per-session payload byte budget (~4 MB newest events). Over → trim-on-put. */ +const DEFAULT_MAX_BYTES_PER_SESSION = 4 * 1024 * 1024; export interface CachedEvent { seq: number; @@ -161,8 +166,28 @@ export function createReplayCache(opts: ReplayCacheOptions = {}): ReplayCache { async function put(sessionId: string, value: ReplayCachePut): Promise { await safe(async () => { - // Over-cap payload: skip persist and drop any stale entry → full replay. - if (JSON.stringify(value.payload).length > maxBytesPerSession) { + // Over-budget: keep newest events that fit instead of dropping the entry. + // Trimming fits the persisted ARRAY serialization (JSON.stringify of the + // whole payload, including commas + brackets), not just the per-entry + // byte sum from selectNewestEventsByBudget — a window whose per-entry sum + // fits can still overflow once array overhead is counted. See change: + // session-tail-rehydrate. + let payload = value.payload; + let maxSeq = value.maxSeq; + if (JSON.stringify(payload).length > maxBytesPerSession) { + const windowed = selectNewestEventsByBudget(payload, maxBytesPerSession); + payload = windowed.events; + maxSeq = windowed.windowMaxSeq || maxSeqOf(payload) || maxSeq; + } + // Recheck the full array serialization: selectNewestEventsByBudget sums + // per-entry JSON.stringify length, but the persisted array adds commas + + // brackets, so the trimmed window can still exceed maxBytesPerSession. + // Drop oldest remaining events until the array fits or the window empties; + // the newest event is retained throughout so maxSeq stays valid. + while (payload.length > 0 && JSON.stringify(payload).length > maxBytesPerSession) { + payload = payload.slice(1); + } + if (payload.length === 0) { await del(sessionId); return; } @@ -170,8 +195,8 @@ export function createReplayCache(opts: ReplayCacheOptions = {}): ReplayCache { const entry: ReplayCacheEntry = { sessionId, schemaVersion, - maxSeq: value.maxSeq, - payload: value.payload, + maxSeq, + payload, lastAccess: nextStamp(), }; const tx = db.transaction(STORE, "readwrite"); @@ -181,6 +206,12 @@ export function createReplayCache(opts: ReplayCacheOptions = {}): ReplayCache { }, undefined); } + function maxSeqOf(buf: CachedEvent[]): number { + let m = 0; + for (const e of buf) if (e.seq > m) m = e.seq; + return m; + } + async function del(sessionId: string): Promise { await safe(async () => { const db = await openDb(); diff --git a/packages/client/src/lib/replay-persist.ts b/packages/client/src/lib/replay-persist.ts index 4476cd9a6..23d4ea8f7 100644 --- a/packages/client/src/lib/replay-persist.ts +++ b/packages/client/src/lib/replay-persist.ts @@ -24,6 +24,10 @@ export interface ReplayPersister { drop(sessionId: string): Promise; /** Force an immediate flush (tests / unmount). */ flush(sessionId: string): Promise; + /** Merge newly-arrived older events into the buffer (dedup by seq), returning the merged snapshot. See change: session-tail-rehydrate. */ + merge(sessionId: string, events: CachedEvent[]): CachedEvent[]; + /** Return a defensive copy of the current raw-event buffer. See change: session-tail-rehydrate. */ + snapshot(sessionId: string): CachedEvent[]; } export function createReplayPersister( @@ -81,6 +85,21 @@ export function createReplayPersister( schedule(sessionId); } + function merge(sessionId: string, events: CachedEvent[]): CachedEvent[] { + if (events.length === 0) return snapshot(sessionId); + const bySeq = new Map(); + for (const e of buffers.get(sessionId) ?? []) bySeq.set(e.seq, e); + for (const e of events) bySeq.set(e.seq, e); + const merged = [...bySeq.values()].sort((a, b) => a.seq - b.seq); + buffers.set(sessionId, merged); + schedule(sessionId); + return merged; + } + + function snapshot(sessionId: string): CachedEvent[] { + return [...(buffers.get(sessionId) ?? [])]; + } + async function drop(sessionId: string): Promise { const t = timers.get(sessionId); if (t) { @@ -91,5 +110,5 @@ export function createReplayPersister( await cache.delete(sessionId); } - return { record, seed, drop, flush }; + return { record, seed, merge, snapshot, drop, flush }; } diff --git a/packages/client/src/lib/session-subscribe.ts b/packages/client/src/lib/session-subscribe.ts new file mode 100644 index 000000000..caa225a2b --- /dev/null +++ b/packages/client/src/lib/session-subscribe.ts @@ -0,0 +1,43 @@ +/** + * Build browser→server `subscribe` messages for session history. + * + * Cold open (`lastSeq === 0`) asks for a tail window so large sessions do not + * force a full replay. Warm/delta (`lastSeq > 0`) stays a plain delta so the + * server only sends events after the client's cursor. + * + * See change: session-tail-rehydrate. + */ +import type { BrowserToServerMessage } from "@blackbelt-technology/pi-dashboard-shared/browser-protocol.js"; + +export type SessionSubscribeMessage = Extract< + BrowserToServerMessage, + { type: "subscribe" } +>; + +/** Cold open / full refresh: newest events under the server's default budget. */ +export function buildColdTailSubscribe(sessionId: string): SessionSubscribeMessage { + return { type: "subscribe", sessionId, lastSeq: 0, mode: "tail" }; +} + +/** + * Subscribe for a known cursor. + * - `lastSeq > 0` → delta only (no mode) + * - `lastSeq === 0` → cold tail + */ +export function buildSessionSubscribe( + sessionId: string, + lastSeq: number, +): SessionSubscribeMessage { + if (lastSeq > 0) { + return { type: "subscribe", sessionId, lastSeq }; + } + return buildColdTailSubscribe(sessionId); +} + +/** Load older history: exclusive upper bound `seq < fromSeq`. */ +export function buildLoadOlderSubscribe( + sessionId: string, + fromSeq: number, +): SessionSubscribeMessage { + return { type: "subscribe", sessionId, fromSeq }; +} diff --git a/packages/client/vite.config.ts b/packages/client/vite.config.ts index 60beb8ed5..6ef9989c2 100644 --- a/packages/client/vite.config.ts +++ b/packages/client/vite.config.ts @@ -128,9 +128,13 @@ export default defineConfig({ }, server: { port: 3000, + // zrok / tunnel hostnames must be allowlisted or Vite returns + // "Blocked request. This host is not allowed." + allowedHosts: true, hmr: { - // HMR WebSocket must connect directly to Vite's port, not the dashboard's. - clientPort: 3000, + // Prefer the actual listen port (CLI --port overrides server.port). + // For public zrok shares HMR often fails anyway; page load still works. + clientPort: Number(process.env.VITE_HMR_CLIENT_PORT) || undefined, }, proxy: { "/api": `http://localhost:${DASHBOARD_PORT}`, diff --git a/packages/server/src/__tests__/subscription-handler.test.ts b/packages/server/src/__tests__/subscription-handler.test.ts index f5c0f576d..519d8b3b4 100644 --- a/packages/server/src/__tests__/subscription-handler.test.ts +++ b/packages/server/src/__tests__/subscription-handler.test.ts @@ -2,7 +2,7 @@ import type { ServerToBrowserMessage } from "@blackbelt-technology/pi-dashboard- import type { DashboardEvent } from "@blackbelt-technology/pi-dashboard-shared/types.js"; import { describe, expect, it, vi } from "vitest"; import type { BrowserHandlerContext } from "../browser-handlers/handler-context.js"; -import { handleSubscribe, replaySessionAssets } from "../browser-handlers/subscription-handler.js"; +import { handleSubscribe, replaySessionAssets, windowEventsForSubscribe } from "../browser-handlers/subscription-handler.js"; import { createMemoryEventStore } from "../memory-event-store.js"; import { createMemorySessionManager } from "../memory-session-manager.js"; @@ -94,6 +94,50 @@ describe("handleSubscribe — stale lastSeq detection", () => { expect(allEvents[0].seq).toBe(1); }); + it("stale lastSeq with omitted mode preserves full replay", async () => { + const ctx = createMockContext(); + for (let i = 0; i < 100; i++) { + ctx.eventStore.insertEvent("s1", { + ...makeEvent(`e${i}`), + data: { pad: "x".repeat(20_000) }, + }); + } + + handleSubscribe({ type: "subscribe", sessionId: "s1", lastSeq: 1000 }, new Set(), ctx); + await new Promise((r) => setTimeout(r, 50)); + + const calls = (ctx.sendTo as any).mock.calls as Array<[any, ServerToBrowserMessage]>; + const replays = calls.filter(([, msg]) => msg.type === "event_replay"); + const allEvents = replays.flatMap(([, msg]: any) => msg.events); + expect(allEvents).toHaveLength(100); + expect((replays[replays.length - 1]![1] as any).hasMoreOlder).toBeUndefined(); + }); + + it("stale lastSeq with mode:tail applies the requested cold window", async () => { + const ctx = createMockContext(); + for (let i = 0; i < 100; i++) { + ctx.eventStore.insertEvent("s1", { + ...makeEvent(`e${i}`), + data: { pad: "x".repeat(20_000) }, + }); + } + + handleSubscribe( + { type: "subscribe", sessionId: "s1", lastSeq: 1000, mode: "tail", windowBytes: 256 * 1024 }, + new Set(), + ctx, + ); + await new Promise((r) => setTimeout(r, 50)); + + const calls = (ctx.sendTo as any).mock.calls as Array<[any, ServerToBrowserMessage]>; + const replays = calls.filter(([, msg]) => msg.type === "event_replay"); + const allEvents = replays.flatMap(([, msg]: any) => msg.events); + expect(allEvents.length).toBeGreaterThan(0); + expect(allEvents.length).toBeLessThan(100); + expect(allEvents[allEvents.length - 1]!.seq).toBe(100); + expect((replays[replays.length - 1]![1] as any).hasMoreOlder).toBe(true); + }); + it("marks replaying during delta replay and clears after", async () => { const markReplaying = vi.fn(); const clearReplaying = vi.fn(); @@ -199,6 +243,91 @@ describe("handleSubscribe — stale lastSeq detection", () => { expect(loadSessionEvents).toHaveBeenCalledWith("s-ctx", "/sessions/s-ctx.jsonl", 1_000_000); }); + it("resets a stale cursor after disk load and preserves legacy full replay", async () => { + const events = Array.from({ length: 100 }, (_, i) => ({ + ...makeEvent(`e${i}`), + data: { pad: "x".repeat(20_000) }, + })); + const loadSessionEvents = vi.fn(async () => ({ success: true, events })); + const ctx = createMockContext({ directoryService: { loadSessionEvents } as any }); + ctx.getSubscribers = () => [ctx.ws]; + ctx.sessionManager.restore({ + id: "s-disk-stale", + cwd: "/test", + source: "tui", + status: "ended", + startedAt: 1000, + endedAt: 2000, + tokensIn: 0, + tokensOut: 0, + cost: 0, + contextWindow: 200_000, + sessionFile: "/sessions/s-disk-stale.jsonl", + sessionDir: "/sessions", + hidden: false, + } as any); + + handleSubscribe({ type: "subscribe", sessionId: "s-disk-stale", lastSeq: 1000 }, new Set(), ctx); + await new Promise((r) => setTimeout(r, 50)); + + const calls = (ctx.sendTo as any).mock.calls as Array<[any, ServerToBrowserMessage]>; + expect(calls.filter(([, msg]) => msg.type === "session_state_reset")).toHaveLength(1); + const replays = calls.filter(([, msg]) => msg.type === "event_replay"); + const allEvents = replays.flatMap(([, msg]: any) => msg.events); + expect(allEvents).toHaveLength(100); + expect((replays[replays.length - 1]![1] as any).hasMoreOlder).toBeUndefined(); + }); + + it("resets a stale cursor after disk load with mode:tail and windows newest", async () => { + // Disk-stale + explicit mode:tail: hydration loads history older than the + // browser cursor, so every subscriber is reset and the cold rebuild applies + // the requested tail window (not a legacy full replay). Fat events force + // the 256 KiB clamp to trim so hasMoreOlder/windowMaxSeq are exercised. + // See change: session-tail-rehydrate (disk-stale tail recovery). + const events = Array.from({ length: 100 }, (_, i) => ({ + ...makeEvent(`e${i}`), + data: { pad: "x".repeat(20_000) }, + })); + const loadSessionEvents = vi.fn(async () => ({ success: true, events })); + const ctx = createMockContext({ directoryService: { loadSessionEvents } as any }); + ctx.getSubscribers = () => [ctx.ws]; + ctx.sessionManager.restore({ + id: "s-disk-tail", + cwd: "/test", + source: "tui", + status: "ended", + startedAt: 1000, + endedAt: 2000, + tokensIn: 0, + tokensOut: 0, + cost: 0, + contextWindow: 200_000, + sessionFile: "/sessions/s-disk-tail.jsonl", + sessionDir: "/sessions", + hidden: false, + } as any); + + handleSubscribe( + { type: "subscribe", sessionId: "s-disk-tail", lastSeq: 1000, mode: "tail", windowBytes: 256 * 1024 }, + new Set(), + ctx, + ); + await new Promise((r) => setTimeout(r, 50)); + + const calls = (ctx.sendTo as any).mock.calls as Array<[any, ServerToBrowserMessage]>; + expect(calls.filter(([, msg]) => msg.type === "session_state_reset")).toHaveLength(1); + const replays = calls.filter(([, msg]) => msg.type === "event_replay") as Array< + [any, Extract] + >; + const allEvents = replays.flatMap(([, msg]) => msg.events); + expect(allEvents.length).toBeGreaterThan(0); + expect(allEvents.length).toBeLessThan(100); + expect(allEvents[allEvents.length - 1]!.seq).toBe(100); + const last = replays[replays.length - 1]![1]; + expect(last.hasMoreOlder).toBe(true); + expect(last.windowMaxSeq).toBe(100); + }); + it("does full replay when lastSeq is 0", async () => { const ctx = createMockContext(); for (let i = 0; i < 3; i++) ctx.eventStore.insertEvent("s1", makeEvent()); @@ -394,3 +523,162 @@ describe("handleSubscribe — asset replay precedes events", () => { expect(firstAssetIdx).toBeLessThan(firstEventIdx); }); }); + +describe("handleSubscribe — tail window + load-older (session-tail-rehydrate)", () => { + function fatEvent(label: string): DashboardEvent { + // ~80 KB each so MIN clamp (256 KiB) still only fits a few newest events. + return { eventType: "test", timestamp: Date.now(), data: { label, pad: "p".repeat(80_000) } }; + } + + it("mode:tail cold open delivers only newest events under budget with hasMoreOlder", async () => { + const ctx = createMockContext(); + // Store truncates strings to 4 KB; ~4100 B/event → need >64 events to exceed + // the 256 KiB MIN clamp. + for (let i = 0; i < 100; i++) ctx.eventStore.insertEvent("s1", fatEvent(`e${i}`)); + + handleSubscribe( + { type: "subscribe", sessionId: "s1", lastSeq: 0, mode: "tail", windowBytes: 8_000 }, + new Set(), + ctx, + ); + await new Promise((r) => setTimeout(r, 50)); + + const calls = (ctx.sendTo as any).mock.calls as Array<[any, ServerToBrowserMessage]>; + const replays = calls.filter(([, m]) => m.type === "event_replay") as Array< + [any, Extract] + >; + expect(replays.length).toBeGreaterThanOrEqual(1); + const allEvents = replays.flatMap(([, m]) => m.events); + expect(allEvents.length).toBeGreaterThan(0); + expect(allEvents.length).toBeLessThan(100); + expect(allEvents[allEvents.length - 1]!.seq).toBe(100); + const last = replays[replays.length - 1]![1]; + expect(last.hasMoreOlder).toBe(true); + expect(last.windowMaxSeq).toBe(100); + expect(last.windowMinSeq).toBe(allEvents[0]!.seq); + }); + it("legacy omit mode still delivers full buffer", async () => { + const ctx = createMockContext(); + for (let i = 0; i < 8; i++) ctx.eventStore.insertEvent("s1", fatEvent(`e${i}`)); + + handleSubscribe({ type: "subscribe", sessionId: "s1", lastSeq: 0 }, new Set(), ctx); + await new Promise((r) => setTimeout(r, 50)); + + const calls = (ctx.sendTo as any).mock.calls as Array<[any, ServerToBrowserMessage]>; + const replays = calls.filter(([, m]) => m.type === "event_replay") as Array< + [any, Extract] + >; + const allEvents = replays.flatMap(([, m]) => m.events); + expect(allEvents).toHaveLength(8); + expect(replays[replays.length - 1]![1].hasMoreOlder).toBeUndefined(); + }); + + it("delta lastSeq>0 is unchanged under mode:tail", async () => { + const ctx = createMockContext(); + for (let i = 0; i < 10; i++) ctx.eventStore.insertEvent("s1", fatEvent(`e${i}`)); + + handleSubscribe( + { type: "subscribe", sessionId: "s1", lastSeq: 7, mode: "tail", windowBytes: 8_000 }, + new Set(), + ctx, + ); + await new Promise((r) => setTimeout(r, 50)); + + const calls = (ctx.sendTo as any).mock.calls as Array<[any, ServerToBrowserMessage]>; + const replays = calls.filter(([, m]) => m.type === "event_replay") as Array< + [any, Extract] + >; + const allEvents = replays.flatMap(([, m]) => m.events); + expect(allEvents.map((e) => e.seq)).toEqual([8, 9, 10]); + }); + + it("fromSeq returns older page strictly below the cursor", async () => { + const ctx = createMockContext(); + for (let i = 0; i < 15; i++) ctx.eventStore.insertEvent("s1", fatEvent(`e${i}`)); + + handleSubscribe( + { type: "subscribe", sessionId: "s1", fromSeq: 12, windowBytes: 1_000_000 }, + new Set(), + ctx, + ); + await new Promise((r) => setTimeout(r, 50)); + + const calls = (ctx.sendTo as any).mock.calls as Array<[any, ServerToBrowserMessage]>; + const replays = calls.filter(([, m]) => m.type === "event_replay") as Array< + [any, Extract] + >; + const allEvents = replays.flatMap(([, m]) => m.events); + expect(allEvents.every((e) => e.seq < 12)).toBe(true); + expect(allEvents[allEvents.length - 1]!.seq).toBe(11); + expect(replays[replays.length - 1]![1].windowMaxSeq).toBe(11); + }); + + it("fromSeq does not mark replaying or clearReplaying catch-up", async () => { + // Load-older must not suppress live events or re-send the live tail via + // clearReplaying(lastSent=windowMax). That catch-up wiped client state. + // See change: session-tail-rehydrate (live-follow after hydrate). + const markReplaying = vi.fn(); + const clearReplaying = vi.fn(); + const ctx = createMockContext({ markReplaying, clearReplaying }); + for (let i = 0; i < 15; i++) ctx.eventStore.insertEvent("s1", fatEvent(`e${i}`)); + + handleSubscribe( + { type: "subscribe", sessionId: "s1", fromSeq: 12, windowBytes: 1_000_000 }, + new Set(), + ctx, + ); + await new Promise((r) => setTimeout(r, 50)); + + expect(markReplaying).not.toHaveBeenCalled(); + expect(clearReplaying).not.toHaveBeenCalled(); + }); +}); + +describe("windowEventsForSubscribe — lastSeq delta vs cold tail", () => { + function stored(n: number, pad = 200) { + return Array.from({ length: n }, (_, i) => ({ + seq: i + 1, + event: { eventType: `e${i}`, timestamp: i, data: { pad: "x".repeat(pad) } }, + })); + } + + it("lastSeq>0 returns only events after the cursor even when buffer starts at seq 1", () => { + // Disk-load path hands a full buffer into windowEvents; must not re-send + // the whole history (or strip hasMoreOlder via a fake cold-tail). + const { events, meta } = windowEventsForSubscribe(stored(10) as any, { + type: "subscribe", + sessionId: "s1", + lastSeq: 7, + }); + expect(events.map((e) => e.seq)).toEqual([8, 9, 10]); + expect(meta.hasMoreOlder).toBeUndefined(); + }); + + it("mode:tail lastSeq 0 windows newest with hasMoreOlder", () => { + // clampTailWindowBytes floors at 256KiB — use fat events so the budget trims. + const { events, meta } = windowEventsForSubscribe(stored(40, 20_000) as any, { + type: "subscribe", + sessionId: "s1", + lastSeq: 0, + mode: "tail", + windowBytes: 256 * 1024, + }); + expect(events.length).toBeGreaterThan(0); + expect(events.length).toBeLessThan(40); + expect(meta.hasMoreOlder).toBe(true); + expect(meta.windowMinSeq).toBe(events[0]!.seq); + expect(meta.windowMaxSeq).toBe(40); + }); + + it("fromSeq page is strictly older than the cursor", () => { + const { events, meta } = windowEventsForSubscribe(stored(20) as any, { + type: "subscribe", + sessionId: "s1", + fromSeq: 12, + windowBytes: 1_000_000, + }); + expect(events.every((e) => e.seq < 12)).toBe(true); + expect(events[events.length - 1]!.seq).toBe(11); + expect(meta.windowMaxSeq).toBe(11); + }); +}); diff --git a/packages/server/src/browser-handlers/subscription-handler.ts b/packages/server/src/browser-handlers/subscription-handler.ts index dbc2a5e28..140fcc397 100644 --- a/packages/server/src/browser-handlers/subscription-handler.ts +++ b/packages/server/src/browser-handlers/subscription-handler.ts @@ -3,6 +3,11 @@ */ import type { BrowserToServerMessage, ServerToBrowserMessage } from "@blackbelt-technology/pi-dashboard-shared/browser-protocol.js"; +import { + clampTailWindowBytes, + selectNewestEventsByBudget, + selectOlderEventsByBudget, +} from "@blackbelt-technology/pi-dashboard-shared/event-window.js"; import type { WebSocket } from "ws"; import { extractStatsFromEvents } from "../event-status-extraction.js"; import type { StoredEvent } from "../memory-event-store.js"; @@ -24,10 +29,69 @@ const BACKPRESSURE_THRESHOLD = 1_024 * 1_024; */ const HYDRATE_HEARTBEAT_MS = 10000; +/** Optional window metadata stamped onto event_replay batches. */ +interface ReplayWindowMeta { + hasMoreOlder?: boolean; + windowMinSeq?: number; + windowMaxSeq?: number; +} + /** - * Send stored events to a WebSocket in batches with backpressure handling. - * Yields between batches to let the event loop flush data and avoid OOM. + * Apply tail / load-older windowing to an ascending event list based on the + * subscribe message. Delta paths (`lastSeq > 0` without `fromSeq`) pass through. + * See change: session-tail-rehydrate. */ +export function windowEventsForSubscribe( + events: StoredEvent[], + msg: Extract, +): { events: StoredEvent[]; meta: ReplayWindowMeta } { + let list = events; + if (MAX_REPLAY_EVENTS > 0 && list.length > MAX_REPLAY_EVENTS) { + list = list.slice(list.length - MAX_REPLAY_EVENTS); + } + + // Load-older: exclusive upper bound (works on full buffer). + if (msg.fromSeq != null && Number.isFinite(msg.fromSeq)) { + const budget = clampTailWindowBytes(msg.windowBytes); + const r = selectOlderEventsByBudget(list, msg.fromSeq, budget); + return { + events: r.events as StoredEvent[], + meta: { + hasMoreOlder: r.hasMoreOlder, + windowMinSeq: r.windowMinSeq || undefined, + windowMaxSeq: r.windowMaxSeq || undefined, + }, + }; + } + + const lastSeq = msg.lastSeq ?? 0; + + // Delta subscribe after IDB rehydrate (or any lastSeq>0): only events after the + // client cursor. Apply BEFORE cold-tail detection — a full buffer from disk + // still starts at seq 1 and must not be re-sent wholesale (that wiped load-older + // meta and defeated the tail). See change: session-tail-rehydrate. + if (lastSeq > 0) { + list = list.filter((e) => e.seq > lastSeq); + return { events: list, meta: {} }; + } + + // Cold open (lastSeq 0): optional tail window. + if (msg.mode === "tail") { + const budget = clampTailWindowBytes(msg.windowBytes); + const r = selectNewestEventsByBudget(list, budget); + return { + events: r.events as StoredEvent[], + meta: { + hasMoreOlder: r.hasMoreOlder, + windowMinSeq: r.windowMinSeq || undefined, + windowMaxSeq: r.windowMaxSeq || undefined, + }, + }; + } + + return { events: list, meta: {} }; +} + /** * Send stored events to a WebSocket in batches with backpressure handling. * Returns the highest seq sent, or 0 if no events were sent. @@ -37,21 +101,30 @@ async function sendEventBatches( sessionId: string, stored: StoredEvent[], sendTo: (ws: WebSocket, msg: ServerToBrowserMessage) => void, + windowMeta: ReplayWindowMeta = {}, ): Promise { + // Empty delivery still needs a terminal event_replay so the client clears + // loadingHistory (and learns hasMoreOlder for empty older pages). + if (stored.length === 0) { + if (ws.readyState === ws.OPEN) { + sendTo(ws, { + type: "event_replay", + sessionId, + events: [], + isLast: true, + ...windowMeta, + }); + } + return 0; + } + for (let i = 0; i < stored.length; i += REPLAY_BATCH_SIZE) { if (ws.readyState !== ws.OPEN) return 0; - const batch = stored.slice(i, i + REPLAY_BATCH_SIZE); - sendTo(ws, { - type: "event_replay", - sessionId, - // Strategy B (reduce-session-replay-traffic): pre-truncate heavy tool - // results to the display form to trim replay bytes. Additive — the store - // keeps the full body for develop's "Show full output" route; small - // results and non-tool events pass through untouched. - events: batch.map((e) => ({ seq: e.seq, event: truncateToolResultForReplay(e.event) })), - isLast: i + REPLAY_BATCH_SIZE >= stored.length, - }); - // Yield to event loop between batches to allow GC and buffer flushing + // Wait BEFORE enqueueing the next batch so sendTo does not drop under + // MAX_WS_BUFFER back-pressure (mobile/tunnel drains slowly). Dropped + // event_replay batches leave the client stuck mid-tail with live events + // suppressed until clearReplaying — or missing entirely if catch-up also + // drops. See change: session-tail-rehydrate (live-follow after hydrate). if (ws.bufferedAmount > BACKPRESSURE_THRESHOLD) { await new Promise((resolve) => { const check = () => { @@ -63,11 +136,26 @@ async function sendEventBatches( }; setTimeout(check, 10); }); - } else { - await new Promise((r) => setImmediate(r)); + if (ws.readyState !== ws.OPEN) return 0; } + const batch = stored.slice(i, i + REPLAY_BATCH_SIZE); + const isLast = i + REPLAY_BATCH_SIZE >= stored.length; + sendTo(ws, { + type: "event_replay", + sessionId, + // Strategy B (reduce-session-replay-traffic): pre-truncate heavy tool + // results to the display form to trim replay bytes. Additive — the store + // keeps the full body for develop's "Show full output" route; small + // results and non-tool events pass through untouched. + events: batch.map((e) => ({ seq: e.seq, event: truncateToolResultForReplay(e.event) })), + isLast, + // Stamp window meta on every batch (stable) so partial clients still see it. + ...windowMeta, + }); + // Yield to event loop between batches to allow GC and buffer flushing + await new Promise((r) => setImmediate(r)); } - return stored.length > 0 ? stored[stored.length - 1].seq : 0; + return stored[stored.length - 1]!.seq; } /** @@ -162,6 +250,21 @@ export function replaySessionAssets( } } +/** + * Build a cold-rebuild subscribe message without turning legacy/full clients + * into tail clients. See change: session-tail-rehydrate. + */ +function coldSubscribeMessage( + msg: Extract, +): Extract { + const { mode, ...rest } = msg; + return { + ...rest, + lastSeq: 0, + ...(mode === "tail" ? { mode: "tail" } : {}), + } as Extract; +} + export function handleSubscribe( msg: Extract, subs: Set, @@ -193,49 +296,64 @@ export function handleSubscribe( const lastSeq = msg.lastSeq ?? 0; const maxSeq = eventStore.getMaxSeq(msg.sessionId); + // Load-older: page from the full buffer regardless of lastSeq. + // Do NOT mark replaying / run clearReplaying catch-up: the client already + // holds the live tail and merges older pages via isOlderPage. Catch-up + // keyed on windowMaxSeq would re-send the entire live tail and the client + // shouldReset path wipes state down to that tail-only batch — freezing + // follow-on live events out of view. See change: session-tail-rehydrate. + if (msg.fromSeq != null && Number.isFinite(msg.fromSeq)) { + const raw = eventStore.getEvents(msg.sessionId, 1); + const { events, meta } = windowEventsForSubscribe(raw, msg); + replaySessionAssets(ws, msg.sessionId, ctx); + void sendEventBatches(ws, msg.sessionId, events, sendTo, meta); + return; + } + // Stale lastSeq: client has higher seq than server (e.g. server restarted) if (lastSeq > 0 && lastSeq > maxSeq) { sendTo(ws, { type: "session_state_reset", sessionId: msg.sessionId }); - // Full replay from seq 1 - let events = eventStore.getEvents(msg.sessionId, 1); - if (MAX_REPLAY_EVENTS > 0 && events.length > MAX_REPLAY_EVENTS) { - events = events.slice(events.length - MAX_REPLAY_EVENTS); - } + const raw = eventStore.getEvents(msg.sessionId, 1); + // Reset is a cold rebuild — do not apply delta filter with the stale cursor. + // Preserve legacy/full mode; only an explicit tail request gets a tail + // window. See change: session-tail-rehydrate. + const coldMsg = coldSubscribeMessage(msg); + const { events, meta } = windowEventsForSubscribe(raw, coldMsg); // Replay asset registry BEFORE events so pi-asset: tokens in // message_update / message_end resolve on first reduce. // See change: chat-markdown-local-images-and-math. replaySessionAssets(ws, msg.sessionId, ctx); - markReplaying(ws, msg.sessionId); - sendEventBatches(ws, msg.sessionId, events, sendTo).then((lastSent) => { - clearReplaying(ws, msg.sessionId, lastSent); - replayPendingUiRequests(ws, msg.sessionId); - replayUiState(ws, msg.sessionId, ctx); - }); - } else { - let events = eventStore.getEvents(msg.sessionId, lastSeq + 1); - if (MAX_REPLAY_EVENTS > 0 && events.length > MAX_REPLAY_EVENTS) { - events = events.slice(events.length - MAX_REPLAY_EVENTS); + if (events.length > 0) { + markReplaying(ws, msg.sessionId); + sendEventBatches(ws, msg.sessionId, events, sendTo, meta).then((lastSent) => { + clearReplaying(ws, msg.sessionId, lastSent); + replayPendingUiRequests(ws, msg.sessionId); + replayUiState(ws, msg.sessionId, ctx); + }); + } else { + sendEventBatches(ws, msg.sessionId, events, sendTo, meta).then(() => { + replayPendingUiRequests(ws, msg.sessionId); + replayUiState(ws, msg.sessionId, ctx); + }); } + } else { + const raw = eventStore.getEvents(msg.sessionId, lastSeq + 1); + const { events, meta } = windowEventsForSubscribe(raw, msg); // Replay asset registry on every subscribe (delta or full). Cheap when // empty; assets already known to the client are simply re-overwritten // with identical bytes. See change: chat-markdown-local-images-and-math. replaySessionAssets(ws, msg.sessionId, ctx); // Suppress live events during paginated replay to prevent out-of-order - // delivery. The client's `event_replay` reset rule (firstSeq <= maxSeq) - // misfires if a live `event` arrives between batches and bumps maxSeq - // past the next batch's firstSeq — wiping state to a fresh build of - // only the last batch. Suppression+catch-up via clearReplaying preserves - // ordering for both cold (lastSeq=0) and warm (lastSeq>0) subscribes. - // See change: fix-cold-subscribe-replay-interleave. + // delivery. See change: fix-cold-subscribe-replay-interleave. if (events.length > 0) { markReplaying(ws, msg.sessionId); - sendEventBatches(ws, msg.sessionId, events, sendTo).then((lastSent) => { + sendEventBatches(ws, msg.sessionId, events, sendTo, meta).then((lastSent) => { clearReplaying(ws, msg.sessionId, lastSent); replayPendingUiRequests(ws, msg.sessionId); replayUiState(ws, msg.sessionId, ctx); }); } else { - sendEventBatches(ws, msg.sessionId, events, sendTo).then(() => { + sendEventBatches(ws, msg.sessionId, events, sendTo, meta).then(() => { replayPendingUiRequests(ws, msg.sessionId); replayUiState(ws, msg.sessionId, ctx); }); @@ -278,17 +396,41 @@ export function handleSubscribe( const metaUpdates: Record = { dataUnavailable: false, ...statsUpdates }; sessionManager.update(msg.sessionId, metaUpdates); broadcast({ type: "session_updated", sessionId: msg.sessionId, updates: metaUpdates }); - let stored = eventStore.getEvents(msg.sessionId, 1); - if (MAX_REPLAY_EVENTS > 0 && stored.length > MAX_REPLAY_EVENTS) { - stored = stored.slice(stored.length - MAX_REPLAY_EVENTS); - } + const raw = eventStore.getEvents(msg.sessionId, 1); + const loadedMax = eventStore.getMaxSeq(msg.sessionId); + const lastSeq = msg.lastSeq ?? 0; const subscribers = getSubscribers(msg.sessionId); - for (const sub of subscribers) { - // Asset registry first — see change: chat-markdown-local-images-and-math. - replaySessionAssets(sub, msg.sessionId, ctx); - await sendEventBatches(sub, msg.sessionId, stored, sendTo); - replayPendingUiRequests(sub, msg.sessionId); - replayUiState(sub, msg.sessionId, ctx); + if (lastSeq > loadedMax) { + // Disk hydration can load a history older than the browser cursor. + // Reset every subscriber before rebuilding from the loaded history. + // See change: session-tail-rehydrate. + const coldMsg = coldSubscribeMessage(msg); + const { events: coldEvents, meta } = windowEventsForSubscribe(raw, coldMsg); + for (const sub of subscribers) { + sendTo(sub, { type: "session_state_reset", sessionId: msg.sessionId }); + // Asset registry first — see change: chat-markdown-local-images-and-math. + replaySessionAssets(sub, msg.sessionId, ctx); + if (coldEvents.length > 0) { + markReplaying(sub, msg.sessionId); + const lastSent = await sendEventBatches(sub, msg.sessionId, coldEvents, sendTo, meta); + clearReplaying(sub, msg.sessionId, lastSent); + } else { + await sendEventBatches(sub, msg.sessionId, coldEvents, sendTo, meta); + } + replayPendingUiRequests(sub, msg.sessionId); + replayUiState(sub, msg.sessionId, ctx); + } + } else { + // Load-older and normal subscribe paths retain their existing window + // selection; stale cursors use the cold message above. + const { events: stored, meta } = windowEventsForSubscribe(raw, msg); + for (const sub of subscribers) { + // Asset registry first — see change: chat-markdown-local-images-and-math. + replaySessionAssets(sub, msg.sessionId, ctx); + await sendEventBatches(sub, msg.sessionId, stored, sendTo, meta); + replayPendingUiRequests(sub, msg.sessionId); + replayUiState(sub, msg.sessionId, ctx); + } } } else if (result.error === "cancelled") { // The load was cancelled because the subscriber left before it diff --git a/packages/server/vitest.config.ts b/packages/server/vitest.config.ts index 3e27d717f..375c23ba8 100644 --- a/packages/server/vitest.config.ts +++ b/packages/server/vitest.config.ts @@ -28,7 +28,7 @@ export default defineConfig({ resolve: { // Worktree-local shared source wins over the hoisted-workspace symlink // (which escapes to the main checkout), so tests see the same code the - // build does. Mirrors packages/client/vitest.config.ts resolve.alias. + // build does — including new modules (e.g. event-window). Mirrors client. alias: { "@blackbelt-technology/pi-dashboard-shared": path.resolve(__dirname, "../shared/src"), }, diff --git a/packages/shared/src/AGENTS.md b/packages/shared/src/AGENTS.md index 357247288..d8546f439 100644 --- a/packages/shared/src/AGENTS.md +++ b/packages/shared/src/AGENTS.md @@ -15,6 +15,7 @@ Files in this directory. One row per source file. | `diff-types.ts` | Session file-diff API types. `EditOperation`, `FileChangeEvent`, `FileDiffEntry`, `SessionDiffResponse` (files, `isGitRepo`, `vcsKind`, `diffBase`, `baseLabel`). `FileDiffEntry` gains optional `additions`/`deletions`; `SessionDiffResponse` gains optional `totalAdditions`/`totalDeletions` (from `git diff --numstat`; absent for non-git/binary). See change: add-change-summary-table. `FileChangeEvent.type` gains `"tool"`; `FileDiffEntry` gains optional `origin` (`write`/`edit`/`tool`/`mixed`), `producedBy` (redacted Bash label), `detectedVia` (`git-status`/`bash-artifact`), `previewable` (reserved), `sessionOwned`; `SessionDiffResponse` gains optional `otherChanges[]` (working-tree changes this session cannot claim). See change: detect-tool-created-files. `FileChangeEvent` gains optional `toolCallId` (lazy-fetch key) + `truncated` (in-memory payload trimmed: `content` ended `…[truncated]` or `edits` collapsed >20); `FileDiffEntry.path` doc widened to allow an ABSOLUTE key for out-of-cwd Write/Edit entries (carried payload-only, `previewable:false`, no fs/git enrichment). See change: opt-in-out-of-cwd-session-diffs. | | `display-prefs.ts` | `DisplayPrefs` interface gates chat-view chrome (thinking, tool calls per kind, turn separators, debug tools, token bar, context bar, etc.). `DISPLAY_PRESETS = { simple, standard, everything }`. `mergeDisplayPrefs(global, override)` deep-merges `toolCalls`. `toolCallPrefKey(toolName)` returns canonical prefs key; returns `null` for `ask_user` (non-hidable). `keepReasoningOpenUntilTurnEnds` bool (default false): holds live reasoning open for the active turn, collapses on turn-end edge; coexists with `reasoningAutoCollapseMs` (ms timer governs when false). `toolGroupDefaultCollapsed` bool (default false): when true every tool GROUP defaults collapsed in all auto states (including running — live header still shows). `changeSummaryTable` bool gates per-turn change-summary block (preset: simple false, standard/everything true). `showOutOfCwdSessionDiffs` bool (default OFF in all presets): opt-in to list files this session wrote OUTSIDE cwd in the change-summary block; display gate only (no server read surface). In all presets + `mergeDisplayPrefs`. See changes: configurable-chat-display, keep-reasoning-open-until-turn-ends, enhance-tool-call-grouping, add-change-summary-table, opt-in-out-of-cwd-session-diffs. | | `doctor-core.ts` | Pure detection core. Types `DoctorCheck` / `DoctorReport` / `DoctorSection` (6: runtime/pi-tooling/server/tunnel/setup/diagnostics) / `ExecFailureKind` / `TunnelWatchdogStatusLike`. `SECTION_OF` + `SUGGESTIONS` maps. Helpers `safeExec` (bounded + classified spawn), `safeCheck` (per-check fault isolation), `assumedMandatory` (logs to `/doctor.log` with 1MB ring rotation, surfaces `diagnostics` row), `stripAnsi`, `formatDoctorReportMarkdown`, `defaultDnsLookup`, `runSharedChecks(deps)`. Tunnel section: `zrok binary`, `zrok environment`, `zrok API reachable`, `tunnel runtime`. Adds git source + bash source rows (Windows-only). See change: embed-git-bash-on-windows. Diagnostics section owns `Legacy install directory` advisory; emits only when `detectLegacyManagedDir().present`. `SharedChecksDeps.detectLegacyManagedDir` is injectable test seam. Obsolete `Managed install (~/.pi-dashboard)` row removed. See changes: add-tunnel-diagnostic-checks, fix-doctor-stale-managed-install-check. Adds `checkAttachedServerVersion({appVersion, healthFetcher})`→`DoctorCheck` (section `setup`, name "Attached server version"): `ok` when versions match, `warning` on skew (launch-source-specific suggestion: standalone→npm, bridge/-orphaned→stop pi session, electron→quit other), `error` when unreachable/no version. Wired into Electron doctor arm ONLY (server self-fetch = loopback tautology). `AttachedHealthLike`/`AttachedServerVersionDeps` types. See change: electron-attach-ownership-fixes. | +| `event-window.ts` | Newest-first byte-budget windowing for session event tails. `selectNewestEventsByBudget` / `selectOlderEventsByBudget`, `clampTailWindowBytes`, `DEFAULT_TAIL_WINDOW_BYTES` (4 MiB). Shared by client IDB tail put + server `mode:"tail"` / load-older subscribe. See change: session-tail-rehydrate. | | `live-server.ts` | Pure browser-safe live-server SSRF boundary. `validateLiveTarget({host,port,label})` — loopback-only (`LOOPBACK_HOSTS` = localhost/127.0.0.1/::1), port 1..65535; rejects cloud-metadata/private/public. `liveServerPath(id)`→`/live//`. Types `LiveServerTarget`, `LiveTargetValidation`. `isLoopbackUrl(href)`→bool: true only for `http(s)://{localhost,127.0.0.1,::1}[:port]/…` (brackets stripped, case-insensitive); credential-in-host/`0.0.0.0`/IPv4-mapped-IPv6/punycode → false. UX router for the split-viewer link routing, not a trust boundary. Shared by client pre-validation + server enforcement. See change: improve-content-editor; open-loopback-links-in-split-viewer. | | `file-kind.ts` | Pure browser-safe viewer classifier. `fileKind(absPath, sniff?)` → `{ kind, mimeType, viewer, editable }`. Discrimination: markdown ext→markdown; `.pdf`→pdf; image ext→image; text allowlist→monaco; else sniff NUL byte→binary, default→monaco/unknown. `editable` always false v1. Throws on relative path. Exports `TEXT_EXTENSIONS`, `IMAGE_EXTENSIONS`, `ViewerKind`, `FileKind`, `FileKindResult`. Re-exported from `index.ts` barrel. See change: add-internal-monaco-editor-pane. `editable` now true for writable markdown subset `.md`/`.mdx` (false elsewhere). WRITABLE_MARKDOWN_EXTENSIONS. See change: directory-settings-page-and-scoped-md-editing. Adds viewer kinds html/mermaid/video/audio: `.html/.htm`→html, `.mmd/.mermaid`→mermaid, `.mp3/.wav/.ogg/.m4a/.flac`→audio, `.mp4/.webm/.mov`→video (`.html`/`.htm` removed from TEXT_EXTENSIONS). New exports `HTML_EXTENSIONS`, `MERMAID_EXTENSIONS`, `AUDIO_EXTENSIONS`, `VIDEO_EXTENSIONS`. See change: improve-content-editor. Adds `"diff"` ViewerKind — opened explicitly, never returned by `fileKind()`. See change: add-change-summary-table. Adds `"terminal"` ViewerKind — opened explicitly under a virtual `term:` tab, never returned by `fileKind()`. See change: terminals-in-tabbed-panes. | | `git-worktree-helpers.ts` | Pure worktree helpers shared by server + client. `slugifyBranch(branch)` → fs-safe slug; `localNameOf(ref)` strips remote prefix; `resolveCheckoutLocalName(base, baseIsLocalBranch)`; `resolveDefaultBase(input)` picks current→develop→main→master→fail. No fs/child_process. | diff --git a/packages/shared/src/__tests__/event-window.test.ts b/packages/shared/src/__tests__/event-window.test.ts new file mode 100644 index 000000000..a743595ea --- /dev/null +++ b/packages/shared/src/__tests__/event-window.test.ts @@ -0,0 +1,118 @@ +import { describe, expect, it } from "vitest"; +import { + DEFAULT_TAIL_WINDOW_BYTES, + MAX_TAIL_WINDOW_BYTES, + MIN_TAIL_WINDOW_BYTES, + clampTailWindowBytes, + estimateSeqEventBytes, + selectNewestEventsByBudget, + selectOlderEventsByBudget, + type SeqEvent, +} from "../event-window.js"; + +function ev(seq: number, padChars: number): SeqEvent<{ n: number; pad: string }> { + return { seq, event: { n: seq, pad: "x".repeat(padChars) } }; +} + +describe("clampTailWindowBytes", () => { + it("defaults for missing / invalid", () => { + expect(clampTailWindowBytes(undefined)).toBe(DEFAULT_TAIL_WINDOW_BYTES); + expect(clampTailWindowBytes(NaN)).toBe(DEFAULT_TAIL_WINDOW_BYTES); + expect(clampTailWindowBytes(0)).toBe(DEFAULT_TAIL_WINDOW_BYTES); + expect(clampTailWindowBytes(-1)).toBe(DEFAULT_TAIL_WINDOW_BYTES); + }); + + it("clamps to min/max", () => { + expect(clampTailWindowBytes(1)).toBe(MIN_TAIL_WINDOW_BYTES); + expect(clampTailWindowBytes(MIN_TAIL_WINDOW_BYTES)).toBe(MIN_TAIL_WINDOW_BYTES); + expect(clampTailWindowBytes(MAX_TAIL_WINDOW_BYTES + 1)).toBe(MAX_TAIL_WINDOW_BYTES); + }); +}); + +describe("selectNewestEventsByBudget", () => { + it("returns empty for empty input", () => { + const r = selectNewestEventsByBudget([], 10_000); + expect(r).toEqual({ + events: [], + hasMoreOlder: false, + windowMinSeq: 0, + windowMaxSeq: 0, + bytes: 0, + }); + }); + + it("returns all when under budget", () => { + const all = [ev(1, 10), ev(2, 10), ev(3, 10)]; + const r = selectNewestEventsByBudget(all, 1_000_000); + expect(r.events.map((e) => e.seq)).toEqual([1, 2, 3]); + expect(r.hasMoreOlder).toBe(false); + expect(r.windowMinSeq).toBe(1); + expect(r.windowMaxSeq).toBe(3); + }); + + it("keeps newest under a tight budget", () => { + // Each event is large; budget fits roughly two of the smaller ones. + const all = [ev(1, 800), ev(2, 800), ev(3, 800), ev(4, 800)]; + const oneSize = estimateSeqEventBytes(all[3]!); + const budget = oneSize * 2 + 10; // fit newest two + const r = selectNewestEventsByBudget(all, budget); + expect(r.events.map((e) => e.seq)).toEqual([3, 4]); + expect(r.hasMoreOlder).toBe(true); + expect(r.windowMinSeq).toBe(3); + expect(r.windowMaxSeq).toBe(4); + expect(r.bytes).toBeLessThanOrEqual(budget); + }); + + it("always keeps the newest event even when it alone exceeds budget", () => { + const huge = ev(1, 50_000); + const size = estimateSeqEventBytes(huge); + // clamp raises tiny budgets to MIN — pass a budget that after clamp still + // is smaller than `size` by using the raw path: select with budget equal + // to MIN when size > MIN. + if (size <= MIN_TAIL_WINDOW_BYTES) { + // pad further + const bigger = ev(1, 300_000); + const r = selectNewestEventsByBudget([bigger], MIN_TAIL_WINDOW_BYTES); + expect(r.events).toHaveLength(1); + expect(r.hasMoreOlder).toBe(false); + expect(r.bytes).toBeGreaterThan(MIN_TAIL_WINDOW_BYTES); + } else { + const r = selectNewestEventsByBudget([huge], MIN_TAIL_WINDOW_BYTES); + expect(r.events).toHaveLength(1); + expect(r.events[0]!.seq).toBe(1); + } + }); + + it("outputs ascending seq order", () => { + const all = [ev(10, 50), ev(20, 50), ev(30, 50)]; + const r = selectNewestEventsByBudget(all, 1_000_000); + const seqs = r.events.map((e) => e.seq); + expect(seqs).toEqual([...seqs].sort((a, b) => a - b)); + }); +}); + +describe("selectOlderEventsByBudget", () => { + it("pages strictly older than fromSeq", () => { + const all = [ev(1, 20), ev(2, 20), ev(3, 20), ev(4, 20), ev(5, 20)]; + const r = selectOlderEventsByBudget(all, 4, 1_000_000); + expect(r.events.map((e) => e.seq)).toEqual([1, 2, 3]); + expect(r.hasMoreOlder).toBe(false); + expect(r.windowMaxSeq).toBe(3); + }); + + it("applies budget on the older prefix", () => { + const all = [ev(1, 800), ev(2, 800), ev(3, 800), ev(4, 800), ev(5, 800)]; + const oneSize = estimateSeqEventBytes(all[2]!); + const r = selectOlderEventsByBudget(all, 5, oneSize * 2 + 10); + // older = 1..4; newest-first of those under budget → 3,4 + expect(r.events.map((e) => e.seq)).toEqual([3, 4]); + expect(r.hasMoreOlder).toBe(true); + }); + + it("empty when fromSeq at/below oldest", () => { + const all = [ev(5, 10), ev(6, 10)]; + const r = selectOlderEventsByBudget(all, 5, 1_000_000); + expect(r.events).toEqual([]); + expect(r.hasMoreOlder).toBe(false); + }); +}); diff --git a/packages/shared/src/browser-protocol.ts b/packages/shared/src/browser-protocol.ts index bb41cd2dd..d987f0034 100644 --- a/packages/shared/src/browser-protocol.ts +++ b/packages/shared/src/browser-protocol.ts @@ -116,6 +116,15 @@ export interface EventReplayMessage { sessionId: string; events: Array<{ seq: number; event: DashboardEvent }>; isLast: boolean; + /** + * True when older history exists below the delivered window. + * See change: session-tail-rehydrate. + */ + hasMoreOlder?: boolean; + /** Lowest seq in the retained/delivered window (tail or older page). */ + windowMinSeq?: number; + /** Highest seq in this delivery. */ + windowMaxSeq?: number; } export interface BrowserCommandsListMessage { @@ -922,6 +931,19 @@ export interface SubscribeMessage { type: "subscribe"; sessionId: string; lastSeq?: number; + /** + * `"tail"` = newest events under a byte budget (cold open). + * `"full"` or omitted = legacy full/delta behavior. + * See change: session-tail-rehydrate. + */ + mode?: "full" | "tail"; + /** Soft budget hint for tail/older pages; server clamps. */ + windowBytes?: number; + /** + * Load-older: exclusive upper bound — return newest events with seq < fromSeq. + * See change: session-tail-rehydrate. + */ + fromSeq?: number; } export interface UnsubscribeMessage { diff --git a/packages/shared/src/event-window.ts b/packages/shared/src/event-window.ts new file mode 100644 index 000000000..81d519ab7 --- /dev/null +++ b/packages/shared/src/event-window.ts @@ -0,0 +1,112 @@ +/** + * Newest-first byte-budget windowing for session event tails. + * + * Used by: + * - client IndexedDB replay cache (trim-on-put for large sessions) + * - server subscribe `mode: "tail"` / load-older (`fromSeq`) paths + * + * See change: session-tail-rehydrate. + */ + +/** Default wire/IDB tail budget (~4 MiB). */ +export const DEFAULT_TAIL_WINDOW_BYTES = 4 * 1024 * 1024; + +/** Server clamp for client-supplied windowBytes. */ +export const MIN_TAIL_WINDOW_BYTES = 256 * 1024; +export const MAX_TAIL_WINDOW_BYTES = 8 * 1024 * 1024; + +/** Minimal event-like shape; both store and wire use `{ seq, event }`. */ +export interface SeqEvent { + seq: number; + event: T; +} + +export interface EventWindowResult { + /** Selected events in ascending seq order. */ + events: SeqEvent[]; + /** True when the input had older events not included. */ + hasMoreOlder: boolean; + /** Lowest seq in `events`, or 0 if empty. */ + windowMinSeq: number; + /** Highest seq in `events`, or 0 if empty. */ + windowMaxSeq: number; + /** Serialized size of the selected payload (sum of per-event sizes). */ + bytes: number; +} + +/** + * Clamp a requested budget into the allowed range. Non-finite / missing → default. + * Use at untrusted API boundaries (subscribe.windowBytes). Pure selectors do not clamp. + */ +export function clampTailWindowBytes(requested?: number): number { + if (requested == null || !Number.isFinite(requested) || requested <= 0) { + return DEFAULT_TAIL_WINDOW_BYTES; + } + return Math.min(MAX_TAIL_WINDOW_BYTES, Math.max(MIN_TAIL_WINDOW_BYTES, Math.floor(requested))); +} + +/** + * Stable size estimate matching IndexedDB put's historical + * `JSON.stringify(payload).length` accounting: per-event + * `JSON.stringify({ seq, event }).length`. + */ +export function estimateSeqEventBytes(entry: SeqEvent): number { + return JSON.stringify(entry).length; +} + +/** + * Select the newest events from an ascending-seq list that fit `budgetBytes`. + * + * - Walks newest → oldest. + * - Never splits an event. + * - Always keeps at least the newest event when the list is non-empty (even if + * that single event exceeds the budget). + * - Returns events in ascending seq order. + * - Does NOT clamp budget (call `clampTailWindowBytes` at API edges). + */ +export function selectNewestEventsByBudget( + eventsAsc: readonly SeqEvent[], + budgetBytes: number = DEFAULT_TAIL_WINDOW_BYTES, +): EventWindowResult { + const budget = + Number.isFinite(budgetBytes) && budgetBytes > 0 + ? Math.floor(budgetBytes) + : DEFAULT_TAIL_WINDOW_BYTES; + + if (eventsAsc.length === 0) { + return { events: [], hasMoreOlder: false, windowMinSeq: 0, windowMaxSeq: 0, bytes: 0 }; + } + + const picked: SeqEvent[] = []; + let bytes = 0; + + for (let i = eventsAsc.length - 1; i >= 0; i--) { + const entry = eventsAsc[i]!; + const size = estimateSeqEventBytes(entry); + if (picked.length > 0 && bytes + size > budget) break; + picked.push(entry); + bytes += size; + } + + // picked is newest-first; reverse to ascending seq. + picked.reverse(); + + const hasMoreOlder = picked.length < eventsAsc.length; + const windowMinSeq = picked[0]?.seq ?? 0; + const windowMaxSeq = picked[picked.length - 1]?.seq ?? 0; + + return { events: picked, hasMoreOlder, windowMinSeq, windowMaxSeq, bytes }; +} + +/** + * Select an older page: newest events with `seq < fromSeq` under the budget. + * `eventsAsc` must be the full ascending buffer (or the prefix older than live tail). + */ +export function selectOlderEventsByBudget( + eventsAsc: readonly SeqEvent[], + fromSeq: number, + budgetBytes: number = DEFAULT_TAIL_WINDOW_BYTES, +): EventWindowResult { + const older = eventsAsc.filter((e) => e.seq < fromSeq); + return selectNewestEventsByBudget(older, budgetBytes); +}