fix(claude-native): make Claude's status file the source of truth

omnigent/claude_native_forwarder.py

@@ -250,17 +250,23 @@ def _hold_assistant_item_for_deltas(
 # published on the per-conversation SSE stream. Unmapped events emit
 # no status.
 #
-# ``Stop`` → idle and ``StopFailure`` → failed are the authoritative
-# turn-end edges (each fires once when Claude finishes / errors a turn);
-# they drive sub-agent terminal delivery via the codex-shared
-# ``external_session_status`` path (→ parent inbox + wake). The
-# PTY-activity ``idle`` cannot: it is a ~1s-quiescence heuristic that
-# oscillates on every mid-turn lull, so delivering on it fired a
-# premature completion and idempotently locked out the real one.
-# ``UserPromptSubmit`` → running stays PTY-derived — the pane watcher
-# drives the UI running/idle badge and catches what ``Stop`` misses
-# (interrupts, compaction failures, TUI edits). ``_publish_status``
-# keeps ``failed`` sticky against the trailing PTY idle.
+# Claude's own ``sessions/<pid>.json`` owns the running/idle badge (see
+# :mod:`omnigent.claude_native_status_file`), so these two hooks exist for
+# what the file cannot express:
+#
+# - ``Stop`` → idle: the sub-agent terminal-delivery edge (→ parent inbox +
+#   wake, via the codex-shared ``external_session_status`` path). It fires
+#   exactly once per finished turn, where the PTY-activity ``idle`` was a
+#   ~1s-quiescence heuristic that oscillated on mid-turn lulls, firing a
+#   premature completion that idempotently locked out the real one. It also
+#   carries the background-shell count. It agrees with the file rather than
+#   competing with it, so arrival order does not matter — the shared edge
+#   dedup collapses the pair.
+# - ``StopFailure`` → failed: the file has no failure literal (it returns to
+#   ``idle`` on a turn error exactly as on success), so this is the only
+#   source of the red pill, ``last_task_error``, and a failed scheduled run.
+#   ``_publish_status`` keeps it sticky against a trailing ``idle``; the
+#   file's next ``busy`` clears it on the following turn.
 _HOOK_EVENT_TO_STATUS: dict[str, str] = {
     "Stop": "idle",
     "StopFailure": "failed",
@@ -657,12 +663,6 @@ class _ForwardDedupeState:
     # sub-agent spend so the gate can block mid-turn. Separate baseline
     # because it can advance while ``posted_cost`` (S) is frozen.
     posted_policy_cost: float | None = None
-    # Response id of the last turn-start ``running`` status POSTed, so the
-    # id-bearing running edge fires exactly once per turn even when an
-    # assistant item is held across polls for delta ordering (which leaves
-    # ``state.current_response_id`` unadvanced). ``None`` until the first
-    # turn-start edge. Reset on /clear and /fork like the other baselines.
-    posted_running_response_id: str | None = None
     # Turn-settle latch driving the scheduled-wake boundary. The Stop edge
     # records the ended turn's id as PENDING; it activates (moves to
     # ``settled_response_id``) only once a fully-consumed transcript batch
@@ -3030,19 +3030,18 @@ async def _forward_available_status_events(
         retry_key = f"hook:{record.event_cursor}:{record.byte_offset}:{status}"
         if retry_tracker.retry_delay_s(retry_key) is not None:
             return durable
-        effective_status = status
-        if status == "idle" and record.background_task_count > 0:
-            effective_status = "waiting"
         try:
             await post_external_session_status(
                 client,
                 session_id=session_id,
-                status=effective_status,
+                status=status,
                 response_id=response_id,
-                # Only the ``Stop`` (idle/waiting) edge carries an authoritative
+                # Only the ``Stop`` (idle) edge carries an authoritative
                 # background-shell count — ``0`` clears the tally, ``N`` sets it.
-                # ``StopFailure`` (failed) clears it on the server regardless, so
-                # leave its count off the wire.
+                # This is the one thing the status file cannot report: its
+                # ``shell`` literal is a boolean, and the indicator renders a
+                # number. ``StopFailure`` (failed) clears it on the server
+                # regardless, so leave its count off the wire.
                 background_task_count=(
                     None if status == "failed" else record.background_task_count
                 ),
@@ -3177,31 +3176,6 @@ async def _ensure_state_for_transcript(
     return state
 
 
-def _turn_has_assistant_output(items: list[ClaudeTranscriptItem], response_id: str) -> bool:
-    """
-    Whether ``response_id`` has assistant-generated output among ``items``.
-
-    The turn-start ``running`` edge should open a streaming turn only for an id
-    that a later ``Stop``/``StopFailure`` hook will close — i.e. one produced by
-    an actual LLM turn. Assistant text (``message`` with ``role=assistant``) and
-    tool calls (``function_call``) qualify; a ``slash_command`` (``/model``,
-    ``/effort``) or ``terminal_command`` (``!cmd``) item opens an id with no LLM
-    turn behind it, so it must not.
-
-    :param items: Transcript items read this poll.
-    :param response_id: The current turn's response id.
-    :returns: ``True`` when an assistant-output item carries ``response_id``.
-    """
-    for item in items:
-        if item.response_id != response_id:
-            continue
-        if item.item_type == "function_call":
-            return True
-        if item.item_type == "message" and item.data.get("role") == "assistant":
-            return True
-    return False
-
-
 def _promote_pending_settle(
     dedupe: _ForwardDedupeState, items: list[ClaudeTranscriptItem]
 ) -> bool:
@@ -3460,55 +3434,15 @@ async def _forward_available_items(
     current_response_id = result.current_response_id
     seen_source_ids = list(state.seen_source_ids)
     seen = set(seen_source_ids)
-    # NOTE: the old "re-assert running on resumed agent output" hack lived
-    # here. It only existed to paper over the hook model's compaction
-    # blind spot (``Stop`` → idle, then an ``isCompactSummary`` resume that
-    # never fired ``UserPromptSubmit``). PTY-activity status makes it
-    # obsolete: the pane keeps changing through a mid-turn compaction, so
-    # the runner's watcher holds the session ``running`` directly.
-    #
-    # Turn-start edge: the first time we see a turn's response id, publish a
-    # ``running`` status carrying it. The PTY watcher already drives the
-    # running/idle BADGE with a bare (id-less) status; this id-bearing edge is
-    # what lets ap-web open a *streaming* ``activeResponse`` for the turn, so
-    # the forwarded tool-call cards (which carry the same response id) render
-    # LIVE — spinner + elapsed timer — instead of as static completed cards.
-    # Deduped on the persistent ``dedupe`` baseline (NOT ``state``): when an
-    # assistant item is held across polls for delta ordering, this function
-    # early-returns with ``state`` unadvanced, so a ``state``-based guard would
-    # re-fire ``running`` every poll of the hold window. Best-effort — a failed
-    # status post must not abort item forwarding (the items below are the
-    # primary payload); the turn-end idle/failed edge still carries the id to
-    # close the lifecycle, and the badge is unaffected either way.
-    #
-    # Only open the streaming turn for an id that has ASSISTANT output in this
-    # poll's items. A surfaced CLI built-in (``/model``, ``/effort``) or a
-    # ``!cmd`` becomes a slash_command / terminal_command item that opens its
-    # own response id but runs no LLM turn, so no ``Stop`` hook ever fires to
-    # close it — a ``running`` opened for it would strand the web composer in
-    # its "Stop"/busy state until the next real message. A skill that DOES
-    # trigger an LLM turn shares its id with the assistant text it produces, so
-    # ``running`` still fires — one poll later, when that output appears.
-    if (
-        current_response_id is not None
-        and dedupe.posted_running_response_id != current_response_id
-        and _turn_has_assistant_output(items, current_response_id)
-    ):
-        try:
-            await post_external_session_status(
-                client,
-                session_id=session_id,
-                status="running",
-                response_id=current_response_id,
-            )
-            dedupe.posted_running_response_id = current_response_id
-        except httpx.HTTPError:
-            _logger.warning(
-                "Failed to forward Claude turn-start running status; session=%s response_id=%s",
-                session_id,
-                current_response_id,
-                exc_info=True,
-            )
+    # This function publishes no session status. Claude's own
+    # ``sessions/<pid>.json`` owns the running/idle badge (see
+    # :mod:`omnigent.claude_native_status_file`), and it reports the turn ending
+    # the moment Claude settles. A status edge derived from the transcript can
+    # only fire once a poll has parsed assistant output, so it lands *after* the
+    # file's ``idle`` on a short turn and re-asserts ``running`` on a session
+    # that already finished — the user sees idle → running → idle. Items carry
+    # their own ``response_id`` (see :func:`_post_external_conversation_item`),
+    # so the transcript's job here is items, not status.
     updated = state
     for item in items:
         if item.source_id in seen:

omnigent/claude_native_status_file.py

@@ -4,8 +4,10 @@
 ``<config_dir>/sessions/<pid>.json`` (its internal "concurrentSessions"
 registry, present since v2.1.139 — the file that also backs
 ``claude agents``). For an interactive session it carries a ``status``
-field that flips ``idle`` ⇄ ``busy`` ⇄ ``waiting`` as the agent works,
-which is a cleaner running/idle signal than diffing the tmux pane.
+field that flips ``idle`` ⇄ ``busy`` ⇄ ``waiting`` as the agent works. It
+reports what Claude is doing rather than inferring it from pane redraws, so
+it — not the tmux pane diff — is the session's running/idle status whenever
+it is readable.
 
 This module owns two pure pieces the claude-native status watcher builds
 on:
@@ -25,12 +27,15 @@
 from __future__ import annotations
 
 import json
+import logging
 import os
 import time
 from collections.abc import Callable
 from dataclasses import dataclass
 from pathlib import Path
 
+_logger = logging.getLogger(__name__)
+
 # Runner-side status vocabulary the file maps onto. ``busy`` and
 # ``waiting`` both mean "the turn is not finished" from the session's point
 # of view, so both map to ``running``; ``waiting`` is distinguished for the
@@ -49,9 +54,9 @@
     "waiting": RUNNING,
     "idle": IDLE,
     # The turn ended but a background shell is still alive (Claude Code
-    # >= v2.1.197). The agent loop is idle, so this maps to ``idle`` — the
-    # Stop hook separately relabels its own ``idle`` to ``waiting`` with the
-    # shell tally, which is what keeps the spinner lit. Mapping ``shell`` to
+    # >= v2.1.197). The agent loop is idle, so this maps to ``idle``; the
+    # working indicator stays lit off the ``Stop`` hook's shell tally, which
+    # carries the count this boolean literal cannot. Mapping ``shell`` to
     # ``running`` would strand the composer on the "(queued)" placeholder,
     # since the session never reads idle while a background shell runs.
     "shell": IDLE,
@@ -273,10 +278,12 @@ class SessionStatusPoller:
     - **Exhausted:** if resolution never succeeds, :attr:`active` stays
       ``False`` permanently and the file contributes nothing.
 
-    The poller never displaces the PTY watcher: it supplies an *additional*
-    status edge at Claude's real turn boundary, plus the freshness-bounded
-    :meth:`asserts_running` level the watcher consults before declaring a
-    quiet pane idle.
+    While :attr:`active` the poller *is* the session's status — it reports what
+    Claude is doing, where the pane diff only infers it from redraws — and the
+    PTY watcher publishes none. The watcher takes over when no file was ever
+    resolved (Claude older than v2.1.139) and always owns pane death, which the
+    file structurally cannot report: a killed Claude leaves its record behind
+    (see :meth:`retire`).
 
     :param on_status: Callback invoked as ``(runner_status, blocked_on)``
         on each transition (and once on first read). Fires when either part
@@ -339,46 +346,61 @@ def tick(self) -> None:
     def _try_resolve(self) -> None:
         """Attempt one resolution, retiring to the PTY watcher on timeout."""
         self._attempts += 1
+        pane_pid = self._pane_pid_getter()
         path = resolve_status_file(
-            pane_pid=self._pane_pid_getter(),
+            pane_pid=pane_pid,
             expected_session_id=self._session_id_getter(),
             config_dir=self._config_dir,
         )
         if path is not None:
+            # Log the hit: whether the file was found at all decides which
+            # source owns the session's status, and without this the answer is
+            # only reachable by re-deriving the resolution by hand.
+            _logger.info(
+                "claude status file resolved: path=%s attempts=%d pane_pid=%s",
+                path,
+                self._attempts,
+                pane_pid,
+            )
             self._path = path
             return
         if self._attempts >= _MAX_RESOLVE_ATTEMPTS:
+            _logger.warning(
+                "claude status file never resolved after %d attempts "
+                "(pane_pid=%s); session status falls back to the pane watcher",
+                self._attempts,
+                pane_pid,
+            )
             self._exhausted = True
 
-    def asserts_running(self, *, ttl_s: float, now: float | None = None) -> bool:
-        """Whether the file *recently* reported the session as running.
-
-        The file is written only when its value changes, so its status is a
-        level that can outlive the truth — Claude keeps reporting ``busy``
-        while a delegate or background task is active, long after the turn
-        itself ended. Callers therefore treat it as authoritative only for
-        *ttl_s* after the write, and fall back to the pane watcher once it
-        goes stale rather than pinning the session to ``running`` forever.
-
-        :param ttl_s: How long after ``statusUpdatedAt`` the level is still
-            trusted, in seconds.
-        :param now: Wall-clock override (tests); uses :func:`time.time`
-            when ``None``.
-        :returns: ``True`` when the last read said running and is still fresh.
+    def retire(self) -> None:
+        """Stop reading the file, permanently.
+
+        Called when the pane's process is gone: a killed Claude does not unlink
+        its file, so the record survives holding whatever it last said. Since
+        the file owns the session's status while it is readable, a dead pane
+        must retire it or that final value would pin the session forever.
         """
-        status = self._last_status
-        if status is None or status.runner_status != RUNNING:
-            return False
-        # ``waiting`` does not decay: a dialog owns Claude's input until it
-        # closes, and closing it changes the value — so a new write is
-        # guaranteed. ``busy`` decays, because a delegate or background task
-        # keeps it set long after the turn it belongs to has ended.
-        if status.raw_status == "waiting":
-            return True
-        if status.status_updated_at is None:
-            return False
-        clock = time.time() if now is None else now
-        return clock - status.status_updated_at / 1000.0 <= ttl_s
+        _logger.info("claude status file retired: path=%s", self._path)
+        self._exhausted = True
+
+    def resync(self) -> None:
+        """Forget what was published so the next tick re-asserts the file.
+
+        The file is written only when its value *changes*, so a poller that
+        already published ``running`` has nothing more to say until Claude's
+        status moves. That is a problem when the *listener* restarts: a server
+        recycle wipes its status cache, and the session would sit on a stale
+        ``idle`` for the rest of the turn because every source believes it
+        already reported. Dropping the edge/mtime baselines makes the next tick
+        publish the file's current value verbatim.
+
+        Keeps the resolved path and the attempt count — this re-asserts a
+        working poller, it does not restart resolution.
+        """
+        _logger.info("claude status file resync: path=%s", self._path)
+        self._last_mtime = None
+        self._last_edge = None
 
     @property
     def blocked_on(self) -> str | None:

omnigent/runner/app.py

@@ -8593,6 +8593,18 @@ async def elicitation(elicitation_id: str, request: Request) -> Response:
             )
 
     async def _catch_up_scan() -> None:
+        # The tunnel just reconnected, which usually means the SERVER restarted
+        # (deploy, crash, replica failover) and lost its in-memory session-status
+        # cache. This runner did not restart, so every status source still
+        # believes its last edge was delivered and nothing re-asserts — a
+        # native session mid-turn during the restart would sit on a stale
+        # ``idle`` for the rest of the turn. Re-arm them before the item scan
+        # below (which skips native harnesses entirely).
+        if resource_registry is not None:
+            try:
+                resource_registry.resync_session_statuses()
+            except Exception:  # noqa: BLE001 — best-effort; never block catch-up.
+                _logger.warning("Session status resync failed after reconnect", exc_info=True)
         for session_id in list(_session_histories):
             if _is_native_harness(session_id):
                 continue