fix(proxy): never inject a cross-account previous_response_id anchor (takeover of #1274)

app/modules/api_keys/repository.py

@@ -380,6 +380,12 @@ async def delete(self, key_id: str) -> bool:
     async def commit(self) -> None:
         await self._session.commit()
 
+    async def update_last_used(self, key_id: str, *, commit: bool = True) -> None:
+        """Compatibility touch for maintenance and durability checks."""
+        await self._session.execute(update(ApiKey).where(ApiKey.id == key_id).values(last_used_at=utcnow()))
+        if commit:
+            await self._session.commit()
+
     async def rollback(self) -> None:
         await self._session.rollback()
 

app/modules/proxy/_service/http_bridge/mixin.py

@@ -2367,6 +2367,7 @@ async def abort_selected_handoff() -> None:
                 await self._unregister_http_bridge_turn_states(session)
                 await self._unregister_http_bridge_previous_response_ids(session)
                 session.last_completed_response_id = None
+                session.last_completed_response_account_id = None
                 session.last_completed_input_count = 0
                 session.last_completed_input_prefix_fingerprint = None
                 session.last_pending_tool_calls.clear()

app/modules/proxy/_service/http_bridge/streaming.py

@@ -2173,29 +2173,58 @@ def switch_to_account_neutral_replay() -> None:
                     and isinstance(recovery_payload.input, list)
                     and len(recovery_payload.input) > durable_full_resend_anchor_count
                 ):
-                    recovery_input = cast(list[JsonValue], recovery_payload.input)
-                    recovery_anchor_input_count = len(recovery_input)
-                    recovery_anchor_input_fingerprint = _fingerprint_input_items(recovery_input)
-                    recovery_payload = recovery_payload.model_copy(
-                        update={
-                            "previous_response_id": durable_lookup.latest_response_id,
-                            "input": recovery_input[durable_full_resend_anchor_count:],
-                        }
-                    )
-                    if durable_lookup.latest_response_id != session.last_completed_response_id:
-                        session.last_pending_tool_calls = {}
-                    session.last_completed_response_id = durable_lookup.latest_response_id
-                    session.last_completed_input_count = durable_full_resend_anchor_count
-                    session.last_completed_input_prefix_fingerprint = durable_full_resend_anchor_fingerprint
-                    _log_http_bridge_event(
-                        "owner_forward_recovery_anchor_injected",
-                        bridge_session_key,
-                        account_id=durable_lookup.account_id,
-                        model=recovery_payload.model,
-                        detail=f"response_id={durable_lookup.latest_response_id}",
-                        cache_key_family=bridge_session_key.affinity_kind,
-                        model_class=_extract_model_class(recovery_payload.model) if recovery_payload.model else None,
-                    )
+                    # The recovery rebind above is allowed to bind this session to
+                    # an account other than the durable owner. A previous_response_id
+                    # is account-scoped upstream, so replaying the durable anchor on
+                    # a different account sends an anchor upstream cannot resolve
+                    # with the history trimmed away: no response.created arrives and
+                    # the per-bridge response-create gate wedges. Resend the full
+                    # history on the serving account instead.
+                    if durable_lookup.account_id != session.account.id:
+                        _log_http_bridge_event(
+                            "cross_account_anchor_declined",
+                            bridge_session_key,
+                            account_id=session.account.id,
+                            model=recovery_payload.model,
+                            detail=(
+                                "site=owner_forward_recovery, "
+                                f"response_id={durable_lookup.latest_response_id}, "
+                                f"anchor_account_id={durable_lookup.account_id}, "
+                                "outcome=full_history_resend"
+                            ),
+                            cache_key_family=bridge_session_key.affinity_kind,
+                            model_class=_extract_model_class(recovery_payload.model)
+                            if recovery_payload.model
+                            else None,
+                            owner_check_applied=True,
+                        )
+                    else:
+                        recovery_input = cast(list[JsonValue], recovery_payload.input)
+                        recovery_anchor_input_count = len(recovery_input)
+                        recovery_anchor_input_fingerprint = _fingerprint_input_items(recovery_input)
+                        recovery_payload = recovery_payload.model_copy(
+                            update={
+                                "previous_response_id": durable_lookup.latest_response_id,
+                                "input": recovery_input[durable_full_resend_anchor_count:],
+                            }
+                        )
+                        if durable_lookup.latest_response_id != session.last_completed_response_id:
+                            session.last_pending_tool_calls = {}
+                        session.last_completed_response_id = durable_lookup.latest_response_id
+                        session.last_completed_response_account_id = durable_lookup.account_id
+                        session.last_completed_input_count = durable_full_resend_anchor_count
+                        session.last_completed_input_prefix_fingerprint = durable_full_resend_anchor_fingerprint
+                        _log_http_bridge_event(
+                            "owner_forward_recovery_anchor_injected",
+                            bridge_session_key,
+                            account_id=durable_lookup.account_id,
+                            model=recovery_payload.model,
+                            detail=f"response_id={durable_lookup.latest_response_id}",
+                            cache_key_family=bridge_session_key.affinity_kind,
+                            model_class=_extract_model_class(recovery_payload.model)
+                            if recovery_payload.model
+                            else None,
+                        )
                 recovery_injected_input = _http_bridge_interrupted_tool_outputs_input(
                     session,
                     payload=recovery_payload,
@@ -2305,6 +2334,12 @@ def switch_to_account_neutral_replay() -> None:
                 # must not trigger interrupted-output injection.
                 session.last_pending_tool_calls = {}
             session.last_completed_response_id = durable_lookup.latest_response_id
+            # The durable anchor is owned by the durable session's account, which
+            # may differ from this session's account after a failover. Record the
+            # owner so the session-anchor injection below can refuse to replay a
+            # cross-account previous_response_id (upstream cannot resolve it and
+            # would stall with no response.created — a wedged response-create gate).
+            session.last_completed_response_account_id = durable_lookup.account_id
             session.last_completed_input_count = durable_full_resend_anchor_count
             session.last_completed_input_prefix_fingerprint = durable_full_resend_anchor_fingerprint
         # --- Session-level previous_response_id injection ---
@@ -2333,17 +2368,45 @@ def switch_to_account_neutral_replay() -> None:
             stored_count=stored_count_preview,
             stored_fingerprint=stored_fingerprint_preview,
         )
+        # A previous_response_id is account-scoped upstream: only the account that
+        # created the response can resume it. If this session's serving account is
+        # not the anchor's owner (e.g. the session failed over after the durable
+        # owner became unavailable), injecting the anchor sends an unresolvable
+        # previous_response_id upstream with the history trimmed away — upstream
+        # then never emits response.created and the response-create gate wedges
+        # ("idle timeout waiting for SSE"). Fall through to a full-history resend.
+        session_anchor_account_owned = (
+            session.last_completed_response_account_id is not None
+            and session.last_completed_response_account_id == session.account.id
+        )
         recovery_session_can_anchor = is_http_bridge_account_neutral_replay(
             kind=session.key.affinity_kind,
             key=session.key.affinity_key,
         ) and (not _http_bridge_payload_looks_like_full_resend(effective_payload) or session_anchor_trimmable)
-        if (
+        session_anchor_candidate = (
             session.codex_session
             and not proxy_injected_previous_response_id
             and effective_payload.previous_response_id is None
             and session.last_completed_response_id is not None
             and (session_anchor_trimmable or recovery_session_can_anchor)
-        ):
+        )
+        if session_anchor_candidate and not session_anchor_account_owned:
+            _log_http_bridge_event(
+                "cross_account_anchor_declined",
+                session.key,
+                account_id=session.account.id,
+                model=effective_payload.model,
+                detail=(
+                    "site=session_anchor, "
+                    f"response_id={session.last_completed_response_id}, "
+                    f"anchor_account_id={session.last_completed_response_account_id}, "
+                    "outcome=full_history_resend"
+                ),
+                cache_key_family=session.key.affinity_kind,
+                model_class=_extract_model_class(effective_payload.model) if effective_payload.model else None,
+                owner_check_applied=True,
+            )
+        if session_anchor_candidate and session_anchor_account_owned:
             fresh_upstream_request_text = text_data
             session_level_payload_looks_like_full_resend = _http_bridge_payload_looks_like_full_resend(
                 effective_payload

app/modules/proxy/_service/http_bridge/upstream_events.py

@@ -2016,6 +2016,10 @@ async def _process_parsed_http_bridge_upstream_event(
             # anchor for continuity lookups.
             if response_id is not None:
                 session.last_completed_response_id = response_id
+                # This response was completed on the session's current account, so
+                # that account owns the anchor. Record it so the anchor is only
+                # replayed on the same account (never after a cross-account failover).
+                session.last_completed_response_account_id = session.account.id
                 # Remember which tool-call items the completed response left
                 # pending so an anchored follow-up that omits their outputs
                 # (interrupted turn) can receive synthetic interrupted

app/modules/proxy/_service/support.py

@@ -1022,6 +1022,12 @@ class _HTTPBridgeSession:
     previous_response_alias_registration_generations: dict[str, int] = field(default_factory=dict)
     last_completed_input_count: int = 0
     last_completed_response_id: str | None = None
+    # Account that owns ``last_completed_response_id``. A previous_response_id
+    # anchor is account-scoped upstream, so it may only be replayed on the same
+    # account; when the session fails over to a different account this diverges
+    # from ``account.id`` and the anchor must NOT be injected. Kept in sync with
+    # ``last_completed_response_id`` at every setter.
+    last_completed_response_account_id: str | None = None
     last_completed_input_prefix_fingerprint: str | None = None
     last_pending_tool_calls: dict[str, str] = field(default_factory=dict)
     durable_session_id: str | None = None

openspec/changes/fix-cross-account-previous-response-anchor/proposal.md

@@ -0,0 +1,65 @@
+# Fix cross-account compact previous_response_id anchor wedge
+
+## Why
+
+The HTTP-bridge "compact anchor" continuity optimization injects
+`previous_response_id = session.last_completed_response_id` and trims the
+already-stored history prefix so a follow-up turn only carries the new items.
+
+A `previous_response_id` is **account-scoped** on the upstream Responses API:
+only the account that created the response can resume it. The injection had no
+account-ownership check. When a Codex session fails over to a different account
+(e.g. the durable owner account became unavailable, or a durable session record
+is restored onto a new session bound to another account), the anchor points at a
+response the serving account never created. Upstream then accepts the WebSocket
+`response.create` but **never emits `response.created`**, and because the history
+was trimmed away there is no fallback. The per-bridge `response_create_gate`
+(a `Semaphore(1)`) stays held: the holder's own client eventually reports
+`stream disconnected before completion: idle timeout waiting for SSE`, and later
+requests on the same session time out as `codex-lb is temporarily overloaded
+during http_bridge_response_create_gate`.
+
+Observed live on 2026-07-10: sol sessions that fanned across three accounts
+repeatedly wedged on the same anchor (`resp_0bc0310d…`) even though the accounts
+had quota. Freeing the gate (the stuck-gate retire backstop) does not stop the
+recurrence because the session re-injects the same cross-account anchor.
+
+## What Changes
+
+- Track the account that owns `last_completed_response_id` on the bridge session
+  (`last_completed_response_account_id`), set in lockstep at every setter: the
+  real upstream `response.completed` path records the session's current account;
+  the durable-restore path records the durable owner account.
+- The session-level compact-anchor injection MUST only fire when the anchor's
+  owning account equals the session's serving account. Otherwise the request
+  falls through to a full-history resend (correct output, slightly more tokens),
+  never a cross-account `previous_response_id`.
+- The owner-forward recovery anchor injection gets the same guard. That rebind
+  runs with `allow_previous_response_recovery_rebind` /
+  `allow_bootstrap_owner_rebind`, which are explicitly allowed to bind the
+  session to an account other than the durable owner, so it is the second site
+  where a proxy-injected anchor can cross an account boundary.
+- Both declines emit a `cross_account_anchor_declined` bridge event naming the
+  injection site and the anchor's owning account, so the wedge family stays
+  observable from bridge event logs.
+
+The remaining proxy-side injection site, the pre-binding durable "fresh reattach"
+anchor, is already covered: setting `previous_response_id` there makes the durable
+owner a *required* account (`require_preferred_account`), so session creation and
+session reuse both refuse to serve that request on another account, and an
+unavailable owner degrades along the account-neutral full-resend path instead.
+
+## Impact
+
+- Affected specs: `sticky-session-operations`
+- Affected code: `_service/support.py` (`_HTTPBridgeSession`),
+  `_service/http_bridge/streaming.py` (both injection guards + durable-restore
+  owner), `_service/http_bridge/upstream_events.py` (completion owner),
+  `_service/http_bridge/mixin.py` (clear the owner with the anchor on an
+  account-changing reconnect).
+- Behavior: on cross-account failover, continuity is preserved by resending full
+  history instead of an unresolvable anchor. No client-visible protocol change.
+- Follow-ups (tracked separately, not in this change): the WebSocket-transport
+  anchor path (`websocket_session_anchor_injected`), and a proactive
+  `response.created` watchdog that replays the stored full-history payload if any
+  anchored request stalls.

openspec/changes/fix-cross-account-previous-response-anchor/specs/sticky-session-operations/spec.md

@@ -0,0 +1,58 @@
+## ADDED Requirements
+
+### Requirement: Compact previous_response_id anchors are account-scoped
+
+codex-lb MUST NOT inject a compact `previous_response_id` anchor whose owning account differs from the account that will serve the request.
+
+The HTTP-bridge compact-anchor injection reduces payload size by replacing
+already-stored history with a proxy-supplied `previous_response_id`, and a
+`previous_response_id` can only be resumed by the account that created it. The
+rule applies to every injection site that runs after the serving account is
+bound: the session-level anchor (`session.last_completed_response_id`) and the
+owner-forward recovery anchor (`durable_lookup.latest_response_id`, injected
+after a rebind that is allowed to land on a different account).
+
+codex-lb MUST record the account that owns `last_completed_response_id` whenever
+that value is set — from a real upstream `response.completed` (the session's
+current account) or from a durable-session restore (the durable owner account) —
+and keep the two in sync.
+
+Injection sites that run before the serving account is bound stay covered by the
+existing required-continuity-owner pin, which fails the request rather than
+serving a proxy-injected anchor on a different account.
+
+#### Scenario: Anchor injected when the serving account owns it
+
+- **WHEN** a Codex session follow-up turn is eligible for compact-anchor injection
+- **AND** the account that owns `last_completed_response_id` equals the session's
+  serving account
+- **THEN** codex-lb injects `previous_response_id = last_completed_response_id`
+  and trims the already-stored history prefix
+
+#### Scenario: Anchor skipped after cross-account failover
+
+- **WHEN** a Codex session follow-up turn is eligible for compact-anchor injection
+- **AND** the account that owns `last_completed_response_id` differs from the
+  session's serving account (for example the session failed over after the durable
+  owner account became unavailable)
+- **THEN** codex-lb MUST NOT inject the anchor
+- **AND** codex-lb resends the full history to the serving account so continuity
+  is preserved without an unresolvable `previous_response_id`
+- **AND** the request MUST NOT stall waiting for a `response.created` that upstream
+  will never send for an anchor the serving account does not own
+
+#### Scenario: Owner-forward recovery anchor skipped after a cross-account rebind
+
+- **WHEN** an owner forward fails and the local recovery rebind binds the session
+  to an account other than the durable record's owner
+- **AND** the durable record still carries a `latest_response_id` the recovery
+  request would otherwise anchor on
+- **THEN** codex-lb MUST NOT inject that anchor
+- **AND** the recovery request keeps its full input instead of a trimmed suffix
+
+#### Scenario: Declined anchors are observable
+
+- **WHEN** codex-lb declines a compact anchor because the serving account does not
+  own it
+- **THEN** codex-lb logs a `cross_account_anchor_declined` bridge event naming the
+  injection site, the anchor's owning account, and the full-history-resend outcome