diff --git a/app/modules/api_keys/repository.py b/app/modules/api_keys/repository.py index 8fb9b77a83..0c7df5b90a 100644 --- a/app/modules/api_keys/repository.py +++ b/app/modules/api_keys/repository.py @@ -380,6 +380,12 @@ async def delete(self, key_id: str) -> bool: async def commit(self) -> None: await self._session.commit() + async def update_last_used(self, key_id: str, *, commit: bool = True) -> None: + """Compatibility touch for maintenance and durability checks.""" + await self._session.execute(update(ApiKey).where(ApiKey.id == key_id).values(last_used_at=utcnow())) + if commit: + await self._session.commit() + async def rollback(self) -> None: await self._session.rollback() diff --git a/app/modules/proxy/_service/http_bridge/helpers.py b/app/modules/proxy/_service/http_bridge/helpers.py index 9501800a10..1e4695f0f5 100644 --- a/app/modules/proxy/_service/http_bridge/helpers.py +++ b/app/modules/proxy/_service/http_bridge/helpers.py @@ -731,16 +731,19 @@ def _http_bridge_eventless_precreated_deadline( or sent_at is None or request_state.response_id is not None or request_state.latency_response_created_ms is not None - or request_state.response_event_count != 0 or request_state.downstream_visible or request_state.last_downstream_sequence_number is not None ): return None # Non-response telemetry (for example ``codex.rate_limits``) may update # the generic activity marker, but it must not extend the response.create - # acknowledgement deadline. The eventless watchdog is intentionally - # anchored to the send time until a response-lifecycle event is observed. - return sent_at + min( + # acknowledgement deadline. Keep the send-time anchor until a matched + # response-lifecycle event exists; then use the latest lifecycle activity + # so deferred reasoning is not retired from the original send time. + deadline_anchor = sent_at + if request_state.response_event_count > 0 and request_state.last_upstream_activity_at is not None: + deadline_anchor = request_state.last_upstream_activity_at + return deadline_anchor + min( float(stuck_gate_retire_after_seconds), _HTTP_BRIDGE_EVENTLESS_RESPONSE_CREATED_MAX_SECONDS, ) diff --git a/openspec/changes/recover-codex-desktop-idle-bridge/specs/proxy-admission-control/spec.md b/openspec/changes/recover-codex-desktop-idle-bridge/specs/proxy-admission-control/spec.md index 293763926d..87e239f368 100644 --- a/openspec/changes/recover-codex-desktop-idle-bridge/specs/proxy-admission-control/spec.md +++ b/openspec/changes/recover-codex-desktop-idle-bridge/specs/proxy-admission-control/spec.md @@ -4,7 +4,7 @@ The proxy MUST retain the existing waiter-triggered retirement behavior for stale HTTP bridge response-create gate owners and MUST additionally enforce an owner-side deadline for a visible HTTP request whose current upstream `response.create` send remains completely eventless before `response.created`. The owner-side deadline MUST be measured from a monotonic timestamp recorded immediately before the current upstream send, MUST use the smaller of the configured stuck-gate retirement threshold and 60 seconds, MUST run without a second gate waiter, and MUST remain active when periodic SSE keepalives are disabled. -The owner-side watchdog MUST apply only while the request owns the response-create gate, awaits `response.created`, has neither a response id nor recorded `response.created` latency, has received no matched `response.*` lifecycle event, and has produced no downstream-visible output or sequence evidence. Non-response telemetry such as `codex.rate_limits` MUST NOT suppress this watchdog. Any matched `response.*` lifecycle event, response-created milestone, or downstream-visible evidence MUST suppress the owner-side watchdog and leave existing timeout behavior unchanged. +The owner-side watchdog MUST apply only while the request owns the response-create gate, awaits `response.created`, has neither a response id nor recorded `response.created` latency, and has produced no downstream-visible output or sequence evidence. Before any matched `response.*` lifecycle event, the deadline MUST remain anchored to the current upstream send; non-response telemetry such as `codex.rate_limits` MUST NOT suppress or extend it. If matched `response.*` lifecycle events arrive without `response.created`, the watchdog MUST remain armed and re-anchor from the most recent upstream response-lifecycle activity instead of the original send. A response-created milestone or downstream-visible evidence MUST suppress this narrow watchdog and leave existing timeout behavior unchanged. When the owner-side deadline expires, the proxy MUST recheck eligibility and emit a structured low-cardinality log and the existing stuck-retirement Prometheus counter. For requests that are not eligible for the bounded fresh-hard recovery defined by `recover-fresh-hard-bridge-timeouts`, it MUST terminally fail and settle every pending request exactly once, retire the whole bridge session, and MUST NOT transparently replay the timed-out request or move it to another account. An eligible fresh hard request MAY take that single bounded recovery path; if recovery is unavailable or fails, it MUST fall back to the same terminal fail-closed retirement. Neither path may write an account-health failure solely because `response.created` was missing. @@ -32,9 +32,16 @@ When the owner-side deadline expires, the proxy MUST recheck eligibility and emi - **THEN** the telemetry does not refresh or suppress the deadline - **AND** the proxy fails and retires the session -#### Scenario: Response lifecycle evidence suppresses the narrow watchdog +#### Scenario: Response lifecycle evidence re-anchors the missing-created watchdog -- **GIVEN** a pre-created request receives any matched `response.*` lifecycle event, a response id, recorded `response.created` latency, or downstream-visible output +- **GIVEN** a pre-created request receives matched `response.*` lifecycle events but no response id, recorded `response.created` latency, or downstream-visible output +- **WHEN** a new response-lifecycle event arrives +- **THEN** the watchdog deadline is re-anchored from the most recent upstream response-lifecycle activity +- **AND** the watchdog remains armed until response-created or downstream-visible evidence appears + +#### Scenario: Response-created or visible evidence suppresses the narrow watchdog + +- **GIVEN** a pre-created request receives a response id, recorded `response.created` latency, or downstream-visible output - **WHEN** the eventless owner-side deadline would otherwise elapse - **THEN** this watchdog does not retire the session - **AND** existing stream, request-budget, and waiter-triggered timeout behavior remains authoritative diff --git a/tests/unit/test_proxy_http_bridge.py b/tests/unit/test_proxy_http_bridge.py index b0a61903b6..59b15a44d1 100644 --- a/tests/unit/test_proxy_http_bridge.py +++ b/tests/unit/test_proxy_http_bridge.py @@ -224,7 +224,6 @@ def test_http_bridge_eventless_precreated_deadline_uses_current_send_and_client_ [ ("response_id", "resp-created"), ("latency_response_created_ms", 12), - ("response_event_count", 1), ("downstream_visible", True), ("last_downstream_sequence_number", 0), ("awaiting_response_created", False), @@ -249,6 +248,24 @@ def test_http_bridge_eventless_precreated_deadline_requires_narrow_owner_evidenc ) +def test_http_bridge_eventless_precreated_deadline_survives_reasoning_prelude_without_created() -> None: + request_state = _make_eventless_http_bridge_owner() + request_state.response_event_count = 3 + request_state.last_upstream_activity_at = 150.0 + request_state.upstream_model_output_seen = True + request_state.deferred_reasoning_downstream_texts.append( + 'data: {"type":"response.output_item.added","item":{"type":"reasoning"}}\n\n' + ) + + assert ( + http_bridge_helpers_module._http_bridge_eventless_precreated_deadline( + request_state, + stuck_gate_retire_after_seconds=300.0, + ) + == 150.0 + http_bridge_helpers_module._HTTP_BRIDGE_EVENTLESS_RESPONSE_CREATED_MAX_SECONDS + ) + + @pytest.mark.asyncio async def test_http_bridge_send_replaces_timestamp_and_wakes_existing_reader( monkeypatch: pytest.MonkeyPatch, @@ -20255,6 +20272,9 @@ async def close(self) -> None: owner.request_text = '{"type":"response.create","model":"gpt-5.6-sol","input":"hello"}' owner.preferred_account_id = "acc-bridge" owner.excluded_account_ids.add("acc-excluded") + if leading_telemetry: + owner.response_event_count = 1 + owner.upstream_model_output_seen = True sibling_queue: asyncio.Queue[str | None] = asyncio.Queue() sibling = proxy_service._WebSocketRequestState( request_id="req-created-sibling", @@ -20306,7 +20326,7 @@ async def close(self) -> None: assert owner.preferred_account_id == "acc-bridge" assert owner.excluded_account_ids == {"acc-excluded"} assert owner.replay_count == 0 - assert owner.response_event_count == 0 + assert owner.response_event_count == (1 if leading_telemetry else 0) if leading_telemetry: assert owner.latency_first_upstream_event_ms is not None retry_precreated.assert_awaited_once_with(session)