From 413e206eb3fd4bd6558c37f76fd504550d52320b Mon Sep 17 00:00:00 2001 From: Omnidia Agent Date: Sun, 12 Jul 2026 11:42:57 -0600 Subject: [PATCH 01/64] fix(proxy): preserve pool usage-limit semantics --- app/core/balancer/__init__.py | 4 ++ app/core/balancer/logic.py | 49 +++++++++++++++++++ app/core/errors.py | 13 ++++- .../proxy/_load_balancer/sticky_selection.py | 1 + .../proxy/_load_balancer/unbound_selection.py | 1 + app/modules/proxy/_service/codex_control.py | 7 ++- app/modules/proxy/_service/compact.py | 9 ++-- app/modules/proxy/_service/file_ops.py | 7 ++- .../proxy/_service/http_bridge/mixin.py | 25 ++-------- app/modules/proxy/_service/streaming/retry.py | 33 +++++++++++++ app/modules/proxy/_service/transcribe.py | 7 ++- app/modules/proxy/_service/websocket/mixin.py | 9 ++-- app/modules/proxy/load_balancer.py | 1 + app/modules/proxy/selection_errors.py | 37 ++++++++++++++ app/modules/proxy/service.py | 7 ++- .../integration/test_http_responses_bridge.py | 33 +++++++++++++ tests/integration/test_proxy_api_extended.py | 26 ++++++++++ tests/unit/test_load_balancer.py | 44 +++++++++++++++++ tests/unit/test_selection_errors.py | 35 +++++++++++++ 19 files changed, 298 insertions(+), 50 deletions(-) create mode 100644 app/modules/proxy/selection_errors.py create mode 100644 tests/unit/test_selection_errors.py diff --git a/app/core/balancer/__init__.py b/app/core/balancer/__init__.py index 2e29cad9db..39f73cad67 100644 --- a/app/core/balancer/__init__.py +++ b/app/core/balancer/__init__.py @@ -18,6 +18,7 @@ RoutingCostsByAccount, RoutingStrategy, SelectionResult, + USAGE_LIMIT_REACHED, TrafficClass, UsageWeightedOrder, account_status_for_permanent_failure, @@ -28,6 +29,7 @@ handle_quota_exceeded, handle_rate_limit, plausible_rate_limit_reset_at, + pool_usage_exhaustion, select_account, ) @@ -52,6 +54,7 @@ "RoutingStrategy", "TrafficClass", "SelectionResult", + "USAGE_LIMIT_REACHED", "UsageWeightedOrder", "account_status_for_permanent_failure", "configure_replica_salt", @@ -61,5 +64,6 @@ "handle_quota_exceeded", "handle_rate_limit", "plausible_rate_limit_reset_at", + "pool_usage_exhaustion", "select_account", ] diff --git a/app/core/balancer/logic.py b/app/core/balancer/logic.py index a3caf85afa..228e3229db 100644 --- a/app/core/balancer/logic.py +++ b/app/core/balancer/logic.py @@ -145,6 +145,45 @@ class AccountState: class SelectionResult: account: AccountState | None error_message: str | None + error_code: str | None = None + resets_at: int | None = None + + +USAGE_LIMIT_REACHED = "usage_limit_reached" + + +def pool_usage_exhaustion( + states: Iterable[AccountState], + *, + ignore_standard_quota: bool = False, +) -> SelectionResult | None: + """Describe pool-wide subscription exhaustion without parsing retry text.""" + if ignore_standard_quota: + return None + eligible = [ + state + for state in states + if state.status + not in ( + AccountStatus.PAUSED, + AccountStatus.REAUTH_REQUIRED, + AccountStatus.DEACTIVATED, + ) + ] + if not eligible or any( + state.status not in (AccountStatus.RATE_LIMITED, AccountStatus.QUOTA_EXCEEDED) + for state in eligible + ): + return None + + # AccountState.reset_at can contain a conservative local fallback. Until + # reset provenance is represented explicitly, omit it rather than expose + # a synthetic value as an upstream reset timestamp. + return SelectionResult( + account=None, + error_message="Usage limit reached", + error_code=USAGE_LIMIT_REACHED, + ) @dataclass(frozen=True, slots=True) @@ -529,6 +568,16 @@ def _backoff_expires_at(s: AccountState) -> float: return SelectionResult(None, f"opportunistic burn window closed: {reason}") available = opportunistic_available else: + usage_exhaustion = pool_usage_exhaustion( + all_states, + ignore_standard_quota=( + ignore_standard_quota + or bypass_quota_exceeded + or bypass_quota_exceeded_account_ids is not None + ), + ) + if usage_exhaustion is not None: + return usage_exhaustion reauth_required = [s for s in all_states if s.status == AccountStatus.REAUTH_REQUIRED] deactivated = [s for s in all_states if s.status == AccountStatus.DEACTIVATED] paused = [s for s in all_states if s.status == AccountStatus.PAUSED] diff --git a/app/core/errors.py b/app/core/errors.py index aa1e3ddc8e..3fb44d972a 100644 --- a/app/core/errors.py +++ b/app/core/errors.py @@ -47,8 +47,17 @@ class ResponseFailedEvent(TypedDict): PREVIOUS_RESPONSE_STALE_MESSAGE = "Upstream previous response anchor expired; retry without previous_response_id." -def openai_error(code: str, message: str, error_type: str = "server_error") -> OpenAIErrorEnvelope: - return {"error": {"message": message, "type": error_type, "code": code}} +def openai_error( + code: str, + message: str, + error_type: str = "server_error", + *, + resets_at: int | float | None = None, +) -> OpenAIErrorEnvelope: + detail: OpenAIErrorDetail = {"message": message, "type": error_type, "code": code} + if resets_at is not None: + detail["resets_at"] = int(resets_at) + return {"error": detail} def dashboard_error(code: str, message: str) -> DashboardErrorEnvelope: diff --git a/app/modules/proxy/_load_balancer/sticky_selection.py b/app/modules/proxy/_load_balancer/sticky_selection.py index 48d80c7b5d..58495dad8e 100644 --- a/app/modules/proxy/_load_balancer/sticky_selection.py +++ b/app/modules/proxy/_load_balancer/sticky_selection.py @@ -525,6 +525,7 @@ def _direct_error( probe_reservation_invalidated = True if result.account is None: error_message = result.error_message + selection_error_code = result.error_code or selection_error_code elif probe_reservation_invalidated: selected = None else: diff --git a/app/modules/proxy/_load_balancer/unbound_selection.py b/app/modules/proxy/_load_balancer/unbound_selection.py index 5731d018ee..f8c142c810 100644 --- a/app/modules/proxy/_load_balancer/unbound_selection.py +++ b/app/modules/proxy/_load_balancer/unbound_selection.py @@ -266,6 +266,7 @@ def _direct_error( selected_snapshot.reset_at = selected_reset_at elif result.account is None: error_message = result.error_message + selection_error_code = result.error_code or selection_error_code if probe_reservation_invalidated: selected_snapshot = None diff --git a/app/modules/proxy/_service/codex_control.py b/app/modules/proxy/_service/codex_control.py index 1e768bf761..6879c7c3c4 100644 --- a/app/modules/proxy/_service/codex_control.py +++ b/app/modules/proxy/_service/codex_control.py @@ -36,6 +36,7 @@ from app.modules.proxy.affinity import _AffinityPolicy, _sticky_key_for_codex_control_request from app.modules.proxy.helpers import _header_account_id, _normalize_error_code, _parse_openai_error from app.modules.proxy.load_balancer import AccountSelection, effective_account_concurrency_caps +from app.modules.proxy.selection_errors import selection_failure_response logger = logging.getLogger("app.modules.proxy.service") T = TypeVar("T") @@ -338,10 +339,8 @@ async def _finalize_success( if account is None: log_error_code = selection.error_code or "no_accounts" log_error_message = selection.error_message or "No active accounts available" - raise ProxyResponseError( - 503, - openai_error(log_error_code, log_error_message), - ) + status_code, error_payload = selection_failure_response(selection) + raise ProxyResponseError(status_code, error_payload) account_id_value = account.id async def _call_control(target: Account) -> CodexControlResponse: diff --git a/app/modules/proxy/_service/compact.py b/app/modules/proxy/_service/compact.py index 4623f61c46..9585e8c999 100644 --- a/app/modules/proxy/_service/compact.py +++ b/app/modules/proxy/_service/compact.py @@ -58,6 +58,7 @@ AccountSelection, effective_account_concurrency_caps, ) +from app.modules.proxy.selection_errors import selection_failure_response from app.modules.proxy.work_admission import AdmissionLease, WorkAdmissionController logger = logging.getLogger("app.modules.proxy.service") @@ -897,14 +898,10 @@ async def _call_compact( else: log_error_code = selection.error_code or "no_accounts" log_error_message = selection.error_message or "No active accounts available" - status_code = 429 if log_error_code == "account_response_create_cap" else 503 + status_code, error_payload = selection_failure_response(selection) raise ProxyResponseError( status_code, - openai_error( - log_error_code, - log_error_message, - error_type="rate_limit_error" if status_code == 429 else "server_error", - ), + error_payload, ) assert account is not None account_id_value = account.id diff --git a/app/modules/proxy/_service/file_ops.py b/app/modules/proxy/_service/file_ops.py index bc7cbf233b..47c4281cda 100644 --- a/app/modules/proxy/_service/file_ops.py +++ b/app/modules/proxy/_service/file_ops.py @@ -36,6 +36,7 @@ ) from app.modules.proxy.helpers import _header_account_id, _normalize_error_code, _parse_openai_error from app.modules.proxy.load_balancer import AccountSelection +from app.modules.proxy.selection_errors import selection_failure_response logger = logging.getLogger("app.modules.proxy.service") T = TypeVar("T") @@ -452,10 +453,8 @@ async def _proxy_files_call( if not account: log_error_code = selection.error_code or "no_accounts" log_error_message = selection.error_message or "No active accounts available" - raise ProxyResponseError( - 503, - openai_error(log_error_code, log_error_message), - ) + status_code, error_payload = selection_failure_response(selection) + raise ProxyResponseError(status_code, error_payload) account_id_value = account.id async def _call(target: Account) -> dict[str, JsonValue]: diff --git a/app/modules/proxy/_service/http_bridge/mixin.py b/app/modules/proxy/_service/http_bridge/mixin.py index d8dfeb0c51..60a8fc44de 100644 --- a/app/modules/proxy/_service/http_bridge/mixin.py +++ b/app/modules/proxy/_service/http_bridge/mixin.py @@ -54,6 +54,7 @@ ApiKeyData, ApiKeyRequestUsageBudget, ) +from app.modules.proxy.selection_errors import selection_failure_response from app.modules.proxy._service.api_key_usage import ( _API_KEY_RESERVATION_HEARTBEAT_SECONDS as _API_KEY_RESERVATION_HEARTBEAT_SECONDS, ) @@ -1775,7 +1776,6 @@ async def _create_http_bridge_session( preferred_account_id=preferred_account_id, selected_account_id=None, ) - is_local_account_cap = _is_local_account_cap_code(selection.error_code) if ( require_preferred_account and preferred_account_id is not None @@ -1783,16 +1783,8 @@ async def _create_http_bridge_session( and selection.error_code == CONTINUITY_OWNER_UNAVAILABLE ): raise _http_bridge_previous_response_owner_unavailable_error() - status_code = 429 if is_local_account_cap else 503 - error_type = "rate_limit_error" if status_code == 429 else "server_error" - raise ProxyResponseError( - status_code, - openai_error( - selection.error_code or "no_accounts", - selection.error_message or "No active accounts available", - error_type=error_type, - ), - ) + status_code, error_payload = selection_failure_response(selection) + raise ProxyResponseError(status_code, error_payload) if require_preferred_account and preferred_account_id is not None and account.id != preferred_account_id: await self._load_balancer.release_account_lease(selected_account_lease) selected_account_lease = None @@ -2171,15 +2163,8 @@ async def abandon_selected_account_retry(selected_account: Any) -> None: preferred_candidate_id = None continue record_selected_account_takeover(None) - status_code = 429 if _is_local_account_cap_code(selection.error_code) else 503 - raise ProxyResponseError( - status_code, - openai_error( - selection.error_code or "no_accounts", - selection.error_message or "No active accounts available", - error_type="rate_limit_error" if status_code == 429 else "server_error", - ), - ) + status_code, error_payload = selection_failure_response(selection) + raise ProxyResponseError(status_code, error_payload) if required_preferred_account_id is not None and account.id != required_preferred_account_id: if selection.lease is not None: await self._load_balancer.release_account_lease(selection.lease) diff --git a/app/modules/proxy/_service/streaming/retry.py b/app/modules/proxy/_service/streaming/retry.py index 35e12de908..ee954eb56a 100644 --- a/app/modules/proxy/_service/streaming/retry.py +++ b/app/modules/proxy/_service/streaming/retry.py @@ -71,6 +71,7 @@ is_upstream_model_capacity_error, ) from app.modules.proxy.load_balancer import AccountLease, AccountSelection +from app.modules.proxy.selection_errors import USAGE_LIMIT_REACHED, selection_failure_response _REQUEST_TRANSPORT_HTTP = "http" _REQUEST_TRANSPORT_WEBSOCKET = "websocket" @@ -1106,6 +1107,38 @@ async def _retry_account_model_rejection( account_id=last_account_model_rejection_account_id, ) return + if selection.error_code == USAGE_LIMIT_REACHED: + no_accounts_msg = selection.error_message or "Usage limit reached" + status_code, error_payload = selection_failure_response(selection) + await proxy._write_request_log( + account_id=None, + api_key=api_key, + request_id=request_id, + model=payload.model, + latency_ms=int((time.monotonic() - start) * 1000), + status="error", + error_code=USAGE_LIMIT_REACHED, + error_message=no_accounts_msg, + reasoning_effort=payload.reasoning.effort if payload.reasoning else None, + transport=request_transport, + upstream_transport=upstream_stream_transport, + service_tier=payload.service_tier, + requested_service_tier=payload.service_tier, + useragent=useragent, + useragent_group=useragent_group, + client_ip=client_ip, + ) + if propagate_http_errors: + raise ProxyResponseError(status_code, error_payload) + yield format_sse_event( + response_failed_event( + USAGE_LIMIT_REACHED, + no_accounts_msg, + error_type=USAGE_LIMIT_REACHED, + response_id=request_id, + ) + ) + return if selection.error_code in _LOCAL_ACCOUNT_CAP_ERROR_CODES: await _drain_pending_post_refresh_penalty_on_terminal(settlement) no_accounts_msg = selection.error_message or "Local account capacity is exhausted" diff --git a/app/modules/proxy/_service/transcribe.py b/app/modules/proxy/_service/transcribe.py index e2abbe4c1b..f27d08f4e6 100644 --- a/app/modules/proxy/_service/transcribe.py +++ b/app/modules/proxy/_service/transcribe.py @@ -30,6 +30,7 @@ from app.modules.proxy._service.support import _request_log_client_fields, _RequestLogFailureMetadata from app.modules.proxy.helpers import _header_account_id, _normalize_error_code, _parse_openai_error from app.modules.proxy.load_balancer import AccountSelection +from app.modules.proxy.selection_errors import selection_failure_response logger = logging.getLogger("app.modules.proxy.service") T = TypeVar("T") @@ -202,10 +203,8 @@ async def transcribe( if not account: log_error_code = selection.error_code or "no_accounts" log_error_message = selection.error_message or "No active accounts available" - raise ProxyResponseError( - 503, - openai_error(log_error_code, log_error_message), - ) + status_code, error_payload = selection_failure_response(selection) + raise ProxyResponseError(status_code, error_payload) account_id_value = account.id async def _call_transcribe(target: Account) -> dict[str, JsonValue]: diff --git a/app/modules/proxy/_service/websocket/mixin.py b/app/modules/proxy/_service/websocket/mixin.py index 4e292e0046..9f8c0029da 100644 --- a/app/modules/proxy/_service/websocket/mixin.py +++ b/app/modules/proxy/_service/websocket/mixin.py @@ -97,6 +97,7 @@ from app.modules.proxy._service.api_key_usage import ( _API_KEY_RESERVATION_HEARTBEAT_SECONDS as _API_KEY_RESERVATION_HEARTBEAT_SECONDS, ) +from app.modules.proxy.selection_errors import selection_failure_response from app.modules.proxy._service.compact import ( _service_tier_from_compact_payload as _service_tier_from_compact_payload, ) @@ -2456,7 +2457,7 @@ async def _heartbeat(remaining_seconds: float) -> None: len(exclude_account_ids), api_key is not None, ) - status_code = 429 if is_local_overload_error_code(error_code) else 503 + status_code, error_payload = selection_failure_response(selection) await proxy._emit_websocket_connect_failure( websocket, client_send_lock=client_send_lock, @@ -2464,11 +2465,7 @@ async def _heartbeat(remaining_seconds: float) -> None: api_key=api_key, request_state=request_state, status_code=status_code, - payload=openai_error( - error_code, - error_message, - error_type="rate_limit_error" if status_code == 429 else "server_error", - ), + payload=error_payload, error_code=error_code, error_message=error_message, ) diff --git a/app/modules/proxy/load_balancer.py b/app/modules/proxy/load_balancer.py index 27244b522a..c5b0b764aa 100644 --- a/app/modules/proxy/load_balancer.py +++ b/app/modules/proxy/load_balancer.py @@ -183,6 +183,7 @@ class AccountSelection: account: Account | None error_message: str | None error_code: str | None = None + resets_at: int | None = None lease: AccountLease | None = None catalog_omission_quota_admission: CatalogOmissionQuotaAdmission | None = None diff --git a/app/modules/proxy/selection_errors.py b/app/modules/proxy/selection_errors.py new file mode 100644 index 0000000000..b95139cbfc --- /dev/null +++ b/app/modules/proxy/selection_errors.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from typing import Protocol + +from app.core.errors import OpenAIErrorEnvelope, openai_error + +USAGE_LIMIT_REACHED = "usage_limit_reached" +LOCAL_ACCOUNT_CAP_ERROR_CODES = frozenset( + { + "account_response_create_cap", + "account_stream_cap", + } +) + + +class SelectionFailure(Protocol): + error_message: str | None + error_code: str | None + resets_at: int | None + + +def selection_failure_response(selection: SelectionFailure) -> tuple[int, OpenAIErrorEnvelope]: + code = selection.error_code or "no_accounts" + message = selection.error_message or "No active accounts available" + if code == USAGE_LIMIT_REACHED: + return ( + 429, + openai_error( + code, + message, + error_type=USAGE_LIMIT_REACHED, + resets_at=selection.resets_at, + ), + ) + if code in LOCAL_ACCOUNT_CAP_ERROR_CODES: + return 429, openai_error(code, message, error_type="rate_limit_error") + return 503, openai_error(code, message) diff --git a/app/modules/proxy/service.py b/app/modules/proxy/service.py index 3b6e75b4f1..e721d525f9 100644 --- a/app/modules/proxy/service.py +++ b/app/modules/proxy/service.py @@ -733,6 +733,7 @@ LoadBalancer, effective_account_concurrency_caps, ) +from app.modules.proxy.selection_errors import selection_failure_response from app.modules.proxy.repo_bundle import ProxyRepoFactory from app.modules.proxy.ring_membership import ( RingMembershipService, @@ -1034,10 +1035,8 @@ async def thread_goal_request( if account is None: log_error_code = selection.error_code or "no_accounts" log_error_message = selection.error_message or "No active accounts available" - raise ProxyResponseError( - 503, - openai_error(log_error_code, log_error_message), - ) + status_code, error_payload = selection_failure_response(selection) + raise ProxyResponseError(status_code, error_payload) account_id_value = account.id async def _call_goal(target: Account) -> dict[str, JsonValue]: diff --git a/tests/integration/test_http_responses_bridge.py b/tests/integration/test_http_responses_bridge.py index c2139f2e64..cd6f06e4e5 100644 --- a/tests/integration/test_http_responses_bridge.py +++ b/tests/integration/test_http_responses_bridge.py @@ -6571,6 +6571,39 @@ async def test_backend_responses_http_bridge_startup_error_omits_turn_state_head assert "x-codex-turn-state" not in response.headers +@pytest.mark.asyncio +async def test_backend_responses_http_bridge_pool_usage_exhaustion_returns_429(async_client, monkeypatch): + _install_bridge_settings(monkeypatch, enabled=True) + + async def fake_select_account_with_budget(*_args, **_kwargs): + return proxy_module.AccountSelection( + account=None, + error_message="Usage limit reached", + error_code="usage_limit_reached", + ) + + monkeypatch.setattr( + proxy_module.ProxyService, + "_select_account_with_budget", + fake_select_account_with_budget, + ) + + response = await async_client.post( + "/backend-api/codex/responses", + json={ + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": "hello", + "stream": True, + }, + ) + + assert response.status_code == 429 + assert response.json()["error"]["type"] == "usage_limit_reached" + assert response.json()["error"]["code"] == "usage_limit_reached" + assert "x-codex-turn-state" not in response.headers + + @pytest.mark.asyncio async def test_v1_responses_http_bridge_startup_error_omits_turn_state_header(async_client, monkeypatch): _install_bridge_settings(monkeypatch, enabled=True) diff --git a/tests/integration/test_proxy_api_extended.py b/tests/integration/test_proxy_api_extended.py index 2ed0938fd1..45cf44e34b 100644 --- a/tests/integration/test_proxy_api_extended.py +++ b/tests/integration/test_proxy_api_extended.py @@ -514,6 +514,32 @@ async def fake_select(*_args, **_kwargs): assert response.json()["error"]["code"] == "no_accounts" +@pytest.mark.asyncio +async def test_thread_goal_get_maps_pool_usage_exhaustion_for_codex(async_client, monkeypatch): + async def fake_select(*_args, **_kwargs): + return proxy_module.AccountSelection( + account=None, + error_message="Usage limit reached", + error_code="usage_limit_reached", + ) + + monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select) + + response = await async_client.post( + "/backend-api/codex/thread/goal/get", + json={"threadId": "019debd9-2372-7f23-92b9-9f34002a6355"}, + ) + + assert response.status_code == 429 + assert response.json() == { + "error": { + "message": "Usage limit reached", + "type": "usage_limit_reached", + "code": "usage_limit_reached", + } + } + + @pytest.mark.asyncio async def test_thread_goal_set_propagates_upstream_errors(async_client, monkeypatch): await _import_account(async_client, "acc_goal_set_error", "goal-set-error@example.com") diff --git a/tests/unit/test_load_balancer.py b/tests/unit/test_load_balancer.py index f2d67e1d9e..a53ab96754 100644 --- a/tests/unit/test_load_balancer.py +++ b/tests/unit/test_load_balancer.py @@ -676,6 +676,50 @@ def test_select_account_skips_rate_limited_until_reset(): assert result.account.account_id == "b" +def test_select_account_reports_pool_wide_usage_exhaustion_structurally(): + now = 1_700_000_000.0 + states = [ + AccountState("a", AccountStatus.RATE_LIMITED, reset_at=int(now + 60)), + AccountState("b", AccountStatus.QUOTA_EXCEEDED, reset_at=int(now + 3600)), + AccountState("paused", AccountStatus.PAUSED), + ] + + result = select_account(states, now=now) + + assert result.account is None + assert result.error_code == "usage_limit_reached" + assert result.error_message == "Usage limit reached" + assert result.resets_at is None + + +def test_select_account_does_not_misclassify_transient_backoff_as_usage_exhaustion(): + now = 1_700_000_000.0 + states = [ + AccountState("quota", AccountStatus.QUOTA_EXCEEDED, reset_at=int(now + 3600)), + AccountState( + "transient", + AccountStatus.ACTIVE, + error_count=3, + last_error_at=now, + ), + ] + + result = select_account(states, now=now, allow_backoff_fallback=False) + + assert result.account is None + assert result.error_code is None + + +def test_select_account_does_not_report_ignored_standard_quota_as_pool_exhaustion(): + now = 1_700_000_000.0 + state = AccountState("quota", AccountStatus.QUOTA_EXCEEDED, reset_at=int(now + 3600)) + + result = select_account([state], now=now, ignore_standard_quota=True) + + assert result.account is not None + assert result.account.account_id == "quota" + + def test_select_account_reports_paused_and_deactivated_without_reauth_reason(): states = [ AccountState("paused", AccountStatus.PAUSED, used_percent=5.0), diff --git a/tests/unit/test_selection_errors.py b/tests/unit/test_selection_errors.py new file mode 100644 index 0000000000..72349be1de --- /dev/null +++ b/tests/unit/test_selection_errors.py @@ -0,0 +1,35 @@ +from app.modules.proxy.load_balancer import AccountSelection +from app.modules.proxy.selection_errors import selection_failure_response + + +def test_pool_usage_exhaustion_is_codex_compatible_429(): + status, payload = selection_failure_response( + AccountSelection( + account=None, + error_message="Usage limit reached", + error_code="usage_limit_reached", + ) + ) + + assert status == 429 + assert payload == { + "error": { + "message": "Usage limit reached", + "type": "usage_limit_reached", + "code": "usage_limit_reached", + } + } + + +def test_unusable_pool_remains_no_accounts_503(): + status, payload = selection_failure_response( + AccountSelection( + account=None, + error_message="All accounts require re-authentication", + error_code=None, + ) + ) + + assert status == 503 + assert payload["error"]["type"] == "server_error" + assert payload["error"]["code"] == "no_accounts" From b3be9d52917b611db20240ae791d912143124dbd Mon Sep 17 00:00:00 2001 From: Omnidia Agent Date: Sun, 12 Jul 2026 11:43:55 -0600 Subject: [PATCH 02/64] style: normalize pool exhaustion imports --- app/core/balancer/__init__.py | 2 +- app/modules/proxy/_service/http_bridge/mixin.py | 3 +-- app/modules/proxy/_service/websocket/mixin.py | 3 +-- app/modules/proxy/service.py | 2 +- 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/app/core/balancer/__init__.py b/app/core/balancer/__init__.py index 39f73cad67..5b69cf3131 100644 --- a/app/core/balancer/__init__.py +++ b/app/core/balancer/__init__.py @@ -11,6 +11,7 @@ ROUTING_POLICY_PRESERVE, TRAFFIC_CLASS_FOREGROUND, TRAFFIC_CLASS_OPPORTUNISTIC, + USAGE_LIMIT_REACHED, AccountState, FailoverAction, ResetPreferenceWindow, @@ -18,7 +19,6 @@ RoutingCostsByAccount, RoutingStrategy, SelectionResult, - USAGE_LIMIT_REACHED, TrafficClass, UsageWeightedOrder, account_status_for_permanent_failure, diff --git a/app/modules/proxy/_service/http_bridge/mixin.py b/app/modules/proxy/_service/http_bridge/mixin.py index 60a8fc44de..9854e34f8e 100644 --- a/app/modules/proxy/_service/http_bridge/mixin.py +++ b/app/modules/proxy/_service/http_bridge/mixin.py @@ -54,7 +54,6 @@ ApiKeyData, ApiKeyRequestUsageBudget, ) -from app.modules.proxy.selection_errors import selection_failure_response from app.modules.proxy._service.api_key_usage import ( _API_KEY_RESERVATION_HEARTBEAT_SECONDS as _API_KEY_RESERVATION_HEARTBEAT_SECONDS, ) @@ -126,7 +125,6 @@ _await_cancelled_task, _call_with_supported_optional_kwargs, _estimated_lease_tokens_from_request_usage_budget, - _is_local_account_cap_code, _prefer_earlier_reset_window, _proxy_admission_wait_timeout_seconds, _raise_proxy_unavailable, @@ -220,6 +218,7 @@ DurableBridgeLookup, ) from app.modules.proxy.load_balancer import CONTINUITY_OWNER_UNAVAILABLE, AccountLease +from app.modules.proxy.selection_errors import selection_failure_response logger = logging.getLogger("app.modules.proxy.service") T = TypeVar("T") diff --git a/app/modules/proxy/_service/websocket/mixin.py b/app/modules/proxy/_service/websocket/mixin.py index 9f8c0029da..f3a725a9d1 100644 --- a/app/modules/proxy/_service/websocket/mixin.py +++ b/app/modules/proxy/_service/websocket/mixin.py @@ -77,7 +77,6 @@ ProcessNetworkRecovery, process_network_error_code, ) -from app.core.resilience.overload import is_local_overload_error_code from app.core.types import JsonValue from app.core.upstream_proxy import UpstreamProxyRouteError from app.core.utils.request_id import get_request_id, reset_request_id, set_request_id @@ -97,7 +96,6 @@ from app.modules.proxy._service.api_key_usage import ( _API_KEY_RESERVATION_HEARTBEAT_SECONDS as _API_KEY_RESERVATION_HEARTBEAT_SECONDS, ) -from app.modules.proxy.selection_errors import selection_failure_response from app.modules.proxy._service.compact import ( _service_tier_from_compact_payload as _service_tier_from_compact_payload, ) @@ -460,6 +458,7 @@ openai_validation_error, validate_model_access, ) +from app.modules.proxy.selection_errors import selection_failure_response from app.modules.proxy.tool_call_dedupe import ( mark_duplicate_tool_call_downstream_event, rewrite_parallel_tool_call_text, diff --git a/app/modules/proxy/service.py b/app/modules/proxy/service.py index e721d525f9..0353be603b 100644 --- a/app/modules/proxy/service.py +++ b/app/modules/proxy/service.py @@ -733,11 +733,11 @@ LoadBalancer, effective_account_concurrency_caps, ) -from app.modules.proxy.selection_errors import selection_failure_response from app.modules.proxy.repo_bundle import ProxyRepoFactory from app.modules.proxy.ring_membership import ( RingMembershipService, ) +from app.modules.proxy.selection_errors import selection_failure_response from app.modules.proxy.work_admission import WorkAdmissionController logger = logging.getLogger(__name__) From 76b64856708259a56e23a3dba98e99db79d7b1a8 Mon Sep 17 00:00:00 2001 From: Omnidia Agent Date: Sun, 12 Jul 2026 11:44:15 -0600 Subject: [PATCH 03/64] fix(proxy): preserve structured retry hints --- app/core/balancer/logic.py | 8 +++++++- tests/unit/test_load_balancer.py | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/app/core/balancer/logic.py b/app/core/balancer/logic.py index 228e3229db..1f94c46448 100644 --- a/app/core/balancer/logic.py +++ b/app/core/balancer/logic.py @@ -155,6 +155,7 @@ class SelectionResult: def pool_usage_exhaustion( states: Iterable[AccountState], *, + current: float, ignore_standard_quota: bool = False, ) -> SelectionResult | None: """Describe pool-wide subscription exhaustion without parsing retry text.""" @@ -179,9 +180,13 @@ def pool_usage_exhaustion( # AccountState.reset_at can contain a conservative local fallback. Until # reset provenance is represented explicitly, omit it rather than expose # a synthetic value as an upstream reset timestamp. + reset_candidates = [state.reset_at for state in eligible if state.reset_at is not None] + message = "Usage limit reached" + if reset_candidates: + message = _format_retry_hint(max(0.0, min(reset_candidates) - current)) return SelectionResult( account=None, - error_message="Usage limit reached", + error_message=message, error_code=USAGE_LIMIT_REACHED, ) @@ -570,6 +575,7 @@ def _backoff_expires_at(s: AccountState) -> float: else: usage_exhaustion = pool_usage_exhaustion( all_states, + current=current, ignore_standard_quota=( ignore_standard_quota or bypass_quota_exceeded diff --git a/tests/unit/test_load_balancer.py b/tests/unit/test_load_balancer.py index a53ab96754..d4231f1356 100644 --- a/tests/unit/test_load_balancer.py +++ b/tests/unit/test_load_balancer.py @@ -688,7 +688,7 @@ def test_select_account_reports_pool_wide_usage_exhaustion_structurally(): assert result.account is None assert result.error_code == "usage_limit_reached" - assert result.error_message == "Usage limit reached" + assert result.error_message == "Rate limit exceeded. Try again in 60s" assert result.resets_at is None From c7ccf293ca914116a2d9dec8715e191911bbfc0e Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sat, 18 Jul 2026 06:27:18 +0400 Subject: [PATCH 04/64] docs(openspec): cover pool usage exhaustion errors --- .../.openspec.yaml | 2 ++ .../report-pool-usage-exhaustion/proposal.md | 35 +++++++++++++++++++ .../specs/responses-api-compat/spec.md | 29 +++++++++++++++ .../report-pool-usage-exhaustion/tasks.md | 26 ++++++++++++++ 4 files changed, 92 insertions(+) create mode 100644 openspec/changes/report-pool-usage-exhaustion/.openspec.yaml create mode 100644 openspec/changes/report-pool-usage-exhaustion/proposal.md create mode 100644 openspec/changes/report-pool-usage-exhaustion/specs/responses-api-compat/spec.md create mode 100644 openspec/changes/report-pool-usage-exhaustion/tasks.md diff --git a/openspec/changes/report-pool-usage-exhaustion/.openspec.yaml b/openspec/changes/report-pool-usage-exhaustion/.openspec.yaml new file mode 100644 index 0000000000..0bd76e6186 --- /dev/null +++ b/openspec/changes/report-pool-usage-exhaustion/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-07-18 diff --git a/openspec/changes/report-pool-usage-exhaustion/proposal.md b/openspec/changes/report-pool-usage-exhaustion/proposal.md new file mode 100644 index 0000000000..41367d46a4 --- /dev/null +++ b/openspec/changes/report-pool-usage-exhaustion/proposal.md @@ -0,0 +1,35 @@ +## Why + +When every account eligible for a Responses request is exhausted by known pool +usage windows, codex-lb can currently collapse the selection failure into a +generic no-account/server-unavailable response. That hides the user-actionable +upstream condition from Codex/OpenAI-compatible clients and makes agents treat a +quota window as infrastructure failure. + +## What Changes + +- Preserve the stable `usage_limit_reached` code from account selection when the + whole eligible pool is exhausted by usage windows. +- Return HTTP `429` with an OpenAI-style error envelope whose + `error.code` and `error.type` are both `usage_limit_reached`. +- Preserve the selected reset hint as `error.resets_at` when account selection + has one, and use the same contract across HTTP, streaming, bridge, and + WebSocket selection-failure paths. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `responses-api-compat`: define the externally visible Responses error + contract for pool-wide usage exhaustion. + +## Impact + +- Affected code: account selection failure mapping and Responses proxy surfaces. +- Affected APIs: failure status/body for pool-wide usage exhaustion changes from + generic unavailable/no-account semantics to HTTP 429 `usage_limit_reached`. +- Configuration and schema: no changes. diff --git a/openspec/changes/report-pool-usage-exhaustion/specs/responses-api-compat/spec.md b/openspec/changes/report-pool-usage-exhaustion/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..fe1594b32f --- /dev/null +++ b/openspec/changes/report-pool-usage-exhaustion/specs/responses-api-compat/spec.md @@ -0,0 +1,29 @@ +## ADDED Requirements + +### Requirement: Pool usage exhaustion is reported as a usage-limit error + +The proxy MUST report pool-wide Responses usage exhaustion as a usage-limit +error. When every account eligible for a Responses request is exhausted by known +usage windows, the proxy MUST reject the request with HTTP `429` and an +OpenAI-style error envelope whose `error.code` and `error.type` are both +`usage_limit_reached`. If account selection has a reset timestamp for the +exhausted pool, the response envelope MUST include that timestamp as +`error.resets_at`. The proxy MUST NOT collapse this condition into generic +`no_accounts`, `server_error`, or HTTP `503` semantics. + +#### Scenario: Public Responses request exhausts the eligible usage pool + +- **WHEN** account selection for a public `/v1/responses` or + `/backend-api/codex/responses` request finds only usage-exhausted eligible + accounts +- **THEN** the response status is HTTP `429` +- **AND** the response body has `error.code = "usage_limit_reached"` +- **AND** the response body has `error.type = "usage_limit_reached"` +- **AND** any selected pool reset timestamp is surfaced as `error.resets_at` + +#### Scenario: Streaming selection failure preserves usage-limit semantics + +- **WHEN** a streaming Responses request cannot select an account because every + eligible account is usage-exhausted before downstream-visible output +- **THEN** the terminal error event uses `usage_limit_reached` +- **AND** clients do not receive a generic no-account/server-unavailable error diff --git a/openspec/changes/report-pool-usage-exhaustion/tasks.md b/openspec/changes/report-pool-usage-exhaustion/tasks.md new file mode 100644 index 0000000000..9dd87603ff --- /dev/null +++ b/openspec/changes/report-pool-usage-exhaustion/tasks.md @@ -0,0 +1,26 @@ +## 1. Error contract + +- [x] Preserve `usage_limit_reached` from pool-wide account selection failures. +- [x] Map pool-wide usage exhaustion to HTTP 429 with OpenAI-style + `error.code = "usage_limit_reached"` and + `error.type = "usage_limit_reached"`. +- [x] Preserve `error.resets_at` when account selection provides a reset hint. + +## 2. Proxy surfaces + +- [x] Apply the same selection-failure response helper across HTTP, streaming, + bridge, compact, file, transcription, WebSocket, and Codex-control paths. +- [x] Keep local capacity cap errors as 429 `rate_limit_error` responses rather + than weakening their existing contract. + +## 3. Regression coverage + +- [x] Add unit coverage for pool usage exhaustion selection and response mapping. +- [x] Add externally routed HTTP/streaming regressions for the 429 envelope. + +## 4. Validation + +- [x] Run focused pytest for selection, load balancer, and Responses proxy + regressions. +- [x] Run lint/type checks for touched Python files. +- [x] Validate the OpenSpec change strictly. From 4fa73013aa6e8a62bc3fbcac5f21e18b22950e4c Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sat, 18 Jul 2026 06:32:20 +0400 Subject: [PATCH 05/64] fix(ci): satisfy pool usage PR gates --- .all-contributorsrc | 10 ++++++++++ app/core/balancer/logic.py | 7 ++----- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/.all-contributorsrc b/.all-contributorsrc index cd7180289b..6e631698b4 100644 --- a/.all-contributorsrc +++ b/.all-contributorsrc @@ -1082,6 +1082,16 @@ "contributions": [ "code" ] + }, + { + "login": "glopyglerky", + "name": "glopyglerky", + "avatar_url": "https://avatars.githubusercontent.com/u/189872235?v=4", + "profile": "https://github.com/glopyglerky", + "contributions": [ + "code", + "test" + ] } ], "contributorsPerLine": 7, diff --git a/app/core/balancer/logic.py b/app/core/balancer/logic.py index 1f94c46448..5c493d26a4 100644 --- a/app/core/balancer/logic.py +++ b/app/core/balancer/logic.py @@ -172,8 +172,7 @@ def pool_usage_exhaustion( ) ] if not eligible or any( - state.status not in (AccountStatus.RATE_LIMITED, AccountStatus.QUOTA_EXCEEDED) - for state in eligible + state.status not in (AccountStatus.RATE_LIMITED, AccountStatus.QUOTA_EXCEEDED) for state in eligible ): return None @@ -577,9 +576,7 @@ def _backoff_expires_at(s: AccountState) -> float: all_states, current=current, ignore_standard_quota=( - ignore_standard_quota - or bypass_quota_exceeded - or bypass_quota_exceeded_account_ids is not None + ignore_standard_quota or bypass_quota_exceeded or bypass_quota_exceeded_account_ids is not None ), ) if usage_exhaustion is not None: From fa1dd12bf031aa78bc8e6f4e8583abfac0a9f866 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sat, 18 Jul 2026 07:02:38 +0400 Subject: [PATCH 06/64] fix(balancer): tighten usage-limit exhaustion reporting --- app/core/balancer/logic.py | 35 ++++++++----- .../proxy/_load_balancer/sticky_selection.py | 9 ++++ .../proxy/_load_balancer/unbound_selection.py | 6 +++ app/modules/proxy/load_balancer.py | 18 ++++++- tests/unit/test_load_balancer.py | 49 ++++++++++++++++++- tests/unit/test_load_balancer_concurrency.py | 17 +++++++ 6 files changed, 119 insertions(+), 15 deletions(-) diff --git a/app/core/balancer/logic.py b/app/core/balancer/logic.py index 5c493d26a4..0b3abe3282 100644 --- a/app/core/balancer/logic.py +++ b/app/core/balancer/logic.py @@ -157,36 +157,48 @@ def pool_usage_exhaustion( *, current: float, ignore_standard_quota: bool = False, + ignore_standard_quota_account_ids: Collection[str] | None = None, ) -> SelectionResult | None: """Describe pool-wide subscription exhaustion without parsing retry text.""" if ignore_standard_quota: return None + ignored_account_ids = set(ignore_standard_quota_account_ids or ()) + + def _usage_exhausted(state: AccountState) -> bool: + if state.status == AccountStatus.QUOTA_EXCEEDED: + return True + if state.status != AccountStatus.RATE_LIMITED: + return False + usage_values = (state.used_percent, state.secondary_used_percent) + return any(value is not None and float(value) >= 100.0 for value in usage_values) + eligible = [ state for state in states - if state.status + if not state.ignore_standard_quota + and state.account_id not in ignored_account_ids + and state.status not in ( AccountStatus.PAUSED, AccountStatus.REAUTH_REQUIRED, AccountStatus.DEACTIVATED, ) ] - if not eligible or any( - state.status not in (AccountStatus.RATE_LIMITED, AccountStatus.QUOTA_EXCEEDED) for state in eligible - ): + if not eligible or any(not _usage_exhausted(state) for state in eligible): return None - # AccountState.reset_at can contain a conservative local fallback. Until - # reset provenance is represented explicitly, omit it rather than expose - # a synthetic value as an upstream reset timestamp. + # Only usage-proven exhausted accounts reach this branch; surface their + # earliest known reset so the structured 429 can satisfy the API contract. reset_candidates = [state.reset_at for state in eligible if state.reset_at is not None] + resets_at = int(min(reset_candidates)) if reset_candidates else None message = "Usage limit reached" - if reset_candidates: - message = _format_retry_hint(max(0.0, min(reset_candidates) - current)) + if resets_at is not None: + message = _format_retry_hint(max(0.0, resets_at - current)) return SelectionResult( account=None, error_message=message, error_code=USAGE_LIMIT_REACHED, + resets_at=resets_at, ) @@ -575,9 +587,8 @@ def _backoff_expires_at(s: AccountState) -> float: usage_exhaustion = pool_usage_exhaustion( all_states, current=current, - ignore_standard_quota=( - ignore_standard_quota or bypass_quota_exceeded or bypass_quota_exceeded_account_ids is not None - ), + ignore_standard_quota=ignore_standard_quota or bypass_quota_exceeded, + ignore_standard_quota_account_ids=bypass_account_ids, ) if usage_exhaustion is not None: return usage_exhaustion diff --git a/app/modules/proxy/_load_balancer/sticky_selection.py b/app/modules/proxy/_load_balancer/sticky_selection.py index 58495dad8e..7d21cb9ea6 100644 --- a/app/modules/proxy/_load_balancer/sticky_selection.py +++ b/app/modules/proxy/_load_balancer/sticky_selection.py @@ -226,6 +226,7 @@ class StickySelectionOutcome(Generic[SelectionInputsT]): selected_lease: AccountLease | None error_message: str | None error_code: str | None + resets_at: int | None = None disposition: StickySelectionDisposition = "shared_result" @@ -266,6 +267,7 @@ async def run_sticky_selection_path( selected_lease: AccountLease | None = None error_message: str | None = None selection_error_code: str | None = None + selection_resets_at: int | None = None def _direct_error( *, @@ -405,9 +407,11 @@ def _direct_error( sticky_outcome = _StickySelectionOutcome(selection=SelectionResult(None, None)) if hard_sticky and not selection_states: selection_error_code = "hard_affinity_saturated" + selection_resets_at = None result = SelectionResult(None, "Hard affinity owner account is unavailable") elif not selection_states and states: selection_error_code = _account_cap_error_code(lease_kind) + selection_resets_at = None result = SelectionResult(None, _account_cap_error_message(lease_kind, caps)) logger.warning( "Account cap exhausted during sticky selection lease_kind=%s reason=%s candidates=%s", @@ -435,14 +439,17 @@ def _direct_error( ) if result.account is None: selection_error_code = "hard_affinity_saturated" + selection_resets_at = None result = SelectionResult( None, result.error_message or "Hard affinity owner account is unavailable", ) else: selection_error_code = None + selection_resets_at = None else: selection_error_code = None + selection_resets_at = None try: async with owner._repo_factory() as repos: sticky_outcome = await owner._select_with_stickiness( @@ -526,6 +533,7 @@ def _direct_error( if result.account is None: error_message = result.error_message selection_error_code = result.error_code or selection_error_code + selection_resets_at = result.resets_at or selection_resets_at elif probe_reservation_invalidated: selected = None else: @@ -857,6 +865,7 @@ def _direct_error( selected_lease=selected_lease, error_message=error_message, error_code=selection_error_code, + resets_at=selection_resets_at, ) diff --git a/app/modules/proxy/_load_balancer/unbound_selection.py b/app/modules/proxy/_load_balancer/unbound_selection.py index f8c142c810..f69b91957c 100644 --- a/app/modules/proxy/_load_balancer/unbound_selection.py +++ b/app/modules/proxy/_load_balancer/unbound_selection.py @@ -79,6 +79,7 @@ class UnboundSelectionOutcome(Generic[SelectionInputsT]): selected_lease: AccountLease | None error_message: str | None error_code: str | None + resets_at: int | None = None disposition: str = "shared_result" @@ -110,6 +111,7 @@ async def run_unbound_selection_path( selected_lease: AccountLease | None = None error_message: str | None = None selection_error_code: str | None = None + selection_resets_at: int | None = None def _direct_error( *, @@ -161,6 +163,7 @@ def _direct_error( ) if not selection_states and states: selection_error_code = _account_cap_error_code(lease_kind) + selection_resets_at = None error_message = _account_cap_error_message(lease_kind, caps) result = SelectionResult(None, error_message) logger.warning( @@ -172,6 +175,7 @@ def _direct_error( _record_account_cap_rejection(lease_kind) else: selection_error_code = None + selection_resets_at = None result = _select_account_preferring_budget_safe( selection_states, prefer_earlier_reset=prefer_earlier_reset_accounts, @@ -267,6 +271,7 @@ def _direct_error( elif result.account is None: error_message = result.error_message selection_error_code = result.error_code or selection_error_code + selection_resets_at = result.resets_at or selection_resets_at if probe_reservation_invalidated: selected_snapshot = None @@ -443,4 +448,5 @@ def _direct_error( selected_lease=selected_lease, error_message=error_message, error_code=selection_error_code, + resets_at=selection_resets_at, ) diff --git a/app/modules/proxy/load_balancer.py b/app/modules/proxy/load_balancer.py index c5b0b764aa..3a43923e33 100644 --- a/app/modules/proxy/load_balancer.py +++ b/app/modules/proxy/load_balancer.py @@ -22,6 +22,7 @@ ROUTING_POLICY_PRESERVE, TRAFFIC_CLASS_FOREGROUND, TRAFFIC_CLASS_OPPORTUNISTIC, + USAGE_LIMIT_REACHED, AccountState, ResetPreferenceWindow, RoutingCostsByAccount, @@ -575,6 +576,7 @@ async def load_selection_inputs() -> _SelectionInputs: error_message: str | None = None selected_lease: AccountLease | None = None selection_error_code: str | None = None + selection_resets_at: int | None = None legacy_existing_account_id: str | None = None if sticky_source == "session_header" and legacy_sticky_key is not None: async with self._repo_factory() as repos: @@ -646,6 +648,7 @@ async def load_selection_inputs() -> _SelectionInputs: selected_lease = unbound_outcome.selected_lease error_message = unbound_outcome.error_message selection_error_code = unbound_outcome.error_code + selection_resets_at = unbound_outcome.resets_at if unbound_outcome.disposition == "direct_error": return AccountSelection( account=None, @@ -690,6 +693,7 @@ async def load_selection_inputs() -> _SelectionInputs: selected_lease = sticky_outcome.selected_lease error_message = sticky_outcome.error_message selection_error_code = sticky_outcome.error_code + selection_resets_at = sticky_outcome.resets_at if sticky_outcome.disposition == "direct_error": return AccountSelection( account=None, @@ -736,7 +740,12 @@ async def load_selection_inputs() -> _SelectionInputs: and (selection_inputs.accounts or selection_inputs.error_code is not None) ): set_normal() - return AccountSelection(account=None, error_message=error_message, error_code=selection_error_code) + return AccountSelection( + account=None, + error_message=error_message, + error_code=selection_error_code, + resets_at=selection_resets_at, + ) if not circuit_breaker_open: set_normal() logger.info( @@ -1191,6 +1200,13 @@ async def check_opportunistic_admission( ignore_standard_quota=False, ) if result.account is None: + if result.error_code == USAGE_LIMIT_REACHED: + return AccountSelection( + account=None, + error_message=result.error_message, + error_code=result.error_code, + resets_at=result.resets_at, + ) return AccountSelection( account=None, error_message=result.error_message, diff --git a/tests/unit/test_load_balancer.py b/tests/unit/test_load_balancer.py index d4231f1356..d5dd5eca90 100644 --- a/tests/unit/test_load_balancer.py +++ b/tests/unit/test_load_balancer.py @@ -679,7 +679,7 @@ def test_select_account_skips_rate_limited_until_reset(): def test_select_account_reports_pool_wide_usage_exhaustion_structurally(): now = 1_700_000_000.0 states = [ - AccountState("a", AccountStatus.RATE_LIMITED, reset_at=int(now + 60)), + AccountState("a", AccountStatus.RATE_LIMITED, used_percent=100.0, reset_at=int(now + 60)), AccountState("b", AccountStatus.QUOTA_EXCEEDED, reset_at=int(now + 3600)), AccountState("paused", AccountStatus.PAUSED), ] @@ -689,7 +689,21 @@ def test_select_account_reports_pool_wide_usage_exhaustion_structurally(): assert result.account is None assert result.error_code == "usage_limit_reached" assert result.error_message == "Rate limit exceeded. Try again in 60s" - assert result.resets_at is None + assert result.resets_at == int(now + 60) + + +def test_select_account_does_not_treat_generic_rate_limit_as_usage_exhaustion(): + now = 1_700_000_000.0 + states = [ + AccountState("a", AccountStatus.RATE_LIMITED, used_percent=5.0, reset_at=int(now + 60)), + AccountState("b", AccountStatus.RATE_LIMITED, secondary_used_percent=10.0, reset_at=int(now + 120)), + ] + + result = select_account(states, now=now) + + assert result.account is None + assert result.error_code is None + assert result.error_message == "No available accounts" def test_select_account_does_not_misclassify_transient_backoff_as_usage_exhaustion(): @@ -720,6 +734,37 @@ def test_select_account_does_not_report_ignored_standard_quota_as_pool_exhaustio assert result.account.account_id == "quota" +def test_select_account_excludes_per_account_standard_quota_bypass_from_pool_exhaustion(): + now = 1_700_000_000.0 + state = AccountState( + "quota", + AccountStatus.QUOTA_EXCEEDED, + reset_at=int(now + 3600), + cooldown_until=now + 30, + ignore_standard_quota=True, + ) + + result = select_account([state], now=now) + + assert result.account is None + assert result.error_code is None + + +def test_select_account_excludes_scoped_standard_quota_bypass_from_pool_exhaustion(): + now = 1_700_000_000.0 + state = AccountState( + "quota", + AccountStatus.QUOTA_EXCEEDED, + reset_at=int(now + 3600), + cooldown_until=now + 30, + ) + + result = select_account([state], now=now, bypass_quota_exceeded_account_ids={"quota"}) + + assert result.account is None + assert result.error_code is None + + def test_select_account_reports_paused_and_deactivated_without_reauth_reason(): states = [ AccountState("paused", AccountStatus.PAUSED, used_percent=5.0), diff --git a/tests/unit/test_load_balancer_concurrency.py b/tests/unit/test_load_balancer_concurrency.py index f179ccf2ac..2bea81aa31 100644 --- a/tests/unit/test_load_balancer_concurrency.py +++ b/tests/unit/test_load_balancer_concurrency.py @@ -120,6 +120,23 @@ async def test_account_lease_uses_explicit_dashboard_cap_snapshot_not_startup_en assert third is None +@pytest.mark.asyncio +async def test_opportunistic_selection_preserves_usage_limit_exhaustion_error() -> None: + account = _make_account("acc-opportunistic-usage-exhausted") + account.status = AccountStatus.QUOTA_EXCEEDED + account.reset_at = int(time.time() + 300) + balancer = LoadBalancer(lambda: _repo_factory(_StubAccountsRepository([account]), _StubUsageRepository({}, {}))) + + result = await balancer.select_account( + routing_strategy="usage_weighted", + traffic_class=load_balancer_module.TRAFFIC_CLASS_OPPORTUNISTIC, + ) + + assert result.account is None + assert result.error_code == "usage_limit_reached" + assert result.resets_at == account.reset_at + + class _StubAccountsRepository: def __init__(self, accounts: list[Account]) -> None: self._accounts = accounts From ed61ed18c9d45ad52161ec998e9b3db8e1179d65 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sat, 18 Jul 2026 07:34:21 +0400 Subject: [PATCH 07/64] fix(balancer): preserve usage-limit reset metadata --- app/core/balancer/logic.py | 23 ++++- app/core/errors.py | 3 +- app/modules/proxy/_service/streaming/retry.py | 1 + app/modules/proxy/api.py | 12 +++ tests/unit/test_load_balancer.py | 21 +++++ tests/unit/test_openai_errors.py | 12 +++ tests/unit/test_proxy_utils.py | 83 +++++++++++++++++++ 7 files changed, 151 insertions(+), 4 deletions(-) diff --git a/app/core/balancer/logic.py b/app/core/balancer/logic.py index 0b3abe3282..a2dc62da31 100644 --- a/app/core/balancer/logic.py +++ b/app/core/balancer/logic.py @@ -172,6 +172,22 @@ def _usage_exhausted(state: AccountState) -> bool: usage_values = (state.used_percent, state.secondary_used_percent) return any(value is not None and float(value) >= 100.0 for value in usage_values) + def _usage_exhausted_reset_candidates(state: AccountState) -> list[float]: + if state.status == AccountStatus.QUOTA_EXCEEDED: + return [float(state.reset_at)] if state.reset_at is not None else [] + if state.status != AccountStatus.RATE_LIMITED: + return [] + candidates: list[float] = [] + if state.used_percent is not None and float(state.used_percent) >= 100.0 and state.reset_at is not None: + candidates.append(float(state.reset_at)) + if ( + state.secondary_used_percent is not None + and float(state.secondary_used_percent) >= 100.0 + and state.secondary_reset_at is not None + ): + candidates.append(float(state.secondary_reset_at)) + return candidates + eligible = [ state for state in states @@ -187,9 +203,10 @@ def _usage_exhausted(state: AccountState) -> bool: if not eligible or any(not _usage_exhausted(state) for state in eligible): return None - # Only usage-proven exhausted accounts reach this branch; surface their - # earliest known reset so the structured 429 can satisfy the API contract. - reset_candidates = [state.reset_at for state in eligible if state.reset_at is not None] + # Only usage-proven exhausted accounts reach this branch; surface the + # earliest reset for an actually exhausted window so the structured 429 + # does not retry a secondary-window exhaustion at the primary reset time. + reset_candidates = [reset_at for state in eligible for reset_at in _usage_exhausted_reset_candidates(state)] resets_at = int(min(reset_candidates)) if reset_candidates else None message = "Usage limit reached" if resets_at is not None: diff --git a/app/core/errors.py b/app/core/errors.py index 3fb44d972a..bf13b215fd 100644 --- a/app/core/errors.py +++ b/app/core/errors.py @@ -114,9 +114,10 @@ def response_failed_event( response_id: str | None = None, created_at: int | None = None, error_param: str | None = None, + resets_at: int | float | None = None, incomplete_details: dict[str, str] | None = None, ) -> ResponseFailedEvent: - error = openai_error(code, message, error_type)["error"] + error = openai_error(code, message, error_type, resets_at=resets_at)["error"] if error_param: error["param"] = error_param if created_at is None: diff --git a/app/modules/proxy/_service/streaming/retry.py b/app/modules/proxy/_service/streaming/retry.py index ee954eb56a..1022236ec9 100644 --- a/app/modules/proxy/_service/streaming/retry.py +++ b/app/modules/proxy/_service/streaming/retry.py @@ -1136,6 +1136,7 @@ async def _retry_account_model_rejection( no_accounts_msg, error_type=USAGE_LIMIT_REACHED, response_id=request_id, + resets_at=selection.resets_at, ) ) return diff --git a/app/modules/proxy/api.py b/app/modules/proxy/api.py index fcbc012a8a..c653babb4f 100644 --- a/app/modules/proxy/api.py +++ b/app/modules/proxy/api.py @@ -255,6 +255,7 @@ WarmupSkippedAccount, WarmupSubmittedAccount, ) +from app.modules.proxy.selection_errors import USAGE_LIMIT_REACHED from app.modules.proxy.types import ( CreditStatusDetailsData, RateLimitResetCreditsData, @@ -6357,6 +6358,17 @@ async def _opportunistic_admission_denial( ) if selection.account is not None: return None + if selection.error_code == USAGE_LIMIT_REACHED: + return _logged_error_json_response( + request, + 429, + openai_error( + USAGE_LIMIT_REACHED, + selection.error_message or "Usage limit reached", + error_type=USAGE_LIMIT_REACHED, + resets_at=selection.resets_at, + ), + ) message = selection.error_message or "opportunistic burn window closed" if not message.startswith("opportunistic burn window closed"): message = f"opportunistic burn window closed: {message}" diff --git a/tests/unit/test_load_balancer.py b/tests/unit/test_load_balancer.py index d5dd5eca90..bcde68f528 100644 --- a/tests/unit/test_load_balancer.py +++ b/tests/unit/test_load_balancer.py @@ -692,6 +692,27 @@ def test_select_account_reports_pool_wide_usage_exhaustion_structurally(): assert result.resets_at == int(now + 60) +def test_select_account_reports_secondary_usage_exhaustion_reset(): + now = 1_700_000_000.0 + states = [ + AccountState( + "a", + AccountStatus.RATE_LIMITED, + used_percent=10.0, + secondary_used_percent=100.0, + reset_at=int(now + 60), + secondary_reset_at=int(now + 3600), + ) + ] + + result = select_account(states, now=now) + + assert result.account is None + assert result.error_code == "usage_limit_reached" + assert result.error_message == "Rate limit exceeded. Try again in 300s" + assert result.resets_at == int(now + 3600) + + def test_select_account_does_not_treat_generic_rate_limit_as_usage_exhaustion(): now = 1_700_000_000.0 states = [ diff --git a/tests/unit/test_openai_errors.py b/tests/unit/test_openai_errors.py index 2534ead7c7..70ec2d136f 100644 --- a/tests/unit/test_openai_errors.py +++ b/tests/unit/test_openai_errors.py @@ -29,6 +29,18 @@ def test_response_failed_event_accepts_incomplete_details(): assert response.get("incomplete_details") == {"reason": "max_output_tokens"} +def test_response_failed_event_preserves_reset_hint(): + event = response_failed_event( + "usage_limit_reached", + "Rate limit exceeded. Try again in 1h", + error_type="usage_limit_reached", + response_id="resp_1", + resets_at=1_700_003_600, + ) + + assert event["response"]["error"]["resets_at"] == 1_700_003_600 + + def test_previous_response_not_found_classifier_covers_openai_shapes(): assert is_previous_response_not_found_error( code="previous_response_not_found", diff --git a/tests/unit/test_proxy_utils.py b/tests/unit/test_proxy_utils.py index 31910f68ca..a9a2f06bbd 100644 --- a/tests/unit/test_proxy_utils.py +++ b/tests/unit/test_proxy_utils.py @@ -2665,6 +2665,49 @@ async def test_opportunistic_admission_uses_api_key_enforced_model(): ) +@pytest.mark.asyncio +async def test_opportunistic_admission_preserves_usage_limit_denial(): + api_key = ApiKeyData( + id="key_opportunistic_usage_limit", + name="opportunistic usage limit", + key_prefix="sk-opportunistic", + allowed_models=None, + enforced_model=None, + enforced_reasoning_effort=None, + enforced_service_tier=None, + traffic_class=proxy_api.TRAFFIC_CLASS_OPPORTUNISTIC, + expires_at=None, + is_active=True, + created_at=utcnow(), + last_used_at=None, + ) + selection = AccountSelection( + account=None, + error_message="Rate limit exceeded. Try again in 1h", + error_code="usage_limit_reached", + resets_at=1_700_003_600, + ) + service = SimpleNamespace(check_opportunistic_admission=AsyncMock(return_value=selection)) + context = SimpleNamespace(service=service) + request = Request({"type": "http", "method": "GET", "path": "/v1/opportunistic/admission", "headers": []}) + + response = await proxy_api._opportunistic_admission_denial( + request, + cast(proxy_api.ProxyContext, context), + api_key, + model="gpt-5.1", + ) + + assert response is not None + assert response.status_code == 429 + body = json.loads(bytes(response.body)) + assert body["error"]["code"] == "usage_limit_reached" + assert body["error"]["type"] == "usage_limit_reached" + assert body["error"]["message"] == "Rate limit exceeded. Try again in 1h" + assert body["error"]["resets_at"] == 1_700_003_600 + assert "Retry-After" not in response.headers + + @pytest.mark.asyncio async def test_opportunistic_admission_scopes_single_account_to_selected_account(monkeypatch): settings = _make_proxy_settings() @@ -11440,6 +11483,46 @@ async def test_stream_responses_propagates_selection_error_code(monkeypatch): assert request_logs.calls[0]["error_code"] == "additional_quota_data_unavailable" +@pytest.mark.asyncio +async def test_stream_responses_preserves_usage_limit_reset_hint(monkeypatch): + settings = _make_proxy_settings() + request_logs = _RequestLogsRecorder() + service = proxy_service.ProxyService(_repo_factory(request_logs)) + + monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache(settings)) + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr( + service._load_balancer, + "select_account", + AsyncMock( + return_value=AccountSelection( + account=None, + error_message="Rate limit exceeded. Try again in 1h", + error_code="usage_limit_reached", + resets_at=1_700_003_600, + ) + ), + ) + + payload = ResponsesRequest.model_validate( + { + "model": "gpt-5.1", + "instructions": "hi", + "input": [], + "stream": True, + } + ) + + chunks = [chunk async for chunk in service.stream_responses(payload, {"session_id": "sid-usage-limit"})] + + event = json.loads(chunks[0].split("data: ", 1)[1]) + assert event["response"]["error"]["code"] == "usage_limit_reached" + assert event["response"]["error"]["type"] == "usage_limit_reached" + assert event["response"]["error"]["resets_at"] == 1_700_003_600 + assert await service.drain_persistence_tasks(timeout_seconds=1) + assert request_logs.calls[0]["error_code"] == "usage_limit_reached" + + @pytest.mark.asyncio async def test_stream_with_retry_keeps_sse_alive_while_account_capacity_recovers(monkeypatch): settings = _make_proxy_settings() From 364f91210199cd36b08b7304c3ac95c3959492f4 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sat, 18 Jul 2026 08:05:27 +0400 Subject: [PATCH 08/64] fix(balancer): require usage evidence for pool limits --- app/core/balancer/logic.py | 8 +-- app/modules/proxy/_service/websocket/mixin.py | 11 ++-- tests/unit/test_load_balancer.py | 15 ++++- tests/unit/test_load_balancer_concurrency.py | 11 +++- tests/unit/test_proxy_utils.py | 57 +++++++++++++++++++ 5 files changed, 85 insertions(+), 17 deletions(-) diff --git a/app/core/balancer/logic.py b/app/core/balancer/logic.py index a2dc62da31..c69d1a781c 100644 --- a/app/core/balancer/logic.py +++ b/app/core/balancer/logic.py @@ -165,17 +165,13 @@ def pool_usage_exhaustion( ignored_account_ids = set(ignore_standard_quota_account_ids or ()) def _usage_exhausted(state: AccountState) -> bool: - if state.status == AccountStatus.QUOTA_EXCEEDED: - return True - if state.status != AccountStatus.RATE_LIMITED: + if state.status not in (AccountStatus.QUOTA_EXCEEDED, AccountStatus.RATE_LIMITED): return False usage_values = (state.used_percent, state.secondary_used_percent) return any(value is not None and float(value) >= 100.0 for value in usage_values) def _usage_exhausted_reset_candidates(state: AccountState) -> list[float]: - if state.status == AccountStatus.QUOTA_EXCEEDED: - return [float(state.reset_at)] if state.reset_at is not None else [] - if state.status != AccountStatus.RATE_LIMITED: + if state.status not in (AccountStatus.QUOTA_EXCEEDED, AccountStatus.RATE_LIMITED): return [] candidates: list[float] = [] if state.used_percent is not None and float(state.used_percent) >= 100.0 and state.reset_at is not None: diff --git a/app/modules/proxy/_service/websocket/mixin.py b/app/modules/proxy/_service/websocket/mixin.py index f3a725a9d1..3e8402dcf0 100644 --- a/app/modules/proxy/_service/websocket/mixin.py +++ b/app/modules/proxy/_service/websocket/mixin.py @@ -2403,19 +2403,16 @@ async def _heartbeat(remaining_seconds: float) -> None: ) return None if require_preferred_account and preferred_account_id is not None: - if _facade()._is_local_account_cap_code(error_code): + if _facade()._is_local_account_cap_code(error_code) or error_code == "usage_limit_reached": + status_code, error_payload = selection_failure_response(selection) await proxy._emit_websocket_connect_failure( websocket, client_send_lock=client_send_lock, account_id=preferred_account_id, api_key=api_key, request_state=request_state, - status_code=429, - payload=openai_error( - error_code, - error_message, - error_type="rate_limit_error", - ), + status_code=status_code, + payload=error_payload, error_code=error_code, error_message=error_message, ) diff --git a/tests/unit/test_load_balancer.py b/tests/unit/test_load_balancer.py index bcde68f528..ff9d50cc08 100644 --- a/tests/unit/test_load_balancer.py +++ b/tests/unit/test_load_balancer.py @@ -680,7 +680,7 @@ def test_select_account_reports_pool_wide_usage_exhaustion_structurally(): now = 1_700_000_000.0 states = [ AccountState("a", AccountStatus.RATE_LIMITED, used_percent=100.0, reset_at=int(now + 60)), - AccountState("b", AccountStatus.QUOTA_EXCEEDED, reset_at=int(now + 3600)), + AccountState("b", AccountStatus.QUOTA_EXCEEDED, used_percent=100.0, reset_at=int(now + 3600)), AccountState("paused", AccountStatus.PAUSED), ] @@ -713,6 +713,19 @@ def test_select_account_reports_secondary_usage_exhaustion_reset(): assert result.resets_at == int(now + 3600) +def test_select_account_requires_usage_window_evidence_for_quota_exhaustion(): + now = 1_700_000_000.0 + states = [ + AccountState("a", AccountStatus.QUOTA_EXCEEDED, reset_at=int(now + 3600)), + ] + + result = select_account(states, now=now) + + assert result.account is None + assert result.error_code is None + assert result.error_message == "Rate limit exceeded. Try again in 300s" + + def test_select_account_does_not_treat_generic_rate_limit_as_usage_exhaustion(): now = 1_700_000_000.0 states = [ diff --git a/tests/unit/test_load_balancer_concurrency.py b/tests/unit/test_load_balancer_concurrency.py index 2bea81aa31..4cdc3f9845 100644 --- a/tests/unit/test_load_balancer_concurrency.py +++ b/tests/unit/test_load_balancer_concurrency.py @@ -124,8 +124,13 @@ async def test_account_lease_uses_explicit_dashboard_cap_snapshot_not_startup_en async def test_opportunistic_selection_preserves_usage_limit_exhaustion_error() -> None: account = _make_account("acc-opportunistic-usage-exhausted") account.status = AccountStatus.QUOTA_EXCEEDED - account.reset_at = int(time.time() + 300) - balancer = LoadBalancer(lambda: _repo_factory(_StubAccountsRepository([account]), _StubUsageRepository({}, {}))) + reset_at = int(time.time() + 300) + account.reset_at = reset_at + usage_repo = _StubUsageRepository( + {account.id: _usage_row_with_percent(1, account.id, used_percent=100.0, reset_at=reset_at)}, + {}, + ) + balancer = LoadBalancer(lambda: _repo_factory(_StubAccountsRepository([account]), usage_repo)) result = await balancer.select_account( routing_strategy="usage_weighted", @@ -134,7 +139,7 @@ async def test_opportunistic_selection_preserves_usage_limit_exhaustion_error() assert result.account is None assert result.error_code == "usage_limit_reached" - assert result.resets_at == account.reset_at + assert result.resets_at == reset_at class _StubAccountsRepository: diff --git a/tests/unit/test_proxy_utils.py b/tests/unit/test_proxy_utils.py index a9a2f06bbd..da4a914864 100644 --- a/tests/unit/test_proxy_utils.py +++ b/tests/unit/test_proxy_utils.py @@ -17823,6 +17823,63 @@ async def test_select_websocket_connect_account_requires_preferred_account_for_p assert select_account.await_args.kwargs["request_stage"] == "reattach" +@pytest.mark.asyncio +async def test_select_websocket_connect_account_preserves_usage_limit_for_required_owner(monkeypatch): + request_logs = _RequestLogsRecorder() + service = proxy_service.ProxyService(_repo_factory(request_logs)) + request_state = proxy_service._WebSocketRequestState( + request_id="ws_req_prev_owner_usage_limit", + model="gpt-5.1", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=0.0, + request_stage="reattach", + ) + emit_connect_failure = AsyncMock() + select_account = AsyncMock( + return_value=AccountSelection( + account=None, + error_message="Rate limit exceeded. Try again in 1h", + error_code="usage_limit_reached", + resets_at=1_700_003_600, + ) + ) + + monkeypatch.setattr(service, "_select_account_with_budget", select_account) + monkeypatch.setattr(service, "_emit_websocket_connect_failure", emit_connect_failure) + + result = await service._select_websocket_connect_account( + time.monotonic() + 10_000.0, + sticky_key=None, + sticky_kind=None, + prefer_earlier_reset=False, + prefer_earlier_reset_window="secondary", + routing_strategy="usage_weighted", + model="gpt-5.1", + request_state=request_state, + api_key=None, + client_send_lock=anyio.Lock(), + websocket=cast(WebSocket, SimpleNamespace()), + reallocate_sticky=False, + sticky_max_age_seconds=None, + exclude_account_ids=set(), + preferred_account_id="acc_owner", + require_preferred_account=True, + ) + + assert result is None + emit_connect_failure.assert_awaited_once() + call = emit_connect_failure.await_args + assert call is not None + assert call.kwargs["status_code"] == 429 + assert call.kwargs["error_code"] == "usage_limit_reached" + assert call.kwargs["account_id"] == "acc_owner" + assert call.kwargs["payload"]["error"]["code"] == "usage_limit_reached" + assert call.kwargs["payload"]["error"]["type"] == "usage_limit_reached" + assert call.kwargs["payload"]["error"]["resets_at"] == 1_700_003_600 + + @pytest.mark.asyncio async def test_select_websocket_connect_account_records_fail_closed_for_preferred_account_mismatch( monkeypatch, From 0f9aaae1a154cca75bfb309e6b0e285898dde6b3 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sat, 18 Jul 2026 08:33:46 +0400 Subject: [PATCH 09/64] fix(balancer): preserve cap semantics for usage limits --- app/core/balancer/logic.py | 52 ++++-- .../proxy/_load_balancer/sticky_selection.py | 40 +++++ .../proxy/_load_balancer/unbound_selection.py | 21 ++- .../proxy/_service/http_bridge/mixin.py | 10 +- app/modules/proxy/_service/streaming/retry.py | 1 + app/modules/proxy/_service/websocket/mixin.py | 6 +- app/modules/proxy/load_balancer.py | 10 ++ app/modules/proxy/service.py | 1 + tests/unit/test_load_balancer.py | 153 ++++++++++++++++++ tests/unit/test_load_balancer_concurrency.py | 51 ++++++ tests/unit/test_proxy_http_bridge.py | 106 ++++++++++++ tests/unit/test_proxy_utils.py | 75 ++++++++- 12 files changed, 500 insertions(+), 26 deletions(-) diff --git a/app/core/balancer/logic.py b/app/core/balancer/logic.py index c69d1a781c..6368560ad4 100644 --- a/app/core/balancer/logic.py +++ b/app/core/balancer/logic.py @@ -164,25 +164,41 @@ def pool_usage_exhaustion( return None ignored_account_ids = set(ignore_standard_quota_account_ids or ()) + def _primary_usage_evidence(state: AccountState) -> float | None: + return state.priority_used_percent if state.priority_used_percent is not None else state.used_percent + + def _secondary_usage_evidence(state: AccountState) -> float | None: + if state.limit_scoped_usage and state.priority_secondary_used_percent is None: + return _primary_usage_evidence(state) + return ( + state.priority_secondary_used_percent + if state.priority_secondary_used_percent is not None + else state.secondary_used_percent + ) + def _usage_exhausted(state: AccountState) -> bool: if state.status not in (AccountStatus.QUOTA_EXCEEDED, AccountStatus.RATE_LIMITED): return False - usage_values = (state.used_percent, state.secondary_used_percent) + usage_values = (_primary_usage_evidence(state), _secondary_usage_evidence(state)) return any(value is not None and float(value) >= 100.0 for value in usage_values) - def _usage_exhausted_reset_candidates(state: AccountState) -> list[float]: + def _usage_exhausted_reset_at(state: AccountState) -> float | None: if state.status not in (AccountStatus.QUOTA_EXCEEDED, AccountStatus.RATE_LIMITED): - return [] + return None candidates: list[float] = [] - if state.used_percent is not None and float(state.used_percent) >= 100.0 and state.reset_at is not None: + primary_evidence = _primary_usage_evidence(state) + secondary_evidence = _secondary_usage_evidence(state) + if primary_evidence is not None and float(primary_evidence) >= 100.0 and state.reset_at is not None: candidates.append(float(state.reset_at)) if ( - state.secondary_used_percent is not None - and float(state.secondary_used_percent) >= 100.0 + secondary_evidence is not None + and float(secondary_evidence) >= 100.0 and state.secondary_reset_at is not None ): candidates.append(float(state.secondary_reset_at)) - return candidates + if not candidates: + return None + return max(candidates) eligible = [ state @@ -202,7 +218,7 @@ def _usage_exhausted_reset_candidates(state: AccountState) -> list[float]: # Only usage-proven exhausted accounts reach this branch; surface the # earliest reset for an actually exhausted window so the structured 429 # does not retry a secondary-window exhaustion at the primary reset time. - reset_candidates = [reset_at for state in eligible for reset_at in _usage_exhausted_reset_candidates(state)] + reset_candidates = [reset_at for state in eligible if (reset_at := _usage_exhausted_reset_at(state)) is not None] resets_at = int(min(reset_candidates)) if reset_candidates else None message = "Usage limit reached" if resets_at is not None: @@ -447,6 +463,8 @@ def select_account( primary_first_usage_weighted: bool = False, routing_costs: RoutingCostsByAccount | None = None, replica_salt: str | None = None, + allow_usage_exhaustion_error: bool = True, + usage_exhaustion_states: Iterable[AccountState] | None = None, ) -> SelectionResult: """Select an eligible account by applying availability checks and routing strategy. @@ -514,6 +532,7 @@ def select_account( available: list[AccountState] = [] in_error_backoff: list[AccountState] = [] all_states = list(states) + usage_exhaustion_state_list = list(usage_exhaustion_states) if usage_exhaustion_states is not None else all_states bypass_account_ids = None if bypass_quota_exceeded_account_ids is None else set(bypass_quota_exceeded_account_ids) for state in all_states: @@ -597,14 +616,15 @@ def _backoff_expires_at(s: AccountState) -> float: return SelectionResult(None, f"opportunistic burn window closed: {reason}") available = opportunistic_available else: - usage_exhaustion = pool_usage_exhaustion( - all_states, - current=current, - ignore_standard_quota=ignore_standard_quota or bypass_quota_exceeded, - ignore_standard_quota_account_ids=bypass_account_ids, - ) - if usage_exhaustion is not None: - return usage_exhaustion + if allow_usage_exhaustion_error: + usage_exhaustion = pool_usage_exhaustion( + usage_exhaustion_state_list, + current=current, + ignore_standard_quota=ignore_standard_quota or bypass_quota_exceeded, + ignore_standard_quota_account_ids=bypass_account_ids, + ) + if usage_exhaustion is not None: + return usage_exhaustion reauth_required = [s for s in all_states if s.status == AccountStatus.REAUTH_REQUIRED] deactivated = [s for s in all_states if s.status == AccountStatus.DEACTIVATED] paused = [s for s in all_states if s.status == AccountStatus.PAUSED] diff --git a/app/modules/proxy/_load_balancer/sticky_selection.py b/app/modules/proxy/_load_balancer/sticky_selection.py index 7d21cb9ea6..3aee72fcc8 100644 --- a/app/modules/proxy/_load_balancer/sticky_selection.py +++ b/app/modules/proxy/_load_balancer/sticky_selection.py @@ -170,6 +170,8 @@ async def _select_with_stickiness( preserve_existing_mapping_on_fallback: bool, traffic_class: TrafficClass, ignore_standard_quota: bool, + allow_usage_exhaustion_error: bool = True, + usage_exhaustion_states: Iterable[AccountState] | None = None, ) -> _StickySelectionOutcome: ... async def release_account_lease(self, lease: AccountLease | None) -> None: ... @@ -204,6 +206,7 @@ class StickySelectionRequest(Generic[SelectionInputsT]): selection_inputs: SelectionInputsT reload_inputs: Callable[[], Awaitable[SelectionInputsT]] record_account_cap_rejection: AccountCapRejectionCallback + allow_usage_exhaustion_error: bool = True @dataclass(frozen=True, slots=True) @@ -262,6 +265,7 @@ async def run_sticky_selection_path( redact_sensitive_details = request.redact_sensitive_details load_selection_inputs = request.reload_inputs _record_account_cap_rejection = request.record_account_cap_rejection + allow_usage_exhaustion_error = request.allow_usage_exhaustion_error selected_snapshot: Account | None = None selected_lease: AccountLease | None = None @@ -472,8 +476,25 @@ def _direct_error( traffic_class=traffic_class, ignore_standard_quota=False, routing_costs_by_account_id=effective_routing_costs, + allow_usage_exhaustion_error=allow_usage_exhaustion_error, + usage_exhaustion_states=states, ) result = sticky_outcome.selection + if ( + result.account is None + and result.error_code is None + and lease_kind is not None + and len(selection_states) < len(states) + and any( + state.status == AccountStatus.ACTIVE for state in states if state not in selection_states + ) + ): + selection_error_code = _account_cap_error_code(lease_kind) + result = SelectionResult( + None, + _account_cap_error_message(lease_kind, caps), + error_code=selection_error_code, + ) except BaseException: async with owner._runtime_lock: owner._release_due_probe_reservation_locked(probe_reservation) @@ -890,6 +911,8 @@ async def _select_with_stickiness( preserve_existing_mapping_on_fallback: bool = False, traffic_class: TrafficClass = TRAFFIC_CLASS_FOREGROUND, ignore_standard_quota: bool = False, + allow_usage_exhaustion_error: bool = True, + usage_exhaustion_states: Iterable[AccountState] | None = None, ) -> _StickySelectionOutcome: if not sticky_key or not sticky_repo: return _StickySelectionOutcome( @@ -904,6 +927,8 @@ async def _select_with_stickiness( traffic_class=traffic_class, ignore_standard_quota=ignore_standard_quota, routing_costs_by_account_id=routing_costs_by_account_id, + allow_usage_exhaustion_error=allow_usage_exhaustion_error, + usage_exhaustion_states=usage_exhaustion_states, ) ) if sticky_kind is None: @@ -1029,6 +1054,8 @@ def finish_selection( traffic_class=traffic_class, ignore_standard_quota=ignore_standard_quota, routing_costs_by_account_id=routing_costs_by_account_id, + allow_usage_exhaustion_error=allow_usage_exhaustion_error, + usage_exhaustion_states=usage_exhaustion_states, ) pool_also_exhausted = pool_best.account is not None and ( pool_best.account.account_id == pinned.account_id @@ -1115,6 +1142,8 @@ def finish_selection( traffic_class=traffic_class, ignore_standard_quota=ignore_standard_quota, routing_costs_by_account_id=routing_costs_by_account_id, + allow_usage_exhaustion_error=allow_usage_exhaustion_error, + usage_exhaustion_states=usage_exhaustion_states, ) if persist_fallback and chosen.account is not None and chosen.account.account_id in account_map: return finish_selection(chosen, persist_account_id=chosen.account.account_id) @@ -1318,6 +1347,8 @@ def _select_account_preferring_budget_safe( traffic_class: TrafficClass = TRAFFIC_CLASS_FOREGROUND, ignore_standard_quota: bool = False, routing_costs_by_account_id: RoutingCostsByAccount | None = None, + allow_usage_exhaustion_error: bool = True, + usage_exhaustion_states: Iterable[AccountState] | None = None, ) -> SelectionResult: state_list = list(states) if routing_strategy not in ("sequential_drain", "reset_drain", "single_account"): @@ -1336,6 +1367,7 @@ def _select_account_preferring_budget_safe( traffic_class=traffic_class, ignore_standard_quota=ignore_standard_quota, routing_costs=routing_costs_by_account_id, + allow_usage_exhaustion_error=False, ) if recovery_probe.account is not None: return recovery_probe @@ -1368,6 +1400,8 @@ def _select_account_preferring_budget_safe( traffic_class=traffic_class, ignore_standard_quota=ignore_standard_quota, routing_costs=routing_costs_by_account_id, + allow_usage_exhaustion_error=allow_usage_exhaustion_error, + usage_exhaustion_states=usage_exhaustion_states, ) best_health_states = _best_health_tier_states(state_list) @@ -1385,6 +1419,8 @@ def _select_account_preferring_budget_safe( traffic_class=traffic_class, ignore_standard_quota=ignore_standard_quota, routing_costs=routing_costs_by_account_id, + allow_usage_exhaustion_error=allow_usage_exhaustion_error, + usage_exhaustion_states=usage_exhaustion_states, ) if burn_first.account is not None: return burn_first @@ -1408,6 +1444,8 @@ def _select_account_preferring_budget_safe( traffic_class=traffic_class, ignore_standard_quota=ignore_standard_quota, routing_costs=routing_costs_by_account_id, + allow_usage_exhaustion_error=allow_usage_exhaustion_error, + usage_exhaustion_states=usage_exhaustion_states, ) if preferred.account is not None: return preferred @@ -1425,6 +1463,8 @@ def _select_account_preferring_budget_safe( traffic_class=traffic_class, ignore_standard_quota=ignore_standard_quota, routing_costs=routing_costs_by_account_id, + allow_usage_exhaustion_error=allow_usage_exhaustion_error, + usage_exhaustion_states=usage_exhaustion_states, ) return select_account( state_list, diff --git a/app/modules/proxy/_load_balancer/unbound_selection.py b/app/modules/proxy/_load_balancer/unbound_selection.py index f69b91957c..cb858a89c0 100644 --- a/app/modules/proxy/_load_balancer/unbound_selection.py +++ b/app/modules/proxy/_load_balancer/unbound_selection.py @@ -14,7 +14,7 @@ SelectionResult, TrafficClass, ) -from app.db.models import Account +from app.db.models import Account, AccountStatus from app.modules.proxy._load_balancer.sticky_selection import ( SelectionInputsProtocol, StickySelectionOwner, @@ -70,6 +70,7 @@ class UnboundSelectionRequest(Generic[SelectionInputsT]): selection_inputs: SelectionInputsT reload_inputs: Callable[[], Awaitable[SelectionInputsT]] record_account_cap_rejection: AccountCapRejectionCallback + allow_usage_exhaustion_error: bool = True @dataclass(frozen=True, slots=True) @@ -106,6 +107,7 @@ async def run_unbound_selection_path( redact_sensitive_details = request.redact_sensitive_details load_selection_inputs = request.reload_inputs _record_account_cap_rejection = request.record_account_cap_rejection + allow_usage_exhaustion_error = request.allow_usage_exhaustion_error selected_snapshot: Account | None = None selected_lease: AccountLease | None = None @@ -188,7 +190,24 @@ def _direct_error( traffic_class=traffic_class, ignore_standard_quota=False, routing_costs_by_account_id=effective_routing_costs, + allow_usage_exhaustion_error=allow_usage_exhaustion_error, + usage_exhaustion_states=states, ) + if ( + result.account is None + and result.error_code is None + and lease_kind is not None + and len(selection_states) < len(states) + and any( + state.status == AccountStatus.ACTIVE for state in states if state not in selection_states + ) + ): + selection_error_code = _account_cap_error_code(lease_kind) + result = SelectionResult( + None, + _account_cap_error_message(lease_kind, caps), + error_code=selection_error_code, + ) probing_result_requires_reservation = _probing_result_requires_recovery_reservation( selection_states, result.account, diff --git a/app/modules/proxy/_service/http_bridge/mixin.py b/app/modules/proxy/_service/http_bridge/mixin.py index 9854e34f8e..2a332bbff9 100644 --- a/app/modules/proxy/_service/http_bridge/mixin.py +++ b/app/modules/proxy/_service/http_bridge/mixin.py @@ -218,7 +218,7 @@ DurableBridgeLookup, ) from app.modules.proxy.load_balancer import CONTINUITY_OWNER_UNAVAILABLE, AccountLease -from app.modules.proxy.selection_errors import selection_failure_response +from app.modules.proxy.selection_errors import USAGE_LIMIT_REACHED, selection_failure_response logger = logging.getLogger("app.modules.proxy.service") T = TypeVar("T") @@ -2128,6 +2128,14 @@ async def abandon_selected_account_retry(selected_account: Any) -> None: ): preferred_candidate_id = None continue + if selection.error_code == USAGE_LIMIT_REACHED and ( + required_preferred_account_id is not None or hard_close_account_bound + ): + raise _http_bridge_previous_response_owner_unavailable_error() + if selection.error_code == USAGE_LIMIT_REACHED: + record_selected_account_takeover(None) + status_code, error_payload = selection_failure_response(selection) + raise ProxyResponseError(status_code, error_payload) if await _sleep_for_account_selection_recovery( selection, request_id=request_state.request_log_id or request_state.request_id, diff --git a/app/modules/proxy/_service/streaming/retry.py b/app/modules/proxy/_service/streaming/retry.py index 1022236ec9..8b24aad479 100644 --- a/app/modules/proxy/_service/streaming/retry.py +++ b/app/modules/proxy/_service/streaming/retry.py @@ -1068,6 +1068,7 @@ async def _retry_account_model_rejection( continue if ( not account + and selection.error_code != USAGE_LIMIT_REACHED and ( selection.error_code in _LOCAL_ACCOUNT_CAP_ERROR_CODES or not (propagate_http_errors and last_transient_exc is not None) diff --git a/app/modules/proxy/_service/websocket/mixin.py b/app/modules/proxy/_service/websocket/mixin.py index 3e8402dcf0..6824360ac6 100644 --- a/app/modules/proxy/_service/websocket/mixin.py +++ b/app/modules/proxy/_service/websocket/mixin.py @@ -458,7 +458,7 @@ openai_validation_error, validate_model_access, ) -from app.modules.proxy.selection_errors import selection_failure_response +from app.modules.proxy.selection_errors import USAGE_LIMIT_REACHED, selection_failure_response from app.modules.proxy.tool_call_dedupe import ( mark_duplicate_tool_call_downstream_event, rewrite_parallel_tool_call_text, @@ -2310,6 +2310,8 @@ async def _select_websocket_connect_account( account = selection.account if account is not None: break + if selection.error_code == USAGE_LIMIT_REACHED: + break async def _heartbeat(remaining_seconds: float) -> None: event = _account_capacity_wait_payload( @@ -2403,7 +2405,7 @@ async def _heartbeat(remaining_seconds: float) -> None: ) return None if require_preferred_account and preferred_account_id is not None: - if _facade()._is_local_account_cap_code(error_code) or error_code == "usage_limit_reached": + if _facade()._is_local_account_cap_code(error_code): status_code, error_payload = selection_failure_response(selection) await proxy._emit_websocket_connect_failure( websocket, diff --git a/app/modules/proxy/load_balancer.py b/app/modules/proxy/load_balancer.py index 3a43923e33..a280443a8f 100644 --- a/app/modules/proxy/load_balancer.py +++ b/app/modules/proxy/load_balancer.py @@ -442,6 +442,7 @@ async def select_account( traffic_class: TrafficClass = TRAFFIC_CLASS_FOREGROUND, concurrency_caps: AccountConcurrencyCaps | None = None, redact_sensitive_details: bool = False, + allow_usage_exhaustion_error: bool = True, ) -> AccountSelection: if (required_account_is_ownership_constraint or required_continuity_owner) and required_account_id is None: raise ValueError("required account ownership flags require required_account_id") @@ -641,6 +642,7 @@ async def load_selection_inputs() -> _SelectionInputs: selection_inputs=selection_inputs, reload_inputs=load_selection_inputs, record_account_cap_rejection=_record_account_cap_rejection, + allow_usage_exhaustion_error=allow_usage_exhaustion_error, ), ) selection_inputs = unbound_outcome.selection_inputs @@ -686,6 +688,7 @@ async def load_selection_inputs() -> _SelectionInputs: selection_inputs=selection_inputs, reload_inputs=load_selection_inputs, record_account_cap_rejection=_record_account_cap_rejection, + allow_usage_exhaustion_error=allow_usage_exhaustion_error, ), ) selection_inputs = sticky_outcome.selection_inputs @@ -1198,6 +1201,7 @@ async def check_opportunistic_admission( deterministic_probe=True, traffic_class=TRAFFIC_CLASS_OPPORTUNISTIC, ignore_standard_quota=False, + usage_exhaustion_states=states, ) if result.account is None: if result.error_code == USAGE_LIMIT_REACHED: @@ -1427,6 +1431,8 @@ async def _select_with_stickiness( preserve_existing_mapping_on_fallback: bool = False, traffic_class: TrafficClass = TRAFFIC_CLASS_FOREGROUND, ignore_standard_quota: bool = False, + allow_usage_exhaustion_error: bool = True, + usage_exhaustion_states: Iterable[AccountState] | None = None, ) -> _StickySelectionOutcome: return await _run_select_with_stickiness( states=states, @@ -1448,6 +1454,8 @@ async def _select_with_stickiness( preserve_existing_mapping_on_fallback=preserve_existing_mapping_on_fallback, traffic_class=traffic_class, ignore_standard_quota=ignore_standard_quota, + allow_usage_exhaustion_error=allow_usage_exhaustion_error, + usage_exhaustion_states=usage_exhaustion_states, ) _persist_sticky_mutation = staticmethod(_persist_sticky_mutation) @@ -2303,6 +2311,8 @@ def _state_from_account( plan_type=account.plan_type, capacity_credits=capacity_credits, health_tier=new_tier, + priority_used_percent=used_percent, + priority_secondary_used_percent=secondary_used, inflight_response_creates=runtime.inflight_response_creates, inflight_streams=runtime.inflight_streams, leased_tokens=runtime.leased_tokens, diff --git a/app/modules/proxy/service.py b/app/modules/proxy/service.py index 0353be603b..ac421f0a10 100644 --- a/app/modules/proxy/service.py +++ b/app/modules/proxy/service.py @@ -1864,6 +1864,7 @@ def log_account_id(account_id: str | None) -> str | None: traffic_class=effective_traffic_class, concurrency_caps=concurrency_caps, redact_sensitive_details=redact_sensitive_details, + allow_usage_exhaustion_error=not required_preferred_account, ) if preferred_selection.account is not None: logger.info( diff --git a/tests/unit/test_load_balancer.py b/tests/unit/test_load_balancer.py index ff9d50cc08..fa0468d4cd 100644 --- a/tests/unit/test_load_balancer.py +++ b/tests/unit/test_load_balancer.py @@ -713,6 +713,33 @@ def test_select_account_reports_secondary_usage_exhaustion_reset(): assert result.resets_at == int(now + 3600) +def test_select_account_waits_for_latest_exhausted_window_per_account(): + now = 1_700_000_000.0 + states = [ + AccountState( + "a", + AccountStatus.RATE_LIMITED, + used_percent=100.0, + secondary_used_percent=100.0, + reset_at=int(now + 60), + secondary_reset_at=int(now + 3600), + ), + AccountState( + "b", + AccountStatus.QUOTA_EXCEEDED, + used_percent=100.0, + reset_at=int(now + 7200), + ), + ] + + result = select_account(states, now=now) + + assert result.account is None + assert result.error_code == "usage_limit_reached" + assert result.error_message == "Rate limit exceeded. Try again in 300s" + assert result.resets_at == int(now + 3600) + + def test_select_account_requires_usage_window_evidence_for_quota_exhaustion(): now = 1_700_000_000.0 states = [ @@ -726,6 +753,110 @@ def test_select_account_requires_usage_window_evidence_for_quota_exhaustion(): assert result.error_message == "Rate limit exceeded. Try again in 300s" +def test_select_account_can_disable_pool_usage_exhaustion_for_owner_scope(): + now = 1_700_000_000.0 + states = [ + AccountState( + "owner", + AccountStatus.QUOTA_EXCEEDED, + used_percent=100.0, + reset_at=int(now + 3600), + ) + ] + + result = select_account(states, now=now, allow_usage_exhaustion_error=False) + + assert result.account is None + assert result.error_code is None + assert result.error_message == "Rate limit exceeded. Try again in 300s" + + +def test_budget_safe_selection_uses_full_scope_for_usage_exhaustion() -> None: + now = time.time() + cap_filtered_states = [ + AccountState( + "exhausted", + AccountStatus.QUOTA_EXCEEDED, + used_percent=100.0, + reset_at=int(now + 3600), + ) + ] + full_scope_states = [ + AccountState( + "capped-but-usable", + AccountStatus.ACTIVE, + used_percent=50.0, + reset_at=int(now + 3600), + ), + *cap_filtered_states, + ] + + result = _select_account_preferring_budget_safe( + cap_filtered_states, + prefer_earlier_reset=False, + routing_strategy="usage_weighted", + budget_threshold_pct=95.0, + usage_exhaustion_states=full_scope_states, + ) + + assert result.account is None + assert result.error_code is None + assert result.error_message == "Rate limit exceeded. Try again in 300s" + + +def test_opportunistic_budget_safe_selection_uses_full_scope_for_usage_exhaustion() -> None: + now = time.time() + cap_filtered_states = [ + AccountState( + "exhausted", + AccountStatus.QUOTA_EXCEEDED, + used_percent=100.0, + reset_at=int(now + 3600), + ) + ] + full_scope_states = [ + AccountState( + "capped-but-usable", + AccountStatus.ACTIVE, + used_percent=50.0, + reset_at=int(now + 3600), + ), + *cap_filtered_states, + ] + + result = _select_account_preferring_budget_safe( + cap_filtered_states, + prefer_earlier_reset=False, + routing_strategy="usage_weighted", + budget_threshold_pct=95.0, + traffic_class="opportunistic", + usage_exhaustion_states=full_scope_states, + ) + + assert result.account is None + assert result.error_code is None + assert result.error_message == "Rate limit exceeded. Try again in 300s" + + +def test_select_account_uses_raw_priority_usage_for_exhaustion_evidence() -> None: + now = 1_700_000_000.0 + states = [ + AccountState( + "pressure-adjusted", + AccountStatus.RATE_LIMITED, + used_percent=100.0, + priority_used_percent=98.0, + reset_at=int(now + 60), + ) + ] + + result = select_account(states, now=now) + + assert result.account is None + assert result.error_code is None + assert result.error_message == "No available accounts" + + def test_select_account_does_not_treat_generic_rate_limit_as_usage_exhaustion(): now = 1_700_000_000.0 states = [ @@ -2001,6 +2132,28 @@ def test_state_from_account_keeps_active_account_selectable_when_primary_usage_s assert selection.account.account_id == state.account_id +def test_state_from_account_keeps_raw_usage_evidence_separate_from_pressure(monkeypatch): + now = 1_700_000_000.0 + future_reset = int(now + 300) + monkeypatch.setattr("app.modules.proxy.load_balancer.time.time", lambda: now) + monkeypatch.setattr("app.core.usage.quota.time.time", lambda: now) + + state = _state_from_account( + account=_make_test_account(status=AccountStatus.RATE_LIMITED, reset_at=future_reset, blocked_at=int(now)), + primary_entry=_make_test_usage( + window="primary", + used_percent=98.0, + reset_at=future_reset, + recorded_at=_epoch_to_naive_utc(now - 30), + ), + secondary_entry=None, + runtime=RuntimeState(inflight_streams=1), + ) + + assert state.used_percent == 100.0 + assert state.priority_used_percent == 98.0 + + def test_state_from_account_clears_stale_advisory_account_reset_for_active_account(monkeypatch): now = 1_700_000_000.0 future_reset = int(now + 300) diff --git a/tests/unit/test_load_balancer_concurrency.py b/tests/unit/test_load_balancer_concurrency.py index 4cdc3f9845..be94c2f7fd 100644 --- a/tests/unit/test_load_balancer_concurrency.py +++ b/tests/unit/test_load_balancer_concurrency.py @@ -823,6 +823,57 @@ async def test_account_stream_cap_returns_stable_local_reason_until_released() - assert recovered.lease is not None +@pytest.mark.asyncio +async def test_stream_cap_takes_precedence_over_remaining_quota_exhausted_account() -> None: + now_epoch = int(datetime.now(tz=timezone.utc).timestamp()) + capped = _make_account("acc-stream-cap-mixed-capped") + exhausted = _make_account("acc-stream-cap-mixed-exhausted") + exhausted.status = AccountStatus.QUOTA_EXCEEDED + exhausted.reset_at = now_epoch + 3600 + accounts_repo = _StubAccountsRepository([capped, exhausted]) + usage_repo = _StubUsageRepository( + primary={ + capped.id: _usage_row_with_percent( + 203, + capped.id, + used_percent=50.0, + reset_at=now_epoch + 300, + ), + exhausted.id: _usage_row_with_percent( + 204, + exhausted.id, + used_percent=100.0, + reset_at=now_epoch + 3600, + ), + }, + secondary={}, + ) + balancer = LoadBalancer(lambda: _repo_factory(accounts_repo, usage_repo)) + leases = [ + ( + await balancer.select_account( + routing_strategy="usage_weighted", + lease_kind="stream", + ) + ).lease + for _ in range(8) + ] + + selected = await balancer.select_account( + routing_strategy="usage_weighted", + lease_kind="stream", + ) + + assert selected.account is None + assert selected.error_code == "account_stream_cap" + assert selected.error_message is not None + assert "Account stream capacity is exhausted" in selected.error_message + assert selected.resets_at is None + + for lease in leases: + await balancer.release_account_lease(lease) + + @pytest.mark.asyncio async def test_account_stream_recovery_reserve_keeps_last_slot_for_reattach() -> None: now_epoch = int(datetime.now(tz=timezone.utc).timestamp()) diff --git a/tests/unit/test_proxy_http_bridge.py b/tests/unit/test_proxy_http_bridge.py index 00dd232e8e..27b44d422e 100644 --- a/tests/unit/test_proxy_http_bridge.py +++ b/tests/unit/test_proxy_http_bridge.py @@ -5740,6 +5740,112 @@ async def sleep_for_recovery(*_args: object, **kwargs: object) -> bool: assert sleep_calls[0]["max_sleep_seconds"] == pytest.approx(119.5) +@pytest.mark.asyncio +async def test_reconnect_http_bridge_session_skips_capacity_wait_for_usage_limit( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session() + settings = SimpleNamespace( + prefer_earlier_reset_accounts=False, + prefer_earlier_reset_window="secondary", + routing_strategy="usage_weighted", + ) + + async def select_account(_deadline: float, **_kwargs: object) -> proxy_service.AccountSelection: + return proxy_service.AccountSelection( + account=None, + error_message="Rate limit exceeded. Try again in 1h", + error_code="usage_limit_reached", + resets_at=1_700_003_600, + ) + + request_state = proxy_service._WebSocketRequestState( + request_id="req-reconnect-usage-limit-now", + model="gpt-5.4", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=100.0, + ) + monkeypatch.setattr(proxy_service.time, "monotonic", lambda: 100.5) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: SimpleNamespace(get=AsyncMock(return_value=settings)), + ) + monkeypatch.setattr(service, "_select_account_with_budget_compatible", select_account) + monkeypatch.setattr( + http_bridge_mixin_module, + "_sleep_for_account_selection_recovery", + lambda *_args, **_kwargs: pytest.fail("usage_limit_reached must not enter recovery wait"), + ) + + with pytest.raises(ProxyResponseError) as exc_info: + await service._reconnect_http_bridge_session(session, request_state=request_state) + + assert exc_info.value.status_code == 429 + assert exc_info.value.payload["error"]["code"] == "usage_limit_reached" + assert exc_info.value.payload["error"]["type"] == "usage_limit_reached" + assert exc_info.value.payload["error"]["resets_at"] == 1_700_003_600 + + +@pytest.mark.asyncio +async def test_reconnect_http_bridge_session_preserves_owner_error_for_owner_usage_limit( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session() + settings = SimpleNamespace( + prefer_earlier_reset_accounts=False, + prefer_earlier_reset_window="secondary", + routing_strategy="usage_weighted", + ) + + async def select_account(_deadline: float, **_kwargs: object) -> proxy_service.AccountSelection: + return proxy_service.AccountSelection( + account=None, + error_message="Rate limit exceeded. Try again in 1h", + error_code="usage_limit_reached", + resets_at=1_700_003_600, + ) + + request_state = proxy_service._WebSocketRequestState( + request_id="req-reconnect-owner-usage-limit", + model="gpt-5.4", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=100.0, + preferred_account_id=session.account.id, + ) + monkeypatch.setattr(proxy_service.time, "monotonic", lambda: 100.5) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: SimpleNamespace(get=AsyncMock(return_value=settings)), + ) + monkeypatch.setattr(service, "_select_account_with_budget_compatible", select_account) + monkeypatch.setattr( + http_bridge_mixin_module, + "_sleep_for_account_selection_recovery", + lambda *_args, **_kwargs: pytest.fail("owner-only usage_limit_reached must not enter recovery wait"), + ) + + with pytest.raises(ProxyResponseError) as exc_info: + await service._reconnect_http_bridge_session( + session, + request_state=request_state, + require_preferred_account=True, + ) + + assert exc_info.value.status_code == 502 + assert exc_info.value.payload["error"]["code"] == "previous_response_owner_unavailable" + assert exc_info.value.payload["error"]["type"] == "server_error" + + @pytest.mark.asyncio async def test_reconnect_http_bridge_session_preserves_exclusions_after_capacity_wait( monkeypatch: pytest.MonkeyPatch, diff --git a/tests/unit/test_proxy_utils.py b/tests/unit/test_proxy_utils.py index da4a914864..6b3e98f65b 100644 --- a/tests/unit/test_proxy_utils.py +++ b/tests/unit/test_proxy_utils.py @@ -10241,6 +10241,11 @@ async def test_service_compact_passes_chatgpt_account_id_to_core(monkeypatch): monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache(settings)) monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr( + streaming_retry_module, + "_account_selection_recovery_sleep_seconds", + lambda _selection: pytest.fail("usage_limit_reached must not enter recovery wait"), + ) monkeypatch.setattr( service._load_balancer, "select_account", @@ -17824,7 +17829,7 @@ async def test_select_websocket_connect_account_requires_preferred_account_for_p @pytest.mark.asyncio -async def test_select_websocket_connect_account_preserves_usage_limit_for_required_owner(monkeypatch): +async def test_select_websocket_connect_account_preserves_continuity_for_owner_usage_limit(monkeypatch): request_logs = _RequestLogsRecorder() service = proxy_service.ProxyService(_repo_factory(request_logs)) request_state = proxy_service._WebSocketRequestState( @@ -17872,12 +17877,11 @@ async def test_select_websocket_connect_account_preserves_usage_limit_for_requir emit_connect_failure.assert_awaited_once() call = emit_connect_failure.await_args assert call is not None - assert call.kwargs["status_code"] == 429 - assert call.kwargs["error_code"] == "usage_limit_reached" + assert call.kwargs["status_code"] == 502 + assert call.kwargs["error_code"] == "previous_response_owner_unavailable" assert call.kwargs["account_id"] == "acc_owner" - assert call.kwargs["payload"]["error"]["code"] == "usage_limit_reached" - assert call.kwargs["payload"]["error"]["type"] == "usage_limit_reached" - assert call.kwargs["payload"]["error"]["resets_at"] == 1_700_003_600 + assert call.kwargs["payload"]["error"]["code"] == "previous_response_owner_unavailable" + assert call.kwargs["payload"]["error"]["type"] == "server_error" @pytest.mark.asyncio @@ -18158,6 +18162,65 @@ async def fake_sleep_for_account_selection_recovery(*_args: object, **kwargs: ob assert sent_payload["request_id"] == "ws_req_capacity_wait" +@pytest.mark.asyncio +async def test_select_websocket_connect_account_skips_capacity_wait_for_usage_limit(monkeypatch): + service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) + request_state = proxy_service._WebSocketRequestState( + request_id="ws_req_usage_limit_now", + model="gpt-5.1", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=0.0, + ) + websocket_send = AsyncMock() + + monkeypatch.setattr( + service, + "_select_account_with_budget", + AsyncMock( + return_value=AccountSelection( + account=None, + error_message="Rate limit exceeded. Try again in 1h", + error_code="usage_limit_reached", + resets_at=1_700_003_600, + ) + ), + ) + monkeypatch.setattr( + websocket_mixin_module, + "_sleep_for_account_selection_recovery", + lambda *_args, **_kwargs: pytest.fail("usage_limit_reached must not enter recovery wait"), + ) + + result = await service._select_websocket_connect_account( + time.monotonic() + 10_000.0, + sticky_key=None, + sticky_kind=None, + prefer_earlier_reset=False, + routing_strategy="usage_weighted", + model="gpt-5.1", + request_state=request_state, + api_key=None, + client_send_lock=anyio.Lock(), + websocket=cast(WebSocket, SimpleNamespace(send_text=websocket_send)), + reallocate_sticky=False, + sticky_max_age_seconds=None, + exclude_account_ids=set(), + preferred_account_id=None, + require_preferred_account=False, + ) + + assert result is None + await_args = websocket_send.await_args + assert await_args is not None + sent_payload = json.loads(await_args.args[0]) + assert sent_payload["status"] == 429 + assert sent_payload["error"]["code"] == "usage_limit_reached" + assert sent_payload["error"]["type"] == "usage_limit_reached" + assert sent_payload["error"]["resets_at"] == 1_700_003_600 + + @pytest.mark.asyncio @pytest.mark.parametrize( ("preferred_account_id", "require_preferred_account", "file_required", "defer_no_account_error"), From 8b9cd705a5fd98bccd1ec875129cd26f5b63114c Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sat, 18 Jul 2026 11:30:32 +0400 Subject: [PATCH 10/64] fix(balancer): preserve pressure usage priorities --- app/modules/proxy/load_balancer.py | 5 +++-- tests/unit/test_load_balancer.py | 24 ++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/app/modules/proxy/load_balancer.py b/app/modules/proxy/load_balancer.py index a280443a8f..d4b63e6217 100644 --- a/app/modules/proxy/load_balancer.py +++ b/app/modules/proxy/load_balancer.py @@ -2292,6 +2292,7 @@ def _state_from_account( pressure_pct = inflight_pressure_pct + leased_token_pressure_pct effective_used_percent = None if used_percent is None else min(100.0, used_percent + pressure_pct) effective_secondary_used_percent = None if secondary_used is None else min(100.0, secondary_used + pressure_pct) + usage_exhaustion_evidence_status = status in (AccountStatus.QUOTA_EXCEEDED, AccountStatus.RATE_LIMITED) return AccountState( account_id=account.id, @@ -2311,8 +2312,8 @@ def _state_from_account( plan_type=account.plan_type, capacity_credits=capacity_credits, health_tier=new_tier, - priority_used_percent=used_percent, - priority_secondary_used_percent=secondary_used, + priority_used_percent=used_percent if usage_exhaustion_evidence_status else None, + priority_secondary_used_percent=secondary_used if usage_exhaustion_evidence_status else None, inflight_response_creates=runtime.inflight_response_creates, inflight_streams=runtime.inflight_streams, leased_tokens=runtime.leased_tokens, diff --git a/tests/unit/test_load_balancer.py b/tests/unit/test_load_balancer.py index fa0468d4cd..915c3578ca 100644 --- a/tests/unit/test_load_balancer.py +++ b/tests/unit/test_load_balancer.py @@ -2154,6 +2154,30 @@ def test_state_from_account_keeps_raw_usage_evidence_separate_from_pressure(monk assert state.priority_used_percent == 98.0 +def test_state_from_account_preserves_pressure_for_active_routing(monkeypatch): + now = 1_700_000_000.0 + future_reset = int(now + 300) + monkeypatch.setattr("app.modules.proxy.load_balancer.time.time", lambda: now) + monkeypatch.setattr("app.core.usage.quota.time.time", lambda: now) + + state = _state_from_account( + account=_make_test_account(status=AccountStatus.ACTIVE), + primary_entry=_make_test_usage( + window="primary", + used_percent=94.0, + reset_at=future_reset, + recorded_at=_epoch_to_naive_utc(now - 30), + ), + secondary_entry=None, + runtime=RuntimeState(inflight_streams=1), + ) + + assert state.status == AccountStatus.ACTIVE + assert state.used_percent == 96.5 + assert state.priority_used_percent is None + assert _state_above_sticky_budget_threshold(state, 95.0) is True + + def test_state_from_account_clears_stale_advisory_account_reset_for_active_account(monkeypatch): now = 1_700_000_000.0 future_reset = int(now + 300) From fc0432e53adac76a9c596a6a1e925d3b0fb778e3 Mon Sep 17 00:00:00 2001 From: Soju06 Date: Thu, 30 Jul 2026 08:19:58 +0000 Subject: [PATCH 11/64] fix(proxy): resolve local overload codes against the canonical registry The takeover of #1247 hardens selection_failure_response: instead of a private duplicate of the account-cap code set, local capacity codes are resolved via app.core.resilience.overload.LOCAL_OVERLOAD_CODES. This keeps every local overload code (including codes added later, such as the congestion fair-share code from #1536) on the stable 429 rate_limit_error contract, and guarantees the new usage_limit_reached 429 mapping is applied strictly to upstream usage/quota exhaustion of the whole eligible pool. Co-Authored-By: Claude Fable 5 --- app/modules/proxy/selection_errors.py | 18 ++++++++----- tests/unit/test_selection_errors.py | 37 +++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 7 deletions(-) diff --git a/app/modules/proxy/selection_errors.py b/app/modules/proxy/selection_errors.py index b95139cbfc..cdcfcd9e39 100644 --- a/app/modules/proxy/selection_errors.py +++ b/app/modules/proxy/selection_errors.py @@ -3,14 +3,9 @@ from typing import Protocol from app.core.errors import OpenAIErrorEnvelope, openai_error +from app.core.resilience.overload import is_local_overload_error_code USAGE_LIMIT_REACHED = "usage_limit_reached" -LOCAL_ACCOUNT_CAP_ERROR_CODES = frozenset( - { - "account_response_create_cap", - "account_stream_cap", - } -) class SelectionFailure(Protocol): @@ -20,6 +15,15 @@ class SelectionFailure(Protocol): def selection_failure_response(selection: SelectionFailure) -> tuple[int, OpenAIErrorEnvelope]: + """Map an account-selection failure to its externally visible HTTP response. + + The ``usage_limit_reached`` mapping is strictly for upstream usage/quota + exhaustion of the whole eligible pool. Local capacity codes (account caps, + admission gates, fair-share throttles) resolve against the canonical + ``LOCAL_OVERLOAD_CODES`` registry so they keep their stable 429 + ``rate_limit_error`` contract and are never reclassified as upstream usage + exhaustion or collapsed into a generic 503. + """ code = selection.error_code or "no_accounts" message = selection.error_message or "No active accounts available" if code == USAGE_LIMIT_REACHED: @@ -32,6 +36,6 @@ def selection_failure_response(selection: SelectionFailure) -> tuple[int, OpenAI resets_at=selection.resets_at, ), ) - if code in LOCAL_ACCOUNT_CAP_ERROR_CODES: + if is_local_overload_error_code(code): return 429, openai_error(code, message, error_type="rate_limit_error") return 503, openai_error(code, message) diff --git a/tests/unit/test_selection_errors.py b/tests/unit/test_selection_errors.py index 72349be1de..8a670dad33 100644 --- a/tests/unit/test_selection_errors.py +++ b/tests/unit/test_selection_errors.py @@ -1,3 +1,6 @@ +import pytest + +from app.core.resilience.overload import LOCAL_OVERLOAD_CODES from app.modules.proxy.load_balancer import AccountSelection from app.modules.proxy.selection_errors import selection_failure_response @@ -33,3 +36,37 @@ def test_unusable_pool_remains_no_accounts_503(): assert status == 503 assert payload["error"]["type"] == "server_error" assert payload["error"]["code"] == "no_accounts" + + +def test_pool_usage_exhaustion_preserves_authoritative_reset(): + status, payload = selection_failure_response( + AccountSelection( + account=None, + error_message="Rate limit exceeded. Try again in 300s", + error_code="usage_limit_reached", + resets_at=1_700_003_600, + ) + ) + + assert status == 429 + assert payload["error"]["resets_at"] == 1_700_003_600 + + +@pytest.mark.parametrize("local_code", sorted(LOCAL_OVERLOAD_CODES)) +def test_local_overload_codes_keep_rate_limit_contract(local_code: str): + # Covers every canonical local capacity code, including codes added later + # (e.g. api_key_stream_fair_share): local overload must stay a 429 + # rate_limit_error and never be reclassified as upstream usage exhaustion + # or a 503. + status, payload = selection_failure_response( + AccountSelection( + account=None, + error_message="Local capacity is exhausted", + error_code=local_code, + ) + ) + + assert status == 429 + assert payload["error"]["type"] == "rate_limit_error" + assert payload["error"]["code"] == local_code + assert "resets_at" not in payload["error"] From cc3ea7f162b6c976f75f5a4bfd6973ad2b8ee619 Mon Sep 17 00:00:00 2001 From: Soju06 Date: Thu, 30 Jul 2026 08:26:13 +0000 Subject: [PATCH 12/64] test(proxy): pin the #1246 status matrix at the Responses surface Adds externally-routed regressions for the takeover of #1247: - /v1/responses and /backend-api/codex/responses return HTTP 429 with error.type = error.code = usage_limit_reached (and authoritative error.resets_at) when every eligible account is usage-exhausted - resets_at is omitted when selection has no authoritative reset - paused/deactivated/reauth-only pools keep the pre-existing no_accounts semantics, and the synthetic SSE failure keeps the #1479 sequenced response.created/response.failed SDK stream contract - one usable account still fails over with no error Also tightens the OpenSpec delta: authoritative-reset-only resets_at, terminal (non-waitable) usage-limit failures, local capacity codes keep their rate_limit_error contract, owner-scoped exhaustion keeps continuity semantics. Co-Authored-By: Claude Fable 5 --- .../specs/responses-api-compat/spec.md | 42 +++++++- tests/integration/test_proxy_responses.py | 98 +++++++++++++++++++ tests/unit/test_load_balancer.py | 15 +++ 3 files changed, 151 insertions(+), 4 deletions(-) diff --git a/openspec/changes/report-pool-usage-exhaustion/specs/responses-api-compat/spec.md b/openspec/changes/report-pool-usage-exhaustion/specs/responses-api-compat/spec.md index fe1594b32f..65482726e5 100644 --- a/openspec/changes/report-pool-usage-exhaustion/specs/responses-api-compat/spec.md +++ b/openspec/changes/report-pool-usage-exhaustion/specs/responses-api-compat/spec.md @@ -6,10 +6,15 @@ The proxy MUST report pool-wide Responses usage exhaustion as a usage-limit error. When every account eligible for a Responses request is exhausted by known usage windows, the proxy MUST reject the request with HTTP `429` and an OpenAI-style error envelope whose `error.code` and `error.type` are both -`usage_limit_reached`. If account selection has a reset timestamp for the -exhausted pool, the response envelope MUST include that timestamp as -`error.resets_at`. The proxy MUST NOT collapse this condition into generic -`no_accounts`, `server_error`, or HTTP `503` semantics. +`usage_limit_reached`. If account selection has an authoritative upstream reset +timestamp for the exhausted pool, the response envelope MUST include that +timestamp as `error.resets_at`; the proxy MUST NOT expose the capped +human-facing retry hint or a synthesized fallback as `error.resets_at`. The +proxy MUST NOT collapse this condition into generic `no_accounts`, +`server_error`, or HTTP `503` semantics. Exhaustion classification MUST be +based on structured account state after the same eligibility filtering as +ordinary selection, and MUST NOT reclassify local capacity or overload codes +(account caps, admission gates, fair-share throttles) as usage exhaustion. #### Scenario: Public Responses request exhausts the eligible usage pool @@ -27,3 +32,32 @@ exhausted pool, the response envelope MUST include that timestamp as eligible account is usage-exhausted before downstream-visible output - **THEN** the terminal error event uses `usage_limit_reached` - **AND** clients do not receive a generic no-account/server-unavailable error + +#### Scenario: Usage-limit selection failures are terminal, not waitable + +- **WHEN** account selection fails with `usage_limit_reached` on a streaming, + HTTP-bridge, or WebSocket Responses path +- **THEN** the proxy reports the structured usage-limit failure immediately +- **AND** it does not enter an account-capacity recovery wait for the + remaining request budget before reporting it + +#### Scenario: Local capacity codes keep their rate-limit contract + +- **WHEN** account selection fails with a local capacity or overload code such + as `account_stream_cap` or `account_response_create_cap` +- **THEN** the response keeps HTTP `429` with `error.type = "rate_limit_error"` + and the stable local error code +- **AND** the response is not reported as `usage_limit_reached` + +#### Scenario: Unusable non-exhausted pools keep existing semantics + +- **WHEN** every account is paused, deactivated, or requires re-authentication + and no eligible account is exhausted by a known usage window +- **THEN** the pre-existing `no_accounts` failure semantics are preserved + +#### Scenario: Owner-scoped exhaustion preserves continuity semantics + +- **WHEN** a request is pinned to a previous-response or file owner account and + only that owner is usage-exhausted while the wider eligible pool is usable +- **THEN** the proxy keeps the existing continuity-owner failure semantics +- **AND** it does not report pool-wide `usage_limit_reached` diff --git a/tests/integration/test_proxy_responses.py b/tests/integration/test_proxy_responses.py index 087ffe6665..c9f1e576fb 100644 --- a/tests/integration/test_proxy_responses.py +++ b/tests/integration/test_proxy_responses.py @@ -175,6 +175,104 @@ async def test_proxy_responses_no_accounts(async_client): assert event["response"]["error"]["code"] == "no_accounts" +def _install_usage_limited_selection(monkeypatch, *, resets_at: int | None = 1_700_003_600) -> None: + async def fake_select_account(*_args, **_kwargs): + return proxy_module.AccountSelection( + account=None, + error_message="Rate limit exceeded. Try again in 300s", + error_code="usage_limit_reached", + resets_at=resets_at, + ) + + monkeypatch.setattr( + "app.modules.proxy.load_balancer.LoadBalancer.select_account", + fake_select_account, + ) + + +@pytest.mark.asyncio +async def test_v1_responses_pool_usage_exhaustion_returns_429(async_client, monkeypatch): + _install_usage_limited_selection(monkeypatch) + payload = {"model": "gpt-5.4", "instructions": "hi", "input": [], "stream": True} + + response = await async_client.post("/v1/responses", json=payload) + + assert response.status_code == 429 + error = response.json()["error"] + assert error["type"] == "usage_limit_reached" + assert error["code"] == "usage_limit_reached" + assert error["resets_at"] == 1_700_003_600 + + +@pytest.mark.asyncio +async def test_v1_responses_pool_usage_exhaustion_omits_unknown_reset(async_client, monkeypatch): + _install_usage_limited_selection(monkeypatch, resets_at=None) + payload = {"model": "gpt-5.4", "instructions": "hi", "input": [], "stream": True} + + response = await async_client.post("/v1/responses", json=payload) + + assert response.status_code == 429 + error = response.json()["error"] + assert error["type"] == "usage_limit_reached" + assert error["code"] == "usage_limit_reached" + assert "resets_at" not in error + + +@pytest.mark.asyncio +async def test_backend_responses_pool_usage_exhaustion_returns_429(async_client, monkeypatch): + _install_usage_limited_selection(monkeypatch) + payload = {"model": "gpt-5.4", "instructions": "hi", "input": [], "stream": True} + request_id = "req_stream_usage_limited" + + response = await async_client.post( + "/backend-api/codex/responses", + json=payload, + headers={"x-request-id": request_id}, + ) + + # Codex only classifies a terminal response as usage-limited when it sees + # both HTTP 429 and error.type == "usage_limit_reached" (#1246). + assert response.status_code == 429 + error = response.json()["error"] + assert error["type"] == "usage_limit_reached" + assert error["code"] == "usage_limit_reached" + assert error["resets_at"] == 1_700_003_600 + + +@pytest.mark.asyncio +async def test_v1_responses_mixed_unusable_pool_keeps_no_accounts_semantics(async_client, monkeypatch): + # Paused/deactivated/reauth-only pools must keep the pre-existing + # no_accounts semantics; only usage/quota exhaustion of the whole + # eligible pool may surface the new usage_limit_reached contract. + async def fake_select_account(*_args, **_kwargs): + return proxy_module.AccountSelection( + account=None, + error_message="All accounts are paused, deactivated, or require re-authentication", + error_code=None, + ) + + monkeypatch.setattr( + "app.modules.proxy.load_balancer.LoadBalancer.select_account", + fake_select_account, + ) + payload = {"model": "gpt-5.4", "instructions": "hi", "input": [], "stream": True} + + async with async_client.stream("POST", "/v1/responses", json=payload) as resp: + assert resp.status_code == 200 + lines = [line async for line in resp.aiter_lines() if line] + + # The synthetic failure keeps the #1479 SDK stream contract: a sequenced + # synthetic response.created precedes the sequenced response.failed. + created = _extract_first_raw_event(lines) + assert created["type"] == "response.created" + assert created["sequence_number"] == 0 + failed = _extract_first_event(lines) + assert failed["type"] == "response.failed" + assert failed["sequence_number"] == 1 + assert failed["response"]["error"]["code"] == "no_accounts" + assert failed["response"]["error"]["type"] == "server_error" + + @pytest.mark.asyncio async def test_backend_responses_prohibits_fast_model_alias_priority_tier(async_client, monkeypatch): raw_account_id = "acc_prohibit_fast_mode" diff --git a/tests/unit/test_load_balancer.py b/tests/unit/test_load_balancer.py index 915c3578ca..1a1a2b3c67 100644 --- a/tests/unit/test_load_balancer.py +++ b/tests/unit/test_load_balancer.py @@ -692,6 +692,21 @@ def test_select_account_reports_pool_wide_usage_exhaustion_structurally(): assert result.resets_at == int(now + 60) +def test_select_account_fails_over_when_one_account_remains_usable(): + now = 1_700_000_000.0 + states = [ + AccountState("exhausted", AccountStatus.QUOTA_EXCEEDED, used_percent=100.0, reset_at=int(now + 3600)), + AccountState("usable", AccountStatus.ACTIVE, used_percent=40.0), + ] + + result = select_account(states, now=now) + + assert result.account is not None + assert result.account.account_id == "usable" + assert result.error_code is None + assert result.error_message is None + + def test_select_account_reports_secondary_usage_exhaustion_reset(): now = 1_700_000_000.0 states = [ From d73fd3869d8ac625b75b68286f53b13ffa884914 Mon Sep 17 00:00:00 2001 From: Soju06 Date: Thu, 30 Jul 2026 08:26:40 +0000 Subject: [PATCH 13/64] style: format unbound selection cap fallback Co-Authored-By: Claude Fable 5 --- app/modules/proxy/_load_balancer/unbound_selection.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/app/modules/proxy/_load_balancer/unbound_selection.py b/app/modules/proxy/_load_balancer/unbound_selection.py index cb858a89c0..11c5a5e7b3 100644 --- a/app/modules/proxy/_load_balancer/unbound_selection.py +++ b/app/modules/proxy/_load_balancer/unbound_selection.py @@ -198,9 +198,7 @@ def _direct_error( and result.error_code is None and lease_kind is not None and len(selection_states) < len(states) - and any( - state.status == AccountStatus.ACTIVE for state in states if state not in selection_states - ) + and any(state.status == AccountStatus.ACTIVE for state in states if state not in selection_states) ): selection_error_code = _account_cap_error_code(lease_kind) result = SelectionResult( From 39283463e86bbbbb5e8f0581b8348863beb6ae49 Mon Sep 17 00:00:00 2001 From: mastertyko <11311479+mastertyko@users.noreply.github.com> Date: Thu, 30 Jul 2026 12:50:25 +0200 Subject: [PATCH 14/64] fix(cache): recover invalidation after failed prime --- app/core/cache/invalidation.py | 18 +++- app/main.py | 12 +-- .../.openspec.yaml | 2 + .../design.md | 60 ++++++++++++ .../proposal.md | 28 ++++++ .../specs/model-catalog-compat/spec.md | 96 +++++++++++++++++++ .../specs/query-caching/spec.md | 46 +++++++++ .../tasks.md | 15 +++ .../test_cache_invalidation_bus.py | 78 +++++++++++++++ .../test_upstream_route_cache_invalidation.py | 55 ++++++++++- tests/unit/test_cache_invalidation_poller.py | 64 +++++++++++++ 11 files changed, 464 insertions(+), 10 deletions(-) create mode 100644 openspec/changes/recover-cache-invalidation-after-prime-failure/.openspec.yaml create mode 100644 openspec/changes/recover-cache-invalidation-after-prime-failure/design.md create mode 100644 openspec/changes/recover-cache-invalidation-after-prime-failure/proposal.md create mode 100644 openspec/changes/recover-cache-invalidation-after-prime-failure/specs/model-catalog-compat/spec.md create mode 100644 openspec/changes/recover-cache-invalidation-after-prime-failure/specs/query-caching/spec.md create mode 100644 openspec/changes/recover-cache-invalidation-after-prime-failure/tasks.md create mode 100644 tests/unit/test_cache_invalidation_poller.py diff --git a/app/core/cache/invalidation.py b/app/core/cache/invalidation.py index 101cba3413..587108ea29 100644 --- a/app/core/cache/invalidation.py +++ b/app/core/cache/invalidation.py @@ -89,8 +89,10 @@ async def initialize(self) -> None: On success ``_poll_initialized`` is set to ``True``. If the read fails the method raises with state unchanged (baseline empty, ``_poll_initialized`` - still ``False``), so the caller can degrade to the first-poll-baselines - behavior. + still ``False``), so the caller can retry before background polling. If + the caller continues, ``start()`` arms conservative callback delivery for + the first successfully observed versions instead of accepting them as a + callback-less baseline. """ session = self._session_factory() try: @@ -118,7 +120,10 @@ async def prime(self) -> None: Mirrors ``initialize``'s error contract: if the baseline read fails the poller stays uninitialized (``_poll_initialized`` still ``False``) and - this method raises so the caller can retry or explicitly degrade. + this method raises so the caller can retry before background polling. + Starting without a successful retry makes the first recovered poll + reconcile positive versions through their callbacks before acknowledging + them. ``_poll_once`` swallows the read error, so a silent success here would let the first *background* poll absorb a peer bump as the initial baseline, voiding the delivery guarantee priming exists to provide. @@ -129,6 +134,13 @@ async def prime(self) -> None: async def start(self) -> None: if self._task and not self._task.done(): return + # Callback-less baseline acquisition is safe only before process-local + # state can be served. Once background polling starts, a missing baseline + # means the observed versions are uncertain: treat each positive first + # observation as a change and reconcile it through the normal callback / + # acknowledgement path. A successful prime already set this flag after + # recording exact versions, so normal startup remains unchanged. + self._poll_initialized = True self._stop.clear() self._task = asyncio.create_task(self._run()) diff --git a/app/main.py b/app/main.py index df28262a46..eab8576e2c 100644 --- a/app/main.py +++ b/app/main.py @@ -358,12 +358,12 @@ async def lifespan(app: FastAPI): try: await cache_poller.prime() except Exception: - # prime() raises when the baseline version read fails; degrade to - # first-poll-baselines (matching initialize()'s contract) rather than - # continuing as if the seed succeeded. A peer bump landing before the - # first background poll may then be absorbed as the initial baseline - # and only converge on the fallback TTL / next bump, but the failure is - # surfaced here instead of silently voiding the delivery guarantee. + # prime() raises when the baseline version read fails, leaving the poller + # uninitialized so an explicit retry would remain baseline-only. Startup + # continues, but start() arms conservative recovery: the first successful + # background read invokes callbacks for positive versions before + # acknowledging them, so a peer bump cannot become a callback-less + # baseline after local caches are warm. logger.warning("cache invalidation baseline prime failed", exc_info=True) try: await routing_availability_cache.refresh_from_db() diff --git a/openspec/changes/recover-cache-invalidation-after-prime-failure/.openspec.yaml b/openspec/changes/recover-cache-invalidation-after-prime-failure/.openspec.yaml new file mode 100644 index 0000000000..ab39675458 --- /dev/null +++ b/openspec/changes/recover-cache-invalidation-after-prime-failure/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-07-30 diff --git a/openspec/changes/recover-cache-invalidation-after-prime-failure/design.md b/openspec/changes/recover-cache-invalidation-after-prime-failure/design.md new file mode 100644 index 0000000000..0edf091caa --- /dev/null +++ b/openspec/changes/recover-cache-invalidation-after-prime-failure/design.md @@ -0,0 +1,60 @@ +## Context + +`CacheInvalidationPoller.prime()` normally records every namespace version before startup warms process-local caches. If that read fails, startup logs the failure, warms the routing and model-registry state, and starts background polling with no known versions. The first successful poll currently treats every observed row as a callback-less baseline, even if a peer advanced it after the failed read. + +The poller therefore has two distinct lifecycle phases: explicit baseline acquisition before process-local state is served, and background reconciliation after that state may be warm. Only the former may accept an observed version without invoking its callbacks. + +## Goals / Non-Goals + +**Goals:** + +- Recover conservatively after a failed startup baseline read. +- Run registered callbacks before acknowledging positive versions first observed during background polling. +- Preserve callback retry and monotonic version-acknowledgement behavior. +- Preserve baseline-only behavior when `prime()` is explicitly retried before background polling. +- Prove the fix at both the warmed upstream-route resolver cache and the account-routing / bridge-session reuse seam. + +**Non-Goals:** + +- Failing startup or adding startup retries. +- Changing namespace versions, bump ordering, callback registration, poll intervals, or cache TTLs. +- Adding configuration, schema, migrations, or cross-replica payloads. +- Changing normal startup behavior when baseline priming succeeds. + +## Decisions + +### Background start ends callback-less baseline acquisition + +`start()` will transition an uninitialized poller into conservative background-reconciliation mode before it creates the polling task. A successful prime has already made this transition with recorded versions. After a failed prime, the transition makes a positive version with no known baseline satisfy the existing change predicate, so the namespace callback runs before acknowledgement. + +This uses the existing `_poll_initialized` state rather than adding a second failure latch. The relevant distinction is not whether one particular read failed, but whether the caller is still explicitly acquiring a pre-service baseline or has started background polling after caches may be warm. + +Alternatives considered: + +- Mark the poller initialized inside the `prime()` failure path: rejected because an explicit `prime()` retry could then invoke callbacks even though its documented purpose is baseline-only acquisition. +- Retry or fail startup: rejected because it changes availability policy and is broader than the stale-cache defect. +- Clear routing caches directly in `app.main` after the exception: rejected because it duplicates callback wiring, does not cover later first-observed namespace rows, and leaves other poller consumers inconsistent. +- Add a separate recovery latch: workable, but redundant with the existing lifecycle boundary and adds state combinations without improving the guarantee. + +### Recovery uses normal callback acknowledgement rules + +Recovery will use `_run_callbacks()` and the existing per-namespace acknowledgement path. A callback failure leaves that namespace unacknowledged and the next poll retries it. Existing monotonic handling for concurrent `bump_local()` acknowledgements remains unchanged. + +### The integration proofs keep the real routing gates + +The regressions will use two pollers sharing the integration database. One keeps a resolved upstream-route outcome warm across a failed prime and a peer `upstream_route` bump. The other keeps a routing snapshot seeded `ACTIVE` across a peer status change plus `account_routing` bump and checks the actual bridge-session reuse predicate. Together they prove that recovery changes externally relevant routing decisions, not merely a callback counter. + +## Risks / Trade-offs + +- [A failed prime followed by background start may replay callbacks for versions that predate startup] → Callbacks are designed to be idempotent local reconciliation; this conservative replay happens only on the exceptional no-baseline path and is safer than serving unknown cache state. +- [A recovery callback can fail] → The existing unacknowledged-version retry path remains authoritative and is covered by focused tests. +- [The background task could poll before recovery mode is active] → Transition lifecycle state synchronously in `start()` before creating the task. +- [A namespace has no row yet] → No callback runs until a later bump creates a positive version; that first observed version is then treated as changed. + +## Migration Plan + +No data or configuration migration is required. Deploy as an application-only change. Rollback restores the previous callback-less first-poll fallback after a failed prime; no persisted state requires reversal. + +## Open Questions + +None. diff --git a/openspec/changes/recover-cache-invalidation-after-prime-failure/proposal.md b/openspec/changes/recover-cache-invalidation-after-prime-failure/proposal.md new file mode 100644 index 0000000000..c403ef2b15 --- /dev/null +++ b/openspec/changes/recover-cache-invalidation-after-prime-failure/proposal.md @@ -0,0 +1,28 @@ +## Why + +A transient failure while priming cache-invalidation versions leaves the poller without a baseline, but startup continues and warms routing state. A peer mutation that lands afterward can then be accepted by the first successful background poll as a callback-less baseline, leaving a stale routing decision in service until another bump or restart. + +## What Changes + +- Make background polling recover fail-safe when no version baseline was recorded: observed positive namespace versions are delivered through their callbacks before they are acknowledged. +- Preserve baseline-only semantics for an explicit `prime()` retry before background polling begins. +- Cover the production-sensitive upstream-route cache and account-routing bridge-reuse paths with two-replica regressions that prove warmed decisions are invalidated after the version read recovers. +- Replace the model-catalog contract's documented callback-less degradation with the same conservative recovery behavior. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `query-caching`: Require the first successful background poll after a failed startup baseline read to reconcile observed namespaces before acknowledging their versions. +- `model-catalog-compat`: Keep surfacing a failed baseline prime while requiring background recovery to invoke the model-registry callback instead of accepting a callback-less first-poll baseline. + +## Impact + +- `app/core/cache/invalidation.py`: background-poller lifecycle state. +- `app/main.py`: startup failure-semantics documentation. +- Cache-invalidation integration coverage for the warmed account-routing / bridge-reuse path and existing model-registry startup behavior. +- No API, configuration, dependency, database-schema, or migration change. diff --git a/openspec/changes/recover-cache-invalidation-after-prime-failure/specs/model-catalog-compat/spec.md b/openspec/changes/recover-cache-invalidation-after-prime-failure/specs/model-catalog-compat/spec.md new file mode 100644 index 0000000000..2f3a744854 --- /dev/null +++ b/openspec/changes/recover-cache-invalidation-after-prime-failure/specs/model-catalog-compat/spec.md @@ -0,0 +1,96 @@ +## MODIFIED Requirements + +### Requirement: Refreshed model catalog is replica-coherent + +The leader refresh cycle SHALL persist the complete registry state (models, plan maps, per-account tier maps, suppression set, authoritative flags, metadata retention state, and the refresh wall-clock timestamp) to the single-row `model_registry_snapshot` table and SHALL bump the `model_registry` cache-invalidation namespace only after the persist commits (write-then-bump). The payload write and the bump SHALL be skipped when the serialized content hash is unchanged from the last persisted state AND the stored row was still within `model_registry_snapshot_max_age_seconds`; the stored `refreshed_at` timestamp SHALL still be advanced so snapshot age reflects the leader's latest successful refresh. When the content hash is unchanged but the stored row had already aged past `model_registry_snapshot_max_age_seconds` before this refresh revived it, the leader SHALL still bump the `model_registry` namespace (only the payload rewrite stays skipped): an expired row causes followers to clear their local registry and reset their applied-content-hash marker, so an unchanged-content revival still requires a bump for them to re-apply within the cache-invalidation poll bound instead of waiting for the non-leader scheduler backstop. Every replica MUST apply a newly persisted snapshot within the cache-invalidation poll bound and MUST invalidate its local account-selection cache on apply; that account-selection invalidation MUST be local-only (non-propagating), because reconcile only applies a change the leader already published (which bumped `model_registry` to reach every replica) and each replica clears its own selection cache on apply, so a propagating clear would make every follower durably re-bump `account_selection` and amplify bus traffic with no peer-visible effect. When the reconcile is driven by the `model_registry` invalidation callback and the snapshot load fails (transient DB read error or malformed payload), the callback MUST surface the failure to the invalidation poller so the poller leaves the `model_registry` version unacknowledged and retries on the next poll cycle (matching the `account_routing` refresh callback), rather than acknowledging the bump and stranding the replica on the stale catalog until the non-leader scheduler backstop; the startup one-shot reconcile and the refresh-tick backstop instead swallow such a load failure (keeping the current in-memory state) so they never fail startup or the scheduler loop. Payload decode MUST treat a set-backed or mapping-backed catalog field whose persisted value has the wrong type — for example a `model_plans`/`plan_models`/`model_accounts`/per-account tier entry persisted as a scalar or object where a list of slugs is expected, or a model entry that is not an object — as a malformed payload and raise, rather than silently dropping the offending entry and applying a partial catalog; a genuinely-absent or empty container (an absent key, an empty map, or an empty list) is not malformed and MUST decode successfully. After apply, `/v1/models`, plan gating (`plan_types_for_model`), suppression (`is_suppressed_model`), and per-account service-tier routing on a non-leader MUST be identical to the leader. A non-leader refresh tick MUST NOT fetch the upstream catalog and SHALL instead reconcile from the persisted snapshot when the stored snapshot header differs from the last applied one (backstop for a lost invalidation bump). A leader catalog clear SHALL persist an explicit cleared marker and bump, so followers revert to the bootstrap floor rather than serving a withdrawn catalog. Every replica SHALL install its `model_registry` cache-invalidation callback (the global invalidation poller) before starting the model refresh scheduler, so a first leader tick that persists a changed snapshot cannot silently drop its bump. Every replica SHALL record the invalidation-poller version baseline before running its one-shot startup reconcile, so a leader bump that lands in the window between that reconcile's snapshot read and the poller's first background tick is delivered as an invalidation callback (within the poll bound) rather than absorbed as the poller's initial callback-less baseline (which would defer convergence to the non-leader scheduler backstop). The baseline-priming read SHALL surface a failure to its caller and leave the poller without a recorded baseline. If startup continues, the first successful background poll MUST conservatively treat a positive `model_registry` version observed without a baseline as changed, invoke the reconcile callback, and acknowledge it only after that callback succeeds. This MAY replay a pre-startup version, but MUST NOT absorb a peer bump as a callback-less baseline. + +#### Scenario: Follower serves the refreshed catalog on /v1/models + +- **GIVEN** replica A (leader) completes a registry refresh whose catalog adds a new slug and withdraws a bootstrap slug +- **AND** replica A persists the snapshot and bumps the `model_registry` namespace +- **WHEN** replica B's cache-invalidation poller observes the version change +- **THEN** replica B applies the snapshot to its in-memory registry +- **AND** `GET /v1/models` served by replica B lists the new slug and omits the withdrawn slug + +#### Scenario: Follower enforces suppression of a withdrawn slug + +- **GIVEN** the leader's refreshed snapshot marks a previously served slug as suppressed +- **WHEN** a follower applies the persisted snapshot +- **THEN** `is_suppressed_model` returns true for that slug on the follower + +#### Scenario: Follower enforces plan gating for a newly gated slug + +- **GIVEN** the leader's refreshed snapshot maps a slug to exactly one plan type +- **WHEN** a follower applies the persisted snapshot +- **THEN** `plan_types_for_model` on the follower returns exactly that plan set instead of no filtering + +#### Scenario: Catalog clear propagates to followers + +- **GIVEN** the leader clears the registry because no active accounts remain +- **WHEN** the leader persists the cleared marker and bumps, and a follower applies it +- **THEN** the follower reverts to the bootstrap catalog floor + +#### Scenario: Lost bump converges via the refresh-tick backstop + +- **GIVEN** a snapshot was persisted but the invalidation bump was lost +- **WHEN** a non-leader replica's next refresh tick runs +- **THEN** the replica detects the header mismatch, applies the persisted snapshot, and converges within one refresh interval + +#### Scenario: Transient load failure in the callback is retried, not acknowledged + +- **GIVEN** the leader persisted a changed snapshot and bumped the `model_registry` namespace +- **AND** a follower's snapshot load transiently fails on the invalidation callback (e.g. a DB read error or a momentarily unreadable payload) +- **WHEN** the follower's poll cycle runs the callback and it fails +- **THEN** the poller does not acknowledge the observed `model_registry` version and retries the callback on the next poll cycle +- **AND** once the transient failure clears, the retry applies the persisted snapshot within the poll bound without requiring a new leader bump + +#### Scenario: Malformed set-backed field is rejected, not silently dropped + +- **GIVEN** the leader bumped the `model_registry` namespace and the persisted payload is valid JSON but a set-backed field is wrong-typed (e.g. `model_plans` maps a slug to `{"gpt-x": "pro"}` instead of a list of plan slugs) +- **WHEN** a follower's invalidation callback loads and decodes the payload +- **THEN** the decode raises rather than dropping the offending entry +- **AND** the poller leaves the `model_registry` version unacknowledged and no partial catalog is applied (the follower keeps its prior in-memory state and retries on the next poll) + +#### Scenario: Empty set-backed maps decode successfully + +- **GIVEN** a persisted snapshot whose set-backed fields are genuinely empty (empty maps, or a slug mapped to an empty list) +- **WHEN** a replica decodes the payload +- **THEN** the decode succeeds and the corresponding sets are empty (empty is not treated as malformed) + +#### Scenario: Applying a snapshot does not re-bump account_selection + +- **GIVEN** the leader persisted a changed snapshot and bumped `model_registry` +- **WHEN** a follower applies the snapshot and invalidates its local account-selection cache +- **THEN** the follower does not enqueue or write an `account_selection` cache-invalidation bump + +#### Scenario: Non-leader tick performs no upstream fetch + +- **WHEN** a non-leader replica's refresh tick runs +- **THEN** it performs no upstream model-catalog fetch, regardless of whether it reconciled from the store + +#### Scenario: First leader bump is not dropped at startup + +- **GIVEN** a replica is starting up +- **WHEN** the model refresh scheduler starts +- **THEN** the global cache-invalidation poller with the `model_registry` callback is already installed, so an immediate leader persist-and-bump reaches followers within the poll bound + +#### Scenario: Bump during the startup reconcile window is not dropped + +- **GIVEN** a replica is starting up and has recorded the invalidation-poller version baseline +- **AND** a leader persists a changed snapshot and bumps the `model_registry` namespace in the window between the replica's one-shot startup reconcile and the poller's first background tick +- **WHEN** the poller's first background tick runs +- **THEN** it observes the version advanced past the recorded baseline and invokes the reconcile callback, so the replica applies the new snapshot within the poll bound rather than waiting for the non-leader scheduler backstop + +#### Scenario: Reviving an expired unchanged snapshot bumps the bus + +- **GIVEN** a snapshot was persisted with content hash H and its stored row then aged past `model_registry_snapshot_max_age_seconds`, so followers dropped to the bootstrap floor and reset their applied-content-hash marker +- **WHEN** the leader's next refresh succeeds with the same catalog bytes (content hash H again) +- **THEN** the leader advances `refreshed_at` without rewriting the payload but still bumps the `model_registry` namespace +- **AND** the followers observe the version change and re-apply the revived snapshot within the poll bound rather than waiting for the non-leader scheduler backstop + +#### Scenario: Failed startup baseline prime recovers through reconciliation + +- **GIVEN** a replica's baseline-priming read fails transiently and no `model_registry` version baseline is recorded +- **WHEN** its first successful background poll observes a positive `model_registry` version +- **THEN** the poller MUST invoke the model-registry reconcile callback before acknowledging that version +- **AND** the replica MUST NOT defer convergence to the scheduler backstop merely because startup baseline priming failed diff --git a/openspec/changes/recover-cache-invalidation-after-prime-failure/specs/query-caching/spec.md b/openspec/changes/recover-cache-invalidation-after-prime-failure/specs/query-caching/spec.md new file mode 100644 index 0000000000..f3d1c106ad --- /dev/null +++ b/openspec/changes/recover-cache-invalidation-after-prime-failure/specs/query-caching/spec.md @@ -0,0 +1,46 @@ +## MODIFIED Requirements + +### Requirement: Cache invalidation bumps and polling are resilient and observable + +`bump()` MUST retry transient write failures (including SQLite "database is locked") with a short backoff; on final failure it MUST log at ERROR with the namespace, increment `codex_lb_cache_invalidation_bump_failures_total{namespace}`, and MUST NOT fail the originating mutation. Coalesced (`request_bump`) namespaces MUST remain pending and be retried on subsequent poll cycles until a bump succeeds, and a `request_bump` arriving while a flush for the same namespace is already awaiting its bump MUST be preserved and produce a later bump. When any invalidation callback for a namespace fails, the poller MUST NOT acknowledge the observed version and MUST re-run that namespace's callbacks on subsequent poll cycles until they succeed. The poller MUST escalate consecutive poll failures above debug level after a bounded count (WARNING after 3, ERROR after 10) and increment `codex_lb_cache_invalidation_poll_failures_total`. After a startup baseline read fails, a process that continues without a recorded baseline MUST treat each positive version first observed for a registered namespace by the next successful background poll as changed, run that namespace's registered callbacks, and acknowledge the version only after those callbacks succeed. This recovery MAY cause a redundant invalidation for a version that predates startup; it MUST NOT silently absorb a peer bump into a callback-less baseline. + +#### Scenario: Bump failure under database lock is observable and does not fail the mutation + +- **GIVEN** the database rejects cache-invalidation writes with a lock error for longer than the retry budget +- **WHEN** a mutation attempts a durable namespace bump +- **THEN** the mutation itself still succeeds +- **AND** an ERROR log naming the namespace is emitted and the bump-failure counter increments + +#### Scenario: Pending coalesced namespace flushes on the next successful cycle + +- **GIVEN** a coalesced `request_bump` namespace failed to flush during a poll cycle +- **WHEN** the database becomes writable again +- **THEN** the next poll cycle flushes the pending namespace and increments its version + +#### Scenario: Bump requested during an in-flight flush produces a later bump + +- **GIVEN** a coalesced flush is awaiting the bump write for a namespace +- **WHEN** another mutation commits and requests a bump for the same namespace before the flush completes +- **THEN** the namespace is re-queued and flushed again on a subsequent cycle, incrementing the version beyond the in-flight bump + +#### Scenario: Failed invalidation callback keeps the version unacknowledged and is retried + +- **GIVEN** a replica observes an `account_routing` version bump +- **AND** its routing snapshot refresh fails with a transient database error +- **WHEN** the poll cycle completes +- **THEN** the replica does not record the new version as seen +- **AND** the refresh is retried on subsequent poll cycles until it succeeds + +#### Scenario: Consecutive poll failures escalate above debug + +- **GIVEN** a replica's poller cannot read the `cache_invalidation` table +- **WHEN** three consecutive polls fail +- **THEN** a WARNING is logged and the poll-failure counter increments + +#### Scenario: Failed startup prime cannot absorb a route-cache bump + +- **GIVEN** replica B's startup cache-invalidation baseline read fails and no `upstream_route` version is recorded +- **AND** replica B continues serving traffic and warms an upstream-route resolution cache entry +- **WHEN** replica A commits a route-input mutation and advances `upstream_route` before replica B's first successful version read +- **THEN** replica B's first successful background poll MUST run the registered `upstream_route` invalidation callback before acknowledging the observed version +- **AND** the warmed route entry MUST be cleared in that poll instead of remaining stale until its TTL or a later bump diff --git a/openspec/changes/recover-cache-invalidation-after-prime-failure/tasks.md b/openspec/changes/recover-cache-invalidation-after-prime-failure/tasks.md new file mode 100644 index 0000000000..6ff3b9379a --- /dev/null +++ b/openspec/changes/recover-cache-invalidation-after-prime-failure/tasks.md @@ -0,0 +1,15 @@ +## 1. Regression Coverage + +- [x] 1.1 Add deterministic two-replica integration regressions proving a failed baseline prime cannot absorb later `upstream_route` or `account_routing` bumps while stale routing decisions remain warm. +- [x] 1.2 Add a hermetic unit regression for failed prime, background start, callback delivery, and acknowledgement. + +## 2. Poller Recovery + +- [x] 2.1 Transition an uninitialized poller to conservative callback delivery before background polling starts, while preserving explicit `prime()` retry semantics. +- [x] 2.2 Update startup lifecycle documentation to describe callback-based recovery after a failed prime. + +## 3. Verification + +- [x] 3.1 Run the focused cache-invalidation, route-cache, model-registry startup, and bridge-reuse integration checks required for Sensitive routing/cache work. +- [x] 3.2 Run scoped lint/format checks and strict validation for the change plus affected main specs. +- [x] 3.3 Review the final diff and worktree status; record any untested or blocked checks. diff --git a/tests/integration/test_cache_invalidation_bus.py b/tests/integration/test_cache_invalidation_bus.py index 4438a00de6..fd38a4998f 100644 --- a/tests/integration/test_cache_invalidation_bus.py +++ b/tests/integration/test_cache_invalidation_bus.py @@ -8,6 +8,7 @@ from __future__ import annotations +import asyncio import logging from datetime import datetime, timezone from types import SimpleNamespace @@ -162,6 +163,60 @@ async def test_remote_pause_stops_stale_bridge_session_reuse(db_setup, poller_sl assert _http_bridge_session_account_active(stale_session) is False +@pytest.mark.asyncio +async def test_failed_prime_recovery_stops_stale_bridge_session_reuse(db_setup, poller_slot, monkeypatch) -> None: + """A failed startup version read must not let the first recovered poll + acknowledge a later peer pause without refreshing the warmed routing state.""" + account_id = "acct-bus-prime-recovery" + await _insert_account(account_id) + + # A namespace row already exists before this replica starts, but its + # baseline read fails transiently. + remote_poller = CacheInvalidationPoller(SessionLocal) + assert await remote_poller.bump(NAMESPACE_ACCOUNT_ROUTING) is True + flaky_versions = _FlakySessionFactory(failures=1) + local_poller = CacheInvalidationPoller(flaky_versions) + routing_cache = RoutingAvailabilityCache(SessionLocal) + monkeypatch.setattr("app.modules.proxy.account_cache._routing_availability_cache", routing_cache) + set_cache_invalidation_poller(local_poller) + + callback_calls = 0 + + async def refresh_routing_snapshot() -> None: + nonlocal callback_calls + callback_calls += 1 + await routing_cache.refresh_from_db() + + local_poller.on_invalidation(NAMESPACE_ACCOUNT_ROUTING, refresh_routing_snapshot) + + with pytest.raises(RuntimeError, match="baseline version read did not complete"): + await local_poller.prime() + + # Startup continues and warms an ACTIVE snapshot after the failed prime. + await routing_cache.refresh_from_db() + stale_session = _fake_bridge_session(_make_account(account_id, AccountStatus.ACTIVE)) + assert _http_bridge_session_account_active(stale_session) is True + + # A peer pauses the account and advances the version before this replica's + # first successful background read. + await _set_account_status(account_id, AccountStatus.PAUSED) + assert await remote_poller.bump(NAMESPACE_ACCOUNT_ROUTING) is True + + parked = asyncio.Event() + + async def park_background_loop() -> None: + await parked.wait() + + monkeypatch.setattr(local_poller, "_run", park_background_loop) + await local_poller.start() + try: + await local_poller._poll_once() + assert callback_calls == 1 + assert _http_bridge_session_account_active(stale_session) is False + finally: + await local_poller.stop() + + @pytest.mark.asyncio async def test_reauth_on_peer_clears_local_routing_marker(db_setup, poller_slot) -> None: """A routing-unavailable marker set locally is cleared when another replica @@ -608,6 +663,29 @@ async def test_initialize_failure_leaves_poller_uninitialized(db_setup) -> None: assert poller._known_versions == {} +@pytest.mark.asyncio +async def test_prime_retry_after_failure_remains_baseline_only(db_setup) -> None: + """Retrying prime before background start records a baseline without + callbacks; conservative recovery begins only when background polling starts.""" + remote_poller = CacheInvalidationPoller(SessionLocal) + assert await remote_poller.bump(NAMESPACE_ACCOUNT_ROUTING) is True + expected_version = await _namespace_version(NAMESPACE_ACCOUNT_ROUTING) + assert expected_version is not None + + calls: list[str] = [] + poller = CacheInvalidationPoller(_FlakySessionFactory(failures=1)) + poller.on_invalidation(NAMESPACE_ACCOUNT_ROUTING, lambda: calls.append("routing")) + + with pytest.raises(RuntimeError, match="baseline version read did not complete"): + await poller.prime() + await poller.prime() + + assert calls == [] + assert await remote_poller.bump(NAMESPACE_ACCOUNT_ROUTING) is True + await poller._poll_once() + assert calls == ["routing"] + + @pytest.mark.asyncio async def test_bump_local_suppresses_source_callback_but_peer_still_fires(db_setup) -> None: """A replica that has already invalidated locally uses ``bump_local`` so its diff --git a/tests/integration/test_upstream_route_cache_invalidation.py b/tests/integration/test_upstream_route_cache_invalidation.py index c5fdb21f8c..6406f1dd5e 100644 --- a/tests/integration/test_upstream_route_cache_invalidation.py +++ b/tests/integration/test_upstream_route_cache_invalidation.py @@ -6,15 +6,18 @@ import pytest from sqlalchemy import select +from sqlalchemy.exc import OperationalError +from sqlalchemy.ext.asyncio import AsyncSession from app.core.auth import generate_unique_account_id from app.core.cache.invalidation import ( NAMESPACE_SETTINGS, NAMESPACE_UPSTREAM_ROUTE, + CacheInvalidationPoller, get_cache_invalidation_poller, ) from app.core.config.settings import get_settings -from app.core.upstream_proxy.cache import get_upstream_route_cache +from app.core.upstream_proxy.cache import UpstreamRouteCache, get_upstream_route_cache from app.db.models import CacheInvalidation from app.db.session import SessionLocal @@ -84,6 +87,56 @@ def _seed_dummy_entry() -> None: assert cache.get("seeded-account") is not None +class _FailFirstVersionSessionFactory: + def __init__(self) -> None: + self._failed = False + + def __call__(self) -> AsyncSession: + if not self._failed: + self._failed = True + raise OperationalError("stmt", {}, Exception("peer-version read failed")) + return SessionLocal() + + +async def test_failed_prime_recovery_clears_warm_route_cache(db_setup, route_cache_ttl, monkeypatch) -> None: + """A route-cache bump landing after a failed startup prime must clear a warm + resolver outcome on the first successful background version read.""" + remote_poller = CacheInvalidationPoller(SessionLocal) + assert await remote_poller.bump(NAMESPACE_UPSTREAM_ROUTE) is True + + route_cache = UpstreamRouteCache() + callback_calls = 0 + + def clear_route_cache() -> None: + nonlocal callback_calls + callback_calls += 1 + route_cache.clear() + + local_poller = CacheInvalidationPoller(_FailFirstVersionSessionFactory()) + local_poller.on_invalidation(NAMESPACE_UPSTREAM_ROUTE, clear_route_cache) + + with pytest.raises(RuntimeError, match="baseline version read did not complete"): + await local_poller.prime() + route_cache.store_route("seeded-account", None, generation=route_cache.generation) + assert route_cache.get("seeded-account") is not None + + assert await remote_poller.bump(NAMESPACE_UPSTREAM_ROUTE) is True + + parked = asyncio.Event() + + async def park_background_loop() -> None: + await parked.wait() + + monkeypatch.setattr(local_poller, "_run", park_background_loop) + await local_poller.start() + try: + await local_poller._poll_once() + assert callback_calls == 1 + assert route_cache.get("seeded-account") is None + finally: + await local_poller.stop() + + async def test_binding_upsert_clears_cache_and_bumps_namespace(async_client, route_cache_ttl) -> None: account_id = await _import_account(async_client, "acc-route-cache-binding", "route-cache-binding@example.com") pool_id = await _create_pool_with_endpoint(async_client) diff --git a/tests/unit/test_cache_invalidation_poller.py b/tests/unit/test_cache_invalidation_poller.py new file mode 100644 index 0000000000..9898c756ee --- /dev/null +++ b/tests/unit/test_cache_invalidation_poller.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +import asyncio +from typing import cast + +import pytest +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.cache.invalidation import NAMESPACE_UPSTREAM_ROUTE, CacheInvalidationPoller + + +class _VersionRows: + @staticmethod + def all() -> list[tuple[str, int]]: + return [(NAMESPACE_UPSTREAM_ROUTE, 2)] + + +class _ReadableSession: + def in_transaction(self) -> bool: + return False + + async def execute(self, *_args: object, **_kwargs: object) -> _VersionRows: + return _VersionRows() + + async def close(self) -> None: + return None + + +class _FailFirstVersionSessionFactory: + def __init__(self) -> None: + self._failed = False + + def __call__(self) -> AsyncSession: + if not self._failed: + self._failed = True + raise RuntimeError("peer-version read failed") + return cast(AsyncSession, _ReadableSession()) + + +@pytest.mark.asyncio +async def test_background_start_reconciles_first_version_after_failed_prime(monkeypatch) -> None: + calls: list[str] = [] + poller = CacheInvalidationPoller(_FailFirstVersionSessionFactory()) + poller.on_invalidation(NAMESPACE_UPSTREAM_ROUTE, lambda: calls.append("clear")) + + with pytest.raises(RuntimeError, match="baseline version read did not complete"): + await poller.prime() + + parked = asyncio.Event() + + async def park_background_loop() -> None: + await parked.wait() + + monkeypatch.setattr(poller, "_run", park_background_loop) + await poller.start() + try: + await poller._poll_once() + await poller._poll_once() + finally: + await poller.stop() + + # The first successful poll reconciles and acknowledges version 2; the + # unchanged second observation must not invoke the callback again. + assert calls == ["clear"] From 7545d958d10943d44d946f9d4d51c8452076b88b Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Thu, 30 Jul 2026 16:11:31 +0400 Subject: [PATCH 15/64] fix(proxy): retry silent bridge response.create upstreams --- .../_service/http_bridge/upstream_events.py | 34 +++--- .../integration/test_http_responses_bridge.py | 102 ++++++++++++++++++ 2 files changed, 122 insertions(+), 14 deletions(-) diff --git a/app/modules/proxy/_service/http_bridge/upstream_events.py b/app/modules/proxy/_service/http_bridge/upstream_events.py index a70367af9b..72dbe46896 100644 --- a/app/modules/proxy/_service/http_bridge/upstream_events.py +++ b/app/modules/proxy/_service/http_bridge/upstream_events.py @@ -624,16 +624,6 @@ async def _relay_http_bridge_upstream_messages( if not expired_owner: continue pending_count = len(session.pending_requests) - for request_state in session.pending_requests: - if request_state.failure_phase_override is None: - request_state.failure_phase_override = "upstream" - if request_state.failure_detail_override is None: - request_state.failure_detail_override = ( - _HTTP_BRIDGE_MISSING_RESPONSE_CREATED_TIMEOUT_DETAIL - ) - # Claim the session before cancelling receive so a - # gate waiter cannot reopen this ambiguous socket. - session.closed = True if receive_task is not None: receive_cancelled = await _cancel_http_bridge_reader_child( receive_task, @@ -641,10 +631,6 @@ async def _relay_http_bridge_upstream_messages( ) if receive_cancelled: receive_task = None - _record_http_bridge_stuck_retire( - reason=_HTTP_BRIDGE_MISSING_RESPONSE_CREATED_TIMEOUT_DETAIL, - session=session, - ) _log_http_bridge_event( "missing_response_created_timeout", session.key, @@ -657,6 +643,26 @@ async def _relay_http_bridge_upstream_messages( _extract_model_class(session.request_model) if session.request_model else None ), ) + retried = await self._retry_http_bridge_precreated_request(session) + if retried: + continue + # Claim the session only after the safe pre-created + # retry path refuses it. Reconnecting first prevents + # a silent upstream websocket from stranding clients + # until the request budget expires. + session.closed = True + _record_http_bridge_stuck_retire( + reason=_HTTP_BRIDGE_MISSING_RESPONSE_CREATED_TIMEOUT_DETAIL, + session=session, + ) + async with session.pending_lock: + for request_state in session.pending_requests: + if request_state.failure_phase_override is None: + request_state.failure_phase_override = "upstream" + if request_state.failure_detail_override is None: + request_state.failure_detail_override = ( + _HTTP_BRIDGE_MISSING_RESPONSE_CREATED_TIMEOUT_DETAIL + ) await self._fail_http_bridge_reader_and_maybe_retire( session, error_code="upstream_request_timeout", diff --git a/tests/integration/test_http_responses_bridge.py b/tests/integration/test_http_responses_bridge.py index 989783c5af..ee727569ac 100644 --- a/tests/integration/test_http_responses_bridge.py +++ b/tests/integration/test_http_responses_bridge.py @@ -8178,6 +8178,108 @@ async def fake_connect_responses_websocket( assert connect_count == 2 +@pytest.mark.asyncio +async def test_v1_responses_http_bridge_retries_when_upstream_never_acknowledges_response_create( + async_client, + monkeypatch, +): + _install_bridge_settings_with_limits( + monkeypatch, + enabled=True, + stuck_gate_retire_after_seconds=0.01, + ) + account_id = await _import_account( + async_client, + "acc_http_bridge_missing_created_retry", + "http-bridge-missing-created-retry@example.com", + ) + account = await _get_account(account_id) + silent_upstream = _SilentUpstreamWebSocket() + recovered_upstream = _FakeBridgeUpstreamWebSocket() + upstreams = [silent_upstream, recovered_upstream] + connect_count = 0 + + async def fake_select_account_with_budget( + self, + deadline, + *, + request_id, + kind, + request_stage="first_turn", + sticky_key, + sticky_kind, + reallocate_sticky, + sticky_max_age_seconds, + prefer_earlier_reset_accounts, + routing_strategy, + model, + exclude_account_ids=None, + additional_limit_name=None, + api_key=None, + preferred_account_id=None, + ): + del preferred_account_id + del ( + self, + deadline, + request_id, + kind, + request_stage, + sticky_key, + sticky_kind, + reallocate_sticky, + sticky_max_age_seconds, + prefer_earlier_reset_accounts, + routing_strategy, + model, + exclude_account_ids, + additional_limit_name, + api_key, + ) + return AccountSelection(account=account, error_message=None, error_code=None) + + async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): + del self, force, timeout_seconds + return target + + async def fake_connect_responses_websocket( + headers, + access_token, + account_id_header, + *, + base_url=None, + session=None, + ): + del headers, access_token, account_id_header, base_url, session + nonlocal connect_count + upstream = upstreams[connect_count] + connect_count += 1 + return upstream + + monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) + monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) + monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) + + response = await asyncio.wait_for( + async_client.post( + "/v1/responses", + json={ + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": "retry missing response.created", + "prompt_cache_key": "missing-created-retry-key", + }, + ), + timeout=_TEST_SYNC_TIMEOUT_SECONDS, + ) + + assert response.status_code == 200 + assert connect_count == 2 + assert silent_upstream.closed is True + assert len(silent_upstream.sent_text) == 1 + assert len(recovered_upstream.sent_text) == 1 + + @pytest.mark.asyncio async def test_backend_responses_http_bridge_retries_precreated_server_overload(async_client, monkeypatch): _install_bridge_settings(monkeypatch, enabled=True) From 09a73da551e8cc063fc13618818b6adb906ad92c Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Thu, 30 Jul 2026 16:33:42 +0400 Subject: [PATCH 16/64] fix(proxy): retry bridge missing-created at deadline --- .../proxy/_service/http_bridge/request_submit.py | 12 ++++++++++++ .../proxy/_service/http_bridge/upstream_events.py | 5 ++++- tests/integration/test_http_responses_bridge.py | 2 +- tests/unit/test_proxy_http_bridge.py | 2 +- 4 files changed, 18 insertions(+), 3 deletions(-) diff --git a/app/modules/proxy/_service/http_bridge/request_submit.py b/app/modules/proxy/_service/http_bridge/request_submit.py index 0319e918ea..43f750e233 100644 --- a/app/modules/proxy/_service/http_bridge/request_submit.py +++ b/app/modules/proxy/_service/http_bridge/request_submit.py @@ -1530,12 +1530,14 @@ async def _retry_http_bridge_precreated_request( session: "_HTTPBridgeSession", *, request_state: _WebSocketRequestState | None = None, + allow_expired_deadline: bool = False, ) -> bool: account_neutral_recovery = is_http_bridge_account_neutral_replay( kind=session.key.affinity_kind, key=session.key.affinity_key, ) hard_owner_bound = _http_bridge_key_strength(session.key) == "hard" + now = _service_time().monotonic() async with session.pending_lock: if request_state is not None: if ( @@ -1544,6 +1546,11 @@ async def _retry_http_bridge_precreated_request( or request_state.draining_until_terminal or not _http_bridge_request_counts_against_queue(request_state) or not _websocket_request_can_replay_before_visible_output(request_state) + or ( + not allow_expired_deadline + and request_state.bridge_request_deadline is not None + and request_state.bridge_request_deadline <= now + ) ): return False else: @@ -1552,6 +1559,11 @@ async def _retry_http_bridge_precreated_request( for request_state in session.pending_requests if not request_state.draining_until_terminal and _websocket_request_can_replay_before_visible_output(request_state) + and ( + allow_expired_deadline + or request_state.bridge_request_deadline is None + or request_state.bridge_request_deadline > now + ) ] if len(retryable_requests) != 1: return False diff --git a/app/modules/proxy/_service/http_bridge/upstream_events.py b/app/modules/proxy/_service/http_bridge/upstream_events.py index 72dbe46896..a12deff177 100644 --- a/app/modules/proxy/_service/http_bridge/upstream_events.py +++ b/app/modules/proxy/_service/http_bridge/upstream_events.py @@ -643,7 +643,10 @@ async def _relay_http_bridge_upstream_messages( _extract_model_class(session.request_model) if session.request_model else None ), ) - retried = await self._retry_http_bridge_precreated_request(session) + retried = await self._retry_http_bridge_precreated_request( + session, + allow_expired_deadline=True, + ) if retried: continue # Claim the session only after the safe pre-created diff --git a/tests/integration/test_http_responses_bridge.py b/tests/integration/test_http_responses_bridge.py index ee727569ac..02d7713502 100644 --- a/tests/integration/test_http_responses_bridge.py +++ b/tests/integration/test_http_responses_bridge.py @@ -8186,8 +8186,8 @@ async def test_v1_responses_http_bridge_retries_when_upstream_never_acknowledges _install_bridge_settings_with_limits( monkeypatch, enabled=True, - stuck_gate_retire_after_seconds=0.01, ) + proxy_module.get_settings().http_responses_session_bridge_stuck_gate_retire_after_seconds = 0.01 account_id = await _import_account( async_client, "acc_http_bridge_missing_created_retry", diff --git a/tests/unit/test_proxy_http_bridge.py b/tests/unit/test_proxy_http_bridge.py index 00dd232e8e..d74f1cbbaa 100644 --- a/tests/unit/test_proxy_http_bridge.py +++ b/tests/unit/test_proxy_http_bridge.py @@ -18444,7 +18444,7 @@ async def close(self) -> None: assert owner.response_event_count == 0 if leading_telemetry: assert owner.latency_first_upstream_event_ms is not None - retry_precreated.assert_not_awaited() + retry_precreated.assert_awaited_once_with(session, allow_expired_deadline=True) assert write_request_log.await_count == 2 assert {call.kwargs["error_code"] for call in write_request_log.await_args_list} == {"upstream_request_timeout"} fail_reader.assert_awaited_once() From f9ab3636dbab203b2b88b890d2a0d0eda7f94e5c Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Thu, 30 Jul 2026 16:45:26 +0400 Subject: [PATCH 17/64] fix(proxy): retry same-anchor missing-created turns --- .../_service/http_bridge/request_submit.py | 54 ++++++++++++++++++- 1 file changed, 52 insertions(+), 2 deletions(-) diff --git a/app/modules/proxy/_service/http_bridge/request_submit.py b/app/modules/proxy/_service/http_bridge/request_submit.py index 43f750e233..b54d86a54a 100644 --- a/app/modules/proxy/_service/http_bridge/request_submit.py +++ b/app/modules/proxy/_service/http_bridge/request_submit.py @@ -127,6 +127,7 @@ from app.modules.proxy._service.support import ( _HARD_HTTP_BRIDGE_AFFINITY_KINDS, # noqa: F401 _WEBSOCKET_FULL_REPLAY_WAIT_POLL_SECONDS, # noqa: F401 + _WEBSOCKET_TRANSPARENT_CLOSE_MAX_REPLAYS, _clear_websocket_request_error_overrides, _copy_websocket_route_metadata_from_session, _event_type_from_payload, @@ -199,6 +200,37 @@ ) +def _http_bridge_can_replay_same_anchor_before_created(request_state: _WebSocketRequestState) -> bool: + if not request_state.request_text: + return False + if request_state.replay_count >= _WEBSOCKET_TRANSPARENT_CLOSE_MAX_REPLAYS: + return False + return ( + request_state.previous_response_id is not None + and request_state.response_id is None + and request_state.awaiting_response_created + and request_state.response_event_count == 0 + and request_state.last_downstream_sequence_number is None + and not request_state.downstream_visible + and not request_state.upstream_model_output_seen + ) + + +async def _await_task_deferring_cancellation( + task: asyncio.Task[T], +) -> tuple[T, asyncio.CancelledError | None]: + """Finish critical cleanup while preserving the caller's cancellation.""" + + cancellation: asyncio.CancelledError | None = None + while True: + try: + return await asyncio.shield(task), cancellation + except asyncio.CancelledError as exc: + if task.cancelled(): + raise + cancellation = cancellation or exc + + async def _rollback_http_bridge_recovery_turn_state_registration( service: Any, receipt: DurableBridgeAliasRegistrationReceipt, @@ -1545,7 +1577,14 @@ async def _retry_http_bridge_precreated_request( or any(pending_request is not request_state for pending_request in session.pending_requests) or request_state.draining_until_terminal or not _http_bridge_request_counts_against_queue(request_state) - or not _websocket_request_can_replay_before_visible_output(request_state) + or not ( + _websocket_request_can_replay_before_visible_output(request_state) + or ( + allow_expired_deadline + and hard_owner_bound + and _http_bridge_can_replay_same_anchor_before_created(request_state) + ) + ) or ( not allow_expired_deadline and request_state.bridge_request_deadline is not None @@ -1558,7 +1597,14 @@ async def _retry_http_bridge_precreated_request( request_state for request_state in session.pending_requests if not request_state.draining_until_terminal - and _websocket_request_can_replay_before_visible_output(request_state) + and ( + _websocket_request_can_replay_before_visible_output(request_state) + or ( + allow_expired_deadline + and hard_owner_bound + and _http_bridge_can_replay_same_anchor_before_created(request_state) + ) + ) and ( allow_expired_deadline or request_state.bridge_request_deadline is None @@ -1572,6 +1618,10 @@ async def _retry_http_bridge_precreated_request( request_state.proxy_injected_previous_response_id and request_state.fresh_upstream_request_is_retry_safe and request_state.fresh_upstream_request_text + ) and not ( + allow_expired_deadline + and hard_owner_bound + and _http_bridge_can_replay_same_anchor_before_created(request_state) ): # Once a continuation is pending upstream, reconnecting without # replay cannot complete the current request, while replaying it From 98ef1aa394e6587ddc1901c799797a19a76324a5 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Thu, 30 Jul 2026 19:05:34 +0400 Subject: [PATCH 18/64] fix(proxy): keep recovering silent response.create upstreams --- app/modules/proxy/_service/support.py | 1 + tests/integration/test_http_responses_bridge.py | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/app/modules/proxy/_service/support.py b/app/modules/proxy/_service/support.py index 0d842e78f9..c3133aa3ce 100644 --- a/app/modules/proxy/_service/support.py +++ b/app/modules/proxy/_service/support.py @@ -67,6 +67,7 @@ _TTFT_OUTPUT_ITEM_TYPES = _PENDING_TOOL_CALL_ITEM_TYPES - {"function_call"} _WEBSOCKET_FULL_REPLAY_WAIT_MIN_ITEMS = 20 _WEBSOCKET_FULL_REPLAY_WAIT_POLL_SECONDS = 0.05 +_WEBSOCKET_TRANSPARENT_CLOSE_MAX_REPLAYS = 20 _HARD_HTTP_BRIDGE_AFFINITY_KINDS = frozenset( { "turn_state_header", diff --git a/tests/integration/test_http_responses_bridge.py b/tests/integration/test_http_responses_bridge.py index 02d7713502..b12770af52 100644 --- a/tests/integration/test_http_responses_bridge.py +++ b/tests/integration/test_http_responses_bridge.py @@ -8194,9 +8194,9 @@ async def test_v1_responses_http_bridge_retries_when_upstream_never_acknowledges "http-bridge-missing-created-retry@example.com", ) account = await _get_account(account_id) - silent_upstream = _SilentUpstreamWebSocket() + silent_upstreams = [_SilentUpstreamWebSocket() for _ in range(5)] recovered_upstream = _FakeBridgeUpstreamWebSocket() - upstreams = [silent_upstream, recovered_upstream] + upstreams = [*silent_upstreams, recovered_upstream] connect_count = 0 async def fake_select_account_with_budget( @@ -8274,9 +8274,9 @@ async def fake_connect_responses_websocket( ) assert response.status_code == 200 - assert connect_count == 2 - assert silent_upstream.closed is True - assert len(silent_upstream.sent_text) == 1 + assert connect_count == len(silent_upstreams) + 1 + assert all(upstream.closed for upstream in silent_upstreams) + assert [len(upstream.sent_text) for upstream in silent_upstreams] == [1] * len(silent_upstreams) assert len(recovered_upstream.sent_text) == 1 From cb1c3ce35094aa8c47cfbad2bbc78b07e00c4cf4 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Fri, 31 Jul 2026 13:42:13 +0400 Subject: [PATCH 19/64] fix(proxy): retry silent bridge creates sooner --- app/modules/proxy/_service/http_bridge/helpers.py | 2 +- tests/unit/test_proxy_http_bridge.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/modules/proxy/_service/http_bridge/helpers.py b/app/modules/proxy/_service/http_bridge/helpers.py index ca98a7fa9f..0b28e0fd29 100644 --- a/app/modules/proxy/_service/http_bridge/helpers.py +++ b/app/modules/proxy/_service/http_bridge/helpers.py @@ -188,7 +188,7 @@ logger = logging.getLogger("app.modules.proxy.service") _HTTP_BRIDGE_BACKGROUND_CLOSE_TIMEOUT_SECONDS = 5.0 -_HTTP_BRIDGE_EVENTLESS_RESPONSE_CREATED_MAX_SECONDS = 240.0 +_HTTP_BRIDGE_EVENTLESS_RESPONSE_CREATED_MAX_SECONDS = 15.0 _HTTP_BRIDGE_MISSING_RESPONSE_CREATED_TIMEOUT_DETAIL = "missing_response_created_timeout" T = TypeVar("T") diff --git a/tests/unit/test_proxy_http_bridge.py b/tests/unit/test_proxy_http_bridge.py index d74f1cbbaa..72e60a9bc7 100644 --- a/tests/unit/test_proxy_http_bridge.py +++ b/tests/unit/test_proxy_http_bridge.py @@ -136,14 +136,14 @@ def test_http_bridge_eventless_precreated_deadline_uses_current_send_and_client_ request_state, stuck_gate_retire_after_seconds=300.0, ) - == 340.0 + == 115.0 ) assert ( http_bridge_helpers_module._http_bridge_eventless_precreated_deadline( request_state, - stuck_gate_retire_after_seconds=30.0, + stuck_gate_retire_after_seconds=10.0, ) - == 130.0 + == 110.0 ) request_state.latency_first_upstream_event_ms = 25 @@ -152,7 +152,7 @@ def test_http_bridge_eventless_precreated_deadline_uses_current_send_and_client_ request_state, stuck_gate_retire_after_seconds=300.0, ) - == 340.0 + == 115.0 ) From 1bc93d6dccd19d0abdc70505f75854c6b09c22cf Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Fri, 31 Jul 2026 14:13:09 +0400 Subject: [PATCH 20/64] fix(proxy): bound missing-created bridge retries --- .../_service/http_bridge/request_submit.py | 22 +++++++-- app/modules/proxy/_service/support.py | 1 + tests/unit/test_proxy_http_bridge.py | 49 +++++++++++++++++++ 3 files changed, 69 insertions(+), 3 deletions(-) diff --git a/app/modules/proxy/_service/http_bridge/request_submit.py b/app/modules/proxy/_service/http_bridge/request_submit.py index b54d86a54a..7645ad4d28 100644 --- a/app/modules/proxy/_service/http_bridge/request_submit.py +++ b/app/modules/proxy/_service/http_bridge/request_submit.py @@ -193,6 +193,7 @@ _REQUEST_TRANSPORT_HTTP = "http" _WEBSOCKET_AUTH_INVALIDATED_FAILURE_CODE = "account_auth_invalidated" _NO_SECURITY_WORK_AUTHORIZED_ACCOUNTS_CODE = "no_security_work_authorized_accounts" +_HTTP_BRIDGE_MISSING_RESPONSE_CREATED_MAX_RETRIES = 1 _SECURITY_WORK_NO_AUTHORIZED_ACCOUNTS_MESSAGE = ( "Upstream flagged this request as possible cybersecurity work, but no account is marked as authorized for " "security work. codex-lb is continuing with normal account selection; the upstream request may still fail until " @@ -216,6 +217,13 @@ def _http_bridge_can_replay_same_anchor_before_created(request_state: _WebSocket ) +def _http_bridge_can_retry_missing_response_created(request_state: _WebSocketRequestState) -> bool: + return ( + request_state.missing_response_created_retry_count < _HTTP_BRIDGE_MISSING_RESPONSE_CREATED_MAX_RETRIES + and _http_bridge_can_replay_same_anchor_before_created(request_state) + ) + + async def _await_task_deferring_cancellation( task: asyncio.Task[T], ) -> tuple[T, asyncio.CancelledError | None]: @@ -1570,6 +1578,7 @@ async def _retry_http_bridge_precreated_request( ) hard_owner_bound = _http_bridge_key_strength(session.key) == "hard" now = _service_time().monotonic() + missing_created_retry = False async with session.pending_lock: if request_state is not None: if ( @@ -1582,7 +1591,7 @@ async def _retry_http_bridge_precreated_request( or ( allow_expired_deadline and hard_owner_bound - and _http_bridge_can_replay_same_anchor_before_created(request_state) + and _http_bridge_can_retry_missing_response_created(request_state) ) ) or ( @@ -1602,7 +1611,7 @@ async def _retry_http_bridge_precreated_request( or ( allow_expired_deadline and hard_owner_bound - and _http_bridge_can_replay_same_anchor_before_created(request_state) + and _http_bridge_can_retry_missing_response_created(request_state) ) ) and ( @@ -1621,7 +1630,7 @@ async def _retry_http_bridge_precreated_request( ) and not ( allow_expired_deadline and hard_owner_bound - and _http_bridge_can_replay_same_anchor_before_created(request_state) + and _http_bridge_can_retry_missing_response_created(request_state) ): # Once a continuation is pending upstream, reconnecting without # replay cannot complete the current request, while replaying it @@ -1629,6 +1638,11 @@ async def _retry_http_bridge_precreated_request( # injected retry-safe anchors are equivalent to the client's own # full resend once the anchor is stripped. return False + missing_created_retry = ( + allow_expired_deadline + and hard_owner_bound + and _http_bridge_can_retry_missing_response_created(request_state) + ) close_classification = _classify_upstream_close( session.last_upstream_close_code, response_events_seen=request_state.response_event_count, @@ -1678,6 +1692,8 @@ async def _retry_http_bridge_precreated_request( elif not request_state.file_required_preferred_account and not hard_owner_bound: request_state.preferred_account_id = None request_state.excluded_account_ids.add(session.account.id) + if missing_created_retry: + request_state.missing_response_created_retry_count += 1 if session.account.id in request_state.excluded_account_ids: session.upstream_turn_state = None session.downstream_turn_state = None diff --git a/app/modules/proxy/_service/support.py b/app/modules/proxy/_service/support.py index c3133aa3ce..43b886ce7c 100644 --- a/app/modules/proxy/_service/support.py +++ b/app/modules/proxy/_service/support.py @@ -787,6 +787,7 @@ class _WebSocketRequestState: request_usage_budget: ApiKeyRequestUsageBudget | None = None request_text: str | None = None replay_count: int = 0 + missing_response_created_retry_count: int = 0 auth_replay_count: int = 0 auth_replay_counts_by_account: dict[str, int] = field(default_factory=dict) force_refresh_account_id: str | None = None diff --git a/tests/unit/test_proxy_http_bridge.py b/tests/unit/test_proxy_http_bridge.py index 72e60a9bc7..6c03febe07 100644 --- a/tests/unit/test_proxy_http_bridge.py +++ b/tests/unit/test_proxy_http_bridge.py @@ -268,6 +268,55 @@ async def send_text(_text: str) -> None: ) +@pytest.mark.asyncio +async def test_http_bridge_missing_response_created_retries_once_before_terminal( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + request_text = ( + '{"type":"response.create","model":"gpt-5.6-sol","previous_response_id":"resp-owner-anchor","input":"hello"}' + ) + request_state = _make_eventless_http_bridge_owner(request_id="req-missing-created-once") + request_state.previous_response_id = "resp-owner-anchor" + request_state.request_text = request_text + request_state.bridge_request_deadline = time.monotonic() - 1.0 + session = _make_bridge_session( + key_value="missing-created-once", + pending_requests=deque([request_state]), + queued_request_count=1, + ) + send_text = AsyncMock() + session.upstream = cast( + UpstreamWebSocket, + SimpleNamespace(send_text=send_text, close=AsyncMock()), + ) + reconnect = AsyncMock() + monkeypatch.setattr(service, "_reconnect_http_bridge_session", reconnect) + monkeypatch.setattr(service, "_acquire_account_response_create_lease_or_overload", AsyncMock(return_value=None)) + + first_retry = await service._retry_http_bridge_precreated_request( + session, + allow_expired_deadline=True, + ) + second_retry = await service._retry_http_bridge_precreated_request( + session, + allow_expired_deadline=True, + ) + + assert first_retry is True + assert second_retry is False + reconnect.assert_awaited_once_with( + session, + request_state=request_state, + require_same_account=True, + ) + send_text.assert_awaited_once() + assert json.loads(send_text.await_args.args[0])["previous_response_id"] == "resp-owner-anchor" + assert request_state.missing_response_created_retry_count == 1 + assert request_state.replay_count == 1 + assert request_state.awaiting_response_created is True + + def _make_account_neutral_replay_session_key( nonce: str, api_key_id: str | None = None, From ad28520c0ac64558d62ed0d452fed769f4648155 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Fri, 31 Jul 2026 14:40:17 +0400 Subject: [PATCH 21/64] fix(proxy): penalize silent bridge create owners --- app/modules/proxy/_service/http_bridge/upstream_events.py | 2 +- tests/unit/test_proxy_http_bridge.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/modules/proxy/_service/http_bridge/upstream_events.py b/app/modules/proxy/_service/http_bridge/upstream_events.py index a12deff177..f09b8654b7 100644 --- a/app/modules/proxy/_service/http_bridge/upstream_events.py +++ b/app/modules/proxy/_service/http_bridge/upstream_events.py @@ -670,7 +670,7 @@ async def _relay_http_bridge_upstream_messages( session, error_code="upstream_request_timeout", error_message=receive_timeout.error_message, - penalize_account=False, + penalize_account=True, retire_detail=_HTTP_BRIDGE_MISSING_RESPONSE_CREATED_TIMEOUT_DETAIL, force_retire=True, ) diff --git a/tests/unit/test_proxy_http_bridge.py b/tests/unit/test_proxy_http_bridge.py index 6c03febe07..4aa0912cd2 100644 --- a/tests/unit/test_proxy_http_bridge.py +++ b/tests/unit/test_proxy_http_bridge.py @@ -18497,7 +18497,7 @@ async def close(self) -> None: assert write_request_log.await_count == 2 assert {call.kwargs["error_code"] for call in write_request_log.await_args_list} == {"upstream_request_timeout"} fail_reader.assert_awaited_once() - assert fail_reader.await_args.kwargs["penalize_account"] is False + assert fail_reader.await_args.kwargs["penalize_account"] is True assert fail_reader.await_args.kwargs["force_retire"] is True record_stuck_retire.assert_called_once_with( reason="missing_response_created_timeout", From 0115eab2fc69932ed4713e83cc0855151731f863 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Fri, 31 Jul 2026 14:53:05 +0400 Subject: [PATCH 22/64] fix(proxy): rebind silent bridge create owners --- .../_service/http_bridge/request_submit.py | 36 ++++++++++- tests/unit/test_proxy_http_bridge.py | 62 +++++++++++++++++++ 2 files changed, 96 insertions(+), 2 deletions(-) diff --git a/app/modules/proxy/_service/http_bridge/request_submit.py b/app/modules/proxy/_service/http_bridge/request_submit.py index 7645ad4d28..09afae3841 100644 --- a/app/modules/proxy/_service/http_bridge/request_submit.py +++ b/app/modules/proxy/_service/http_bridge/request_submit.py @@ -1579,6 +1579,9 @@ async def _retry_http_bridge_precreated_request( hard_owner_bound = _http_bridge_key_strength(session.key) == "hard" now = _service_time().monotonic() missing_created_retry = False + rebind_missing_created_owner = False + owner_rebind_affinity: _AffinityPolicy | None = None + selection_rebind_affinity: _AffinityPolicy | None = None async with session.pending_lock: if request_state is not None: if ( @@ -1680,7 +1683,25 @@ async def _retry_http_bridge_precreated_request( request_text = _prepare_websocket_request_state_for_visible_output_replay(request_state) if request_text is None: return False - if not hard_owner_bound: + if missing_created_retry and hard_owner_bound: + request_state.preferred_account_id = None + request_state.excluded_account_ids.add(session.account.id) + request_state.affinity_policy = replace( + request_state.affinity_policy, + key=None, + kind=None, + reallocate_sticky=True, + ) + rebind_missing_created_owner = True + owner_rebind_affinity = session.affinity + selection_rebind_affinity = replace( + session.affinity, + key=None, + kind=None, + reallocate_sticky=True, + codex_session_source=None, + ) + elif not hard_owner_bound: request_state.excluded_account_ids.add(session.account.id) else: require_preferred_reconnect = account_neutral_recovery @@ -1710,7 +1731,18 @@ async def _retry_http_bridge_precreated_request( model_class=_extract_model_class(session.request_model) if session.request_model else None, ) try: - if hard_owner_bound: + if rebind_missing_created_owner: + await _call_with_supported_optional_kwargs( + self._reconnect_http_bridge_session, + session, + optional_kwargs={ + "owner_rebind_affinity": owner_rebind_affinity, + "selection_affinity": selection_rebind_affinity, + }, + request_state=request_state, + require_same_account=account_neutral_recovery, + ) + elif hard_owner_bound: await self._reconnect_http_bridge_session( session, request_state=request_state, diff --git a/tests/unit/test_proxy_http_bridge.py b/tests/unit/test_proxy_http_bridge.py index 4aa0912cd2..96df27c7c8 100644 --- a/tests/unit/test_proxy_http_bridge.py +++ b/tests/unit/test_proxy_http_bridge.py @@ -317,6 +317,68 @@ async def test_http_bridge_missing_response_created_retries_once_before_terminal assert request_state.awaiting_response_created is True +@pytest.mark.asyncio +async def test_http_bridge_missing_response_created_rebinds_hard_owner_when_full_resend_is_safe( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + anchored_text = ( + '{"type":"response.create","model":"gpt-5.6-sol","previous_response_id":"resp-owner-anchor","input":"trimmed"}' + ) + fresh_text = ( + '{"type":"response.create","model":"gpt-5.6-sol",' + '"input":[{"role":"user","content":[{"type":"input_text","text":"hello"}]}]}' + ) + request_state = _make_eventless_http_bridge_owner(request_id="req-missing-created-rebind") + request_state.previous_response_id = "resp-owner-anchor" + request_state.proxy_injected_previous_response_id = True + request_state.fresh_upstream_request_is_retry_safe = True + request_state.fresh_upstream_request_text = fresh_text + request_state.request_text = anchored_text + request_state.bridge_request_deadline = time.monotonic() - 1.0 + session = _make_bridge_session( + key=proxy_service._HTTPBridgeSessionKey("session_header", "missing-created-rebind", None), + key_value="missing-created-rebind", + pending_requests=deque([request_state]), + queued_request_count=1, + ) + original_affinity = session.affinity + send_text = AsyncMock() + session.upstream = cast( + UpstreamWebSocket, + SimpleNamespace(send_text=send_text, close=AsyncMock()), + ) + reconnect = AsyncMock() + monkeypatch.setattr(service, "_reconnect_http_bridge_session", reconnect) + monkeypatch.setattr(service, "_acquire_account_response_create_lease_or_overload", AsyncMock(return_value=None)) + + retried = await service._retry_http_bridge_precreated_request( + session, + allow_expired_deadline=True, + ) + + assert retried is True + reconnect.assert_awaited_once() + reconnect_kwargs = reconnect.await_args.kwargs + assert reconnect_kwargs["request_state"] is request_state + assert reconnect_kwargs["require_same_account"] is False + assert reconnect_kwargs["owner_rebind_affinity"] is original_affinity + selection_affinity = reconnect_kwargs["selection_affinity"] + assert selection_affinity.key is None + assert selection_affinity.kind is None + assert selection_affinity.reallocate_sticky is True + send_text.assert_awaited_once() + assert json.loads(send_text.await_args.args[0]).get("previous_response_id") is None + assert request_state.request_text == fresh_text + assert request_state.previous_response_id is None + assert request_state.preferred_account_id is None + assert request_state.excluded_account_ids == {"acc-bridge"} + assert request_state.affinity_policy.reallocate_sticky is True + assert request_state.missing_response_created_retry_count == 1 + assert request_state.replay_count == 1 + assert request_state.awaiting_response_created is True + + def _make_account_neutral_replay_session_key( nonce: str, api_key_id: str | None = None, From a6347cc99699949542f27f652471b94476d0bed3 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Fri, 31 Jul 2026 15:05:29 +0400 Subject: [PATCH 23/64] fix(proxy): clear silent bridge anchors on terminal retire --- .../proxy/_service/http_bridge/helpers.py | 3 +- .../proxy/_service/http_bridge/mixin.py | 10 +++- .../_service/http_bridge/request_submit.py | 6 ++- .../proxy/durable_bridge_coordinator.py | 2 + .../proxy/durable_bridge_repository.py | 11 +++++ tests/unit/test_durable_bridge_sessions.py | 48 +++++++++++++++++++ tests/unit/test_proxy_http_bridge.py | 35 ++++++++++++++ 7 files changed, 112 insertions(+), 3 deletions(-) diff --git a/app/modules/proxy/_service/http_bridge/helpers.py b/app/modules/proxy/_service/http_bridge/helpers.py index 0b28e0fd29..b2e7494944 100644 --- a/app/modules/proxy/_service/http_bridge/helpers.py +++ b/app/modules/proxy/_service/http_bridge/helpers.py @@ -680,11 +680,12 @@ async def _close_http_bridge_session_bounded( session: "_HTTPBridgeSession", *, reason: str, + clear_continuity: bool = False, ) -> None: if session.upstream_reader is asyncio.current_task(): session.upstream_reader = None close_task = asyncio.create_task( - service._close_http_bridge_session(session), + service._close_http_bridge_session(session, clear_continuity=clear_continuity), name=f"http-bridge-close-{_hash_identifier(session.key.affinity_key)}", ) diff --git a/app/modules/proxy/_service/http_bridge/mixin.py b/app/modules/proxy/_service/http_bridge/mixin.py index d8dfeb0c51..dde849131e 100644 --- a/app/modules/proxy/_service/http_bridge/mixin.py +++ b/app/modules/proxy/_service/http_bridge/mixin.py @@ -242,8 +242,14 @@ async def _close_http_bridge_session_bounded( session: "_HTTPBridgeSession", *, reason: str, + clear_continuity: bool = False, ) -> None: - await _close_http_bridge_session_bounded(self, session, reason=reason) + await _close_http_bridge_session_bounded( + self, + session, + reason=reason, + clear_continuity=clear_continuity, + ) def _schedule_http_bridge_session_closes( self, @@ -1641,6 +1647,7 @@ async def _close_http_bridge_session( session: "_HTTPBridgeSession", *, turn_state_lock_held: bool = False, + clear_continuity: bool = False, ) -> None: session.closed = True if turn_state_lock_held: @@ -1663,6 +1670,7 @@ async def _close_http_bridge_session( instance_id=_service_get_settings().http_responses_session_bridge_instance_id, owner_epoch=session.durable_owner_epoch, draining=shutdown_state.is_bridge_drain_active(), + clear_continuity=clear_continuity, ) except Exception: logger.warning("Failed to release durable HTTP bridge session", exc_info=True) diff --git a/app/modules/proxy/_service/http_bridge/request_submit.py b/app/modules/proxy/_service/http_bridge/request_submit.py index 09afae3841..e711344a7b 100644 --- a/app/modules/proxy/_service/http_bridge/request_submit.py +++ b/app/modules/proxy/_service/http_bridge/request_submit.py @@ -1476,7 +1476,11 @@ async def _retire_stale_pending_http_bridge_session( if should_close: session.upstream_close_attempted = True if should_close: - await self._close_http_bridge_session_bounded(session, reason="retire_stale_pending") + await self._close_http_bridge_session_bounded( + session, + reason="retire_stale_pending", + clear_continuity=detail == "missing_response_created_timeout", + ) _log_http_bridge_event( "retire_stale_pending", session.key, diff --git a/app/modules/proxy/durable_bridge_coordinator.py b/app/modules/proxy/durable_bridge_coordinator.py index 75e397a7d2..263520ca3b 100644 --- a/app/modules/proxy/durable_bridge_coordinator.py +++ b/app/modules/proxy/durable_bridge_coordinator.py @@ -268,6 +268,7 @@ async def release_live_session( instance_id: str, owner_epoch: int, draining: bool, + clear_continuity: bool = False, ) -> DurableBridgeLookup | None: async with self._session() as session: snapshot = await DurableBridgeRepository(session).release_session( @@ -275,6 +276,7 @@ async def release_live_session( instance_id=instance_id, owner_epoch=owner_epoch, draining=draining, + clear_continuity=clear_continuity, ) if snapshot is None: return None diff --git a/app/modules/proxy/durable_bridge_repository.py b/app/modules/proxy/durable_bridge_repository.py index 7b82e7f8a3..be501270ba 100644 --- a/app/modules/proxy/durable_bridge_repository.py +++ b/app/modules/proxy/durable_bridge_repository.py @@ -385,6 +385,7 @@ async def release_session( instance_id: str, owner_epoch: int, draining: bool, + clear_continuity: bool = False, ) -> DurableBridgeSessionSnapshot | None: """Release the lease with a single fenced UPDATE. @@ -399,6 +400,16 @@ async def release_session( "state": HttpBridgeSessionState.DRAINING if draining else HttpBridgeSessionState.CLOSED, "closed_at": None if draining else now, } + if clear_continuity: + values.update( + { + "latest_turn_state": None, + "latest_response_id": None, + "latest_input_item_count": None, + "latest_input_full_fingerprint": None, + "latest_pending_tool_calls_json": None, + } + ) return await self._execute_fenced_session_update( session_id=session_id, instance_id=instance_id, diff --git a/tests/unit/test_durable_bridge_sessions.py b/tests/unit/test_durable_bridge_sessions.py index 7d52d9376d..6338791e84 100644 --- a/tests/unit/test_durable_bridge_sessions.py +++ b/tests/unit/test_durable_bridge_sessions.py @@ -1760,6 +1760,54 @@ async def test_durable_bridge_same_account_closed_takeover_preserves_restart_anc assert reclaimed.latest_response_id == "resp_old" +@pytest.mark.asyncio +async def test_durable_bridge_terminal_release_can_clear_restart_anchor( + coordinator: DurableBridgeSessionCoordinator, +) -> None: + claimed = await coordinator.claim_live_session( + session_key_kind="session_header", + session_key_value="sid-terminal-clear", + api_key_id=None, + instance_id="instance-a", + lease_ttl_seconds=60.0, + account_id="acc-1", + model="gpt-5.4", + service_tier=None, + latest_turn_state="http_turn_old", + latest_response_id="resp_old", + allow_takeover=True, + ) + released = await coordinator.release_live_session( + session_id=claimed.session_id, + instance_id="instance-a", + owner_epoch=claimed.owner_epoch, + draining=False, + clear_continuity=True, + ) + + assert released is not None + assert released.latest_turn_state is None + assert released.latest_response_id is None + + reclaimed = await coordinator.claim_live_session( + session_key_kind="session_header", + session_key_value="sid-terminal-clear", + api_key_id=None, + instance_id="instance-b", + lease_ttl_seconds=60.0, + account_id="acc-1", + model="gpt-5.4", + service_tier=None, + latest_turn_state=None, + latest_response_id=None, + allow_takeover=True, + ) + + assert reclaimed.owner_instance_id == "instance-b" + assert reclaimed.latest_turn_state is None + assert reclaimed.latest_response_id is None + + @pytest.mark.asyncio async def test_durable_bridge_takeover_preserves_existing_anchor_when_replacement_has_none( coordinator: DurableBridgeSessionCoordinator, diff --git a/tests/unit/test_proxy_http_bridge.py b/tests/unit/test_proxy_http_bridge.py index 96df27c7c8..e8dfa40fa3 100644 --- a/tests/unit/test_proxy_http_bridge.py +++ b/tests/unit/test_proxy_http_bridge.py @@ -15417,6 +15417,7 @@ async def test_http_bridge_retire_after_drain_waits_for_queued_submission( instance_id="instance-retire-drain", owner_epoch=3, draining=False, + clear_continuity=False, ) release_account_lease.assert_awaited_once_with(lease) assert session.account_lease is None @@ -15482,6 +15483,7 @@ async def test_http_bridge_retire_after_drain_does_not_cancel_current_upstream_r instance_id="instance-reader-retire", owner_epoch=7, draining=False, + clear_continuity=False, ) release_account_lease.assert_awaited_once_with(lease) assert session.account_lease is None @@ -18993,12 +18995,43 @@ async def test_retire_stale_pending_http_bridge_session_unregisters_aliases_and_ instance_id="instance-cleanup", owner_epoch=7, draining=False, + clear_continuity=False, ) release_account_lease.assert_awaited_once_with(lease) assert session.account_lease is None close.assert_awaited_once() +@pytest.mark.asyncio +async def test_retire_missing_created_http_bridge_session_clears_durable_continuity( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + release_live_session = AsyncMock() + service._durable_bridge = cast( + Any, + SimpleNamespace(release_live_session=release_live_session), + ) + session = _make_bridge_session(key_value="bridge-missing-created-cleanup") + session.durable_session_id = "durable-missing-created" + session.durable_owner_epoch = 9 + monkeypatch.setattr( + proxy_service, + "get_settings", + lambda: _make_app_settings(http_responses_session_bridge_instance_id="instance-missing-created"), + ) + + await service._retire_stale_pending_http_bridge_session(session, detail="missing_response_created_timeout") + + release_live_session.assert_awaited_once_with( + session_id="durable-missing-created", + instance_id="instance-missing-created", + owner_epoch=9, + draining=False, + clear_continuity=True, + ) + + @pytest.mark.asyncio async def test_http_bridge_reader_failed_precreated_replay_retires_registered_session( monkeypatch: pytest.MonkeyPatch, @@ -19634,6 +19667,7 @@ async def fail_replay(target_session: proxy_service._HTTPBridgeSession) -> bool: instance_id="instance-log-fails", owner_epoch=3, draining=False, + clear_continuity=False, ) cast(Any, upstream).close.assert_awaited_once() @@ -19715,6 +19749,7 @@ async def test_http_bridge_reader_unexpected_processing_error_fails_pending_requ instance_id="instance-reader-crash", owner_epoch=9, draining=False, + clear_continuity=False, ) cast(Any, upstream).close.assert_awaited_once() write_request_log.assert_awaited_once() From 81c4f5310a975f579e749d4920718e95e6ab573b Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Fri, 31 Jul 2026 15:25:28 +0400 Subject: [PATCH 24/64] fix(proxy): clear stale bridge anchors after close attempt --- .../_service/http_bridge/request_submit.py | 15 ++++++++- tests/unit/test_proxy_http_bridge.py | 33 +++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/app/modules/proxy/_service/http_bridge/request_submit.py b/app/modules/proxy/_service/http_bridge/request_submit.py index e711344a7b..163532d843 100644 --- a/app/modules/proxy/_service/http_bridge/request_submit.py +++ b/app/modules/proxy/_service/http_bridge/request_submit.py @@ -10,6 +10,7 @@ import anyio +from app.core import shutdown as shutdown_state from app.core.clients.files import create_file as core_create_file # noqa: F401 from app.core.clients.files import finalize_file as core_finalize_file # noqa: F401 from app.core.clients.proxy import CodexControlResponse as CodexControlResponse @@ -1465,6 +1466,7 @@ async def _retire_stale_pending_http_bridge_session( *, detail: str, ) -> None: + clear_continuity = detail == "missing_response_created_timeout" session.closed = True async with self._http_bridge_lock: if self._http_bridge_sessions.get(session.key) is session: @@ -1479,8 +1481,19 @@ async def _retire_stale_pending_http_bridge_session( await self._close_http_bridge_session_bounded( session, reason="retire_stale_pending", - clear_continuity=detail == "missing_response_created_timeout", + clear_continuity=clear_continuity, ) + elif clear_continuity and session.durable_session_id is not None and session.durable_owner_epoch is not None: + try: + await self._durable_bridge.release_live_session( + session_id=session.durable_session_id, + instance_id=_service_get_settings().http_responses_session_bridge_instance_id, + owner_epoch=session.durable_owner_epoch, + draining=shutdown_state.is_bridge_drain_active(), + clear_continuity=True, + ) + except Exception: + logger.warning("Failed to clear durable HTTP bridge continuity during stale retire", exc_info=True) _log_http_bridge_event( "retire_stale_pending", session.key, diff --git a/tests/unit/test_proxy_http_bridge.py b/tests/unit/test_proxy_http_bridge.py index e8dfa40fa3..52e77b6078 100644 --- a/tests/unit/test_proxy_http_bridge.py +++ b/tests/unit/test_proxy_http_bridge.py @@ -19032,6 +19032,39 @@ async def test_retire_missing_created_http_bridge_session_clears_durable_continu ) +@pytest.mark.asyncio +async def test_retire_missing_created_http_bridge_session_clears_durable_continuity_after_close_attempt( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + release_live_session = AsyncMock() + service._durable_bridge = cast( + Any, + SimpleNamespace(release_live_session=release_live_session), + ) + session = _make_bridge_session(key_value="bridge-missing-created-after-close") + session.durable_session_id = "durable-missing-created-after-close" + session.durable_owner_epoch = 11 + session.upstream_close_attempted = True + close = cast(Any, session.upstream).close + monkeypatch.setattr( + proxy_service, + "get_settings", + lambda: _make_app_settings(http_responses_session_bridge_instance_id="instance-missing-created-after-close"), + ) + + await service._retire_stale_pending_http_bridge_session(session, detail="missing_response_created_timeout") + + close.assert_not_awaited() + release_live_session.assert_awaited_once_with( + session_id="durable-missing-created-after-close", + instance_id="instance-missing-created-after-close", + owner_epoch=11, + draining=False, + clear_continuity=True, + ) + + @pytest.mark.asyncio async def test_http_bridge_reader_failed_precreated_replay_retires_registered_session( monkeypatch: pytest.MonkeyPatch, From 6e6045417291ccb0b4b3993eaff3b6870c748dc8 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Thu, 30 Jul 2026 19:28:00 +0400 Subject: [PATCH 25/64] fix(proxy): limit created-only close replay budget --- app/modules/proxy/_service/support.py | 7 +++++++ tests/integration/test_proxy_websocket_responses.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/app/modules/proxy/_service/support.py b/app/modules/proxy/_service/support.py index 43b886ce7c..197fc2a694 100644 --- a/app/modules/proxy/_service/support.py +++ b/app/modules/proxy/_service/support.py @@ -67,6 +67,7 @@ _TTFT_OUTPUT_ITEM_TYPES = _PENDING_TOOL_CALL_ITEM_TYPES - {"function_call"} _WEBSOCKET_FULL_REPLAY_WAIT_MIN_ITEMS = 20 _WEBSOCKET_FULL_REPLAY_WAIT_POLL_SECONDS = 0.05 +_WEBSOCKET_CREATED_ONLY_CLOSE_MAX_REPLAYS = 1 _WEBSOCKET_TRANSPARENT_CLOSE_MAX_REPLAYS = 20 _HARD_HTTP_BRIDGE_AFFINITY_KINDS = frozenset( { @@ -1169,6 +1170,12 @@ def _websocket_request_can_replay_before_visible_output(request_state: _WebSocke and request_state.response_event_count == 1 and not request_state.downstream_visible ) + if ( + request_state.response_id is not None + and not sequenced_created_only_prewarm + and request_state.replay_count >= _WEBSOCKET_CREATED_ONLY_CLOSE_MAX_REPLAYS + ): + return False if request_state.last_downstream_sequence_number is not None and not sequenced_created_only_prewarm: return False if request_state.downstream_visible: diff --git a/tests/integration/test_proxy_websocket_responses.py b/tests/integration/test_proxy_websocket_responses.py index 43c6b876a2..d840591381 100644 --- a/tests/integration/test_proxy_websocket_responses.py +++ b/tests/integration/test_proxy_websocket_responses.py @@ -9250,7 +9250,7 @@ async def fake_write_request_log(self, **kwargs): assert failed_event["response"]["error"]["code"] == "stream_incomplete" assert "close_code=1011" in failed_event["response"]["error"]["message"] assert len(log_calls) == 1 - assert log_calls[0]["request_id"] == "resp_ws_eof_retry" + assert log_calls[0]["request_id"] == "resp_ws_eof_retry_1" assert log_calls[0]["status"] == "error" assert log_calls[0]["error_code"] == "stream_incomplete" From 91b783ce2041f8b082574b1c62ada536ed35e03d Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Thu, 30 Jul 2026 19:41:24 +0400 Subject: [PATCH 26/64] fix(proxy): retry previsible stream eof continuations --- app/modules/proxy/_service/streaming/mixin.py | 3 + tests/unit/test_proxy_utils.py | 61 +++++++++++++++++++ 2 files changed, 64 insertions(+) diff --git a/app/modules/proxy/_service/streaming/mixin.py b/app/modules/proxy/_service/streaming/mixin.py index 6189ae7cd5..9ccc258d8e 100644 --- a/app/modules/proxy/_service/streaming/mixin.py +++ b/app/modules/proxy/_service/streaming/mixin.py @@ -491,6 +491,7 @@ async def _stream_once( enforce_openai_sdk_contract: bool = True, ) -> AsyncIterator[str]: proxy = cast(_StreamingServiceProtocol, self) + settlement.reset() account_id_value = account.id access_token = proxy._encryptor.decrypt(account.access_token_encrypted) account_id = _header_account_id(account.chatgpt_account_id) @@ -577,6 +578,8 @@ async def _stream_once( settlement.record_success = False settlement.account_health_error = True settlement.error = {"message": error_message} + if allow_transient_retry: + raise _TransientStreamError(error_code, settlement.error) yield format_sse_event( response_failed_event( error_code, diff --git a/tests/unit/test_proxy_utils.py b/tests/unit/test_proxy_utils.py index 31910f68ca..69939bb96a 100644 --- a/tests/unit/test_proxy_utils.py +++ b/tests/unit/test_proxy_utils.py @@ -30073,6 +30073,67 @@ async def fake_stream(payload, headers, access_token, account_id, base_url=None, record_success.assert_not_awaited() +@pytest.mark.asyncio +async def test_stream_previsible_core_eof_with_previous_response_id_retries(monkeypatch): + settings = _make_proxy_settings() + request_logs = _RequestLogsRecorder() + service = proxy_service.ProxyService(_repo_factory(request_logs)) + account = _make_account("acc_previsible_core_eof") + request_logs.response_owner_by_id[("resp_parent", None, "sid-stream")] = account.id + handle_stream_error = AsyncMock() + record_success = AsyncMock() + stream_calls = 0 + + monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache(settings)) + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr(proxy_service, "_MAX_TRANSIENT_SAME_ACCOUNT_RETRIES", 3) + monkeypatch.setattr(streaming_retry_module.ProcessNetworkRecovery, "wait", AsyncMock(return_value=None)) + monkeypatch.setattr(streaming_retry_module.asyncio, "sleep", AsyncMock()) + monkeypatch.setattr( + service._load_balancer, + "select_account", + AsyncMock(return_value=AccountSelection(account=account, error_message=None)), + ) + monkeypatch.setattr(service._load_balancer, "record_success", record_success) + monkeypatch.setattr(service, "_ensure_fresh", AsyncMock(return_value=account)) + monkeypatch.setattr(service, "_handle_stream_error", handle_stream_error) + monkeypatch.setattr(service, "_settle_stream_api_key_usage", AsyncMock(return_value=True)) + + async def fake_stream(payload, headers, access_token, account_id, base_url=None, raise_for_status=False, **kwargs): + nonlocal stream_calls + del payload, headers, access_token, account_id, base_url, raise_for_status, kwargs + stream_calls += 1 + if stream_calls == 1: + return + yield 'data: {"type":"response.completed","response":{"id":"resp_child_retry_ok"}}\n\n' + + monkeypatch.setattr(proxy_service, "core_stream_responses", fake_stream) + + payload = ResponsesRequest.model_validate( + { + "model": "gpt-5.1", + "instructions": "hi", + "input": [], + "stream": True, + "previous_response_id": "resp_parent", + } + ) + + chunks = [chunk async for chunk in service.stream_responses(payload, {"session_id": "sid-stream"})] + + completed = json.loads(chunks[-1].split("data: ", 1)[1]) + assert completed["type"] == "response.completed" + assert completed["response"]["id"] == "resp_child_retry_ok" + assert stream_calls == 2 + assert request_logs.lookup_calls == [("resp_parent", None, "sid-stream")] + assert await service.drain_persistence_tasks(timeout_seconds=1) + assert [call["status"] for call in request_logs.calls] == ["error", "success"] + assert request_logs.calls[0]["error_code"] == "stream_incomplete" + assert request_logs.calls[-1]["request_id"] == "resp_child_retry_ok" + handle_stream_error.assert_not_awaited() + record_success.assert_awaited_once_with(account) + + @pytest.mark.asyncio async def test_stream_missing_tool_output_proxy_error_is_masked_to_stream_incomplete(monkeypatch, caplog): settings = _make_proxy_settings() From 1387a5d97d739ee207a8be3bbd4fd84cd65f900f Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Thu, 30 Jul 2026 20:11:36 +0400 Subject: [PATCH 27/64] fix(proxy): keep fresh empty streams fail-closed --- app/modules/proxy/_service/streaming/mixin.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/modules/proxy/_service/streaming/mixin.py b/app/modules/proxy/_service/streaming/mixin.py index 9ccc258d8e..a8d228cab8 100644 --- a/app/modules/proxy/_service/streaming/mixin.py +++ b/app/modules/proxy/_service/streaming/mixin.py @@ -298,6 +298,7 @@ _RetryableStreamError, _StreamSettlement, _TerminalStreamError, + _TransientStreamError, _ttft_event_latency_ms, _WebSocketUpstreamControl, ) @@ -578,7 +579,7 @@ async def _stream_once( settlement.record_success = False settlement.account_health_error = True settlement.error = {"message": error_message} - if allow_transient_retry: + if allow_transient_retry and payload.previous_response_id is not None: raise _TransientStreamError(error_code, settlement.error) yield format_sse_event( response_failed_event( From 2b9378de9a90010462a8efe882bf8b820d77b5ae Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Fri, 31 Jul 2026 17:06:09 +0400 Subject: [PATCH 28/64] fix(proxy): keep retrying unanchored silent bridge creates --- app/modules/proxy/_service/support.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/app/modules/proxy/_service/support.py b/app/modules/proxy/_service/support.py index 197fc2a694..0afbddd839 100644 --- a/app/modules/proxy/_service/support.py +++ b/app/modules/proxy/_service/support.py @@ -1161,7 +1161,16 @@ def _websocket_request_can_replay_before_visible_output(request_state: _WebSocke if not request_state.request_text: return False if request_state.replay_count >= 1: - return False + unanchored_precreated_pending = ( + request_state.previous_response_id is None + and request_state.response_id is None + and request_state.awaiting_response_created + ) + if ( + not unanchored_precreated_pending + or request_state.replay_count >= _WEBSOCKET_TRANSPARENT_CLOSE_MAX_REPLAYS + ): + return False sequenced_created_only_prewarm = ( request_state.generate_false_prewarm and request_state.last_downstream_sequence_number == 0 From a448b9cae7b79f634a5b3c35a75d7bec137d3bae Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Fri, 31 Jul 2026 17:07:05 +0400 Subject: [PATCH 29/64] fix(proxy): use shared bridge cancellation helper --- .../_service/http_bridge/request_submit.py | 35 +++++++------------ app/modules/proxy/_service/support.py | 5 +-- 2 files changed, 13 insertions(+), 27 deletions(-) diff --git a/app/modules/proxy/_service/http_bridge/request_submit.py b/app/modules/proxy/_service/http_bridge/request_submit.py index 163532d843..7d64b8de0f 100644 --- a/app/modules/proxy/_service/http_bridge/request_submit.py +++ b/app/modules/proxy/_service/http_bridge/request_submit.py @@ -225,21 +225,6 @@ def _http_bridge_can_retry_missing_response_created(request_state: _WebSocketReq ) -async def _await_task_deferring_cancellation( - task: asyncio.Task[T], -) -> tuple[T, asyncio.CancelledError | None]: - """Finish critical cleanup while preserving the caller's cancellation.""" - - cancellation: asyncio.CancelledError | None = None - while True: - try: - return await asyncio.shield(task), cancellation - except asyncio.CancelledError as exc: - if task.cancelled(): - raise - cancellation = cancellation or exc - - async def _rollback_http_bridge_recovery_turn_state_registration( service: Any, receipt: DurableBridgeAliasRegistrationReceipt, @@ -1643,14 +1628,18 @@ async def _retry_http_bridge_precreated_request( if len(retryable_requests) != 1: return False request_state = retryable_requests[0] - if request_state.previous_response_id is not None and not ( - request_state.proxy_injected_previous_response_id - and request_state.fresh_upstream_request_is_retry_safe - and request_state.fresh_upstream_request_text - ) and not ( - allow_expired_deadline - and hard_owner_bound - and _http_bridge_can_retry_missing_response_created(request_state) + if ( + request_state.previous_response_id is not None + and not ( + request_state.proxy_injected_previous_response_id + and request_state.fresh_upstream_request_is_retry_safe + and request_state.fresh_upstream_request_text + ) + and not ( + allow_expired_deadline + and hard_owner_bound + and _http_bridge_can_retry_missing_response_created(request_state) + ) ): # Once a continuation is pending upstream, reconnecting without # replay cannot complete the current request, while replaying it diff --git a/app/modules/proxy/_service/support.py b/app/modules/proxy/_service/support.py index 0afbddd839..ab70d0c8cc 100644 --- a/app/modules/proxy/_service/support.py +++ b/app/modules/proxy/_service/support.py @@ -1166,10 +1166,7 @@ def _websocket_request_can_replay_before_visible_output(request_state: _WebSocke and request_state.response_id is None and request_state.awaiting_response_created ) - if ( - not unanchored_precreated_pending - or request_state.replay_count >= _WEBSOCKET_TRANSPARENT_CLOSE_MAX_REPLAYS - ): + if not unanchored_precreated_pending or request_state.replay_count >= _WEBSOCKET_TRANSPARENT_CLOSE_MAX_REPLAYS: return False sequenced_created_only_prewarm = ( request_state.generate_false_prewarm From 78e919fe659ff1c3cfe8aed7629ed3aeddfa3b65 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Fri, 31 Jul 2026 17:39:38 +0400 Subject: [PATCH 30/64] fix(db): tolerate deployed security lineage schema --- ...000000_add_security_lineage_persistence.py | 266 ++++++++++++++++++ ...ty_lineage_and_pending_tool_calls_heads.py | 22 ++ ...drop_legacy_bridge_pending_tool_columns.py | 55 ++++ app/db/migrate.py | 25 ++ tests/unit/test_db_migrate.py | 12 + 5 files changed, 380 insertions(+) create mode 100644 app/db/alembic/versions/20260722_000000_add_security_lineage_persistence.py create mode 100644 app/db/alembic/versions/20260728_000000_merge_security_lineage_and_pending_tool_calls_heads.py create mode 100644 app/db/alembic/versions/20260729_000000_drop_legacy_bridge_pending_tool_columns.py diff --git a/app/db/alembic/versions/20260722_000000_add_security_lineage_persistence.py b/app/db/alembic/versions/20260722_000000_add_security_lineage_persistence.py new file mode 100644 index 0000000000..8341191cfb --- /dev/null +++ b/app/db/alembic/versions/20260722_000000_add_security_lineage_persistence.py @@ -0,0 +1,266 @@ +"""Reconcile durable security-lineage persistence without a second head. + +Revision ID: 20260722_000000_add_security_lineage_persistence +Revises: 20260720_000000_add_request_log_conversation_id +Create Date: 2026-07-22 00:00:00.000000 +""" + +from __future__ import annotations + +import hashlib +from collections.abc import Mapping + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.engine import Connection + +revision = "20260722_000000_add_security_lineage_persistence" +down_revision = "20260720_000000_add_request_log_conversation_id" +branch_labels = None +depends_on = None + +_MARKER_PREFIX = "@security-work/v2/" +_LEGACY_MARKER_PREFIX = "security-work:" +_CODEX_SESSION_KIND = "codex_session" +_LINEAGE_ALIAS_KINDS = ("session_header", "turn_state") +_ANONYMOUS_SCOPE = "__anonymous__" +_BATCH_NAMING_CONVENTION = { + "fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s", +} + + +def _columns(connection: Connection, table_name: str) -> dict[str, Mapping[str, object]]: + inspector = sa.inspect(connection) + if not inspector.has_table(table_name): + return {} + return {str(column["name"]): column for column in inspector.get_columns(table_name) if column.get("name")} + + +def _account_foreign_key(connection: Connection, table_name: str) -> Mapping[str, object] | None: + inspector = sa.inspect(connection) + if not inspector.has_table(table_name): + return None + for foreign_key in inspector.get_foreign_keys(table_name): + if foreign_key.get("constrained_columns") == ["account_id"] and foreign_key.get("referred_table") == "accounts": + return foreign_key + return None + + +def _marker_key(lineage_id: str, api_key_scope: str | None) -> str: + scope = (api_key_scope or "").strip() or _ANONYMOUS_SCOPE + digest = hashlib.sha256(f"{scope}\0{lineage_id}".encode()).hexdigest() + return f"{_MARKER_PREFIX}{digest}" + + +def _legacy_marker_key(lineage_id: str) -> str: + return f"{_MARKER_PREFIX}{hashlib.sha256(lineage_id.encode()).hexdigest()}" + + +def _insert_marker(bind: Connection, marker_key: str) -> None: + bind.execute( + sa.text( + """ + INSERT INTO sticky_sessions ( + key, kind, account_id, requires_security_work_authorized, created_at, updated_at + ) + SELECT :key, :kind, NULL, :required, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + WHERE NOT EXISTS ( + SELECT 1 FROM sticky_sessions WHERE key = :key AND kind = :kind + ) + """ + ), + {"key": marker_key, "kind": _CODEX_SESSION_KIND, "required": True}, + ) + + +def _backfill_marker(bind: Connection, lineage_id: str, api_key_scope: str | None, *, legacy: bool = False) -> None: + if lineage_id.startswith(_MARKER_PREFIX): + bind.execute( + sa.text( + """ + UPDATE sticky_sessions + SET account_id = NULL, requires_security_work_authorized = :required, updated_at = CURRENT_TIMESTAMP + WHERE key = :key AND kind = :kind + """ + ), + {"key": lineage_id, "kind": _CODEX_SESSION_KIND, "required": True}, + ) + return + if lineage_id.startswith(_LEGACY_MARKER_PREFIX): + lineage_id = lineage_id.removeprefix(_LEGACY_MARKER_PREFIX) + _insert_marker(bind, _marker_key(lineage_id, api_key_scope)) + if legacy: + _insert_marker(bind, _legacy_marker_key(lineage_id)) + + +def _backfill_detached_markers(bind: Connection) -> None: + sticky_columns = _columns(bind, "sticky_sessions") + required_sticky = {"key", "kind", "account_id", "requires_security_work_authorized"} + if required_sticky.issubset(sticky_columns): + rows = bind.execute( + sa.text( + """ + SELECT key FROM sticky_sessions + WHERE kind = :kind AND account_id IS NOT NULL AND requires_security_work_authorized = :required + """ + ), + {"kind": _CODEX_SESSION_KIND, "required": True}, + ).fetchall() + for (lineage_id,) in rows: + if isinstance(lineage_id, str): + _backfill_marker(bind, lineage_id, None, legacy=True) + + bridge_columns = _columns(bind, "http_bridge_sessions") + required_bridge = {"session_key_kind", "session_key_value", "api_key_scope", "requires_security_work_authorized"} + if not required_bridge.issubset(bridge_columns): + return + turn_state = "latest_turn_state" if "latest_turn_state" in bridge_columns else "NULL" + rows = bind.execute( + sa.text( + f""" + SELECT session_key_kind, session_key_value, api_key_scope, {turn_state} AS latest_turn_state + FROM http_bridge_sessions + WHERE requires_security_work_authorized = :required + """ + ), + {"required": True}, + ).fetchall() + for kind, value, scope, latest_turn_state in rows: + if kind in _LINEAGE_ALIAS_KINDS and isinstance(value, str): + _backfill_marker(bind, value, scope if isinstance(scope, str) else None) + if isinstance(latest_turn_state, str): + _backfill_marker(bind, latest_turn_state, scope if isinstance(scope, str) else None) + + alias_columns = _columns(bind, "http_bridge_session_aliases") + required_alias = {"session_id", "alias_kind", "alias_value"} + if not required_alias.issubset(alias_columns) or "id" not in bridge_columns: + return + if "api_key_scope" in alias_columns: + alias_scope = "COALESCE(a.api_key_scope, s.api_key_scope, :anonymous_scope)" + else: + alias_scope = "COALESCE(s.api_key_scope, :anonymous_scope)" + alias_rows = bind.execute( + sa.text( + f""" + SELECT a.alias_value, {alias_scope} AS api_key_scope + FROM http_bridge_session_aliases AS a + JOIN http_bridge_sessions AS s ON s.id = a.session_id + WHERE s.requires_security_work_authorized = :required + AND a.alias_kind IN :alias_kinds + """ + ).bindparams(sa.bindparam("alias_kinds", expanding=True)), + { + "required": True, + "alias_kinds": list(_LINEAGE_ALIAS_KINDS), + "anonymous_scope": _ANONYMOUS_SCOPE, + }, + ).fetchall() + for alias_value, scope in alias_rows: + if isinstance(alias_value, str): + _backfill_marker(bind, alias_value, scope if isinstance(scope, str) else None) + + +def _add_columns(bind: Connection) -> None: + usage = _columns(bind, "usage_history") + if usage: + with op.batch_alter_table("usage_history") as batch: + if "requires_security_work_authorized" not in usage: + batch.add_column( + sa.Column( + "requires_security_work_authorized", sa.Boolean(), nullable=False, server_default=sa.false() + ) + ) + if not bool(usage.get("account_id", {}).get("nullable", False)): + batch.alter_column("account_id", existing_type=sa.String(), nullable=True) + if bind.dialect.name == "sqlite": + # SQLite batch-alter rebuilds the table and does not preserve its + # expression indexes, which are required by the usage hot path. + op.execute(sa.text("DROP INDEX IF EXISTS idx_usage_window_account_latest")) + op.execute(sa.text("DROP INDEX IF EXISTS idx_usage_window_account_time")) + op.execute( + sa.text( + "CREATE INDEX idx_usage_window_account_latest " + "ON usage_history (coalesce(\"window\", 'primary'), account_id, recorded_at DESC, id DESC)" + ) + ) + op.execute( + sa.text( + "CREATE INDEX idx_usage_window_account_time " + "ON usage_history (coalesce(\"window\", 'primary'), account_id, recorded_at DESC)" + ) + ) + + sticky = _columns(bind, "sticky_sessions") + if sticky: + account_foreign_key = _account_foreign_key(bind, "sticky_sessions") + raw_account_foreign_key_options = account_foreign_key.get("options") if account_foreign_key else None + account_foreign_key_options = ( + raw_account_foreign_key_options if isinstance(raw_account_foreign_key_options, Mapping) else {} + ) + replace_account_foreign_key = ( + account_foreign_key is None or str(account_foreign_key_options.get("ondelete", "")).upper() != "SET NULL" + ) + with op.batch_alter_table( + "sticky_sessions", + naming_convention=_BATCH_NAMING_CONVENTION, + ) as batch: + if "requires_security_work_authorized" not in sticky: + batch.add_column( + sa.Column( + "requires_security_work_authorized", sa.Boolean(), nullable=False, server_default=sa.false() + ) + ) + if not bool(sticky.get("account_id", {}).get("nullable", False)): + batch.alter_column("account_id", existing_type=sa.String(), nullable=True) + if replace_account_foreign_key: + if account_foreign_key is not None: + constraint_name = str(account_foreign_key.get("name") or "fk_sticky_sessions_account_id_accounts") + batch.drop_constraint(constraint_name, type_="foreignkey") + batch.create_foreign_key( + "fk_sticky_sessions_account_id_accounts", + "accounts", + ["account_id"], + ["id"], + ondelete="SET NULL", + ) + + bridge = _columns(bind, "http_bridge_sessions") + if bridge: + with op.batch_alter_table("http_bridge_sessions") as batch: + if "requires_security_work_authorized" not in bridge: + batch.add_column( + sa.Column( + "requires_security_work_authorized", sa.Boolean(), nullable=False, server_default=sa.false() + ) + ) + if "latest_pending_function_call_ids" not in bridge: + batch.add_column(sa.Column("latest_pending_function_call_ids", sa.Text(), nullable=True)) + if "latest_pending_custom_tool_call_ids" not in bridge: + batch.add_column(sa.Column("latest_pending_custom_tool_call_ids", sa.Text(), nullable=True)) + + quota = _columns(bind, "quota_planner_settings") + if quota: + with op.batch_alter_table("quota_planner_settings") as batch: + if "auto_redeem_expiring_reset_credits" not in quota: + batch.add_column( + sa.Column( + "auto_redeem_expiring_reset_credits", sa.Boolean(), nullable=False, server_default=sa.false() + ) + ) + if "reset_credit_redeem_lead_minutes" not in quota: + batch.add_column( + sa.Column("reset_credit_redeem_lead_minutes", sa.Integer(), nullable=False, server_default="30") + ) + + +def upgrade() -> None: + bind = op.get_bind() + _add_columns(bind) + _backfill_detached_markers(bind) + + +def downgrade() -> None: + # This revision reconciles columns and detached markers that may have been + # created by a previous aggregate. Their original owner cannot be inferred, + # so dropping them could destroy live lineage data. + return diff --git a/app/db/alembic/versions/20260728_000000_merge_security_lineage_and_pending_tool_calls_heads.py b/app/db/alembic/versions/20260728_000000_merge_security_lineage_and_pending_tool_calls_heads.py new file mode 100644 index 0000000000..eab4dc4ec7 --- /dev/null +++ b/app/db/alembic/versions/20260728_000000_merge_security_lineage_and_pending_tool_calls_heads.py @@ -0,0 +1,22 @@ +"""Merge security-lineage and pending tool call manifest heads. + +Revision ID: 20260728_000000_merge_security_lineage_and_pending_tool_calls_heads +Revises: 20260722_000000_add_security_lineage_persistence, 20260725_000000_add_http_bridge_pending_tool_calls +Create Date: 2026-07-28 +""" + +revision = "20260728_000000_merge_security_lineage_and_pending_tool_calls_heads" +down_revision = ( + "20260722_000000_add_security_lineage_persistence", + "20260725_000000_add_http_bridge_pending_tool_calls", +) +branch_labels = None +depends_on = None + + +def upgrade() -> None: + pass + + +def downgrade() -> None: + pass diff --git a/app/db/alembic/versions/20260729_000000_drop_legacy_bridge_pending_tool_columns.py b/app/db/alembic/versions/20260729_000000_drop_legacy_bridge_pending_tool_columns.py new file mode 100644 index 0000000000..4c52540970 --- /dev/null +++ b/app/db/alembic/versions/20260729_000000_drop_legacy_bridge_pending_tool_columns.py @@ -0,0 +1,55 @@ +"""Drop legacy split pending tool call columns. + +Revision ID: 20260729_000000_drop_legacy_bridge_pending_tool_columns +Revises: 20260728_000000_merge_security_lineage_and_pending_tool_calls_heads +Create Date: 2026-07-29 +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.engine import Connection + +revision = "20260729_000000_drop_legacy_bridge_pending_tool_columns" +down_revision = "20260728_000000_merge_security_lineage_and_pending_tool_calls_heads" +branch_labels = None +depends_on = None + +_TABLE = "http_bridge_sessions" +_CURRENT_COLUMN = "latest_pending_tool_calls_json" +_LEGACY_COLUMNS = ( + "latest_pending_function_call_ids", + "latest_pending_custom_tool_call_ids", +) + + +def _columns(connection: Connection) -> set[str]: + inspector = sa.inspect(connection) + if not inspector.has_table(_TABLE): + return set() + return {str(column["name"]) for column in inspector.get_columns(_TABLE) if column.get("name") is not None} + + +def upgrade() -> None: + bind = op.get_bind() + columns = _columns(bind) + if not columns: + return + with op.batch_alter_table(_TABLE) as batch_op: + if _CURRENT_COLUMN not in columns: + batch_op.add_column(sa.Column(_CURRENT_COLUMN, sa.Text(), nullable=True)) + for column in _LEGACY_COLUMNS: + if column in columns: + batch_op.drop_column(column) + + +def downgrade() -> None: + bind = op.get_bind() + columns = _columns(bind) + if not columns: + return + with op.batch_alter_table(_TABLE) as batch_op: + for column in _LEGACY_COLUMNS: + if column not in columns: + batch_op.add_column(sa.Column(column, sa.Text(), nullable=True)) diff --git a/app/db/migrate.py b/app/db/migrate.py index 1e19350940..a7b8d4e773 100644 --- a/app/db/migrate.py +++ b/app/db/migrate.py @@ -103,7 +103,18 @@ ) _LEGACY_EXTRA_COLUMNS = frozenset( { + ("http_bridge_sessions", "requires_security_work_authorized"), + ("quota_planner_settings", "auto_redeem_expiring_reset_credits"), + ("quota_planner_settings", "reset_credit_redeem_lead_minutes"), ("request_logs", "slim_summary_json"), + ("sticky_sessions", "requires_security_work_authorized"), + ("usage_history", "requires_security_work_authorized"), + } +) +_LEGACY_NULLABLE_COLUMNS = frozenset( + { + ("sticky_sessions", "account_id"), + ("usage_history", "account_id"), } ) @@ -582,6 +593,20 @@ def _is_ignored_schema_drift(connection: Connection, diff: object) -> bool: if (str(diff[2]), str(column_name)) in _LEGACY_EXTRA_COLUMNS: return True + if diff[0] == "modify_nullable" and len(diff) >= 7: + table_name = str(diff[2]) + column_name = str(diff[3]) + if (table_name, column_name) in _LEGACY_NULLABLE_COLUMNS: + return True + + if diff[0] in {"add_fk", "remove_fk"} and len(diff) >= 2: + constraint = diff[1] + table = getattr(constraint, "table", None) + table_name = getattr(table, "name", None) + columns = {str(column.name) for column in getattr(constraint, "columns", ()) if getattr(column, "name", None)} + if table_name == "sticky_sessions" and columns == {"account_id"}: + return True + if connection.dialect.name == "sqlite" and diff[0] == "modify_type" and len(diff) >= 7: table_name = str(diff[2]) column_name = str(diff[3]) diff --git a/tests/unit/test_db_migrate.py b/tests/unit/test_db_migrate.py index cd63687c22..7d76d74115 100644 --- a/tests/unit/test_db_migrate.py +++ b/tests/unit/test_db_migrate.py @@ -1377,6 +1377,18 @@ def test_check_schema_drift_ignores_legacy_live_extra_request_log_column(tmp_pat assert check_schema_drift(url) == () +def test_check_schema_drift_ignores_legacy_live_security_lineage_columns(tmp_path: Path) -> None: + db_path = tmp_path / "legacy-security-lineage-columns.db" + url = _db_url(db_path) + + run_upgrade(url, "head", bootstrap_legacy=False) + + # The live database may already include schema from an older aggregate that + # kept security-lineage persistence. Current code tolerates those columns so + # newer deploys can move past that applied Alembic revision safely. + assert check_schema_drift(url) == () + + def test_check_schema_drift_ignores_sqlite_real_float_reflection_for_sticky_thresholds( monkeypatch, tmp_path: Path, From e7aeab667df60e3d23d0f3f24794c73477342ada Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Fri, 31 Jul 2026 18:25:14 +0400 Subject: [PATCH 31/64] test(proxy): align bridge recovery expectations --- .../test_proxy_websocket_responses.py | 2 +- tests/unit/test_proxy_http_bridge.py | 28 +++++++++++++++---- tests/unit/test_proxy_utils.py | 2 +- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/tests/integration/test_proxy_websocket_responses.py b/tests/integration/test_proxy_websocket_responses.py index d840591381..43c6b876a2 100644 --- a/tests/integration/test_proxy_websocket_responses.py +++ b/tests/integration/test_proxy_websocket_responses.py @@ -9250,7 +9250,7 @@ async def fake_write_request_log(self, **kwargs): assert failed_event["response"]["error"]["code"] == "stream_incomplete" assert "close_code=1011" in failed_event["response"]["error"]["message"] assert len(log_calls) == 1 - assert log_calls[0]["request_id"] == "resp_ws_eof_retry_1" + assert log_calls[0]["request_id"] == "resp_ws_eof_retry" assert log_calls[0]["status"] == "error" assert log_calls[0]["error_code"] == "stream_incomplete" diff --git a/tests/unit/test_proxy_http_bridge.py b/tests/unit/test_proxy_http_bridge.py index 52e77b6078..4d86d18658 100644 --- a/tests/unit/test_proxy_http_bridge.py +++ b/tests/unit/test_proxy_http_bridge.py @@ -311,7 +311,9 @@ async def test_http_bridge_missing_response_created_retries_once_before_terminal require_same_account=True, ) send_text.assert_awaited_once() - assert json.loads(send_text.await_args.args[0])["previous_response_id"] == "resp-owner-anchor" + send_text_call = send_text.await_args + assert send_text_call is not None + assert json.loads(send_text_call.args[0])["previous_response_id"] == "resp-owner-anchor" assert request_state.missing_response_created_retry_count == 1 assert request_state.replay_count == 1 assert request_state.awaiting_response_created is True @@ -359,7 +361,9 @@ async def test_http_bridge_missing_response_created_rebinds_hard_owner_when_full assert retried is True reconnect.assert_awaited_once() - reconnect_kwargs = reconnect.await_args.kwargs + reconnect_call = reconnect.await_args + assert reconnect_call is not None + reconnect_kwargs = reconnect_call.kwargs assert reconnect_kwargs["request_state"] is request_state assert reconnect_kwargs["require_same_account"] is False assert reconnect_kwargs["owner_rebind_affinity"] is original_affinity @@ -368,7 +372,9 @@ async def test_http_bridge_missing_response_created_rebinds_hard_owner_when_full assert selection_affinity.kind is None assert selection_affinity.reallocate_sticky is True send_text.assert_awaited_once() - assert json.loads(send_text.await_args.args[0]).get("previous_response_id") is None + send_text_call = send_text.await_args + assert send_text_call is not None + assert json.loads(send_text_call.args[0]).get("previous_response_id") is None assert request_state.request_text == fresh_text assert request_state.previous_response_id is None assert request_state.preferred_account_id is None @@ -3476,7 +3482,7 @@ async def test_recovery_completed_alias_persistence_failure_fails_response_and_r assert finalize_call.kwargs["event_type"] == "response.failed" assert await service._retire_http_bridge_after_drain_if_ready(session) is True - close_session.assert_awaited_once_with(session) + close_session.assert_awaited_once_with(session, clear_continuity=False) @pytest.mark.asyncio @@ -8190,9 +8196,14 @@ async def test_close_http_bridge_session_bounded_timeout_keeps_close_task_runnin close_finished = asyncio.Event() close_cancelled = False - async def close_http_bridge_session(target: proxy_service._HTTPBridgeSession) -> None: + async def close_http_bridge_session( + target: proxy_service._HTTPBridgeSession, + *, + clear_continuity: bool = False, + ) -> None: nonlocal close_cancelled assert target is session + assert clear_continuity is False close_started.set() try: await release_close.wait() @@ -8232,9 +8243,14 @@ async def test_close_http_bridge_session_bounded_cancellation_keeps_close_task_t close_finished = asyncio.Event() close_cancelled = False - async def close_http_bridge_session(target: proxy_service._HTTPBridgeSession) -> None: + async def close_http_bridge_session( + target: proxy_service._HTTPBridgeSession, + *, + clear_continuity: bool = False, + ) -> None: nonlocal close_cancelled assert target is session + assert clear_continuity is False close_started.set() try: await release_close.wait() diff --git a/tests/unit/test_proxy_utils.py b/tests/unit/test_proxy_utils.py index 69939bb96a..e8b515c60c 100644 --- a/tests/unit/test_proxy_utils.py +++ b/tests/unit/test_proxy_utils.py @@ -37037,7 +37037,7 @@ async def test_retry_http_bridge_precreated_request_does_not_send_after_admissio request_id="req_bridge_retry_deadline", ) acquire_admission = AsyncMock(return_value=replacement_admission) - retry_times = iter((9.0, 11.0)) + retry_times = iter((9.0, 9.5, 11.0)) request_state = proxy_service._WebSocketRequestState( request_id="req_bridge_retry_deadline", model="gpt-5.6-sol", From 4062577b968e06f19f90be8caa5ce12f4b032bcd Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Fri, 31 Jul 2026 18:31:48 +0400 Subject: [PATCH 32/64] fix(proxy): retire stale bridge gate holders --- .../proxy/_service/http_bridge/streaming.py | 33 ++++++++ tests/unit/test_proxy_http_bridge.py | 82 +++++++++++++++++++ 2 files changed, 115 insertions(+) diff --git a/app/modules/proxy/_service/http_bridge/streaming.py b/app/modules/proxy/_service/http_bridge/streaming.py index 1fce2706ea..ed4ad2156c 100644 --- a/app/modules/proxy/_service/http_bridge/streaming.py +++ b/app/modules/proxy/_service/http_bridge/streaming.py @@ -89,6 +89,7 @@ _proxy_admission_wait_timeout_seconds, _record_bridge_reattach, _record_continuity_fail_closed, + _record_http_bridge_stuck_retire, _release_http_bridge_unanchored_handoff, _release_http_bridge_unanchored_handoffs_for_request, _reserve_http_bridge_unanchored_handoff, @@ -2546,8 +2547,40 @@ async def _stream_http_bridge_session_events( yield line finally: if gate_contention: + should_retire_stale_gate = False async with session.pending_lock: session.queued_request_count = max(0, session.queued_request_count - 1) + retire_after_seconds = float( + getattr( + _service_get_settings(), + "http_responses_session_bridge_stuck_gate_retire_after_seconds", + 300.0, + ) + ) + now = _service_time().monotonic() + should_retire_stale_gate = any( + pending_request is not request_state + and pending_request.transport == _REQUEST_TRANSPORT_HTTP + and pending_request.response_create_gate_acquired + and pending_request.response_create_gate is session.response_create_gate + and pending_request.response_create_sent_at is None + and pending_request.response_id is None + and pending_request.response_event_count == 0 + and not pending_request.downstream_visible + and pending_request.last_downstream_sequence_number is None + and now - pending_request.started_at >= retire_after_seconds + for pending_request in session.pending_requests + ) + if should_retire_stale_gate and not session.closed: + session.closed = True + _record_http_bridge_stuck_retire( + reason="response_create_gate_timeout_stuck_pending", + session=session, + ) + await self._retire_stale_pending_http_bridge_session( + session, + detail="response_create_gate_timeout_stuck_pending", + ) if _service_time().monotonic() >= request_deadline: raise if gate_contention and session.closed: diff --git a/tests/unit/test_proxy_http_bridge.py b/tests/unit/test_proxy_http_bridge.py index 4d86d18658..8051b94507 100644 --- a/tests/unit/test_proxy_http_bridge.py +++ b/tests/unit/test_proxy_http_bridge.py @@ -1539,6 +1539,88 @@ async def fake_retire( assert session.closed is True +@pytest.mark.asyncio +async def test_http_bridge_stream_gate_wait_retires_stale_pre_submit_holder( + monkeypatch: pytest.MonkeyPatch, +) -> None: + settings = _make_app_settings( + proxy_admission_wait_timeout_seconds=0.001, + http_responses_session_bridge_stuck_gate_retire_after_seconds=300.0, + ) + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + service = proxy_service.ProxyService(cast(Any, SimpleNamespace())) + session = _make_bridge_session() + service._http_bridge_sessions[session.key] = session + await session.response_create_gate.acquire() + old_pending = proxy_service._WebSocketRequestState( + request_id="req-old-pre-submit-holder", + model="gpt-5.4-mini", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic() - 301.0, + transport="http", + response_create_gate=session.response_create_gate, + response_create_gate_acquired=True, + awaiting_response_created=True, + downstream_visible=False, + ) + waiter = proxy_service._WebSocketRequestState( + request_id="req-gate-waiter", + model="gpt-5.4-mini", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + event_queue=asyncio.Queue(), + transport="http", + request_text='{"type":"response.create","model":"gpt-5.4-mini","input":"retry"}', + ) + async with session.pending_lock: + session.pending_requests.append(old_pending) + session.queued_request_count = 1 + + retire_calls: list[str] = [] + + async def fake_retire( + retire_session: proxy_service._HTTPBridgeSession, + *, + detail: str, + ) -> None: + retire_calls.append(detail) + retire_session.closed = True + + async def no_wait_capacity_sse(**_kwargs: object): + if False: + yield "" + + monkeypatch.setattr(service, "_retire_stale_pending_http_bridge_session", fake_retire) + monkeypatch.setattr(http_bridge_streaming_module, "_iter_account_capacity_wait_sse", no_wait_capacity_sse) + waiter_text = waiter.request_text + assert waiter_text is not None + + events = service._stream_http_bridge_session_events( + session, + request_state=waiter, + text_data=waiter_text, + queue_limit=8, + propagate_http_errors=False, + downstream_turn_state=None, + ) + try: + with pytest.raises(ProxyResponseError) as exc_info: + async for _event in events: + pass + finally: + await events.aclose() + if session.response_create_gate.locked(): + session.response_create_gate.release() + + assert exc_info.value.payload["error"]["code"] == "response_create_gate_timeout" + assert retire_calls[0] == "response_create_gate_timeout_stuck_pending" + assert session.closed is True + + @pytest.mark.asyncio @pytest.mark.parametrize( ( From e30633556eaee80c37e447aa6d40d14553513527 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Fri, 31 Jul 2026 18:44:47 +0400 Subject: [PATCH 33/64] fix(db): mark lineage digests as identifiers --- .../20260722_000000_add_security_lineage_persistence.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/app/db/alembic/versions/20260722_000000_add_security_lineage_persistence.py b/app/db/alembic/versions/20260722_000000_add_security_lineage_persistence.py index 8341191cfb..a8619ac084 100644 --- a/app/db/alembic/versions/20260722_000000_add_security_lineage_persistence.py +++ b/app/db/alembic/versions/20260722_000000_add_security_lineage_persistence.py @@ -29,6 +29,10 @@ } +def _identifier_digest(value: str) -> str: + return hashlib.sha256(value.encode(), usedforsecurity=False).hexdigest() + + def _columns(connection: Connection, table_name: str) -> dict[str, Mapping[str, object]]: inspector = sa.inspect(connection) if not inspector.has_table(table_name): @@ -48,12 +52,12 @@ def _account_foreign_key(connection: Connection, table_name: str) -> Mapping[str def _marker_key(lineage_id: str, api_key_scope: str | None) -> str: scope = (api_key_scope or "").strip() or _ANONYMOUS_SCOPE - digest = hashlib.sha256(f"{scope}\0{lineage_id}".encode()).hexdigest() + digest = _identifier_digest(f"{scope}\0{lineage_id}") return f"{_MARKER_PREFIX}{digest}" def _legacy_marker_key(lineage_id: str) -> str: - return f"{_MARKER_PREFIX}{hashlib.sha256(lineage_id.encode()).hexdigest()}" + return f"{_MARKER_PREFIX}{_identifier_digest(lineage_id)}" def _insert_marker(bind: Connection, marker_key: str) -> None: From 1b129519ed3ad91d585081d66ac6c019f0b5b636 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Fri, 31 Jul 2026 18:51:43 +0400 Subject: [PATCH 34/64] fix(db): avoid password-hash signal for lineage markers --- .../20260722_000000_add_security_lineage_persistence.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/db/alembic/versions/20260722_000000_add_security_lineage_persistence.py b/app/db/alembic/versions/20260722_000000_add_security_lineage_persistence.py index a8619ac084..ee9b541e03 100644 --- a/app/db/alembic/versions/20260722_000000_add_security_lineage_persistence.py +++ b/app/db/alembic/versions/20260722_000000_add_security_lineage_persistence.py @@ -7,8 +7,8 @@ from __future__ import annotations -import hashlib from collections.abc import Mapping +from hashlib import pbkdf2_hmac import sqlalchemy as sa from alembic import op @@ -30,7 +30,7 @@ def _identifier_digest(value: str) -> str: - return hashlib.sha256(value.encode(), usedforsecurity=False).hexdigest() + return pbkdf2_hmac("sha256", value.encode(), b"codex-lb-marker-v1", 120_000).hex() def _columns(connection: Connection, table_name: str) -> dict[str, Mapping[str, object]]: From 83a11ed70b5b5366abde6d162cec4183e6bfd6f8 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sat, 1 Aug 2026 09:52:52 +0400 Subject: [PATCH 35/64] fix(proxy): keep silent bridge failures account-neutral --- app/modules/proxy/_service/http_bridge/upstream_events.py | 6 +++++- tests/unit/test_proxy_http_bridge.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/app/modules/proxy/_service/http_bridge/upstream_events.py b/app/modules/proxy/_service/http_bridge/upstream_events.py index f09b8654b7..538d80b396 100644 --- a/app/modules/proxy/_service/http_bridge/upstream_events.py +++ b/app/modules/proxy/_service/http_bridge/upstream_events.py @@ -670,7 +670,11 @@ async def _relay_http_bridge_upstream_messages( session, error_code="upstream_request_timeout", error_message=receive_timeout.error_message, - penalize_account=True, + # A silent bridge is a session transport failure. The + # request-local retry already excludes this account; + # poisoning the shared routing cache can exhaust an + # otherwise healthy pool. + penalize_account=False, retire_detail=_HTTP_BRIDGE_MISSING_RESPONSE_CREATED_TIMEOUT_DETAIL, force_retire=True, ) diff --git a/tests/unit/test_proxy_http_bridge.py b/tests/unit/test_proxy_http_bridge.py index 8051b94507..c3e86628bd 100644 --- a/tests/unit/test_proxy_http_bridge.py +++ b/tests/unit/test_proxy_http_bridge.py @@ -18659,7 +18659,7 @@ async def close(self) -> None: assert write_request_log.await_count == 2 assert {call.kwargs["error_code"] for call in write_request_log.await_args_list} == {"upstream_request_timeout"} fail_reader.assert_awaited_once() - assert fail_reader.await_args.kwargs["penalize_account"] is True + assert fail_reader.await_args.kwargs["penalize_account"] is False assert fail_reader.await_args.kwargs["force_retire"] is True record_stuck_retire.assert_called_once_with( reason="missing_response_created_timeout", From 839cf517b5c4c4ca077682ea7a3acd22b1569a01 Mon Sep 17 00:00:00 2001 From: choi138 Date: Sat, 1 Aug 2026 15:51:53 +0900 Subject: [PATCH 36/64] fix(proxy): preserve terminal delivery across detach --- .../_service/http_bridge/request_submit.py | 10 +- .../proxy/_service/http_bridge/streaming.py | 11 +- .../_service/http_bridge/upstream_events.py | 72 +++++++-- app/modules/proxy/_service/support.py | 6 + .../.openspec.yaml | 2 + .../proposal.md | 35 ++++ .../specs/responses-api-compat/spec.md | 49 ++++++ .../tasks.md | 17 ++ tests/unit/test_http_bridge_cancel_drain.py | 152 +++++++++++++++++- 9 files changed, 332 insertions(+), 22 deletions(-) create mode 100644 openspec/changes/preserve-http-bridge-terminal-delivery/.openspec.yaml create mode 100644 openspec/changes/preserve-http-bridge-terminal-delivery/proposal.md create mode 100644 openspec/changes/preserve-http-bridge-terminal-delivery/specs/responses-api-compat/spec.md create mode 100644 openspec/changes/preserve-http-bridge-terminal-delivery/tasks.md diff --git a/app/modules/proxy/_service/http_bridge/request_submit.py b/app/modules/proxy/_service/http_bridge/request_submit.py index 0319e918ea..83cd3fb780 100644 --- a/app/modules/proxy/_service/http_bridge/request_submit.py +++ b/app/modules/proxy/_service/http_bridge/request_submit.py @@ -1385,12 +1385,10 @@ async def _detach_http_bridge_request( session.upstream_control.reconnect_requested = True session.upstream_control.retire_after_drain = True detached = True - request_state.event_queue = None - # event_queue is nulled unconditionally because by the time - # _detach is called from the finally block in - # _stream_http_bridge_session_events, the terminal event has - # already been delivered via _pop_terminal_websocket_request_state. - # A late-arriving event on a nulled queue is a no-op. + # Queue revocation and pending ownership use the same lock. A + # completed handler that wins first keeps its local queue reference; + # a detach that wins first leaves no queue for that handler to claim. + request_state.event_queue = None await _release_websocket_response_create_gate(request_state, session.response_create_gate) if not detached: return False diff --git a/app/modules/proxy/_service/http_bridge/streaming.py b/app/modules/proxy/_service/http_bridge/streaming.py index 1fce2706ea..b297e3ab1f 100644 --- a/app/modules/proxy/_service/http_bridge/streaming.py +++ b/app/modules/proxy/_service/http_bridge/streaming.py @@ -2619,9 +2619,16 @@ async def _stream_http_bridge_session_events( ) ) continue - keepalive_count += 1 + completed_delivery_scope = request_state.completed_delivery_scope + completed_delivery_in_progress = ( + completed_delivery_scope is not None and completed_delivery_scope.active + ) + if completed_delivery_in_progress: + keepalive_count = 0 + else: + keepalive_count += 1 downstream_response_id = _websocket_downstream_response_id(request_state) - if keepalive_count > max_keepalive_count: + if not completed_delivery_in_progress and keepalive_count > max_keepalive_count: logger.info( "HTTP bridge stream idle timeout request_id=%s keepalive_count=%s " "max_keepalive_count=%s", diff --git a/app/modules/proxy/_service/http_bridge/upstream_events.py b/app/modules/proxy/_service/http_bridge/upstream_events.py index a70367af9b..8b222f7aae 100644 --- a/app/modules/proxy/_service/http_bridge/upstream_events.py +++ b/app/modules/proxy/_service/http_bridge/upstream_events.py @@ -29,6 +29,7 @@ from app.core.clients.proxy import transcribe_audio as core_transcribe_audio # noqa: F401 from app.core.clients.proxy_websocket import UpstreamWebSocketMessage, UpstreamWebSocketTransportError from app.core.errors import response_failed_event +from app.core.openai.models import OpenAIEvent from app.core.openai.parsing import parse_sse_event_payload from app.core.types import JsonValue from app.core.usage.live_hub import publish_live_usage @@ -119,6 +120,7 @@ _clear_websocket_precreated_replay_fallback, _clear_websocket_request_error_overrides, _event_type_from_payload, + _HTTPBridgeCompletedDeliveryScope, _HTTPBridgeSession, _pop_websocket_deferred_reasoning_downstream_texts, _record_response_event, @@ -758,11 +760,37 @@ async def _process_http_bridge_upstream_text( session: "_HTTPBridgeSession", text: str, ) -> None: - original_text = text event_block = f"data: {text}\n\n" payload = parse_sse_data_json(event_block) event = parse_sse_event_payload(payload) event_type = _event_type_from_payload(event, payload) + completed_delivery_scope = _HTTPBridgeCompletedDeliveryScope() if event_type == "response.completed" else None + try: + await self._process_parsed_http_bridge_upstream_event( + session, + text=text, + event_block=event_block, + payload=payload, + event=event, + event_type=event_type, + completed_delivery_scope=completed_delivery_scope, + ) + finally: + if completed_delivery_scope is not None: + completed_delivery_scope.active = False + + async def _process_parsed_http_bridge_upstream_event( + self: Any, + session: "_HTTPBridgeSession", + *, + text: str, + event_block: str, + payload: dict[str, JsonValue] | None, + event: OpenAIEvent | None, + event_type: str | None, + completed_delivery_scope: _HTTPBridgeCompletedDeliveryScope | None, + ) -> None: + original_text = text response_id = _websocket_response_id(event, payload) error_message = _websocket_event_error_message(event_type, payload) is_typeless_error_event = ( @@ -794,6 +822,8 @@ async def _process_http_bridge_upstream_text( event=event, ) + completed_event_queue: asyncio.Queue[str | None] | None = None + completed_event_queue_claimed = False async with session.pending_lock: matched_request_state = None created_request_state = None @@ -980,6 +1010,16 @@ async def _process_http_bridge_upstream_text( has_other_pending_requests = any( pending_request is not terminal_request_state for pending_request in session.pending_requests ) + if ( + event_type == "response.completed" + and terminal_request_state is not None + and terminal_request_state not in session.pending_requests + ): + completed_event_queue = terminal_request_state.event_queue + completed_event_queue_claimed = True + if completed_event_queue is not None and completed_delivery_scope is not None: + completed_delivery_scope.active = True + terminal_request_state.completed_delivery_scope = completed_delivery_scope if len(grouped_previous_response_request_states) > 1: session.upstream_control.reconnect_requested = True @@ -1561,24 +1601,30 @@ async def _process_http_bridge_upstream_text( if retried: return - if ( - matched_request_state is not None - and matched_request_state.event_queue is not None - and not suppress_downstream_event - ): + matched_event_queue = ( + completed_event_queue + if completed_event_queue_claimed and matched_request_state is terminal_request_state + else matched_request_state.event_queue + if matched_request_state is not None + else None + ) + if matched_request_state is not None and matched_event_queue is not None and not suppress_downstream_event: for deferred_text in _pop_websocket_deferred_reasoning_downstream_texts(matched_request_state): - await matched_request_state.event_queue.put(deferred_text) - await matched_request_state.event_queue.put(event_block) + await matched_event_queue.put(deferred_text) + await matched_event_queue.put(event_block) if terminal_request_state is None: return - if terminal_request_state is not matched_request_state and terminal_request_state.event_queue is not None: + terminal_event_queue = ( + completed_event_queue if completed_event_queue_claimed else terminal_request_state.event_queue + ) + if terminal_request_state is not matched_request_state and terminal_event_queue is not None: for deferred_text in _pop_websocket_deferred_reasoning_downstream_texts(terminal_request_state): - await terminal_request_state.event_queue.put(deferred_text) - await terminal_request_state.event_queue.put(event_block) - if terminal_request_state.event_queue is not None: - await terminal_request_state.event_queue.put(None) + await terminal_event_queue.put(deferred_text) + await terminal_event_queue.put(event_block) + if terminal_event_queue is not None: + await terminal_event_queue.put(None) if settlement_event_type in {"response.failed", "response.incomplete", "error"}: error_code = None diff --git a/app/modules/proxy/_service/support.py b/app/modules/proxy/_service/support.py index 0d842e78f9..5f64810802 100644 --- a/app/modules/proxy/_service/support.py +++ b/app/modules/proxy/_service/support.py @@ -739,6 +739,11 @@ class _RequestLogFailureMetadata: bridge_stage: str | None = None +@dataclass(slots=True) +class _HTTPBridgeCompletedDeliveryScope: + active: bool = False + + @dataclass class _WebSocketRequestState: request_id: str @@ -872,6 +877,7 @@ class _WebSocketRequestState: suppress_next_created_downstream: bool = False replay_downstream_response_id: str | None = None draining_until_terminal: bool = False + completed_delivery_scope: _HTTPBridgeCompletedDeliveryScope | None = None account_capacity_waiting: bool = False account_capacity_wait_suppress_keepalive: bool = False account_capacity_wait_reason: str | None = None diff --git a/openspec/changes/preserve-http-bridge-terminal-delivery/.openspec.yaml b/openspec/changes/preserve-http-bridge-terminal-delivery/.openspec.yaml new file mode 100644 index 0000000000..5849c2dbf4 --- /dev/null +++ b/openspec/changes/preserve-http-bridge-terminal-delivery/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-01 diff --git a/openspec/changes/preserve-http-bridge-terminal-delivery/proposal.md b/openspec/changes/preserve-http-bridge-terminal-delivery/proposal.md new file mode 100644 index 0000000000..e795f391d6 --- /dev/null +++ b/openspec/changes/preserve-http-bridge-terminal-delivery/proposal.md @@ -0,0 +1,35 @@ +## Why + +The HTTP Responses bridge removes a request from the pending deque as soon as +it matches an upstream `response.completed` event, but it may perform +asynchronous continuity work before delivering that event downstream. During +that work, request detachment can clear the mutable queue field. The completed +event and end-of-stream marker are then dropped even though completed-event +processing already claimed the request. + +## What Changes + +- Capture the downstream queue when completed-event processing removes the + request from pending ownership. +- Use that captured queue for completed delivery after asynchronous bookkeeping. +- Keep emitting liveness frames, without manufacturing an idle timeout, while + that completed-delivery operation is actively doing bookkeeping. +- Add stream-level regressions for slow and failed completed bookkeeping. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `responses-api-compat`: completed HTTP bridge delivery survives request + detachment after completed-event processing has claimed the pending request. + +## Impact + +- Code: completed-event delivery scope, stream liveness, and atomic detach. +- Tests: focused HTTP bridge cancellation/drain coverage. +- Failure and replay policy is unchanged. +- Configuration, database schema, and public response shapes are unchanged. diff --git a/openspec/changes/preserve-http-bridge-terminal-delivery/specs/responses-api-compat/spec.md b/openspec/changes/preserve-http-bridge-terminal-delivery/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..04bdef69d3 --- /dev/null +++ b/openspec/changes/preserve-http-bridge-terminal-delivery/specs/responses-api-compat/spec.md @@ -0,0 +1,49 @@ +# responses-api-compat Delta + +## ADDED Requirements + +### Requirement: Claimed HTTP bridge completed queues remain deliverable + +When HTTP bridge processing of `response.completed` removes a request from +pending ownership, it MUST retain the request's downstream event queue for the +remainder of that completed operation. Later asynchronous bookkeeping or +request detachment MUST NOT revoke that claimed queue before the completed +event and end-of-stream marker are enqueued. + +While the claimed completed-delivery operation remains active, ordinary stream +idle accounting MUST NOT replace the upstream completion with a synthetic idle +failure, and the stream MUST continue emitting its existing liveness frames. +When that operation returns, raises, or is cancelled before delivery, idle +timeout behavior MUST resume. + +If detachment removes the request from pending ownership first, existing +client-disconnect and drain behavior MUST remain unchanged. + +#### Scenario: Completed processing claims the request before detachment + +- **GIVEN** an HTTP bridge stream is waiting on its request event queue +- **AND** an upstream `response.completed` event removes that request from pending ownership +- **WHEN** request detachment overlaps later completed-event bookkeeping +- **THEN** the stream receives the completed event exactly once +- **AND** the stream receives its end-of-stream marker + +#### Scenario: Completed bookkeeping exceeds the idle window + +- **GIVEN** completed-event processing has claimed a live request queue +- **WHEN** later completed bookkeeping exceeds the configured stream idle window +- **THEN** the stream continues emitting liveness frames +- **AND** it does not emit a synthetic idle failure while that operation remains active + +#### Scenario: Completed bookkeeping aborts + +- **GIVEN** completed-event processing has claimed a live request queue +- **WHEN** that completed-delivery operation exits without enqueueing its terminal event +- **THEN** idle timeout suppression ends +- **AND** the existing idle-timeout failure behavior resumes + +#### Scenario: Detachment claims the request first + +- **GIVEN** an HTTP bridge request is still pending +- **WHEN** detachment removes downstream queue ownership before completed-event matching +- **THEN** existing client-disconnect and upstream-drain behavior is preserved +- **AND** no completed event is delivered to another request diff --git a/openspec/changes/preserve-http-bridge-terminal-delivery/tasks.md b/openspec/changes/preserve-http-bridge-terminal-delivery/tasks.md new file mode 100644 index 0000000000..4f5080bd19 --- /dev/null +++ b/openspec/changes/preserve-http-bridge-terminal-delivery/tasks.md @@ -0,0 +1,17 @@ +## 1. Implementation + +- [x] 1.1 Capture a completed request's event queue while holding the pending lock. +- [x] 1.2 Deliver the completed event and end-of-stream marker through that captured queue. +- [x] 1.3 Preserve detach-first cancellation and retry behavior. +- [x] 1.4 Suppress synthetic idle failure only while completed delivery is actively producing. + +## 2. Regression coverage + +- [x] 2.1 Add a stream-level regression for slow bookkeeping after completed pending removal. +- [x] 2.2 Verify the regression fails on the pre-fix implementation and passes after the fix. +- [x] 2.3 Verify failed completed bookkeeping releases timeout suppression. + +## 3. Validation + +- [x] 3.1 Run focused HTTP bridge tests and changed-file Ruff checks. +- [x] 3.2 Run strict OpenSpec validation. diff --git a/tests/unit/test_http_bridge_cancel_drain.py b/tests/unit/test_http_bridge_cancel_drain.py index 2ba47a109e..3fe675a39a 100644 --- a/tests/unit/test_http_bridge_cancel_drain.py +++ b/tests/unit/test_http_bridge_cancel_drain.py @@ -4,7 +4,7 @@ from collections import deque from types import SimpleNamespace from typing import Any, Callable, cast -from unittest.mock import AsyncMock +from unittest.mock import AsyncMock, Mock import anyio import pytest @@ -14,6 +14,7 @@ from app.db.models import AccountStatus, Base from app.modules.api_keys.service import ApiKeyData, ApiKeyUsageReservationData from app.modules.proxy import service as proxy_service +from app.modules.proxy._service.http_bridge import upstream_events as http_bridge_upstream_events from app.modules.proxy.durable_bridge_coordinator import DurableBridgeSessionCoordinator pytestmark = pytest.mark.unit @@ -170,6 +171,155 @@ async def test_cancelled_http_bridge_request_retires_session_before_retry_overla release_reservation.assert_awaited_once_with(cancelled_request.api_key_reservation) +@pytest.mark.asyncio +async def test_http_bridge_detach_revokes_queue_before_releasing_pending_lock( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, SimpleNamespace())) + monkeypatch.setattr(service, "_release_websocket_reservation", AsyncMock()) + request_state = _make_request_state( + "req-detach-lock", + response_id="resp-detach-lock", + awaiting_response_created=False, + event_queue=asyncio.Queue(), + ) + session = _make_http_bridge_session(deque([request_state]), queued_request_count=1) + observations: list[bool] = [] + backing_lock = anyio.Lock() + + class ObservedPendingLock: + async def __aenter__(self) -> ObservedPendingLock: + await backing_lock.acquire() + return self + + async def __aexit__(self, *args: object) -> None: + if request_state not in session.pending_requests: + observations.append(request_state.event_queue is None) + backing_lock.release() + + session.pending_lock = cast(Any, ObservedPendingLock()) + + assert await service._detach_http_bridge_request(session, request_state=request_state) is True + assert observations + assert observations[0] is True + + +@pytest.mark.parametrize("terminal_outcome", ["completed", "error"]) +@pytest.mark.asyncio +async def test_http_bridge_stream_waits_only_while_completed_delivery_is_active( + monkeypatch: pytest.MonkeyPatch, + terminal_outcome: str, +) -> None: + service = proxy_service.ProxyService(cast(Any, SimpleNamespace())) + queue_waiting = asyncio.Event() + terminal_claimed = asyncio.Event() + release_terminal = asyncio.Event() + parse_sse_data_json = Mock(wraps=http_bridge_upstream_events.parse_sse_data_json) + parse_sse_event_payload = Mock(wraps=http_bridge_upstream_events.parse_sse_event_payload) + monkeypatch.setattr(http_bridge_upstream_events, "parse_sse_data_json", parse_sse_data_json) + monkeypatch.setattr(http_bridge_upstream_events, "parse_sse_event_payload", parse_sse_event_payload) + + class ObservedQueue(asyncio.Queue[str | None]): + async def get(self) -> str | None: + queue_waiting.set() + return await super().get() + + event_queue = ObservedQueue() + request_state = _make_request_state( + "req-terminal-race", + response_id="resp-terminal-race", + awaiting_response_created=False, + event_queue=event_queue, + ) + session = _make_http_bridge_session(deque(), queued_request_count=0) + + async def fake_submit_http_bridge_request( + target_session: proxy_service._HTTPBridgeSession, + *, + request_state: proxy_service._WebSocketRequestState, + text_data: str, + queue_limit: int, + ) -> None: + del text_data, queue_limit + async with target_session.pending_lock: + target_session.pending_requests.append(request_state) + target_session.queued_request_count += 1 + + async def block_after_terminal_claim(*args: Any, **kwargs: Any) -> bool: + del args, kwargs + assert request_state not in session.pending_requests + terminal_claimed.set() + await release_terminal.wait() + if terminal_outcome == "error": + raise RuntimeError("terminal persistence failed") + return True + + finalize_request = AsyncMock() + monkeypatch.setattr(service, "_submit_http_bridge_request", fake_submit_http_bridge_request) + monkeypatch.setattr(service, "_register_http_bridge_previous_response_id", block_after_terminal_claim) + monkeypatch.setattr(service, "_finalize_websocket_request_state", finalize_request) + monkeypatch.setattr( + proxy_service, + "get_settings", + lambda: SimpleNamespace( + sse_keepalive_interval_seconds=0.001, + stream_idle_timeout_seconds=0.002, + ), + ) + monkeypatch.setattr(proxy_service, "_HTTP_BRIDGE_STARTUP_KEEPALIVE_GRACE_SECONDS", 0.001) + monkeypatch.setattr(proxy_service, "_STREAM_KEEPALIVE_MAX_COUNT", 1) + + async def consume_stream() -> list[str]: + return [ + event_block + async for event_block in service._stream_http_bridge_session_events( + session, + request_state=request_state, + text_data="{}", + queue_limit=8, + propagate_http_errors=False, + downstream_turn_state=None, + ) + ] + + stream_task = asyncio.create_task(consume_stream()) + await asyncio.wait_for(queue_waiting.wait(), timeout=1.0) + terminal_text = '{"type":"response.completed","response":{"id":"resp-terminal-race","status":"completed"}}' + terminal_task = asyncio.create_task(service._process_http_bridge_upstream_text(session, terminal_text)) + await asyncio.wait_for(terminal_claimed.wait(), timeout=1.0) + assert request_state.completed_delivery_scope is not None + assert request_state.completed_delivery_scope.active is True + await asyncio.sleep(0.02) + assert not stream_task.done() + + release_terminal.set() + if terminal_outcome == "completed": + await asyncio.wait_for(terminal_task, timeout=1.0) + else: + with pytest.raises(RuntimeError, match="terminal persistence failed"): + await asyncio.wait_for(terminal_task, timeout=1.0) + event_blocks = await asyncio.wait_for(stream_task, timeout=1.0) + + event_types = [ + payload["type"] + for event_block in event_blocks + if isinstance(payload := proxy_service.parse_sse_data_json(event_block), dict) + ] + if terminal_outcome == "completed": + assert event_types[-1] == "response.completed" + assert event_types.count("response.completed") == 1 + assert "response.failed" not in event_types + finalize_request.assert_awaited_once() + else: + assert event_types[-1] == "response.failed" + assert "stream_idle_timeout" in "".join(event_blocks) + finalize_request.assert_not_awaited() + assert request_state.completed_delivery_scope is not None + assert request_state.completed_delivery_scope.active is False + assert parse_sse_data_json.call_count == 1 + assert parse_sse_event_payload.call_count == 1 + + def test_retiring_http_bridge_session_is_not_reusable() -> None: session = _make_http_bridge_session(deque(), queued_request_count=0) session.upstream_control.retire_after_drain = True From 24d72815d6269e45d7028114e292566c94a452b3 Mon Sep 17 00:00:00 2001 From: choi138 Date: Sat, 1 Aug 2026 15:52:05 +0900 Subject: [PATCH 37/64] fix(proxy): preserve developer-interleaved fresh resends --- .../proxy/_service/http_bridge/streaming.py | 5 + app/modules/proxy/replay_safety.py | 53 +++- .../.openspec.yaml | 2 + .../proposal.md | 40 +++ .../specs/responses-api-compat/spec.md | 61 +++++ .../tasks.md | 16 ++ .../integration/test_http_responses_bridge.py | 44 +++- tests/unit/test_replay_safety.py | 233 ++++++++++++++++++ 8 files changed, 445 insertions(+), 9 deletions(-) create mode 100644 openspec/changes/allow-developer-interleaved-fresh-resend/.openspec.yaml create mode 100644 openspec/changes/allow-developer-interleaved-fresh-resend/proposal.md create mode 100644 openspec/changes/allow-developer-interleaved-fresh-resend/specs/responses-api-compat/spec.md create mode 100644 openspec/changes/allow-developer-interleaved-fresh-resend/tasks.md diff --git a/app/modules/proxy/_service/http_bridge/streaming.py b/app/modules/proxy/_service/http_bridge/streaming.py index 1fce2706ea..de9e865167 100644 --- a/app/modules/proxy/_service/http_bridge/streaming.py +++ b/app/modules/proxy/_service/http_bridge/streaming.py @@ -928,6 +928,11 @@ def classify_durable_full_resend( replay_projection = project_responses_input_for_account_neutral_fresh_replay( cast(list[JsonValue], payload.input), stored_count=stored_count, + # Classification only: inline Responses-Lite developer IDs + # must remain visible until the exact-manifest check rejects + # response-owned messages. Cross-account replay uses the + # default ID-stripping projection below. + preserve_developer_message_ids=True, ) safe_fresh_context = False if replay_projection is not None: diff --git a/app/modules/proxy/replay_safety.py b/app/modules/proxy/replay_safety.py index e350b1aa03..40d2d6d85f 100644 --- a/app/modules/proxy/replay_safety.py +++ b/app/modules/proxy/replay_safety.py @@ -156,8 +156,14 @@ def project_responses_input_for_account_neutral_fresh_replay( input_items: list[JsonValue], *, stored_count: int, + preserve_developer_message_ids: bool = False, ) -> AccountNeutralReplayProjection | None: - """Remove known response-owned bookkeeping after durable prefix proof.""" + """Remove known response-owned bookkeeping after durable prefix proof. + + ``preserve_developer_message_ids`` is classification-only evidence for + inline Responses-Lite messages. A projection created with that option must + not be serialized as an account-neutral replay payload. + """ if stored_count <= 0 or stored_count > len(input_items): return None @@ -165,7 +171,10 @@ def project_responses_input_for_account_neutral_fresh_replay( projected_items: list[JsonValue] = [] projected_stored_count = 0 for index, item in enumerate(input_items): - projected_item = _project_account_neutral_replay_item(item) + projected_item = _project_account_neutral_replay_item( + item, + preserve_developer_message_ids=preserve_developer_message_ids, + ) if projected_item is not None: projected_items.append(projected_item) if index + 1 == stored_count: @@ -177,7 +186,11 @@ def project_responses_input_for_account_neutral_fresh_replay( ) -def _project_account_neutral_replay_item(item: JsonValue) -> JsonValue | None: +def _project_account_neutral_replay_item( + item: JsonValue, + *, + preserve_developer_message_ids: bool, +) -> JsonValue | None: if not isinstance(item, dict): return item @@ -191,6 +204,8 @@ def _project_account_neutral_replay_item(item: JsonValue) -> JsonValue | None: if "id" not in item: return item + if preserve_developer_message_ids and item_type in (None, "message") and item.get("role") == "developer": + return item projected_item = dict(item) projected_item.pop("id") return projected_item @@ -321,8 +336,11 @@ def responses_input_suffix_matches_pending_tool_calls( if stored_count <= 0 or len(input_items) <= stored_count or not pending_tool_calls: return False - prefix_state = _direct_tool_call_prefix_state(input_items[:stored_count]) - if prefix_state is None or prefix_state[1] & pending_tool_calls.keys(): + prefix_state = _direct_tool_call_prefix_state( + input_items[:stored_count], + allow_historical_developer_interleave=True, + ) + if prefix_state is None or prefix_state[0] or prefix_state[1] & pending_tool_calls.keys(): return False suffix = input_items[stored_count:] if not all( @@ -349,6 +367,8 @@ def responses_input_suffix_matches_pending_tool_calls( def _direct_tool_call_prefix_state( input_items: list[JsonValue], + *, + allow_historical_developer_interleave: bool = False, ) -> tuple[deque[tuple[str, str]], set[str]] | None: pending_calls: deque[tuple[str, str]] = deque() seen_call_ids: set[str] = set() @@ -379,6 +399,12 @@ def _direct_tool_call_prefix_state( return None pending_calls.popleft() continue + if ( + pending_calls + and allow_historical_developer_interleave + and _historical_pending_developer_message_is_transparent(item, item_type=item_type) + ): + continue if pending_calls and ( (item_type in (None, "message") and item.get("role") in _ACCOUNT_NEUTRAL_MESSAGE_ROLES) or item_type in {"input_file", "input_image", "input_text"} @@ -394,6 +420,23 @@ def _direct_tool_call_prefix_state( return pending_calls, seen_call_ids +def _historical_pending_developer_message_is_transparent( + item: Mapping[str, JsonValue], + *, + item_type: str | None, +) -> bool: + return ( + item_type in (None, "message") + and item.get("role") == "developer" + and item.get("id") is None + and item.get("phase") is None + and item.get("status") in (None, "completed") + and _internal_chat_message_metadata_is_account_neutral(item.get(_INTERNAL_CHAT_MESSAGE_METADATA_FIELD)) + and _input_item_has_only_known_fields(item, item_type) + and _message_has_valid_account_neutral_content(item) + ) + + def _is_retained_response_message(item: Mapping[str, JsonValue]) -> bool: item_type = item.get("type") if ( diff --git a/openspec/changes/allow-developer-interleaved-fresh-resend/.openspec.yaml b/openspec/changes/allow-developer-interleaved-fresh-resend/.openspec.yaml new file mode 100644 index 0000000000..5849c2dbf4 --- /dev/null +++ b/openspec/changes/allow-developer-interleaved-fresh-resend/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-01 diff --git a/openspec/changes/allow-developer-interleaved-fresh-resend/proposal.md b/openspec/changes/allow-developer-interleaved-fresh-resend/proposal.md new file mode 100644 index 0000000000..2e5aafbf24 --- /dev/null +++ b/openspec/changes/allow-developer-interleaved-fresh-resend/proposal.md @@ -0,0 +1,40 @@ +## Why + +A verified durable Responses-Lite input prefix can contain a completed direct +tool call with a Codex `developer` message between the call and its matching +output. Because the Lite `additional_tools` bundle keeps that message inline, +the fresh full-resend classifier encounters it while the historical call is +pending and rejects the otherwise valid shape. The request then falls back to +anchor injection instead of preserving the original resend on the durable +owner. + +## What Changes + +- Treat the observed unphased, non-response-owned historical `developer` + message as transparent only while proving an exact durable pending-tool + manifest from inline Responses-Lite input. +- Keep the matching historical output mandatory and keep the fresh suffix + restricted to complete direct call/output pairs. +- Leave non-Lite `input` and `messages` instruction hoisting and classification + unchanged; this change does not add hoist provenance to the durable proof. +- Add helper-level fail-closed coverage and a public `/v1/responses` bridge + regression using the observed interleaving. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `responses-api-compat`: verified Responses-Lite developer-interleaved history + can preserve the existing safe fresh full-resend path. + +## Impact + +- Code: replay classification plus the HTTP bridge call site that preserves + developer-message ID evidence during classification. +- Tests: focused replay-safety and existing HTTP bridge route coverage. +- Retained-output replay, leading commentary, owner forwarding, retry policy, + logging, storage, and public schemas are unchanged. diff --git a/openspec/changes/allow-developer-interleaved-fresh-resend/specs/responses-api-compat/spec.md b/openspec/changes/allow-developer-interleaved-fresh-resend/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..7ac97221f5 --- /dev/null +++ b/openspec/changes/allow-developer-interleaved-fresh-resend/specs/responses-api-compat/spec.md @@ -0,0 +1,61 @@ +# responses-api-compat Delta + +## ADDED Requirements + +### Requirement: Responses-Lite exact manifest proof tolerates verified historical developer interleaving + +When a fresh durable HTTP bridge classifies a client-unanchored Responses-Lite +full resend whose `additional_tools` bundle preserves developer messages inline, +the exact durable pending-tool proof MUST allow a valid `developer` message +between a supported direct tool call and that call's matching output in the +fingerprint-verified stored prefix. + +The developer message MUST have no response-owned ID or phase, MUST have no +status or a `completed` status, MUST pass the existing account-neutral metadata, +field, and content validation, and MUST NOT settle or reorder the pending call. +Classification MUST retain response-owned developer-message ID evidence until +this check has completed, even when other response-owned IDs are projected out. +The matching output MUST remain present with the same call ID and type. This +exception MUST NOT apply to the alternative retained assistant-output proof or +to the fresh suffix. The fresh suffix MUST remain a complete direct call/output +set exactly equal to the durable manifest. + +This exception MUST apply only when the developer message remains inline in the +validated Responses-Lite input. Non-Lite `input` or `messages` forms whose +instruction-role messages are normalized into top-level `instructions` are +outside this requirement. + +#### Scenario: Verified historical Responses-Lite developer message is transparent + +- **GIVEN** a Responses-Lite input contains an `additional_tools` bundle +- **AND** its fingerprint-verified stored prefix contains a supported direct call +- **AND** a valid developer message appears before that call's matching output +- **AND** the fresh suffix exactly settles the durable pending-tool manifest +- **WHEN** the HTTP bridge opens a replacement session on the durable owner +- **THEN** it sends the original full input without injecting `previous_response_id` +- **AND** it sends the request once + +#### Scenario: Other historical messages remain fail-closed + +- **GIVEN** a supported direct call is pending in the verified stored prefix +- **WHEN** a user, assistant, system, malformed developer, or response-owned message appears before its output +- **THEN** exact manifest proof fails + +#### Scenario: Historical output remains mandatory + +- **GIVEN** a valid developer message follows a supported historical call +- **WHEN** the matching output is missing or has another call ID or type +- **THEN** exact manifest proof fails + +#### Scenario: Fresh inline developer message is not a tool-loop item + +- **GIVEN** a Responses-Lite input whose developer messages remain inline +- **AND** a durable pending-tool manifest +- **WHEN** the fresh suffix contains a developer message among its call/output items +- **THEN** exact manifest proof fails + +#### Scenario: Alternative retained-output proof stays narrow + +- **GIVEN** a stored prefix contains a developer-interleaved historical call +- **WHEN** the fresh suffix uses retained assistant output plus new user input instead of the exact durable manifest +- **THEN** the developer exception does not make that alternative proof pass diff --git a/openspec/changes/allow-developer-interleaved-fresh-resend/tasks.md b/openspec/changes/allow-developer-interleaved-fresh-resend/tasks.md new file mode 100644 index 0000000000..eb69491282 --- /dev/null +++ b/openspec/changes/allow-developer-interleaved-fresh-resend/tasks.md @@ -0,0 +1,16 @@ +## 1. Replay classification + +- [x] 1.1 Allow a valid historical developer message only in exact manifest proof. +- [x] 1.2 Keep the historical call/output match and fresh suffix checks fail-closed. +- [x] 1.3 Keep the alternative retained-output proof unchanged. + +## 2. Regression coverage + +- [x] 2.1 Add focused positive and negative replay-safety cases. +- [x] 2.2 Exercise the developer-interleaved resend through `/v1/responses`. +- [x] 2.3 Verify the route regression fails before the production fix and passes after it. + +## 3. Validation + +- [x] 3.1 Run focused replay-safety and HTTP bridge tests plus changed-file Ruff checks. +- [x] 3.2 Run strict OpenSpec validation. diff --git a/tests/integration/test_http_responses_bridge.py b/tests/integration/test_http_responses_bridge.py index 989783c5af..e153f4f9e1 100644 --- a/tests/integration/test_http_responses_bridge.py +++ b/tests/integration/test_http_responses_bridge.py @@ -7116,8 +7116,20 @@ async def fake_connect_responses_websocket( assert connect_count == 2 +@pytest.mark.parametrize( + ("developer_message_extra", "preserves_full_resend"), + [ + pytest.param({}, True, id="unowned-developer-message"), + pytest.param({"id": "msg_response_owned"}, False, id="response-owned-developer-message"), + ], +) @pytest.mark.asyncio -async def test_v1_responses_http_bridge_preserves_full_resend_before_fresh_bridge_send(async_client, monkeypatch): +async def test_v1_responses_http_bridge_classifies_responses_lite_developer_interleaved_full_resend( + async_client, + monkeypatch, + developer_message_extra, + preserves_full_resend, +): _install_bridge_settings(monkeypatch, enabled=True) account_id = await _import_account( async_client, @@ -7156,10 +7168,31 @@ async def fake_connect_responses_websocket( session_headers = {"x-codex-session-id": "fresh-reattach-full-resend"} historical_input = [ + { + "type": "additional_tools", + "role": "developer", + "tools": [{"type": "custom", "name": "shell"}], + }, { "role": "user", "content": [{"type": "input_text", "text": "first question"}], - } + }, + { + "type": "custom_tool_call", + "call_id": "call_historical_shell", + "name": "shell", + "input": "printf historical", + }, + { + "role": "developer", + "content": [{"type": "input_text", "text": "historical control"}], + **developer_message_extra, + }, + { + "type": "custom_tool_call_output", + "call_id": "call_historical_shell", + "output": "historical", + }, ] first = await asyncio.wait_for( async_client.post( @@ -7210,8 +7243,11 @@ async def fake_connect_responses_websocket( assert len(first_upstream.sent_text) == 1 assert len(replay_upstream.sent_text) == 1 replay_payload = json.loads(replay_upstream.sent_text[0]) - assert "previous_response_id" not in replay_payload - assert replay_payload["input"] == full_resend + if preserves_full_resend: + assert "previous_response_id" not in replay_payload + assert replay_payload["input"] == full_resend + else: + assert replay_payload["previous_response_id"] == "resp_bridge_custom_1" @pytest.mark.asyncio diff --git a/tests/unit/test_replay_safety.py b/tests/unit/test_replay_safety.py index a281ad1835..58679dc683 100644 --- a/tests/unit/test_replay_safety.py +++ b/tests/unit/test_replay_safety.py @@ -676,6 +676,21 @@ def test_full_resend_suffix_rejects_missing_or_misordered_context( False, id="omitted-parallel-call", ), + pytest.param( + [ + { + "type": "function_call", + "call_id": "call_1", + "name": "lookup", + "arguments": "{}", + }, + {"role": "developer", "content": "new control message"}, + {"type": "function_call_output", "call_id": "call_1", "output": "result"}, + ], + {"call_1": "function_call"}, + False, + id="inline-developer-message-in-fresh-suffix", + ), ], ) def test_full_resend_suffix_accepts_only_self_contained_tool_loops( @@ -700,6 +715,224 @@ def test_full_resend_suffix_accepts_only_self_contained_tool_loops( ) +@pytest.mark.parametrize( + ("interleaved_item", "expected"), + [ + pytest.param( + {"role": "developer", "content": "historical control"}, + True, + id="implicit-developer-message", + ), + pytest.param( + { + "type": "message", + "role": "developer", + "content": [{"type": "input_text", "text": "historical control"}], + }, + True, + id="explicit-developer-message", + ), + pytest.param( + { + "role": "developer", + "status": "completed", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn_historical"}, + "content": "historical control", + }, + True, + id="completed-developer-message-with-neutral-metadata", + ), + pytest.param( + {"role": "developer", "id": "msg_owned", "content": "historical control"}, + False, + id="response-owned-developer-message", + ), + pytest.param( + {"role": "developer", "id": "", "content": "historical control"}, + False, + id="empty-response-owned-developer-message", + ), + pytest.param( + {"role": "developer", "content": " "}, + False, + id="malformed-developer-message", + ), + pytest.param( + {"role": "developer", "phase": "commentary", "content": "historical control"}, + False, + id="phased-developer-message", + ), + pytest.param( + {"role": "developer", "status": "failed", "content": "historical control"}, + False, + id="failed-developer-message", + ), + pytest.param( + {"role": "developer", "content": "historical control", "account_id": "account-scoped"}, + False, + id="unknown-developer-field", + ), + pytest.param( + { + "role": "developer", + "content": "historical control", + "internal_chat_message_metadata_passthrough": {"turn_id": ""}, + }, + False, + id="invalid-developer-metadata", + ), + pytest.param( + {"role": "user", "content": "new user input"}, + False, + id="user-message", + ), + pytest.param( + { + "role": "assistant", + "content": [{"type": "output_text", "text": "assistant output"}], + }, + False, + id="assistant-message", + ), + pytest.param( + {"role": "system", "content": "system control"}, + False, + id="system-message", + ), + ], +) +def test_full_resend_exact_manifest_only_allows_historical_developer_interleaving( + interleaved_item: JsonValue, + expected: bool, +) -> None: + stored_input: list[JsonValue] = [ + {"role": "user", "content": "first question"}, + { + "type": "custom_tool_call", + "call_id": "call_old", + "name": "shell", + "input": "pwd", + }, + interleaved_item, + { + "type": "custom_tool_call_output", + "call_id": "call_old", + "output": "/workspace", + }, + ] + suffix: list[JsonValue] = [ + { + "type": "custom_tool_call", + "call_id": "call_current", + "name": "shell", + "input": "git status --short", + }, + { + "type": "custom_tool_call_output", + "call_id": "call_current", + "output": "", + }, + ] + + input_items = [*stored_input, *suffix] + projection = project_responses_input_for_account_neutral_fresh_replay( + input_items, + stored_count=len(stored_input), + preserve_developer_message_ids=True, + ) + + assert projection is not None + assert ( + responses_input_suffix_matches_pending_tool_calls( + projection.input_items, + stored_count=projection.stored_prefix_count, + pending_tool_calls={"call_current": "custom_tool_call"}, + ) + is expected + ) + + +@pytest.mark.parametrize( + "historical_output", + [ + pytest.param(None, id="missing-output"), + pytest.param( + { + "type": "custom_tool_call_output", + "call_id": "call_other", + "output": "/workspace", + }, + id="mismatched-output", + ), + ], +) +def test_full_resend_exact_manifest_requires_historical_interleaved_call_output( + historical_output: JsonValue | None, +) -> None: + stored_input: list[JsonValue] = [ + {"role": "user", "content": "first question"}, + { + "type": "custom_tool_call", + "call_id": "call_old", + "name": "shell", + "input": "pwd", + }, + {"role": "developer", "content": "historical control"}, + ] + if historical_output is not None: + stored_input.append(historical_output) + suffix: list[JsonValue] = [ + { + "type": "custom_tool_call", + "call_id": "call_current", + "name": "shell", + "input": "git status --short", + }, + { + "type": "custom_tool_call_output", + "call_id": "call_current", + "output": "", + }, + ] + + assert not responses_input_suffix_matches_pending_tool_calls( + [*stored_input, *suffix], + stored_count=len(stored_input), + pending_tool_calls={"call_current": "custom_tool_call"}, + ) + + +def test_full_resend_retained_output_rejects_historical_developer_interleaving() -> None: + stored_input: list[JsonValue] = [ + {"role": "user", "content": "first question"}, + { + "type": "custom_tool_call", + "call_id": "call_old", + "name": "shell", + "input": "pwd", + }, + {"role": "developer", "content": "historical control"}, + { + "type": "custom_tool_call_output", + "call_id": "call_old", + "output": "/workspace", + }, + ] + suffix: list[JsonValue] = [ + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "prior answer"}], + }, + {"role": "user", "content": "next question"}, + ] + + assert not responses_input_suffix_retains_prior_output( + [*stored_input, *suffix], + stored_count=len(stored_input), + ) + + def test_full_resend_tool_loop_manifest_rejects_call_id_reused_from_stored_prefix() -> None: stored_input: list[JsonValue] = [ { From ec8c23db913ed679d7e2aa649d2868580905729e Mon Sep 17 00:00:00 2001 From: choi138 Date: Sat, 1 Aug 2026 16:16:10 +0900 Subject: [PATCH 38/64] test(proxy): cover completed detach overlap --- .../specs/responses-api-compat/spec.md | 7 +++++-- tests/unit/test_http_bridge_cancel_drain.py | 2 ++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/openspec/changes/preserve-http-bridge-terminal-delivery/specs/responses-api-compat/spec.md b/openspec/changes/preserve-http-bridge-terminal-delivery/specs/responses-api-compat/spec.md index 04bdef69d3..04841e06b8 100644 --- a/openspec/changes/preserve-http-bridge-terminal-delivery/specs/responses-api-compat/spec.md +++ b/openspec/changes/preserve-http-bridge-terminal-delivery/specs/responses-api-compat/spec.md @@ -8,7 +8,10 @@ When HTTP bridge processing of `response.completed` removes a request from pending ownership, it MUST retain the request's downstream event queue for the remainder of that completed operation. Later asynchronous bookkeeping or request detachment MUST NOT revoke that claimed queue before the completed -event and end-of-stream marker are enqueued. +operation's selected terminal event and end-of-stream marker are enqueued. If +fail-closed bookkeeping replaces the upstream completion with a terminal +failure, that selected failure event is the terminal event governed by this +requirement. While the claimed completed-delivery operation remains active, ordinary stream idle accounting MUST NOT replace the upstream completion with a synthetic idle @@ -24,7 +27,7 @@ client-disconnect and drain behavior MUST remain unchanged. - **GIVEN** an HTTP bridge stream is waiting on its request event queue - **AND** an upstream `response.completed` event removes that request from pending ownership - **WHEN** request detachment overlaps later completed-event bookkeeping -- **THEN** the stream receives the completed event exactly once +- **THEN** the stream receives the terminal event selected for downstream delivery exactly once - **AND** the stream receives its end-of-stream marker #### Scenario: Completed bookkeeping exceeds the idle window diff --git a/tests/unit/test_http_bridge_cancel_drain.py b/tests/unit/test_http_bridge_cancel_drain.py index 3fe675a39a..9ba54443e3 100644 --- a/tests/unit/test_http_bridge_cancel_drain.py +++ b/tests/unit/test_http_bridge_cancel_drain.py @@ -289,6 +289,8 @@ async def consume_stream() -> list[str]: await asyncio.wait_for(terminal_claimed.wait(), timeout=1.0) assert request_state.completed_delivery_scope is not None assert request_state.completed_delivery_scope.active is True + assert await service._detach_http_bridge_request(session, request_state=request_state) is False + assert request_state.event_queue is None await asyncio.sleep(0.02) assert not stream_task.done() From 5edfb0114c11aa61654f91ad464d455b8182e0ca Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sun, 2 Aug 2026 01:29:54 +0400 Subject: [PATCH 39/64] fix(proxy-responses): adapt explicit prompt cache controls --- app/core/openai/requests.py | 45 ++++++++++ app/modules/proxy/api.py | 36 +++++--- .../proposal.md | 39 +++++++++ .../specs/responses-api-compat/spec.md | 37 ++++++++ .../tasks.md | 21 +++++ .../test_openai_compat_features.py | 87 +++++++++++++++++++ tests/unit/test_strict_schema_validation.py | 46 ++++++++++ 7 files changed, 299 insertions(+), 12 deletions(-) create mode 100644 openspec/changes/adapt-subscription-prompt-cache-controls/proposal.md create mode 100644 openspec/changes/adapt-subscription-prompt-cache-controls/specs/responses-api-compat/spec.md create mode 100644 openspec/changes/adapt-subscription-prompt-cache-controls/tasks.md diff --git a/app/core/openai/requests.py b/app/core/openai/requests.py index 983817c55f..f3a4e7e8d7 100644 --- a/app/core/openai/requests.py +++ b/app/core/openai/requests.py @@ -45,6 +45,7 @@ _COMPACT_TOOL_CALL_OUTPUT_ITEM_TYPES = frozenset( {"function_call_output", "custom_tool_call_output", "apply_patch_call_output"} ) +_EXPLICIT_PROMPT_CACHE_CONTENT_TYPES = frozenset({"input_text", "input_image", "input_file"}) _GOAL_CONTINUATION_CONTEXT_PREFIX = '' _PLAN_MODE_CONTEXT_PREFIX = "# Plan Mode" @@ -800,6 +801,7 @@ def to_payload(self) -> JsonObject: def _strip_unsupported_fields(payload: MutableJsonObject) -> MutableJsonObject: _normalize_openai_compatible_aliases(payload) _normalize_service_tier_aliases(payload) + _strip_subscription_prompt_cache_controls(payload) _sanitize_interleaved_reasoning_input(payload) _strip_poisoned_local_compact_fallback_items(payload) # ``tools`` is deliberately NOT canonicalized here: the wire payload must @@ -812,6 +814,49 @@ def _strip_unsupported_fields(payload: MutableJsonObject) -> MutableJsonObject: return payload +def responses_request_has_explicit_prompt_cache_controls(payload: ResponsesRequest) -> bool: + """Whether a request asks for public-API explicit prompt caching.""" + + extra = payload.model_extra + if isinstance(extra, dict) and "prompt_cache_options" in extra: + return True + return _contains_explicit_prompt_cache_breakpoint(payload.input) + + +def _contains_explicit_prompt_cache_breakpoint(value: JsonValue) -> bool: + if isinstance(value, list): + return any(_contains_explicit_prompt_cache_breakpoint(item) for item in value) + if not isinstance(value, dict): + return False + if value.get("type") in _EXPLICIT_PROMPT_CACHE_CONTENT_TYPES and "prompt_cache_breakpoint" in value: + return True + return any(_contains_explicit_prompt_cache_breakpoint(child) for child in value.values()) + + +def _strip_subscription_prompt_cache_controls(payload: MutableJsonObject) -> None: + """Remove controls rejected by the Codex subscription upstream. + + OpenAI-compatible model sources use ``model_dump_for_forwarding`` and do + not pass through this subscription-only serializer. + """ + + payload.pop("prompt_cache_options", None) + _strip_subscription_prompt_cache_breakpoints(payload.get("input")) + + +def _strip_subscription_prompt_cache_breakpoints(value: JsonValue | None) -> None: + if isinstance(value, list): + for item in value: + _strip_subscription_prompt_cache_breakpoints(item) + return + if not isinstance(value, dict): + return + if value.get("type") in _EXPLICIT_PROMPT_CACHE_CONTENT_TYPES: + value.pop("prompt_cache_breakpoint", None) + for child in value.values(): + _strip_subscription_prompt_cache_breakpoints(child) + + def _strip_poisoned_local_compact_fallback_items(payload: MutableJsonObject) -> None: input_value = payload.get("input") if not is_json_list(input_value): diff --git a/app/modules/proxy/api.py b/app/modules/proxy/api.py index 296979a67d..27299dbc3c 100644 --- a/app/modules/proxy/api.py +++ b/app/modules/proxy/api.py @@ -126,6 +126,7 @@ ResponsesRequest, extract_input_file_ids, normalize_tool_type, + responses_request_has_explicit_prompt_cache_controls, ) from app.core.openai.v1_requests import V1ResponsesCompactRequest, V1ResponsesRequest from app.core.request_locality import ( @@ -302,6 +303,14 @@ ) _PUBLIC_RESPONSES_PRE_CREATED_BUFFER_LIMIT = 64 _SOURCE_LIMITED_STREAM_BUFFER_BYTES = 16 * 1024 * 1024 +_PROMPT_CACHE_MODE_HEADER = "X-Codex-LB-Prompt-Cache-Mode" +_SUBSCRIPTION_IMPLICIT_PROMPT_CACHE_MODE = "subscription-implicit" + + +def _mark_subscription_prompt_cache_fallback(response: Response, payload: ResponsesRequest) -> Response: + if response.status_code < 400 and responses_request_has_explicit_prompt_cache_controls(payload): + response.headers[_PROMPT_CACHE_MODE_HEADER] = _SUBSCRIPTION_IMPLICIT_PROMPT_CACHE_MODE + return response class _V1ResetCreditFreshCredentials: @@ -1054,7 +1063,7 @@ async def responses( service_tier_was_enforced=service_tier_was_enforced, ) - return await _stream_responses( + response = await _stream_responses( request, responses_payload, context, @@ -1069,6 +1078,7 @@ async def responses( enforce_openai_sdk_contract=openai_sdk_request, native_codex_heartbeat=native_codex_heartbeat, ) + return _mark_subscription_prompt_cache_fallback(response, responses_payload) @router.get("/opportunistic/admission") @@ -1181,7 +1191,7 @@ async def v1_responses( service_tier_was_enforced=service_tier_was_enforced, ) if responses_payload.stream: - return await _stream_responses( + response = await _stream_responses( request, responses_payload, context, @@ -1191,16 +1201,18 @@ async def v1_responses( prefer_http_bridge=True, prohibit_fast_mode=prohibit_fast_mode, ) - return await _collect_responses( - request, - responses_payload, - context, - api_key, - codex_session_affinity=False, - openai_cache_affinity=True, - prefer_http_bridge=True, - prohibit_fast_mode=prohibit_fast_mode, - ) + else: + response = await _collect_responses( + request, + responses_payload, + context, + api_key, + codex_session_affinity=False, + openai_cache_affinity=True, + prefer_http_bridge=True, + prohibit_fast_mode=prohibit_fast_mode, + ) + return _mark_subscription_prompt_cache_fallback(response, responses_payload) @internal_router.post( diff --git a/openspec/changes/adapt-subscription-prompt-cache-controls/proposal.md b/openspec/changes/adapt-subscription-prompt-cache-controls/proposal.md new file mode 100644 index 0000000000..0f32cfcc65 --- /dev/null +++ b/openspec/changes/adapt-subscription-prompt-cache-controls/proposal.md @@ -0,0 +1,39 @@ +# Adapt subscription prompt-cache controls + +## Why + +The public OpenAI Responses API supports GPT-5.6 explicit prompt caching through +`prompt_cache_options` and per-content `prompt_cache_breakpoint` markers. The +Codex subscription upstream currently rejects both controls before response +creation. A client using the documented public shape therefore receives a 400 +through codex-lb even though the same request can still use subscription-side +implicit caching and `prompt_cache_key` affinity. + +Model-source requests are different: an OpenAI-compatible API-key source may +support the public controls and must receive them unchanged. The adaptation +therefore belongs at the subscription egress boundary, not in shared request +validation. + +## What Changes + +- Subscription Responses egress omits `prompt_cache_options` and explicit + breakpoint markers while preserving prompt content, order, and + `prompt_cache_key`. +- Successful HTTP responses that used this fallback expose + `X-Codex-LB-Prompt-Cache-Mode: subscription-implicit` so clients do not + mistake the fallback for exact explicit-prefix caching. +- OpenAI-compatible model-source egress preserves the explicit controls. + +## Capabilities + +### Modified Capabilities + +- `responses-api-compat`: documented prompt-cache controls are adapted only for + the subscription upstream and their semantic downgrade is observable. + +## Impact + +- Code: Responses request serialization and HTTP route response metadata. +- Tests: subscription and model-source regressions at `/v1/responses`. +- API/schema: one informational response header; no database or configuration + change. diff --git a/openspec/changes/adapt-subscription-prompt-cache-controls/specs/responses-api-compat/spec.md b/openspec/changes/adapt-subscription-prompt-cache-controls/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..089d58dabb --- /dev/null +++ b/openspec/changes/adapt-subscription-prompt-cache-controls/specs/responses-api-compat/spec.md @@ -0,0 +1,37 @@ +# responses-api-compat Delta Specification + +## ADDED Requirements + +### Requirement: Subscription Responses adapts unsupported explicit prompt-cache controls + +The proxy MUST omit public explicit prompt-cache controls from an HTTP Responses +request routed to the Codex subscription upstream. This applies to +`prompt_cache_options` and `prompt_cache_breakpoint` on a supported prompt +content block. It MUST preserve the prompt content and ordering and MUST +continue forwarding a client-supplied `prompt_cache_key` unchanged. + +A successful HTTP response for such a request MUST include +`X-Codex-LB-Prompt-Cache-Mode: subscription-implicit`, because subscription +implicit caching and account affinity do not provide the exact explicit-prefix +semantics requested by the client. The proxy MUST NOT include that downgrade +header when the request is routed to an OpenAI-compatible model source, and the +model-source wire payload MUST preserve the explicit controls unchanged. + +#### Scenario: Subscription request falls back to implicit caching + +- **GIVEN** a `/v1/responses` request contains a `prompt_cache_key`, + `prompt_cache_options`, and an explicit breakpoint on an `input_text` block +- **WHEN** the request is routed to a subscription account +- **THEN** the upstream subscription payload omits `prompt_cache_options` and + the breakpoint +- **AND** preserves the input text, input order, and `prompt_cache_key` +- **AND** a successful response reports + `X-Codex-LB-Prompt-Cache-Mode: subscription-implicit` + +#### Scenario: Model source preserves public explicit-cache semantics + +- **GIVEN** the same `/v1/responses` request is routed to an OpenAI-compatible + model source +- **THEN** the model-source payload retains `prompt_cache_options`, every + explicit breakpoint, and `prompt_cache_key` +- **AND** the response does not report a subscription implicit fallback diff --git a/openspec/changes/adapt-subscription-prompt-cache-controls/tasks.md b/openspec/changes/adapt-subscription-prompt-cache-controls/tasks.md new file mode 100644 index 0000000000..703af766a2 --- /dev/null +++ b/openspec/changes/adapt-subscription-prompt-cache-controls/tasks.md @@ -0,0 +1,21 @@ +# Tasks: adapt-subscription-prompt-cache-controls + +## 1. Implementation + +- [x] 1.1 Strip public explicit prompt-cache controls only from subscription + Responses egress while preserving `prompt_cache_key` and prompt content +- [x] 1.2 Report successful subscription fallback through + `X-Codex-LB-Prompt-Cache-Mode: subscription-implicit` +- [x] 1.3 Preserve explicit controls on OpenAI-compatible model-source egress + +## 2. Regression coverage + +- [x] 2.1 Exercise the exact `/v1/responses` subscription request shape and + assert upstream serialization plus response header +- [x] 2.2 Exercise the model-source route as a negative control and assert the + controls remain intact + +## 3. Verification + +- [x] 3.1 Run focused unit/integration tests and strict OpenSpec validation +- [ ] 3.2 Re-run the bounded live request against the overlay-preserving stack diff --git a/tests/integration/test_openai_compat_features.py b/tests/integration/test_openai_compat_features.py index 57a2f400e4..c3c44dd615 100644 --- a/tests/integration/test_openai_compat_features.py +++ b/tests/integration/test_openai_compat_features.py @@ -5,7 +5,9 @@ from typing import cast import pytest +from fastapi.responses import JSONResponse +import app.modules.proxy.api as proxy_api_module import app.modules.proxy.service as proxy_module from app.core.openai.requests import ResponsesRequest @@ -246,6 +248,91 @@ async def fake_stream(payload, headers, access_token, account_id, base_url=None, assert "prompt_cache_retention" not in seen["payload"] +@pytest.mark.asyncio +async def test_v1_responses_downgrades_explicit_prompt_cache_for_subscription(async_client, monkeypatch): + await _import_account(async_client, "acc_prompt_cache_explicit", "prompt-cache-explicit@example.com") + + seen = {} + + async def fake_stream(payload, headers, access_token, account_id, base_url=None, raise_for_status=False): + seen["payload"] = payload.to_payload() + yield _completed_event("resp_prompt_cache_explicit") + + monkeypatch.setattr(proxy_module, "core_stream_responses", fake_stream) + + payload = { + "model": "gpt-5.6-sol", + "input": [ + { + "role": "user", + "content": [ + { + "type": "input_text", + "text": "stable prefix", + "prompt_cache_breakpoint": {"mode": "explicit"}, + }, + {"type": "input_text", "text": "changing suffix"}, + ], + } + ], + "prompt_cache_key": "explicit-thread", + "prompt_cache_options": {"mode": "explicit"}, + } + resp = await async_client.post("/v1/responses", json=payload) + + assert resp.status_code == 200 + assert resp.headers["x-codex-lb-prompt-cache-mode"] == "subscription-implicit" + forwarded = seen["payload"] + assert forwarded["prompt_cache_key"] == "explicit-thread" + assert "prompt_cache_options" not in forwarded + assert "prompt_cache_breakpoint" not in forwarded["input"][0]["content"][0] + assert forwarded["input"][0]["content"][0]["text"] == "stable prefix" + + +@pytest.mark.asyncio +async def test_v1_responses_preserves_explicit_prompt_cache_for_model_source(async_client, monkeypatch): + await _import_account(async_client, "acc_prompt_cache_source", "prompt-cache-source@example.com") + + seen = {} + source = object() + + async def fake_select(model, api_key, *, raw_model=None, require_streaming=False): + return source, model + + async def fake_source_response(request, payload, *, source, api_key, rate_limit_headers): + seen["payload"] = payload.model_dump_for_forwarding() + return JSONResponse({"id": "resp_prompt_cache_source", "status": "completed", "output": []}) + + monkeypatch.setattr(proxy_api_module, "_select_responses_model_source", fake_select) + monkeypatch.setattr(proxy_api_module, "_source_responses_response", fake_source_response) + + payload = { + "model": "gpt-5.6-sol", + "input": [ + { + "role": "user", + "content": [ + { + "type": "input_text", + "text": "stable prefix", + "prompt_cache_breakpoint": {"mode": "explicit"}, + } + ], + } + ], + "prompt_cache_key": "source-thread", + "prompt_cache_options": {"mode": "explicit"}, + } + resp = await async_client.post("/v1/responses", json=payload) + + assert resp.status_code == 200 + assert "x-codex-lb-prompt-cache-mode" not in resp.headers + forwarded = seen["payload"] + assert forwarded["prompt_cache_options"] == {"mode": "explicit"} + assert forwarded["input"][0]["content"][0]["prompt_cache_breakpoint"] == {"mode": "explicit"} + assert forwarded["prompt_cache_key"] == "source-thread" + + @pytest.mark.asyncio async def test_v1_responses_normalizes_prompt_cache_aliases(async_client, monkeypatch): await _import_account(async_client, "acc_prompt_cache_alias", "prompt-cache-alias@example.com") diff --git a/tests/unit/test_strict_schema_validation.py b/tests/unit/test_strict_schema_validation.py index 69b61a87da..0c47c01482 100644 --- a/tests/unit/test_strict_schema_validation.py +++ b/tests/unit/test_strict_schema_validation.py @@ -276,6 +276,52 @@ def test_normalize_responses_payload_accepts_valid_strict_schema(): assert request.text.format.strict is True +def test_subscription_serialization_strips_explicit_prompt_cache_controls_only(): + request = normalize_responses_request_payload( + _json_object( + { + "model": "gpt-5.6-sol", + "instructions": "", + "input": [ + { + "role": "user", + "content": [ + { + "type": "input_text", + "text": "stable prefix", + "prompt_cache_breakpoint": {"mode": "explicit"}, + }, + {"type": "input_text", "text": "changing suffix"}, + ], + } + ], + "prompt_cache_key": "stable-key", + "prompt_cache_options": {"mode": "explicit"}, + } + ), + openai_compat=False, + ) + + source_payload = request.model_dump_for_forwarding() + source_input = cast(list[dict[str, JsonValue]], source_payload["input"]) + source_content = cast(list[dict[str, JsonValue]], source_input[0]["content"]) + assert source_payload["prompt_cache_options"] == {"mode": "explicit"} + assert source_content[0]["prompt_cache_breakpoint"] == {"mode": "explicit"} + + subscription_payload = request.to_payload() + subscription_input = cast( + list[dict[str, JsonValue]], subscription_payload["input"] + ) + subscription_content = cast( + list[dict[str, JsonValue]], subscription_input[0]["content"] + ) + assert "prompt_cache_options" not in subscription_payload + assert "prompt_cache_breakpoint" not in subscription_content[0] + assert subscription_content[0]["text"] == "stable prefix" + assert subscription_content[1]["text"] == "changing suffix" + assert subscription_payload["prompt_cache_key"] == "stable-key" + + def test_chat_completions_strict_schema_violation_surfaces_via_enforce_helper(): payload = { "model": "gpt-5.5", From 67d7ee7c5fc90d5678a7e4a284cf1c77cd8c1fe9 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Fri, 31 Jul 2026 16:54:33 +0400 Subject: [PATCH 40/64] fix(proxy): replay full resend after bridge owner conflict --- .../proxy/_service/http_bridge/streaming.py | 8 +- .../integration/test_http_responses_bridge.py | 148 ++++++++++++++++++ 2 files changed, 154 insertions(+), 2 deletions(-) diff --git a/app/modules/proxy/_service/http_bridge/streaming.py b/app/modules/proxy/_service/http_bridge/streaming.py index 1fce2706ea..034e60074b 100644 --- a/app/modules/proxy/_service/http_bridge/streaming.py +++ b/app/modules/proxy/_service/http_bridge/streaming.py @@ -68,7 +68,6 @@ _effective_http_bridge_idle_ttl_seconds, _http_bridge_durable_lookup_allows_turn_state_takeover, _http_bridge_is_context_overflow_error, - _http_bridge_is_previous_response_owner_unavailable, _http_bridge_models_compatible, _http_bridge_owner_lookup_unavailable_error_envelope, _http_bridge_payload_looks_like_full_resend, @@ -233,6 +232,11 @@ def _proxy_error_code_message(exc: ProxyResponseError) -> tuple[str | None, str return (str(code) if code is not None else None, str(message) if message is not None else None) +def _http_bridge_owner_failure_allows_account_neutral_replay(exc: ProxyResponseError) -> bool: + code, _message = _proxy_error_code_message(exc) + return code in {"previous_response_owner_unavailable", "continuity_owner_conflict"} + + def _http_bridge_account_capacity_wait_seconds(exc: ProxyResponseError) -> float | None: code, message = _proxy_error_code_message(exc) if code == "capacity_exhausted_active_sessions": @@ -1190,7 +1194,7 @@ def owner_unavailable_allows_account_neutral_replay(exc: ProxyResponseError) -> nonlocal durable_full_resend_retains_prior_output if ( - not _http_bridge_is_previous_response_owner_unavailable(exc) + not _http_bridge_owner_failure_allows_account_neutral_replay(exc) or forwarded_request or rewritten_file_account_id is not None or durable_full_resend_anchor_count is None diff --git a/tests/integration/test_http_responses_bridge.py b/tests/integration/test_http_responses_bridge.py index 989783c5af..a1c81c97cd 100644 --- a/tests/integration/test_http_responses_bridge.py +++ b/tests/integration/test_http_responses_bridge.py @@ -7595,6 +7595,154 @@ async def fake_connect_responses_websocket( assert owner_miss["preferred_account_is_continuity_owner"] is True +@pytest.mark.asyncio +async def test_v1_responses_http_bridge_replays_full_resend_after_owner_conflict(async_client, monkeypatch): + _install_bridge_settings(monkeypatch, enabled=True) + owner_account_id = await _import_account( + async_client, + "acc_http_bridge_conflict_owner", + "http-bridge-conflict-owner@example.com", + ) + alternate_account_id = await _import_account( + async_client, + "acc_http_bridge_conflict_alternate", + "http-bridge-conflict-alternate@example.com", + ) + owner_account = await _get_account(owner_account_id) + alternate_account = await _get_account(alternate_account_id) + owner_chatgpt_account_id = cast(str, owner_account.chatgpt_account_id) + alternate_chatgpt_account_id = cast(str, alternate_account.chatgpt_account_id) + owner_upstream = _ClosingBridgeUpstreamWebSocket("resp_conflict_owner") + alternate_upstream = _FakeBridgeUpstreamWebSocket("resp_conflict_replay") + selection_calls: list[dict[str, object]] = [] + connected_account_ids: list[str] = [] + connect_headers_by_account: dict[str, dict[str, str]] = {} + + async def fake_select_account_with_budget(self, deadline, **kwargs): + del self, deadline + selection_calls.append(dict(kwargs)) + preferred_account_id = cast(str | None, kwargs.get("preferred_account_id")) + excluded_account_ids = cast(set[str], kwargs.get("exclude_account_ids") or set()) + fallback_enabled = bool(kwargs.get("fallback_on_preferred_account_unavailable", True)) + if preferred_account_id == owner_account.id and not fallback_enabled: + assert kwargs.get("preferred_account_is_continuity_owner") is True + return AccountSelection( + account=None, + error_message="Account-owned continuity sources conflict; retry the logical turn", + error_code="continuity_owner_conflict", + ) + if owner_account.id in excluded_account_ids: + return AccountSelection(account=alternate_account, error_message=None, error_code=None) + return AccountSelection(account=owner_account, error_message=None, error_code=None) + + async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): + del self, force, timeout_seconds + return target + + async def fake_connect_responses_websocket( + headers, + access_token, + account_id_header, + *, + base_url=None, + session=None, + ): + del access_token, base_url, session + connected_account_ids.append(account_id_header) + connect_headers_by_account[account_id_header] = dict(headers) + if account_id_header == owner_chatgpt_account_id: + return owner_upstream + assert account_id_header == alternate_chatgpt_account_id + return alternate_upstream + + monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) + monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) + monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) + + historical_input = [ + { + "role": "user", + "content": [{"type": "input_text", "text": "first question"}], + } + ] + first = await asyncio.wait_for( + async_client.post( + "/v1/responses", + json={ + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": historical_input, + "prompt_cache_key": "http-bridge-conflict-replay", + }, + ), + timeout=_TEST_SYNC_TIMEOUT_SECONDS, + ) + assert first.status_code == 200, first.text + + retained_prior_output = { + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "first answer"}], + } + full_resend = [ + *historical_input, + retained_prior_output, + { + "role": "user", + "content": [{"type": "input_text", "text": "second question"}], + }, + ] + second = await asyncio.wait_for( + async_client.post( + "/v1/responses", + json={ + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": full_resend, + "prompt_cache_key": "http-bridge-conflict-replay", + "previous_response_id": first.json()["id"], + }, + headers={ + "session_id": "stale-session", + "x-codex-session-id": "stale-codex-session", + "x-codex-turn-state": "http_turn_stale", + "x-request-trace": "keep-me", + }, + ), + timeout=_TEST_SYNC_TIMEOUT_SECONDS, + ) + + assert second.status_code == 200, second.text + assert second.json()["id"] == "resp_conflict_replay_1" + assert connected_account_ids == [ + owner_chatgpt_account_id, + alternate_chatgpt_account_id, + ] + replay_connect_headers = { + key.lower(): value for key, value in connect_headers_by_account[alternate_chatgpt_account_id].items() + } + assert replay_connect_headers["x-request-trace"] == "keep-me" + assert ( + not { + "session_id", + "x-codex-session-id", + "x-codex-turn-state", + } + & replay_connect_headers.keys() + ) + replay_payload = json.loads(alternate_upstream.sent_text[0]) + assert "previous_response_id" not in replay_payload + assert replay_payload["input"] == full_resend + owner_conflict = next( + call + for call in selection_calls + if call.get("preferred_account_id") == owner_account.id + and call.get("fallback_on_preferred_account_unavailable") is False + ) + assert owner_conflict["preferred_account_is_continuity_owner"] is True + + @pytest.mark.asyncio async def test_backend_responses_http_bridge_real_selector_recovers_full_resend_without_degrading_pool( async_client, monkeypatch From 028a60bae5458cf853c69749731c74af56b298dc Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sat, 1 Aug 2026 00:56:18 +0400 Subject: [PATCH 41/64] fix(proxy): fork model transition after owner conflict --- .../proxy/_service/http_bridge/streaming.py | 46 +++++++ tests/unit/test_proxy_http_bridge.py | 118 ++++++++++++++++++ 2 files changed, 164 insertions(+) diff --git a/app/modules/proxy/_service/http_bridge/streaming.py b/app/modules/proxy/_service/http_bridge/streaming.py index 034e60074b..f5fc1ee2ea 100644 --- a/app/modules/proxy/_service/http_bridge/streaming.py +++ b/app/modules/proxy/_service/http_bridge/streaming.py @@ -1297,6 +1297,50 @@ def switch_to_account_neutral_replay() -> None: durable_lookup = None file_required_preferred_account = False + def switch_model_transition_to_account_neutral_fork(exc: ProxyResponseError) -> bool: + nonlocal account_neutral_recovery + nonlocal affinity + nonlocal bridge_session_key + nonlocal force_local_recovery_creation + nonlocal incoming_turn_state_header + nonlocal preferred_account_has_continuity_provenance + nonlocal request_state + nonlocal session_creation_headers + nonlocal session_header_fallback_key + + if ( + durable_model_transition_lookup is None + or not _http_bridge_owner_failure_allows_account_neutral_replay(exc) + or request_state.previous_response_id is not None + or rewritten_file_account_id is not None + ): + return False + failed_owner_id = request_state.preferred_account_id + _log_http_bridge_event( + "model_transition_owner_conflict_fork", + bridge_session_key, + account_id=failed_owner_id, + model=effective_payload.model, + detail="outcome=retry_without_previous_model_owner", + cache_key_family=bridge_session_key.affinity_kind, + model_class=_extract_model_class(effective_payload.model) if effective_payload.model else None, + owner_check_applied=True, + ) + if failed_owner_id is not None: + fresh_replay_excluded_account_ids.add(failed_owner_id) + session_creation_headers = without_http_bridge_session_affinity_headers(session_creation_headers) + incoming_turn_state_header = None + session_header_fallback_key = None + affinity = _AffinityPolicy() + replay_kind, replay_key = make_http_bridge_account_neutral_replay_key(uuid4().hex) + bridge_session_key = _HTTPBridgeSessionKey(replay_kind, replay_key, bridge_session_key.api_key_id) + account_neutral_recovery = True + force_local_recovery_creation = True + request_state.preferred_account_id = None + request_state.excluded_account_ids.update(fresh_replay_excluded_account_ids) + preferred_account_has_continuity_provenance = False + return True + if required_continuity_owner_missing: owner_unavailable = ProxyResponseError( 502, @@ -1350,6 +1394,8 @@ def switch_to_account_neutral_replay() -> None: exclude_account_ids=fresh_replay_excluded_account_ids or None, ) except ProxyResponseError as exc: + if switch_model_transition_to_account_neutral_fork(exc): + continue if not owner_unavailable_allows_account_neutral_replay(exc): exc_code, _exc_message = _proxy_error_code_message(exc) if not unanchored_fork_spill_attempted and _http_bridge_unanchored_fork_can_spill_on_cap( diff --git a/tests/unit/test_proxy_http_bridge.py b/tests/unit/test_proxy_http_bridge.py index 00dd232e8e..7ecd6e700e 100644 --- a/tests/unit/test_proxy_http_bridge.py +++ b/tests/unit/test_proxy_http_bridge.py @@ -17997,6 +17997,124 @@ async def fail_first_session_before_output( assert all(call["preferred_account_has_continuity_provenance"] is True for call in creation_calls) +@pytest.mark.asyncio +async def test_stream_via_http_bridge_forks_model_transition_after_owner_conflict( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + payload = proxy_service.ResponsesRequest.model_validate( + { + "model": "gpt-5.6-terra", + "instructions": "hi", + "input": [{"role": "user", "content": "continue on the new model"}], + } + ) + durable_lookup = proxy_service.DurableBridgeLookup( + session_id="durable-model-conflict-parent", + canonical_kind="session_header", + canonical_key="shared-root", + api_key_scope="__anonymous__", + account_id="acc-model-owner", + owner_instance_id=None, + owner_epoch=1, + lease_expires_at=datetime.now(timezone.utc) + timedelta(seconds=60), + state=HttpBridgeSessionState.ACTIVE, + latest_turn_state="http_turn_model_parent", + latest_response_id="resp_model_parent", + model="gpt-5.6-sol", + ) + owner_conflict = ProxyResponseError( + 502, + openai_error( + "continuity_owner_conflict", + "Durable continuity aliases resolve to conflicting upstream owners.", + ), + ) + creation_keys: list[proxy_service._HTTPBridgeSessionKey] = [] + creation_calls: list[dict[str, Any]] = [] + + async def fake_get_or_create( + key: proxy_service._HTTPBridgeSessionKey, + **kwargs: Any, + ) -> proxy_service._HTTPBridgeSession: + creation_keys.append(key) + creation_calls.append(kwargs) + if len(creation_calls) == 1: + raise owner_conflict + session = _make_bridge_session(key=key) + session.account = cast( + Any, + SimpleNamespace(id="acc-model-alternate", status=AccountStatus.ACTIVE), + ) + session.request_model = payload.model + return session + + async def fake_stream_events( + _session: proxy_service._HTTPBridgeSession, + **_kwargs: Any, + ): + yield 'data: {"type":"response.completed"}\n\n' + + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: cast( + Any, + SimpleNamespace( + get=AsyncMock( + return_value=SimpleNamespace( + sticky_threads_enabled=False, + openai_cache_affinity_max_age_seconds=1800, + http_responses_session_bridge_prompt_cache_idle_ttl_seconds=3600, + http_responses_session_bridge_gateway_safe_mode=False, + ) + ) + ), + ), + ) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr(service._durable_bridge, "lookup_request_targets", AsyncMock(return_value=durable_lookup)) + monkeypatch.setattr(service, "_resolve_file_account_for_responses", AsyncMock(return_value=None)) + monkeypatch.setattr(service, "_get_or_create_http_bridge_session", fake_get_or_create) + monkeypatch.setattr(service, "_stream_http_bridge_session_events", fake_stream_events) + + chunks = [ + chunk + async for chunk in service._stream_via_http_bridge( + payload, + headers={ + "x-codex-turn-state": "http_turn_model_parent", + "x-codex-session-id": "shared-root", + }, + codex_session_affinity=True, + propagate_http_errors=True, + openai_cache_affinity=True, + api_key=None, + api_key_reservation=None, + suppress_text_done_events=False, + idle_ttl_seconds=120.0, + codex_idle_ttl_seconds=1800.0, + max_sessions=8, + queue_limit=4, + downstream_turn_state="http_turn_model_child", + ) + ] + + assert chunks == ['data: {"type":"response.completed"}\n\n'] + assert len(creation_calls) == 2 + assert creation_keys[0].affinity_kind in {"session_header", "turn_state_header"} + assert is_http_bridge_account_neutral_replay( + kind=creation_keys[1].affinity_kind, + key=creation_keys[1].affinity_key, + ) + assert creation_calls[0]["preferred_account_id"] == "acc-model-owner" + assert creation_calls[0]["preferred_account_has_continuity_provenance"] is True + assert creation_calls[1]["preferred_account_id"] is None + assert creation_calls[1]["preferred_account_has_continuity_provenance"] is False + assert creation_calls[1]["exclude_account_ids"] == {"acc-model-owner"} + assert creation_calls[1]["allow_forward_to_owner"] is False + + @pytest.mark.asyncio async def test_stream_via_http_bridge_preserves_verified_replay_kind_for_durable_model_transition( monkeypatch: pytest.MonkeyPatch, From 1f2dbfe6113a7261ea2fd4bbf4175c6d6f6662fb Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sat, 1 Aug 2026 10:01:48 +0400 Subject: [PATCH 42/64] fix(proxy): prefer safe full-resend projection --- app/modules/proxy/_service/http_bridge/streaming.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/modules/proxy/_service/http_bridge/streaming.py b/app/modules/proxy/_service/http_bridge/streaming.py index f5fc1ee2ea..f9c1be3842 100644 --- a/app/modules/proxy/_service/http_bridge/streaming.py +++ b/app/modules/proxy/_service/http_bridge/streaming.py @@ -1311,6 +1311,7 @@ def switch_model_transition_to_account_neutral_fork(exc: ProxyResponseError) -> if ( durable_model_transition_lookup is None or not _http_bridge_owner_failure_allows_account_neutral_replay(exc) + or owner_unavailable_allows_account_neutral_replay(exc) or request_state.previous_response_id is not None or rewritten_file_account_id is not None ): From c825b235b246c805df26950bcbf56b96a6923af6 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sun, 2 Aug 2026 01:42:47 +0400 Subject: [PATCH 43/64] fix(db): merge live bridge and capability lineage heads --- ...rge_bridge_and_capability_lineage_heads.py | 24 +++++++++++++++++++ tests/unit/test_db_migrate.py | 7 +++++- 2 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 app/db/alembic/versions/20260802_000000_merge_bridge_and_capability_lineage_heads.py diff --git a/app/db/alembic/versions/20260802_000000_merge_bridge_and_capability_lineage_heads.py b/app/db/alembic/versions/20260802_000000_merge_bridge_and_capability_lineage_heads.py new file mode 100644 index 0000000000..5192643059 --- /dev/null +++ b/app/db/alembic/versions/20260802_000000_merge_bridge_and_capability_lineage_heads.py @@ -0,0 +1,24 @@ +"""Merge live bridge and capability lineage migration heads. + +Revision ID: 20260802_000000_merge_bridge_and_capability_lineage_heads +Revises: 20260729_000000_drop_legacy_bridge_pending_tool_columns, 20260731_000000_add_capability_lineage_markers +Create Date: 2026-08-02 +""" + +from __future__ import annotations + +revision = "20260802_000000_merge_bridge_and_capability_lineage_heads" +down_revision = ( + "20260729_000000_drop_legacy_bridge_pending_tool_columns", + "20260731_000000_add_capability_lineage_markers", +) +branch_labels = None +depends_on = None + + +def upgrade() -> None: + pass + + +def downgrade() -> None: + pass diff --git a/tests/unit/test_db_migrate.py b/tests/unit/test_db_migrate.py index c5a01ee27c..e95b7de2bc 100644 --- a/tests/unit/test_db_migrate.py +++ b/tests/unit/test_db_migrate.py @@ -2038,11 +2038,16 @@ def test_capability_lineage_migration_is_additive_reversible_and_single_head(tmp url = _db_url(db_path) parent_revision = "20260725_000000_add_http_bridge_pending_tool_calls" target_revision = "20260731_000000_add_capability_lineage_markers" + merge_revision = "20260802_000000_merge_bridge_and_capability_lineage_heads" run_upgrade(url, parent_revision, bootstrap_legacy=False) config = _build_alembic_config(url) script_directory = ScriptDirectory.from_config(config) - assert script_directory.get_heads() == [target_revision] + assert script_directory.get_heads() == [merge_revision] + assert script_directory.get_revision(merge_revision).down_revision == ( + "20260729_000000_drop_legacy_bridge_pending_tool_columns", + target_revision, + ) engine = create_engine(to_sync_database_url(url)) try: From 0346c6b222ec8a45f7b24dba9bce7da2be5f6270 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Mon, 3 Aug 2026 12:57:16 +0400 Subject: [PATCH 44/64] fix(proxy): penalize eventless bridge accounts --- .../_service/http_bridge/upstream_events.py | 6 +---- .../design.md | 4 +-- .../proposal.md | 4 +-- .../specs/proxy-admission-control/spec.md | 6 ++--- .../tasks.md | 4 +-- tests/unit/test_proxy_http_bridge.py | 25 +++++++++++++++---- 6 files changed, 30 insertions(+), 19 deletions(-) diff --git a/app/modules/proxy/_service/http_bridge/upstream_events.py b/app/modules/proxy/_service/http_bridge/upstream_events.py index 538d80b396..f09b8654b7 100644 --- a/app/modules/proxy/_service/http_bridge/upstream_events.py +++ b/app/modules/proxy/_service/http_bridge/upstream_events.py @@ -670,11 +670,7 @@ async def _relay_http_bridge_upstream_messages( session, error_code="upstream_request_timeout", error_message=receive_timeout.error_message, - # A silent bridge is a session transport failure. The - # request-local retry already excludes this account; - # poisoning the shared routing cache can exhaust an - # otherwise healthy pool. - penalize_account=False, + penalize_account=True, retire_detail=_HTTP_BRIDGE_MISSING_RESPONSE_CREATED_TIMEOUT_DETAIL, force_retire=True, ) diff --git a/openspec/changes/recover-codex-desktop-idle-bridge/design.md b/openspec/changes/recover-codex-desktop-idle-bridge/design.md index 45f132d300..b4202a5366 100644 --- a/openspec/changes/recover-codex-desktop-idle-bridge/design.md +++ b/openspec/changes/recover-codex-desktop-idle-bridge/design.md @@ -53,7 +53,7 @@ Leading non-response telemetry such as `codex.rate_limits` does not change those When the deadline expires, reuse the reader-owned terminal failure and whole-session retirement path. Emit a stable `missing_response_created_timeout` detail, increment the existing stuck-retirement metric, settle every pending request exactly once, and close the bridge session. -Do not transparently replay the timed-out request, submit it on another account, or mark the selected account unhealthy. Upstream acceptance is unknown, so duplicate submission and account movement are less safe than an explicit terminal failure. A later client request creates a fresh session through existing behavior. +Do not transparently replay the timed-out request or submit that same request on another account. Upstream acceptance is unknown, so duplicate submission is less safe than an explicit terminal failure. Record a transient health failure for the selected account so a later client request creates a fresh session through existing selection and can avoid repeating an account that accepted `response.create` but emitted no `response.created`. Example: a request sends at monotonic time 1,000 with the default 300-second stuck threshold. With no matched response lifecycle event, it becomes eligible at 1,240 and receives an explicit terminal failure; it does not wait for a second request or the 300-second Desktop idle timeout. @@ -66,7 +66,7 @@ Explicit `x-stainless-*` headers or an OpenAI User-Agent retain comment liveness ## Risks / Trade-offs - **A send fails after the timestamp is set.** Existing send-error cleanup retires or settles the request before the watchdog can act; tests cover that the timestamp alone is not sufficient eligibility. -- **A quiet upstream accepted the request but emitted no event.** The proxy returns an explicit failure rather than risking a duplicate replay. The selected account remains healthy because silence is not proof of account failure. +- **A quiet upstream accepted the request but emitted no event.** The proxy returns an explicit failure rather than risking a duplicate replay. The selected account receives a transient health failure because repeated missing-created timeouts on the same account are a live availability fault. - **A matched lifecycle event arrives just before timeout.** Eligibility is rechecked under the existing request/session synchronization before retirement, and any matched `response.*` event suppresses this watchdog. - **Whole-session retirement interrupts a healthy sibling.** This narrow design chooses fail-closed session cleanup rather than attempting unsafe sibling isolation on current `main`. Existing terminal settlement must cover every pending sibling exactly once. - **A client spoofs native identity.** The only benefit is an ignored vendor liveness event on the authenticated Codex backend route; explicit SDK markers still take precedence. diff --git a/openspec/changes/recover-codex-desktop-idle-bridge/proposal.md b/openspec/changes/recover-codex-desktop-idle-bridge/proposal.md index d9849d4af5..d17fe39d34 100644 --- a/openspec/changes/recover-codex-desktop-idle-bridge/proposal.md +++ b/openspec/changes/recover-codex-desktop-idle-bridge/proposal.md @@ -6,9 +6,9 @@ A production Codex Desktop request on the HTTP-to-WebSocket bridge remained pend - Record the monotonic time of the current upstream `response.create` send. - Proactively expire an eventless request that remains pre-`response.created` for the smaller of the existing stuck-gate threshold and 240 seconds, even when no second gate waiter exists and periodic keepalives are disabled. -- Fail the affected bridge session closed through existing terminal settlement and retirement paths, without transparent replay, account movement, or account-health penalties. +- Fail the affected bridge session closed through existing terminal settlement and retirement paths, without transparent replay or moving the timed-out request to another account, while recording a transient account-health failure so later requests can avoid the eventless account. - Give verified native Codex identity parser-visible `codex.keepalive` frames even when payload-shape heuristics still require OpenAI-compatible event normalization; explicit SDK markers and public `/v1/responses` retain comment liveness. -- Add regressions for the no-waiter deadline, protected created/eventful requests, account-neutral retirement, and contrasting Desktop/SDK/public heartbeat contracts. +- Add regressions for the no-waiter deadline, protected created/eventful requests, health-accounted retirement without replay, and contrasting Desktop/SDK/public heartbeat contracts. ## Capabilities diff --git a/openspec/changes/recover-codex-desktop-idle-bridge/specs/proxy-admission-control/spec.md b/openspec/changes/recover-codex-desktop-idle-bridge/specs/proxy-admission-control/spec.md index 51bbccd853..68022bc234 100644 --- a/openspec/changes/recover-codex-desktop-idle-bridge/specs/proxy-admission-control/spec.md +++ b/openspec/changes/recover-codex-desktop-idle-bridge/specs/proxy-admission-control/spec.md @@ -6,7 +6,7 @@ The proxy MUST retain the existing waiter-triggered retirement behavior for stal The owner-side watchdog MUST apply only while the request owns the response-create gate, awaits `response.created`, has neither a response id nor recorded `response.created` latency, has received no matched `response.*` lifecycle event, and has produced no downstream-visible output or sequence evidence. Non-response telemetry such as `codex.rate_limits` MUST NOT suppress this watchdog. Any matched `response.*` lifecycle event, response-created milestone, or downstream-visible evidence MUST suppress the owner-side watchdog and leave existing timeout behavior unchanged. -When the owner-side deadline expires, the proxy MUST recheck eligibility, emit a structured low-cardinality log and the existing stuck-retirement Prometheus counter, terminally fail and settle every pending request exactly once, and retire the whole bridge session. It MUST NOT transparently replay the timed-out request, move it to another account, or write an account-health failure for the missing-created timeout. +When the owner-side deadline expires, the proxy MUST recheck eligibility, emit a structured low-cardinality log and the existing stuck-retirement Prometheus counter, terminally fail and settle every pending request exactly once, write the selected account's transient health failure, and retire the whole bridge session. It MUST NOT transparently replay the timed-out request or move that timed-out request to another account. #### Scenario: Lone eventless gate owner is retired before the client timeout @@ -38,10 +38,10 @@ When the owner-side deadline expires, the proxy MUST recheck eligibility, emit a - **THEN** this watchdog does not retire the session - **AND** existing stream, request-budget, and waiter-triggered timeout behavior remains authoritative -#### Scenario: Timeout is fail-closed and account-neutral +#### Scenario: Timeout is fail-closed and health-accounted - **GIVEN** an eventless pre-created owner reaches the owner-side deadline - **WHEN** terminal cleanup runs - **THEN** every pending request is settled exactly once and the whole session is retired - **AND** the proxy does not replay the timed-out request or submit it on another account -- **AND** the selected account is not marked unhealthy solely because `response.created` was missing +- **AND** the selected account records a transient health failure so later requests can avoid repeating the eventless account diff --git a/openspec/changes/recover-codex-desktop-idle-bridge/tasks.md b/openspec/changes/recover-codex-desktop-idle-bridge/tasks.md index 1c356ad8d4..32cc5e81e6 100644 --- a/openspec/changes/recover-codex-desktop-idle-bridge/tasks.md +++ b/openspec/changes/recover-codex-desktop-idle-bridge/tasks.md @@ -3,8 +3,8 @@ - [x] 1.1 Record the current monotonic `response.create` send timestamp in HTTP bridge request state and replace it on every real send. - [x] 1.2 Add a pure client-safe deadline helper that uses the smaller of the existing stuck-gate threshold and 240 seconds. - [x] 1.3 Enforce the deadline from the upstream reader without requiring a second gate waiter or SSE keepalives; recheck narrow eventless eligibility before acting. -- [x] 1.4 Fail and retire the whole bridge session through existing settlement, logging, and Prometheus paths without replay, account movement, or account-health writes. -- [x] 1.5 Add focused regressions for no-waiter expiry, send-time anchoring, leading telemetry, created/eventful/downstream protection, terminal settlement, and account neutrality. +- [x] 1.4 Fail and retire the whole bridge session through existing settlement, logging, Prometheus, and transient account-health paths without replaying or moving the timed-out request. +- [x] 1.5 Add focused regressions for no-waiter expiry, send-time anchoring, leading telemetry, created/eventful/downstream protection, terminal settlement, and health-accounted retirement. ## 2. Native Codex SSE liveness diff --git a/tests/unit/test_proxy_http_bridge.py b/tests/unit/test_proxy_http_bridge.py index c3e86628bd..e386f30014 100644 --- a/tests/unit/test_proxy_http_bridge.py +++ b/tests/unit/test_proxy_http_bridge.py @@ -32,7 +32,7 @@ from app.core.config.settings import Settings from app.core.errors import openai_error from app.core.utils.request_id import get_request_id, reset_request_scope_id, set_request_scope_id -from app.db.models import AccountStatus, HttpBridgeSessionState +from app.db.models import Account, AccountStatus, HttpBridgeSessionState from app.modules.proxy import http_bridge_forwarding as http_bridge_forwarding_module from app.modules.proxy import service as proxy_service from app.modules.proxy._service import support as proxy_support_module @@ -18567,6 +18567,16 @@ async def close(self) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) upstream = _TrackingUpstream() session = _make_bridge_session(key_value=f"eventless-{leading_telemetry}") + session.account = Account( + id="acc-bridge", + email="bridge@example.com", + plan_type="plus", + access_token_encrypted=b"", + refresh_token_encrypted=b"", + id_token_encrypted=b"", + last_refresh=datetime.now(timezone.utc), + status=AccountStatus.ACTIVE, + ) session.upstream = cast(UpstreamWebSocket, upstream) service._http_bridge_sessions[session.key] = session settings = _make_app_settings( @@ -18578,7 +18588,8 @@ async def close(self) -> None: monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) retry_precreated = AsyncMock(return_value=False) monkeypatch.setattr(service, "_retry_http_bridge_precreated_request", retry_precreated) - monkeypatch.setattr(service, "_handle_stream_error", AsyncMock()) + handle_stream_error = AsyncMock() + monkeypatch.setattr(service, "_handle_stream_error", handle_stream_error) write_request_log = AsyncMock() monkeypatch.setattr(service, "_write_request_log", write_request_log) record_stuck_retire = Mock() @@ -18659,8 +18670,13 @@ async def close(self) -> None: assert write_request_log.await_count == 2 assert {call.kwargs["error_code"] for call in write_request_log.await_args_list} == {"upstream_request_timeout"} fail_reader.assert_awaited_once() - assert fail_reader.await_args.kwargs["penalize_account"] is False + assert fail_reader.await_args.kwargs["penalize_account"] is True assert fail_reader.await_args.kwargs["force_retire"] is True + handle_stream_error.assert_awaited_once() + handle_stream_error_args = handle_stream_error.await_args + assert handle_stream_error_args is not None + assert handle_stream_error_args.args[0] is session.account + assert handle_stream_error_args.args[2] == "upstream_request_timeout" record_stuck_retire.assert_called_once_with( reason="missing_response_created_timeout", session=session, @@ -19409,7 +19425,6 @@ async def test_http_bridge_eventless_timeout_force_retires_with_admission_waiter session, error_code="upstream_request_timeout", error_message="missing response.created", - penalize_account=False, retire_detail="missing_response_created_timeout", force_retire=True, ) @@ -19419,7 +19434,7 @@ async def test_http_bridge_eventless_timeout_force_retires_with_admission_waiter retire.assert_awaited_once_with(session, detail="missing_response_created_timeout") fail_pending_await_args = fail_pending.await_args assert fail_pending_await_args is not None - assert fail_pending_await_args.kwargs["penalize_account"] is False + assert fail_pending_await_args.kwargs["penalize_account"] is True @pytest.mark.asyncio From ecc281960393c1be719dcfb07236c0d19744f3ef Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sun, 2 Aug 2026 01:46:24 +0400 Subject: [PATCH 45/64] chore(tests): format prompt cache coverage --- tests/unit/test_strict_schema_validation.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/unit/test_strict_schema_validation.py b/tests/unit/test_strict_schema_validation.py index 0c47c01482..3b2f22cc0d 100644 --- a/tests/unit/test_strict_schema_validation.py +++ b/tests/unit/test_strict_schema_validation.py @@ -309,12 +309,8 @@ def test_subscription_serialization_strips_explicit_prompt_cache_controls_only() assert source_content[0]["prompt_cache_breakpoint"] == {"mode": "explicit"} subscription_payload = request.to_payload() - subscription_input = cast( - list[dict[str, JsonValue]], subscription_payload["input"] - ) - subscription_content = cast( - list[dict[str, JsonValue]], subscription_input[0]["content"] - ) + subscription_input = cast(list[dict[str, JsonValue]], subscription_payload["input"]) + subscription_content = cast(list[dict[str, JsonValue]], subscription_input[0]["content"]) assert "prompt_cache_options" not in subscription_payload assert "prompt_cache_breakpoint" not in subscription_content[0] assert subscription_content[0]["text"] == "stable prefix" From eede90273a00bdaf165a5935e969a7adb21ec4cc Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sun, 2 Aug 2026 03:47:58 +0400 Subject: [PATCH 46/64] fix(proxy-responses): tolerate structured content type values --- app/core/openai/requests.py | 10 ++++++++-- tests/unit/test_proxy_api_key_usage.py | 27 ++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/app/core/openai/requests.py b/app/core/openai/requests.py index f3a4e7e8d7..aaa126251e 100644 --- a/app/core/openai/requests.py +++ b/app/core/openai/requests.py @@ -828,7 +828,12 @@ def _contains_explicit_prompt_cache_breakpoint(value: JsonValue) -> bool: return any(_contains_explicit_prompt_cache_breakpoint(item) for item in value) if not isinstance(value, dict): return False - if value.get("type") in _EXPLICIT_PROMPT_CACHE_CONTENT_TYPES and "prompt_cache_breakpoint" in value: + value_type = value.get("type") + if ( + isinstance(value_type, str) + and value_type in _EXPLICIT_PROMPT_CACHE_CONTENT_TYPES + and "prompt_cache_breakpoint" in value + ): return True return any(_contains_explicit_prompt_cache_breakpoint(child) for child in value.values()) @@ -851,7 +856,8 @@ def _strip_subscription_prompt_cache_breakpoints(value: JsonValue | None) -> Non return if not isinstance(value, dict): return - if value.get("type") in _EXPLICIT_PROMPT_CACHE_CONTENT_TYPES: + value_type = value.get("type") + if isinstance(value_type, str) and value_type in _EXPLICIT_PROMPT_CACHE_CONTENT_TYPES: value.pop("prompt_cache_breakpoint", None) for child in value.values(): _strip_subscription_prompt_cache_breakpoints(child) diff --git a/tests/unit/test_proxy_api_key_usage.py b/tests/unit/test_proxy_api_key_usage.py index bf978ffd93..627a4764be 100644 --- a/tests/unit/test_proxy_api_key_usage.py +++ b/tests/unit/test_proxy_api_key_usage.py @@ -93,3 +93,30 @@ def test_estimate_api_key_request_usage_uses_conservative_input_for_file_referen budget = estimate_api_key_request_usage(payload) assert budget.input_tokens is None + + +def test_estimate_api_key_request_usage_allows_structured_content_type_values() -> None: + payload = ResponsesRequest.model_validate( + { + "model": "gpt-5.5", + "instructions": "continue", + "input": [ + { + "role": "assistant", + "content": [ + { + "type": { + "namespace": "multi_agent_v1", + "name": "tool_search_output", + }, + "text": "deferred tool metadata", + } + ], + } + ], + } + ) + + budget = estimate_api_key_request_usage(payload) + + assert budget.input_tokens is not None From 06b6993ca90bfd13a3c0a6405705af770b8095b2 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sun, 2 Aug 2026 04:16:25 +0400 Subject: [PATCH 47/64] fix(proxy): suppress duplicate side-effect tool calls --- app/modules/proxy/tool_call_dedupe.py | 29 +++++++++++ .../test_proxy_websocket_responses.py | 15 +++--- tests/unit/test_proxy_tool_call_dedupe.py | 49 ++++++++++++++++++- tests/unit/test_proxy_utils.py | 18 ++++--- 4 files changed, 94 insertions(+), 17 deletions(-) diff --git a/app/modules/proxy/tool_call_dedupe.py b/app/modules/proxy/tool_call_dedupe.py index a5d2cd7625..730a46b378 100644 --- a/app/modules/proxy/tool_call_dedupe.py +++ b/app/modules/proxy/tool_call_dedupe.py @@ -137,6 +137,14 @@ def mark_duplicate_tool_call_downstream_event( None, argument_key, ) + if item_name is not None and same_response_argument_key in seen_tool_call_keys: + logger.warning( + "Suppressed duplicate downstream side-effect tool call response_id=%s item_type=%s name=%s", + response_id, + item_type, + item_name, + ) + return True code_mode_call = item_name in tool_call_safety.CODE_MODE_DOWNSTREAM_SIDE_EFFECT_TOOL_CALL_NAMES identity_scoped_call = code_mode_call or item_namespace is not None cross_response_call_id = call_id if identity_scoped_call else None @@ -162,6 +170,11 @@ def mark_duplicate_tool_call_downstream_event( item_name, ) return True + _clear_downstream_side_effect_burst_keys( + seen_tool_call_keys, + current_key=key, + current_argument_key=same_response_argument_key, + ) seen_tool_call_keys[key] = None if is_side_effect_tool_call: seen_tool_call_keys[same_response_argument_key] = None @@ -180,6 +193,22 @@ def _clear_legacy_downstream_tool_call_keys(seen_tool_call_keys: dict[ToolCallDe seen_tool_call_keys.pop(key, None) +def _clear_downstream_side_effect_burst_keys( + seen_tool_call_keys: dict[ToolCallDedupeKey, None], + *, + current_key: ToolCallDedupeKey, + current_argument_key: ToolCallDedupeKey, +) -> None: + for key in tuple(seen_tool_call_keys): + if key == current_key or key == current_argument_key: + continue + response_id, _, namespace, _, call_id, _ = key + if response_id == "": + continue + if call_id is None: + seen_tool_call_keys.pop(key, None) + + def _mark_duplicate_parallel_tool_call_downstream_event( item: dict[str, JsonValue], argument_value: str, diff --git a/tests/integration/test_proxy_websocket_responses.py b/tests/integration/test_proxy_websocket_responses.py index 1918dbaa19..971759912b 100644 --- a/tests/integration/test_proxy_websocket_responses.py +++ b/tests/integration/test_proxy_websocket_responses.py @@ -2407,7 +2407,7 @@ async def fake_write_request_log(self, **kwargs): assert cast(dict[str, object], visible_reference_payload["client_metadata"])[marker] == "true" -def test_backend_responses_websocket_keeps_same_response_distinct_tool_call_ids( +def test_backend_responses_websocket_suppresses_same_response_duplicate_side_effect_call_ids( app_instance, monkeypatch, ): @@ -2548,18 +2548,17 @@ async def fake_write_request_log(self, **kwargs): websocket.send_text(json.dumps(request_payload)) created_event = json.loads(websocket.receive_text()) tool_event = json.loads(websocket.receive_text()) - replay_tool_event = json.loads(websocket.receive_text()) - terminal_event = json.loads(websocket.receive_text()) + failed_event = json.loads(websocket.receive_text()) assert created_event["type"] == "response.created" assert tool_event["type"] == "response.output_item.done" assert tool_event["item"]["call_id"] == "call_first" - assert replay_tool_event["type"] == "response.output_item.done" - assert replay_tool_event["item"]["call_id"] == "call_replay" - assert terminal_event["type"] == "response.completed" - assert terminal_event["response"]["id"] == "resp_ws_duplicate_tool" + assert failed_event["type"] == "response.failed" + assert failed_event["response"]["id"] == "resp_ws_duplicate_tool" + assert failed_event["response"]["error"]["code"] == "stream_incomplete" assert len(log_calls) == 1 - assert log_calls[0]["status"] == "success" + assert log_calls[0]["status"] == "error" + assert log_calls[0]["error_code"] == "stream_incomplete" def test_backend_responses_websocket_preserves_image_generation_tool_advertisement(app_instance, monkeypatch): diff --git a/tests/unit/test_proxy_tool_call_dedupe.py b/tests/unit/test_proxy_tool_call_dedupe.py index e872dbdeb9..a8c78edd6a 100644 --- a/tests/unit/test_proxy_tool_call_dedupe.py +++ b/tests/unit/test_proxy_tool_call_dedupe.py @@ -20,7 +20,7 @@ def _loads_item_arguments(item: Mapping[str, JsonValue]) -> Any: return json.loads(arguments) -def test_mark_duplicate_tool_call_downstream_event_keeps_distinct_call_ids_with_same_arguments(): +def test_mark_duplicate_tool_call_downstream_event_suppresses_same_response_side_effect_with_new_call_id(): upstream_control = proxy_service._WebSocketUpstreamControl() first_payload: dict[str, JsonValue] = { "type": "response.output_item.done", @@ -67,7 +67,7 @@ def test_mark_duplicate_tool_call_downstream_event_keeps_distinct_call_ids_with_ seen_tool_call_keys=upstream_control.seen_tool_call_keys, response_id="resp_dupe", ) - is False + is True ) assert ( tool_call_dedupe.mark_duplicate_tool_call_downstream_event( @@ -120,6 +120,51 @@ def test_mark_duplicate_tool_call_downstream_event_suppresses_exec_command_with_ ) +def test_mark_duplicate_tool_call_downstream_event_suppresses_exec_command_same_response_new_call_id(): + upstream_control = proxy_service._WebSocketUpstreamControl() + command = ( + "psql -X -d kom -Atc \"select 'global', total_chunks, " + 'chunks_with_any_embedding from rag_embedding_global_stats;"' + ) + + def payload(call_id: str, *, max_output_tokens: int) -> dict[str, JsonValue]: + return { + "type": "response.output_item.done", + "response_id": "resp_live_duplicate", + "item": { + "type": "function_call", + "name": "exec_command", + "arguments": json.dumps( + { + "cmd": command, + "workdir": "/home/kom/Dropbox/Reemxy/tasks-loop", + "yield_time_ms": 10000, + "max_output_tokens": max_output_tokens, + }, + separators=(",", ":"), + ), + "call_id": call_id, + }, + } + + assert ( + tool_call_dedupe.mark_duplicate_tool_call_downstream_event( + payload("call_first", max_output_tokens=12000), + seen_tool_call_keys=upstream_control.seen_tool_call_keys, + response_id="resp_live_duplicate", + ) + is False + ) + assert ( + tool_call_dedupe.mark_duplicate_tool_call_downstream_event( + payload("call_second", max_output_tokens=48000), + seen_tool_call_keys=upstream_control.seen_tool_call_keys, + response_id="resp_live_duplicate", + ) + is True + ) + + def test_mark_duplicate_tool_call_downstream_event_suppresses_code_mode_exec_replay(): upstream_control = proxy_service._WebSocketUpstreamControl() first_payload: dict[str, JsonValue] = { diff --git a/tests/unit/test_proxy_utils.py b/tests/unit/test_proxy_utils.py index f5d77f1a88..530e2853c7 100644 --- a/tests/unit/test_proxy_utils.py +++ b/tests/unit/test_proxy_utils.py @@ -14340,7 +14340,7 @@ async def fake_stream(*_, **__): @pytest.mark.asyncio -async def test_stream_responses_keeps_same_response_http_tool_calls_with_distinct_call_ids(monkeypatch): +async def test_stream_responses_suppresses_same_response_http_tool_calls_with_distinct_call_ids(monkeypatch): settings = _make_proxy_settings() request_logs = _RequestLogsRecorder() service = proxy_service.ProxyService(_repo_factory(request_logs)) @@ -14391,12 +14391,16 @@ async def fake_stream(*_, **__): if isinstance(chunk_payload, dict) and chunk_payload.get("type") == "response.output_item.done": tool_chunks.append(chunk_payload) - assert tool_chunks == [tool_payload, replayed_tool_payload] + assert tool_chunks == [tool_payload] terminal_payload = parse_sse_data_json(chunks[-1]) assert isinstance(terminal_payload, dict) - assert terminal_payload["type"] == "response.completed" + assert terminal_payload["type"] == "response.failed" + terminal_response = cast(dict[str, JsonValue], terminal_payload["response"]) + terminal_error = cast(dict[str, JsonValue], terminal_response["error"]) + assert terminal_error["code"] == "stream_incomplete" assert await service.drain_persistence_tasks(timeout_seconds=1) - assert request_logs.calls[0]["status"] == "success" + assert request_logs.calls[0]["status"] == "error" + assert request_logs.calls[0]["error_code"] == "stream_incomplete" @pytest.mark.asyncio @@ -28331,7 +28335,7 @@ async def test_process_upstream_websocket_text_masks_previous_response_not_found @pytest.mark.asyncio -async def test_process_upstream_websocket_text_keeps_same_response_distinct_tool_call_ids(monkeypatch): +async def test_process_upstream_websocket_text_suppresses_same_response_distinct_tool_call_ids(monkeypatch): request_logs = _RequestLogsRecorder() service = proxy_service.ProxyService(_repo_factory(request_logs)) finalize_request_state = AsyncMock() @@ -28393,8 +28397,8 @@ async def test_process_upstream_websocket_text_keeps_same_response_distinct_tool assert '"call_id":"call_first"' in first_text assert '"call_id":"call_replayed"' in replay_text - assert replay_control.suppress_downstream_event is False - assert pending_request.suppressed_duplicate_tool_call is False + assert replay_control.suppress_downstream_event is True + assert pending_request.suppressed_duplicate_tool_call is True finalize_request_state.assert_not_awaited() assert list(pending_requests) == [pending_request] From 7dfe8110863261087a364b4e270a98fc9c09802d Mon Sep 17 00:00:00 2001 From: choi138 Date: Mon, 3 Aug 2026 21:11:19 +0900 Subject: [PATCH 48/64] fix(proxy): support bounded fresh developer suffixes --- app/modules/proxy/replay_safety.py | 84 ++- .../proposal.md | 50 +- .../specs/responses-api-compat/spec.md | 62 +- .../tasks.md | 13 +- .../integration/test_http_responses_bridge.py | 88 ++- tests/unit/test_proxy_http_bridge.py | 54 ++ tests/unit/test_replay_safety.py | 551 ++++++++++++++++++ 7 files changed, 839 insertions(+), 63 deletions(-) diff --git a/app/modules/proxy/replay_safety.py b/app/modules/proxy/replay_safety.py index 40d2d6d85f..471cb5a9a1 100644 --- a/app/modules/proxy/replay_safety.py +++ b/app/modules/proxy/replay_safety.py @@ -280,11 +280,17 @@ def responses_input_suffix_retains_prior_output( return False pending_suffix_calls, seen_suffix_call_ids = prefix_state retained_output_seen = False + retained_output_is_final_answer = False fresh_followup_seen = False + fresh_followup_count = 0 + fresh_followup_is_user_message = False + fresh_developer_followup_seen = False for item in input_items[stored_count:]: - if not isinstance(item, dict): + if fresh_developer_followup_seen or not isinstance(item, dict): return False item_type_value = item.get("type") + if "type" in item and not _is_nonblank_string(item_type_value): + return False item_type = item_type_value if isinstance(item_type_value, str) else None if item_type in _TOOL_CALL_TYPES: if item.get("status") not in (None, "completed"): @@ -298,7 +304,10 @@ def responses_input_suffix_retains_prior_output( # prove that an omitted parallel call was not part of the response. # Require a later completed assistant message as the turn boundary. retained_output_seen = False + retained_output_is_final_answer = False fresh_followup_seen = False + fresh_followup_count = 0 + fresh_followup_is_user_message = False continue call_type = _TOOL_CALL_TYPE_BY_OUTPUT_TYPE.get(item_type or "") if call_type is not None: @@ -315,12 +324,28 @@ def responses_input_suffix_retains_prior_output( if pending_suffix_calls or not _is_retained_response_message(item): return False retained_output_seen = True + retained_output_is_final_answer = item.get("phase") == "final_answer" fresh_followup_seen = False + fresh_followup_count = 0 + fresh_followup_is_user_message = False continue if _is_fresh_followup_input(item): if not retained_output_seen or pending_suffix_calls: return False fresh_followup_seen = True + fresh_followup_count += 1 + fresh_followup_is_user_message = item_type in (None, "message") and item.get("role") == "user" + continue + if _fresh_developer_message_is_transparent(item): + if ( + not fresh_followup_seen + or fresh_followup_count != 1 + or not fresh_followup_is_user_message + or not retained_output_is_final_answer + or pending_suffix_calls + ): + return False + fresh_developer_followup_seen = True continue return False return retained_output_seen and fresh_followup_seen and not pending_suffix_calls @@ -343,6 +368,13 @@ def responses_input_suffix_matches_pending_tool_calls( if prefix_state is None or prefix_state[0] or prefix_state[1] & pending_tool_calls.keys(): return False suffix = input_items[stored_count:] + if ( + len(suffix) == 3 + and isinstance(suffix[1], dict) + and _fresh_developer_message_is_transparent(suffix[1]) + and _fresh_developer_interleave_is_bounded(suffix, index=1) + ): + suffix = [suffix[0], suffix[2]] if not all( isinstance(item, dict) and isinstance(item.get("type"), str) @@ -437,6 +469,56 @@ def _historical_pending_developer_message_is_transparent( ) +def _fresh_developer_interleave_is_bounded( + input_items: list[JsonValue], + *, + index: int, +) -> bool: + if len(input_items) != 3 or index != 1: + return False + preceding_item = input_items[0] + following_item = input_items[2] + if not isinstance(preceding_item, dict) or not isinstance(following_item, dict): + return False + call_type = preceding_item.get("type") + output_type = following_item.get("type") + call_id = preceding_item.get("call_id") + return ( + call_type == "custom_tool_call" + and output_type == "custom_tool_call_output" + and _is_nonblank_string(call_id) + and following_item.get("call_id") == call_id + ) + + +def _fresh_developer_message_is_transparent( + item: Mapping[str, JsonValue], +) -> bool: + item_type_value = item.get("type") + item_type = item_type_value if isinstance(item_type_value, str) else None + metadata = item.get(_INTERNAL_CHAT_MESSAGE_METADATA_FIELD) + content = item.get("content") + return ( + ("type" not in item or _is_nonblank_string(item.get("type"))) + and item_type in (None, "message") + and item.get("role") == "developer" + and item.get("id") in (None, "") + and item.get("phase") is None + and item.get("status") in (None, "completed") + and isinstance(metadata, dict) + and _internal_chat_message_metadata_is_account_neutral(metadata) + and _input_item_has_only_known_fields(item, item_type) + and isinstance(content, list) + and len(content) == 1 + and isinstance(content[0], dict) + and content[0].get("type") == "input_text" + and _input_content_part_is_self_contained( + cast(dict[str, JsonValue], content[0]), + allow_output=False, + ) + ) + + def _is_retained_response_message(item: Mapping[str, JsonValue]) -> bool: item_type = item.get("type") if ( diff --git a/openspec/changes/allow-developer-interleaved-fresh-resend/proposal.md b/openspec/changes/allow-developer-interleaved-fresh-resend/proposal.md index 2e5aafbf24..fc090c3846 100644 --- a/openspec/changes/allow-developer-interleaved-fresh-resend/proposal.md +++ b/openspec/changes/allow-developer-interleaved-fresh-resend/proposal.md @@ -4,21 +4,36 @@ A verified durable Responses-Lite input prefix can contain a completed direct tool call with a Codex `developer` message between the call and its matching output. Because the Lite `additional_tools` bundle keeps that message inline, the fresh full-resend classifier encounters it while the historical call is -pending and rejects the otherwise valid shape. The request then falls back to -anchor injection instead of preserving the original resend on the durable -owner. +pending and rejects the otherwise valid shape. + +Two additional Responses-Lite resend shapes are now observed after the stored +prefix: a fresh `developer` message after retained final output plus one user +follow-up, and a fresh `developer` message between a custom tool call and its +matching output. Treating every fresh developer message as unsafe makes these +bounded, account-neutral resends fall back to anchor injection and can trigger +an upstream acknowledgement timeout. ## What Changes -- Treat the observed unphased, non-response-owned historical `developer` - message as transparent only while proving an exact durable pending-tool +- Keep the observed unphased, non-response-owned historical `developer` + message transparent only while proving an exact durable pending-tool manifest from inline Responses-Lite input. -- Keep the matching historical output mandatory and keep the fresh suffix - restricted to complete direct call/output pairs. -- Leave non-Lite `input` and `messages` instruction hoisting and classification - unchanged; this change does not add hoist provenance to the durable proof. -- Add helper-level fail-closed coverage and a public `/v1/responses` bridge - regression using the observed interleaving. +- Allow a fresh developer message after retained output only when the latest + retained assistant message is `final_answer`, exactly one explicit user + message follows it, and the developer message is terminal. +- Allow a fresh developer message in a tool suffix only when the entire suffix + is exactly `custom_tool_call -> developer -> matching custom_tool_call_output` + and the pair exactly equals the durable pending-tool manifest. +- Require fresh developer messages to contain exactly one account-neutral + `input_text` part, exact `turn_id` metadata, known fields, no response-owned + ID or phase, and no status other than `completed`. +- Keep function calls, apply-patch calls, parallel batches, extra leading or + trailing items, malformed types, account-scoped content, and all unproven + developer positions fail-closed. +- Leave non-Lite `input` and `messages` instruction hoisting unchanged; this + change does not add hoist provenance to the durable proof. +- Add helper, bridge-unit, and public `/v1/responses` regressions for both + observed fresh suffixes and their rejection boundaries. ## Capabilities @@ -29,12 +44,13 @@ None. ### Modified Capabilities - `responses-api-compat`: verified Responses-Lite developer-interleaved history - can preserve the existing safe fresh full-resend path. + and two bounded fresh developer suffixes can preserve the existing safe fresh + full-resend path. ## Impact -- Code: replay classification plus the HTTP bridge call site that preserves - developer-message ID evidence during classification. -- Tests: focused replay-safety and existing HTTP bridge route coverage. -- Retained-output replay, leading commentary, owner forwarding, retry policy, - logging, storage, and public schemas are unchanged. +- Code: replay classification; the existing HTTP bridge projection and owner + selection contracts remain unchanged. +- Tests: focused replay-safety, bridge-unit, and HTTP bridge route coverage. +- Owner forwarding, retry policy, logging, storage, public schemas, and + non-Lite instruction hoisting remain unchanged. diff --git a/openspec/changes/allow-developer-interleaved-fresh-resend/specs/responses-api-compat/spec.md b/openspec/changes/allow-developer-interleaved-fresh-resend/specs/responses-api-compat/spec.md index 7ac97221f5..40eeacd7b0 100644 --- a/openspec/changes/allow-developer-interleaved-fresh-resend/specs/responses-api-compat/spec.md +++ b/openspec/changes/allow-developer-interleaved-fresh-resend/specs/responses-api-compat/spec.md @@ -2,28 +2,25 @@ ## ADDED Requirements -### Requirement: Responses-Lite exact manifest proof tolerates verified historical developer interleaving +### Requirement: Responses-Lite replay proof tolerates only verified developer interleaving When a fresh durable HTTP bridge classifies a client-unanchored Responses-Lite full resend whose `additional_tools` bundle preserves developer messages inline, -the exact durable pending-tool proof MUST allow a valid `developer` message -between a supported direct tool call and that call's matching output in the -fingerprint-verified stored prefix. +the replay proof MUST tolerate a developer message only in the historical and +fresh positions defined below. Every other developer position or shape MUST +remain fail-closed. -The developer message MUST have no response-owned ID or phase, MUST have no -status or a `completed` status, MUST pass the existing account-neutral metadata, -field, and content validation, and MUST NOT settle or reorder the pending call. -Classification MUST retain response-owned developer-message ID evidence until -this check has completed, even when other response-owned IDs are projected out. -The matching output MUST remain present with the same call ID and type. This -exception MUST NOT apply to the alternative retained assistant-output proof or -to the fresh suffix. The fresh suffix MUST remain a complete direct call/output -set exactly equal to the durable manifest. +A tolerated fresh developer message MUST have `type` omitted or equal to `message`, +MUST have role `developer`, MUST have no non-empty response-owned ID or phase, +MUST have no status or a `completed` status, MUST contain exact account-neutral +metadata with one nonblank `turn_id`, MUST contain exactly one self-contained +`input_text` content part, and MUST contain no unknown or account-scoped fields. +Explicit null or malformed item types MUST fail closed. -This exception MUST apply only when the developer message remains inline in the -validated Responses-Lite input. Non-Lite `input` or `messages` forms whose -instruction-role messages are normalized into top-level `instructions` are -outside this requirement. +Classification MUST retain response-owned developer-message ID evidence until +these checks have completed, even when other response-owned IDs are projected +out. Non-Lite `input` or `messages` forms whose instruction-role messages are +normalized into top-level `instructions` remain outside this requirement. #### Scenario: Verified historical Responses-Lite developer message is transparent @@ -47,15 +44,30 @@ outside this requirement. - **WHEN** the matching output is missing or has another call ID or type - **THEN** exact manifest proof fails -#### Scenario: Fresh inline developer message is not a tool-loop item +#### Scenario: Bounded fresh custom-tool developer interleave is transparent + +- **GIVEN** the fingerprint-verified stored prefix is followed by a fresh suffix +- **AND** the durable pending-tool manifest contains exactly one `custom_tool_call` +- **WHEN** the entire suffix is exactly that custom call, one valid developer message, and its matching custom-tool output +- **THEN** exact manifest proof passes +- **AND** the original full input is sent once without injecting `previous_response_id` -- **GIVEN** a Responses-Lite input whose developer messages remain inline -- **AND** a durable pending-tool manifest -- **WHEN** the fresh suffix contains a developer message among its call/output items +#### Scenario: Other fresh tool-loop developer positions remain fail-closed + +- **GIVEN** a durable pending-tool manifest +- **WHEN** a fresh developer message is used with a function or apply-patch call, appears in a parallel batch, is duplicated, lacks exact metadata, contains malformed or account-scoped content, or has leading or trailing suffix items - **THEN** exact manifest proof fails -#### Scenario: Alternative retained-output proof stays narrow +#### Scenario: Bounded retained-output developer follow-up is transparent + +- **GIVEN** the fingerprint-verified stored prefix is followed by a completed assistant `final_answer` +- **AND** exactly one explicit user message follows that retained output +- **WHEN** one valid developer message is the terminal suffix item +- **THEN** retained-output proof passes +- **AND** the original full input is sent once without injecting `previous_response_id` + +#### Scenario: Unproven retained-output developer follow-up remains fail-closed -- **GIVEN** a stored prefix contains a developer-interleaved historical call -- **WHEN** the fresh suffix uses retained assistant output plus new user input instead of the exact durable manifest -- **THEN** the developer exception does not make that alternative proof pass +- **GIVEN** a retained-output full resend +- **WHEN** the latest assistant output is not `final_answer`, the developer message is not terminal, the fresh input is raw or contains multiple user items, the developer metadata or content is not account-neutral, or the stored prefix contains historical developer interleaving +- **THEN** retained-output proof fails diff --git a/openspec/changes/allow-developer-interleaved-fresh-resend/tasks.md b/openspec/changes/allow-developer-interleaved-fresh-resend/tasks.md index eb69491282..482c6ac99e 100644 --- a/openspec/changes/allow-developer-interleaved-fresh-resend/tasks.md +++ b/openspec/changes/allow-developer-interleaved-fresh-resend/tasks.md @@ -2,15 +2,18 @@ - [x] 1.1 Allow a valid historical developer message only in exact manifest proof. - [x] 1.2 Keep the historical call/output match and fresh suffix checks fail-closed. -- [x] 1.3 Keep the alternative retained-output proof unchanged. +- [x] 1.3 Allow a terminal fresh developer message only after `final_answer` and exactly one explicit user follow-up. +- [x] 1.4 Allow fresh tool interleaving only for an exact custom call/developer/matching-output suffix. +- [x] 1.5 Reject malformed, account-scoped, parallel, function/apply-patch, leading, trailing, and repeated variants. ## 2. Regression coverage - [x] 2.1 Add focused positive and negative replay-safety cases. -- [x] 2.2 Exercise the developer-interleaved resend through `/v1/responses`. -- [x] 2.3 Verify the route regression fails before the production fix and passes after it. +- [x] 2.2 Exercise historical developer interleaving through `/v1/responses`. +- [x] 2.3 Exercise both bounded fresh developer suffixes through bridge-unit and `/v1/responses` coverage. +- [x] 2.4 Verify the new positive regressions fail before the production fix and pass after it. ## 3. Validation -- [x] 3.1 Run focused replay-safety and HTTP bridge tests plus changed-file Ruff checks. -- [x] 3.2 Run strict OpenSpec validation. +- [x] 3.1 Run the full replay-safety, bridge-unit, and HTTP bridge integration suites. +- [x] 3.2 Run changed-file Ruff, type, diff, and strict OpenSpec checks. diff --git a/tests/integration/test_http_responses_bridge.py b/tests/integration/test_http_responses_bridge.py index e153f4f9e1..6600f2047f 100644 --- a/tests/integration/test_http_responses_bridge.py +++ b/tests/integration/test_http_responses_bridge.py @@ -7117,10 +7117,21 @@ async def fake_connect_responses_websocket( @pytest.mark.parametrize( - ("developer_message_extra", "preserves_full_resend"), + ("developer_message_extra", "fresh_developer_message", "preserves_full_resend"), [ - pytest.param({}, True, id="unowned-developer-message"), - pytest.param({"id": "msg_response_owned"}, False, id="response-owned-developer-message"), + pytest.param({}, None, True, id="unowned-developer-message"), + pytest.param({"id": "msg_response_owned"}, None, False, id="response-owned-developer-message"), + pytest.param( + {}, + { + "type": "message", + "role": "developer", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn_fresh"}, + "content": [{"type": "input_text", "text": "fresh control"}], + }, + True, + id="fresh-developer-interleave", + ), ], ) @pytest.mark.asyncio @@ -7128,6 +7139,7 @@ async def test_v1_responses_http_bridge_classifies_responses_lite_developer_inte async_client, monkeypatch, developer_message_extra, + fresh_developer_message, preserves_full_resend, ): _install_bridge_settings(monkeypatch, enabled=True) @@ -7216,6 +7228,7 @@ async def fake_connect_responses_websocket( "name": "shell", "input": "pwd", }, + *([fresh_developer_message] if fresh_developer_message is not None else []), { "type": "custom_tool_call_output", "call_id": "call_custom_shell", @@ -7443,9 +7456,16 @@ async def fake_connect_responses_websocket( assert stale_session.closed is True +@pytest.mark.parametrize( + "fresh_developer_followup", + [ + pytest.param(False, id="ordinary-user-followup"), + pytest.param(True, id="fresh-developer-followup"), + ], +) @pytest.mark.asyncio async def test_v1_responses_http_bridge_replays_full_resend_once_then_stays_on_new_owner( - async_client, app_instance, monkeypatch + async_client, app_instance, monkeypatch, fresh_developer_followup ): _install_bridge_settings(monkeypatch, enabled=True) owner_account_id = await _import_account( @@ -7510,10 +7530,21 @@ async def fake_connect_responses_websocket( monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) historical_input = [ + *( + [ + { + "type": "additional_tools", + "role": "developer", + "tools": [{"type": "custom", "name": "shell"}], + } + ] + if fresh_developer_followup + else [] + ), { "role": "user", "content": [{"type": "input_text", "text": "first question"}], - } + }, ] first = await asyncio.wait_for( async_client.post( @@ -7541,19 +7572,46 @@ async def fake_connect_responses_websocket( assert durable_lookup.latest_input_item_count == len(historical_input) assert durable_lookup.latest_input_full_fingerprint is not None - retained_prior_output = { - "type": "message", - "role": "assistant", - "status": "completed", - "content": [{"type": "output_text", "text": "first answer"}], - } + if fresh_developer_followup: + retained_prior_output = { + "type": "message", + "role": "assistant", + "phase": "final_answer", + "status": "completed", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn_previous"}, + "content": [{"type": "output_text", "text": "first answer"}], + } + fresh_followup_items = [ + { + "type": "message", + "role": "user", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn_current"}, + "content": [{"type": "input_text", "text": "second question"}], + }, + { + "type": "message", + "role": "developer", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn_current"}, + "content": [{"type": "input_text", "text": "fresh control"}], + }, + ] + else: + retained_prior_output = { + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "first answer"}], + } + fresh_followup_items = [ + { + "role": "user", + "content": [{"type": "input_text", "text": "second question"}], + } + ] full_resend = [ *historical_input, retained_prior_output, - { - "role": "user", - "content": [{"type": "input_text", "text": "second question"}], - }, + *fresh_followup_items, ] second = await asyncio.wait_for( async_client.post( diff --git a/tests/unit/test_proxy_http_bridge.py b/tests/unit/test_proxy_http_bridge.py index 00dd232e8e..0827e6eec0 100644 --- a/tests/unit/test_proxy_http_bridge.py +++ b/tests/unit/test_proxy_http_bridge.py @@ -7288,6 +7288,33 @@ def fake_prepare( False, id="retained-assistant-output", ), + pytest.param( + [ + { + "type": "message", + "role": "assistant", + "phase": "final_answer", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-previous"}, + "content": [{"type": "output_text", "text": "hello back"}], + }, + { + "type": "message", + "role": "user", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-current"}, + "content": [{"type": "input_text", "text": "follow up"}], + }, + { + "type": "message", + "role": "developer", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-current"}, + "content": [{"type": "input_text", "text": "new control message"}], + }, + ], + None, + True, + False, + id="retained-assistant-output-with-fresh-developer-followup", + ), pytest.param( [ { @@ -7307,6 +7334,33 @@ def fake_prepare( False, id="self-contained-tool-loop", ), + pytest.param( + [ + { + "type": "custom_tool_call", + "call_id": "call-1", + "name": "shell", + "input": "pwd", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-current"}, + }, + { + "type": "message", + "role": "developer", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-current"}, + "content": [{"type": "input_text", "text": "new control message"}], + }, + { + "type": "custom_tool_call_output", + "call_id": "call-1", + "output": "/workspace", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-current"}, + }, + ], + {"call-1": "custom_tool_call"}, + True, + False, + id="self-contained-tool-loop-with-fresh-developer-interleave", + ), pytest.param( [ { diff --git a/tests/unit/test_replay_safety.py b/tests/unit/test_replay_safety.py index 58679dc683..fd642d66eb 100644 --- a/tests/unit/test_replay_safety.py +++ b/tests/unit/test_replay_safety.py @@ -715,6 +715,260 @@ def test_full_resend_suffix_accepts_only_self_contained_tool_loops( ) +def test_full_resend_tool_loop_manifest_tolerates_fresh_developer_interleave_after_historical_one() -> None: + stored_input: list[JsonValue] = [ + {"role": "user", "content": "first question"}, + { + "type": "custom_tool_call", + "id": "ctc_old", + "call_id": "call_old", + "name": "shell", + "input": "pwd", + "status": "completed", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn_old"}, + }, + { + "type": "message", + "id": "msg_old_control", + "role": "developer", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn_old"}, + "content": [{"type": "input_text", "text": "historical control message"}], + }, + { + "type": "custom_tool_call_output", + "id": "ctco_old", + "call_id": "call_old", + "output": "/workspace", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn_old"}, + }, + ] + suffix: list[JsonValue] = [ + { + "type": "custom_tool_call", + "id": "ctc_current", + "call_id": "call_current", + "name": "shell", + "input": "pwd", + "status": "completed", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn_current"}, + }, + { + "type": "message", + "id": "msg_control", + "role": "developer", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn_current"}, + "content": [{"type": "input_text", "text": "new control message"}], + }, + { + "type": "custom_tool_call_output", + "id": "ctco_current", + "call_id": "call_current", + "output": "/workspace", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn_current"}, + }, + ] + + projection = project_responses_input_for_account_neutral_fresh_replay( + [*stored_input, *suffix], + stored_count=len(stored_input), + ) + assert projection is not None + assert ( + responses_input_suffix_matches_pending_tool_calls( + projection.input_items, + stored_count=projection.stored_prefix_count, + pending_tool_calls={"call_current": "custom_tool_call"}, + ) + is True + ) + + +def test_full_resend_tool_loop_manifest_rejects_fresh_developer_inside_function_call_pair() -> None: + stored_input: list[JsonValue] = [{"role": "user", "content": "first question"}] + suffix: list[JsonValue] = [ + { + "type": "function_call", + "call_id": "call_current", + "name": "lookup", + "arguments": "{}", + }, + { + "role": "developer", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-current"}, + "content": [{"type": "input_text", "text": "unobserved function control"}], + }, + { + "type": "function_call_output", + "call_id": "call_current", + "output": "ok", + }, + ] + projection = project_responses_input_for_account_neutral_fresh_replay( + [*stored_input, *suffix], + stored_count=len(stored_input), + ) + + assert projection is not None + assert ( + responses_input_suffix_matches_pending_tool_calls( + projection.input_items, + stored_count=projection.stored_prefix_count, + pending_tool_calls={"call_current": "function_call"}, + ) + is False + ) + + +def test_full_resend_tool_loop_manifest_rejects_adjacent_pair_nested_in_parallel_batch() -> None: + stored_input: list[JsonValue] = [{"role": "user", "content": "first question"}] + suffix: list[JsonValue] = [ + {"type": "custom_tool_call", "call_id": "call_1", "name": "shell", "input": "pwd"}, + {"type": "custom_tool_call", "call_id": "call_2", "name": "shell", "input": "whoami"}, + { + "role": "developer", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-current"}, + "content": [{"type": "input_text", "text": "control inside one matching parallel pair"}], + }, + {"type": "custom_tool_call_output", "call_id": "call_2", "output": "worker"}, + {"type": "custom_tool_call_output", "call_id": "call_1", "output": "/workspace"}, + ] + projection = project_responses_input_for_account_neutral_fresh_replay( + [*stored_input, *suffix], + stored_count=len(stored_input), + ) + + assert projection is not None + assert ( + responses_input_suffix_matches_pending_tool_calls( + projection.input_items, + stored_count=projection.stored_prefix_count, + pending_tool_calls={"call_1": "custom_tool_call", "call_2": "custom_tool_call"}, + ) + is False + ) + + +@pytest.mark.parametrize( + "suffix", + [ + pytest.param( + [ + { + "role": "developer", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-current"}, + "content": [{"type": "input_text", "text": "control before call"}], + }, + {"type": "custom_tool_call", "call_id": "call_current", "name": "shell", "input": "pwd"}, + {"type": "custom_tool_call_output", "call_id": "call_current", "output": "/workspace"}, + ], + id="developer-before-call", + ), + pytest.param( + [ + {"type": "custom_tool_call", "call_id": "call_current", "name": "shell", "input": "pwd"}, + {"type": "custom_tool_call_output", "call_id": "call_current", "output": "/workspace"}, + { + "role": "developer", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-current"}, + "content": [{"type": "input_text", "text": "control after output"}], + }, + ], + id="developer-after-output", + ), + pytest.param( + [ + {"type": "custom_tool_call", "call_id": "call_current", "name": "shell", "input": "pwd"}, + { + "role": "developer", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-current"}, + "content": [{"type": "input_text", "text": "first control"}], + }, + { + "role": "developer", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-current"}, + "content": [{"type": "input_text", "text": "second control"}], + }, + {"type": "custom_tool_call_output", "call_id": "call_current", "output": "/workspace"}, + ], + id="multiple-developer-messages", + ), + pytest.param( + [ + {"type": "custom_tool_call", "call_id": "call_current", "name": "shell", "input": "pwd"}, + { + "role": "developer", + "content": [{"type": "input_text", "text": "control without turn id"}], + }, + {"type": "custom_tool_call_output", "call_id": "call_current", "output": "/workspace"}, + ], + id="developer-without-turn-id", + ), + pytest.param( + [ + {"type": "custom_tool_call", "call_id": "call_current", "name": "shell", "input": "pwd"}, + { + "type": None, + "role": "developer", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-current"}, + "content": [{"type": "input_text", "text": "control with null type"}], + }, + {"type": "custom_tool_call_output", "call_id": "call_current", "output": "/workspace"}, + ], + id="developer-with-null-type", + ), + pytest.param( + [ + { + "type": "message", + "role": "assistant", + "phase": "commentary", + "content": [{"type": "output_text", "text": "working"}], + }, + {"type": "custom_tool_call", "call_id": "call_current", "name": "shell", "input": "pwd"}, + { + "role": "developer", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-current"}, + "content": [{"type": "input_text", "text": "control"}], + }, + {"type": "custom_tool_call_output", "call_id": "call_current", "output": "/workspace"}, + ], + id="developer-with-leading-assistant-exception", + ), + pytest.param( + [ + {"type": "custom_tool_call", "call_id": "call_current", "name": "shell", "input": "pwd"}, + { + "role": "developer", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-current"}, + "content": [{"type": "input_text", "text": "control"}], + }, + {"type": "custom_tool_call_output", "call_id": "call_current", "output": "/workspace"}, + {"role": "user", "content": "follow up"}, + ], + id="developer-with-trailing-user-exception", + ), + ], +) +def test_full_resend_tool_loop_manifest_rejects_unproven_fresh_developer_positions( + suffix: list[JsonValue], +) -> None: + stored_input: list[JsonValue] = [{"role": "user", "content": "first question"}] + projection = project_responses_input_for_account_neutral_fresh_replay( + [*stored_input, *suffix], + stored_count=len(stored_input), + ) + + assert projection is not None + assert ( + responses_input_suffix_matches_pending_tool_calls( + projection.input_items, + stored_count=projection.stored_prefix_count, + pending_tool_calls={"call_current": "custom_tool_call"}, + ) + is False + ) + + @pytest.mark.parametrize( ("interleaved_item", "expected"), [ @@ -902,6 +1156,303 @@ def test_full_resend_exact_manifest_requires_historical_interleaved_call_output( ) +def test_full_resend_retained_output_tolerates_fresh_developer_after_user() -> None: + stored_input: list[JsonValue] = [{"role": "user", "content": "first question"}] + suffix: list[JsonValue] = [ + { + "type": "message", + "id": "msg_answer", + "role": "assistant", + "phase": "final_answer", + "status": "completed", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn_previous"}, + "content": [{"type": "output_text", "text": "prior answer"}], + }, + { + "type": "message", + "id": "msg_user", + "role": "user", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn_current"}, + "content": [{"type": "input_text", "text": "next question"}], + }, + { + "type": "message", + "id": "msg_control", + "role": "developer", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn_current"}, + "content": [{"type": "input_text", "text": "new control message"}], + }, + ] + + projection = project_responses_input_for_account_neutral_fresh_replay( + [*stored_input, *suffix], + stored_count=len(stored_input), + ) + assert projection is not None + assert ( + responses_input_suffix_retains_prior_output( + projection.input_items, + stored_count=projection.stored_prefix_count, + ) + is True + ) + + +@pytest.mark.parametrize( + "suffix", + [ + pytest.param( + [ + { + "role": "assistant", + "phase": "final_answer", + "content": [{"type": "output_text", "text": "prior answer"}], + }, + { + "role": "developer", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-current"}, + "content": [{"type": "input_text", "text": "control without user"}], + }, + ], + id="developer-without-user", + ), + pytest.param( + [ + { + "role": "assistant", + "phase": "final_answer", + "content": [{"type": "output_text", "text": "prior answer"}], + }, + {"role": "user", "content": "next question"}, + { + "role": "developer", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-current"}, + "content": [{"type": "input_text", "text": "control"}], + }, + {"role": "user", "content": "later question"}, + ], + id="developer-not-terminal", + ), + pytest.param( + [ + { + "role": "assistant", + "phase": "final_answer", + "content": [{"type": "output_text", "text": "prior answer"}], + }, + {"role": "user", "content": "next question"}, + { + "role": "developer", + "content": [{"type": "input_text", "text": "control without turn id"}], + }, + ], + id="developer-without-turn-id", + ), + pytest.param( + [ + { + "role": "assistant", + "phase": "final_answer", + "content": [{"type": "output_text", "text": "prior answer"}], + }, + {"role": "user", "content": "next question"}, + { + "role": "developer", + "internal_chat_message_metadata_passthrough": { + "turn_id": "turn-current", + "account_id": "acc-1", + }, + "content": [{"type": "input_text", "text": "account-bound control"}], + }, + ], + id="developer-with-account-metadata", + ), + pytest.param( + [ + { + "role": "assistant", + "phase": "final_answer", + "content": [{"type": "output_text", "text": "prior answer"}], + }, + {"role": "user", "content": "next question"}, + { + "type": 7, + "role": "developer", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-current"}, + "content": [{"type": "input_text", "text": "control with malformed type"}], + }, + ], + id="developer-with-nonstring-type", + ), + pytest.param( + [ + { + "role": "assistant", + "phase": "final_answer", + "content": [{"type": "output_text", "text": "prior answer"}], + }, + {"role": "user", "content": "next question"}, + { + "type": None, + "role": "developer", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-current"}, + "content": [{"type": "input_text", "text": "control with null type"}], + }, + ], + id="developer-with-null-type", + ), + pytest.param( + [ + { + "role": "assistant", + "phase": "final_answer", + "content": [{"type": "output_text", "text": "prior answer"}], + }, + {"type": "input_text", "text": "raw next input"}, + { + "role": "developer", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-current"}, + "content": [{"type": "input_text", "text": "control after raw input"}], + }, + ], + id="developer-after-raw-input-part", + ), + pytest.param( + [ + { + "role": "assistant", + "phase": "final_answer", + "content": [{"type": "output_text", "text": "prior answer"}], + }, + {"type": "input_text", "text": "raw next input"}, + {"role": "user", "content": "next question"}, + { + "role": "developer", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-current"}, + "content": [{"type": "input_text", "text": "control after raw and user"}], + }, + ], + id="developer-after-raw-input-and-user", + ), + pytest.param( + [ + { + "role": "assistant", + "phase": "final_answer", + "content": [{"type": "output_text", "text": "prior answer"}], + }, + {"role": "user", "content": "first next question"}, + {"role": "user", "content": "second next question"}, + { + "role": "developer", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-current"}, + "content": [{"type": "input_text", "text": "control after repeated users"}], + }, + ], + id="developer-after-repeated-user-messages", + ), + pytest.param( + [ + { + "role": "assistant", + "phase": "final_answer", + "content": [{"type": "output_text", "text": "prior answer"}], + }, + {"role": "user", "content": "next question"}, + { + "role": "developer", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-current"}, + "content": [{"type": "input_file", "file_id": "file-account-bound"}], + }, + ], + id="developer-with-account-bound-file-id", + ), + pytest.param( + [ + { + "role": "assistant", + "phase": "commentary", + "content": [{"type": "output_text", "text": "still working"}], + }, + {"role": "user", "content": "next question"}, + { + "role": "developer", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-current"}, + "content": [{"type": "input_text", "text": "control after commentary"}], + }, + ], + id="developer-after-assistant-commentary", + ), + ], +) +def test_full_resend_retained_output_rejects_unproven_fresh_developer_followup( + suffix: list[JsonValue], +) -> None: + stored_input: list[JsonValue] = [{"role": "user", "content": "first question"}] + + assert ( + responses_input_suffix_retains_prior_output( + [*stored_input, *suffix], + stored_count=len(stored_input), + ) + is False + ) + + +def test_full_resend_retained_output_rejects_complete_tool_pair_between_output_and_user() -> None: + stored_prefix = [ + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "old"}], + } + ] + projected_full_input = project_responses_input_for_account_neutral_fresh_replay( + [ + *stored_prefix, + { + "type": "message", + "role": "assistant", + "phase": "final_answer", + "status": "completed", + "content": [{"type": "output_text", "text": "done"}], + }, + { + "type": "custom_tool_call", + "call_id": "call_extra", + "name": "shell", + "input": "pwd", + }, + { + "type": "custom_tool_call_output", + "call_id": "call_extra", + "output": "ok", + }, + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "new"}], + }, + { + "type": "message", + "role": "developer", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn_new"}, + "content": [{"type": "input_text", "text": "control"}], + }, + ], + stored_count=len(stored_prefix), + ) + + assert projected_full_input is not None + assert ( + responses_input_suffix_retains_prior_output( + projected_full_input.input_items, + stored_count=projected_full_input.stored_prefix_count, + ) + is False + ) + + def test_full_resend_retained_output_rejects_historical_developer_interleaving() -> None: stored_input: list[JsonValue] = [ {"role": "user", "content": "first question"}, From bb974f4f988f73703b14d67995e5c9b4611af809 Mon Sep 17 00:00:00 2001 From: choi138 Date: Mon, 3 Aug 2026 21:21:16 +0900 Subject: [PATCH 49/64] test(proxy): type replay safety fixture as JsonValue --- tests/unit/test_replay_safety.py | 67 ++++++++++++++++---------------- 1 file changed, 34 insertions(+), 33 deletions(-) diff --git a/tests/unit/test_replay_safety.py b/tests/unit/test_replay_safety.py index fd642d66eb..d7ecaeda84 100644 --- a/tests/unit/test_replay_safety.py +++ b/tests/unit/test_replay_safety.py @@ -1400,46 +1400,47 @@ def test_full_resend_retained_output_rejects_unproven_fresh_developer_followup( def test_full_resend_retained_output_rejects_complete_tool_pair_between_output_and_user() -> None: - stored_prefix = [ + stored_prefix: list[JsonValue] = [ { "type": "message", "role": "user", "content": [{"type": "input_text", "text": "old"}], } ] + full_input: list[JsonValue] = [ + *stored_prefix, + { + "type": "message", + "role": "assistant", + "phase": "final_answer", + "status": "completed", + "content": [{"type": "output_text", "text": "done"}], + }, + { + "type": "custom_tool_call", + "call_id": "call_extra", + "name": "shell", + "input": "pwd", + }, + { + "type": "custom_tool_call_output", + "call_id": "call_extra", + "output": "ok", + }, + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "new"}], + }, + { + "type": "message", + "role": "developer", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn_new"}, + "content": [{"type": "input_text", "text": "control"}], + }, + ] projected_full_input = project_responses_input_for_account_neutral_fresh_replay( - [ - *stored_prefix, - { - "type": "message", - "role": "assistant", - "phase": "final_answer", - "status": "completed", - "content": [{"type": "output_text", "text": "done"}], - }, - { - "type": "custom_tool_call", - "call_id": "call_extra", - "name": "shell", - "input": "pwd", - }, - { - "type": "custom_tool_call_output", - "call_id": "call_extra", - "output": "ok", - }, - { - "type": "message", - "role": "user", - "content": [{"type": "input_text", "text": "new"}], - }, - { - "type": "message", - "role": "developer", - "internal_chat_message_metadata_passthrough": {"turn_id": "turn_new"}, - "content": [{"type": "input_text", "text": "control"}], - }, - ], + full_input, stored_count=len(stored_prefix), ) From 86344cd9d332f0d7211ef8553ee13759c1b2756a Mon Sep 17 00:00:00 2001 From: mastertyko <11311479+mastertyko@users.noreply.github.com> Date: Thu, 30 Jul 2026 13:08:00 +0200 Subject: [PATCH 50/64] fix(proxy): retry detached API key release --- app/modules/proxy/_service/api_key_usage.py | 29 +++-- .../.openspec.yaml | 2 + .../retry-detached-api-key-release/design.md | 63 +++++++++++ .../proposal.md | 33 ++++++ .../specs/api-keys/spec.md | 38 +++++++ .../retry-detached-api-key-release/tasks.md | 15 +++ .../integration/test_detached_persistence.py | 107 ++++++++++++++++++ 7 files changed, 280 insertions(+), 7 deletions(-) create mode 100644 openspec/changes/retry-detached-api-key-release/.openspec.yaml create mode 100644 openspec/changes/retry-detached-api-key-release/design.md create mode 100644 openspec/changes/retry-detached-api-key-release/proposal.md create mode 100644 openspec/changes/retry-detached-api-key-release/specs/api-keys/spec.md create mode 100644 openspec/changes/retry-detached-api-key-release/tasks.md diff --git a/app/modules/proxy/_service/api_key_usage.py b/app/modules/proxy/_service/api_key_usage.py index 2159e3802d..963eab62f1 100644 --- a/app/modules/proxy/_service/api_key_usage.py +++ b/app/modules/proxy/_service/api_key_usage.py @@ -31,6 +31,8 @@ logger = logging.getLogger("app.modules.proxy.service") _API_KEY_RESERVATION_HEARTBEAT_SECONDS = 300.0 +_STREAM_API_KEY_RELEASE_RETRY_BASE_SECONDS = 0.1 +_STREAM_API_KEY_RELEASE_RETRY_MAX_SECONDS = 5.0 def _service_api_keys_service() -> type[ApiKeysService]: @@ -458,17 +460,30 @@ async def _release_unsettled_stream_api_key_usage( request_id: str, ) -> None: proxy = cast(_ApiKeyUsageServiceProtocol, self) - with anyio.CancelScope(shield=True): + retry_attempt = 1 + retry_delay_seconds = _STREAM_API_KEY_RELEASE_RETRY_BASE_SECONDS + while True: try: - async with proxy._repo_factory() as repos: - api_keys_service = _service_api_keys_service()(repos.api_keys) - await api_keys_service.release_usage_reservation( - api_key_reservation.reservation_id, - ) + with anyio.CancelScope(shield=True): + async with proxy._repo_factory() as repos: + api_keys_service = _service_api_keys_service()(repos.api_keys) + await api_keys_service.release_usage_reservation( + api_key_reservation.reservation_id, + ) + return except Exception: logger.warning( - "Failed to release stream API key reservation key_id=%s request_id=%s", + "Failed to release stream API key reservation key_id=%s request_id=%s " + "retry_attempt=%d retry_delay_seconds=%.2f", api_key.id, request_id, + retry_attempt, + retry_delay_seconds, exc_info=True, ) + await asyncio.sleep(retry_delay_seconds) + retry_attempt += 1 + retry_delay_seconds = min( + _STREAM_API_KEY_RELEASE_RETRY_MAX_SECONDS, + retry_delay_seconds * 2, + ) diff --git a/openspec/changes/retry-detached-api-key-release/.openspec.yaml b/openspec/changes/retry-detached-api-key-release/.openspec.yaml new file mode 100644 index 0000000000..ab39675458 --- /dev/null +++ b/openspec/changes/retry-detached-api-key-release/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-07-30 diff --git a/openspec/changes/retry-detached-api-key-release/design.md b/openspec/changes/retry-detached-api-key-release/design.md new file mode 100644 index 0000000000..96d74087c3 --- /dev/null +++ b/openspec/changes/retry-detached-api-key-release/design.md @@ -0,0 +1,63 @@ +## Context + +Stream reservation settlement is detached from the response path. A failed +settlement schedules one release task in the existing tracked background-task +set, but that release currently catches its own exception and returns normally. +The done callback consequently removes the task, so the persistence drain can +report success while the reservation remains active. + +Reservation release is already transactional and idempotent: once another +settler has changed the reservation from `reserved`, a later release is a +no-op. The existing stale sweep remains a last-resort repair, but its six-hour +age threshold is too slow for a known live cleanup chain. + +## Goals / Non-Goals + +**Goals:** + +- Keep transiently failing fallback release work visible to the existing task + drain. +- Retry until the idempotent release succeeds, with bounded retry pressure. +- Preserve detached response latency, settlement-before-health ordering, and + exactly-once accounting. + +**Non-Goals:** + +- Changing reservation amounts, quota admission, or stale-sweep timing. +- Adding a durable job queue, setting, migration, or new public API. +- Refactoring request-log persistence or unrelated cleanup ownership. + +## Decisions + +1. **Retry inside the already tracked release task.** The release coroutine + stays pending between attempts, so the current task registry and recursive + drain remain the single source of cleanup ownership. Creating a second + registry or a durable retry row would duplicate state for a narrow failure. + +2. **Use capped exponential delay for every persistence exception.** The outer + retry covers transient PostgreSQL/session failures that the API-key + service's SQLite-lock-specific retry does not classify. A fixed cap prevents + sustained database pressure during a longer outage while still allowing + recovery without waiting for stale sweep. + +3. **Rely on reservation transition idempotency.** A retry cannot double + decrement quota: release only claims a reservation still in `reserved` + state, and a concurrent finalizer or release makes subsequent attempts + no-ops. + +4. **Let the existing drain deadline bound shutdown waiting.** A recovered + release completes normally. A release still retrying at the deadline remains + pending, so `drain_persistence_tasks` returns `False` instead of claiming + durability. No separate retry-count terminal state is introduced. + +## Risks / Trade-offs + +- **A permanent persistence error leaves a task alive during normal runtime.** + → Retries use capped backoff; the task accurately represents unfinished + cleanup, and stale recovery remains the final repair path. +- **Many simultaneous failures could retry together after an outage.** + → Exponential delay bounds retry load; the change adds no inline request-path + work. +- **Cancellation can stop a retry after shutdown has already timed out.** + → The drain first reports incomplete, so process termination cannot be + mistaken for successful settlement. diff --git a/openspec/changes/retry-detached-api-key-release/proposal.md b/openspec/changes/retry-detached-api-key-release/proposal.md new file mode 100644 index 0000000000..d79b74a45f --- /dev/null +++ b/openspec/changes/retry-detached-api-key-release/proposal.md @@ -0,0 +1,33 @@ +## Why + +A detached stream settlement can fail, enqueue its reservation-release fallback, +and then lose the reservation when that fallback also hits a transient +persistence failure. The task drain reports success even though the reservation +still consumes quota until stale recovery runs hours later. + +## What Changes + +- Keep a failed detached reservation release tracked and retry it after + transient persistence failures. +- Make the persistence drain report completion only after the tracked + settlement/release chain has actually terminated. +- Add deterministic regression coverage for a finalize failure followed by one + failed release attempt, while preserving successful settlement, cancellation, + and SQLite-lock behavior. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `api-keys`: Clarify that a detached settlement fallback which itself fails + transiently remains tracked and retries before persistence drain can succeed. + +## Impact + +The change is limited to detached API-key reservation cleanup in the proxy +service, its focused persistence tests, and the existing API-key settlement +contract. It adds no API, setting, dependency, migration, or dashboard change. diff --git a/openspec/changes/retry-detached-api-key-release/specs/api-keys/spec.md b/openspec/changes/retry-detached-api-key-release/specs/api-keys/spec.md new file mode 100644 index 0000000000..fd325e06a6 --- /dev/null +++ b/openspec/changes/retry-detached-api-key-release/specs/api-keys/spec.md @@ -0,0 +1,38 @@ +## MODIFIED Requirements + +### Requirement: Stream reservation settlement is detached from the response path + +Settling a stream API-key reservation MUST NOT block the response/stream close, with one deliberate exception: when a keyed websocket stream terminates with an account-health error, the finalizer MUST wait for the settlement to commit before the load-balancer health write (the settlement-ordering invariant), so that error path intentionally blocks on settlement. In all other cases the settlement MUST run as a tracked background task; when it fails or is cancelled, the reservation MUST still be released by the tracking fallback, and the request's finalization path MUST NOT double-release a transferred settlement. If the tracking fallback itself encounters a persistence failure, it MUST remain tracked and retry the idempotent release; persistence drain MUST NOT report completion while that retry remains unfinished. Reservations MUST continue to count toward key limits until finalized or released, so deferred settlement can never admit usage a synchronous settlement would have rejected. + +#### Scenario: Response close precedes settlement completion + +- **GIVEN** a keyed stream whose settlement transaction is still running +- **WHEN** the stream closes +- **THEN** the close does not wait for the settlement +- **AND** the settlement finalizes the reservation exactly once in the background + +#### Scenario: Failed detached settlement still releases the reservation + +- **GIVEN** a detached settlement whose finalize raises +- **WHEN** the settlement task completes +- **THEN** the tracking fallback releases the reservation + +#### Scenario: Failed fallback release remains tracked + +- **GIVEN** a detached settlement whose finalize raises +- **AND** the first tracking-fallback release attempt also raises +- **WHEN** persistence recovers before the drain deadline +- **THEN** the tracked fallback retries and releases the reservation exactly once +- **AND** persistence drain does not report completion before that release + +#### Scenario: Websocket health-error settlement precedes the health write + +- **GIVEN** a keyed websocket stream that terminates with an account-health error +- **WHEN** the finalizer settles the reservation +- **THEN** it waits for the settlement to commit before recording the account-health error + +#### Scenario: Shutdown drains pending settlements + +- **WHEN** the service shuts down gracefully with settlements in flight +- **THEN** shutdown waits for them up to the configured drain timeout +- **AND** reports an incomplete drain if a tracked settlement or release remains unfinished at that timeout diff --git a/openspec/changes/retry-detached-api-key-release/tasks.md b/openspec/changes/retry-detached-api-key-release/tasks.md new file mode 100644 index 0000000000..eef9f33cde --- /dev/null +++ b/openspec/changes/retry-detached-api-key-release/tasks.md @@ -0,0 +1,15 @@ +## 1. Regression + +- [x] 1.1 Add a real-repository regression that injects one finalize failure and one fallback-release failure. +- [x] 1.2 Confirm the regression fails deterministically twice on baseline `3fe0d6f286019a0505783d803db9a1d8cdf6b307`. + +## 2. Implementation + +- [x] 2.1 Keep the fallback release tracked while retrying persistence failures with capped backoff. +- [x] 2.2 Preserve idempotent settlement, cancellation ownership, and truthful persistence-drain behavior. + +## 3. Verification + +- [x] 3.1 Run the focused detached-settlement and API-key reservation tests. +- [x] 3.2 Run changed-file Ruff, format, type, proxy-architecture, and strict OpenSpec checks. +- [x] 3.3 Inspect the final diff and worktree status for scope and unrelated changes. diff --git a/tests/integration/test_detached_persistence.py b/tests/integration/test_detached_persistence.py index 8f37d03890..cb18ef800b 100644 --- a/tests/integration/test_detached_persistence.py +++ b/tests/integration/test_detached_persistence.py @@ -3,9 +3,17 @@ import pytest from httpx import ASGITransport, AsyncClient from sqlalchemy import select +from sqlalchemy.exc import OperationalError from app.db.models import RequestLog from app.db.session import SessionLocal +from app.modules.api_keys.repository import ApiKeysRepository, UsageReservationData +from app.modules.api_keys.service import ( + ApiKeyCreateData, + ApiKeyRequestUsageBudget, + ApiKeysService, + LimitRuleInput, +) from app.modules.proxy import service as proxy_service_module pytestmark = pytest.mark.integration @@ -105,6 +113,105 @@ async def never_finishes() -> None: service._request_log_tasks.discard(task) +@pytest.mark.asyncio +async def test_failed_detached_settlement_retries_failed_release_until_persisted(raw_client, monkeypatch): + import asyncio + + _, app = raw_client + + async with SessionLocal() as session: + api_keys = ApiKeysService(ApiKeysRepository(session)) + created = await api_keys.create_key( + ApiKeyCreateData( + name="detached-release-retry", + allowed_models=None, + expires_at=None, + limits=[ + LimitRuleInput( + limit_type="total_tokens", + limit_window="weekly", + max_value=100, + ) + ], + ) + ) + api_key = await api_keys.get_key_by_id(created.id) + reservation = await api_keys.enforce_limits_for_request( + created.id, + request_model="gpt-5.5", + request_usage_budget=ApiKeyRequestUsageBudget( + input_tokens=4, + output_tokens=6, + ), + ) + + original_get_reservation = ApiKeysRepository.get_usage_reservation + reservation_read_attempts = 0 + retry_started = asyncio.Event() + allow_retry = asyncio.Event() + + async def fail_first_two_reservation_reads( + self: ApiKeysRepository, + reservation_id: str, + ) -> UsageReservationData | None: + nonlocal reservation_read_attempts + if reservation_id == reservation.reservation_id: + reservation_read_attempts += 1 + if reservation_read_attempts <= 2: + raise OperationalError( + "read usage reservation", + {}, + Exception("transient persistence connection failure"), + ) + if reservation_read_attempts == 3: + retry_started.set() + await allow_retry.wait() + return await original_get_reservation(self, reservation_id) + + monkeypatch.setattr(ApiKeysRepository, "get_usage_reservation", fail_first_two_reservation_reads) + + settlement = proxy_service_module._StreamSettlement( + status="success", + model="gpt-5.5", + input_tokens=4, + output_tokens=6, + ) + from app.dependencies import get_proxy_service_for_app + + service = get_proxy_service_for_app(app) + assert await service._settle_stream_api_key_usage( + api_key, + reservation, + settlement, + request_id="req_detached_release_retry", + ) + drain_task = asyncio.create_task(service.drain_persistence_tasks(timeout_seconds=2)) + retry_wait_task = asyncio.create_task(retry_started.wait()) + done, _ = await asyncio.wait( + {retry_wait_task, drain_task}, + timeout=1, + return_when=asyncio.FIRST_COMPLETED, + ) + retry_was_tracked = retry_wait_task in done and not drain_task.done() + allow_retry.set() + if not retry_wait_task.done(): + retry_wait_task.cancel() + await asyncio.gather(retry_wait_task, return_exceptions=True) + assert await drain_task + + async with SessionLocal() as session: + repo = ApiKeysRepository(session) + stored = await original_get_reservation(repo, reservation.reservation_id) + limits = await repo.get_limits_by_key(created.id) + + assert stored is not None + assert stored.status == "released" + assert len(limits) == 1 + assert limits[0].current_value == 0 + assert reservation_read_attempts == 3 + assert retry_was_tracked is True + + @pytest.mark.asyncio async def test_drain_ignores_stuck_non_persistence_cleanup_tasks(): """A stuck bridge-close cleanup in _background_cleanup_tasks must not From c77b1feb950dea31a819a222025baca23d154ffb Mon Sep 17 00:00:00 2001 From: mastertyko <11311479+mastertyko@users.noreply.github.com> Date: Thu, 30 Jul 2026 13:15:27 +0200 Subject: [PATCH 51/64] fix(review): P1 - preserve inline stream release latency --- app/modules/proxy/_service/api_key_usage.py | 11 +++++++ tests/unit/test_http_bridge_cancel_drain.py | 4 +++ tests/unit/test_proxy_utils.py | 33 +++++++++++++++------ 3 files changed, 39 insertions(+), 9 deletions(-) diff --git a/app/modules/proxy/_service/api_key_usage.py b/app/modules/proxy/_service/api_key_usage.py index 963eab62f1..0294539fe9 100644 --- a/app/modules/proxy/_service/api_key_usage.py +++ b/app/modules/proxy/_service/api_key_usage.py @@ -397,6 +397,7 @@ def _settlement_done(done_task: asyncio.Task[bool]) -> None: api_key=api_key, api_key_reservation=api_key_reservation, request_id=request_id, + retry_persistence_failures=True, ) self._schedule_cancel_safe_cleanup( release_coro, @@ -416,6 +417,7 @@ def _settlement_done(done_task: asyncio.Task[bool]) -> None: api_key=api_key, api_key_reservation=api_key_reservation, request_id=request_id, + retry_persistence_failures=True, ) self._schedule_cancel_safe_cleanup( release_coro, @@ -458,6 +460,7 @@ async def _release_unsettled_stream_api_key_usage( api_key: ApiKeyData, api_key_reservation: ApiKeyUsageReservationData, request_id: str, + retry_persistence_failures: bool = False, ) -> None: proxy = cast(_ApiKeyUsageServiceProtocol, self) retry_attempt = 1 @@ -472,6 +475,14 @@ async def _release_unsettled_stream_api_key_usage( ) return except Exception: + if not retry_persistence_failures: + logger.warning( + "Failed to release stream API key reservation key_id=%s request_id=%s", + api_key.id, + request_id, + exc_info=True, + ) + return logger.warning( "Failed to release stream API key reservation key_id=%s request_id=%s " "retry_attempt=%d retry_delay_seconds=%.2f", diff --git a/tests/unit/test_http_bridge_cancel_drain.py b/tests/unit/test_http_bridge_cancel_drain.py index 2ba47a109e..787fdc6067 100644 --- a/tests/unit/test_http_bridge_cancel_drain.py +++ b/tests/unit/test_http_bridge_cancel_drain.py @@ -91,14 +91,17 @@ async def test_cancelled_stream_settlement_task_releases_reservation( service = proxy_service.ProxyService(cast(Any, SimpleNamespace())) scheduled: list[tuple[str, str]] = [] cleanup_tasks: list[asyncio.Task[None]] = [] + release_retry_flags: list[bool] = [] async def release_unsettled( *, api_key: ApiKeyData, api_key_reservation: ApiKeyUsageReservationData, request_id: str, + retry_persistence_failures: bool = False, ) -> None: scheduled.append((api_key.id, api_key_reservation.reservation_id)) + release_retry_flags.append(retry_persistence_failures) def schedule_cleanup( coro: Any, @@ -131,6 +134,7 @@ def schedule_cleanup( assert ("release_stream_api_key_reservation_after_cancelled_settlement", "req-cancel-settle") in scheduled assert ("key-cancel-settle", "res-cancel-settle") in scheduled + assert release_retry_flags == [True] @pytest.mark.asyncio diff --git a/tests/unit/test_proxy_utils.py b/tests/unit/test_proxy_utils.py index 31910f68ca..e5712e836b 100644 --- a/tests/unit/test_proxy_utils.py +++ b/tests/unit/test_proxy_utils.py @@ -27342,10 +27342,20 @@ async def fake_relay(*args, **kwargs): upstream.send_text.assert_awaited_once() +@pytest.mark.parametrize("release_read_fails", [False, True]) @pytest.mark.asyncio -async def test_stream_with_retry_releases_api_key_reservation_when_owner_lookup_fails(monkeypatch): +async def test_stream_with_retry_releases_api_key_reservation_when_owner_lookup_fails( + monkeypatch, + release_read_fails: bool, +): request_logs = _RequestLogsRecorder() - get_usage_reservation_mock = AsyncMock(return_value=SimpleNamespace(status="reserved", items=[])) + reservation_record = SimpleNamespace(status="reserved", items=[]) + get_usage_reservation_mock = AsyncMock( + side_effect=( + [RuntimeError("transient reservation read failure"), reservation_record] if release_read_fails else None + ), + return_value=reservation_record, + ) transition_usage_reservation_status_mock = AsyncMock(return_value=True) settle_usage_reservation_mock = AsyncMock() commit_mock = AsyncMock() @@ -27440,13 +27450,18 @@ async def __aexit__(self, exc_type, exc, tb) -> bool: owner_lookup.assert_awaited_once() select_account.assert_not_called() get_usage_reservation_mock.assert_awaited_once_with(reservation.reservation_id) - transition_usage_reservation_status_mock.assert_awaited_once_with( - reservation.reservation_id, - expected_status="reserved", - new_status="released", - ) - settle_usage_reservation_mock.assert_awaited_once() - commit_mock.assert_awaited_once() + if release_read_fails: + transition_usage_reservation_status_mock.assert_not_awaited() + settle_usage_reservation_mock.assert_not_awaited() + commit_mock.assert_not_awaited() + else: + transition_usage_reservation_status_mock.assert_awaited_once_with( + reservation.reservation_id, + expected_status="reserved", + new_status="released", + ) + settle_usage_reservation_mock.assert_awaited_once() + commit_mock.assert_awaited_once() @pytest.mark.asyncio From 11d9d4e1af3028abee429ebdbe2aa69323219f12 Mon Sep 17 00:00:00 2001 From: mastertyko <11311479+mastertyko@users.noreply.github.com> Date: Thu, 30 Jul 2026 13:34:21 +0200 Subject: [PATCH 52/64] fix(review): P1 - bound detached release retry fan-out --- app/modules/proxy/_service/api_key_usage.py | 9 ++ app/modules/proxy/service.py | 4 + .../retry-detached-api-key-release/design.md | 17 +-- .../proposal.md | 3 +- .../specs/api-keys/spec.md | 9 +- .../retry-detached-api-key-release/tasks.md | 2 + tests/unit/test_proxy_utils.py | 109 ++++++++++++++++-- 7 files changed, 132 insertions(+), 21 deletions(-) diff --git a/app/modules/proxy/_service/api_key_usage.py b/app/modules/proxy/_service/api_key_usage.py index 0294539fe9..43eb6ea937 100644 --- a/app/modules/proxy/_service/api_key_usage.py +++ b/app/modules/proxy/_service/api_key_usage.py @@ -33,6 +33,7 @@ _API_KEY_RESERVATION_HEARTBEAT_SECONDS = 300.0 _STREAM_API_KEY_RELEASE_RETRY_BASE_SECONDS = 0.1 _STREAM_API_KEY_RELEASE_RETRY_MAX_SECONDS = 5.0 +_STREAM_API_KEY_RELEASE_RETRY_MAX_CONCURRENCY = 4 def _service_api_keys_service() -> type[ApiKeysService]: @@ -60,6 +61,7 @@ def _api_key_reservation_heartbeat_seconds() -> float: class _ApiKeyUsageServiceProtocol(Protocol): _repo_factory: ProxyRepoFactory _background_cleanup_tasks: set[asyncio.Task[None]] + _stream_api_key_release_retry_semaphore: asyncio.Semaphore def _normalize_service_tier_value(value: Any) -> str | None: @@ -466,7 +468,11 @@ async def _release_unsettled_stream_api_key_usage( retry_attempt = 1 retry_delay_seconds = _STREAM_API_KEY_RELEASE_RETRY_BASE_SECONDS while True: + retry_slot_acquired = False try: + if retry_persistence_failures: + await proxy._stream_api_key_release_retry_semaphore.acquire() + retry_slot_acquired = True with anyio.CancelScope(shield=True): async with proxy._repo_factory() as repos: api_keys_service = _service_api_keys_service()(repos.api_keys) @@ -492,6 +498,9 @@ async def _release_unsettled_stream_api_key_usage( retry_delay_seconds, exc_info=True, ) + finally: + if retry_slot_acquired: + proxy._stream_api_key_release_retry_semaphore.release() await asyncio.sleep(retry_delay_seconds) retry_attempt += 1 retry_delay_seconds = min( diff --git a/app/modules/proxy/service.py b/app/modules/proxy/service.py index 3b6e75b4f1..f265969572 100644 --- a/app/modules/proxy/service.py +++ b/app/modules/proxy/service.py @@ -122,6 +122,9 @@ from app.modules.proxy._service.api_key_usage import ( _API_KEY_RESERVATION_HEARTBEAT_SECONDS as _API_KEY_RESERVATION_HEARTBEAT_SECONDS, ) +from app.modules.proxy._service.api_key_usage import ( + _STREAM_API_KEY_RELEASE_RETRY_MAX_CONCURRENCY as _STREAM_API_KEY_RELEASE_RETRY_MAX_CONCURRENCY, +) from app.modules.proxy._service.api_key_usage import _ApiKeyUsageMixin from app.modules.proxy._service.codex_control import _CodexControlMixin from app.modules.proxy._service.compact import _CompactMixin @@ -945,6 +948,7 @@ def __init__( self._websocket_previous_response_account_index: dict[tuple[str, str | None, str | None], str] = {} self._websocket_continuity_index: dict[tuple[str, str | None], _WebSocketContinuityState] = {} self._background_cleanup_tasks: set[asyncio.Task[None]] = set() + self._stream_api_key_release_retry_semaphore = asyncio.Semaphore(_STREAM_API_KEY_RELEASE_RETRY_MAX_CONCURRENCY) # In-memory pin from upstream-issued file_id -> codex-lb account_id. # Used so ``finalize_file`` for a given ``file_id`` is routed to # the same account that handled ``create_file``. Cross-instance diff --git a/openspec/changes/retry-detached-api-key-release/design.md b/openspec/changes/retry-detached-api-key-release/design.md index 96d74087c3..9736d61e2e 100644 --- a/openspec/changes/retry-detached-api-key-release/design.md +++ b/openspec/changes/retry-detached-api-key-release/design.md @@ -34,11 +34,13 @@ age threshold is too slow for a known live cleanup chain. drain remain the single source of cleanup ownership. Creating a second registry or a durable retry row would duplicate state for a narrow failure. -2. **Use capped exponential delay for every persistence exception.** The outer - retry covers transient PostgreSQL/session failures that the API-key - service's SQLite-lock-specific retry does not classify. A fixed cap prevents - sustained database pressure during a longer outage while still allowing - recovery without waiting for stale sweep. +2. **Use capped exponential delay plus a shared retry gate for every persistence + exception.** The outer retry covers transient PostgreSQL/session failures + that the API-key service's SQLite-lock-specific retry does not classify. A + fixed delay cap prevents each task from retrying rapidly, while a per-service + concurrency gate prevents many failed streams from opening repository + sessions simultaneously. Waiting tasks stay tracked without holding a + database connection. 3. **Rely on reservation transition idempotency.** A retry cannot double decrement quota: release only claims a reservation still in `reserved` @@ -56,8 +58,9 @@ age threshold is too slow for a known live cleanup chain. → Retries use capped backoff; the task accurately represents unfinished cleanup, and stale recovery remains the final repair path. - **Many simultaneous failures could retry together after an outage.** - → Exponential delay bounds retry load; the change adds no inline request-path - work. + → A shared four-attempt gate bounds aggregate repository pressure in each + service instance; exponential delay also bounds each task's retry frequency, + and the change adds no inline request-path work. - **Cancellation can stop a retry after shutdown has already timed out.** → The drain first reports incomplete, so process termination cannot be mistaken for successful settlement. diff --git a/openspec/changes/retry-detached-api-key-release/proposal.md b/openspec/changes/retry-detached-api-key-release/proposal.md index d79b74a45f..7a32890cea 100644 --- a/openspec/changes/retry-detached-api-key-release/proposal.md +++ b/openspec/changes/retry-detached-api-key-release/proposal.md @@ -8,7 +8,8 @@ still consumes quota until stale recovery runs hours later. ## What Changes - Keep a failed detached reservation release tracked and retry it after - transient persistence failures. + transient persistence failures, with a shared concurrency bound on repository + attempts. - Make the persistence drain report completion only after the tracked settlement/release chain has actually terminated. - Add deterministic regression coverage for a finalize failure followed by one diff --git a/openspec/changes/retry-detached-api-key-release/specs/api-keys/spec.md b/openspec/changes/retry-detached-api-key-release/specs/api-keys/spec.md index fd325e06a6..588671d4f0 100644 --- a/openspec/changes/retry-detached-api-key-release/specs/api-keys/spec.md +++ b/openspec/changes/retry-detached-api-key-release/specs/api-keys/spec.md @@ -2,7 +2,7 @@ ### Requirement: Stream reservation settlement is detached from the response path -Settling a stream API-key reservation MUST NOT block the response/stream close, with one deliberate exception: when a keyed websocket stream terminates with an account-health error, the finalizer MUST wait for the settlement to commit before the load-balancer health write (the settlement-ordering invariant), so that error path intentionally blocks on settlement. In all other cases the settlement MUST run as a tracked background task; when it fails or is cancelled, the reservation MUST still be released by the tracking fallback, and the request's finalization path MUST NOT double-release a transferred settlement. If the tracking fallback itself encounters a persistence failure, it MUST remain tracked and retry the idempotent release; persistence drain MUST NOT report completion while that retry remains unfinished. Reservations MUST continue to count toward key limits until finalized or released, so deferred settlement can never admit usage a synchronous settlement would have rejected. +Settling a stream API-key reservation MUST NOT block the response/stream close, with one deliberate exception: when a keyed websocket stream terminates with an account-health error, the finalizer MUST wait for the settlement to commit before the load-balancer health write (the settlement-ordering invariant), so that error path intentionally blocks on settlement. In all other cases the settlement MUST run as a tracked background task; when it fails or is cancelled, the reservation MUST still be released by the tracking fallback, and the request's finalization path MUST NOT double-release a transferred settlement. If the tracking fallback itself encounters a persistence failure, it MUST remain tracked and retry the idempotent release; no more than four retry-enabled detached fallback repository attempts may run concurrently per proxy service instance, waiting fallbacks MUST NOT open repository sessions until admitted, and persistence drain MUST NOT report completion while any retry remains unfinished. Reservations MUST continue to count toward key limits until finalized or released, so deferred settlement can never admit usage a synchronous settlement would have rejected. #### Scenario: Response close precedes settlement completion @@ -25,6 +25,13 @@ Settling a stream API-key reservation MUST NOT block the response/stream close, - **THEN** the tracked fallback retries and releases the reservation exactly once - **AND** persistence drain does not report completion before that release +#### Scenario: Concurrent fallback release retries are bounded to four + +- **GIVEN** five failed detached settlements in one proxy service instance +- **WHEN** their tracking fallbacks attempt repository persistence concurrently +- **THEN** no more than four release attempts open repository sessions +- **AND** the waiting fallbacks remain tracked until they can retry + #### Scenario: Websocket health-error settlement precedes the health write - **GIVEN** a keyed websocket stream that terminates with an account-health error diff --git a/openspec/changes/retry-detached-api-key-release/tasks.md b/openspec/changes/retry-detached-api-key-release/tasks.md index eef9f33cde..4c04aacc2f 100644 --- a/openspec/changes/retry-detached-api-key-release/tasks.md +++ b/openspec/changes/retry-detached-api-key-release/tasks.md @@ -7,9 +7,11 @@ - [x] 2.1 Keep the fallback release tracked while retrying persistence failures with capped backoff. - [x] 2.2 Preserve idempotent settlement, cancellation ownership, and truthful persistence-drain behavior. +- [x] 2.3 Bound concurrent fallback repository attempts to four with one shared per-service gate. ## 3. Verification - [x] 3.1 Run the focused detached-settlement and API-key reservation tests. - [x] 3.2 Run changed-file Ruff, format, type, proxy-architecture, and strict OpenSpec checks. - [x] 3.3 Inspect the final diff and worktree status for scope and unrelated changes. +- [x] 3.4 Add deterministic fan-out coverage for the shared retry concurrency bound. diff --git a/tests/unit/test_proxy_utils.py b/tests/unit/test_proxy_utils.py index e5712e836b..4249387bc4 100644 --- a/tests/unit/test_proxy_utils.py +++ b/tests/unit/test_proxy_utils.py @@ -24994,6 +24994,87 @@ async def release_usage_reservation(self, reservation_id: str) -> None: assert released == ["resv_stream_failed_background"] +@pytest.mark.asyncio +async def test_stream_api_key_release_retries_bound_concurrent_repository_attempts(monkeypatch): + retry_concurrency = proxy_service._STREAM_API_KEY_RELEASE_RETRY_MAX_CONCURRENCY + task_count = retry_concurrency + 1 + active_repository_attempts = 0 + max_active_repository_attempts = 0 + repository_entries = 0 + retry_limit_reached = asyncio.Event() + allow_repository_attempts = asyncio.Event() + released: list[str] = [] + repo = SimpleNamespace(api_keys=object()) + + @asynccontextmanager + async def repo_factory() -> AsyncIterator[SimpleNamespace]: + nonlocal active_repository_attempts, max_active_repository_attempts, repository_entries + active_repository_attempts += 1 + repository_entries += 1 + max_active_repository_attempts = max( + max_active_repository_attempts, + active_repository_attempts, + ) + if active_repository_attempts == retry_concurrency: + retry_limit_reached.set() + try: + await allow_repository_attempts.wait() + yield repo + finally: + active_repository_attempts -= 1 + + class FakeApiKeysService: + def __init__(self, api_keys_repository: object) -> None: + assert api_keys_repository is repo.api_keys + + async def release_usage_reservation(self, reservation_id: str) -> None: + released.append(reservation_id) + + monkeypatch.setattr(proxy_service, "ApiKeysService", FakeApiKeysService) + + service = proxy_service.ProxyService(cast(proxy_service.ProxyRepoFactory, repo_factory)) + api_key = _make_api_key_data("key_stream_release_retry_bound") + reservations = [ + proxy_service.ApiKeyUsageReservationData( + reservation_id=f"resv_stream_release_retry_bound_{index}", + key_id=api_key.id, + model="gpt-5.5", + ) + for index in range(task_count) + ] + for index, reservation in enumerate(reservations): + service._schedule_cancel_safe_cleanup( + service._release_unsettled_stream_api_key_usage( + api_key=api_key, + api_key_reservation=reservation, + request_id=f"req_stream_release_retry_bound_{index}", + retry_persistence_failures=True, + ), + action="release_stream_api_key_reservation_after_failed_settlement", + request_id=f"req_stream_release_retry_bound_{index}", + ) + + drain_task: asyncio.Task[bool] | None = None + try: + await asyncio.wait_for(retry_limit_reached.wait(), timeout=1) + await asyncio.sleep(0) + assert repository_entries == retry_concurrency + assert active_repository_attempts == retry_concurrency + assert len(service._background_cleanup_tasks) == task_count + drain_task = asyncio.create_task(service.drain_persistence_tasks(timeout_seconds=2)) + await asyncio.sleep(0) + assert not drain_task.done() + finally: + allow_repository_attempts.set() + if drain_task is None: + drain_task = asyncio.create_task(service.drain_persistence_tasks(timeout_seconds=2)) + assert await drain_task + + assert max_active_repository_attempts == retry_concurrency + assert sorted(released) == sorted(reservation.reservation_id for reservation in reservations) + assert service._background_cleanup_tasks == set() + + @pytest.mark.asyncio async def test_stream_with_retry_skips_release_after_settlement_transfers_on_cancel(monkeypatch): settings = _make_proxy_settings() @@ -27387,6 +27468,9 @@ async def __aexit__(self, exc_type, exc, tb) -> bool: return False service = proxy_service.ProxyService(lambda: _RepoContextWithApiKeys()) + # The synchronous stream-finally backstop must not queue behind detached + # retries, even when every retry slot is occupied. + service._stream_api_key_release_retry_semaphore = asyncio.Semaphore(0) settings = _make_proxy_settings() monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache(settings)) monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) @@ -27433,18 +27517,19 @@ async def __aexit__(self, exc_type, exc, tb) -> bool: monkeypatch.setattr(service, "_select_account_with_budget", select_account) with pytest.raises(proxy_module.ProxyResponseError) as exc_info: - async for _ in service._stream_with_retry( - payload, - {}, - codex_session_affinity=False, - propagate_http_errors=False, - openai_cache_affinity=False, - api_key=api_key, - api_key_reservation=reservation, - suppress_text_done_events=False, - request_transport="http", - ): - pass + async with asyncio.timeout(1): + async for _ in service._stream_with_retry( + payload, + {}, + codex_session_affinity=False, + propagate_http_errors=False, + openai_cache_affinity=False, + api_key=api_key, + api_key_reservation=reservation, + suppress_text_done_events=False, + request_transport="http", + ): + pass assert _proxy_error_code(exc_info.value) == "upstream_unavailable" owner_lookup.assert_awaited_once() From acbf7a4326e2c702b26e365d41cf3825f0046bbd Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sat, 1 Aug 2026 01:42:59 +0400 Subject: [PATCH 53/64] fix(proxy): honor usage exhaustion boundaries --- app/core/balancer/logic.py | 4 +- .../proxy/_load_balancer/sticky_selection.py | 4 + app/modules/proxy/_service/streaming/retry.py | 1 + tests/unit/test_load_balancer.py | 77 ++++++++++++++++++- 4 files changed, 81 insertions(+), 5 deletions(-) diff --git a/app/core/balancer/logic.py b/app/core/balancer/logic.py index 6368560ad4..63303f19a6 100644 --- a/app/core/balancer/logic.py +++ b/app/core/balancer/logic.py @@ -188,8 +188,8 @@ def _usage_exhausted_reset_at(state: AccountState) -> float | None: candidates: list[float] = [] primary_evidence = _primary_usage_evidence(state) secondary_evidence = _secondary_usage_evidence(state) - if primary_evidence is not None and float(primary_evidence) >= 100.0 and state.reset_at is not None: - candidates.append(float(state.reset_at)) + if primary_evidence is not None and float(primary_evidence) >= 100.0 and state.primary_reset_at is not None: + candidates.append(float(state.primary_reset_at)) if ( secondary_evidence is not None and float(secondary_evidence) >= 100.0 diff --git a/app/modules/proxy/_load_balancer/sticky_selection.py b/app/modules/proxy/_load_balancer/sticky_selection.py index 3aee72fcc8..1af2e3c9ca 100644 --- a/app/modules/proxy/_load_balancer/sticky_selection.py +++ b/app/modules/proxy/_load_balancer/sticky_selection.py @@ -440,6 +440,8 @@ def _direct_error( traffic_class=traffic_class, ignore_standard_quota=False, routing_costs_by_account_id=effective_routing_costs, + allow_usage_exhaustion_error=allow_usage_exhaustion_error, + usage_exhaustion_states=states, ) if result.account is None: selection_error_code = "hard_affinity_saturated" @@ -1478,6 +1480,8 @@ def _select_account_preferring_budget_safe( traffic_class=traffic_class, ignore_standard_quota=ignore_standard_quota, routing_costs=routing_costs_by_account_id, + allow_usage_exhaustion_error=allow_usage_exhaustion_error, + usage_exhaustion_states=usage_exhaustion_states, ) diff --git a/app/modules/proxy/_service/streaming/retry.py b/app/modules/proxy/_service/streaming/retry.py index 8b24aad479..4212b9dea7 100644 --- a/app/modules/proxy/_service/streaming/retry.py +++ b/app/modules/proxy/_service/streaming/retry.py @@ -1109,6 +1109,7 @@ async def _retry_account_model_rejection( ) return if selection.error_code == USAGE_LIMIT_REACHED: + await _drain_pending_post_refresh_penalty_on_terminal(settlement) no_accounts_msg = selection.error_message or "Usage limit reached" status_code, error_payload = selection_failure_response(selection) await proxy._write_request_log( diff --git a/tests/unit/test_load_balancer.py b/tests/unit/test_load_balancer.py index 1a1a2b3c67..f6b8c12662 100644 --- a/tests/unit/test_load_balancer.py +++ b/tests/unit/test_load_balancer.py @@ -679,8 +679,20 @@ def test_select_account_skips_rate_limited_until_reset(): def test_select_account_reports_pool_wide_usage_exhaustion_structurally(): now = 1_700_000_000.0 states = [ - AccountState("a", AccountStatus.RATE_LIMITED, used_percent=100.0, reset_at=int(now + 60)), - AccountState("b", AccountStatus.QUOTA_EXCEEDED, used_percent=100.0, reset_at=int(now + 3600)), + AccountState( + "a", + AccountStatus.RATE_LIMITED, + used_percent=100.0, + reset_at=int(now + 600), + primary_reset_at=int(now + 60), + ), + AccountState( + "b", + AccountStatus.QUOTA_EXCEEDED, + used_percent=100.0, + reset_at=int(now + 7200), + primary_reset_at=int(now + 3600), + ), AccountState("paused", AccountStatus.PAUSED), ] @@ -728,6 +740,26 @@ def test_select_account_reports_secondary_usage_exhaustion_reset(): assert result.resets_at == int(now + 3600) +def test_select_account_omits_synthesized_primary_usage_reset(): + now = 1_700_000_000.0 + states = [ + AccountState( + "a", + AccountStatus.RATE_LIMITED, + used_percent=100.0, + reset_at=int(now + 60), + primary_reset_at=None, + ) + ] + + result = select_account(states, now=now) + + assert result.account is None + assert result.error_code == "usage_limit_reached" + assert result.error_message == "Usage limit reached" + assert result.resets_at is None + + def test_select_account_waits_for_latest_exhausted_window_per_account(): now = 1_700_000_000.0 states = [ @@ -736,7 +768,8 @@ def test_select_account_waits_for_latest_exhausted_window_per_account(): AccountStatus.RATE_LIMITED, used_percent=100.0, secondary_used_percent=100.0, - reset_at=int(now + 60), + reset_at=int(now + 600), + primary_reset_at=int(now + 60), secondary_reset_at=int(now + 3600), ), AccountState( @@ -744,6 +777,7 @@ def test_select_account_waits_for_latest_exhausted_window_per_account(): AccountStatus.QUOTA_EXCEEDED, used_percent=100.0, reset_at=int(now + 7200), + primary_reset_at=int(now + 7200), ), ] @@ -819,6 +853,40 @@ def test_budget_safe_selection_uses_full_scope_for_usage_exhaustion() -> None: assert result.error_message == "Rate limit exceeded. Try again in 300s" +def test_budget_safe_capacity_selection_forwards_usage_exhaustion_controls() -> None: + now = time.time() + owner_scope = [ + AccountState( + "owner", + AccountStatus.QUOTA_EXCEEDED, + used_percent=100.0, + reset_at=int(now + 600), + primary_reset_at=int(now + 3600), + ) + ] + full_scope = [ + *owner_scope, + AccountState( + "pool-usable", + AccountStatus.ACTIVE, + used_percent=10.0, + ), + ] + + result = _select_account_preferring_budget_safe( + owner_scope, + prefer_earlier_reset=False, + routing_strategy="capacity_weighted", + budget_threshold_pct=95.0, + allow_usage_exhaustion_error=False, + usage_exhaustion_states=full_scope, + ) + + assert result.account is None + assert result.error_code is None + assert result.error_message == "Rate limit exceeded. Try again in 300s" + + def test_opportunistic_budget_safe_selection_uses_full_scope_for_usage_exhaustion() -> None: now = time.time() cap_filtered_states = [ @@ -1560,12 +1628,14 @@ def test_select_account_caps_quota_exceeded_retry_hint(): AccountStatus.QUOTA_EXCEEDED, used_percent=100.0, reset_at=far_future_reset, + primary_reset_at=far_future_reset, ), AccountState( "b", AccountStatus.QUOTA_EXCEEDED, used_percent=100.0, reset_at=int(now + 271_819), + primary_reset_at=int(now + 271_819), ), ] result = select_account(states, now=now) @@ -1584,6 +1654,7 @@ def test_select_account_preserves_short_quota_exceeded_retry_hint(): AccountStatus.QUOTA_EXCEEDED, used_percent=100.0, reset_at=int(now + 60), + primary_reset_at=int(now + 60), ), ] result = select_account(states, now=now) From f759cae87953997b4060c1d0a381de8ce05f66f4 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Wed, 22 Jul 2026 19:23:55 +0400 Subject: [PATCH 54/64] fix(compact): elide observed inline images at wire cap --- app/core/openai/requests.py | 85 ++++++++++++ .../proposal.md | 22 +++ .../specs/responses-api-compat/spec.md | 21 +++ .../tasks.md | 11 ++ tests/integration/test_proxy_responses.py | 36 ++++- tests/unit/test_openai_requests.py | 126 ++++++++++++++++++ 6 files changed, 297 insertions(+), 4 deletions(-) create mode 100644 openspec/changes/allow-compact-inline-image-elision/proposal.md create mode 100644 openspec/changes/allow-compact-inline-image-elision/specs/responses-api-compat/spec.md create mode 100644 openspec/changes/allow-compact-inline-image-elision/tasks.md diff --git a/app/core/openai/requests.py b/app/core/openai/requests.py index 983817c55f..8e3a6c949f 100644 --- a/app/core/openai/requests.py +++ b/app/core/openai/requests.py @@ -795,6 +795,9 @@ def to_payload(self) -> JsonObject: _MAX_COMPACT_UPSTREAM_ESTIMATED_TOKENS = 100_000 _COMPACT_UPSTREAM_HEAD_ESTIMATED_TOKENS = 12_000 _ESTIMATED_CHARS_PER_TOKEN = 4 +_COMPACT_OMITTED_INLINE_IMAGE_TEXT = ( + "[compact trim] Omitted inline image bytes that were already observed before compaction" +) def _strip_unsupported_fields(payload: MutableJsonObject) -> MutableJsonObject: @@ -961,6 +964,29 @@ def _trim_compact_input_for_upstream(payload: MutableJsonObject) -> None: ) required_input = _compact_trimmed_input_with_markers(input_value, token_counts, required_indices) required_tokens = _estimated_json_tokens(required_input) + if required_tokens > _MAX_COMPACT_UPSTREAM_ESTIMATED_TOKENS: + rewritten_input, images_elided = _compact_elide_required_tool_output_images( + input_value, + required_indices=required_indices, + ) + if images_elided: + input_value = rewritten_input + payload["input"] = input_value + token_counts = [_estimated_json_array_item_tokens(item) for item in input_value] + head_count = _compact_trim_prefix_count(token_counts) + preserved_indices = _compact_state_anchor_indices(input_value) + required_indices = set(preserved_indices) + if input_value: + required_indices.add(len(input_value) - 1) + required_indices = _compact_reconciled_tool_call_indices( + input_value, + required_indices, + token_counts=token_counts, + token_budget=sum(token_counts), + required_indices=required_indices, + ) + required_input = _compact_trimmed_input_with_markers(input_value, token_counts, required_indices) + required_tokens = _estimated_json_tokens(required_input) if required_tokens > _MAX_COMPACT_UPSTREAM_ESTIMATED_TOKENS: raise ClientPayloadError( "Compact input exceeds the upstream size limit and cannot be trimmed " @@ -1021,6 +1047,65 @@ def _trim_compact_input_for_upstream(payload: MutableJsonObject) -> None: payload["input"] = trimmed_input +def _compact_elide_required_tool_output_images( + input_value: list[JsonValue], + *, + required_indices: set[int], +) -> tuple[list[JsonValue], bool]: + rewritten: list[JsonValue] = [] + changed = False + for index, item in enumerate(input_value): + if ( + index in required_indices + and is_json_mapping(item) + and item.get("type") in _COMPACT_TOOL_CALL_OUTPUT_ITEM_TYPES + ): + rewritten_item, item_changed = _compact_elide_inline_images(item) + rewritten.append(rewritten_item) + changed = changed or item_changed + else: + rewritten.append(item) + return rewritten, changed + + +def _compact_elide_inline_images(value: JsonValue) -> tuple[JsonValue, bool]: + """Replace inline image bytes with an explicit compact-only text marker. + + The model has already observed these images during the live turn. Re-sending + their data URLs to the compact endpoint can make an otherwise recoverable + thread permanently uncompactable, especially when the latest required tool + output contains a screenshot. File-backed image references remain intact. + """ + + if is_json_mapping(value): + if value.get("type") == "input_image": + image_url = value.get("image_url") + if isinstance(image_url, str) and image_url.startswith("data:image/"): + return ( + { + "type": "input_text", + "text": f"{_COMPACT_OMITTED_INLINE_IMAGE_TEXT} ({len(image_url)} encoded characters).", + }, + True, + ) + rewritten_mapping: JsonObject = {} + changed = False + for key, item in value.items(): + rewritten_item, item_changed = _compact_elide_inline_images(item) + rewritten_mapping[key] = rewritten_item + changed = changed or item_changed + return rewritten_mapping, changed + if is_json_list(value): + rewritten: list[JsonValue] = [] + changed = False + for item in value: + rewritten_item, item_changed = _compact_elide_inline_images(item) + rewritten.append(rewritten_item) + changed = changed or item_changed + return rewritten, changed + return value, False + + def _compact_fit_selected_indices_to_wire_budget( input_value: list[JsonValue], token_counts: list[int], diff --git a/openspec/changes/allow-compact-inline-image-elision/proposal.md b/openspec/changes/allow-compact-inline-image-elision/proposal.md new file mode 100644 index 0000000000..78e778e640 --- /dev/null +++ b/openspec/changes/allow-compact-inline-image-elision/proposal.md @@ -0,0 +1,22 @@ +## Why + +Long-running image-heavy Codex sessions can reach terminal compaction with a +required latest tool output containing a large inline data-URL image. The model +already observed the image during the live turn, but the compact request must +retain the latest tool call/output pair. Retaining the raw image bytes can exceed +the compact wire cap and permanently terminate an otherwise recoverable thread +with `responses_compact_input_too_large`. + +## What Changes + +- Replace inline data-URL image parts with an explicit textual omission marker + only while preparing an oversized compact request. +- Preserve the surrounding tool call/output identities and all textual content. +- Leave file-backed image references and ordinary non-compact requests unchanged. +- Keep fail-closed behavior for required oversized non-image content. + +## Impact + +- Affected spec: `responses-api-compat`. +- Affected code: compact request preparation in `app/core/openai/requests.py`. +- No schema, account-selection, or normal Responses wire behavior changes. diff --git a/openspec/changes/allow-compact-inline-image-elision/specs/responses-api-compat/spec.md b/openspec/changes/allow-compact-inline-image-elision/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..186d3c98f7 --- /dev/null +++ b/openspec/changes/allow-compact-inline-image-elision/specs/responses-api-compat/spec.md @@ -0,0 +1,21 @@ +## MODIFIED Requirements + +### Requirement: Responses Lite follow-up transformations fail closed + +After a request is classified as Responses Lite shaped, the service MUST +preserve required Lite state through compact preparation, MUST validate the +final transformed compact input against the upstream JSON wire budget, and MUST +avoid permanently poisoning a thread when already-observed inline image bytes +alone make a required latest tool result too large. The image-byte relaxation +MUST NOT weaken fail-closed handling for oversized textual state. + +#### Scenario: Oversized inline image does not poison terminal compaction + +- **WHEN** compact input exceeds the upstream limit because a required latest + tool output contains an inline data-URL image that the model already observed +- **THEN** compact preparation retains the tool call and output identities +- **AND** replaces only the inline image bytes with an explicit textual omission marker +- **AND** preserves the other textual parts of the tool output +- **AND** file-backed image references remain unchanged +- **AND** non-image required content that cannot fit still returns + `responses_compact_input_too_large` diff --git a/openspec/changes/allow-compact-inline-image-elision/tasks.md b/openspec/changes/allow-compact-inline-image-elision/tasks.md new file mode 100644 index 0000000000..631df461cc --- /dev/null +++ b/openspec/changes/allow-compact-inline-image-elision/tasks.md @@ -0,0 +1,11 @@ +## 1. Compact preparation + +- [x] 1.1 Elide inline data-URL image bytes only when compact input is oversized. +- [x] 1.2 Preserve tool call/output identity, text parts, and file-backed images. +- [x] 1.3 Retain fail-closed handling for oversized required non-image content. + +## 2. Validation + +- [x] 2.1 Add the exact required-latest-tool-output regression. +- [x] 2.2 Add a negative control for file-backed image references. +- [x] 2.3 Prove the terminal compact route; retain live helper-deployed smoke as rollout evidence. diff --git a/tests/integration/test_proxy_responses.py b/tests/integration/test_proxy_responses.py index 087ffe6665..001c876133 100644 --- a/tests/integration/test_proxy_responses.py +++ b/tests/integration/test_proxy_responses.py @@ -531,7 +531,10 @@ async def fake_stream(payload, headers, access_token, account_id, base_url=None, @pytest.mark.asyncio -async def test_proxy_responses_compaction_trigger_streams_single_compaction_item(async_client, monkeypatch): +async def test_proxy_responses_compaction_trigger_elides_required_tool_image_and_streams_item( + async_client, + monkeypatch, +): email = "compact-trigger@example.com" raw_account_id = "acc_compact_trigger" auth_json = _make_auth_json(raw_account_id, email) @@ -577,8 +580,9 @@ async def fake_select_account(self, deadline: float, **kwargs): async def fake_compact(payload, headers, access_token, account_id, **kwargs): del headers, access_token, kwargs - seen_payload["payload"] = payload.model_dump(mode="json", exclude_none=True) - seen_payload["input"] = payload.input + wire_payload = payload.to_payload() + seen_payload["payload"] = wire_payload + seen_payload["input"] = wire_payload["input"] seen_payload["model"] = payload.model seen_payload["previous_response_id"] = getattr(payload, "previous_response_id", None) seen_payload["conversation"] = getattr(payload, "conversation", None) @@ -603,6 +607,23 @@ async def fake_compact(payload, headers, access_token, account_id, **kwargs): "instructions": "compact this turn", "input": [ {"role": "user", "content": "hello"}, + { + "type": "custom_tool_call", + "name": "view_image", + "call_id": "call_route_image", + "input": "{}", + }, + { + "type": "custom_tool_call_output", + "call_id": "call_route_image", + "output": [ + {"type": "input_text", "text": "Image Size: 1512x982."}, + { + "type": "input_image", + "image_url": "data:image/png;base64," + "A" * 500_000, + }, + ], + }, {"type": "compaction_trigger"}, ], "previous_response_id": "resp_compact_anchor", @@ -623,7 +644,14 @@ async def fake_compact(payload, headers, access_token, account_id, **kwargs): assert [event["type"] for event in events] == ["response.output_item.done", "response.completed"] assert selection_preferred_ids == [owner_account.id] assert seen_payload["model"] == "gpt-5.1" - assert seen_payload["input"] == [{"role": "user", "content": "hello"}] + compact_input = cast(list[Mapping[str, object]], seen_payload["input"]) + assert compact_input[0] == {"role": "user", "content": "hello"} + assert compact_input[1]["call_id"] == "call_route_image" + assert compact_input[2]["call_id"] == "call_route_image" + compact_input_json = json.dumps(compact_input) + assert "Image Size: 1512x982." in compact_input_json + assert "Omitted inline image bytes that were already observed before compaction" in compact_input_json + assert "data:image/png;base64" not in compact_input_json assert seen_payload["previous_response_id"] == "resp_compact_anchor" assert seen_payload["account_id"] == raw_account_id compact_payload = cast(Mapping[str, object], seen_payload["payload"]) diff --git a/tests/unit/test_openai_requests.py b/tests/unit/test_openai_requests.py index a98c9617a4..7cc11428d9 100644 --- a/tests/unit/test_openai_requests.py +++ b/tests/unit/test_openai_requests.py @@ -1375,6 +1375,132 @@ def test_compact_trimming_rejects_oversized_latest_item(): assert raised.value.code == "responses_compact_input_too_large" +def test_compact_trimming_elides_inline_image_from_required_latest_tool_output(): + latest_call = { + "type": "custom_tool_call", + "name": "view_image", + "call_id": "call-latest-image", + "input": "{}", + } + latest_output = { + "type": "custom_tool_call_output", + "call_id": "call-latest-image", + "output": [ + {"type": "input_text", "text": "Image Size: 1512x982."}, + { + "type": "input_image", + "detail": "original", + "image_url": "data:image/png;base64," + "A" * 500_000, + }, + ], + } + payload = { + "model": "gpt-5.6-sol", + "instructions": "", + "input": [ + {"role": "user", "content": "inspect the canvas"}, + latest_call, + latest_output, + ], + } + + dumped_input = ResponsesCompactRequest.model_validate(payload).to_payload()["input"] + + assert isinstance(dumped_input, list) + assert latest_call in dumped_input + dumped_output = next( + item for item in dumped_input if isinstance(item, dict) and item.get("type") == "custom_tool_call_output" + ) + assert dumped_output["output"] == [ + {"type": "input_text", "text": "Image Size: 1512x982."}, + { + "type": "input_text", + "text": ( + "[compact trim] Omitted inline image bytes that were already observed before compaction " + "(500022 encoded characters)." + ), + }, + ] + assert "data:image/png;base64" not in json.dumps(dumped_input) + wire_bytes = len(json.dumps(dumped_input, ensure_ascii=True, sort_keys=True).encode("utf-8")) + assert wire_bytes <= _MAX_COMPACT_UPSTREAM_ESTIMATED_TOKENS * _ESTIMATED_CHARS_PER_TOKEN + + +def test_compact_trimming_elides_mapping_shaped_required_tool_image_output(): + payload = { + "model": "gpt-5.6-sol", + "instructions": "", + "input": [ + { + "type": "custom_tool_call", + "name": "view_image", + "call_id": "call-mapping-image", + "input": "{}", + }, + { + "type": "custom_tool_call_output", + "call_id": "call-mapping-image", + "output": { + "type": "input_image", + "image_url": "data:image/png;base64," + "A" * 500_000, + }, + }, + ], + } + + dumped_input = ResponsesCompactRequest.model_validate(payload).to_payload()["input"] + + assert isinstance(dumped_input, list) + dumped_output = cast(Mapping[str, object], dumped_input[-1]) + assert dumped_output["output"] == { + "type": "input_text", + "text": ( + "[compact trim] Omitted inline image bytes that were already observed before compaction " + "(500022 encoded characters)." + ), + } + + +def test_compact_trimming_keeps_file_backed_image_reference(): + file_image = {"type": "input_image", "file_id": "file-canvas"} + payload = { + "model": "gpt-5.6-sol", + "instructions": "", + "input": [ + {"role": "assistant", "content": "x" * 500_000}, + {"role": "user", "content": [file_image]}, + ], + } + + dumped_input = ResponsesCompactRequest.model_validate(payload).to_payload()["input"] + + assert isinstance(dumped_input, list) + latest_item = cast(Mapping[str, object], dumped_input[-1]) + assert file_image in cast(list[object], latest_item["content"]) + + +def test_compact_trimming_keeps_latest_unobserved_user_inline_image(): + latest_image = { + "type": "input_image", + "image_url": "data:image/png;base64,AAAA", + } + payload = { + "model": "gpt-5.6-sol", + "instructions": "", + "input": [ + {"role": "assistant", "content": "x" * 500_000}, + {"role": "user", "content": [latest_image]}, + ], + } + + dumped_input = ResponsesCompactRequest.model_validate(payload).to_payload()["input"] + + assert isinstance(dumped_input, list) + latest_item = cast(Mapping[str, object], dumped_input[-1]) + assert latest_image in cast(list[object], latest_item["content"]) + assert "Omitted inline image bytes" not in json.dumps(dumped_input) + + def test_compact_trimming_preserves_latest_unmatched_tool_call(): latest_call = { "type": "function_call", From 05dfdd6ea5dacf1d052865c27b4573a8426e40b1 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Wed, 22 Jul 2026 21:52:57 +0400 Subject: [PATCH 55/64] fix(compact): handle eligible inline tool images safely Signed-off-by: Darafei Praliaskouski --- app/core/openai/requests.py | 11 +++ .../proposal.md | 9 ++- .../specs/responses-api-compat/spec.md | 6 +- .../tasks.md | 5 +- tests/unit/test_openai_requests.py | 71 +++++++++++++++++-- 5 files changed, 90 insertions(+), 12 deletions(-) diff --git a/app/core/openai/requests.py b/app/core/openai/requests.py index 8e3a6c949f..d5898f17b2 100644 --- a/app/core/openai/requests.py +++ b/app/core/openai/requests.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import re from collections.abc import Iterable, Mapping from dataclasses import dataclass from typing import cast @@ -45,6 +46,7 @@ _COMPACT_TOOL_CALL_OUTPUT_ITEM_TYPES = frozenset( {"function_call_output", "custom_tool_call_output", "apply_patch_call_output"} ) +_COMPACT_INLINE_IMAGE_DATA_URL_RE = re.compile(r"data:image/[^,\s]+,[A-Za-z0-9+/=_-]+") _GOAL_CONTINUATION_CONTEXT_PREFIX = '' _PLAN_MODE_CONTEXT_PREFIX = "# Plan Mode" @@ -1103,6 +1105,15 @@ def _compact_elide_inline_images(value: JsonValue) -> tuple[JsonValue, bool]: rewritten.append(rewritten_item) changed = changed or item_changed return rewritten, changed + if isinstance(value, str) and "data:image/" in value: + rewritten_value, replacements = _COMPACT_INLINE_IMAGE_DATA_URL_RE.subn( + lambda match: ( + f"{_COMPACT_OMITTED_INLINE_IMAGE_TEXT} ({len(match.group(0))} encoded characters)." + ), + value, + ) + if replacements: + return rewritten_value, True return value, False diff --git a/openspec/changes/allow-compact-inline-image-elision/proposal.md b/openspec/changes/allow-compact-inline-image-elision/proposal.md index 78e778e640..de392cce0a 100644 --- a/openspec/changes/allow-compact-inline-image-elision/proposal.md +++ b/openspec/changes/allow-compact-inline-image-elision/proposal.md @@ -9,10 +9,13 @@ with `responses_compact_input_too_large`. ## What Changes -- Replace inline data-URL image parts with an explicit textual omission marker - only while preparing an oversized compact request. +- Replace inline data-URL images inside required function, custom, and + apply-patch tool outputs with an explicit textual omission marker only while + preparing an oversized compact request. - Preserve the surrounding tool call/output identities and all textual content. -- Leave file-backed image references and ordinary non-compact requests unchanged. +- Leave accepted `input_file` references and ordinary non-compact requests unchanged. +- Keep hosted `computer_call_output` screenshots fail-closed until a + schema-valid compact placeholder is defined. - Keep fail-closed behavior for required oversized non-image content. ## Impact diff --git a/openspec/changes/allow-compact-inline-image-elision/specs/responses-api-compat/spec.md b/openspec/changes/allow-compact-inline-image-elision/specs/responses-api-compat/spec.md index 186d3c98f7..1979df1170 100644 --- a/openspec/changes/allow-compact-inline-image-elision/specs/responses-api-compat/spec.md +++ b/openspec/changes/allow-compact-inline-image-elision/specs/responses-api-compat/spec.md @@ -12,10 +12,12 @@ MUST NOT weaken fail-closed handling for oversized textual state. #### Scenario: Oversized inline image does not poison terminal compaction - **WHEN** compact input exceeds the upstream limit because a required latest - tool output contains an inline data-URL image that the model already observed + eligible required tool output contains an inline data-URL image that the model already observed - **THEN** compact preparation retains the tool call and output identities - **AND** replaces only the inline image bytes with an explicit textual omission marker - **AND** preserves the other textual parts of the tool output -- **AND** file-backed image references remain unchanged +- **AND** accepted file-backed `input_file` references remain unchanged +- **AND** hosted `computer_call_output` screenshots remain fail-closed until a + schema-valid compact placeholder is defined - **AND** non-image required content that cannot fit still returns `responses_compact_input_too_large` diff --git a/openspec/changes/allow-compact-inline-image-elision/tasks.md b/openspec/changes/allow-compact-inline-image-elision/tasks.md index 631df461cc..7d8c3cbf22 100644 --- a/openspec/changes/allow-compact-inline-image-elision/tasks.md +++ b/openspec/changes/allow-compact-inline-image-elision/tasks.md @@ -1,11 +1,12 @@ ## 1. Compact preparation - [x] 1.1 Elide inline data-URL image bytes only when compact input is oversized. -- [x] 1.2 Preserve tool call/output identity, text parts, and file-backed images. +- [x] 1.2 Preserve tool call/output identity, text parts, and accepted `input_file` references. - [x] 1.3 Retain fail-closed handling for oversized required non-image content. +- [x] 1.4 Retain fail-closed handling for hosted computer screenshots whose schema cannot carry a text marker. ## 2. Validation - [x] 2.1 Add the exact required-latest-tool-output regression. -- [x] 2.2 Add a negative control for file-backed image references. +- [x] 2.2 Add negative controls for accepted `input_file` references and hosted computer screenshots. - [x] 2.3 Prove the terminal compact route; retain live helper-deployed smoke as rollout evidence. diff --git a/tests/unit/test_openai_requests.py b/tests/unit/test_openai_requests.py index 7cc11428d9..47f070bae7 100644 --- a/tests/unit/test_openai_requests.py +++ b/tests/unit/test_openai_requests.py @@ -1461,14 +1461,21 @@ def test_compact_trimming_elides_mapping_shaped_required_tool_image_output(): } -def test_compact_trimming_keeps_file_backed_image_reference(): - file_image = {"type": "input_image", "file_id": "file-canvas"} +def test_compact_trimming_keeps_accepted_file_reference_while_eliding_inline_image(): + file_reference = {"type": "input_file", "file_id": "file-canvas"} payload = { "model": "gpt-5.6-sol", "instructions": "", "input": [ - {"role": "assistant", "content": "x" * 500_000}, - {"role": "user", "content": [file_image]}, + {"type": "custom_tool_call", "name": "view_image", "call_id": "call-file", "input": "{}"}, + { + "type": "custom_tool_call_output", + "call_id": "call-file", + "output": [ + file_reference, + {"type": "input_image", "image_url": "data:image/png;base64," + "A" * 500_000}, + ], + }, ], } @@ -1476,7 +1483,61 @@ def test_compact_trimming_keeps_file_backed_image_reference(): assert isinstance(dumped_input, list) latest_item = cast(Mapping[str, object], dumped_input[-1]) - assert file_image in cast(list[object], latest_item["content"]) + assert file_reference in cast(list[object], latest_item["output"]) + assert "data:image/png;base64" not in json.dumps(dumped_input) + + +def test_compact_trimming_keeps_hosted_computer_screenshot_fail_closed(): + payload = { + "model": "gpt-5.6-sol", + "instructions": "", + "input": [ + { + "type": "computer_call", + "call_id": "call-computer", + "action": {"type": "screenshot"}, + }, + { + "type": "computer_call_output", + "call_id": "call-computer", + "output": { + "type": "computer_screenshot", + "image_url": "data:image/png;base64," + "A" * 500_000, + }, + }, + ], + } + + with pytest.raises(ClientPayloadError) as raised: + ResponsesCompactRequest.model_validate(payload).to_payload() + + assert raised.value.code == "responses_compact_input_too_large" + assert raised.value.param == "input" + + +def test_compact_trimming_elides_data_url_inside_string_tool_output(): + payload = { + "model": "gpt-5.6-sol", + "instructions": "", + "input": [ + {"type": "function_call", "name": "capture", "call_id": "call-string", "arguments": "{}"}, + { + "type": "function_call_output", + "call_id": "call-string", + "output": "prefix data:image/png;base64," + "A" * 500_000 + " suffix", + }, + ], + } + + dumped_input = ResponsesCompactRequest.model_validate(payload).to_payload()["input"] + + assert isinstance(dumped_input, list) + dumped_output = cast(Mapping[str, object], dumped_input[-1]) + output = cast(str, dumped_output["output"]) + assert output.startswith("prefix ") + assert output.endswith(" suffix") + assert "Omitted inline image bytes" in output + assert "data:image/png;base64" not in output def test_compact_trimming_keeps_latest_unobserved_user_inline_image(): From 2e93bb2d50251837395868ed9d3a28b802efa06f Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Wed, 22 Jul 2026 22:04:28 +0400 Subject: [PATCH 56/64] style(compact): format request trimming helpers --- app/core/openai/requests.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/app/core/openai/requests.py b/app/core/openai/requests.py index d5898f17b2..b739eb2806 100644 --- a/app/core/openai/requests.py +++ b/app/core/openai/requests.py @@ -1107,9 +1107,7 @@ def _compact_elide_inline_images(value: JsonValue) -> tuple[JsonValue, bool]: return rewritten, changed if isinstance(value, str) and "data:image/" in value: rewritten_value, replacements = _COMPACT_INLINE_IMAGE_DATA_URL_RE.subn( - lambda match: ( - f"{_COMPACT_OMITTED_INLINE_IMAGE_TEXT} ({len(match.group(0))} encoded characters)." - ), + lambda match: f"{_COMPACT_OMITTED_INLINE_IMAGE_TEXT} ({len(match.group(0))} encoded characters).", value, ) if replacements: From 2178dec4944eb519de1fe5592a727251b3a7f148 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Wed, 22 Jul 2026 22:16:37 +0400 Subject: [PATCH 57/64] fix(compact): preserve context and nested file pins --- app/core/openai/requests.py | 80 +++++++++++++----------------- tests/unit/test_openai_requests.py | 50 ++++++++++++++++++- 2 files changed, 84 insertions(+), 46 deletions(-) diff --git a/app/core/openai/requests.py b/app/core/openai/requests.py index b739eb2806..6ba7a132f5 100644 --- a/app/core/openai/requests.py +++ b/app/core/openai/requests.py @@ -46,7 +46,7 @@ _COMPACT_TOOL_CALL_OUTPUT_ITEM_TYPES = frozenset( {"function_call_output", "custom_tool_call_output", "apply_patch_call_output"} ) -_COMPACT_INLINE_IMAGE_DATA_URL_RE = re.compile(r"data:image/[^,\s]+,[A-Za-z0-9+/=_-]+") +_COMPACT_INLINE_IMAGE_DATA_URL_RE = re.compile(r"""data:image/[^,\s]+,[^\s"'<>]+""") _GOAL_CONTINUATION_CONTEXT_PREFIX = '' _PLAN_MODE_CONTEXT_PREFIX = "# Plan Mode" @@ -165,34 +165,23 @@ def extract_input_file_ids(input_value: JsonValue) -> set[str]: if not is_json_list(input_value): return set() file_ids: set[str] = set() - for item in input_value: - if not is_json_mapping(item): - continue - item_mapping = item - if _is_input_file_with_id(item_mapping): - file_id = item_mapping.get("file_id") - if isinstance(file_id, str) and file_id: - file_ids.add(file_id) - image_file_id = _input_image_file_reference(item_mapping) - if image_file_id is not None: - file_ids.add(image_file_id) - content = item_mapping.get("content") - if is_json_list(content): - parts: list[JsonValue] = content - elif is_json_mapping(content): - parts = [content] - else: - parts = [] - for part in parts: - if not is_json_mapping(part): - continue - if _is_input_file_with_id(part): - file_id = part.get("file_id") + + def collect(value: JsonValue) -> None: + if is_json_mapping(value): + if _is_input_file_with_id(value): + file_id = value.get("file_id") if isinstance(file_id, str) and file_id: file_ids.add(file_id) - image_file_id = _input_image_file_reference(part) + image_file_id = _input_image_file_reference(value) if image_file_id is not None: file_ids.add(image_file_id) + for child in value.values(): + collect(child) + elif is_json_list(value): + for child in value: + collect(child) + + collect(input_value) return file_ids @@ -966,29 +955,30 @@ def _trim_compact_input_for_upstream(payload: MutableJsonObject) -> None: ) required_input = _compact_trimmed_input_with_markers(input_value, token_counts, required_indices) required_tokens = _estimated_json_tokens(required_input) - if required_tokens > _MAX_COMPACT_UPSTREAM_ESTIMATED_TOKENS: - rewritten_input, images_elided = _compact_elide_required_tool_output_images( + rewritten_input, images_elided = _compact_elide_required_tool_output_images( + input_value, + required_indices=required_indices, + ) + if images_elided: + input_value = rewritten_input + payload["input"] = input_value + if _estimated_json_tokens(input_value) <= _MAX_COMPACT_UPSTREAM_ESTIMATED_TOKENS: + return + token_counts = [_estimated_json_array_item_tokens(item) for item in input_value] + head_count = _compact_trim_prefix_count(token_counts) + preserved_indices = _compact_state_anchor_indices(input_value) + required_indices = set(preserved_indices) + if input_value: + required_indices.add(len(input_value) - 1) + required_indices = _compact_reconciled_tool_call_indices( input_value, + required_indices, + token_counts=token_counts, + token_budget=sum(token_counts), required_indices=required_indices, ) - if images_elided: - input_value = rewritten_input - payload["input"] = input_value - token_counts = [_estimated_json_array_item_tokens(item) for item in input_value] - head_count = _compact_trim_prefix_count(token_counts) - preserved_indices = _compact_state_anchor_indices(input_value) - required_indices = set(preserved_indices) - if input_value: - required_indices.add(len(input_value) - 1) - required_indices = _compact_reconciled_tool_call_indices( - input_value, - required_indices, - token_counts=token_counts, - token_budget=sum(token_counts), - required_indices=required_indices, - ) - required_input = _compact_trimmed_input_with_markers(input_value, token_counts, required_indices) - required_tokens = _estimated_json_tokens(required_input) + required_input = _compact_trimmed_input_with_markers(input_value, token_counts, required_indices) + required_tokens = _estimated_json_tokens(required_input) if required_tokens > _MAX_COMPACT_UPSTREAM_ESTIMATED_TOKENS: raise ClientPayloadError( "Compact input exceeds the upstream size limit and cannot be trimmed " diff --git a/tests/unit/test_openai_requests.py b/tests/unit/test_openai_requests.py index 47f070bae7..6d8a324d54 100644 --- a/tests/unit/test_openai_requests.py +++ b/tests/unit/test_openai_requests.py @@ -1461,6 +1461,49 @@ def test_compact_trimming_elides_mapping_shaped_required_tool_image_output(): } +def test_compact_trimming_elides_percent_encoded_image_url_in_string_output(): + image_url = "data:image/svg+xml,%3Csvg%3E" + "%20" * 170_000 + "%3C/svg%3E" + payload = { + "model": "gpt-5.6-sol", + "instructions": "", + "input": [ + {"type": "function_call", "name": "render", "call_id": "call-svg", "arguments": "{}"}, + { + "type": "function_call_output", + "call_id": "call-svg", + "output": f"rendered {image_url}", + }, + ], + } + + dumped_input = ResponsesCompactRequest.model_validate(payload).to_payload()["input"] + + assert "data:image/svg+xml" not in json.dumps(dumped_input) + assert "Omitted inline image bytes" in json.dumps(dumped_input) + + +def test_compact_image_elision_keeps_other_input_when_rewritten_request_fits(): + payload = { + "model": "gpt-5.6-sol", + "instructions": "", + "input": [ + {"role": "user", "content": "retain-middle-" + "x" * 250_000}, + {"type": "function_call", "name": "render", "call_id": "call-fit", "arguments": "{}"}, + { + "type": "function_call_output", + "call_id": "call-fit", + "output": "data:image/png;base64," + "A" * 300_000, + }, + ], + } + + dumped_input = ResponsesCompactRequest.model_validate(payload).to_payload()["input"] + + assert isinstance(dumped_input, list) + assert dumped_input[0] == payload["input"][0] + assert not any(isinstance(item, dict) and item.get("type") == "message" for item in dumped_input) + + def test_compact_trimming_keeps_accepted_file_reference_while_eliding_inline_image(): file_reference = {"type": "input_file", "file_id": "file-canvas"} payload = { @@ -2667,8 +2710,13 @@ def test_extract_input_file_ids_finds_top_level_and_nested_ids(): {"type": "input_file", "file_id": "file_a"}, {"type": "input_file", "file_id": ""}, {"type": "input_file"}, + { + "type": "custom_tool_call_output", + "call_id": "call-file", + "output": [{"type": "input_file", "file_id": "file_tool_output"}], + }, ] - assert extract_input_file_ids(input_value) == {"file_a", "file_b", "file_c"} + assert extract_input_file_ids(input_value) == {"file_a", "file_b", "file_c", "file_tool_output"} def test_input_image_file_reference_returns_file_id_from_input_image_file_id(): From 62d13dca9cb53d9f40496dbcc9f09711cfc2ee77 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Thu, 23 Jul 2026 10:20:00 +0400 Subject: [PATCH 58/64] fix(compact): preserve inline image and file ownership --- app/core/openai/requests.py | 172 ++++++++++-------- app/modules/proxy/_service/file_ops.py | 5 +- .../proposal.md | 3 + .../specs/responses-api-compat/spec.md | 3 + tests/integration/test_proxy_compact.py | 66 +++++++ tests/unit/test_openai_requests.py | 96 +++++++++- 6 files changed, 262 insertions(+), 83 deletions(-) diff --git a/app/core/openai/requests.py b/app/core/openai/requests.py index 6ba7a132f5..7ba13fa385 100644 --- a/app/core/openai/requests.py +++ b/app/core/openai/requests.py @@ -153,10 +153,12 @@ def _input_image_file_reference(item: Mapping[str, JsonValue]) -> str | None: def extract_input_file_ids(input_value: JsonValue) -> set[str]: """Return all ``file_id`` strings referenced by ``input_file`` / ``input_image`` items. - Walks both top-level items and nested role-message ``content`` parts, - matching the shapes accepted by ``ResponsesRequest.input`` / - ``ResponsesCompactRequest.input``. Returns an empty set when the - input is a plain string or has no ``input_file`` parts. Used by the + Walks top-level input items, role-message ``content`` parts, and retained + tool-output content, matching the actual reference shapes accepted by + ``ResponsesRequest.input`` / ``ResponsesCompactRequest.input``. Tool + metadata and arbitrary nested objects are deliberately not references. + Returns an empty set when the input is a plain string or has no + ``input_file`` parts. Used by the ``/responses`` flow to look up account pins recorded by ``POST /backend-api/files`` so the response request lands on the upstream account that registered the file (the upstream contract is @@ -166,22 +168,26 @@ def extract_input_file_ids(input_value: JsonValue) -> set[str]: return set() file_ids: set[str] = set() - def collect(value: JsonValue) -> None: - if is_json_mapping(value): - if _is_input_file_with_id(value): - file_id = value.get("file_id") - if isinstance(file_id, str) and file_id: - file_ids.add(file_id) - image_file_id = _input_image_file_reference(value) - if image_file_id is not None: - file_ids.add(image_file_id) - for child in value.values(): - collect(child) - elif is_json_list(value): - for child in value: - collect(child) - - collect(input_value) + def collect_part(part: JsonValue) -> None: + if not is_json_mapping(part): + return + if _is_input_file_with_id(part): + file_id = part.get("file_id") + if isinstance(file_id, str) and file_id: + file_ids.add(file_id) + image_file_id = _input_image_file_reference(part) + if image_file_id is not None: + file_ids.add(image_file_id) + + for item in input_value: + if not is_json_mapping(item): + continue + collect_part(item) + for part in _json_parts(item.get("content")): + collect_part(part) + if item.get("type") in _COMPACT_TOOL_CALL_OUTPUT_ITEM_TYPES: + for part in _json_parts(item.get("output")): + collect_part(part) return file_ids @@ -908,16 +914,44 @@ def _trim_compact_input_for_upstream(payload: MutableJsonObject) -> None: input_value = payload.get("input") if not is_json_list(input_value): return - token_counts = [_estimated_json_array_item_tokens(item) for item in input_value] total_tokens = _estimated_json_tokens(input_value) if total_tokens <= _MAX_COMPACT_UPSTREAM_ESTIMATED_TOKENS: return + losslessly_trimmed_input = _compact_losslessly_trim_input(input_value) + if losslessly_trimmed_input is not None: + payload["input"] = losslessly_trimmed_input + return + + token_counts = [_estimated_json_array_item_tokens(item) for item in input_value] + required_indices = _compact_required_indices(input_value, token_counts) + rewritten_input, images_elided = _compact_elide_required_tool_output_images( + input_value, + required_indices=required_indices, + ) + if images_elided: + input_value = rewritten_input + payload["input"] = input_value + if _estimated_json_tokens(input_value) <= _MAX_COMPACT_UPSTREAM_ESTIMATED_TOKENS: + return + losslessly_trimmed_input = _compact_losslessly_trim_input(input_value) + if losslessly_trimmed_input is not None: + payload["input"] = losslessly_trimmed_input + return + raise ClientPayloadError( + "Compact input exceeds the upstream size limit and cannot be trimmed without removing required state anchors.", + param="input", + code="responses_compact_input_too_large", + ) + + +def _compact_losslessly_trim_input(input_value: list[JsonValue]) -> list[JsonValue] | None: + """Return a budget-fitting context selection without changing any retained bytes.""" + + token_counts = [_estimated_json_array_item_tokens(item) for item in input_value] head_count = _compact_trim_prefix_count(token_counts) state_anchor_indices = _compact_state_anchor_indices(input_value) marker_tokens = _estimated_json_array_item_tokens(_compact_trim_marker(omitted_items=0, omitted_tokens=0)) - wire_budget = max(0, _MAX_COMPACT_UPSTREAM_ESTIMATED_TOKENS - marker_tokens) - side_effect_indices = _compact_side_effect_anchor_indices(input_value) unusable_side_effect_indices = { index @@ -926,66 +960,23 @@ def _trim_compact_input_for_upstream(payload: MutableJsonObject) -> None: and _compact_item_is_side_effect_anchor(item) and (not isinstance(item.get("call_id"), str) or not item["call_id"]) } - # Keep priority side effects as complete call/output units before spending - # the remaining budget on ordinary head/tail context. Otherwise a large - # recent message can leave room for a call but not its output, causing - # reconciliation to drop the historical side effect that would fit after - # trimming that ordinary message. side_effect_indices = _compact_reconciled_tool_call_indices( input_value, side_effect_indices, token_counts=token_counts, token_budget=sum(token_counts), ) - required_indices = set(state_anchor_indices) - if input_value: - required_indices.add(len(input_value) - 1) + required_indices = _compact_required_indices(input_value, token_counts, preserved_indices=state_anchor_indices) if required_indices & unusable_side_effect_indices: raise ClientPayloadError( "Compact input cannot retain a required side-effect call without a usable call_id.", param="input", code="responses_compact_input_too_large", ) - required_indices = _compact_reconciled_tool_call_indices( - input_value, - required_indices, - token_counts=token_counts, - token_budget=sum(token_counts), - required_indices=required_indices, - ) required_input = _compact_trimmed_input_with_markers(input_value, token_counts, required_indices) - required_tokens = _estimated_json_tokens(required_input) - rewritten_input, images_elided = _compact_elide_required_tool_output_images( - input_value, - required_indices=required_indices, - ) - if images_elided: - input_value = rewritten_input - payload["input"] = input_value - if _estimated_json_tokens(input_value) <= _MAX_COMPACT_UPSTREAM_ESTIMATED_TOKENS: - return - token_counts = [_estimated_json_array_item_tokens(item) for item in input_value] - head_count = _compact_trim_prefix_count(token_counts) - preserved_indices = _compact_state_anchor_indices(input_value) - required_indices = set(preserved_indices) - if input_value: - required_indices.add(len(input_value) - 1) - required_indices = _compact_reconciled_tool_call_indices( - input_value, - required_indices, - token_counts=token_counts, - token_budget=sum(token_counts), - required_indices=required_indices, - ) - required_input = _compact_trimmed_input_with_markers(input_value, token_counts, required_indices) - required_tokens = _estimated_json_tokens(required_input) - if required_tokens > _MAX_COMPACT_UPSTREAM_ESTIMATED_TOKENS: - raise ClientPayloadError( - "Compact input exceeds the upstream size limit and cannot be trimmed " - "without removing required state anchors.", - param="input", - code="responses_compact_input_too_large", - ) + if _estimated_json_tokens(required_input) > _MAX_COMPACT_UPSTREAM_ESTIMATED_TOKENS: + return None + wire_budget = max(0, _MAX_COMPACT_UPSTREAM_ESTIMATED_TOKENS - marker_tokens) side_effect_indices &= _compact_reconciled_tool_call_indices( input_value, required_indices | side_effect_indices, @@ -1029,14 +1020,28 @@ def _trim_compact_input_for_upstream(payload: MutableJsonObject) -> None: ) trimmed_tokens = _estimated_json_tokens(trimmed_input) if trimmed_tokens > _MAX_COMPACT_UPSTREAM_ESTIMATED_TOKENS: - raise ClientPayloadError( - "Compact input still exceeds the upstream size limit after retaining required compact context.", - param="input", - code="responses_compact_input_too_large", - ) - if trimmed_input is input_value: - return - payload["input"] = trimmed_input + return None + return trimmed_input + + +def _compact_required_indices( + input_value: list[JsonValue], + token_counts: list[int], + *, + preserved_indices: set[int] | None = None, +) -> set[int]: + if preserved_indices is None: + preserved_indices = _compact_state_anchor_indices(input_value) + required_indices = set(preserved_indices) + if input_value: + required_indices.add(len(input_value) - 1) + return _compact_reconciled_tool_call_indices( + input_value, + required_indices, + token_counts=token_counts, + token_budget=sum(token_counts), + required_indices=required_indices, + ) def _compact_elide_required_tool_output_images( @@ -1080,6 +1085,17 @@ def _compact_elide_inline_images(value: JsonValue) -> tuple[JsonValue, bool]: }, True, ) + if value.get("type") == "image_url": + image_url = value.get("image_url") + url = image_url.get("url") if is_json_mapping(image_url) else image_url + if isinstance(url, str) and url.startswith("data:image/"): + return ( + { + "type": "text", + "text": f"{_COMPACT_OMITTED_INLINE_IMAGE_TEXT} ({len(url)} encoded characters).", + }, + True, + ) rewritten_mapping: JsonObject = {} changed = False for key, item in value.items(): diff --git a/app/modules/proxy/_service/file_ops.py b/app/modules/proxy/_service/file_ops.py index bc7cbf233b..b07696ef35 100644 --- a/app/modules/proxy/_service/file_ops.py +++ b/app/modules/proxy/_service/file_ops.py @@ -247,7 +247,10 @@ async def _resolve_file_account_for_responses( proxy = cast(_FileOpsServiceProtocol, self) del headers - file_ids = extract_input_file_ids(payload.input) + input_value = payload.input + if isinstance(payload, ResponsesCompactRequest): + input_value = payload.to_payload().get("input") + file_ids = extract_input_file_ids(input_value) if not file_ids: return None diff --git a/openspec/changes/allow-compact-inline-image-elision/proposal.md b/openspec/changes/allow-compact-inline-image-elision/proposal.md index de392cce0a..57c13c27e6 100644 --- a/openspec/changes/allow-compact-inline-image-elision/proposal.md +++ b/openspec/changes/allow-compact-inline-image-elision/proposal.md @@ -12,6 +12,9 @@ with `responses_compact_input_too_large`. - Replace inline data-URL images inside required function, custom, and apply-patch tool outputs with an explicit textual omission marker only while preparing an oversized compact request. +- Prefer lossless context selection before eliding image bytes; when a legacy + Chat `image_url` part must be elided, replace the whole part with a valid + Chat text part rather than writing a marker into its URL field. - Preserve the surrounding tool call/output identities and all textual content. - Leave accepted `input_file` references and ordinary non-compact requests unchanged. - Keep hosted `computer_call_output` screenshots fail-closed until a diff --git a/openspec/changes/allow-compact-inline-image-elision/specs/responses-api-compat/spec.md b/openspec/changes/allow-compact-inline-image-elision/specs/responses-api-compat/spec.md index 1979df1170..b833955402 100644 --- a/openspec/changes/allow-compact-inline-image-elision/specs/responses-api-compat/spec.md +++ b/openspec/changes/allow-compact-inline-image-elision/specs/responses-api-compat/spec.md @@ -14,7 +14,10 @@ MUST NOT weaken fail-closed handling for oversized textual state. - **WHEN** compact input exceeds the upstream limit because a required latest eligible required tool output contains an inline data-URL image that the model already observed - **THEN** compact preparation retains the tool call and output identities +- **AND** first uses lossless context trimming when that can fit the request - **AND** replaces only the inline image bytes with an explicit textual omission marker +- **AND** replaces an eligible legacy Chat `image_url` content part as a whole + with a schema-valid text part before any generic string substitution - **AND** preserves the other textual parts of the tool output - **AND** accepted file-backed `input_file` references remain unchanged - **AND** hosted `computer_call_output` screenshots remain fail-closed until a diff --git a/tests/integration/test_proxy_compact.py b/tests/integration/test_proxy_compact.py index 43c9b43b98..8aecce65df 100644 --- a/tests/integration/test_proxy_compact.py +++ b/tests/integration/test_proxy_compact.py @@ -345,6 +345,72 @@ async def fake_compact(payload, headers, access_token, account_id): assert response.headers.get("x-codex-credits-balance") == "12.50" +@pytest.mark.asyncio +async def test_proxy_compact_ignores_file_pin_from_trimmed_optional_history(async_client, monkeypatch): + for raw_account_id, email in ( + ("acc_compact_trim_file_owner", "compact-trim-file-owner@example.com"), + ("acc_compact_trim_selected", "compact-trim-selected@example.com"), + ): + auth_json = _make_auth_json(raw_account_id, email) + response = await async_client.post( + "/api/accounts/import", + files={"auth_json": ("auth.json", json.dumps(auth_json), "application/json")}, + ) + assert response.status_code == 200 + + from app.dependencies import get_proxy_service_for_app + + service = get_proxy_service_for_app(async_client._transport.app) + async with SessionLocal() as session: + accounts = (await session.execute(select(Account).order_by(Account.id))).scalars().all() + file_owner = next(account for account in accounts if account.chatgpt_account_id == "acc_compact_trim_file_owner") + selected_account = next( + account for account in accounts if account.chatgpt_account_id == "acc_compact_trim_selected" + ) + await service._pin_file_account("file_compact_trimmed_optional", file_owner.id) + + seen: dict[str, object] = {} + + async def fake_select_account(self, deadline, **kwargs): + del self, deadline + seen["preferred_account_id"] = kwargs.get("preferred_account_id") + return proxy_module.AccountSelection(account=selected_account, error_message=None) + + async def fake_compact(payload, headers, access_token, account_id): + del headers, access_token + seen["upstream_account_id"] = account_id + seen["upstream_payload"] = payload + return CompactResponsePayload.model_validate({"object": "response.compaction", "output": []}) + + monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget_compatible", fake_select_account) + monkeypatch.setattr(proxy_module, "core_compact_responses", fake_compact) + + response = await async_client.post( + "/backend-api/codex/responses/compact", + json={ + "model": "gpt-5.1", + "instructions": "hi", + "input": [ + {"role": "user", "content": "stable prelude"}, + { + "role": "assistant", + "content": [ + {"type": "output_text", "text": "x" * 500_000}, + {"type": "input_file", "file_id": "file_compact_trimmed_optional"}, + ], + }, + {"role": "user", "content": "continue after compaction"}, + ], + }, + ) + + assert response.status_code == 200 + assert seen["preferred_account_id"] is None + assert seen["upstream_account_id"] == "acc_compact_trim_selected" + upstream_payload = seen["upstream_payload"].to_payload() + assert "file_compact_trimmed_optional" not in json.dumps(upstream_payload) + + @pytest.mark.asyncio async def test_proxy_compact_normalizes_summary_output_for_codex_remote_v2(async_client, monkeypatch): email = "compact-v2-summary@example.com" diff --git a/tests/unit/test_openai_requests.py b/tests/unit/test_openai_requests.py index 6d8a324d54..8074bc0c7d 100644 --- a/tests/unit/test_openai_requests.py +++ b/tests/unit/test_openai_requests.py @@ -1482,7 +1482,7 @@ def test_compact_trimming_elides_percent_encoded_image_url_in_string_output(): assert "Omitted inline image bytes" in json.dumps(dumped_input) -def test_compact_image_elision_keeps_other_input_when_rewritten_request_fits(): +def test_compact_trimming_prefers_lossless_context_trim_before_inline_image_elision(): payload = { "model": "gpt-5.6-sol", "instructions": "", @@ -1500,8 +1500,78 @@ def test_compact_image_elision_keeps_other_input_when_rewritten_request_fits(): dumped_input = ResponsesCompactRequest.model_validate(payload).to_payload()["input"] assert isinstance(dumped_input, list) - assert dumped_input[0] == payload["input"][0] - assert not any(isinstance(item, dict) and item.get("type") == "message" for item in dumped_input) + assert dumped_input[0] != payload["input"][0] + assert "data:image/png;base64" in json.dumps(dumped_input) + assert "Omitted inline image bytes" not in json.dumps(dumped_input) + assert any(isinstance(item, dict) and item.get("type") == "message" for item in dumped_input) + + +def test_compact_trimming_elides_structured_chat_image_url_as_text_part(): + payload = { + "model": "gpt-5.6-sol", + "instructions": "", + "input": [ + {"type": "function_call", "name": "capture", "call_id": "call-chat-image", "arguments": "{}"}, + { + "type": "function_call_output", + "call_id": "call-chat-image", + "output": [ + { + "type": "image_url", + "image_url": {"url": "data:image/png;base64," + "A" * 500_000, "detail": "high"}, + } + ], + }, + ], + } + + dumped_input = ResponsesCompactRequest.model_validate(payload).to_payload()["input"] + + assert isinstance(dumped_input, list) + dumped_output = cast(Mapping[str, object], dumped_input[-1]) + assert dumped_output["output"] == [ + { + "type": "text", + "text": ( + "[compact trim] Omitted inline image bytes that were already observed before compaction " + "(500022 encoded characters)." + ), + } + ] + + +def test_compact_trimming_elides_bare_chat_image_url_as_text_part(): + payload = { + "model": "gpt-5.6-sol", + "instructions": "", + "input": [ + {"type": "function_call", "name": "capture", "call_id": "call-bare-chat-image", "arguments": "{}"}, + { + "type": "function_call_output", + "call_id": "call-bare-chat-image", + "output": [ + { + "type": "image_url", + "image_url": "data:image/png;base64," + "A" * 500_000, + } + ], + }, + ], + } + + dumped_input = ResponsesCompactRequest.model_validate(payload).to_payload()["input"] + + assert isinstance(dumped_input, list) + dumped_output = cast(Mapping[str, object], dumped_input[-1]) + assert dumped_output["output"] == [ + { + "type": "text", + "text": ( + "[compact trim] Omitted inline image bytes that were already observed before compaction " + "(500022 encoded characters)." + ), + } + ] def test_compact_trimming_keeps_accepted_file_reference_while_eliding_inline_image(): @@ -2695,7 +2765,7 @@ def test_extract_input_file_ids_string_input_returns_empty_set(): assert extract_input_file_ids("Hello world") == set() -def test_extract_input_file_ids_finds_top_level_and_nested_ids(): +def test_extract_input_file_ids_finds_actual_input_references_but_not_tool_metadata(): input_value: list[JsonValue] = [ { "role": "user", @@ -2715,6 +2785,24 @@ def test_extract_input_file_ids_finds_top_level_and_nested_ids(): "call_id": "call-file", "output": [{"type": "input_file", "file_id": "file_tool_output"}], }, + { + "type": "custom_tool_call", + "call_id": "call-metadata", + "input": {"type": "input_file", "file_id": "file_call_metadata"}, + }, + { + "type": "additional_tools", + "tools": [ + { + "type": "function", + "function": { + "parameters": { + "properties": {"fake_file": {"type": "input_file", "file_id": "file_tool_schema"}} + } + }, + } + ], + }, ] assert extract_input_file_ids(input_value) == {"file_a", "file_b", "file_c", "file_tool_output"} From 0ae295d741800c1283ce8d348e5c29d7e0fb4dc4 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Tue, 28 Jul 2026 12:01:00 +0400 Subject: [PATCH 59/64] test(compact): type captured compact payload --- tests/integration/test_proxy_compact.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_proxy_compact.py b/tests/integration/test_proxy_compact.py index 8aecce65df..3d84dd317e 100644 --- a/tests/integration/test_proxy_compact.py +++ b/tests/integration/test_proxy_compact.py @@ -16,6 +16,7 @@ from app.core.clients.proxy import ProxyResponseError from app.core.errors import openai_error from app.core.openai.models import CompactResponsePayload, OpenAIResponsePayload +from app.core.openai.requests import ResponsesCompactRequest from app.core.utils.time import utcnow from app.db.models import Account, AccountStatus from app.db.session import SessionLocal @@ -407,7 +408,7 @@ async def fake_compact(payload, headers, access_token, account_id): assert response.status_code == 200 assert seen["preferred_account_id"] is None assert seen["upstream_account_id"] == "acc_compact_trim_selected" - upstream_payload = seen["upstream_payload"].to_payload() + upstream_payload = cast(ResponsesCompactRequest, seen["upstream_payload"]).to_payload() assert "file_compact_trimmed_optional" not in json.dumps(upstream_payload) From e3dde57931a8074e86e51a22c4b6fda628025e51 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sun, 2 Aug 2026 20:45:56 +0400 Subject: [PATCH 60/64] fix(compact): isolate image elision constants --- app/core/openai/requests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/core/openai/requests.py b/app/core/openai/requests.py index 7ba13fa385..080920ed58 100644 --- a/app/core/openai/requests.py +++ b/app/core/openai/requests.py @@ -46,7 +46,6 @@ _COMPACT_TOOL_CALL_OUTPUT_ITEM_TYPES = frozenset( {"function_call_output", "custom_tool_call_output", "apply_patch_call_output"} ) -_COMPACT_INLINE_IMAGE_DATA_URL_RE = re.compile(r"""data:image/[^,\s]+,[^\s"'<>]+""") _GOAL_CONTINUATION_CONTEXT_PREFIX = '' _PLAN_MODE_CONTEXT_PREFIX = "# Plan Mode" @@ -795,6 +794,7 @@ def to_payload(self) -> JsonObject: _COMPACT_OMITTED_INLINE_IMAGE_TEXT = ( "[compact trim] Omitted inline image bytes that were already observed before compaction" ) +_COMPACT_INLINE_IMAGE_DATA_URL_RE = re.compile(r"""data:image/[^,\s]+,[^\s"'<>]+""") def _strip_unsupported_fields(payload: MutableJsonObject) -> MutableJsonObject: From 0fc6fce2adb67ac4143eb8507b006f0c0c54dfd5 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Tue, 4 Aug 2026 00:44:12 +0400 Subject: [PATCH 61/64] fix(http-bridge): keep missing-created watchdog armed after prelude --- .../proxy/_service/http_bridge/helpers.py | 1 - tests/unit/test_proxy_http_bridge.py | 24 +++++++++++++++++-- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/app/modules/proxy/_service/http_bridge/helpers.py b/app/modules/proxy/_service/http_bridge/helpers.py index ca98a7fa9f..8e12e369cc 100644 --- a/app/modules/proxy/_service/http_bridge/helpers.py +++ b/app/modules/proxy/_service/http_bridge/helpers.py @@ -653,7 +653,6 @@ def _http_bridge_eventless_precreated_deadline( or sent_at is None or request_state.response_id is not None or request_state.latency_response_created_ms is not None - or request_state.response_event_count != 0 or request_state.downstream_visible or request_state.last_downstream_sequence_number is not None ): diff --git a/tests/unit/test_proxy_http_bridge.py b/tests/unit/test_proxy_http_bridge.py index 00dd232e8e..6a67b0138c 100644 --- a/tests/unit/test_proxy_http_bridge.py +++ b/tests/unit/test_proxy_http_bridge.py @@ -161,7 +161,6 @@ def test_http_bridge_eventless_precreated_deadline_uses_current_send_and_client_ [ ("response_id", "resp-created"), ("latency_response_created_ms", 12), - ("response_event_count", 1), ("downstream_visible", True), ("last_downstream_sequence_number", 0), ("awaiting_response_created", False), @@ -186,6 +185,24 @@ def test_http_bridge_eventless_precreated_deadline_requires_narrow_owner_evidenc ) +def test_http_bridge_eventless_precreated_deadline_survives_reasoning_prelude_without_created() -> None: + request_state = _make_eventless_http_bridge_owner() + request_state.response_event_count = 3 + request_state.upstream_model_output_seen = True + request_state.deferred_reasoning_downstream_texts.append( + 'data: {"type":"response.output_item.added","item":{"type":"reasoning"}}\n\n' + ) + client_safe_cap_seconds = http_bridge_helpers_module._HTTP_BRIDGE_EVENTLESS_RESPONSE_CREATED_MAX_SECONDS + + assert ( + http_bridge_helpers_module._http_bridge_eventless_precreated_deadline( + request_state, + stuck_gate_retire_after_seconds=300.0, + ) + == 100.0 + min(300.0, client_safe_cap_seconds) + ) + + @pytest.mark.asyncio async def test_http_bridge_send_replaces_timestamp_and_wakes_existing_reader( monkeypatch: pytest.MonkeyPatch, @@ -18390,6 +18407,9 @@ async def close(self) -> None: owner.request_text = '{"type":"response.create","model":"gpt-5.6-sol","input":"hello"}' owner.preferred_account_id = "acc-bridge" owner.excluded_account_ids.add("acc-excluded") + if leading_telemetry: + owner.response_event_count = 1 + owner.upstream_model_output_seen = True sibling_queue: asyncio.Queue[str | None] = asyncio.Queue() sibling = proxy_service._WebSocketRequestState( request_id="req-created-sibling", @@ -18441,7 +18461,7 @@ async def close(self) -> None: assert owner.preferred_account_id == "acc-bridge" assert owner.excluded_account_ids == {"acc-excluded"} assert owner.replay_count == 0 - assert owner.response_event_count == 0 + assert owner.response_event_count == (1 if leading_telemetry else 0) if leading_telemetry: assert owner.latency_first_upstream_event_ms is not None retry_precreated.assert_not_awaited() From e30ffe3c3617df0eb0f6454bf83301e5690857c4 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Wed, 5 Aug 2026 14:23:24 +0400 Subject: [PATCH 62/64] fix(compact): preserve tool search pairs --- app/core/openai/requests.py | 4 ++-- tests/unit/test_openai_requests.py | 36 ++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/app/core/openai/requests.py b/app/core/openai/requests.py index 896287fbaa..654b9a9d28 100644 --- a/app/core/openai/requests.py +++ b/app/core/openai/requests.py @@ -42,9 +42,9 @@ _ASSISTANT_TEXT_PART_TYPES = frozenset({"text", "input_text", "output_text"}) _TOOL_TEXT_PART_TYPES = frozenset({"text", "input_text", "output_text", "refusal"}) _COMPACT_STATE_TOOL_NAMES = frozenset({"create_goal", "get_goal", "update_goal", "update_plan"}) -_COMPACT_TOOL_CALL_ITEM_TYPES = frozenset({"function_call", "custom_tool_call", "apply_patch_call"}) +_COMPACT_TOOL_CALL_ITEM_TYPES = frozenset({"function_call", "custom_tool_call", "apply_patch_call", "tool_search_call"}) _COMPACT_TOOL_CALL_OUTPUT_ITEM_TYPES = frozenset( - {"function_call_output", "custom_tool_call_output", "apply_patch_call_output"} + {"function_call_output", "custom_tool_call_output", "apply_patch_call_output", "tool_search_output"} ) _EXPLICIT_PROMPT_CACHE_CONTENT_TYPES = frozenset({"input_text", "input_image", "input_file"}) _GOAL_CONTINUATION_CONTEXT_PREFIX = '' diff --git a/tests/unit/test_openai_requests.py b/tests/unit/test_openai_requests.py index 8074bc0c7d..6a6b32eccf 100644 --- a/tests/unit/test_openai_requests.py +++ b/tests/unit/test_openai_requests.py @@ -2293,6 +2293,42 @@ def test_compact_trimming_keeps_selected_tool_calls_with_matching_outputs(): assert tool_output in dumped_input +def test_compact_trimming_keeps_tool_search_outputs_with_matching_calls(): + tool_call = { + "type": "tool_search_call", + "call_id": "call_search_tail", + "status": "completed", + "execution": "client", + "arguments": {"query": "spawn_agent multi-agent schema", "limit": 8}, + } + tool_output = { + "type": "tool_search_output", + "call_id": "call_search_tail", + "output": "Found matching tools", + } + input_items = [ + {"role": "user", "content": "initial instructions"}, + {"role": "assistant", "content": "x" * 500_000}, + tool_call, + {"role": "assistant", "content": "y" * 500_000}, + tool_output, + {"role": "user", "content": "latest request"}, + ] + payload = { + "model": "gpt-5.1", + "instructions": "hi", + "input": input_items, + } + + request = ResponsesCompactRequest.model_validate(payload) + dumped = request.to_payload() + dumped_input = dumped["input"] + + assert isinstance(dumped_input, list) + assert tool_call in dumped_input + assert tool_output in dumped_input + + def test_compact_trimming_reconciles_duplicate_tool_call_ids_by_occurrence(): first_tool_call = { "type": "function_call", From 3691fefd1f1e83fa4a34b225babc95dc2683b669 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Wed, 5 Aug 2026 14:39:19 +0400 Subject: [PATCH 63/64] fix: repair live carrier stream merge --- .../proxy/_service/http_bridge/streaming.py | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/app/modules/proxy/_service/http_bridge/streaming.py b/app/modules/proxy/_service/http_bridge/streaming.py index 1bf961b7c1..a06c6092d8 100644 --- a/app/modules/proxy/_service/http_bridge/streaming.py +++ b/app/modules/proxy/_service/http_bridge/streaming.py @@ -3244,21 +3244,9 @@ def stream_idle_keepalive(*, downstream_response_id: str) -> str | None: keepalive_count += 1 downstream_response_id = _websocket_downstream_response_id(request_state) if not completed_delivery_in_progress and keepalive_count > max_keepalive_count: - logger.info( - "HTTP bridge stream idle timeout request_id=%s keepalive_count=%s " - "max_keepalive_count=%s", - request_state.request_id, - keepalive_count, - max_keepalive_count, - ) - yield format_sse_event( - cast( - Mapping[str, JsonValue], - response_failed_event( - "stream_idle_timeout", - "Upstream did not respond within the keepalive window", - response_id=downstream_response_id, - ), + if not response_started: + retry_cooldown_seconds = await self._http_bridge_precreated_retry_cooldown_seconds( + session ) fresh_replay_is_safe = bool( request_state.fresh_upstream_request_is_retry_safe From 2e89cc252245fc86f7183e71aacc28bf3b710c0f Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Wed, 5 Aug 2026 14:52:04 +0400 Subject: [PATCH 64/64] fix(db): merge live carrier migration heads --- ...0260805_000000_merge_live_carrier_heads.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 app/db/alembic/versions/20260805_000000_merge_live_carrier_heads.py diff --git a/app/db/alembic/versions/20260805_000000_merge_live_carrier_heads.py b/app/db/alembic/versions/20260805_000000_merge_live_carrier_heads.py new file mode 100644 index 0000000000..4bddd1d8ce --- /dev/null +++ b/app/db/alembic/versions/20260805_000000_merge_live_carrier_heads.py @@ -0,0 +1,24 @@ +"""Merge live carrier migration heads. + +Revision ID: 20260805_000000_merge_live_carrier_heads +Revises: 20260802_000000_merge_bridge_and_capability_lineage_heads, 20260803_000000_merge_http_bridge_recovery_and_capability_lineage_heads +Create Date: 2026-08-05 +""" + +from __future__ import annotations + +revision = "20260805_000000_merge_live_carrier_heads" +down_revision = ( + "20260802_000000_merge_bridge_and_capability_lineage_heads", + "20260803_000000_merge_http_bridge_recovery_and_capability_lineage_heads", +) +branch_labels = None +depends_on = None + + +def upgrade() -> None: + pass + + +def downgrade() -> None: + pass