fix: handle session.status cooldown retries in runtime fallback

src/hooks/runtime-fallback/constants.ts

@@ -26,6 +26,11 @@ export const RETRYABLE_ERROR_PATTERNS = [
   /rate.?limit/i,
   /too.?many.?requests/i,
   /quota.?exceeded/i,
+  /quota\s+will\s+reset\s+after/i,
+  /all\s+credentials\s+for\s+model/i,
+  /cool(?:ing)?\s+down/i,
+  /cooldown/i,
+  /exhausted\s+your\s+capacity/i,
   /usage\s+limit\s+has\s+been\s+reached/i,
   /service.?unavailable/i,
   /overloaded/i,

src/hooks/runtime-fallback/event-handler.ts

@@ -6,9 +6,11 @@ import { extractStatusCode, extractErrorName, classifyErrorType, isRetryableErro
 import { createFallbackState, prepareFallback } from "./fallback-state"
 import { getFallbackModelsForSession } from "./fallback-models"
 import { SessionCategoryRegistry } from "../../shared/session-category-registry"
+import { createSessionStatusHandler } from "./session-status-handler"
 
 export function createEventHandler(deps: HookDeps, helpers: AutoRetryHelpers) {
   const { config, pluginConfig, sessionStates, sessionLastAccess, sessionRetryInFlight, sessionAwaitingFallbackResult, sessionFallbackTimeouts } = deps
+  const sessionStatusHandler = createSessionStatusHandler(deps, helpers)
 
   const handleSessionCreated = (props: Record<string, unknown> | undefined) => {
     const sessionInfo = props?.info as { id?: string; model?: string } | undefined
@@ -33,6 +35,7 @@ export function createEventHandler(deps: HookDeps, helpers: AutoRetryHelpers) {
       sessionRetryInFlight.delete(sessionID)
       sessionAwaitingFallbackResult.delete(sessionID)
       helpers.clearSessionFallbackTimeout(sessionID)
+      sessionStatusHandler.clearRetryKey(sessionID)
       SessionCategoryRegistry.remove(sessionID)
     }
   }
@@ -191,6 +194,7 @@ export function createEventHandler(deps: HookDeps, helpers: AutoRetryHelpers) {
     if (event.type === "session.deleted") { handleSessionDeleted(props); return }
     if (event.type === "session.stop") { await handleSessionStop(props); return }
     if (event.type === "session.idle") { handleSessionIdle(props); return }
+    if (event.type === "session.status") { await sessionStatusHandler.handleSessionStatus(props); return }
     if (event.type === "session.error") { await handleSessionError(props); return }
   }
 }

src/hooks/runtime-fallback/index.test.ts

@@ -387,6 +387,133 @@ describe("runtime-fallback", () => {
       expect(fallbackLog?.data).toMatchObject({ from: "openai/gpt-5.3-codex", to: "anthropic/claude-opus-4-6" })
     })
 
+    test("should trigger fallback on session.status auto-retry signal", async () => {
+      const promptCalls: unknown[] = []
+      const hook = createRuntimeFallbackHook(
+        createMockPluginInput({
+          session: {
+            messages: async () => ({
+              data: [
+                {
+                  info: { role: "user" },
+                  parts: [{ type: "text", text: "continue" }],
+                },
+              ],
+            }),
+            promptAsync: async (args: unknown) => {
+              promptCalls.push(args)
+              return {}
+            },
+          },
+        }),
+        {
+          config: createMockConfig({ notify_on_fallback: false }),
+          pluginConfig: createMockPluginConfigWithCategoryFallback(["openai/gpt-5.4"]),
+        }
+      )
+
+      const sessionID = "test-session-status-auto-retry"
+      SessionCategoryRegistry.register(sessionID, "test")
+
+      await hook.event({
+        event: {
+          type: "session.created",
+          properties: { info: { id: sessionID, model: "quotio/claude-opus-4-6" } },
+        },
+      })
+
+      await hook.event({
+        event: {
+          type: "session.status",
+          properties: {
+            sessionID,
+            status: {
+              type: "retry",
+              attempt: 1,
+              next: 476,
+              message: "All credentials for model claude-opus-4-6 are cooling down [retrying in 7m 56s attempt #1]",
+            },
+          },
+        },
+      })
+
+      const signalLog = logCalls.find((c) => c.msg.includes("Detected provider auto-retry signal in session.status"))
+      expect(signalLog).toBeDefined()
+
+      const fallbackLog = logCalls.find((c) => c.msg.includes("Preparing fallback"))
+      expect(fallbackLog).toBeDefined()
+      expect(fallbackLog?.data).toMatchObject({ from: "quotio/claude-opus-4-6", to: "openai/gpt-5.4" })
+      expect(promptCalls).toHaveLength(1)
+    })
+
+    test("should deduplicate session.status countdown updates for the same retry attempt", async () => {
+      const promptCalls: unknown[] = []
+      const hook = createRuntimeFallbackHook(
+        createMockPluginInput({
+          session: {
+            messages: async () => ({
+              data: [
+                {
+                  info: { role: "user" },
+                  parts: [{ type: "text", text: "continue" }],
+                },
+              ],
+            }),
+            promptAsync: async (args: unknown) => {
+              promptCalls.push(args)
+              return {}
+            },
+          },
+        }),
+        {
+          config: createMockConfig({ notify_on_fallback: false }),
+          pluginConfig: createMockPluginConfigWithCategoryFallback(["openai/gpt-5.4"]),
+        }
+      )
+
+      const sessionID = "test-session-status-countdown-dedup"
+      SessionCategoryRegistry.register(sessionID, "test")
+
+      await hook.event({
+        event: {
+          type: "session.created",
+          properties: { info: { id: sessionID, model: "quotio/claude-opus-4-6" } },
+        },
+      })
+
+      await hook.event({
+        event: {
+          type: "session.status",
+          properties: {
+            sessionID,
+            status: {
+              type: "retry",
+              attempt: 1,
+              next: 476,
+              message: "All credentials for model claude-opus-4-6 are cooling down [retrying in 7m 56s attempt #1]",
+            },
+          },
+        },
+      })
+
+      await hook.event({
+        event: {
+          type: "session.status",
+          properties: {
+            sessionID,
+            status: {
+              type: "retry",
+              attempt: 1,
+              next: 475,
+              message: "All credentials for model claude-opus-4-6 are cooling down [retrying in 7m 55s attempt #1]",
+            },
+          },
+        },
+      })
+
+      expect(promptCalls).toHaveLength(1)
+    })
+
     test("should NOT trigger fallback on auto-retry signal when timeout_seconds is 0", async () => {
       const hook = createRuntimeFallbackHook(createMockPluginInput(), {
         config: createMockConfig({ notify_on_fallback: false, timeout_seconds: 0 }),

src/hooks/runtime-fallback/session-status-handler.ts

@@ -0,0 +1,160 @@
+import type { HookDeps } from "./types"
+import type { AutoRetryHelpers } from "./auto-retry"
+import { HOOK_NAME } from "./constants"
+import { log } from "../../shared/logger"
+import { isRetryableError } from "./error-classifier"
+import { createFallbackState, prepareFallback } from "./fallback-state"
+import { getFallbackModelsForSession } from "./fallback-models"
+import { extractRetryAttempt, extractRetryStatusModel, normalizeRetryStatusMessage } from "../../shared/retry-status-utils"
+
+type SessionStatus = {
+  type?: string
+  message?: string
+  attempt?: number
+}
+
+function resolveInitialModel(
+  props: Record<string, unknown> | undefined,
+  retryMessage: string,
+  resolvedAgent: string | undefined,
+  pluginConfig: HookDeps["pluginConfig"],
+): string | undefined {
+  const eventModel = typeof props?.model === "string" ? props.model : undefined
+  if (eventModel) {
+    return eventModel
+  }
+
+  const retryModel = extractRetryStatusModel(retryMessage)
+  if (retryModel) {
+    return retryModel
+  }
+
+  const agentConfig = resolvedAgent
+    ? pluginConfig?.agents?.[resolvedAgent as keyof typeof pluginConfig.agents]
+    : undefined
+
+  return typeof agentConfig?.model === "string" ? agentConfig.model : undefined
+}
+
+export function createSessionStatusHandler(deps: HookDeps, helpers: AutoRetryHelpers): {
+  clearRetryKey: (sessionID: string) => void
+  handleSessionStatus: (props: Record<string, unknown> | undefined) => Promise<void>
+} {
+  const {
+    config,
+    pluginConfig,
+    sessionStates,
+    sessionLastAccess,
+    sessionRetryInFlight,
+    sessionAwaitingFallbackResult,
+  } = deps
+  const sessionStatusRetryKeys = new Map<string, string>()
+
+  const clearRetryKey = (sessionID: string): void => {
+    sessionStatusRetryKeys.delete(sessionID)
+  }
+
+  const handleSessionStatus = async (props: Record<string, unknown> | undefined): Promise<void> => {
+    const sessionID = props?.sessionID as string | undefined
+    const status = props?.status as SessionStatus | undefined
+    const agent = props?.agent as string | undefined
+    const timeoutEnabled = config.timeout_seconds > 0
+
+    if (!sessionID || status?.type !== "retry" || !timeoutEnabled) {
+      return
+    }
+
+    const retryMessage = typeof status.message === "string" ? status.message : ""
+    if (!retryMessage || !isRetryableError({ message: retryMessage }, config.retry_on_errors)) {
+      return
+    }
+
+    const currentState = sessionStates.get(sessionID)
+    const retryAttempt = extractRetryAttempt(status.attempt, retryMessage)
+    const retryModel =
+      (typeof props?.model === "string" ? props.model : undefined) ??
+      extractRetryStatusModel(retryMessage) ??
+      currentState?.currentModel ??
+      "unknown-model"
+    const retryKey = `${retryAttempt}:${retryModel}:${normalizeRetryStatusMessage(retryMessage)}`
+
+    if (sessionStatusRetryKeys.get(sessionID) === retryKey) {
+      return
+    }
+    sessionStatusRetryKeys.set(sessionID, retryKey)
+
+    if (sessionRetryInFlight.has(sessionID)) {
+      log(`[${HOOK_NAME}] Overriding in-flight retry due to provider session.status retry signal`, {
+        sessionID,
+        retryModel,
+      })
+      await helpers.abortSessionRequest(sessionID, "session.status.retry-signal")
+      sessionRetryInFlight.delete(sessionID)
+    }
+
+    sessionAwaitingFallbackResult.delete(sessionID)
+
+    const resolvedAgent = await helpers.resolveAgentForSessionFromContext(sessionID, agent)
+    const fallbackModels = getFallbackModelsForSession(sessionID, resolvedAgent, pluginConfig)
+
+    if (fallbackModels.length === 0) {
+      log(`[${HOOK_NAME}] No fallback models configured`, { sessionID, agent: resolvedAgent ?? agent })
+      return
+    }
+
+    let state = currentState
+    if (!state) {
+      const initialModel = resolveInitialModel(props, retryMessage, resolvedAgent, pluginConfig)
+      if (!initialModel) {
+        log(`[${HOOK_NAME}] session.status retry missing model info, cannot fallback`, { sessionID })
+        return
+      }
+
+      state = createFallbackState(initialModel)
+      sessionStates.set(sessionID, state)
+    }
+
+    sessionLastAccess.set(sessionID, Date.now())
+
+    if (state.pendingFallbackModel) {
+      log(`[${HOOK_NAME}] Clearing pending fallback due to provider session.status retry signal`, {
+        sessionID,
+        pendingFallbackModel: state.pendingFallbackModel,
+      })
+      state.pendingFallbackModel = undefined
+    }
+
+    log(`[${HOOK_NAME}] Detected provider auto-retry signal in session.status`, {
+      sessionID,
+      model: state.currentModel,
+      retryAttempt,
+    })
+
+    const result = prepareFallback(sessionID, state, fallbackModels, config)
+
+    if (result.success && config.notify_on_fallback) {
+      await deps.ctx.client.tui
+        .showToast({
+          body: {
+            title: "Model Fallback",
+            message: `Switching to ${result.newModel?.split("/").pop() || result.newModel} for next request`,
+            variant: "warning",
+            duration: 5000,
+          },
+        })
+        .catch(() => {})
+    }
+
+    if (result.success && result.newModel) {
+      await helpers.autoRetryWithFallback(sessionID, result.newModel, resolvedAgent, "session.status")
+      return
+    }
+
+    log(`[${HOOK_NAME}] Fallback preparation failed`, { sessionID, error: result.error })
+  }
+
+  return {
+    clearRetryKey,
+    handleSessionStatus,
+  }
+}