fix(codex): add hierarchical quota waterfall to codex-global exhaustion

Previously codex-global only mirrored primary window (5h-limit) exhaustion.
When a secondary window (weekly-limit or monthly-limit) was exhausted, the
cooldown was applied only to the display group — not codex-global. This left
the credential selectable by CooldownChecker and showed "mixed" status instead
of "exhausted".

Port the same tier hierarchy pattern from opencode_go_provider: walk tiers
from highest (monthly > weekly > 5h) and block codex-global on the first
exhausted tier. Pro-tier credentials with no monthly window now correctly
exhaust when weekly-limit is consumed.

Author: b3nw

src/rotator_library/providers/utilities/codex_quota_tracker.py

@@ -34,6 +34,12 @@
 
 lib_logger = logging.getLogger("rotator_library")
 
+# Tier hierarchy: higher-tier exhaustion implies all lower tiers are blocked.
+# monthly > weekly > primary (5h).  When weekly is exhausted the credential
+# cannot be used even if 5h-limit still shows remaining capacity.
+# Mirrors opencode_go_provider.QUOTA_TIER_HIERARCHY.
+QUOTA_TIER_HIERARCHY = ["5h-limit", "weekly-limit", "monthly-limit"]
+
 
 # =============================================================================
 # HELPER FUNCTIONS
@@ -550,9 +556,14 @@ def _push_quota_to_usage_manager(
 
         async def _push():
             try:
+                now = time.time()
+                # Collect per-window data for hierarchical exhaustion waterfall.
+                # Each display window is pushed independently (apply_exhaustion=False);
+                # blocking is determined by codex-global via the tier hierarchy.
+                window_tiers: Dict[str, Dict[str, Any]] = {}
+
                 if snapshot.primary:
                     used_pct = snapshot.primary.used_percent
-                    # Convert percentage to a request count on a 100-scale
                     quota_used = int(used_pct)
                     primary_label = _window_label_from_seconds(
                         snapshot.primary.window_minutes * 60 if snapshot.primary.window_minutes else None
@@ -567,24 +578,13 @@ async def _push():
                         quota_used=quota_used,
                         quota_group=primary_label,
                         force=True,
-                        apply_exhaustion=snapshot.primary.is_exhausted,
-                    )
-                    # Also push to codex-global so the executor's quota display
-                    # can find the limit when looking up the model's quota group.
-                    # apply_exhaustion MUST also be set here — codex-global is
-                    # the key the CooldownChecker uses during credential selection.
-                    # Without it, the primary-limit cooldown is invisible to the
-                    # selection engine and exhausted credentials keep getting picked.
-                    await self._usage_manager.update_quota_baseline(
-                        accessor=credential_path,
-                        model=f"{provider_prefix}/_global_quota",
-                        quota_max_requests=100,
-                        quota_reset_ts=snapshot.primary.reset_at,
-                        quota_used=quota_used,
-                        quota_group="codex-global",
-                        force=True,
-                        apply_exhaustion=snapshot.primary.is_exhausted,
+                        apply_exhaustion=False,
                     )
+                    window_tiers[primary_label] = {
+                        "used_percent": used_pct,
+                        "reset_ts": snapshot.primary.reset_at or 0,
+                        "quota_used": quota_used,
+                    }
 
                 if snapshot.secondary:
                     used_pct = snapshot.secondary.used_percent
@@ -602,8 +602,42 @@ async def _push():
                         quota_used=quota_used,
                         quota_group=secondary_label,
                         force=True,
-                        apply_exhaustion=snapshot.secondary.is_exhausted,
+                        apply_exhaustion=False,
                     )
+                    window_tiers[secondary_label] = {
+                        "used_percent": used_pct,
+                        "reset_ts": snapshot.secondary.reset_at or 0,
+                        "quota_used": quota_used,
+                    }
+
+                # Hierarchical exhaustion waterfall for codex-global.
+                # Walk tiers from highest to lowest; the first exhausted tier
+                # blocks the credential (its reset_ts becomes the cooldown).
+                global_exhausted = False
+                global_reset_ts = None
+                global_quota_used = 0
+                for tier_key in reversed(QUOTA_TIER_HIERARCHY):
+                    wd = window_tiers.get(tier_key)
+                    if wd and wd["used_percent"] >= 100.0 and wd["reset_ts"] > now:
+                        global_exhausted = True
+                        global_reset_ts = wd["reset_ts"]
+                        global_quota_used = 100
+                        break
+
+                if not global_exhausted and snapshot.primary:
+                    global_quota_used = int(snapshot.primary.used_percent)
+                    global_reset_ts = snapshot.primary.reset_at
+
+                await self._usage_manager.update_quota_baseline(
+                    accessor=credential_path,
+                    model=f"{provider_prefix}/_global_quota",
+                    quota_max_requests=100,
+                    quota_reset_ts=global_reset_ts,
+                    quota_used=global_quota_used,
+                    quota_group="codex-global",
+                    force=True,
+                    apply_exhaustion=global_exhausted,
+                )
             except Exception as e:
                 lib_logger.debug(
                     f"Failed to push Codex quota to UsageManager: {e}"
@@ -707,8 +741,11 @@ async def get_all_quota_info(
                     plan_type_counts.get(snapshot.plan_type, 0) + 1
                 )
 
-            # Check if exhausted
-            if snapshot.primary and snapshot.primary.is_exhausted:
+            # Check if exhausted (any tier in the hierarchy blocks the credential)
+            if (
+                (snapshot.primary and snapshot.primary.is_exhausted)
+                or (snapshot.secondary and snapshot.secondary.is_exhausted)
+            ):
                 exhausted_count += 1
 
             # Build result entry
@@ -902,11 +939,12 @@ async def _store_baselines_to_usage_manager(
         models = getattr(self, "_available_models_for_quota", [])
         provider_prefix = getattr(self, "provider_env_name", "codex")
 
+        now = time.time()
+
         for cred_path, quota_data in quota_results.items():
             if quota_data.get("status") != "success":
                 continue
 
-            # Get remaining fraction from primary and secondary windows
             primary = quota_data.get("primary")
             secondary = quota_data.get("secondary")
 
@@ -916,13 +954,16 @@ async def _store_baselines_to_usage_manager(
             else:
                 short_cred = Path(cred_path).stem
 
-            # Store primary window under dynamically-named virtual model
+            # Collect per-window tier data for hierarchical exhaustion.
+            # Display windows are pushed with apply_exhaustion=False;
+            # blocking is decided once via codex-global.
+            window_tiers: Dict[str, Dict[str, Any]] = {}
+
             if primary:
                 primary_remaining = primary.get("remaining_fraction", 1.0)
                 primary_used_pct = primary.get("used_percent", 0)
                 primary_reset = primary.get("reset_at")
                 primary_window_minutes = primary.get("window_minutes")
-                is_exhausted = primary.get("is_exhausted", False)
                 primary_label = _window_label_from_seconds(
                     primary_window_minutes * 60 if primary_window_minutes else None
                 )
@@ -937,23 +978,14 @@ async def _store_baselines_to_usage_manager(
                         quota_used=int(primary_used_pct),
                         quota_group=primary_label,
                         force=force,
-                        apply_exhaustion=is_exhausted and is_initial_fetch,
-                    )
-                    # Also store in codex-global so the executor's quota display
-                    # can find the limit when looking up the model's quota group.
-                    # apply_exhaustion MUST mirror the primary-limit push — codex-global
-                    # is the key the CooldownChecker uses during credential selection.
-                    await usage_manager.update_quota_baseline(
-                        accessor=cred_path,
-                        model=f"{provider_prefix}/_global_quota",
-                        quota_max_requests=100,
-                        quota_reset_ts=primary_reset,
-                        quota_used=int(primary_used_pct),
-                        quota_group="codex-global",
-                        force=force,
-                        apply_exhaustion=is_exhausted and is_initial_fetch,
+                        apply_exhaustion=False,
                     )
                     stored_count += 1
+                    window_tiers[primary_label] = {
+                        "used_percent": primary_used_pct,
+                        "reset_ts": primary_reset or 0,
+                        "quota_used": int(primary_used_pct),
+                    }
                     lib_logger.debug(
                         f"Stored Codex {primary_label} baseline for {short_cred}: "
                         f"{primary_remaining * 100:.1f}% remaining"
@@ -963,13 +995,11 @@ async def _store_baselines_to_usage_manager(
                         f"Failed to store Codex {primary_label} baseline for {short_cred}: {e}"
                     )
 
-            # Store secondary window under dynamically-named virtual model
             if secondary:
                 secondary_remaining = secondary.get("remaining_fraction", 1.0)
                 secondary_used_pct = secondary.get("used_percent", 0)
                 secondary_reset = secondary.get("reset_at")
                 secondary_window_minutes = secondary.get("window_minutes")
-                is_exhausted = secondary.get("is_exhausted", False)
                 secondary_label = _window_label_from_seconds(
                     secondary_window_minutes * 60 if secondary_window_minutes else None
                 )
@@ -984,9 +1014,14 @@ async def _store_baselines_to_usage_manager(
                         quota_used=int(secondary_used_pct),
                         quota_group=secondary_label,
                         force=force,
-                        apply_exhaustion=is_exhausted and is_initial_fetch,
+                        apply_exhaustion=False,
                     )
                     stored_count += 1
+                    window_tiers[secondary_label] = {
+                        "used_percent": secondary_used_pct,
+                        "reset_ts": secondary_reset or 0,
+                        "quota_used": int(secondary_used_pct),
+                    }
                     lib_logger.debug(
                         f"Stored Codex {secondary_label} baseline for {short_cred}: "
                         f"{secondary_remaining * 100:.1f}% remaining"
@@ -996,6 +1031,40 @@ async def _store_baselines_to_usage_manager(
                         f"Failed to store Codex {secondary_label} baseline for {short_cred}: {e}"
                     )
 
+            # Hierarchical exhaustion waterfall for codex-global.
+            # Walk tiers from highest (monthly) to lowest (5h); the first
+            # exhausted tier blocks the credential entirely.
+            global_exhausted = False
+            global_reset_ts = None
+            global_quota_used = 0
+            for tier_key in reversed(QUOTA_TIER_HIERARCHY):
+                wd = window_tiers.get(tier_key)
+                if wd and wd["used_percent"] >= 100.0 and wd["reset_ts"] > now:
+                    global_exhausted = True
+                    global_reset_ts = wd["reset_ts"]
+                    global_quota_used = 100
+                    break
+
+            if not global_exhausted and primary:
+                global_quota_used = int(primary.get("used_percent", 0))
+                global_reset_ts = primary.get("reset_at")
+
+            try:
+                await usage_manager.update_quota_baseline(
+                    accessor=cred_path,
+                    model=f"{provider_prefix}/_global_quota",
+                    quota_max_requests=100,
+                    quota_reset_ts=global_reset_ts,
+                    quota_used=global_quota_used,
+                    quota_group="codex-global",
+                    force=force,
+                    apply_exhaustion=global_exhausted and is_initial_fetch,
+                )
+            except Exception as e:
+                lib_logger.warning(
+                    f"Failed to store Codex global baseline for {short_cred}: {e}"
+                )
+
         return stored_count
 
     async def fetch_initial_baselines(