From 6bf64adbae89973935c6dd035e873adea066bcd3 Mon Sep 17 00:00:00 2001 From: Liam Steiner Date: Tue, 14 Jul 2026 12:26:24 +0300 Subject: [PATCH 1/2] refactor(scripts): extract shared Gemini quota-error predicate (9 sites) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds scripts/_gemini_quota.py with a single is_quota_error(exc) predicate and re-points the 9 duplicated inline "429"/RESOURCE_EXHAUSTED checks in drift_check.py, trec_atom_benchmark.py, memory_indexer.py, rule_following_judge.py, and attention_dilution_probe.py to use it. Scope is intentionally minimal — this is the one piece of the Gemini call-with-fallback logic proven safe to consolidate across prior review rounds. The surrounding retry/fallback bodies (temperatures, token limits, exhaustion-tracking, retry cadence) differ meaningfully per file and are left untouched. Co-Authored-By: Claude Sonnet 5 --- scripts/_gemini_quota.py | 17 ++++++++++ scripts/bench/attention_dilution_probe.py | 28 ++++++++++++++-- scripts/bench/rule_following_judge.py | 19 ++++++++++- scripts/drift_check.py | 11 +++---- scripts/memory_indexer.py | 9 ++--- scripts/tests/test_gemini_quota.py | 40 +++++++++++++++++++++++ scripts/trec_atom_benchmark.py | 6 ++-- 7 files changed, 112 insertions(+), 18 deletions(-) create mode 100644 scripts/_gemini_quota.py create mode 100644 scripts/tests/test_gemini_quota.py diff --git a/scripts/_gemini_quota.py b/scripts/_gemini_quota.py new file mode 100644 index 00000000..236c327e --- /dev/null +++ b/scripts/_gemini_quota.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 +"""Shared Gemini quota-error predicate. + +Extracted from duplicated `"429" in str(e) or "RESOURCE_EXHAUSTED" in str(e)` +checks scattered across scripts/drift_check.py, scripts/trec_atom_benchmark.py, +scripts/memory_indexer.py, scripts/bench/rule_following_judge.py, and +scripts/bench/attention_dilution_probe.py. Intentionally scoped to ONLY this +predicate — the surrounding retry/fallback bodies differ meaningfully per +file (temperatures, token limits, exhaustion-tracking, retry cadence) and are +NOT consolidated here. +""" + + +def is_quota_error(exc: Exception) -> bool: + """Return True if `exc` looks like a Gemini per-minute/per-day quota error.""" + msg = str(exc) + return "429" in msg or "RESOURCE_EXHAUSTED" in msg diff --git a/scripts/bench/attention_dilution_probe.py b/scripts/bench/attention_dilution_probe.py index 4fc5355b..362993f1 100644 --- a/scripts/bench/attention_dilution_probe.py +++ b/scripts/bench/attention_dilution_probe.py @@ -35,6 +35,7 @@ import argparse import datetime import hashlib +import importlib.util import itertools import json import os @@ -54,6 +55,7 @@ _SCRIPTS_DIR = _BENCH_DIR.parent _REPO_ROOT = _SCRIPTS_DIR.parent _SP_PATH = _SCRIPTS_DIR / "standards_pack.py" +_GQ_PATH = _SCRIPTS_DIR / "_gemini_quota.py" _FIXTURES_DIR = _BENCH_DIR / "fixtures" _DEFAULT_OUTPUT_DIR = _BENCH_DIR / "results" _PAIRWISE_CACHE_PATH = _BENCH_DIR / "attention_dilution_pairwise_cache.json" @@ -109,6 +111,28 @@ """ +# --------------------------------------------------------------------------- +# Module loaders +# --------------------------------------------------------------------------- + + +def _load_gq(): + """Load _gemini_quota as a module. scripts/bench/ is not on sys.path (this + file invokes standards_pack.py as a subprocess rather than in-process, so + unlike rule_following_judge.py it had no prior importlib plumbing) — same + spec_from_file_location pattern as rule_following_judge.py's `_load_gq`. + """ + if "_gemini_quota" in sys.modules: + return sys.modules["_gemini_quota"] + spec = importlib.util.spec_from_file_location("_gemini_quota", _GQ_PATH) + if spec is None or spec.loader is None: + raise RuntimeError(f"Cannot load _gemini_quota from {_GQ_PATH}") + mod = importlib.util.module_from_spec(spec) + sys.modules["_gemini_quota"] = mod + spec.loader.exec_module(mod) + return mod + + # --------------------------------------------------------------------------- # Standards-pack subprocess invocation # --------------------------------------------------------------------------- @@ -433,7 +457,7 @@ def _judge_recall( except Exception as e: msg = str(e) last_err = msg - if "429" in msg or "RESOURCE_EXHAUSTED" in msg: + if _load_gq().is_quota_error(e): if "PerDay" in msg: exhausted.add(model) else: @@ -655,7 +679,7 @@ def _judge_pair( except Exception as e: msg = str(e) last_err = msg - if "429" in msg or "RESOURCE_EXHAUSTED" in msg: + if _load_gq().is_quota_error(e): if "PerDay" in msg: exhausted.add(model) else: diff --git a/scripts/bench/rule_following_judge.py b/scripts/bench/rule_following_judge.py index ab1774af..7a19a8d5 100644 --- a/scripts/bench/rule_following_judge.py +++ b/scripts/bench/rule_following_judge.py @@ -56,6 +56,7 @@ _SCRIPTS_DIR = _BENCH_DIR.parent _REPO_ROOT = _SCRIPTS_DIR.parent _SP_PATH = _SCRIPTS_DIR / "standards_pack.py" +_GQ_PATH = _SCRIPTS_DIR / "_gemini_quota.py" _DEFAULT_PROBES = _SCRIPTS_DIR / "tests" / "fixtures" / "rule_following_probes.jsonl" _DEFAULT_OUTPUT_DIR = _BENCH_DIR / "results" _JUDGE_CACHE_PATH = _BENCH_DIR / "rule_following_judge_cache.json" @@ -131,6 +132,22 @@ def _load_sp(): return mod +def _load_gq(): + """Load _gemini_quota as a module. Same importlib pattern as `_load_sp` — + scripts/bench/ is not on sys.path, so a plain `from _gemini_quota import + is_quota_error` would break direct script execution. + """ + if "_gemini_quota" in sys.modules: + return sys.modules["_gemini_quota"] + spec = importlib.util.spec_from_file_location("_gemini_quota", _GQ_PATH) + if spec is None or spec.loader is None: + raise RuntimeError(f"Cannot load _gemini_quota from {_GQ_PATH}") + mod = importlib.util.module_from_spec(spec) + sys.modules["_gemini_quota"] = mod + spec.loader.exec_module(mod) + return mod + + # --------------------------------------------------------------------------- # Fixture I/O # --------------------------------------------------------------------------- @@ -372,7 +389,7 @@ def _judge_one( except Exception as e: # noqa: BLE001 — broad on purpose for fallback msg = str(e) last_err = msg[:200] - if "429" in msg or "RESOURCE_EXHAUSTED" in msg: + if _load_gq().is_quota_error(e): if "PerDay" in msg: exhausted.add(model) print(f" {model} daily quota exhausted, skipping", file=sys.stderr) diff --git a/scripts/drift_check.py b/scripts/drift_check.py index f0a065cd..317a2bf9 100644 --- a/scripts/drift_check.py +++ b/scripts/drift_check.py @@ -37,6 +37,8 @@ import sys from pathlib import Path +from _gemini_quota import is_quota_error + PROJECT_ROOT = Path(__file__).parent.parent @@ -841,8 +843,7 @@ def call_gemini(prompt: str) -> str | None: ) return (response.text or "").strip() except Exception as e: - err = str(e) - if "429" in err or "RESOURCE_EXHAUSTED" in err: + if is_quota_error(e): continue print(f" WARN: Gemini error ({model}): {e}", file=sys.stderr) return None @@ -1047,8 +1048,7 @@ def call_gemini(prompt: str) -> str | None: ) return (response.text or "").strip() except Exception as e: - err = str(e) - if "429" in err or "RESOURCE_EXHAUSTED" in err: + if is_quota_error(e): continue print(f" WARN: Gemini error ({model}): {e}", file=sys.stderr) return None @@ -1195,8 +1195,7 @@ def check_contradictions(project_root: Path, pattern_filter: str | None = None) response_text = (response.text or "").strip() break except Exception as e: - err = str(e) - if "429" in err or "RESOURCE_EXHAUSTED" in err: + if is_quota_error(e): continue print(f"WARN: Gemini error ({model}): {e}", file=sys.stderr) return 0 diff --git a/scripts/memory_indexer.py b/scripts/memory_indexer.py index a7e6cdbc..a6e828dc 100644 --- a/scripts/memory_indexer.py +++ b/scripts/memory_indexer.py @@ -34,6 +34,7 @@ from _time import local_now, utc_now # noqa: E402 from _exit_codes import SUCCESS, ABSTAIN, USAGE_ERROR, NOT_FOUND, AUTH_ERROR, INTERNAL_ERROR from _agent_io import is_agent_context, compact_json, select_fields +from _gemini_quota import is_quota_error import sqlite_vec from google import genai @@ -1930,12 +1931,6 @@ def _extract_content_for_llm(content: str, max_chars: int = 6000) -> str: return trimmed[:max_chars] -def _is_quota_error(exc: Exception) -> bool: - """Return True if `exc` looks like a Gemini per-minute / per-day quota error.""" - msg = str(exc) - return "429" in msg or "RESOURCE_EXHAUSTED" in msg - - def _generate_with_fallback( prompt, *, @@ -1969,7 +1964,7 @@ def _generate_with_fallback( try: return client.models.generate_content(model=model, **kwargs) except Exception as exc: - if _is_quota_error(exc): + if is_quota_error(exc): if attempt == 1: # brief within-model backoff — helps per-minute RPM caps time.sleep(1) diff --git a/scripts/tests/test_gemini_quota.py b/scripts/tests/test_gemini_quota.py new file mode 100644 index 00000000..4c28aa9c --- /dev/null +++ b/scripts/tests/test_gemini_quota.py @@ -0,0 +1,40 @@ +"""Tests for scripts/_gemini_quota.py — the shared Gemini quota-error predicate. + +Extracted from duplicated `"429" in str(e) or "RESOURCE_EXHAUSTED" in str(e)` +checks across drift_check.py, trec_atom_benchmark.py, memory_indexer.py, and +the two bench judge scripts. This test only covers the predicate itself. +""" +from __future__ import annotations + +import sys +from pathlib import Path + +_SCRIPTS_DIR = str(Path(__file__).resolve().parent.parent) +if _SCRIPTS_DIR not in sys.path: + sys.path.insert(0, _SCRIPTS_DIR) + +from _gemini_quota import is_quota_error + + +def test_matches_429(): + assert is_quota_error(Exception("429 Too Many Requests")) is True + + +def test_matches_resource_exhausted(): + assert is_quota_error(Exception("RESOURCE_EXHAUSTED: quota exceeded")) is True + + +def test_matches_both_markers_present(): + assert is_quota_error(Exception("429 RESOURCE_EXHAUSTED PerDay limit")) is True + + +def test_does_not_match_unrelated_error(): + assert is_quota_error(Exception("500 Internal Server Error")) is False + + +def test_does_not_match_empty_message(): + assert is_quota_error(Exception("")) is False + + +def test_does_not_match_connection_error(): + assert is_quota_error(ConnectionError("connection reset by peer")) is False diff --git a/scripts/trec_atom_benchmark.py b/scripts/trec_atom_benchmark.py index 424e4279..f0f44c33 100644 --- a/scripts/trec_atom_benchmark.py +++ b/scripts/trec_atom_benchmark.py @@ -42,6 +42,8 @@ if _SCRIPTS_DIR not in sys.path: sys.path.insert(0, _SCRIPTS_DIR) +from _gemini_quota import is_quota_error # noqa: E402 + BENCH_DIR = Path(os.path.expanduser("~/.deus/bench")) LOG_PATH = Path(os.path.expanduser("~/.deus/memory_tree_queries.jsonl")) DB_PATH = Path(os.path.expanduser("~/.deus/memory.db")) @@ -412,7 +414,7 @@ def stage_judge(judge: str = "gemini", judge_model: str = "gemma4:e4b", fresh: b time.sleep(4) break except Exception as e: - if "429" in str(e) or "RESOURCE_EXHAUSTED" in str(e): + if is_quota_error(e): if "PerDay" in str(e): exhausted_models.add(model) print(f" {model} daily quota exhausted, skipping", file=sys.stderr) @@ -463,7 +465,7 @@ def stage_judge(judge: str = "gemini", judge_model: str = "gemma4:e4b", fresh: b time.sleep(4) break except Exception as e: - if "429" in str(e) or "RESOURCE_EXHAUSTED" in str(e): + if is_quota_error(e): if "PerDay" in str(e): exhausted_models.add(model) else: From b09ff3dd9dcf992d34d1a87405256e6321fa6d42 Mon Sep 17 00:00:00 2001 From: Liam Steiner Date: Fri, 17 Jul 2026 18:52:09 +0300 Subject: [PATCH 2/2] chore(patterns): bump eval-change.md drift check --- patterns/eval-change.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/patterns/eval-change.md b/patterns/eval-change.md index 88e06eef..3af31235 100644 --- a/patterns/eval-change.md +++ b/patterns/eval-change.md @@ -3,7 +3,7 @@ governs: - evolution/ - eval/ - scripts/memory_indexer.py -last_verified: "2026-07-16" # auto-bump @1784178260 +last_verified: "2026-07-17" # auto-bump @1784303525 test_tasks: - "Add a new DeepEval metric under eval/ for the core_qa test suite" - "Add a new judge backend to evolution/judge/ using the provider registry"