Fetch rate limit config variables from Coop

edsl/coop/coop.py

@@ -665,6 +665,10 @@ def web(
         return response_json
 
     def fetch_prices(self) -> dict:
+        """
+        Fetch model prices from Coop. If the request fails, return an empty dict.
+        """
+
         from edsl.coop.PriceFetcher import PriceFetcher
 
         from edsl.config import CONFIG
@@ -675,6 +679,20 @@ def fetch_prices(self) -> dict:
         else:
             return {}
 
+    def fetch_rate_limit_config_vars(self) -> dict:
+        """
+        Fetch a dict of rate limit config vars from Coop.
+
+        The dict keys are RPM and TPM variables like EDSL_SERVICE_RPM_OPENAI.
+        """
+        response = self._send_server_request(
+            uri="api/v0/config-vars",
+            method="GET",
+        )
+        self._resolve_server_response(response)
+        data = response.json()
+        return data
+
 
 if __name__ == "__main__":
     sheet_data = fetch_sheet_data()

edsl/inference_services/InferenceServiceABC.py

@@ -1,6 +1,7 @@
 from abc import abstractmethod, ABC
 import os
 import re
+from datetime import datetime, timedelta
 from edsl.config import CONFIG
 
 
@@ -10,6 +11,8 @@ class InferenceServiceABC(ABC):
     Anthropic: https://docs.anthropic.com/en/api/rate-limits
     """
 
+    _coop_config_vars = None
+
     default_levels = {
         "google": {"tpm": 2_000_000, "rpm": 15},
         "openai": {"tpm": 2_000_000, "rpm": 10_000},
@@ -31,12 +34,37 @@ def __init_subclass__(cls):
                 f"Class {cls.__name__} must have a 'model_exclude_list' attribute."
             )
 
+    @classmethod
+    def _should_refresh_coop_config_vars(cls):
+        """
+        Returns True if config vars have been fetched over 24 hours ago, and False otherwise.
+        """
+
+        if cls._last_config_fetch is None:
+            return True
+        return (datetime.now() - cls._last_config_fetch) > timedelta(hours=24)
+
     @classmethod
     def _get_limt(cls, limit_type: str) -> int:
         key = f"EDSL_SERVICE_{limit_type.upper()}_{cls._inference_service_.upper()}"
         if key in os.environ:
             return int(os.getenv(key))
 
+        if cls._coop_config_vars is None or cls._should_refresh_coop_config_vars():
+            try:
+                from edsl import Coop
+
+                c = Coop()
+                cls._coop_config_vars = c.fetch_rate_limit_config_vars()
+                cls._last_config_fetch = datetime.now()
+                if key in cls._coop_config_vars:
+                    return cls._coop_config_vars[key]
+            except Exception:
+                cls._coop_config_vars = None
+        else:
+            if key in cls._coop_config_vars:
+                return cls._coop_config_vars[key]
+
         if cls._inference_service_ in cls.default_levels:
             return int(cls.default_levels[cls._inference_service_][limit_type])