From 34c42b5ad4709ac10fce5c81effddc88ec150851 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Mon, 6 Jul 2026 16:43:46 -0500 Subject: [PATCH 1/3] Use GitHub authentication and retries for metadata requests --- src/rapids_metadata/remote.py | 76 +++++++++++++++++- tests/test_remote.py | 142 +++++++++++++++++++++++++++++++++- 2 files changed, 211 insertions(+), 7 deletions(-) diff --git a/src/rapids_metadata/remote.py b/src/rapids_metadata/remote.py index b867f1a..4dc1f9e 100644 --- a/src/rapids_metadata/remote.py +++ b/src/rapids_metadata/remote.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024-2025, NVIDIA CORPORATION. +# Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,7 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +import math +import os +import time +import urllib.error import urllib.request +from collections.abc import Mapping +from typing import Optional import pydantic @@ -21,12 +27,74 @@ __all__ = ["fetch_latest"] _GITHUB_METADATA_URL = "https://raw.githubusercontent.com/rapidsai/rapids-metadata/main/rapids-metadata.json" +_GITHUB_API_METADATA_URL = "https://api.github.com/repos/rapidsai/rapids-metadata/contents/rapids-metadata.json" +_GITHUB_API_VERSION = "2026-03-10" +_MAX_ATTEMPTS = 3 +_RETRYABLE_HTTP_STATUS_CODES = {408, 429, 500, 502, 503, 504} +_RATE_LIMIT_RETRY_SECONDS = 60.0 -def _fetch_from_url(url: str) -> RAPIDSMetadata: - with urllib.request.urlopen(url) as f: - return pydantic.TypeAdapter(RAPIDSMetadata).validate_json(f.read()) +def _header_as_float(error: urllib.error.HTTPError, name: str) -> Optional[float]: + try: + value = float(error.headers[name]) + except (KeyError, TypeError, ValueError): + return None + return value if math.isfinite(value) else None + + +def _retry_delay(error: urllib.error.HTTPError, attempt: int) -> float: + if (retry_after := _header_as_float(error, "Retry-After")) is not None: + return max(0.0, retry_after) + + if ( + error.headers.get("X-RateLimit-Remaining") == "0" + and (reset_at := _header_as_float(error, "X-RateLimit-Reset")) is not None + ): + return max(0.0, reset_at - time.time()) + 1.0 + + if error.code in {403, 429}: + return _RATE_LIMIT_RETRY_SECONDS * 2**attempt + return float(2**attempt) + + +def _is_retryable(error: urllib.error.HTTPError) -> bool: + return error.code in _RETRYABLE_HTTP_STATUS_CODES or ( + error.code == 403 + and ( + "Retry-After" in error.headers + or error.headers.get("X-RateLimit-Remaining") == "0" + ) + ) + + +def _fetch_from_url( + url: str, *, headers: Optional[Mapping[str, str]] = None +) -> RAPIDSMetadata: + request = urllib.request.Request(url, headers=dict(headers or {})) + for attempt in range(_MAX_ATTEMPTS): + try: + with urllib.request.urlopen(request) as f: + return pydantic.TypeAdapter(RAPIDSMetadata).validate_json(f.read()) + except urllib.error.HTTPError as error: + if attempt == _MAX_ATTEMPTS - 1 or not _is_retryable(error): + raise + time.sleep(_retry_delay(error, attempt)) + except urllib.error.URLError: + if attempt == _MAX_ATTEMPTS - 1: + raise + time.sleep(2**attempt) + + raise AssertionError("unreachable") def fetch_latest() -> RAPIDSMetadata: + if token := os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN"): + return _fetch_from_url( + _GITHUB_API_METADATA_URL, + headers={ + "Accept": "application/vnd.github.raw+json", + "Authorization": f"Bearer {token}", + "X-GitHub-Api-Version": _GITHUB_API_VERSION, + }, + ) return _fetch_from_url(_GITHUB_METADATA_URL) diff --git a/tests/test_remote.py b/tests/test_remote.py index d77f488..1b9c625 100644 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024-2025, NVIDIA CORPORATION. +# Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,8 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -from unittest.mock import patch +import urllib.error +from unittest.mock import MagicMock, call, patch +import pytest import rapids_metadata.remote as rapids_remote from pydantic import TypeAdapter from pytest_httpserver import HTTPServer @@ -31,8 +33,142 @@ def test_fetch_from_url(httpserver: HTTPServer): ) -def test_fetch_latest(): +def test_fetch_latest_without_token(monkeypatch): + monkeypatch.delenv("GH_TOKEN", raising=False) + monkeypatch.delenv("GITHUB_TOKEN", raising=False) with patch("rapids_metadata.remote._fetch_from_url") as patch_fetch_from_url: return_value = rapids_remote.fetch_latest() patch_fetch_from_url.assert_called_once_with(rapids_remote._GITHUB_METADATA_URL) assert return_value == patch_fetch_from_url() + + +@pytest.mark.parametrize( + ("environment", "expected_token"), + [ + ({"GITHUB_TOKEN": "github-token"}, "github-token"), + ( + {"GH_TOKEN": "gh-token", "GITHUB_TOKEN": "github-token"}, + "gh-token", + ), + ], +) +def test_fetch_latest_with_token(monkeypatch, environment, expected_token): + monkeypatch.delenv("GH_TOKEN", raising=False) + monkeypatch.delenv("GITHUB_TOKEN", raising=False) + for name, value in environment.items(): + monkeypatch.setenv(name, value) + + with patch("rapids_metadata.remote._fetch_from_url") as patch_fetch_from_url: + return_value = rapids_remote.fetch_latest() + + patch_fetch_from_url.assert_called_once_with( + rapids_remote._GITHUB_API_METADATA_URL, + headers={ + "Accept": "application/vnd.github.raw+json", + "Authorization": f"Bearer {expected_token}", + "X-GitHub-Api-Version": rapids_remote._GITHUB_API_VERSION, + }, + ) + assert return_value == patch_fetch_from_url() + + +def _metadata_response(): + response = MagicMock() + response.__enter__.return_value = response + response.read.return_value = TypeAdapter(RAPIDSMetadata).dump_json(all_metadata) + return response + + +def _http_error(status, headers=None): + return urllib.error.HTTPError( + rapids_remote._GITHUB_METADATA_URL, + status, + "request failed", + headers or {}, + None, + ) + + +@patch("rapids_metadata.remote.time.sleep") +@patch("rapids_metadata.remote.urllib.request.urlopen") +def test_fetch_retries_429(patch_urlopen, patch_sleep): + patch_urlopen.side_effect = [ + _http_error(429, {"Retry-After": "7"}), + _metadata_response(), + ] + + assert rapids_remote._fetch_from_url("https://example.com") == all_metadata + assert patch_urlopen.call_count == 2 + patch_sleep.assert_called_once_with(7.0) + + +@patch("rapids_metadata.remote.time.sleep") +@patch("rapids_metadata.remote.time.time", return_value=100.0) +@patch("rapids_metadata.remote.urllib.request.urlopen") +def test_fetch_honors_rate_limit_reset(patch_urlopen, patch_time, patch_sleep): + patch_urlopen.side_effect = [ + _http_error( + 403, + { + "X-RateLimit-Remaining": "0", + "X-RateLimit-Reset": "109", + }, + ), + _metadata_response(), + ] + + assert rapids_remote._fetch_from_url("https://example.com") == all_metadata + patch_time.assert_called_once_with() + patch_sleep.assert_called_once_with(10.0) + + +@patch("rapids_metadata.remote.time.sleep") +@patch("rapids_metadata.remote.urllib.request.urlopen") +def test_fetch_retries_transient_server_error(patch_urlopen, patch_sleep): + patch_urlopen.side_effect = [_http_error(503), _metadata_response()] + + assert rapids_remote._fetch_from_url("https://example.com") == all_metadata + patch_sleep.assert_called_once_with(1.0) + + +@patch("rapids_metadata.remote.time.sleep") +@patch("rapids_metadata.remote.urllib.request.urlopen") +def test_fetch_retries_url_error(patch_urlopen, patch_sleep): + patch_urlopen.side_effect = [ + urllib.error.URLError("connection reset"), + _metadata_response(), + ] + + assert rapids_remote._fetch_from_url("https://example.com") == all_metadata + patch_sleep.assert_called_once_with(1) + + +@patch("rapids_metadata.remote.time.sleep") +@patch("rapids_metadata.remote.urllib.request.urlopen") +def test_fetch_stops_after_max_attempts(patch_urlopen, patch_sleep): + errors = [ + _http_error(429, {"Retry-After": "0"}) + for _ in range(rapids_remote._MAX_ATTEMPTS) + ] + patch_urlopen.side_effect = errors + + with pytest.raises(urllib.error.HTTPError) as raised: + rapids_remote._fetch_from_url("https://example.com") + + assert raised.value is errors[-1] + assert patch_urlopen.call_count == rapids_remote._MAX_ATTEMPTS + assert patch_sleep.call_args_list == [call(0.0), call(0.0)] + + +@patch("rapids_metadata.remote.time.sleep") +@patch("rapids_metadata.remote.urllib.request.urlopen") +def test_fetch_does_not_retry_nonretryable_error(patch_urlopen, patch_sleep): + error = _http_error(404) + patch_urlopen.side_effect = error + + with pytest.raises(urllib.error.HTTPError) as raised: + rapids_remote._fetch_from_url("https://example.com") + + assert raised.value is error + patch_urlopen.assert_called_once() + patch_sleep.assert_not_called() From 422118e1a615c380103a301bf84027daeb622c5d Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Mon, 6 Jul 2026 17:48:56 -0500 Subject: [PATCH 2/3] Simplify metadata request retries --- src/rapids_metadata/remote.py | 51 ++++++------------------------ tests/test_remote.py | 58 ++++++++--------------------------- 2 files changed, 23 insertions(+), 86 deletions(-) diff --git a/src/rapids_metadata/remote.py b/src/rapids_metadata/remote.py index 4dc1f9e..da947ea 100644 --- a/src/rapids_metadata/remote.py +++ b/src/rapids_metadata/remote.py @@ -12,12 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import math import os import time import urllib.error import urllib.request -from collections.abc import Mapping from typing import Optional import pydantic @@ -31,58 +29,29 @@ _GITHUB_API_VERSION = "2026-03-10" _MAX_ATTEMPTS = 3 _RETRYABLE_HTTP_STATUS_CODES = {408, 429, 500, 502, 503, 504} -_RATE_LIMIT_RETRY_SECONDS = 60.0 - - -def _header_as_float(error: urllib.error.HTTPError, name: str) -> Optional[float]: - try: - value = float(error.headers[name]) - except (KeyError, TypeError, ValueError): - return None - return value if math.isfinite(value) else None - - -def _retry_delay(error: urllib.error.HTTPError, attempt: int) -> float: - if (retry_after := _header_as_float(error, "Retry-After")) is not None: - return max(0.0, retry_after) - - if ( - error.headers.get("X-RateLimit-Remaining") == "0" - and (reset_at := _header_as_float(error, "X-RateLimit-Reset")) is not None - ): - return max(0.0, reset_at - time.time()) + 1.0 - - if error.code in {403, 429}: - return _RATE_LIMIT_RETRY_SECONDS * 2**attempt - return float(2**attempt) - - -def _is_retryable(error: urllib.error.HTTPError) -> bool: - return error.code in _RETRYABLE_HTTP_STATUS_CODES or ( - error.code == 403 - and ( - "Retry-After" in error.headers - or error.headers.get("X-RateLimit-Remaining") == "0" - ) - ) def _fetch_from_url( - url: str, *, headers: Optional[Mapping[str, str]] = None + url: str, *, headers: Optional[dict[str, str]] = None ) -> RAPIDSMetadata: - request = urllib.request.Request(url, headers=dict(headers or {})) + request = urllib.request.Request(url, headers=headers or {}) for attempt in range(_MAX_ATTEMPTS): try: with urllib.request.urlopen(request) as f: return pydantic.TypeAdapter(RAPIDSMetadata).validate_json(f.read()) except urllib.error.HTTPError as error: - if attempt == _MAX_ATTEMPTS - 1 or not _is_retryable(error): + if ( + attempt == _MAX_ATTEMPTS - 1 + or error.code not in _RETRYABLE_HTTP_STATUS_CODES + ): raise - time.sleep(_retry_delay(error, attempt)) + default_delay = (60 if error.code == 429 else 1) * 2**attempt + delay = max(0, int(error.headers.get("Retry-After", default_delay))) except urllib.error.URLError: if attempt == _MAX_ATTEMPTS - 1: raise - time.sleep(2**attempt) + delay = 2**attempt + time.sleep(delay) raise AssertionError("unreachable") diff --git a/tests/test_remote.py b/tests/test_remote.py index 1b9c625..5031ba8 100644 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -12,8 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +import io import urllib.error -from unittest.mock import MagicMock, call, patch +from unittest.mock import call, patch import pytest import rapids_metadata.remote as rapids_remote @@ -37,9 +38,8 @@ def test_fetch_latest_without_token(monkeypatch): monkeypatch.delenv("GH_TOKEN", raising=False) monkeypatch.delenv("GITHUB_TOKEN", raising=False) with patch("rapids_metadata.remote._fetch_from_url") as patch_fetch_from_url: - return_value = rapids_remote.fetch_latest() + rapids_remote.fetch_latest() patch_fetch_from_url.assert_called_once_with(rapids_remote._GITHUB_METADATA_URL) - assert return_value == patch_fetch_from_url() @pytest.mark.parametrize( @@ -59,7 +59,7 @@ def test_fetch_latest_with_token(monkeypatch, environment, expected_token): monkeypatch.setenv(name, value) with patch("rapids_metadata.remote._fetch_from_url") as patch_fetch_from_url: - return_value = rapids_remote.fetch_latest() + rapids_remote.fetch_latest() patch_fetch_from_url.assert_called_once_with( rapids_remote._GITHUB_API_METADATA_URL, @@ -69,14 +69,10 @@ def test_fetch_latest_with_token(monkeypatch, environment, expected_token): "X-GitHub-Api-Version": rapids_remote._GITHUB_API_VERSION, }, ) - assert return_value == patch_fetch_from_url() def _metadata_response(): - response = MagicMock() - response.__enter__.return_value = response - response.read.return_value = TypeAdapter(RAPIDSMetadata).dump_json(all_metadata) - return response + return io.BytesIO(TypeAdapter(RAPIDSMetadata).dump_json(all_metadata)) def _http_error(status, headers=None): @@ -99,45 +95,17 @@ def test_fetch_retries_429(patch_urlopen, patch_sleep): assert rapids_remote._fetch_from_url("https://example.com") == all_metadata assert patch_urlopen.call_count == 2 - patch_sleep.assert_called_once_with(7.0) - - -@patch("rapids_metadata.remote.time.sleep") -@patch("rapids_metadata.remote.time.time", return_value=100.0) -@patch("rapids_metadata.remote.urllib.request.urlopen") -def test_fetch_honors_rate_limit_reset(patch_urlopen, patch_time, patch_sleep): - patch_urlopen.side_effect = [ - _http_error( - 403, - { - "X-RateLimit-Remaining": "0", - "X-RateLimit-Reset": "109", - }, - ), - _metadata_response(), - ] - - assert rapids_remote._fetch_from_url("https://example.com") == all_metadata - patch_time.assert_called_once_with() - patch_sleep.assert_called_once_with(10.0) - - -@patch("rapids_metadata.remote.time.sleep") -@patch("rapids_metadata.remote.urllib.request.urlopen") -def test_fetch_retries_transient_server_error(patch_urlopen, patch_sleep): - patch_urlopen.side_effect = [_http_error(503), _metadata_response()] - - assert rapids_remote._fetch_from_url("https://example.com") == all_metadata - patch_sleep.assert_called_once_with(1.0) + patch_sleep.assert_called_once_with(7) +@pytest.mark.parametrize( + "error", + [_http_error(503), urllib.error.URLError("connection reset")], +) @patch("rapids_metadata.remote.time.sleep") @patch("rapids_metadata.remote.urllib.request.urlopen") -def test_fetch_retries_url_error(patch_urlopen, patch_sleep): - patch_urlopen.side_effect = [ - urllib.error.URLError("connection reset"), - _metadata_response(), - ] +def test_fetch_retries_transient_error(patch_urlopen, patch_sleep, error): + patch_urlopen.side_effect = [error, _metadata_response()] assert rapids_remote._fetch_from_url("https://example.com") == all_metadata patch_sleep.assert_called_once_with(1) @@ -157,7 +125,7 @@ def test_fetch_stops_after_max_attempts(patch_urlopen, patch_sleep): assert raised.value is errors[-1] assert patch_urlopen.call_count == rapids_remote._MAX_ATTEMPTS - assert patch_sleep.call_args_list == [call(0.0), call(0.0)] + assert patch_sleep.call_args_list == [call(0), call(0)] @patch("rapids_metadata.remote.time.sleep") From c43b033402b356b547dc288ebeac3ac806ca13cd Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Tue, 7 Jul 2026 14:37:54 -0500 Subject: [PATCH 3/3] Use context managers for remote test patches --- tests/test_remote.py | 68 ++++++++++++++++++++++---------------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/tests/test_remote.py b/tests/test_remote.py index 5031ba8..e67e945 100644 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -85,58 +85,58 @@ def _http_error(status, headers=None): ) -@patch("rapids_metadata.remote.time.sleep") -@patch("rapids_metadata.remote.urllib.request.urlopen") -def test_fetch_retries_429(patch_urlopen, patch_sleep): - patch_urlopen.side_effect = [ - _http_error(429, {"Retry-After": "7"}), - _metadata_response(), - ] +def test_fetch_retries_429(): + with patch("rapids_metadata.remote.urllib.request.urlopen") as patch_urlopen: + with patch("rapids_metadata.remote.time.sleep") as patch_sleep: + patch_urlopen.side_effect = [ + _http_error(429, {"Retry-After": "7"}), + _metadata_response(), + ] - assert rapids_remote._fetch_from_url("https://example.com") == all_metadata - assert patch_urlopen.call_count == 2 - patch_sleep.assert_called_once_with(7) + assert rapids_remote._fetch_from_url("https://example.com") == all_metadata + assert patch_urlopen.call_count == 2 + patch_sleep.assert_called_once_with(7) @pytest.mark.parametrize( "error", [_http_error(503), urllib.error.URLError("connection reset")], ) -@patch("rapids_metadata.remote.time.sleep") -@patch("rapids_metadata.remote.urllib.request.urlopen") -def test_fetch_retries_transient_error(patch_urlopen, patch_sleep, error): - patch_urlopen.side_effect = [error, _metadata_response()] +def test_fetch_retries_transient_error(error): + with patch("rapids_metadata.remote.urllib.request.urlopen") as patch_urlopen: + with patch("rapids_metadata.remote.time.sleep") as patch_sleep: + patch_urlopen.side_effect = [error, _metadata_response()] - assert rapids_remote._fetch_from_url("https://example.com") == all_metadata - patch_sleep.assert_called_once_with(1) + assert rapids_remote._fetch_from_url("https://example.com") == all_metadata + patch_sleep.assert_called_once_with(1) -@patch("rapids_metadata.remote.time.sleep") -@patch("rapids_metadata.remote.urllib.request.urlopen") -def test_fetch_stops_after_max_attempts(patch_urlopen, patch_sleep): +def test_fetch_stops_after_max_attempts(): errors = [ _http_error(429, {"Retry-After": "0"}) for _ in range(rapids_remote._MAX_ATTEMPTS) ] - patch_urlopen.side_effect = errors + with patch("rapids_metadata.remote.urllib.request.urlopen") as patch_urlopen: + with patch("rapids_metadata.remote.time.sleep") as patch_sleep: + patch_urlopen.side_effect = errors - with pytest.raises(urllib.error.HTTPError) as raised: - rapids_remote._fetch_from_url("https://example.com") + with pytest.raises(urllib.error.HTTPError) as raised: + rapids_remote._fetch_from_url("https://example.com") - assert raised.value is errors[-1] - assert patch_urlopen.call_count == rapids_remote._MAX_ATTEMPTS - assert patch_sleep.call_args_list == [call(0), call(0)] + assert raised.value is errors[-1] + assert patch_urlopen.call_count == rapids_remote._MAX_ATTEMPTS + assert patch_sleep.call_args_list == [call(0), call(0)] -@patch("rapids_metadata.remote.time.sleep") -@patch("rapids_metadata.remote.urllib.request.urlopen") -def test_fetch_does_not_retry_nonretryable_error(patch_urlopen, patch_sleep): +def test_fetch_does_not_retry_nonretryable_error(): error = _http_error(404) - patch_urlopen.side_effect = error + with patch("rapids_metadata.remote.urllib.request.urlopen") as patch_urlopen: + with patch("rapids_metadata.remote.time.sleep") as patch_sleep: + patch_urlopen.side_effect = error - with pytest.raises(urllib.error.HTTPError) as raised: - rapids_remote._fetch_from_url("https://example.com") + with pytest.raises(urllib.error.HTTPError) as raised: + rapids_remote._fetch_from_url("https://example.com") - assert raised.value is error - patch_urlopen.assert_called_once() - patch_sleep.assert_not_called() + assert raised.value is error + patch_urlopen.assert_called_once() + patch_sleep.assert_not_called()