Skip to content

Normalize HTTP error handling #46

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
wilsonfreitas merged 1 commit into from
Jun 14, 2026
Merged

Normalize HTTP error handling #46

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
166 changes: 59 additions & 107 deletions bcb/currency.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,25 +6,23 @@
import threading
from datetime import date, timedelta
from io import BytesIO, StringIO
from typing import TYPE_CHECKING, Dict, List, Literal, NamedTuple, Union, overload
from typing import Dict, List, Literal, NamedTuple, Union, overload
from urllib.parse import urlencode

import httpx
import numpy as np
import pandas as pd
from lxml import html

from bcb.http import _CLIENT, _ASYNC_CLIENT
from bcb.exceptions import (
BCBAPIError,
BCBAPINotFoundError,
BCBRateLimitError,
CurrencyNotFoundError,
from bcb.http import (
_CLIENT,
_ASYNC_CLIENT,
raise_for_request_error,
raise_for_status,
)
from bcb.exceptions import BCBAPIError, CurrencyNotFoundError
from bcb.utils import Date, DateInput

if TYPE_CHECKING:
import httpx

logger = logging.getLogger(__name__)

"""
Expand Down Expand Up @@ -165,28 +163,18 @@ def _currency_id_list(
"method=exibeFormularioConsultaBoletim"
)
logger.debug(f"Fetching currency ID list from {url1}")
res = _CLIENT.get(url1)
try:
res = _CLIENT.get(url1)
except httpx.HTTPError as ex:
raise_for_request_error(ex, context="Currency ID list")
logger.debug(
f"Currency ID list response: status={res.status_code}, length={len(res.content)}"
)
if res.status_code == 429:
raise BCBRateLimitError(
"BCB API rate limit exceeded. Please try again later.",
status_code=429,
)
if res.status_code == 404:
raise BCBAPINotFoundError(
"BCB API endpoint not found (404)",
status_code=404,
)
if res.status_code >= 500:
raise BCBAPIError(
f"BCB API server error (status {res.status_code})",
status_code=res.status_code,
)
if res.status_code != 200:
msg = f"BCB API Request error, status code = {res.status_code}"
raise BCBAPIError(msg, res.status_code)
raise_for_status(
res,
context="Currency ID list",
not_found_message="BCB API endpoint not found (404)",
)

doc = html.parse(BytesIO(res.content)).getroot()
xpath = "//select[@name='ChkMoeda']/option"
Expand Down Expand Up @@ -238,10 +226,10 @@ def _get_valid_currency_list(
logger.debug(f"Fetching currency list from {url2}")
try:
res = _CLIENT.get(url2)
except Exception as ex:
except httpx.HTTPError as ex:
# Connection error: retry same date up to 3 times
if n >= 3:
raise ex
raise_for_request_error(ex, context="Currency list")
logger.warning(
f"Connection error fetching {url2}, retrying (attempt {n + 1}/3)"
)
Expand All @@ -252,12 +240,12 @@ def _get_valid_currency_list(
)
if res.status_code == 200:
return res
else:
# Non-200 response (file not found for date): roll back to previous day
logger.debug(
f"Currency list not found for {_date}, rolling back to previous day"
)
return _get_valid_currency_list(_date - timedelta(1), 0, max_rollback)
if res.status_code == 429 or res.status_code >= 500:
raise_for_status(res, context="Currency list")

# Non-200 response (file not found for date): roll back to previous day
logger.debug(f"Currency list not found for {_date}, rolling back to previous day")
return _get_valid_currency_list(_date - timedelta(1), 0, max_rollback)


def get_currency_list(
Expand Down Expand Up @@ -347,13 +335,16 @@ def _fetch_symbol_response(
cid = _get_currency_id(symbol) # Raises CurrencyNotFoundError if not found
url = _currency_url(cid, start_date, end_date)
logger.debug(f"Fetching currency data for {symbol} from {url.split('?')[0]}")
res = _CLIENT.get(url)
try:
res = _CLIENT.get(url)
except httpx.HTTPError as ex:
raise_for_request_error(ex, context=f"Currency data for {symbol}")
logger.debug(
f"Currency data response: status={res.status_code}, length={len(res.content)}"
)

# Handle HTML error response (e.g., no data for date range)
if res.headers["Content-Type"].startswith("text/html"):
if res.headers.get("Content-Type", "").startswith("text/html"):
doc = html.parse(BytesIO(res.content)).getroot()
xpath = "//div[@class='msgErro']"
elm = doc.xpath(xpath)[0]
Expand All @@ -363,27 +354,11 @@ def _fetch_symbol_response(
msg = f"BCB API returned error: {x} - {symbol}"
raise BCBAPIError(msg, status_code=400)

# Handle HTTP error responses
if res.status_code == 429:
raise BCBRateLimitError(
"BCB API rate limit exceeded. Please try again later.",
status_code=429,
)
if res.status_code == 404:
raise BCBAPINotFoundError(
f"Currency data not found for {symbol}",
status_code=404,
)
if res.status_code >= 500:
raise BCBAPIError(
f"BCB API server error (status {res.status_code})",
status_code=res.status_code,
)
if res.status_code != 200:
raise BCBAPIError(
f"BCB API request failed with status {res.status_code}",
status_code=res.status_code,
)
raise_for_status(
res,
context=f"Currency data for {symbol}",
not_found_message=f"Currency data not found for {symbol}",
)

return res

Expand Down Expand Up @@ -689,25 +664,15 @@ async def _async_currency_id_list(
"https://ptax.bcb.gov.br/ptax_internet/consultaBoletim.do?"
"method=exibeFormularioConsultaBoletim"
)
res = await _ASYNC_CLIENT.get(url1)
if res.status_code == 429:
raise BCBRateLimitError(
"BCB API rate limit exceeded. Please try again later.",
status_code=429,
)
if res.status_code == 404:
raise BCBAPINotFoundError(
"BCB API endpoint not found (404)",
status_code=404,
)
if res.status_code >= 500:
raise BCBAPIError(
f"BCB API server error (status {res.status_code})",
status_code=res.status_code,
)
if res.status_code != 200:
msg = f"BCB API Request error, status code = {res.status_code}"
raise BCBAPIError(msg, res.status_code)
try:
res = await _ASYNC_CLIENT.get(url1)
except httpx.HTTPError as ex:
raise_for_request_error(ex, context="Currency ID list")
raise_for_status(
res,
context="Currency ID list",
not_found_message="BCB API endpoint not found (404)",
)

doc = html.parse(BytesIO(res.content)).getroot()
xpath = "//select[@name='ChkMoeda']/option"
Expand All @@ -732,17 +697,16 @@ async def _async_get_valid_currency_list(
url2 = f"https://www4.bcb.gov.br/Download/fechamento/M{_date:%Y%m%d}.csv"
try:
res = await _ASYNC_CLIENT.get(url2)
except Exception as ex:
except httpx.HTTPError as ex:
if n >= 3:
raise ex
raise_for_request_error(ex, context="Currency list")
return await _async_get_valid_currency_list(_date, n + 1, max_rollback)

if res.status_code == 200:
return res
else:
return await _async_get_valid_currency_list(
_date - timedelta(1), 0, max_rollback
)
if res.status_code == 429 or res.status_code >= 500:
raise_for_status(res, context="Currency list")
return await _async_get_valid_currency_list(_date - timedelta(1), 0, max_rollback)


async def _async_get_currency_list(
Expand Down Expand Up @@ -794,9 +758,12 @@ async def _async_fetch_symbol_response(
"""Async version of _fetch_symbol_response()."""
cid = await _async_get_currency_id(symbol)
url = _currency_url(cid, start_date, end_date)
res = await _ASYNC_CLIENT.get(url)
try:
res = await _ASYNC_CLIENT.get(url)
except httpx.HTTPError as ex:
raise_for_request_error(ex, context=f"Currency data for {symbol}")

if res.headers["Content-Type"].startswith("text/html"):
if res.headers.get("Content-Type", "").startswith("text/html"):
doc = html.parse(BytesIO(res.content)).getroot()
xpath = "//div[@class='msgErro']"
elm = doc.xpath(xpath)[0]
Expand All @@ -806,26 +773,11 @@ async def _async_fetch_symbol_response(
msg = f"BCB API returned error: {x} - {symbol}"
raise BCBAPIError(msg, status_code=400)

if res.status_code == 429:
raise BCBRateLimitError(
"BCB API rate limit exceeded. Please try again later.",
status_code=429,
)
if res.status_code == 404:
raise BCBAPINotFoundError(
f"Currency data not found for {symbol}",
status_code=404,
)
if res.status_code >= 500:
raise BCBAPIError(
f"BCB API server error (status {res.status_code})",
status_code=res.status_code,
)
if res.status_code != 200:
raise BCBAPIError(
f"BCB API request failed with status {res.status_code}",
status_code=res.status_code,
)
raise_for_status(
res,
context=f"Currency data for {symbol}",
not_found_message=f"Currency data not found for {symbol}",
)

return res

Expand Down
70 changes: 69 additions & 1 deletion bcb/http.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

from __future__ import annotations

from typing import Callable, TypeVar
from typing import Callable, NoReturn, TypeVar

import httpx
from tenacity import (
Expand All @@ -11,6 +11,13 @@
wait_exponential,
)

from bcb.exceptions import (
BCBAPIError,
BCBAPINotFoundError,
BCBAPIServerError,
BCBRateLimitError,
)

# Default timeout for all HTTP requests (seconds)
DEFAULT_TIMEOUT = 30.0

Expand Down Expand Up @@ -82,6 +89,67 @@ def close_async_client() -> None:
T = TypeVar("T")


def _raise_error(
error_cls: type[Exception],
message: str,
status_code: int,
) -> NoReturn:
"""Raise project exceptions with or without an HTTP status constructor."""
if issubclass(error_cls, BCBAPIError):
raise error_cls(message, status_code)
raise error_cls(message)


def raise_for_status(
response: httpx.Response,
*,
context: str,
expected_status: int | tuple[int, ...] = 200,
error_cls: type[Exception] = BCBAPIError,
not_found_cls: type[Exception] = BCBAPINotFoundError,
rate_limit_cls: type[Exception] = BCBRateLimitError,
server_error_cls: type[Exception] = BCBAPIServerError,
rate_limit_message: str | None = None,
not_found_message: str | None = None,
server_error_message: str | None = None,
error_message: str | None = None,
) -> None:
"""Raise a consistent project exception for unexpected HTTP statuses."""
expected = (
(expected_status,) if isinstance(expected_status, int) else expected_status
)
status_code = response.status_code
if status_code in expected:
return

if status_code == 429:
message = (
rate_limit_message or "BCB API rate limit exceeded. Please try again later."
)
_raise_error(rate_limit_cls, message, status_code)
if status_code == 404:
message = not_found_message or f"{context} not found (status 404)"
_raise_error(not_found_cls, message, status_code)
if status_code >= 500:
message = (
server_error_message or f"{context} server error (status {status_code})"
)
_raise_error(server_error_cls, message, status_code)

message = error_message or f"{context} request failed with status {status_code}"
_raise_error(error_cls, message, status_code)


def raise_for_request_error(
exc: httpx.HTTPError,
*,
context: str,
error_cls: type[Exception] = BCBAPIError,
) -> NoReturn:
"""Raise a consistent project exception for HTTP client failures."""
_raise_error(error_cls, f"{context} request failed: {exc}", status_code=0)


def with_retry(func: Callable[..., T]) -> Callable[..., T]:
"""Decorator to add automatic retry with exponential backoff to any function.

Expand Down
Loading
Loading