From a575e539b97aa230a61a5a05fbbcf5afc05880fa Mon Sep 17 00:00:00 2001 From: Matt Seddon Date: Wed, 5 Mar 2025 13:28:40 +1100 Subject: [PATCH] fix chess examples by using dlt rest client --- docs/examples/chess/chess.py | 11 ++++++----- docs/examples/chess_production/chess_production.py | 8 +++++--- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/docs/examples/chess/chess.py b/docs/examples/chess/chess.py index 6af431b330..24cd2c3dcf 100644 --- a/docs/examples/chess/chess.py +++ b/docs/examples/chess/chess.py @@ -5,7 +5,7 @@ from dlt.common import sleep from dlt.common.typing import StrAny, TDataItems -from dlt.sources.helpers.requests import client +from dlt.sources.helpers.rest_client import RESTClient @dlt.source @@ -16,15 +16,16 @@ def chess( year: int = 2022, month: int = 10, ) -> Any: + client = RESTClient(base_url=chess_url) + def _get_data_with_retry(path: str) -> StrAny: - r = client.get(f"{chess_url}{path}") + r = client.get(path) return r.json() # type: ignore @dlt.resource(write_disposition="replace") def players() -> Iterator[TDataItems]: # return players one by one, you could also return a list that would be faster but we want to pass players item by item to the transformer - for p in _get_data_with_retry(f"titled/{title}")["players"][:max_players]: - yield p + yield from _get_data_with_retry(f"titled/{title}")["players"][:max_players] # this resource takes data from players and returns profiles # it uses `defer` decorator to enable parallel run in thread pool. defer requires return at the end so we convert yield into return (we return one item anyway) @@ -41,7 +42,7 @@ def players_profiles(username: Any) -> TDataItems: def players_games(username: Any) -> Iterator[TDataItems]: # https://api.chess.com/pub/player/{username}/games/{YYYY}/{MM} path = f"player/{username}/games/{year:04d}/{month:02d}" - yield _get_data_with_retry(path)["games"] + yield _get_data_with_retry(path).get("games") return players(), players_profiles, players_games diff --git a/docs/examples/chess_production/chess_production.py b/docs/examples/chess_production/chess_production.py index 196bc13e18..632faba31e 100644 --- a/docs/examples/chess_production/chess_production.py +++ b/docs/examples/chess_production/chess_production.py @@ -30,9 +30,9 @@ import dlt from dlt.common import sleep, logger from dlt.common.typing import StrAny, TDataItems -from dlt.sources.helpers.requests import client from dlt.pipeline.helpers import retry_load from dlt.common.runtime.slack import send_slack_message +from dlt.sources.helpers.rest_client import RESTClient @dlt.source @@ -43,8 +43,10 @@ def chess( year: int = 2022, month: int = 10, ) -> Any: + client = RESTClient(base_url=chess_url) + def _get_data_with_retry(path: str) -> StrAny: - r = client.get(f"{chess_url}{path}") + r = client.get(path) return r.json() # type: ignore @dlt.resource(write_disposition="replace") @@ -67,7 +69,7 @@ def players_profiles(username: Any) -> TDataItems: def players_games(username: Any) -> Iterator[TDataItems]: # https://api.chess.com/pub/player/{username}/games/{YYYY}/{MM} path = f"player/{username}/games/{year:04d}/{month:02d}" - yield _get_data_with_retry(path)["games"] + yield _get_data_with_retry(path).get("games") return players(), players_profiles, players_games