Skip to content

Commit

Permalink
fix chess examples by using dlt rest client
Browse files Browse the repository at this point in the history
  • Loading branch information
mattseddon committed Mar 6, 2025
1 parent 63de7c5 commit bade748
Show file tree
Hide file tree
Showing 2 changed files with 11 additions and 8 deletions.
11 changes: 6 additions & 5 deletions docs/examples/chess/chess.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

from dlt.common import sleep
from dlt.common.typing import StrAny, TDataItems
from dlt.sources.helpers.requests import client
from dlt.sources.helpers.rest_client import RESTClient


@dlt.source
Expand All @@ -16,15 +16,16 @@ def chess(
year: int = 2022,
month: int = 10,
) -> Any:
client = RESTClient(base_url=chess_url)

def _get_data_with_retry(path: str) -> StrAny:
r = client.get(f"{chess_url}{path}")
r = client.get(path)
return r.json() # type: ignore

@dlt.resource(write_disposition="replace")
def players() -> Iterator[TDataItems]:
# return players one by one, you could also return a list that would be faster but we want to pass players item by item to the transformer
for p in _get_data_with_retry(f"titled/{title}")["players"][:max_players]:
yield p
yield from _get_data_with_retry(f"titled/{title}")["players"][:max_players]

# this resource takes data from players and returns profiles
# it uses `defer` decorator to enable parallel run in thread pool. defer requires return at the end so we convert yield into return (we return one item anyway)
Expand All @@ -41,7 +42,7 @@ def players_profiles(username: Any) -> TDataItems:
def players_games(username: Any) -> Iterator[TDataItems]:
# https://api.chess.com/pub/player/{username}/games/{YYYY}/{MM}
path = f"player/{username}/games/{year:04d}/{month:02d}"
yield _get_data_with_retry(path)["games"]
yield _get_data_with_retry(path).get("games")

return players(), players_profiles, players_games

Expand Down
8 changes: 5 additions & 3 deletions docs/examples/chess_production/chess_production.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,9 @@
import dlt
from dlt.common import sleep, logger
from dlt.common.typing import StrAny, TDataItems
from dlt.sources.helpers.requests import client
from dlt.pipeline.helpers import retry_load
from dlt.common.runtime.slack import send_slack_message
from dlt.sources.helpers.rest_client import RESTClient


@dlt.source
Expand All @@ -43,8 +43,10 @@ def chess(
year: int = 2022,
month: int = 10,
) -> Any:
client = RESTClient(base_url=chess_url)

def _get_data_with_retry(path: str) -> StrAny:
r = client.get(f"{chess_url}{path}")
r = client.get(path)
return r.json() # type: ignore

@dlt.resource(write_disposition="replace")
Expand All @@ -67,7 +69,7 @@ def players_profiles(username: Any) -> TDataItems:
def players_games(username: Any) -> Iterator[TDataItems]:
# https://api.chess.com/pub/player/{username}/games/{YYYY}/{MM}
path = f"player/{username}/games/{year:04d}/{month:02d}"
yield _get_data_with_retry(path)["games"]
yield _get_data_with_retry(path).get("games")

return players(), players_profiles, players_games

Expand Down

0 comments on commit bade748

Please sign in to comment.