VER: Release 0.33.0

See release notes.

Author: nmacholl

CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## 0.33.0 - 2024-04-16
+
+#### Enhancements
+- The `Historical.batch.download` and `Historical.batch.download_async` methods will now automatically retry the download if a rate limit (HTTP 429) error is received
+- The `Historical.batch.download` and `Historical.batch.download_async` methods will now retry failed downloads automatically
+- The `Historical.batch.download` and `Historical.batch.download_async` methods will now download files concurrently
+- The `output_dir` parameter for `Historical.batch.download` and `Historical.batch.download_async` is now optional and will default to the current working directory if unspecified
+
+#### Breaking changes
+- The `enable_partial_downloads` parameter for `Historical.batch.download` and `Historical.batch.download_async` has been removed, partial files will always be resumed which was the default behavior
+- The parameters for `Historical.batch.download` and `Historical.batch.download_async` have been reordered because `output_dir` is now optional, `job_id` now comes first
+
 ## 0.32.0 - 2024-04-04
 
 #### Enhancements

databento/common/constants.py

@@ -25,8 +25,6 @@
     x[0]: np.iinfo(x[1]).max for x in InstrumentDefMsg._dtypes if not isinstance(x[1], str)
 }
 
-INT64_NULL: Final = 9223372036854775807
-
 SCHEMA_STRUCT_MAP: Final[dict[Schema, type[DBNRecord]]] = {
     Schema.DEFINITION: InstrumentDefMsg,
     Schema.IMBALANCE: ImbalanceMsg,

databento/common/dbnstore.py

@@ -29,6 +29,7 @@
 import pytz
 import zstandard
 from databento_dbn import FIXED_PRICE_SCALE
+from databento_dbn import UNDEF_PRICE
 from databento_dbn import Compression
 from databento_dbn import DBNDecoder
 from databento_dbn import Encoding
@@ -42,7 +43,6 @@
 from databento_dbn import VersionUpgradePolicy
 
 from databento.common.constants import DEFINITION_TYPE_MAX_MAP
-from databento.common.constants import INT64_NULL
 from databento.common.constants import SCHEMA_STRUCT_MAP
 from databento.common.constants import SCHEMA_STRUCT_MAP_V1
 from databento.common.error import BentoError
@@ -1422,12 +1422,18 @@ def _format_map_symbols(self, df: pd.DataFrame) -> None:
         # the first ordered field will be ts_recv or ts_event when appropriate
         ts_name = self._struct_type._ordered_fields[0]
 
-        df_index = df[ts_name] if self._pretty_ts else pd.to_datetime(df[ts_name], utc=True)
-        dates = [ts.date() for ts in df_index]
-        df["symbol"] = [
-            self._instrument_map.resolve(inst, dates[i])
-            for i, inst in enumerate(df["instrument_id"])
-        ]
+        if df.empty:
+            df["symbol"] = []
+        else:
+            df["symbol"] = df.apply(
+                lambda r: self._instrument_map.resolve(
+                    r["instrument_id"],
+                    (
+                        r[ts_name] if self._pretty_ts else pd.to_datetime(r[ts_name], utc=True)
+                    ).date(),
+                ),
+                axis=1,
+            )
 
     def _format_timezone(self, df: pd.DataFrame) -> None:
         for field in self._struct_type._timestamp_fields:
@@ -1441,13 +1447,12 @@ def _format_px(
         px_fields = self._struct_type._price_fields
 
         if price_type == "decimal":
-            for field in px_fields:
-                df[field] = (
-                    df[field].replace(INT64_NULL, np.nan).apply(decimal.Decimal) / FIXED_PRICE_SCALE
-                )
+            df[px_fields] = (
+                df[px_fields].replace(UNDEF_PRICE, np.nan).applymap(decimal.Decimal)
+                / FIXED_PRICE_SCALE
+            )
         elif price_type == "float":
-            for field in px_fields:
-                df[field] = df[field].replace(INT64_NULL, np.nan) / FIXED_PRICE_SCALE
+            df[px_fields] = df[px_fields].replace(UNDEF_PRICE, np.nan) / FIXED_PRICE_SCALE
         else:
             return  # do nothing
 

databento/common/validation.py

@@ -38,7 +38,7 @@ def validate_path(value: PathLike[str] | str, param: str) -> Path:
         return Path(value)
     except TypeError:
         raise TypeError(
-            f"The `{param}` was not a valid path type. " "Use any of [str, bytes, os.PathLike].",
+            f"The `{param}` was not a valid path type. " "Use any of [PathLike[str], str].",
         ) from None