VER: Release 0.10.0

See release notes.

Author: cjdsellers

.github/workflows/test.yml

@@ -12,7 +12,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: ["3.7", "3.8", "3.9", "3.10"]
+        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
     name: build - Python ${{ matrix.python-version }} (${{ matrix.os }})
     runs-on: ${{ matrix.os }}
 
@@ -31,5 +31,9 @@ jobs:
           scripts/build.sh
 
       # Run tests
+      - name: Run tests in release mode
+        if: ${{ github.ref == 'refs/heads/main' }}
+        run: scripts/test.sh --release
       - name: Run tests
+        if: ${{ github.ref != 'refs/heads/main' }}
         run: scripts/test.sh

CHANGELOG.md

@@ -1,6 +1,18 @@
 # Changelog
 
-## 0.8.2 - 2023-03-10
+## 0.10.0 - 2023-04-07
+- Upgraded `databento-dbn` to 0.4.3
+- Renamed `Bento` class to `DBNStore`
+- Removed `metadata.list_compressions` (redundant with docs)
+- Removed `metadata.list_encodings` (redundant with docs)
+- Removed optional `start` and `end` params from `metadata.list_schemas` (redundant)
+- Removed `related` and `related_security_id` from definition schema
+- Added `instrument_class`, `strike_price`, and `strike_price_currency` to definition
+  schema
+- Added support for `imbalance` schema
+- Improved exception messages for server and client timeouts
+
+## 0.9.0 - 2023-03-10
 - Removed `record_count` property from Bento class
 - Fixed bug in `Bento` where invalid metadata would prevent iteration
 - Improved use of the logging module

README.md

@@ -31,7 +31,7 @@ The library is fully compatible with the latest distribution of Anaconda 3.7 and
 The minimum dependencies as found in the `requirements.txt` are also listed below:
 - Python (>=3.7)
 - aiohttp (>=3.7.2)
-- databento-dbn (>=0.3.2)
+- databento-dbn (==0.4.3)
 - numpy (>=1.17.0)
 - pandas (>=1.1.3)
 - requests (>=2.24.0)

databento/__init__.py

@@ -1,7 +1,7 @@
 import logging
 
 from databento.common import utility
-from databento.common.bento import Bento
+from databento.common.dbnstore import DBNStore
 from databento.common.enums import (
     Compression,
     Dataset,
@@ -18,20 +18,20 @@
     SType,
     SymbologyResolution,
 )
-from databento.historical.api import API_VERSION
-from databento.historical.client import Historical
-from databento.historical.error import (
+from databento.common.error import (
     BentoClientError,
     BentoError,
     BentoHttpError,
     BentoServerError,
 )
+from databento.historical.api import API_VERSION
+from databento.historical.client import Historical
 from databento.version import __version__  # noqa
 
 
 __all__ = [
     "API_VERSION",
-    "Bento",
+    "DBNStore",
     "BentoClientError",
     "BentoError",
     "BentoHttpError",
@@ -58,4 +58,4 @@
 
 # Convenience imports
 enable_logging = utility.enable_logging
-from_dbn = Bento.from_file
+from_dbn = DBNStore.from_file

databento/common/data.py

@@ -114,7 +114,7 @@ def get_deriv_ba_types(level: int) -> List[Tuple[str, Union[type, str]]]:
     ("contract_multiplier", np.int32),
     ("decay_quantity", np.int32),
     ("original_contract_size", np.int32),
-    ("related_security_id", np.uint32),
+    ("reserved1", "S4"),
     ("trading_reference_date", np.uint16),
     ("appl_id", np.int16),
     ("maturity_year", np.uint16),
@@ -131,7 +131,11 @@ def get_deriv_ba_types(level: int) -> List[Tuple[str, Union[type, str]]]:
     ("security_type", "S7"),  # 7 byte chararray
     ("unit_of_measure", "S31"),  # 31 byte chararray
     ("underlying", "S21"),  # 21 byte chararray
-    ("related", "S21"),  # 21 byte chararray
+    ("strike_price_currency", "S4"),
+    ("instrument_class", "S1"),
+    ("reserved2", "S2"),
+    ("strike_price", np.int64),
+    ("reserved3", "S6"),
     ("match_algorithm", "S1"),  # 1 byte chararray
     ("md_security_trading_status", np.uint8),
     ("main_fraction", np.uint8),
@@ -150,6 +154,30 @@ def get_deriv_ba_types(level: int) -> List[Tuple[str, Union[type, str]]]:
     ("dummy", "S3"),  # 3 byte chararray (Adjustment filler for 8-bytes alignment)
 ]
 
+IMBALANCE_MSG: List[Tuple[str, Union[type, str]]] = RECORD_HEADER + [
+    ("ts_recv", np.uint64),
+    ("ref_price", np.int64),
+    ("auction_time", np.uint64),
+    ("cont_book_clr_price", np.int64),
+    ("auct_interest_clr_price", np.int64),
+    ("ssr_filling_price", np.int64),
+    ("ind_match_price", np.int64),
+    ("upper_collar", np.int64),
+    ("lower_collar", np.int64),
+    ("paired_qty", np.uint32),
+    ("total_imbalance_qty", np.uint32),
+    ("market_imbalance_qty", np.uint32),
+    ("unpaired_qty", np.uint32),
+    ("auction_type", "S1"),
+    ("side", "S1"),
+    ("auction_status", np.uint8),
+    ("freeze_status", np.uint8),
+    ("num_extensions", np.uint8),
+    ("unpaired_side", "S1"),
+    ("significant_imbalance", "S1"),
+    ("dummy", "S1"),
+]
+
 
 STRUCT_MAP: Dict[Schema, List[Tuple[str, Union[type, str]]]] = {
     Schema.MBO: MBO_MSG,
@@ -173,6 +201,7 @@ def get_deriv_ba_types(level: int) -> List[Tuple[str, Union[type, str]]]:
     Schema.OHLCV_1D: OHLCV_MSG,
     Schema.STATUS: STATUS_MSG,
     Schema.DEFINITION: DEFINITION_MSG,
+    Schema.IMBALANCE: IMBALANCE_MSG,
     Schema.GATEWAY_ERROR: RECORD_HEADER
     + [
         ("error", "S64"),
@@ -199,10 +228,10 @@ def get_deriv_ba_types(level: int) -> List[Tuple[str, Union[type, str]]]:
     "security_type",
     "unit_of_measure",
     "underlying",
-    "related",
     "match_algorithm",
     "security_update_action",
     "user_defined_instrument",
+    "strike_price_currency",
 ]
 
 DEFINITION_PRICE_COLUMNS = [
@@ -213,6 +242,7 @@ def get_deriv_ba_types(level: int) -> List[Tuple[str, Union[type, str]]]:
     "max_price_variation",
     "trading_reference_price",
     "min_price_increment_amount",
+    "strike_price",
 ]
 
 DEFINITION_TYPE_MAX_MAP = {
@@ -261,14 +291,37 @@ def get_deriv_ba_fields(level: int) -> List[str]:
     "volume",
 ]
 
-STATUS_COLUMNS = [x for x in np.dtype(STATUS_MSG).names or ()]
-STATUS_COLUMNS.remove("ts_recv")  # Index
+STATUS_DROP_COLUMNS = ["ts_recv"]
+DEFINITION_DROP_COLUMNS = [
+    "ts_recv",
+    "length",
+    "rtype",
+    "reserved1",
+    "reserved2",
+    "reserved3",
+    "dummy",
+]
+
+IMBALANCE_DROP_COLUMNS = [
+    "ts_recv",
+    "length",
+    "rtype",
+    "dummy",
+]
 
-DEFINITION_COLUMNS = [x for x in np.dtype(DEFINITION_MSG).names or ()]
-DEFINITION_COLUMNS.remove("ts_recv")  # Index
-DEFINITION_COLUMNS.remove("length")
-DEFINITION_COLUMNS.remove("rtype")
-DEFINITION_COLUMNS.remove("dummy")
+STATUS_COLUMNS = [
+    x for x in (np.dtype(STATUS_MSG).names or ()) if x not in STATUS_DROP_COLUMNS
+]
+
+DEFINITION_COLUMNS = [
+    x
+    for x in (np.dtype(DEFINITION_MSG).names or ())
+    if x not in DEFINITION_DROP_COLUMNS
+]
+
+IMBALANCE_COLUMNS = [
+    x for x in (np.dtype(IMBALANCE_MSG).names or ()) if x not in IMBALANCE_DROP_COLUMNS
+]
 
 
 COLUMNS = {
@@ -306,4 +359,5 @@ def get_deriv_ba_fields(level: int) -> List[str]:
     Schema.OHLCV_1D: OHLCV_HEADER_COLUMNS,
     Schema.STATUS: STATUS_COLUMNS,
     Schema.DEFINITION: DEFINITION_COLUMNS,
+    Schema.IMBALANCE: IMBALANCE_COLUMNS,
 }

databento/common/bento.py renamed to databento/common/dbnstore.py

@@ -30,6 +30,7 @@
     STRUCT_MAP,
 )
 from databento.common.enums import Compression, Schema, SType
+from databento.common.error import BentoError
 from databento.common.metadata import MetadataDecoder
 from databento.common.symbology import ProductIdMappingInterval
 
@@ -83,7 +84,7 @@ def is_dbn(reader: IO[bytes]) -> bool:
 
 
 class DataSource(abc.ABC):
-    """Abstract base class for backing Bento classes with data."""
+    """Abstract base class for backing DBNStore instances with data."""
 
     def __init__(self, source: object) -> None:
         ...
@@ -103,7 +104,7 @@ def reader(self) -> IO[bytes]:
 
 class FileDataSource(DataSource):
     """
-    A file-backed data source for a Bento object.
+    A file-backed data source for a DBNStore object.
 
     Attributes
     ----------
@@ -181,7 +182,7 @@ def reader(self) -> IO[bytes]:
 
 class MemoryDataSource(DataSource):
     """
-    A memory-backed data source for a Bento object.
+    A memory-backed data source for a DBNStore object.
 
     Attributes
     ----------
@@ -243,7 +244,7 @@ def reader(self) -> IO[bytes]:
         return self.__buffer
 
 
-class Bento:
+class DBNStore:
     """
     A container for Databento Binary Encoding (DBN) data.
 
@@ -345,11 +346,18 @@ def __init__(self, data_source: DataSource) -> None:
 
     def __iter__(self) -> Generator[np.void, None, None]:
         reader = self.reader
+        dtype = STRUCT_MAP[self.schema]
         while True:
             raw = reader.read(self.record_size)
             if raw:
-                rec = np.frombuffer(raw, dtype=STRUCT_MAP[self.schema])
-                yield rec[0]
+                try:
+                    rec = np.frombuffer(raw, dtype)
+                except ValueError as value_error:
+                    raise BentoError(
+                        f"Error decoding {len(raw)} bytes for {self.schema} iteration",
+                    ) from value_error
+                else:
+                    yield rec[0]
             else:
                 break
 
@@ -576,7 +584,7 @@ def raw(self) -> bytes:
 
         See Also
         --------
-        Bento.reader
+        DBNStore.reader
 
         """
         return self._data_source.reader.read()
@@ -592,7 +600,7 @@ def reader(self) -> IO[bytes]:
 
         See Also
         --------
-        Bento.raw
+        DBNStore.raw
 
         """
         if self.compression == Compression.ZSTD:
@@ -704,7 +712,7 @@ def symbols(self) -> List[str]:
         return self._metadata["symbols"]
 
     @classmethod
-    def from_file(cls, path: Union[PathLike[str], str]) -> "Bento":
+    def from_file(cls, path: Union[PathLike[str], str]) -> "DBNStore":
         """
         Load the data from a DBN file at the given path.
 
@@ -715,7 +723,7 @@ def from_file(cls, path: Union[PathLike[str], str]) -> "Bento":
 
         Returns
         -------
-        Bento
+        DBNStore
 
         Raises
         ------
@@ -726,7 +734,7 @@ def from_file(cls, path: Union[PathLike[str], str]) -> "Bento":
         return cls(FileDataSource(path))
 
     @classmethod
-    def from_bytes(cls, data: bytes) -> "Bento":
+    def from_bytes(cls, data: bytes) -> "DBNStore":
         """
         Load the data from a raw bytes.
 
@@ -737,7 +745,7 @@ def from_bytes(cls, data: bytes) -> "Bento":
 
         Returns
         -------
-        Bento
+        DBNStore
 
         Raises
         ------
@@ -771,7 +779,7 @@ def request_full_definitions(
         self,
         client: "Historical",
         path: Optional[Union[Path, str]] = None,
-    ) -> "Bento":
+    ) -> "DBNStore":
         """
         Request full instrument definitions based on the metadata properties.
 
@@ -782,11 +790,11 @@ def request_full_definitions(
         client : Historical
             The historical client to use for the request (contains the API key).
         path : Path or str, optional
-            The path to stream the data to on disk (will then return a `Bento`).
+            The path to stream the data to on disk (will then return a `DBNStore`).
 
         Returns
         -------
-        Bento
+        DBNStore
 
         Warnings
         --------
@@ -971,4 +979,11 @@ def to_ndarray(self) -> np.ndarray[Any, Any]:
 
         """
         data: bytes = self.reader.read()
-        return np.frombuffer(data, dtype=self.dtype)
+        try:
+            nd_array = np.frombuffer(data, dtype=self.dtype)
+        except ValueError as value_error:
+            raise BentoError(
+                f"Error decoding {len(data)} bytes to {self.schema} `ndarray`",
+            ) from value_error
+        else:
+            return nd_array

databento/common/enums.py

@@ -149,6 +149,7 @@ class Schema(StringyMixin, str, Enum):
     OHLCV_1H = "ohlcv-1h"
     OHLCV_1D = "ohlcv-1d"
     DEFINITION = "definition"
+    IMBALANCE = "imbalance"
     STATISTICS = "statistics"
     STATUS = "status"
     GATEWAY_ERROR = "gateway_error"

databento/historical/error.py renamed to databento/common/error.py

@@ -6,8 +6,6 @@ class BentoError(Exception):
     Represents a Databento specific error.
     """
 
-    pass
-
 
 class BentoHttpError(BentoError):
     """
@@ -105,5 +103,3 @@ class BentoWarning(Warning):
     """
     Represents a Databento specific warning.
     """
-
-    pass