diff --git a/harp/io.py b/harp/io.py index 5b9091b..2b750c0 100644 --- a/harp/io.py +++ b/harp/io.py @@ -6,7 +6,7 @@ import numpy as np import numpy.typing as npt import pandas as pd -from pandas._typing import Axes +from pandas._typing import Axes # pyright: ignore[reportPrivateImportUsage] from harp.typing import _BufferLike, _FileLike diff --git a/harp/reader.py b/harp/reader.py index 590164c..27a91f4 100644 --- a/harp/reader.py +++ b/harp/reader.py @@ -3,14 +3,17 @@ from dataclasses import dataclass from datetime import datetime from functools import partial +from io import StringIO from math import log2 from os import PathLike from pathlib import Path from typing import Callable, Iterable, Mapping, Optional, Protocol, Union +import requests from numpy import dtype from pandas import DataFrame, Series -from pandas._typing import Axes +from pandas._typing import Axes # pyright: ignore[reportPrivateImportUsage] +from typing_extensions import deprecated from harp.io import MessageType, read from harp.model import BitMask, GroupMask, Model, PayloadMember, Register @@ -75,6 +78,238 @@ def __dir__(self) -> Iterable[str]: def __getattr__(self, __name: str) -> RegisterReader: return self.registers[__name] + @classmethod + def from_file( + cls, + filepath: Union[PathLike, str], + base_path: Optional[Union[PathLike, str]] = None, + include_common_registers: bool = True, + epoch: Optional[datetime] = None, + keep_type: bool = False, + ) -> "DeviceReader": + """Creates a device reader object from the specified schema yml file. + + Parameters + ---------- + filepath + A path to the device yml schema describing the device. + base_path + The path to attempt to resolve the location of data files. + include_common_registers + Specifies whether to include the set of Harp common registers in the + parsed device schema object. If a parsed device schema object is provided, + this parameter is ignored. + epoch + The default reference datetime at which time zero begins. If specified, + the data frames returned by each register reader will have a datetime index. + keep_type + Specifies whether to include a column with the message type by default. + + Returns + ------- + A device reader object which can be used to read binary data for each + register or to access metadata about each register. Individual registers + can be accessed using dot notation using the name of the register as the + key. + """ + + device = read_schema(filepath, include_common_registers) + if base_path is None: + path = Path(filepath).absolute().resolve() + base_path = path.parent / device.device + else: + base_path = Path(base_path).absolute().resolve() / device.device + + reg_readers = { + name: _create_register_handler(device, name, _ReaderParams(base_path, epoch, keep_type)) + for name in device.registers.keys() + } + return cls(device, reg_readers) + + @classmethod + def from_url( + cls, + url: str, + base_path: Optional[Union[PathLike, str]] = None, + include_common_registers: bool = True, + epoch: Optional[datetime] = None, + keep_type: bool = False, + timeout: int = 5, + ) -> "DeviceReader": + """Creates a device reader object from a url pointing to a device.yml file. + + Parameters + ---------- + url + The url pointing to the device.yml schema describing the device. + base_path + The path to attempt to resolve the location of data files. + include_common_registers + Specifies whether to include the set of Harp common registers in the + parsed device schema object. If a parsed device schema object is provided, + this parameter is ignored. + epoch + The default reference datetime at which time zero begins. If specified, + the data frames returned by each register reader will have a datetime index. + keep_type + Specifies whether to include a column with the message type by default. + timeout + The number of seconds to wait for the server to send data before giving up. + Returns + ------- + A device reader object which can be used to read binary data for each + register or to access metadata about each register. Individual registers + can be accessed using dot notation using the name of the register as the + key. + """ + + response = requests.get(url, timeout=timeout) + response.raise_for_status() + text = response.text + + return cls.from_str( + text, + base_path=base_path, + include_common_registers=include_common_registers, + epoch=epoch, + keep_type=keep_type, + ) + + @classmethod + def from_str( + cls, + schema: str, + base_path: Optional[Union[PathLike, str]] = None, + include_common_registers: bool = True, + epoch: Optional[datetime] = None, + keep_type: bool = False, + ) -> "DeviceReader": + """Creates a device reader object from a string containing a device.yml schema. + + Parameters + ---------- + schema + The string containing the device.yml schema describing the device. + base_path + The path to attempt to resolve the location of data files. + include_common_registers + Specifies whether to include the set of Harp common registers in the + parsed device schema object. If a parsed device schema object is provided, + this parameter is ignored. + epoch + The default reference datetime at which time zero begins. If specified, + the data frames returned by each register reader will have a datetime index. + keep_type + Specifies whether to include a column with the message type by default. + + Returns + ------- + A device reader object which can be used to read binary data for each + register or to access metadata about each register. Individual registers + can be accessed using dot notation using the name of the register as the + key. + """ + device = read_schema(StringIO(schema), include_common_registers) + if base_path is None: + base_path = Path(device.device).absolute().resolve() + else: + base_path = Path(base_path).absolute().resolve() + base_path = base_path / device.device + + reg_readers = { + name: _create_register_handler(device, name, _ReaderParams(base_path, epoch, keep_type)) + for name in device.registers.keys() + } + return cls(device, reg_readers) + + @classmethod + def from_model( + cls, + model: Model, + base_path: Optional[Union[PathLike, str]] = None, + epoch: Optional[datetime] = None, + keep_type: bool = False, + ) -> "DeviceReader": + """Creates a device reader object from a parsed device schema object. + + Parameters + ---------- + model + The parsed device schema object describing the device. + base_path + The path to attempt to resolve the location of data files. + epoch + The default reference datetime at which time zero begins. If specified, + the data frames returned by each register reader will have a datetime index. + keep_type + Specifies whether to include a column with the message type by default. + + Returns + ------- + A device reader object which can be used to read binary data for each + register or to access metadata about each register. Individual registers + can be accessed using dot notation using the name of the register as the + key. + """ + + if base_path is None: + base_path = Path(model.device).absolute().resolve() + else: + base_path = Path(base_path).absolute().resolve() + base_path = base_path / model.device + + reg_readers = { + name: _create_register_handler(model, name, _ReaderParams(base_path, epoch, keep_type)) + for name in model.registers.keys() + } + return cls(model, reg_readers) + + @classmethod + def from_dataset( + cls, + dataset: Union[PathLike, str], + include_common_registers: bool = True, + epoch: Optional[datetime] = None, + keep_type: bool = False, + ) -> "DeviceReader": + """Creates a device reader object from the specified dataset folder. + + Parameters + ---------- + dataset + A path to the dataset folder containing a device.yml schema describing the device. + include_common_registers + Specifies whether to include the set of Harp common registers in the + parsed device schema object. If a parsed device schema object is provided, + this parameter is ignored. + epoch + The default reference datetime at which time zero begins. If specified, + the data frames returned by each register reader will have a datetime index. + keep_type + Specifies whether to include a column with the message type by default. + + Returns + ------- + A device reader object which can be used to read binary data for each + register or to access metadata about each register. Individual registers + can be accessed using dot notation using the name of the register as the + key. + """ + + path = Path(dataset).absolute().resolve() + is_dir = os.path.isdir(path) + if is_dir: + filepath = path / "device.yml" + return cls.from_file( + filepath=filepath, + base_path=path, + include_common_registers=include_common_registers, + epoch=epoch, + keep_type=keep_type, + ) + else: + raise ValueError("The dataset must be a directory containing a device.yml file.") + def _compose_parser( f: Callable[[DataFrame], DataFrame], @@ -82,7 +317,7 @@ def _compose_parser( params: _ReaderParams, ) -> Callable[..., DataFrame]: def parser( - data, + data: Optional[Union[_FileLike, _BufferLike]] = None, columns: Optional[Axes] = None, epoch: Optional[datetime] = params.epoch, keep_type: bool = params.keep_type, @@ -229,6 +464,7 @@ def payload_parser(df: DataFrame): return RegisterReader(register, reader) +@deprecated("This function is deprecated. Use DeviceReader.from_* methods instead.") def create_reader( device: Union[str, PathLike, Model], include_common_registers: bool = True, diff --git a/pyproject.toml b/pyproject.toml index f8264b0..102ece7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,11 +7,12 @@ description = "A low-level interface for loading binary Harp protocol data" readme = "README.md" requires-python = ">=3.9.0" dynamic = ["version"] -license = {text = "MIT License"} +license = "MIT" dependencies = [ "pydantic-yaml", - "pandas" + "pandas", + "requests" ] classifiers = [ @@ -20,8 +21,7 @@ classifiers = [ "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", - "Operating System :: OS Independent", - "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent" ] [project.urls] diff --git a/tests/data/device.yml b/tests/data/device.yml index 583f290..636a6cf 100644 --- a/tests/data/device.yml +++ b/tests/data/device.yml @@ -24,6 +24,22 @@ registers: length: 3 access: Event description: Reports the current values of the analog input lines. + AnalogDataPayloadSpec: + address: 44 + type: S16 + length: 3 + access: Event + description: Reports the current values of the analog input lines. + payloadSpec: + AnalogInput0: + offset: 0 + description: The voltage at the output of the ADC channel 0. + Encoder: + offset: 1 + description: The quadrature counter value on Port 2 + AnalogInput1: + offset: 2 + description: The voltage at the output of the ADC channel 1. bitMasks: DigitalInputs: description: Specifies the state of the digital input lines. diff --git a/tests/test_reader.py b/tests/test_reader.py index 41c2776..23676b0 100644 --- a/tests/test_reader.py +++ b/tests/test_reader.py @@ -1,8 +1,10 @@ +from unittest.mock import Mock, patch + import pandas as pd from pytest import mark from harp.io import REFERENCE_EPOCH, MessageType -from harp.reader import create_reader +from harp.reader import DeviceReader, Model, create_reader, read_schema from tests.params import DeviceSchemaParam testdata = [ @@ -16,12 +18,15 @@ expected_whoAmI=0, expected_registers=["AnalogData"], ), + DeviceSchemaParam( + path="data/device.yml", + expected_whoAmI=0, + expected_registers=["AnalogDataPayloadSpec"], + ), ] -@mark.parametrize("schemaFile", testdata) -def test_create_reader(schemaFile: DeviceSchemaParam): - reader = create_reader(schemaFile.path, epoch=REFERENCE_EPOCH) +def helper_test_reader(reader: DeviceReader, schemaFile: DeviceSchemaParam) -> None: schemaFile.assert_schema(reader.device) whoAmI = reader.WhoAmI.read() @@ -36,3 +41,49 @@ def test_create_reader(schemaFile: DeviceSchemaParam): for register_name in schemaFile.expected_registers: data = reader.registers[register_name].read() assert isinstance(data.index, pd.DatetimeIndex) + + +@mark.parametrize("schemaFile", testdata) +@mark.filterwarnings("ignore:Call to deprecated") +def test_create_reader(schemaFile: DeviceSchemaParam): + reader = create_reader(schemaFile.path, epoch=REFERENCE_EPOCH) + helper_test_reader(reader, schemaFile) + + +@mark.parametrize("schemaFile", testdata) +def test_create_reader_from_file(schemaFile: DeviceSchemaParam): + reader = DeviceReader.from_file("./tests/data/device.yml", epoch=REFERENCE_EPOCH) + helper_test_reader(reader, schemaFile) + + +@mark.parametrize("schemaFile", testdata) +def test_create_reader_from_dataset(schemaFile: DeviceSchemaParam): + reader = DeviceReader.from_dataset("./tests/data", epoch=REFERENCE_EPOCH) + helper_test_reader(reader, schemaFile) + + +@mark.parametrize("schemaFile", testdata) +def test_create_reader_from_str(schemaFile: DeviceSchemaParam): + with open("./tests/data/device.yml", "r", encoding="utf-8") as f: + reader = DeviceReader.from_str(f.read(), base_path="./tests/data/", epoch=REFERENCE_EPOCH) + helper_test_reader(reader, schemaFile) + + +@mark.parametrize("schemaFile", testdata) +def test_create_reader_from_model(schemaFile: DeviceSchemaParam): + model = read_schema("./tests/data/device.yml", include_common_registers=True) + reader = DeviceReader.from_model(model=model, base_path="./tests/data/", epoch=REFERENCE_EPOCH) + helper_test_reader(reader, schemaFile) + + +@mark.parametrize("schemaFile", testdata) +@patch("requests.get") +def test_create_reader_from_url(mock_url_get, schemaFile: DeviceSchemaParam): + mock_response = Mock() + mock_response.status_code = 200 + mock_response.text = open("./tests/data/device.yml", "r", encoding="utf-8").read() + + mock_url_get.return_value = mock_response + + reader = DeviceReader.from_url("mocked_url", base_path="./tests/data/", epoch=REFERENCE_EPOCH) + helper_test_reader(reader, schemaFile)