From ee6997d1efe4d018c37d1e4859a1afe72b29d644 Mon Sep 17 00:00:00 2001 From: Tim Paine <3105306+timkpaine@users.noreply.github.com> Date: Sat, 13 Jul 2024 13:32:26 -0400 Subject: [PATCH] Bump ruff to 0.7, normalize conda recipes, clean docs to only use makefile commands Add ruff isort to rules apply isort update settings Signed-off-by: Tim Paine <3105306+timkpaine@users.noreply.github.com> --- Makefile | 3 +- conda/dev-environment-unix.yml | 4 +- conda/dev-environment-win.yml | 8 +- csp/adapters/db.py | 3 +- csp/adapters/output_adapters/parquet.py | 5 +- .../output_adapters/parquet_utility_nodes.py | 3 +- csp/adapters/parquet.py | 5 +- csp/baselib.py | 5 +- csp/curve.py | 5 +- csp/impl/pandas.py | 3 +- csp/impl/pandas_accessor.py | 5 +- csp/impl/pandas_ext_type.py | 9 +- csp/impl/pandas_perspective.py | 5 +- csp/impl/struct.py | 3 +- csp/impl/types/container_type_normalizer.py | 3 +- csp/impl/types/instantiation_type_resolver.py | 3 +- csp/impl/types/pydantic_type_resolver.py | 3 +- csp/impl/types/pydantic_types.py | 3 +- csp/impl/types/typing_utils.py | 3 +- csp/impl/wiring/adapters.py | 1 + .../wiring/numba_utils/datetime_extension.py | 1 + .../wiring/numba_utils/numba_type_resolver.py | 3 +- csp/impl/wiring/runtime.py | 3 +- csp/math.py | 3 +- csp/profiler.py | 3 +- csp/random.py | 3 +- csp/stats.py | 5 +- csp/tests/adapters/test_db.py | 5 +- csp/tests/adapters/test_kafka.py | 3 +- csp/tests/adapters/test_numpy.py | 3 +- csp/tests/adapters/test_parquet.py | 9 +- csp/tests/adapters/test_status.py | 3 +- csp/tests/adapters/test_websocket.py | 3 +- csp/tests/impl/test_pandas.py | 3 +- csp/tests/impl/test_pandas_accessor.py | 5 +- csp/tests/impl/test_pandas_ext_type.py | 5 +- csp/tests/impl/test_pandas_perspective.py | 5 +- csp/tests/impl/test_struct.py | 17 +-- .../impl/types/test_pydantic_type_resolver.py | 5 +- csp/tests/impl/types/test_tstype.py | 7 +- csp/tests/test_baselib.py | 3 +- csp/tests/test_baskets.py | 3 +- csp/tests/test_curve.py | 3 +- csp/tests/test_dynamic.py | 3 +- csp/tests/test_engine.py | 5 +- csp/tests/test_examples.py | 3 +- csp/tests/test_history.py | 5 +- csp/tests/test_math.py | 3 +- csp/tests/test_profiler.py | 7 +- csp/tests/test_random.py | 3 +- csp/tests/test_stats.py | 7 +- csp/tests/test_type_checking.py | 3 +- csp/tests/test_typing.py | 3 +- csp/typing.py | 3 +- csp/utils/csp_watch_profile.py | 3 +- docs/wiki/dev-guides/Build-CSP-from-Source.md | 42 +------ examples/02_intermediate/e3_numpy_stats.py | 3 +- examples/05_cpp/1_cpp_node/setup.py | 1 + .../05_cpp/2_cpp_node_with_struct/setup.py | 1 + examples/06_advanced/e2_pandas_extension.py | 5 +- examples/07_end_to_end/earthquake.ipynb | 29 +++-- examples/07_end_to_end/mta.ipynb | 36 +++--- examples/07_end_to_end/seismic_waveform.ipynb | 110 +++++++++++------- examples/07_end_to_end/wikimedia.ipynb | 90 +++++++------- pyproject.toml | 29 ++--- setup.py | 1 + 66 files changed, 329 insertions(+), 251 deletions(-) diff --git a/Makefile b/Makefile index 6c8bf66f0..0d4098638 100644 --- a/Makefile +++ b/Makefile @@ -38,7 +38,6 @@ install: ## install library .PHONY: lint-py lint-cpp lint lints fix-py fix-cpp fix format check checks lint-py: - python -m isort --check csp/ examples/ setup.py python -m ruff check csp/ examples/ setup.py python -m ruff format --check csp/ examples/ setup.py @@ -57,7 +56,7 @@ lint: lint-py lint-docs ## run lints lints: lint fix-py: - python -m isort csp/ examples/ setup.py + python -m ruff check --fix csp/ examples/ setup.py python -m ruff format csp/ examples/ setup.py fix-cpp: diff --git a/conda/dev-environment-unix.yml b/conda/dev-environment-unix.yml index 19ab9a6b5..d084721c2 100644 --- a/conda/dev-environment-unix.yml +++ b/conda/dev-environment-unix.yml @@ -16,13 +16,13 @@ dependencies: - graphviz - gtest - httpx>=0.20,<1 - - isort>=5,<6 - libarrow=16 - libboost>=1.80.0 - libboost-headers>=1.80.0 - libprotobuf<5 - librdkafka - lz4-c + - make - mamba - mdformat>=0.7.17,<0.8 - ninja @@ -45,7 +45,7 @@ dependencies: - rapidjson - requests - ruamel.yaml - - ruff>=0.3,<0.4 + - ruff>=0.7,<0.8 - scikit-build - setuptools>=69,<74 - sqlalchemy diff --git a/conda/dev-environment-win.yml b/conda/dev-environment-win.yml index 11ed5c7e4..2b7dddd97 100644 --- a/conda/dev-environment-win.yml +++ b/conda/dev-environment-win.yml @@ -3,6 +3,7 @@ channels: - conda-forge - nodefaults dependencies: +# - bison # not available on windows - brotli - bump2version>=1 - cmake @@ -11,10 +12,10 @@ dependencies: - cyrus-sasl - deprecated - exprtk +# - flex # not available on windows - graphviz - gtest - httpx>=0.20,<1 - - isort>=5,<6 - libarrow=16 - libboost>=1.80.0 - libboost-headers>=1.80.0 @@ -44,12 +45,15 @@ dependencies: - rapidjson - requests - ruamel.yaml - - ruff>=0.3,<0.4 + - ruff>=0.7,<0.8 - scikit-build - setuptools>=69,<74 - sqlalchemy +# - tar # not available on windows - threadpoolctl - tornado - twine - typing-extensions +# - unzip # not available on windows - wheel +# - zip # not available on windows diff --git a/csp/adapters/db.py b/csp/adapters/db.py index d4529bc26..362d99874 100644 --- a/csp/adapters/db.py +++ b/csp/adapters/db.py @@ -8,8 +8,9 @@ except ImportError: from backports import zoneinfo -import pytz from importlib.metadata import PackageNotFoundError, version as get_package_version + +import pytz from packaging import version from csp import PushMode, ts diff --git a/csp/adapters/output_adapters/parquet.py b/csp/adapters/output_adapters/parquet.py index 1d6f8a183..594e2d281 100644 --- a/csp/adapters/output_adapters/parquet.py +++ b/csp/adapters/output_adapters/parquet.py @@ -1,9 +1,10 @@ -import numpy import os from importlib.metadata import PackageNotFoundError, version as get_package_version -from packaging import version from typing import Callable, Dict, Optional, TypeVar +import numpy +from packaging import version + import csp from csp.impl.struct import Struct from csp.impl.types.container_type_normalizer import ContainerTypeNormalizer diff --git a/csp/adapters/output_adapters/parquet_utility_nodes.py b/csp/adapters/output_adapters/parquet_utility_nodes.py index 7d1028e2f..1a856d415 100644 --- a/csp/adapters/output_adapters/parquet_utility_nodes.py +++ b/csp/adapters/output_adapters/parquet_utility_nodes.py @@ -1,6 +1,7 @@ -import numpy import typing +import numpy + import csp T = typing.TypeVar("T") diff --git a/csp/adapters/parquet.py b/csp/adapters/parquet.py index e88fcfeaa..d70662e68 100644 --- a/csp/adapters/parquet.py +++ b/csp/adapters/parquet.py @@ -1,11 +1,12 @@ import datetime import io +from importlib.metadata import PackageNotFoundError, version as get_package_version +from typing import TypeVar + import numpy import pyarrow import pyarrow.parquet -from importlib.metadata import PackageNotFoundError, version as get_package_version from packaging import version -from typing import TypeVar import csp from csp.adapters.output_adapters.parquet import ParquetOutputConfig, ParquetWriter, resolve_array_shape_column_name diff --git a/csp/baselib.py b/csp/baselib.py index 74e874b5e..1a628afe1 100644 --- a/csp/baselib.py +++ b/csp/baselib.py @@ -2,13 +2,14 @@ import contextlib import logging import math -import numpy as np -import pytz import queue import threading from datetime import datetime, timedelta from typing import Callable, Dict, List, Optional, TypeVar, Union +import numpy as np +import pytz + import csp from csp.impl.__cspimpl import _cspimpl from csp.impl.constants import UNSET diff --git a/csp/curve.py b/csp/curve.py index 32d4032ec..9237144fe 100644 --- a/csp/curve.py +++ b/csp/curve.py @@ -1,9 +1,10 @@ import copy -import numpy as np -import pytz from datetime import timedelta from typing import Union +import numpy as np +import pytz + from csp import null_ts from csp.impl.__cspimpl import _cspimpl from csp.impl.pulladapter import PullInputAdapter diff --git a/csp/impl/pandas.py b/csp/impl/pandas.py index c6c9f427c..00498e704 100644 --- a/csp/impl/pandas.py +++ b/csp/impl/pandas.py @@ -1,6 +1,7 @@ -import pandas from datetime import timedelta +import pandas + import csp from csp import ts diff --git a/csp/impl/pandas_accessor.py b/csp/impl/pandas_accessor.py index 3d7a0eb1b..0b9e1142f 100644 --- a/csp/impl/pandas_accessor.py +++ b/csp/impl/pandas_accessor.py @@ -1,9 +1,10 @@ +from datetime import datetime, timedelta +from typing import Dict, List, TypeVar, Union + import numpy as np import pandas as pd -from datetime import datetime, timedelta from pandas.api.extensions import register_dataframe_accessor, register_series_accessor from pandas.core.arrays import ExtensionArray -from typing import Dict, List, TypeVar, Union import csp from csp import ts diff --git a/csp/impl/pandas_ext_type.py b/csp/impl/pandas_ext_type.py index c4c681002..bd6b12c69 100644 --- a/csp/impl/pandas_ext_type.py +++ b/csp/impl/pandas_ext_type.py @@ -1,15 +1,16 @@ import copy import functools -import numpy as np import operator -import pandas as pd import re +from pydoc import locate +from typing import Any, List, Type, TypeVar, Union, cast + +import numpy as np +import pandas as pd from pandas.api.types import is_integer, is_list_like from pandas.core.arrays import ExtensionArray, ExtensionScalarOpsMixin, IntervalArray from pandas.core.dtypes.dtypes import PandasExtensionDtype, register_extension_dtype from pandas.core.indexers import check_array_indexer -from pydoc import locate -from typing import Any, List, Type, TypeVar, Union, cast import csp from csp.impl.types.tstype import TsType, isTsType, ts diff --git a/csp/impl/pandas_perspective.py b/csp/impl/pandas_perspective.py index 92b4f4e9d..769b639fe 100644 --- a/csp/impl/pandas_perspective.py +++ b/csp/impl/pandas_perspective.py @@ -1,8 +1,9 @@ +from datetime import datetime, timedelta +from typing import Optional + import pandas as pd import pytz -from datetime import datetime, timedelta from pandas.compat import set_function_name -from typing import Optional import csp import csp.impl.pandas_accessor # To ensure that the csp accessors are registered diff --git a/csp/impl/struct.py b/csp/impl/struct.py index 777cca8d4..db324d4f9 100644 --- a/csp/impl/struct.py +++ b/csp/impl/struct.py @@ -1,7 +1,8 @@ import io -import ruamel.yaml import typing from copy import deepcopy + +import ruamel.yaml from deprecated import deprecated import csp diff --git a/csp/impl/types/container_type_normalizer.py b/csp/impl/types/container_type_normalizer.py index 9b4d50360..973c84530 100644 --- a/csp/impl/types/container_type_normalizer.py +++ b/csp/impl/types/container_type_normalizer.py @@ -1,5 +1,6 @@ -import numpy import typing + +import numpy import typing_extensions import csp.typing diff --git a/csp/impl/types/instantiation_type_resolver.py b/csp/impl/types/instantiation_type_resolver.py index 33b964b70..974529573 100644 --- a/csp/impl/types/instantiation_type_resolver.py +++ b/csp/impl/types/instantiation_type_resolver.py @@ -1,9 +1,10 @@ import collections import inspect -import numpy import typing from abc import ABCMeta, abstractmethod +import numpy + import csp.typing from csp.impl.types import tstype from csp.impl.types.common_definitions import ArgKind, BasketKind, InputDef, OutputDef diff --git a/csp/impl/types/pydantic_type_resolver.py b/csp/impl/types/pydantic_type_resolver.py index 3a1dcf3b5..855c38f67 100644 --- a/csp/impl/types/pydantic_type_resolver.py +++ b/csp/impl/types/pydantic_type_resolver.py @@ -1,6 +1,7 @@ +from typing import Any, Dict, List, Set, Tuple, Type, Union, get_args + import numpy from pydantic import TypeAdapter, ValidationError -from typing import Any, Dict, List, Set, Tuple, Type, Union, get_args import csp.typing from csp.impl.types.container_type_normalizer import ContainerTypeNormalizer diff --git a/csp/impl/types/pydantic_types.py b/csp/impl/types/pydantic_types.py index b821934c3..14d42b48a 100644 --- a/csp/impl/types/pydantic_types.py +++ b/csp/impl/types/pydantic_types.py @@ -2,10 +2,11 @@ import sys import types import typing +from typing import Any, ForwardRef, Generic, Optional, Type, TypeVar, Union, get_args, get_origin + import typing_extensions from pydantic import GetCoreSchemaHandler, ValidationInfo, ValidatorFunctionWrapHandler from pydantic_core import CoreSchema, core_schema -from typing import Any, ForwardRef, Generic, Optional, Type, TypeVar, Union, get_args, get_origin from csp.impl.types.common_definitions import OutputBasket, OutputBasketContainer from csp.impl.types.tstype import SnapKeyType, SnapType, isTsDynamicBasket diff --git a/csp/impl/types/typing_utils.py b/csp/impl/types/typing_utils.py index 71630767b..e55bfde81 100644 --- a/csp/impl/types/typing_utils.py +++ b/csp/impl/types/typing_utils.py @@ -1,9 +1,10 @@ # utils for dealing with typing types -import numpy import sys import types import typing +import numpy + import csp.typing T = typing.TypeVar("T") diff --git a/csp/impl/wiring/adapters.py b/csp/impl/wiring/adapters.py index 794779e80..904095790 100644 --- a/csp/impl/wiring/adapters.py +++ b/csp/impl/wiring/adapters.py @@ -1,6 +1,7 @@ import inspect from datetime import timedelta from typing import List + from typing_extensions import override from csp.impl.__cspimpl import _cspimpl diff --git a/csp/impl/wiring/numba_utils/datetime_extension.py b/csp/impl/wiring/numba_utils/datetime_extension.py index b1668d534..739cd1b0b 100644 --- a/csp/impl/wiring/numba_utils/datetime_extension.py +++ b/csp/impl/wiring/numba_utils/datetime_extension.py @@ -7,6 +7,7 @@ """ import datetime + import numba from csp.impl.wiring.numba_utils.csp_cpp_numba_interface import C as csp_c diff --git a/csp/impl/wiring/numba_utils/numba_type_resolver.py b/csp/impl/wiring/numba_utils/numba_type_resolver.py index 290cf78e6..670642200 100644 --- a/csp/impl/wiring/numba_utils/numba_type_resolver.py +++ b/csp/impl/wiring/numba_utils/numba_type_resolver.py @@ -1,7 +1,8 @@ import datetime -import numba import typing +import numba + from csp.impl.types.typing_utils import CspTypingUtils from csp.impl.wiring.numba_utils import datetime_extension diff --git a/csp/impl/wiring/runtime.py b/csp/impl/wiring/runtime.py index e8d173057..a14388d71 100644 --- a/csp/impl/wiring/runtime.py +++ b/csp/impl/wiring/runtime.py @@ -1,9 +1,10 @@ -import pytz import threading import time from collections import deque from datetime import datetime, timedelta +import pytz + from csp.impl.__cspimpl import _cspimpl from csp.impl.error_handling import ExceptionContext from csp.impl.wiring.adapters import _graph_return_adapter diff --git a/csp/math.py b/csp/math.py index d1415d99c..d60b630ce 100644 --- a/csp/math.py +++ b/csp/math.py @@ -1,8 +1,9 @@ import math -import numpy as np from functools import lru_cache from typing import List, TypeVar, get_origin +import numpy as np + import csp from csp.impl.types.tstype import ts from csp.impl.wiring import node diff --git a/csp/profiler.py b/csp/profiler.py index 56f74825d..d6fd612ff 100644 --- a/csp/profiler.py +++ b/csp/profiler.py @@ -1,6 +1,5 @@ import base64 import gc -import numpy as np import pickle import sys import threading @@ -11,6 +10,8 @@ from io import BytesIO from typing import Dict, List +import numpy as np + import csp from csp.impl.genericpushadapter import GenericPushAdapter from csp.impl.struct import Struct diff --git a/csp/random.py b/csp/random.py index f5d749dff..e9411d86f 100644 --- a/csp/random.py +++ b/csp/random.py @@ -1,7 +1,8 @@ -import numpy as np from datetime import timedelta from typing import TypeVar +import numpy as np + import csp from csp import ts from csp.stats import numpy_to_list diff --git a/csp/stats.py b/csp/stats.py index d357970f3..1131fd33f 100644 --- a/csp/stats.py +++ b/csp/stats.py @@ -1,7 +1,8 @@ -import numpy as np from datetime import datetime, timedelta from typing import Any, List, Optional, TypeVar, Union +import numpy as np + import csp from csp import ts from csp.lib import _cspnpstatsimpl, _cspstatsimpl @@ -232,7 +233,7 @@ def _setup(x, interval, min_window, trigger, sampler, reset, weights=None, ignor if min_window is None: min_window = interval else: - if type(min_window) != type(interval): + if type(min_window) is not type(interval): raise TypeError("Interval and min_window must be of the same type") if sampler is None: diff --git a/csp/tests/adapters/test_db.py b/csp/tests/adapters/test_db.py index 87c247cf3..dd7fcd295 100644 --- a/csp/tests/adapters/test_db.py +++ b/csp/tests/adapters/test_db.py @@ -1,9 +1,10 @@ import os -import pytz -import sqlalchemy as db import unittest from datetime import date, datetime, time +import pytz +import sqlalchemy as db + import csp from csp.adapters.db import _SQLALCHEMY_2, DateTimeAccessor, DBReader, EngineStartTimeAccessor, TimestampAccessor diff --git a/csp/tests/adapters/test_kafka.py b/csp/tests/adapters/test_kafka.py index 36d67614c..06b9519e1 100644 --- a/csp/tests/adapters/test_kafka.py +++ b/csp/tests/adapters/test_kafka.py @@ -1,7 +1,8 @@ import os -import pytest from datetime import datetime, timedelta +import pytest + import csp from csp import ts from csp.adapters.kafka import ( diff --git a/csp/tests/adapters/test_numpy.py b/csp/tests/adapters/test_numpy.py index a12f5346d..ff9042b00 100644 --- a/csp/tests/adapters/test_numpy.py +++ b/csp/tests/adapters/test_numpy.py @@ -1,8 +1,9 @@ -import numpy as np import unittest from datetime import datetime, timedelta from typing import List +import numpy as np + import csp diff --git a/csp/tests/adapters/test_parquet.py b/csp/tests/adapters/test_parquet.py index 87590e30b..6d18b3e4a 100644 --- a/csp/tests/adapters/test_parquet.py +++ b/csp/tests/adapters/test_parquet.py @@ -1,14 +1,15 @@ import math -import numpy import os +import tempfile +import unittest +from datetime import datetime, timedelta + +import numpy import pandas import polars import pyarrow import pyarrow.parquet import pytz -import tempfile -import unittest -from datetime import datetime, timedelta import csp from csp.adapters.output_adapters.parquet import ParquetOutputConfig diff --git a/csp/tests/adapters/test_status.py b/csp/tests/adapters/test_status.py index 66cd41dd6..23c924df6 100644 --- a/csp/tests/adapters/test_status.py +++ b/csp/tests/adapters/test_status.py @@ -1,7 +1,8 @@ import os -import pytest from datetime import datetime, timedelta +import pytest + import csp from csp import ts from csp.adapters.kafka import DateTimeType, JSONTextMessageMapper, KafkaStatusMessageType diff --git a/csp/tests/adapters/test_websocket.py b/csp/tests/adapters/test_websocket.py index 41aeb3311..083628cfa 100644 --- a/csp/tests/adapters/test_websocket.py +++ b/csp/tests/adapters/test_websocket.py @@ -1,10 +1,11 @@ import os -import pytz import threading import unittest from datetime import datetime, timedelta from typing import List +import pytz + import csp from csp import ts diff --git a/csp/tests/impl/test_pandas.py b/csp/tests/impl/test_pandas.py index ae9d6e592..715a3f4a7 100644 --- a/csp/tests/impl/test_pandas.py +++ b/csp/tests/impl/test_pandas.py @@ -1,6 +1,7 @@ -import pandas as pd import unittest from datetime import datetime, timedelta + +import pandas as pd from numpy import NaN import csp diff --git a/csp/tests/impl/test_pandas_accessor.py b/csp/tests/impl/test_pandas_accessor.py index f42e64c45..696b49023 100644 --- a/csp/tests/impl/test_pandas_accessor.py +++ b/csp/tests/impl/test_pandas_accessor.py @@ -1,8 +1,9 @@ +import unittest +from datetime import date, datetime, timedelta + import numpy as np import pandas as pd import pytest -import unittest -from datetime import date, datetime, timedelta import csp import csp.impl.pandas_accessor diff --git a/csp/tests/impl/test_pandas_ext_type.py b/csp/tests/impl/test_pandas_ext_type.py index 453e67f46..f1d1d6f39 100644 --- a/csp/tests/impl/test_pandas_ext_type.py +++ b/csp/tests/impl/test_pandas_ext_type.py @@ -1,10 +1,11 @@ -import numpy as np import operator +from datetime import datetime + +import numpy as np import pandas as pd import pandas._testing as pd_test import pandas.tests.extension.base as base import pytest -from datetime import datetime from pandas import Series from pandas.tests.extension.conftest import * diff --git a/csp/tests/impl/test_pandas_perspective.py b/csp/tests/impl/test_pandas_perspective.py index 528ea8166..ed0e728d5 100644 --- a/csp/tests/impl/test_pandas_perspective.py +++ b/csp/tests/impl/test_pandas_perspective.py @@ -1,7 +1,8 @@ -import numpy as np -import pandas as pd import unittest from datetime import date, datetime, timedelta + +import numpy as np +import pandas as pd from packaging import version import csp diff --git a/csp/tests/impl/test_struct.py b/csp/tests/impl/test_struct.py index 1a5605b13..727497455 100644 --- a/csp/tests/impl/test_struct.py +++ b/csp/tests/impl/test_struct.py @@ -1,12 +1,13 @@ import enum import json -import numpy as np import pickle -import pytz import typing import unittest from datetime import date, datetime, time, timedelta from typing import Dict, List, Set, Tuple + +import numpy as np +import pytz from typing_extensions import Annotated import csp @@ -859,16 +860,16 @@ class S(csp.Struct): i: 2 f: 2.5 b: false - ls: - - 1 - - 2 + ls: + - 1 + - 2 - 3 lc: - value: [1,2,3] set_value: ["x","y","z"] - - value: + value: - 4 """ @@ -2541,7 +2542,7 @@ def test_list_field_repr(self): for ann_typ in struct_list_annotation_types: for typ, v in struct_list_test_values.items(): # Excluding str due to own repr implementation - if typ != str: + if typ is not str: class A(csp.Struct): a: ann_typ[typ] @@ -2563,7 +2564,7 @@ def test_list_field_str(self): for ann_typ in struct_list_annotation_types: for typ, v in struct_list_test_values.items(): # Excluding str due to own repr implementation - if typ != str: + if typ is not str: class A(csp.Struct): a: ann_typ[typ] diff --git a/csp/tests/impl/types/test_pydantic_type_resolver.py b/csp/tests/impl/types/test_pydantic_type_resolver.py index 3fef60355..e56b941d5 100644 --- a/csp/tests/impl/types/test_pydantic_type_resolver.py +++ b/csp/tests/impl/types/test_pydantic_type_resolver.py @@ -1,8 +1,9 @@ -import numpy as np -from pydantic import BaseModel, TypeAdapter, ValidationInfo, field_validator, model_validator from typing import Dict, Generic, List, Set, TypeVar, get_args, get_origin from unittest import TestCase +import numpy as np +from pydantic import BaseModel, TypeAdapter, ValidationInfo, field_validator, model_validator + import csp import csp.typing from csp import ts diff --git a/csp/tests/impl/types/test_tstype.py b/csp/tests/impl/types/test_tstype.py index ade10a836..1c0185e8f 100644 --- a/csp/tests/impl/types/test_tstype.py +++ b/csp/tests/impl/types/test_tstype.py @@ -1,10 +1,11 @@ -import numpy as np -import pytest import sys -from pydantic import TypeAdapter from typing import Dict, ForwardRef, Generic, List, Mapping, TypeVar, Union, get_args, get_origin from unittest import TestCase +import numpy as np +import pytest +from pydantic import TypeAdapter + import csp from csp import dynamic_demultiplex, ts from csp.impl.types.common_definitions import OutputBasket, Outputs diff --git a/csp/tests/test_baselib.py b/csp/tests/test_baselib.py index 08c8d56c9..a6aacab60 100644 --- a/csp/tests/test_baselib.py +++ b/csp/tests/test_baselib.py @@ -2,12 +2,13 @@ import itertools import logging import math -import numpy as np import unittest from datetime import date, datetime, timedelta, timezone from enum import Enum, auto from typing import List +import numpy as np + import csp from csp import ts from csp.baselib import _convert_ts_object_for_print diff --git a/csp/tests/test_baskets.py b/csp/tests/test_baskets.py index 22c95ef00..7fa8ef41b 100644 --- a/csp/tests/test_baskets.py +++ b/csp/tests/test_baskets.py @@ -1,4 +1,3 @@ -import numpy import random import time import unittest @@ -6,6 +5,8 @@ from datetime import datetime, timedelta from typing import Dict, List +import numpy + import csp import csp.impl from csp import ts diff --git a/csp/tests/test_curve.py b/csp/tests/test_curve.py index c0209ab13..3ce860240 100644 --- a/csp/tests/test_curve.py +++ b/csp/tests/test_curve.py @@ -1,7 +1,8 @@ -import numpy import unittest from datetime import datetime, timedelta +import numpy + import csp diff --git a/csp/tests/test_dynamic.py b/csp/tests/test_dynamic.py index 5b55f7823..d852d828a 100644 --- a/csp/tests/test_dynamic.py +++ b/csp/tests/test_dynamic.py @@ -1,5 +1,4 @@ import itertools -import numpy import random import string import time @@ -8,6 +7,8 @@ from datetime import datetime, timedelta from typing import Dict, List +import numpy + import csp from csp import ts diff --git a/csp/tests/test_engine.py b/csp/tests/test_engine.py index bc6c9b3d8..56686cfa3 100644 --- a/csp/tests/test_engine.py +++ b/csp/tests/test_engine.py @@ -1,9 +1,7 @@ import collections import gc -import numpy as np import os import pickle -import psutil import random import re import sys @@ -14,6 +12,9 @@ from datetime import datetime, timedelta from typing import Callable, Dict, List +import numpy as np +import psutil + import csp from csp import PushMode, ts from csp.impl.types.instantiation_type_resolver import ArgTypeMismatchError, TSArgTypeMismatchError diff --git a/csp/tests/test_examples.py b/csp/tests/test_examples.py index 2d2c03f6d..8dde99882 100644 --- a/csp/tests/test_examples.py +++ b/csp/tests/test_examples.py @@ -1,8 +1,9 @@ import importlib import os.path -import pytest import sys +import pytest + EXAMPLES_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "examples")) # make examples importable without keeping in source tree diff --git a/csp/tests/test_history.py b/csp/tests/test_history.py index 291da8bb5..ab7befdf4 100644 --- a/csp/tests/test_history.py +++ b/csp/tests/test_history.py @@ -1,10 +1,11 @@ -import numpy as np import os -import psutil import unittest from datetime import datetime, timedelta from typing import List +import numpy as np +import psutil + import csp from csp import ts diff --git a/csp/tests/test_math.py b/csp/tests/test_math.py index 2dcf0a283..f42572d5f 100644 --- a/csp/tests/test_math.py +++ b/csp/tests/test_math.py @@ -1,9 +1,10 @@ import math -import numpy as np import sys import unittest from datetime import datetime, timedelta +import numpy as np + import csp diff --git a/csp/tests/test_profiler.py b/csp/tests/test_profiler.py index 80ad1e871..f0a886d0e 100644 --- a/csp/tests/test_profiler.py +++ b/csp/tests/test_profiler.py @@ -1,7 +1,4 @@ -import numpy as np import os -import pandas as pd -import pytz import string import sys import tempfile @@ -11,6 +8,10 @@ from functools import reduce from typing import List +import numpy as np +import pandas as pd +import pytz + import csp import csp.stats as stats from csp import profiler, ts diff --git a/csp/tests/test_random.py b/csp/tests/test_random.py index 3a875ce1d..69d42e1a1 100644 --- a/csp/tests/test_random.py +++ b/csp/tests/test_random.py @@ -1,7 +1,8 @@ -import numpy as np import unittest from datetime import datetime, timedelta +import numpy as np + import csp from csp.random import brownian_motion, brownian_motion_1d, poisson_timer from csp.typing import Numpy1DArray, NumpyNDArray diff --git a/csp/tests/test_stats.py b/csp/tests/test_stats.py index 3e765dd75..bf2722ace 100644 --- a/csp/tests/test_stats.py +++ b/csp/tests/test_stats.py @@ -1,11 +1,12 @@ import math -import numpy as np -import numpy.testing -import pandas as pd import sys import unittest from datetime import datetime, timedelta +import numpy as np +import numpy.testing +import pandas as pd + import csp from csp.stats import _window_updates from csp.typing import Numpy1DArray, NumpyNDArray diff --git a/csp/tests/test_type_checking.py b/csp/tests/test_type_checking.py index 7c0493c34..97a2ec697 100644 --- a/csp/tests/test_type_checking.py +++ b/csp/tests/test_type_checking.py @@ -1,4 +1,3 @@ -import numpy as np import os import pickle import re @@ -6,6 +5,8 @@ import unittest from datetime import datetime, time, timedelta +import numpy as np + import csp import csp.impl.types.instantiation_type_resolver as type_resolver from csp import ts diff --git a/csp/tests/test_typing.py b/csp/tests/test_typing.py index c20f14c85..041e1aa39 100644 --- a/csp/tests/test_typing.py +++ b/csp/tests/test_typing.py @@ -1,6 +1,7 @@ +from unittest import TestCase + import numpy as np from pydantic import TypeAdapter -from unittest import TestCase from csp.typing import Numpy1DArray, NumpyNDArray diff --git a/csp/typing.py b/csp/typing.py index 4e1024e74..c242480b9 100644 --- a/csp/typing.py +++ b/csp/typing.py @@ -1,6 +1,7 @@ -import numpy from typing import Generic, TypeVar, get_args +import numpy + T = TypeVar("T") diff --git a/csp/utils/csp_watch_profile.py b/csp/utils/csp_watch_profile.py index 22446ed16..679259ed2 100644 --- a/csp/utils/csp_watch_profile.py +++ b/csp/utils/csp_watch_profile.py @@ -1,8 +1,9 @@ import argparse -import requests import time from datetime import datetime +import requests + from csp.profiler import ProfilerInfo diff --git a/docs/wiki/dev-guides/Build-CSP-from-Source.md b/docs/wiki/dev-guides/Build-CSP-from-Source.md index c24299f08..fbd4b2d52 100644 --- a/docs/wiki/dev-guides/Build-CSP-from-Source.md +++ b/docs/wiki/dev-guides/Build-CSP-from-Source.md @@ -124,9 +124,6 @@ git submodule update --init --recursive ```bash # for vcpkg sudo make dependencies-debian -# or -# sudo apt-get install -y automake bison cmake curl flex ninja-build tar unzip zip - # for g++ sudo apt install build-essential ``` @@ -136,9 +133,6 @@ sudo apt install build-essential ```bash # for vcpkg sudo make dependencies-fedora -# or -# yum install -y automake bison cmake curl flex perl-IPC-Cmd tar unzip zip - # for g++ sudo dnf group install "Development Tools" ``` @@ -150,8 +144,6 @@ sudo dnf group install "Development Tools" ```bash # for vcpkg make dependencies-mac -# or -# brew install bison cmake flex make ninja ``` ### Install Python dependencies @@ -160,11 +152,6 @@ Python build and develop dependencies are specified in the `pyproject.toml`, but ```bash make requirements - -# or -# python -m pip install toml -# python -m pip install `python -c 'import toml; c = toml.load("pyproject.toml"); print("\n".join(c["build-system"]["requires"]))'` -# python -m pip install `python -c 'import toml; c = toml.load("pyproject.toml"); print("\n".join(c["project"]["optional-dependencies"]["develop"]))'` ``` Note that these dependencies would otherwise be installed normally as part of [PEP517](https://peps.python.org/pep-0517/) / [PEP518](https://peps.python.org/pep-0518/). @@ -178,8 +165,6 @@ make build # on aarch64 linux, comment the above command and use this instead # VCPKG_FORCE_SYSTEM_BINARIES=1 make build -# or -# python setup.py build build_ext --inplace ``` ### Building on `aarch64` Linux @@ -198,56 +183,41 @@ By default, we pull and build dependencies with [vcpkg](https://vcpkg.io/en/). W CSP has linting and auto formatting. -| Language | Linter | Autoformatter | Description | -| :------- | :------------- | :------------- | :---------- | -| C++ | `clang-format` | `clang-format` | Style | -| Python | `ruff` | `ruff` | Style | -| Python | `isort` | `isort` | Imports | -| Markdown | `mdformat` | `mdformat` | Style | -| Markdown | `codespell` | | Spelling | +| Language | Linter | Autoformatter | Description | +| :------- | :------------- | :------------- | :------------- | +| C++ | `clang-format` | `clang-format` | Style | +| Python | `ruff` | `ruff` | Style, Imports | +| Markdown | `mdformat` | `mdformat` | Style | +| Markdown | `codespell` | | Spelling | **C++ Linting** ```bash make lint-cpp -# or -# clang-format --dry-run -Werror -i -style=file `find ./cpp/ -name "*.*pp"` ``` **C++ Autoformatting** ```bash make fix-cpp -# or -# clang-format -i -style=file `find ./cpp/ -name "*.*pp"` ``` **Python Linting** ```bash make lint-py -# or -# python -m isort --check csp/ setup.py -# python -m ruff check csp/ setup.py -# python -m ruff format --check csp/ setup.py ``` **Python Autoformatting** ```bash make fix-py -# or -# python -m isort csp/ setup.py -# python -m ruff format csp/ setup.py ``` **Documentation Linting** ```bash make lint-docs -# or -# python -m mdformat --check docs/wiki/ README.md examples/README.md -# python -m codespell_lib docs/wiki/ README.md examples/README.md ``` **Documentation Autoformatting** diff --git a/examples/02_intermediate/e3_numpy_stats.py b/examples/02_intermediate/e3_numpy_stats.py index 0debaf210..7f3e28877 100644 --- a/examples/02_intermediate/e3_numpy_stats.py +++ b/examples/02_intermediate/e3_numpy_stats.py @@ -1,6 +1,7 @@ -import numpy as np from datetime import datetime, timedelta +import numpy as np + import csp """ diff --git a/examples/05_cpp/1_cpp_node/setup.py b/examples/05_cpp/1_cpp_node/setup.py index 37696a368..3b178fe25 100644 --- a/examples/05_cpp/1_cpp_node/setup.py +++ b/examples/05_cpp/1_cpp_node/setup.py @@ -1,6 +1,7 @@ import os import os.path import sys + from skbuild import setup python_version = f"{sys.version_info.major}.{sys.version_info.minor}" diff --git a/examples/05_cpp/2_cpp_node_with_struct/setup.py b/examples/05_cpp/2_cpp_node_with_struct/setup.py index ace4a35c4..8975b5cdd 100644 --- a/examples/05_cpp/2_cpp_node_with_struct/setup.py +++ b/examples/05_cpp/2_cpp_node_with_struct/setup.py @@ -1,6 +1,7 @@ import os import os.path import sys + from skbuild import setup python_version = f"{sys.version_info.major}.{sys.version_info.minor}" diff --git a/examples/06_advanced/e2_pandas_extension.py b/examples/06_advanced/e2_pandas_extension.py index 29064c3cb..f7de9ddab 100644 --- a/examples/06_advanced/e2_pandas_extension.py +++ b/examples/06_advanced/e2_pandas_extension.py @@ -2,11 +2,12 @@ This example illustrates how csp edges can be used inside a pandas data frame via the pandas extension type mechanism. """ -import numpy as np -import pandas as pd import random from datetime import datetime, timedelta +import numpy as np +import pandas as pd + import csp import csp.impl.pandas_accessor # This registers the "csp" accessors on pd.Series and pd.DataFrame from csp.impl.pandas_ext_type import TsDtype diff --git a/examples/07_end_to_end/earthquake.ipynb b/examples/07_end_to_end/earthquake.ipynb index 29c0fb369..3acbd86a8 100644 --- a/examples/07_end_to_end/earthquake.ipynb +++ b/examples/07_end_to_end/earthquake.ipynb @@ -204,10 +204,11 @@ "metadata": {}, "outputs": [], "source": [ - "from perspective import PerspectiveWidget, Plugin\n", - "import ipywidgets as widgets\n", "from datetime import datetime\n", "\n", + "import ipywidgets as widgets\n", + "from perspective import PerspectiveWidget, Plugin\n", + "\n", "# Data schema for Perspective widget\n", "data = {\n", " \"longitude\": float,\n", @@ -269,12 +270,15 @@ "source": [ "# PushPullAdapter\n", "import threading\n", - "import csp\n", + "import time\n", + "from datetime import datetime, timedelta\n", + "\n", "from obspy import read_events\n", - "from datetime import datetime, timedelta, timezone\n", + "\n", + "import csp\n", "from csp.impl.pushpulladapter import PushPullInputAdapter\n", "from csp.impl.wiring import py_pushpull_adapter_def\n", - "import time\n", + "\n", "\n", "# We use a csp.Struct to store the earthquake event data\n", "class EventData(csp.Struct):\n", @@ -282,7 +286,8 @@ " longitude: float\n", " latitude: float\n", " magnitude: float\n", - " \n", + "\n", + "\n", "# Create a runtime implementation of the adapter\n", "class EarthquakeEventAdapter(PushPullInputAdapter):\n", " def __init__(self, interval, url):\n", @@ -345,7 +350,7 @@ " print(\"-------------------------------------------------------------------\")\n", " print(f\"{datetime.utcnow()}: Refreshing earthquake live feed with {len(new_events)} events\")\n", " print(\"-------------------------------------------------------------------\")\n", - " \n", + "\n", " for event in reversed(new_events):\n", " # Push live data\n", " event_data = EventData(\n", @@ -359,9 +364,13 @@ " self.push_tick(True, event_data.time, event_data)\n", "\n", " time.sleep(self._interval.total_seconds())\n", - " \n", + "\n", + "\n", "# Create the graph-time representation of our adapter\n", - "EarthquakeEvent = py_pushpull_adapter_def(\"EarthquakeEventAdapter\", EarthquakeEventAdapter, csp.ts[EventData], interval=timedelta, url=str)\n", + "EarthquakeEvent = py_pushpull_adapter_def(\n", + " \"EarthquakeEventAdapter\", EarthquakeEventAdapter, csp.ts[EventData], interval=timedelta, url=str\n", + ")\n", + "\n", "\n", "@csp.node\n", "def update_widget(event: csp.ts[EventData], widget: widgets.widgets.widget_selectioncontainer.Tab):\n", @@ -376,6 +385,7 @@ " widget.children[0].update(data)\n", " widget.children[1].update(data)\n", "\n", + "\n", "@csp.graph\n", "def earthquake_graph():\n", " print(\"Start of graph building\")\n", @@ -386,6 +396,7 @@ " csp.add_graph_output(\"Earthquakes\", earthquakes)\n", " print(\"End of graph building\")\n", "\n", + "\n", "start = datetime.utcnow() - timedelta(hours=24)\n", "end = datetime.utcnow() + timedelta(minutes=10)\n", "csp.run(earthquake_graph, starttime=start, endtime=end, realtime=True)\n", diff --git a/examples/07_end_to_end/mta.ipynb b/examples/07_end_to_end/mta.ipynb index 07ff0ffa4..8eede791d 100644 --- a/examples/07_end_to_end/mta.ipynb +++ b/examples/07_end_to_end/mta.ipynb @@ -222,7 +222,7 @@ "source": [ "# This cell can be run multiple times, and data will be refreshed every 30s\n", "feed.refresh()\n", - "trains = feed.filter_trips(underway=True, headed_for_stop_id=['128S', '128N'])\n", + "trains = feed.filter_trips(underway=True, headed_for_stop_id=[\"128S\", \"128N\"])\n", "trains" ] }, @@ -351,7 +351,7 @@ "print(\"Station | Line | Direction | Arrival time\")\n", "for train in trains:\n", " for update in train.stop_time_updates:\n", - " if update.stop_id in ['128S', '128N']:\n", + " if update.stop_id in [\"128S\", \"128N\"]:\n", " print(f\"{update.stop_name} | {train.route_id} | {train.headsign_text} | {update.arrival}\")\n", " trains_at_penn.append((train, update))" ] @@ -647,16 +647,15 @@ } ], "source": [ - "import csp\n", - "from csp.impl.pushadapter import PushInputAdapter\n", - "from csp.impl.wiring import py_push_adapter_def\n", + "import threading\n", + "import time\n", + "from datetime import datetime, timedelta\n", "\n", "import nyct_gtfs\n", "\n", - "import os\n", - "import time\n", - "import threading\n", - "from datetime import datetime, timedelta\n", + "import csp\n", + "from csp.impl.pushadapter import PushInputAdapter\n", + "from csp.impl.wiring import py_push_adapter_def\n", "\n", "\n", "class Event(csp.Struct):\n", @@ -665,6 +664,7 @@ " arrival: datetime\n", " direction: str\n", "\n", + "\n", "# Create a runtime implementation of the adapter\n", "class FetchTrainDataAdapter(PushInputAdapter):\n", " def __init__(self, interval, stations):\n", @@ -700,21 +700,28 @@ " for train in trains:\n", " for update in train.stop_time_updates:\n", " if update.stop_id in self._stations:\n", - " self.push_tick(Event(train=train, update=update, direction=train.direction, arrival=update.arrival))\n", + " self.push_tick(\n", + " Event(train=train, update=update, direction=train.direction, arrival=update.arrival)\n", + " )\n", " time.sleep(self._interval.total_seconds())\n", "\n", + "\n", "# Create the graph-time representation of our adapter\n", - "FetchTrainData = py_push_adapter_def(\"FetchTrainData\", FetchTrainDataAdapter, csp.ts[Event], interval=timedelta, stations=list)\n", + "FetchTrainData = py_push_adapter_def(\n", + " \"FetchTrainData\", FetchTrainDataAdapter, csp.ts[Event], interval=timedelta, stations=list\n", + ")\n", + "\n", "\n", "@csp.node\n", "def pretty_print(train_data: csp.ts[Event], count: csp.ts[float]) -> csp.ts[str]:\n", " message = f\" {train_data.update.stop_name} | {train_data.train.route_id} | {train_data.train.headsign_text} | {train_data.update.arrival} | Southbound train count: {int(count)}\"\n", " return message\n", "\n", + "\n", "@csp.graph\n", "def mta_graph():\n", " print(\"Start of graph building\")\n", - " stations = ['128S', '128N']\n", + " stations = [\"128S\", \"128N\"]\n", " interval = timedelta(seconds=30)\n", " trains_at_penn = FetchTrainData(interval, stations=stations)\n", " # trains_at_penn is an edge that can be processed through a node.\n", @@ -723,11 +730,14 @@ " # Convert timestamps to unique float values\n", " timestamp = csp.apply(south_trains.arrival, datetime.timestamp, float)\n", " # Count the number of unique entries in this timeseries block (reset every 30 seconds)\n", - " count = csp.stats.count(csp.stats.unique(timestamp), interval=timedelta(seconds=30), min_window=timedelta(seconds=1))\n", + " count = csp.stats.count(\n", + " csp.stats.unique(timestamp), interval=timedelta(seconds=30), min_window=timedelta(seconds=1)\n", + " )\n", " result = pretty_print(trains_at_penn, count)\n", " csp.print(\":\", result)\n", " print(\"End of graph building\")\n", "\n", + "\n", "start = datetime.utcnow()\n", "end = start + timedelta(minutes=3)\n", "csp.run(mta_graph, starttime=start, realtime=True, endtime=end)\n", diff --git a/examples/07_end_to_end/seismic_waveform.ipynb b/examples/07_end_to_end/seismic_waveform.ipynb index 9ee2e29bd..d7940eb05 100644 --- a/examples/07_end_to_end/seismic_waveform.ipynb +++ b/examples/07_end_to_end/seismic_waveform.ipynb @@ -37,19 +37,21 @@ "metadata": {}, "outputs": [], "source": [ - "import csp\n", "import logging\n", - "import numpy as np\n", - "import pandas as pd\n", "import sys\n", "import warnings\n", - "from csp import ts\n", - "from datetime import timedelta, datetime\n", - "from obspy import read, Trace, Stream, UTCDateTime\n", - "from scipy.signal import iirfilter, sosfilt, zpk2sos\n", + "from datetime import datetime, timedelta\n", "from typing import Dict, List, Optional, Tuple\n", "\n", - "warnings.filterwarnings('ignore') " + "import numpy as np\n", + "import pandas as pd\n", + "from obspy import Stream, Trace, UTCDateTime, read\n", + "from scipy.signal import iirfilter, sosfilt, zpk2sos\n", + "\n", + "import csp\n", + "from csp import ts\n", + "\n", + "warnings.filterwarnings(\"ignore\")" ] }, { @@ -230,6 +232,7 @@ " for k, tr in sim.items():\n", " csp.print(f\"Received simulated trace {k}\", tr)\n", "\n", + "\n", "starttime = st[0].stats.starttime.datetime\n", "csp.run(print_slices, st, lag=timedelta(seconds=0.01), starttime=starttime)" ] @@ -271,7 +274,14 @@ } ], "source": [ - "csp.run(print_slices, st, lag=datetime.utcnow()-starttime, starttime=datetime.utcnow(), endtime=timedelta(seconds=30), realtime=True)" + "csp.run(\n", + " print_slices,\n", + " st,\n", + " lag=datetime.utcnow() - starttime,\n", + " starttime=datetime.utcnow(),\n", + " endtime=timedelta(seconds=30),\n", + " realtime=True,\n", + ")" ] }, { @@ -302,7 +312,7 @@ " if csp.ticked(trace):\n", " if s_sos is None: # Design the filter the first time we see a trace obser\n", " s_df = trace.stats.sampling_rate\n", - " z, p, k = iirfilter(4, [2*freqmin/s_df, 2*freqmax/s_df], btype=\"band\", ftype=\"butter\", output=\"zpk\")\n", + " z, p, k = iirfilter(4, [2 * freqmin / s_df, 2 * freqmax / s_df], btype=\"band\", ftype=\"butter\", output=\"zpk\")\n", " s_sos = zpk2sos(z, p, k)\n", " s_zi = np.zeros((s_sos.shape[0], 2))\n", " else:\n", @@ -357,6 +367,7 @@ " csp.print(f\"Calculated filtered trace {k}\", filtered)\n", " csp.add_graph_output(k, filtered)\n", "\n", + "\n", "freqmin = 1\n", "freqmax = 20\n", "st = read()\n", @@ -450,13 +461,13 @@ "@csp.graph\n", "def sta_lta_ts(trace: ts[Trace], nsta: int, nlta: int) -> ts[float]:\n", " # In the line below, we extract the data from each Trace mini-batch and \"unroll\" the data so that we can process it in sequence with csp.\n", - " # Thus, on each tick of trace we will immediately process a batch of the past observations, but one at a time. \n", + " # Thus, on each tick of trace we will immediately process a batch of the past observations, but one at a time.\n", " inputs = csp.unroll(csp.apply(trace, lambda tr: tr.data.tolist(), List[float]))\n", " square = inputs**2\n", - " sta = csp.stats.ema(square, alpha=1./nsta, min_periods=nlta, adjust=False)\n", - " lta = csp.stats.ema(square, alpha=1./nlta, min_periods=nlta, adjust=False)\n", - " # (Optional) Return zero if sta/lta hasn't ticked yet (due to the window not being full). We do this to reconcile more closely with the obspy behavior. \n", - " return csp.merge(sta/lta, csp.sample(square, csp.const(0)))" + " sta = csp.stats.ema(square, alpha=1.0 / nsta, min_periods=nlta, adjust=False)\n", + " lta = csp.stats.ema(square, alpha=1.0 / nlta, min_periods=nlta, adjust=False)\n", + " # (Optional) Return zero if sta/lta hasn't ticked yet (due to the window not being full). We do this to reconcile more closely with the obspy behavior.\n", + " return csp.merge(sta / lta, csp.sample(square, csp.const(0)))" ] }, { @@ -503,12 +514,14 @@ "source": [ "starttime = st[0].stats.starttime.datetime\n", "tr_filt = bandpass_filter_ts(\n", - " trace_sim(\n", - " st, \n", - " interval=timedelta(seconds=5), # Assume we receive new data every 5 seconds \n", - " lag=timedelta(seconds=0.01))[\"BW.RJOB..EHZ\"], # Assume data is delayed some tiny amount\n", - " freqmin=1, \n", - " freqmax=20)\n", + " trace_sim(\n", + " st,\n", + " interval=timedelta(seconds=5), # Assume we receive new data every 5 seconds\n", + " lag=timedelta(seconds=0.01),\n", + " )[\"BW.RJOB..EHZ\"], # Assume data is delayed some tiny amount\n", + " freqmin=1,\n", + " freqmax=20,\n", + ")\n", "timestamps_csp, cft_csp = csp.run(sta_lta_ts(tr_filt, nsta, nlta), starttime=starttime, output_numpy=True)[0]" ] }, @@ -529,7 +542,8 @@ "outputs": [], "source": [ "from obspy.signal.trigger import recursive_sta_lta_py\n", - "tr = tr.copy().filter(\"bandpass\", freqmin=1, freqmax=20) \n", + "\n", + "tr = tr.copy().filter(\"bandpass\", freqmin=1, freqmax=20)\n", "cft = recursive_sta_lta_py(tr.data, nsta, nlta)" ] }, @@ -569,7 +583,7 @@ } ], "source": [ - "pd.DataFrame({\"cft_csp\":cft_csp, \"cft\":cft}).plot()" + "pd.DataFrame({\"cft_csp\": cft_csp, \"cft\": cft}).plot()" ] }, { @@ -594,7 +608,7 @@ " if not s_on and cft > thrOn:\n", " s_on = True\n", " return s_on\n", - " elif s_on and cft< thrOff:\n", + " elif s_on and cft < thrOff:\n", " s_on = False\n", " return s_on" ] @@ -630,10 +644,14 @@ "source": [ "thrOn = 4\n", "thrOff = 0.7\n", + "\n", + "\n", "@csp.graph\n", "def print_trace_and_trigger(tr_filt: ts[Trace]):\n", " csp.print(\"Trace\", tr_filt)\n", " csp.print(\"Trigger On?\", trigger_node(sta_lta_ts(tr_filt, nsta, nlta), thrOn, thrOff))\n", + "\n", + "\n", "csp.run(print_trace_and_trigger, tr_filt, starttime=starttime)" ] }, @@ -667,10 +685,11 @@ } ], "source": [ - "from obspy.signal.trigger import plot_trigger, trigger_onset\n", + "from obspy.signal.trigger import plot_trigger\n", + "\n", "thrOn = 4\n", "thrOff = 0.7\n", - "plot_trigger(tr, cft, thrOn, thrOff) " + "plot_trigger(tr, cft, thrOn, thrOff)" ] }, { @@ -691,6 +710,7 @@ "source": [ "from obspy.clients.fdsn import Client\n", "\n", + "\n", "@csp.graph\n", "def fdsn_waveforms(\n", " client: Client,\n", @@ -818,12 +838,15 @@ "t = UTCDateTime(\"2012-12-14T10:36:01.6Z\")\n", "starttime = (t + 300).datetime\n", "endtime = (t + 400).datetime\n", + "\n", + "\n", "@csp.graph\n", "def graph():\n", " basket = fdsn_waveforms(client, \"TA\", \"E42A\", \"*\", \"BH?\", timedelta(seconds=4))\n", " for k, v in basket.items():\n", " csp.print(k, v)\n", "\n", + "\n", "out = csp.run(graph, starttime=starttime, endtime=endtime)" ] }, @@ -851,13 +874,13 @@ "metadata": {}, "outputs": [], "source": [ - "import logging\n", "import threading\n", "from fnmatch import fnmatch\n", + "from urllib.parse import urlparse\n", + "\n", "from obspy.clients.seedlink.client.seedlinkconnection import SeedLinkConnection\n", "from obspy.clients.seedlink.client.slstate import SLState\n", "from obspy.clients.seedlink.slpacket import SLPacket\n", - "from urllib.parse import urlparse\n", "\n", "from csp.impl.adaptermanager import AdapterManagerImpl\n", "from csp.impl.pushadapter import PushInputAdapter\n", @@ -865,16 +888,16 @@ "\n", "\n", "class EasySeedLinkAdapter:\n", - " \"\"\"Real time adapter (technically an adapter manager of push adapters) that manages subscription to multiple seedlink streams for the same connection. \"\"\"\n", - " \n", + " \"\"\"Real time adapter (technically an adapter manager of push adapters) that manages subscription to multiple seedlink streams for the same connection.\"\"\"\n", + "\n", " def __init__(self, server_url: str):\n", - " \"\"\" Constructor\n", - " \n", + " \"\"\"Constructor\n", + "\n", " Args:\n", " server_url: The url and port of the seedlink server we connect to.\n", " \"\"\"\n", " self._server_url = server_url\n", - " \n", + "\n", " self._server_url = server_url\n", " parsed_url = urlparse(server_url, scheme=\"seedlink\")\n", " self._conn = SeedLinkConnection()\n", @@ -902,8 +925,8 @@ "\n", "\n", "class _EasySeedLinkAdapterManagerImpl(AdapterManagerImpl):\n", - " \"\"\" This is the implementation of the Adapter Manager, which is shared across multiple adapter implementations \"\"\"\n", - " \n", + " \"\"\"This is the implementation of the Adapter Manager, which is shared across multiple adapter implementations\"\"\"\n", + "\n", " def __init__(self, engine, server_url: str):\n", " super().__init__(engine)\n", " parsed_url = urlparse(server_url, scheme=\"seedlink\")\n", @@ -918,9 +941,9 @@ " self._thread = None\n", " self._starttime = None\n", " self._endtime = None\n", - " \n", + "\n", " def start(self, starttime, endtime):\n", - " \"\"\"start will get called at the start of the engine run. At this point one would start up the realtime data source / spawn the driving \n", + " \"\"\"start will get called at the start of the engine run. At this point one would start up the realtime data source / spawn the driving\n", " thread(s) and subscribe to the needed data\"\"\"\n", " self._running = True\n", " self._starttime = starttime\n", @@ -933,7 +956,7 @@ "\n", " def stop(self):\n", " \"\"\"This will be called at the end of the engine run, at which point resources should be closed and cleaned up\"\"\"\n", - " if self._running: \n", + " if self._running:\n", " self._running = False\n", " self._thread.join()\n", "\n", @@ -967,7 +990,7 @@ " break\n", " elif data == SLPacket.SLERROR:\n", " continue\n", - " \n", + "\n", " if data.get_type() not in (SLPacket.TYPE_SLINF, SLPacket.TYPE_SLINFT):\n", " trace = data.get_trace()\n", " for (network, station, selector), adapters in self._inputs.items():\n", @@ -983,6 +1006,7 @@ "\n", "class _EasySeedLinkAdapterImpl(PushInputAdapter):\n", " \"\"\"Our input adapter is a very simple implementation, and just defers its work back to the manager who is expected to deal with sharing a single connection.\"\"\"\n", + "\n", " def __init__(self, manager_impl, network, station, selector):\n", " manager_impl.register_input_adapter(network, station, selector, self)\n", " super().__init__()\n", @@ -1124,9 +1148,9 @@ "source": [ "# Connect to the iris seedlink server, which covers many US stations\n", "address = \"http://rtserve.iris.washington.edu:18000\"\n", - "symbols = [(\"LD\", \"PAL\", \"??Z\"), (\"LD\", \"CUNY\", \"??Z\")] # Palisades, NY and Queens College, CUNY, NY \n", + "symbols = [(\"LD\", \"PAL\", \"??Z\"), (\"LD\", \"CUNY\", \"??Z\")] # Palisades, NY and Queens College, CUNY, NY\n", "# Could also subscribe to station via wildcard, to get a given channel from all available stations!\n", - "#symbols = [(\"LD\", \"*\", \"HHZ\")]\n", + "# symbols = [(\"LD\", \"*\", \"HHZ\")]\n", "\n", "logging.basicConfig(stream=sys.stdout, format=\"[%(asctime)s] %(levelname)s: %(message)s\", level=logging.INFO)\n", "csp.run(\n", @@ -1170,13 +1194,13 @@ " adapter_manager = EasySeedLinkAdapter(address)\n", " for symbol in symbols:\n", " data = adapter_manager.subscribe(*symbol)\n", - " csp.log(logging.INFO, f\"Raw data:\", data)\n", + " csp.log(logging.INFO, \"Raw data:\", data)\n", " tr_filt = bandpass_filter_ts(data, freqmin=1, freqmax=20)\n", " cft = sta_lta_ts(tr_filt, nsta, nlta)\n", " # Because on each tick of \"data\", we generate several values of \"cft\" by unrolling the observations, we want to trigger reporting of the maxim cft over each batch\n", " trigger = csp.delay(data, timedelta.resolution)\n", " min_cft = csp.stats.min(cft, trigger=trigger, reset=data)\n", - " max_cft = csp.stats.max(cft, trigger=trigger, reset=data) \n", + " max_cft = csp.stats.max(cft, trigger=trigger, reset=data)\n", " csp.log(logging.INFO, f\"Characteristic function for {symbol}, Min\", min_cft)\n", " csp.log(logging.INFO, f\"Characteristic function for {symbol}, Max\", max_cft)" ] diff --git a/examples/07_end_to_end/wikimedia.ipynb b/examples/07_end_to_end/wikimedia.ipynb index 91c85b18a..b673464a1 100644 --- a/examples/07_end_to_end/wikimedia.ipynb +++ b/examples/07_end_to_end/wikimedia.ipynb @@ -112,7 +112,7 @@ "source": [ "from datetime import datetime, timedelta\n", "\n", - "URL = 'https://stream.wikimedia.org/v2/stream/recentchange'" + "URL = \"https://stream.wikimedia.org/v2/stream/recentchange\"" ] }, { @@ -134,7 +134,9 @@ "\n", "# Data schema\n", "data = {\"servername\": str, \"timestamp\": datetime, \"event\": str, \"servername_count\": int}\n", - "widget = PerspectiveWidget(data, plugin=Plugin.XBAR, group_by=[\"servername\"], columns=[\"servername_count\"], theme='Pro Light')\n", + "widget = PerspectiveWidget(\n", + " data, plugin=Plugin.XBAR, group_by=[\"servername\"], columns=[\"servername_count\"], theme=\"Pro Light\"\n", + ")\n", "widget" ] }, @@ -157,14 +159,15 @@ }, "outputs": [], "source": [ - "import re\n", "import json\n", + "import re\n", "import threading\n", "\n", "from sseclient import SSEClient as EventSource\n", "\n", "import csp\n", "\n", + "\n", "class WikiData(csp.Struct):\n", " servername: str\n", " servername_count: int\n", @@ -194,6 +197,7 @@ "from csp.impl.pushadapter import PushInputAdapter\n", "from csp.impl.wiring import py_push_adapter_def\n", "\n", + "\n", "# Define the runtime implementation of our adapter\n", "class FetchWikiDataAdapter(PushInputAdapter):\n", " def __init__(self, url: str):\n", @@ -216,36 +220,31 @@ " self._source.close()\n", "\n", " def _run(self):\n", - " servernames = dict([])\n", " while self._running:\n", " for item in self._source:\n", - " if item.event == 'message':\n", + " if item.event == \"message\":\n", " try:\n", " change = json.loads(item.data)\n", " except ValueError:\n", " pass\n", " else:\n", " # discard canary events\n", - " # WMF Data Engineering team produces artificial 'canary' events into \n", + " # WMF Data Engineering team produces artificial 'canary' events into\n", " # each stream multiple times an hour. The presence of these canary\n", " # events in a stream allow us to differentiate between a broken event\n", " # stream, and an empty one.\n", " # We will also filter bot-generated events\n", - " if change['meta']['domain'] == 'canary' or re.search('bot', change['user'], re.IGNORECASE):\n", + " if change[\"meta\"][\"domain\"] == \"canary\" or re.search(\"bot\", change[\"user\"], re.IGNORECASE):\n", " continue\n", - " timestamp = change['meta']['dt']\n", + " timestamp = change[\"meta\"][\"dt\"]\n", " event = f\"{timestamp}:: {change['user']} edited {change['title']}\"\n", - " servername = change['server_name']\n", + " servername = change[\"server_name\"]\n", " # Manually \"tick\" this edge\n", " self.push_tick(\n", - " WikiData(\n", - " servername=servername,\n", - " servername_count=0,\n", - " timestamp=timestamp,\n", - " event=event\n", - " )\n", + " WikiData(servername=servername, servername_count=0, timestamp=timestamp, event=event)\n", " )\n", "\n", + "\n", "# Create the graph-time representation of our adapter\n", "FetchWikiData = py_push_adapter_def(\"FetchWikiData\", FetchWikiDataAdapter, csp.ts[WikiData], url=str)" ] @@ -270,12 +269,15 @@ "@csp.node\n", "def update_widget(wiki_event: csp.ts[WikiData], widget: PerspectiveWidget):\n", " if csp.ticked(wiki_event):\n", - " widget.update({\n", - " \"servername\": [wiki_event.servername],\n", - " \"servername_count\": [wiki_event.servername_count],\n", - " \"timestamp\": [wiki_event.timestamp],\n", - " \"event\": [wiki_event.event],\n", - " })\n", + " widget.update(\n", + " {\n", + " \"servername\": [wiki_event.servername],\n", + " \"servername_count\": [wiki_event.servername_count],\n", + " \"timestamp\": [wiki_event.timestamp],\n", + " \"event\": [wiki_event.event],\n", + " }\n", + " )\n", + "\n", "\n", "@csp.node\n", "def compute_server_count(wiki_event: csp.ts[WikiData]) -> csp.ts[WikiData]:\n", @@ -290,10 +292,11 @@ " wiki_event.servername_count = s_servernames[wiki_event.servername]\n", " return wiki_event\n", "\n", + "\n", "@csp.graph\n", "def wiki_graph():\n", " print(\"Start of graph building\")\n", - " URL = 'https://stream.wikimedia.org/v2/stream/recentchange'\n", + " URL = \"https://stream.wikimedia.org/v2/stream/recentchange\"\n", " # Create an edge in the graph with the events fetched by the adapter\n", " result = FetchWikiData(url=URL)\n", " result = compute_server_count(result)\n", @@ -305,8 +308,9 @@ " csp.add_graph_output(\"Wiki events\", result.event)\n", " print(\"End of graph building\")\n", "\n", + "\n", "start = datetime.utcnow()\n", - "csp.run(wiki_graph, starttime=start, endtime=start+timedelta(seconds=30), realtime=True)\n", + "csp.run(wiki_graph, starttime=start, endtime=start + timedelta(seconds=30), realtime=True)\n", "print(\"Done.\")" ] }, @@ -365,6 +369,7 @@ "from csp.impl.pulladapter import PullInputAdapter\n", "from csp.impl.wiring import py_pull_adapter_def\n", "\n", + "\n", "# Define the runtime implementation of our adapter\n", "class HistoricalWikiDataAdapter(PullInputAdapter):\n", " def __init__(self, url: str):\n", @@ -378,35 +383,35 @@ "\n", " def stop(self):\n", " print(\"HistoricalWikiDataAdapter::stop\")\n", - " \n", + "\n", " def next(self):\n", " \"\"\"Return tuple of datetime, value of next tick, or None if no more data is available\"\"\"\n", " self._next_event = next(self._events)\n", " time = self._start_time\n", " if self._next_event and time < self._end_time:\n", - " while True:\n", + " while True:\n", " item = self._next_event\n", - " if item.event == 'message':\n", + " if item.event == \"message\":\n", " try:\n", - " change = json.loads(item.data) \n", + " change = json.loads(item.data)\n", " except ValueError:\n", " self._next_event = next(self._events)\n", " continue\n", " else:\n", " # discard canary events\n", - " # WMF Data Engineering team produces artificial 'canary' events into \n", + " # WMF Data Engineering team produces artificial 'canary' events into\n", " # each stream multiple times an hour. The presence of these canary\n", " # events in a stream allow us to differentiate between a broken event\n", " # stream, and an empty one.\n", " # We will also filter bot-generated events\n", - " if change['meta']['domain'] == 'canary' or re.search('bot', change['user'], re.IGNORECASE):\n", + " if change[\"meta\"][\"domain\"] == \"canary\" or re.search(\"bot\", change[\"user\"], re.IGNORECASE):\n", " self._next_event = next(self._events)\n", " continue\n", - " timestamp = change['meta']['dt']\n", - " time = datetime.fromisoformat(timestamp.rstrip('Z'))\n", + " timestamp = change[\"meta\"][\"dt\"]\n", + " time = datetime.fromisoformat(timestamp.rstrip(\"Z\"))\n", " if time <= self._end_time:\n", " event = f\"{timestamp}:: {change['user']} edited {change['title']}\"\n", - " servername = change['server_name']\n", + " servername = change[\"server_name\"]\n", " # Manually \"tick\" this edge\n", " return (\n", " time,\n", @@ -415,23 +420,21 @@ " timestamp=timestamp,\n", " event=event,\n", " servername_count=0,\n", - " )\n", + " ),\n", " )\n", " return None\n", "\n", + "\n", "# Create the graph-time representation of our adapter\n", - "HistoricalWikiData = py_pull_adapter_def(\n", - " \"HistoricalWikiData\",\n", - " HistoricalWikiDataAdapter,\n", - " csp.ts[WikiData],\n", - " url=str\n", - ")\n", + "HistoricalWikiData = py_pull_adapter_def(\"HistoricalWikiData\", HistoricalWikiDataAdapter, csp.ts[WikiData], url=str)\n", + "\n", "\n", "@csp.node\n", "def pretty_print(event: csp.ts[WikiData]) -> csp.ts[str]:\n", " if csp.ticked(event):\n", " return f\"{event.event}\"\n", "\n", + "\n", "@csp.graph\n", "def wiki_graph():\n", " URL = f\"https://stream.wikimedia.org/v2/stream/recentchange?since={start_time}\"\n", @@ -450,6 +453,7 @@ " csp.add_graph_output(\"en_wiki\", en_wiki)\n", " print(\"End of graph building\")\n", "\n", + "\n", "csp.run(wiki_graph, starttime=start_time, endtime=end_time)\n", "print(\"Done.\")" ] @@ -469,11 +473,6 @@ "metadata": {}, "outputs": [], "source": [ - "@csp.node\n", - "def pretty_print(event: csp.ts[WikiData]) -> csp.ts[str]:\n", - " if csp.ticked(event):\n", - " return f\"{event.event}\"\n", - "\n", "@csp.graph\n", "def wiki_graph():\n", " print(\"Start of graph building\")\n", @@ -496,8 +495,9 @@ " csp.add_graph_output(\"en_wiki\", en_wiki)\n", " print(\"End of graph building\")\n", "\n", + "\n", "start_time = datetime.utcnow()\n", - "end_time = start_time+timedelta(seconds=30)\n", + "end_time = start_time + timedelta(seconds=30)\n", "csp.run(wiki_graph, starttime=start_time, endtime=end_time, realtime=True)\n", "print(\"Done.\")" ] diff --git a/pyproject.toml b/pyproject.toml index fd93098af..2b30af372 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,9 +66,8 @@ develop = [ "wheel", # lint "codespell>=2.2.6,<2.3", - "isort>=5,<6", "mdformat>=0.7.17,<0.8", - "ruff>=0.3,<0.4", + "ruff>=0.7,<0.8", # test "pytest", "pytest-asyncio", @@ -151,17 +150,6 @@ before-build = "make requirements" archs = "AMD64" skip = "*win32 *arm_64" -[tool.isort] -combine_as_imports = true -include_trailing_comma = true -line_length = 120 -profile = "black" - -default_section = "THIRDPARTY" -sections = "FUTURE,THIRDPARTY,FIRSTPARTY,LOCALFOLDER" - -known_first_party = "csp" - [tool.pytest.ini_options] asyncio_mode = "strict" testpaths = "csp/tests" @@ -169,6 +157,21 @@ testpaths = "csp/tests" [tool.ruff] line-length = 120 +[tool.ruff.lint] +extend-select = ["I"] + +[tool.ruff.lint.isort] +combine-as-imports = true +default-section = "third-party" +known-first-party = ["csp"] +section-order = [ + "future", + "standard-library", + "third-party", + "first-party", + "local-folder", +] + [tool.ruff.lint.per-file-ignores] "__init__.py" = ["F401", "F403"] "csp/impl/__*.py" = ["F401"] diff --git a/setup.py b/setup.py index ea2a1fa68..0f6684de5 100644 --- a/setup.py +++ b/setup.py @@ -5,6 +5,7 @@ import subprocess import sys from shutil import which + from skbuild import setup CSP_USE_VCPKG = os.environ.get("CSP_USE_VCPKG", "1").lower() in ("1", "on")