diff --git a/.lockfiles/py310-dev.lock b/.lockfiles/py310-dev.lock index d0da179b99..7c37147154 100644 --- a/.lockfiles/py310-dev.lock +++ b/.lockfiles/py310-dev.lock @@ -8,7 +8,7 @@ anyio==4.4.0 # via # httpx # jupyter-server -appnope==0.1.4 ; platform_system == 'Darwin' +appnope==0.1.4 ; sys_platform == 'darwin' # via ipykernel argon2-cffi==23.1.0 # via jupyter-server @@ -61,7 +61,7 @@ cachetools==5.4.0 # via # streamlit # tox -cattrs==23.2.3 +cattrs==24.1.2 # via baybe (pyproject.toml) certifi==2024.7.4 # via @@ -240,7 +240,7 @@ importlib-metadata==7.1.0 # opentelemetry-api iniconfig==2.0.0 # via pytest -intel-openmp==2021.4.0 ; platform_system == 'Windows' +intel-openmp==2021.4.0 ; sys_platform == 'win32' # via mkl interface-meta==1.3.0 # via formulaic @@ -393,7 +393,7 @@ mdurl==0.1.2 # via markdown-it-py mistune==3.0.2 # via nbconvert -mkl==2021.4.0 ; platform_system == 'Windows' +mkl==2021.4.0 ; sys_platform == 'win32' # via torch mmh3==5.0.1 # via e3fp @@ -487,36 +487,36 @@ numpy==1.26.4 # types-seaborn # xarray # xyzpy -nvidia-cublas-cu12==12.1.3.1 ; platform_machine == 'x86_64' and platform_system == 'Linux' +nvidia-cublas-cu12==12.1.3.1 ; platform_machine == 'x86_64' and sys_platform == 'linux' # via # nvidia-cudnn-cu12 # nvidia-cusolver-cu12 # torch -nvidia-cuda-cupti-cu12==12.1.105 ; platform_machine == 'x86_64' and platform_system == 'Linux' +nvidia-cuda-cupti-cu12==12.1.105 ; platform_machine == 'x86_64' and sys_platform == 'linux' # via torch -nvidia-cuda-nvrtc-cu12==12.1.105 ; platform_machine == 'x86_64' and platform_system == 'Linux' +nvidia-cuda-nvrtc-cu12==12.1.105 ; platform_machine == 'x86_64' and sys_platform == 'linux' # via torch -nvidia-cuda-runtime-cu12==12.1.105 ; platform_machine == 'x86_64' and platform_system == 'Linux' +nvidia-cuda-runtime-cu12==12.1.105 ; platform_machine == 'x86_64' and sys_platform == 'linux' # via torch -nvidia-cudnn-cu12==8.9.2.26 ; platform_machine == 'x86_64' and platform_system == 'Linux' +nvidia-cudnn-cu12==8.9.2.26 ; platform_machine == 'x86_64' and sys_platform == 'linux' # via torch -nvidia-cufft-cu12==11.0.2.54 ; platform_machine == 'x86_64' and platform_system == 'Linux' +nvidia-cufft-cu12==11.0.2.54 ; platform_machine == 'x86_64' and sys_platform == 'linux' # via torch -nvidia-curand-cu12==10.3.2.106 ; platform_machine == 'x86_64' and platform_system == 'Linux' +nvidia-curand-cu12==10.3.2.106 ; platform_machine == 'x86_64' and sys_platform == 'linux' # via torch -nvidia-cusolver-cu12==11.4.5.107 ; platform_machine == 'x86_64' and platform_system == 'Linux' +nvidia-cusolver-cu12==11.4.5.107 ; platform_machine == 'x86_64' and sys_platform == 'linux' # via torch -nvidia-cusparse-cu12==12.1.0.106 ; platform_machine == 'x86_64' and platform_system == 'Linux' +nvidia-cusparse-cu12==12.1.0.106 ; platform_machine == 'x86_64' and sys_platform == 'linux' # via # nvidia-cusolver-cu12 # torch -nvidia-nccl-cu12==2.20.5 ; platform_machine == 'x86_64' and platform_system == 'Linux' +nvidia-nccl-cu12==2.20.5 ; platform_machine == 'x86_64' and sys_platform == 'linux' # via torch -nvidia-nvjitlink-cu12==12.5.82 ; platform_machine == 'x86_64' and platform_system == 'Linux' +nvidia-nvjitlink-cu12==12.5.82 ; platform_machine == 'x86_64' and sys_platform == 'linux' # via # nvidia-cusolver-cu12 # nvidia-cusparse-cu12 -nvidia-nvtx-cu12==12.1.105 ; platform_machine == 'x86_64' and platform_system == 'Linux' +nvidia-nvtx-cu12==12.1.105 ; platform_machine == 'x86_64' and sys_platform == 'linux' # via torch onnx==1.16.1 # via @@ -922,7 +922,7 @@ sympy==1.13.1 # via # onnxruntime # torch -tbb==2021.13.0 ; platform_system == 'Windows' +tbb==2021.13.0 ; sys_platform == 'win32' # via mkl tenacity==8.5.0 # via @@ -1007,7 +1007,7 @@ traitlets==5.14.3 # nbclient # nbconvert # nbformat -triton==2.3.1 ; python_full_version < '3.12' and platform_machine == 'x86_64' and platform_system == 'Linux' +triton==2.3.1 ; python_full_version < '3.12' and platform_machine == 'x86_64' and sys_platform == 'linux' # via torch typeguard==2.13.3 # via @@ -1050,7 +1050,7 @@ virtualenv==20.26.3 # via # pre-commit # tox -watchdog==4.0.1 ; platform_system != 'Darwin' +watchdog==4.0.1 ; sys_platform != 'darwin' # via streamlit wcwidth==0.2.13 # via prompt-toolkit diff --git a/benchmarks/__init__.py b/benchmarks/__init__.py index fb631ee0c9..d1221e3ec1 100644 --- a/benchmarks/__init__.py +++ b/benchmarks/__init__.py @@ -1,9 +1,17 @@ """Benchmarking module for performance tracking.""" -from benchmarks.definition import Benchmark +from benchmarks.definition import ( + Benchmark, + BenchmarkSettings, + ConvergenceBenchmark, + ConvergenceBenchmarkSettings, +) from benchmarks.result import Result __all__ = [ - "Result", "Benchmark", + "BenchmarkSettings", + "ConvergenceBenchmark", + "ConvergenceBenchmarkSettings", + "Result", ] diff --git a/benchmarks/definition/__init__.py b/benchmarks/definition/__init__.py index 970a3dacbd..c5895d8ecd 100644 --- a/benchmarks/definition/__init__.py +++ b/benchmarks/definition/__init__.py @@ -1,13 +1,17 @@ -"""Benchmark task definitions.""" +"""Benchmark definitions.""" -from benchmarks.definition.config import ( +from benchmarks.definition.base import ( Benchmark, BenchmarkSettings, - ConvergenceExperimentSettings, +) +from benchmarks.definition.convergence import ( + ConvergenceBenchmark, + ConvergenceBenchmarkSettings, ) __all__ = [ - "ConvergenceExperimentSettings", "Benchmark", "BenchmarkSettings", + "ConvergenceBenchmark", + "ConvergenceBenchmarkSettings", ] diff --git a/benchmarks/definition/base.py b/benchmarks/definition/base.py new file mode 100644 index 0000000000..d1f6858684 --- /dev/null +++ b/benchmarks/definition/base.py @@ -0,0 +1,86 @@ +"""Basic benchmark configuration.""" + +import time +from abc import ABC +from collections.abc import Callable +from datetime import datetime, timedelta, timezone +from typing import Generic, TypeVar + +from attrs import define, field +from attrs.validators import instance_of +from cattrs import override +from cattrs.gen import make_dict_unstructure_fn +from pandas import DataFrame + +from baybe.utils.random import temporary_seed +from benchmarks.result import Result, ResultMetadata +from benchmarks.serialization import BenchmarkSerialization, converter + + +@define(frozen=True, kw_only=True) +class BenchmarkSettings(ABC, BenchmarkSerialization): + """The basic benchmark configuration.""" + + random_seed: int = field(validator=instance_of(int), default=1337) + """The used random seed.""" + + +BenchmarkSettingsType = TypeVar("BenchmarkSettingsType", bound=BenchmarkSettings) + + +@define(frozen=True) +class Benchmark(Generic[BenchmarkSettingsType], BenchmarkSerialization): + """The base class for all benchmark definitions.""" + + function: Callable[[BenchmarkSettingsType], DataFrame] = field() + """The callable containing the benchmarking logic.""" + + settings: BenchmarkSettingsType = field() + """The benchmark configuration.""" + + @function.validator + def _validate_function(self, _, function) -> None: + if function.__doc__ is None: + raise ValueError("The benchmark function must have a docstring.") + + @property + def name(self) -> str: + """The name of the benchmark function.""" + return self.function.__name__ + + @property + def description(self) -> str: + """The description of the benchmark function.""" + assert self.function.__doc__ is not None + return self.function.__doc__ + + def __call__(self) -> Result: + """Execute the benchmark and return the result.""" + start_datetime = datetime.now(timezone.utc) + + with temporary_seed(self.settings.random_seed): + start_sec = time.perf_counter() + result = self.function(self.settings) + stop_sec = time.perf_counter() + + duration = timedelta(seconds=stop_sec - start_sec) + + metadata = ResultMetadata( + start_datetime=start_datetime, + duration=duration, + ) + + return Result(self.name, result, metadata) + + +@converter.register_unstructure_hook +def unstructure_benchmark(benchmark: Benchmark) -> dict: + """Unstructure a benchmark instance.""" + fn = make_dict_unstructure_fn( + type(benchmark), converter, function=override(omit=True) + ) + return { + "name": benchmark.name, + "description": benchmark.description, + **fn(benchmark), + } diff --git a/benchmarks/definition/config.py b/benchmarks/definition/config.py deleted file mode 100644 index 211c0c48ac..0000000000 --- a/benchmarks/definition/config.py +++ /dev/null @@ -1,103 +0,0 @@ -"""Benchmark configurations.""" - -import time -from abc import ABC -from collections.abc import Callable -from datetime import datetime, timedelta, timezone -from typing import Any, Generic, TypeVar - -from attrs import define, field -from attrs.validators import instance_of -from cattr.gen import make_dict_unstructure_fn, override -from pandas import DataFrame - -from baybe.utils.random import temporary_seed -from benchmarks.result import Result, ResultMetadata -from benchmarks.serialization import BenchmarkSerialization, converter - - -@define(frozen=True) -class BenchmarkSettings(ABC, BenchmarkSerialization): - """Benchmark configuration for recommender analyses.""" - - random_seed: int = field(validator=instance_of(int), kw_only=True, default=1337) - """The random seed for reproducibility.""" - - -BenchmarkSettingsType = TypeVar("BenchmarkSettingsType", bound=BenchmarkSettings) - - -@define(frozen=True) -class ConvergenceExperimentSettings(BenchmarkSettings): - """Benchmark configuration for recommender convergence analyses.""" - - batch_size: int = field(validator=instance_of(int)) - """The recommendation batch size.""" - - n_doe_iterations: int = field(validator=instance_of(int)) - """The number of Design of Experiment iterations.""" - - n_mc_iterations: int = field(validator=instance_of(int)) - """The number of Monte Carlo iterations.""" - - -@define(frozen=True) -class Benchmark(Generic[BenchmarkSettingsType], BenchmarkSerialization): - """The base class for a benchmark executable.""" - - settings: BenchmarkSettingsType = field() - """The benchmark configuration.""" - - function: Callable[[BenchmarkSettingsType], DataFrame] = field() - """The callable which contains the benchmarking logic.""" - - name: str = field(init=False) - """The name of the benchmark.""" - - best_possible_result: float | None = field(default=None) - """The best possible result which can be achieved in the optimization process.""" - - optimal_function_inputs: list[dict[str, Any]] | None = field(default=None) - """An input that creates the best_possible_result.""" - - @property - def description(self) -> str: - """The description of the benchmark function.""" - if self.function.__doc__ is not None: - return self.function.__doc__ - return "No description available." - - @name.default - def _default_name(self): - """Return the name of the benchmark function.""" - return self.function.__name__ - - def __call__(self) -> Result: - """Execute the benchmark and return the result.""" - start_datetime = datetime.now(timezone.utc) - - with temporary_seed(self.settings.random_seed): - start_sec = time.perf_counter() - result = self.function(self.settings) - stop_sec = time.perf_counter() - - duration = timedelta(seconds=stop_sec - start_sec) - - metadata = ResultMetadata( - start_datetime=start_datetime, - duration=duration, - ) - - return Result(self.name, result, metadata) - - -# Register un-/structure hooks -converter.register_unstructure_hook( - Benchmark, - lambda o: dict( - {"description": o.description}, - **make_dict_unstructure_fn(Benchmark, converter, function=override(omit=True))( - o - ), - ), -) diff --git a/benchmarks/definition/convergence.py b/benchmarks/definition/convergence.py new file mode 100644 index 0000000000..c6abb5b458 --- /dev/null +++ b/benchmarks/definition/convergence.py @@ -0,0 +1,39 @@ +"""Convergence benchmark configuration.""" + +from typing import Any + +from attrs import define, field +from attrs.validators import deep_mapping, instance_of, optional + +from benchmarks.definition.base import Benchmark, BenchmarkSettings + + +@define(frozen=True, kw_only=True) +class ConvergenceBenchmarkSettings(BenchmarkSettings): + """Benchmark configuration for recommender convergence analyses.""" + + batch_size: int = field(validator=instance_of(int)) + """The recommendation batch size.""" + + n_doe_iterations: int = field(validator=instance_of(int)) + """The number of Design of Experiment iterations.""" + + n_mc_iterations: int = field(validator=instance_of(int)) + """The number of Monte Carlo iterations.""" + + +@define(frozen=True) +class ConvergenceBenchmark(Benchmark[ConvergenceBenchmarkSettings]): + """A class for defining convergence benchmarks.""" + + optimal_target_values: dict[str, Any] | None = field( + default=None, + validator=optional( + deep_mapping( + key_validator=instance_of(str), + mapping_validator=instance_of(dict), + value_validator=lambda *_: None, + ) + ), + ) + """The optimal values that can be achieved for the targets **individually**.""" diff --git a/benchmarks/domains/__init__.py b/benchmarks/domains/__init__.py index 4a0e956a83..cf0d577824 100644 --- a/benchmarks/domains/__init__.py +++ b/benchmarks/domains/__init__.py @@ -1,6 +1,6 @@ """Benchmark domains.""" -from benchmarks.definition.config import Benchmark +from benchmarks.definition.base import Benchmark from benchmarks.domains.synthetic_2C1D_1C import synthetic_2C1D_1C_benchmark BENCHMARKS: list[Benchmark] = [ diff --git a/benchmarks/domains/synthetic_2C1D_1C.py b/benchmarks/domains/synthetic_2C1D_1C.py index 94c475ac4c..d5cb92e926 100644 --- a/benchmarks/domains/synthetic_2C1D_1C.py +++ b/benchmarks/domains/synthetic_2C1D_1C.py @@ -15,9 +15,9 @@ from baybe.searchspace import SearchSpace from baybe.simulation import simulate_scenarios from baybe.targets import NumericalTarget -from benchmarks.definition import ( - Benchmark, - ConvergenceExperimentSettings, +from benchmarks.definition.convergence import ( + ConvergenceBenchmark, + ConvergenceBenchmarkSettings, ) if TYPE_CHECKING: @@ -49,7 +49,7 @@ def lookup(df: pd.DataFrame, /) -> pd.DataFrame: ) -def synthetic_2C1D_1C(settings: ConvergenceExperimentSettings) -> DataFrame: +def synthetic_2C1D_1C(settings: ConvergenceBenchmarkSettings) -> DataFrame: """Hybrid synthetic test function. Inputs: @@ -95,20 +95,16 @@ def synthetic_2C1D_1C(settings: ConvergenceExperimentSettings) -> DataFrame: ) -benchmark_config = ConvergenceExperimentSettings( +benchmark_config = ConvergenceBenchmarkSettings( batch_size=5, n_doe_iterations=30, n_mc_iterations=50, ) -synthetic_2C1D_1C_benchmark = Benchmark( +synthetic_2C1D_1C_benchmark = ConvergenceBenchmark( function=synthetic_2C1D_1C, - best_possible_result=4.09685, + optimal_target_values={"target": 4.09685}, settings=benchmark_config, - optimal_function_inputs=[ - {"x": 1.610, "y": 1.571, "z": 3}, - {"x": 1.610, "y": -4.712, "z": 3}, - ], ) diff --git a/benchmarks/persistence/persistence.py b/benchmarks/persistence/persistence.py index 83dfce13d6..01ff02d625 100644 --- a/benchmarks/persistence/persistence.py +++ b/benchmarks/persistence/persistence.py @@ -48,8 +48,12 @@ class PathConstructor: benchmark_name: str = field(validator=instance_of(str)) """The name of the benchmark for which the path should be constructed.""" - branch: str = field(validator=instance_of(str)) - """The branch checked out at benchmark execution time.""" + branch: str = field( + converter=lambda x: x or "-branchless-", + validator=instance_of(str), + ) + """The branch checked out at benchmark execution time. + In case of detached head state the branch is set to '-branchless-'.""" latest_baybe_tag: str = field(validator=instance_of(str)) """The latest BayBE version tag existing at benchmark execution time.""" @@ -108,6 +112,7 @@ def get_path(self, strategy: PathStrategy) -> Path: separator = "/" if strategy is PathStrategy.HIERARCHICAL else "_" file_usable_date = self.execution_date_time.strftime("%Y-%m-%d") + components = [ self.benchmark_name, self.branch, diff --git a/benchmarks/result/metadata.py b/benchmarks/result/metadata.py index 75f368e92a..e777f89f05 100644 --- a/benchmarks/result/metadata.py +++ b/benchmarks/result/metadata.py @@ -4,7 +4,7 @@ import git from attrs import define, field -from attrs.validators import instance_of +from attrs.validators import instance_of, optional from cattrs.gen import make_dict_unstructure_fn from benchmarks.serialization import BenchmarkSerialization, converter @@ -26,15 +26,18 @@ class ResultMetadata(BenchmarkSerialization): latest_baybe_tag: str = field(validator=instance_of(str), init=False) """The latest BayBE tag reachable in the ancestor commit history.""" - branch: str = field(validator=instance_of(str), init=False) - """The branch currently checked out.""" + branch: str | None = field(validator=optional(instance_of(str)), init=False) + """The branch checked out during benchmark execution.""" @branch.default - def _default_branch(self) -> str: + def _default_branch(self) -> str | None: """Set the current checkout branch.""" repo = git.Repo(search_parent_directories=True) - current_branch = repo.active_branch.name - return current_branch + try: + current_branch = repo.active_branch.name + return current_branch + except TypeError: + return None @commit_hash.default def _default_commit_hash(self) -> str: diff --git a/benchmarks/result/result.py b/benchmarks/result/result.py index 69a9b9104a..173b71d010 100644 --- a/benchmarks/result/result.py +++ b/benchmarks/result/result.py @@ -16,7 +16,7 @@ @define(frozen=True) class Result(BenchmarkSerialization): - """A single result of the benchmarking.""" + """A single benchmarking result.""" benchmark_identifier: str = field(validator=instance_of(str)) """The identifier of the benchmark that produced the result.""" diff --git a/pyproject.toml b/pyproject.toml index 3bf1c36ace..352d0629fa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,7 @@ dynamic = ['version'] dependencies = [ "attrs>=24.1.0", "botorch>=0.9.3,<1", - "cattrs>=23.2.0", + "cattrs>=24.1.0", "exceptiongroup", "funcy>=1.17,<2", "gpytorch>=1.9.1,<2",