Parameter optimization (WIP)

examples/benchmarks/liander2024_optimization_example.py

@@ -0,0 +1,121 @@
+# SPDX-FileCopyrightText: 2025 Contributors to the OpenSTEF project <[email protected]>
+#
+# SPDX-License-Identifier: MPL-2.0
+
+"""Backtesting pipeline for evaluating energy forecasting models.
+
+Simulates realistic forecasting scenarios by replaying historical data with
+proper temporal constraints. Executes prediction and retraining schedules
+that mirror operational deployment conditions, ensuring evaluation results
+accurately reflect real-world model performance.
+"""
+
+import logging
+import sys
+from datetime import timedelta
+from pathlib import Path
+
+from openstef_beam.backtesting.backtest_forecaster.mixins import BacktestForecasterConfig
+from openstef_beam.backtesting.backtest_pipeline import BacktestConfig
+from openstef_beam.benchmarking.benchmarks.liander2024 import Liander2024TargetProvider
+from openstef_beam.evaluation.metric_providers import RCRPSSampleWeightedProvider
+from openstef_beam.parameter_tuning.models import (
+    FloatDistribution,
+    IntDistribution,
+    LGBMParameterSpace,
+    OptimizationMetric,
+)
+from openstef_beam.parameter_tuning.optimizer.benchmark_optimizer import BenchmarkOptimizer
+from openstef_beam.parameter_tuning.optimizer.optimizer import (
+    OptimizerConfig,
+)
+from openstef_core.types import LeadTime, Quantile
+from openstef_models.presets import ForecastingWorkflowConfig
+
+logger = logging.getLogger(__name__)
+handler = logging.StreamHandler(stream=sys.stdout)
+logger.addHandler(handler)
+
+
+single_target = False
+
+horizons = [LeadTime.from_string("PT12H")]
+quantiles = [Quantile(0.1), Quantile(0.3), Quantile(0.5), Quantile(0.7), Quantile(0.9)]
+forecaster_name = "lgbm"  # Choose model type: "lgbm", "xgboost", "gblinear", "lgbmlinear", "hybrid", "flatliner"
+
+
+# Base Forecasting Workflow Config
+base_config = ForecastingWorkflowConfig(
+    model_id="common_model_",
+    run_name=None,
+    model=forecaster_name,
+    horizons=horizons,
+    quantiles=quantiles,
+    model_reuse_enable=True,
+    mlflow_storage=None,
+    radiation_column="shortwave_radiation",
+    rolling_aggregate_features=["mean", "median", "max", "min"],
+    wind_speed_column="wind_speed_80m",
+    pressure_column="surface_pressure",
+    temperature_column="temperature_2m",
+    relative_humidity_column="relative_humidity_2m",
+    energy_price_column="EPEX_NL",
+)
+
+
+# Define hyperparameter search space
+params = LGBMParameterSpace(
+    learning_rate=FloatDistribution(low=0.01, high=0.5, log=True),
+    num_leaves=IntDistribution(low=5, high=120),
+    max_depth=IntDistribution(low=1, high=3),
+    reg_lambda=0.2,
+)
+
+# Set optimization goal
+optimization_metric = OptimizationMetric(
+    metric=RCRPSSampleWeightedProvider(lower_quantile=0.01, upper_quantile=0.99),
+    direction_minimize=True,
+)
+
+# Load target provider with historical data
+target_provider = Liander2024TargetProvider(
+    data_dir=Path("../data/liander2024-energy-forecasting-benchmark"),
+)
+
+# Create the backtest configuration
+backtest_config = BacktestConfig(
+    prediction_sample_interval=timedelta(minutes=15),
+    predict_interval=timedelta(hours=6),
+    train_interval=timedelta(days=7),
+)
+
+# Configure the backtest forecaster
+backtest_forecaster_config = BacktestForecasterConfig(
+    requires_training=True,
+    predict_length=timedelta(hours=13),
+    predict_min_length=timedelta(hours=11),
+    predict_context_length=timedelta(days=14),  # Context needed for lag features
+    predict_context_min_coverage=0.5,
+    training_context_length=timedelta(days=90),  # Three months of training data
+    training_context_min_coverage=0.5,
+    predict_sample_interval=timedelta(minutes=15),
+)
+
+
+# Create the optimizer configuration
+optimizer_config = OptimizerConfig(
+    base_config=base_config,
+    parameter_space=params,
+    backtest_config=backtest_config,
+    backtest_forecaster_config=backtest_forecaster_config,
+    optimization_metric=optimization_metric,
+    n_jobs=4,
+    n_trials=20,
+)
+
+optimizer = BenchmarkOptimizer(config=optimizer_config)
+
+best_hyperparams = optimizer.optimize(experiment_name="Liander2024 Benchmark", target_provider=target_provider)
+
+msg = f"{forecaster_name} - Best hyperparameters found: {best_hyperparams}"
+logger.info(msg)

examples/benchmarks/liander_2024_benchmark_xgboost_gblinear.py

@@ -40,6 +40,7 @@
 BENCHMARK_RESULTS_PATH_GBLINEAR = OUTPUT_PATH / "GBLinear"
 N_PROCESSES = multiprocessing.cpu_count()  # Amount of parallel processes to use for the benchmark
 
+
 # Model configuration
 FORECAST_HORIZONS = [LeadTime.from_string("P3D")]  # Forecast horizon(s)
 PREDICTION_QUANTILES = [

packages/openstef-beam/src/openstef_beam/benchmarking/baselines/openstef4.py

@@ -14,15 +14,26 @@
 from pydantic import Field, PrivateAttr
 from pydantic_extra_types.coordinate import Coordinate
 
-from openstef_beam.backtesting.backtest_forecaster.mixins import BacktestForecasterConfig, BacktestForecasterMixin
-from openstef_beam.backtesting.restricted_horizon_timeseries import RestrictedHorizonVersionedTimeSeries
-from openstef_beam.benchmarking.benchmark_pipeline import BenchmarkContext, BenchmarkTarget, ForecasterFactory
+from openstef_beam.backtesting.backtest_forecaster.mixins import (
+    BacktestForecasterConfig,
+    BacktestForecasterMixin,
+)
+from openstef_beam.backtesting.restricted_horizon_timeseries import (
+    RestrictedHorizonVersionedTimeSeries,
+)
+from openstef_beam.benchmarking.benchmark_pipeline import (
+    BenchmarkContext,
+    BenchmarkTarget,
+    ForecasterFactory,
+)
 from openstef_core.base_model import BaseConfig, BaseModel
 from openstef_core.datasets import TimeSeriesDataset
 from openstef_core.exceptions import FlatlinerDetectedError, NotFittedError
 from openstef_core.types import Q
 from openstef_models.presets import ForecastingWorkflowConfig
-from openstef_models.workflows.custom_forecasting_workflow import CustomForecastingWorkflow
+from openstef_models.workflows.custom_forecasting_workflow import (
+    CustomForecastingWorkflow,
+)
 
 
 class WorkflowCreationContext(BaseConfig):
@@ -54,6 +65,10 @@ class OpenSTEF4BacktestForecaster(BaseModel, BacktestForecasterMixin):
         default=False,
         description="When True, saves intermediate input data for debugging",
     )
+    contributions: bool = Field(
+        default=False,
+        description="When True, saves base Forecaster prediction contributions for ensemble models in cache_dir",
+    )
 
     _workflow: CustomForecastingWorkflow | None = PrivateAttr(default=None)
     _is_flatliner_detected: bool = PrivateAttr(default=False)
@@ -62,7 +77,7 @@ class OpenSTEF4BacktestForecaster(BaseModel, BacktestForecasterMixin):
 
     @override
     def model_post_init(self, context: Any) -> None:
-        if self.debug:
+        if self.debug or self.contributions:
             self.cache_dir.mkdir(parents=True, exist_ok=True)
 
     @property
@@ -82,7 +97,9 @@ def fit(self, data: RestrictedHorizonVersionedTimeSeries) -> None:
 
         # Extract the dataset for training
         training_data = data.get_window(
-            start=data.horizon - self.config.training_context_length, end=data.horizon, available_before=data.horizon
+            start=data.horizon - self.config.training_context_length,
+            end=data.horizon,
+            available_before=data.horizon,
         )
 
         if self.debug:
@@ -225,4 +242,8 @@ def create_openstef4_preset_backtest_forecaster(
     )
 
 
-__all__ = ["OpenSTEF4BacktestForecaster", "WorkflowCreationContext", "create_openstef4_preset_backtest_forecaster"]
+__all__ = [
+    "OpenSTEF4BacktestForecaster",
+    "WorkflowCreationContext",
+    "create_openstef4_preset_backtest_forecaster",
+]

packages/openstef-beam/src/openstef_beam/evaluation/evaluation_pipeline.py

@@ -235,7 +235,7 @@ def _iterate_subsets(
         for lead_time in self.config.lead_times:
             predictions_filtered = predictions.filter_by_lead_time(lead_time=lead_time).select_version()
             if evaluation_mask is not None:
-                predictions_filtered = predictions_filtered.filter_index(evaluation_mask)
+                predictions_filtered = predictions_filtered.filter_index(evaluation_mask)  # type: ignore
 
             # Remove target column from predictions to avoid duplication
             if target_column in predictions_filtered.data.columns:

packages/openstef-beam/src/openstef_beam/parameter_tuning/__init__.py

@@ -0,0 +1,5 @@
+# SPDX-FileCopyrightText: 2025 Contributors to the OpenSTEF project <[email protected]>
+#
+# SPDX-License-Identifier: MPL-2.0
+
+"""Parameter tuning module for OpenSTEF Beam."""