From 25abc44804cb3164f70e6f45fa3c67c204f11860 Mon Sep 17 00:00:00 2001 From: cwasicki <126617870+cwasicki@users.noreply.github.com> Date: Wed, 30 Apr 2025 01:16:57 +0200 Subject: [PATCH 1/6] Add asset optimization reporting module Signed-off-by: cwasicki <126617870+cwasicki@users.noreply.github.com> --- .../notebooks/reporting/asset_optimization.py | 330 ++++++++++++++++++ 1 file changed, 330 insertions(+) create mode 100644 src/frequenz/lib/notebooks/reporting/asset_optimization.py diff --git a/src/frequenz/lib/notebooks/reporting/asset_optimization.py b/src/frequenz/lib/notebooks/reporting/asset_optimization.py new file mode 100644 index 00000000..f014609b --- /dev/null +++ b/src/frequenz/lib/notebooks/reporting/asset_optimization.py @@ -0,0 +1,330 @@ +# License: MIT +# Copyright © 2025 Frequenz Energy-as-a-Service GmbH + +"""Reporting for asset optimization.""" + + +from datetime import datetime, timedelta + +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +from matplotlib.axes import Axes + +from frequenz.data.microgrid import MicrogridData + + +# pylint: disable=too-many-arguments +async def fetch_data( + mdata: MicrogridData, + *, + component_types: tuple[str], + mid: int, + start_time: datetime, + end_time: datetime, + resampling_period: timedelta, + splits: bool = False, + fetch_soc: bool = False, +) -> pd.DataFrame: + """ + Fetch data of a microgrid and processes it for plotting. + + Args: + mdata: MicrogridData object to fetch data from. + component_types: List of component types to fetch data for. + mid: Microgrid ID. + start_time: Start time for data fetching. + end_time: End time for data fetching. + resampling_period: Time resolution for data fetching. + splits: Whether to split the data into positive and negative parts. + fetch_soc: Whether to fetch state of charge (SOC) data. + + Returns: + pd.DataFrame: DataFrame containing the processed data. + + Raises: + ValueError: If no data is found for the given microgrid and time range or if + unexpected component types are present in the data. + """ + print( + f"Requesting data from {start_time} to {end_time} at {resampling_period} resolution" + ) + df = await mdata.ac_active_power( + microgrid_id=mid, + component_types=component_types, + start=start_time, + end=end_time, + resampling_period=resampling_period, + keep_components=False, + splits=splits, + unit="kW", + ) + if df is None or df.empty: + raise ValueError( + f"No data found for microgrid {mid} between {start_time} and {end_time}" + ) + + print(f"Received {df.shape[0]} rows and {df.shape[1]} columns") + + if fetch_soc: + soc_df = await mdata.soc( + microgrid_id=mid, + start=start_time, + end=end_time, + resampling_period=resampling_period, + keep_components=False, + ) + if soc_df is None or soc_df.empty: + raise ValueError( + f"No SOC data found for microgrid {mid} between {start_time} and {end_time}" + ) + df = pd.concat([df, soc_df.rename(columns={"battery": "soc"})[["soc"]]], axis=1) + + df["soc"] = df.get("soc", np.nan) + # For later vizualization we default to zero + df["battery"] = df.get("battery", 0) + df["chp"] = df["chp"].clip(upper=0) if "chp" in df.columns else 0 + df["pv"] = df["pv"].clip(upper=0) if "pv" in df.columns else 0 + + # Determine consumption if not present + if "consumption" not in df.columns: + if any( + ct not in ["grid", "pv", "battery", "chp", "consumption"] + for ct in df.columns + ): + raise ValueError( + "Consumption not found in data and unexpected component types present." + ) + df["consumption"] = df["grid"] - (df["chp"] + df["pv"] + df["battery"]) + + return df + + +def plot_power_flow(df: pd.DataFrame, ax: Axes | None = None) -> None: + """Plot the power flow of the microgrid.""" + d = -df.copy() + i = d.index + cons = -d["consumption"].to_numpy() + + has_chp = "chp" in d.columns + has_pv = "pv" in d.columns + chp = d["chp"] if has_chp else 0 * cons + prod = chp + (d["pv"].clip(lower=0) if has_pv else 0) + + if ax is None: + fig, ax = plt.subplots(figsize=(30, 10), sharex=True) + + if has_pv: + ax.fill_between( + i, + chp, + prod, + color="gold", + alpha=0.7, + label="PV" + (" (on CHP)" if has_chp else ""), + ) + if has_chp: + ax.fill_between(i, chp, color="cornflowerblue", alpha=0.5, label="CHP") + + if "battery" in d.columns: + bat_cons = -(d["consumption"].to_numpy() + d["battery"].to_numpy()) + charge = bat_cons > cons + discharge = bat_cons < cons + ax.fill_between( + i, + cons, + bat_cons, + where=charge, + color="green", + alpha=0.2, + label="Charge", + ) + ax.fill_between( + i, + cons, + bat_cons, + where=discharge, + color="lightcoral", + alpha=0.5, + label="Discharge", + ) + + if "grid" in d.columns: + ax.plot(i, -d["grid"], color="grey", label="Grid") + + ax.plot(i, cons, "k-", label="Consumption") + ax.set_ylabel("Power [kW]") + ax.legend() + ax.grid(True) + ax.set_ylim(bottom=min(0, ax.get_ylim()[0])) + + +def plot_energy_trade(df: pd.DataFrame, ax: Axes | None = None) -> None: + """Plot the energy trade of the microgrid.""" + d = -df.copy() + cons = -d["consumption"] + trade = cons.copy() + + has_chp = "chp" in d.columns + has_pv = "pv" in d.columns + chp = d["chp"] if has_chp else 0 * cons + prod = chp + (d["pv"].clip(lower=0) if has_pv else 0) + trade -= prod + + g = trade.resample("15min").mean() / 4 + + if ax is None: + fig, ax = plt.subplots(figsize=(30, 10), sharex=True) + ax.fill_between( + g.index, 0, g.clip(lower=0).to_numpy(), color="darkred", label="Buy", step="pre" + ) + ax.fill_between( + g.index, + 0, + g.clip(upper=0).to_numpy(), + color="darkgreen", + label="Sell", + step="pre", + ) + ax.set_ylabel("Energy [kWh]") + ax.legend() + ax.grid(True) + + +def plot_power_flow_trade(df: pd.DataFrame) -> None: + """Plot both power flow and energy trade of the microgrid.""" + fig, (ax1, ax2) = plt.subplots( + 2, 1, figsize=(30, 10), sharex=True, height_ratios=[4, 1] + ) + plot_power_flow(df, ax=ax1) + plot_energy_trade(df, ax=ax2) + plt.tight_layout() + plt.show() + + +def plot_battery_power(df: pd.DataFrame) -> None: + """Plot the battery power and state of charge (SOC) of the microgrid.""" + if "soc" not in df.columns: + raise ValueError( + "DataFrame must contain 'soc' column for battery SOC plotting." + ) + + fig, ax1 = plt.subplots(figsize=(30, 6.66)) # Increased the figure height + + # Plot Battery SOC + twin_ax = ax1.twinx() + assert df["soc"].ndim == 1, "SOC data should be 1D" + soc = df["soc"] # .iloc[:, 0] if df["soc"].ndim > 1 else df["soc"] + twin_ax.grid(False) # Turn off the grid for the SOC plot + twin_ax.fill_between( + df.index, + soc.to_numpy() * 0, + soc.to_numpy(), + color="grey", + alpha=0.4, + label="SOC", + ) + twin_ax.set_ylim(0, 100) # Set SOC range between 0 and 100 + twin_ax.set_ylabel("Battery SOC", fontsize=14) # Increased the font size + twin_ax.tick_params( + axis="y", labelcolor="grey", labelsize=14 + ) # Increased the font size for ticks + + # Available power + available = df["battery"] - df["grid"] + ax1.plot( + df.index, + available, + color="black", + linestyle="-", + label="Available power", + alpha=1, + ) + + # Plot Battery Power on primary y-axis + ax1.axhline(y=0, color="grey", linestyle="--", alpha=0.5) + # Make battery power range symmetric + max_abs_bat = max( + abs(df["battery"].min()), + abs(df["battery"].max()), + abs(available.min()), + abs(available.max()), + ) + ax1.set_ylim(-max_abs_bat * 1.1, max_abs_bat * 1.1) + ax1.set_ylabel( + "Battery Power", fontsize=14 + ) # Updated the label to include Grid - Bat + ax1.tick_params( + axis="y", labelcolor="black", labelsize=14 + ) # Increased the font size for ticks + + # Fill Battery Power around zero (reverse sign) + ax1.fill_between( + df.index, + 0, + df["battery"], + where=(df["battery"].to_numpy() > 0).tolist(), + interpolate=False, + color="green", + alpha=0.9, + label="Charge", + ) + ax1.fill_between( + df.index, + 0, + df["battery"], + where=(df["battery"].to_numpy() <= 0).tolist(), + interpolate=False, + color="red", + alpha=0.9, + label="Discharge", + ) + + fig.tight_layout() + fig.legend(loc="upper left", fontsize=14) # Increased font size for legend + plt.show() + + +def plot_monthly(df: pd.DataFrame) -> pd.DataFrame: + """Plot monthly aggregate data.""" + months = df.resample("1MS").sum() + resolution = (df.index[1] - df.index[0]).total_seconds() + kW2MWh = resolution / 3600 / 1000 # pylint: disable=invalid-name + months *= kW2MWh + # Ensure the index is a datetime + if not isinstance(months.index, pd.DatetimeIndex): + months.index = pd.to_datetime(months.index) + months.index = pd.Index(months.index.date) + pos, neg = ( + months[[c for c in months.columns if "_pos" in c]], + months[[c for c in months.columns if "_neg" in c]], + ) + + pos = pos.rename( + columns={ + "grid_pos": "Grid Consumption", + "battery_pos": "Battery Charge", + "consumption_pos": "Consumption", + "pv_pos": "PV Consumption", + "chp_pos": "CHP Consumption", + } + ) + neg = neg.rename( + columns={ + "grid_neg": "Grid Feed-in", + "battery_neg": "Battery Discharge", + "consumption_neg": "Unknown Production", + "pv_neg": "PV Production", + "chp_neg": "CHP Production", + } + ) + + # Remove zero columns + pos = pos.loc[:, pos.abs().sum(axis=0) > 0] + neg = neg.loc[:, neg.abs().sum(axis=0) > 0] + + ax = pos.plot.bar() + neg.plot.bar(ax=ax, alpha=0.7) + plt.xticks(rotation=0) + plt.ylabel("Energy [MWh]") + return months From 98eb3dff29c86cf91937a3a381a4e857ebf65377 Mon Sep 17 00:00:00 2001 From: cwasicki <126617870+cwasicki@users.noreply.github.com> Date: Wed, 28 May 2025 20:17:05 +0200 Subject: [PATCH 2/6] Update release notes Signed-off-by: cwasicki <126617870+cwasicki@users.noreply.github.com> --- RELEASE_NOTES.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index a2c59919..40977cc4 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -10,6 +10,8 @@ ## New Features +* Add reporting module for asset optimization. + ## Bug Fixes From c03252045d709ed7bbf0bee3c80e499119acb9b0 Mon Sep 17 00:00:00 2001 From: cwasicki <126617870+cwasicki@users.noreply.github.com> Date: Wed, 28 May 2025 19:31:34 +0200 Subject: [PATCH 3/6] Add Asset optimization example notebook Signed-off-by: cwasicki <126617870+cwasicki@users.noreply.github.com> --- RELEASE_NOTES.md | 2 +- examples/Asset-optimization.ipynb | 912 ++++++++++++++++++++++++++++++ 2 files changed, 913 insertions(+), 1 deletion(-) create mode 100644 examples/Asset-optimization.ipynb diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 40977cc4..6dc68741 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -10,7 +10,7 @@ ## New Features -* Add reporting module for asset optimization. +* Add reporting module for asset optimization and example notebook. diff --git a/examples/Asset-optimization.ipynb b/examples/Asset-optimization.ipynb new file mode 100644 index 00000000..a3f74680 --- /dev/null +++ b/examples/Asset-optimization.ipynb @@ -0,0 +1,912 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "cell_id": "ecb69413fe59458a93d7284bd427b595", + "deepnote_app_block_group_id": null, + "deepnote_app_block_order": 0, + "deepnote_app_block_visible": true, + "deepnote_cell_type": "text-cell-h1", + "formattedRanges": [] + }, + "source": [ + "# PV Optimization Monitoring" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "cell_id": "2c26245ed9274e3b9c82323fd0dbe3dc", + "deepnote_app_block_group_id": null, + "deepnote_app_block_order": 1, + "deepnote_app_block_visible": true, + "deepnote_cell_type": "text-cell-p", + "formattedRanges": [ + { + "fromCodePoint": 0, + "marks": { + "bold": true + }, + "toCodePoint": 85, + "type": "marks" + } + ] + }, + "source": [ + "Please ensure to select exactly the component types that exist on the selected site. " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "cell_id": "26aab8e8fe79411a81d212d809df0692", + "deepnote_cell_type": "markdown" + }, + "source": [ + "This notebook provides a workflow for monitoring and optimizing photovoltaic (PV) systems within a microgrid. It guides the user through selecting the correct component types for a specific site, fetching and visualizing power flow and battery data, and analyzing monthly aggregates. " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "cell_id": "e224ca0fb56f4ff1bdc61cb5fc150ee8", + "deepnote_app_block_group_id": null, + "deepnote_app_block_order": 2, + "deepnote_app_block_visible": true, + "deepnote_cell_type": "text-cell-p", + "formattedRanges": [ + { + "fromCodePoint": 0, + "marks": { + "bold": true + }, + "toCodePoint": 49, + "type": "marks" + } + ] + }, + "source": [ + "Otherwise will cause errors or incorrect results!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cell_id": "cfe326c6efb94dada9a7f6ce36f81d7c", + "deepnote_cell_type": "code", + "execution_context_id": "332de439-32ca-4300-9e80-0064a729953c", + "execution_millis": 14951, + "execution_start": 1748455637248, + "source_hash": "3d35b38e" + }, + "outputs": [], + "source": [ + "!pip install matplotlib pandas\n", + "!pip install frequenz-lib-notebooks" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cell_id": "40984e17ae8046ec962cf11183c8d966", + "deepnote_app_block_visible": false, + "deepnote_app_is_code_hidden": true, + "deepnote_app_is_output_hidden": true, + "deepnote_cell_type": "code", + "execution_context_id": "332de439-32ca-4300-9e80-0064a729953c", + "execution_millis": 401, + "execution_start": 1748455659428, + "is_code_hidden": false, + "source_hash": "a8f850cf" + }, + "outputs": [], + "source": [ + "import pandas as pd \n", + "import matplotlib.pyplot as plt\n", + "from datetime import datetime, timedelta\n", + "import os\n", + "from dotenv import load_dotenv\n", + "\n", + "\n", + "import frequenz.lib.notebooks.reporting.asset_optimization as ao \n", + "\n", + "import sys\n", + "from frequenz.data.microgrid import MicrogridData \n", + "\n", + "plt.rcParams[\"figure.figsize\"] = (30,6.66)\n", + "plt.rcParams[\"axes.grid\"] = True" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cell_id": "08e0dbad2cd64250b49ac2b94e0840b1", + "deepnote_app_block_visible": false, + "deepnote_app_is_code_hidden": true, + "deepnote_app_is_output_hidden": true, + "deepnote_cell_type": "code", + "execution_context_id": "332de439-32ca-4300-9e80-0064a729953c", + "execution_millis": 102, + "execution_start": 1748455660273, + "source_hash": "8d5e3fdf" + }, + "outputs": [], + "source": [ + "SERVICE_ADDRESS = os.environ[\"REPORTING_API_URL\"]\n", + "API_KEY = os.environ[\"REPORTING_API_KEY\"]\n", + "\n", + "mdata = MicrogridData(\n", + " server_url=SERVICE_ADDRESS,\n", + " key=API_KEY, \n", + " microgrid_config_path=\"./microgrids.toml\",\n", + ")\n", + "\n", + "mids = [i.replace(\"iot\", \"\") for i in mdata.microgrid_ids] " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "cell_id": "1967a3f81896441895256ec6207117e2", + "deepnote_cell_type": "markdown" + }, + "source": [ + "# Power flow" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cell_id": "64d33604c6064afa93936db9dcd2a093", + "deepnote_app_block_group_id": null, + "deepnote_app_block_order": 4, + "deepnote_app_block_visible": true, + "deepnote_cell_type": "input-select", + "deepnote_input_label": "", + "deepnote_variable_custom_options": [ + "Option 1", + "Option 2" + ], + "deepnote_variable_name": "microgrid_id", + "deepnote_variable_options": [ + "1", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10", + "11", + "12", + "13", + "14", + "15", + "16", + "17", + "19", + "20", + "23", + "24", + "25", + "26", + "27", + "28", + "29", + "30", + "31", + "33", + "34", + "35", + "36", + "40", + "41", + "42", + "43", + "44", + "45", + "48", + "50", + "52", + "53", + "54", + "56", + "57", + "58", + "59", + "60", + "62", + "64", + "65", + "66", + "67", + "69", + "70", + "71", + "72", + "73", + "74", + "75", + "76", + "77", + "78", + "79", + "80", + "81", + "82", + "84", + "85", + "86", + "87", + "88", + "90", + "91", + "92", + "93", + "106", + "107", + "108", + "109", + "111", + "113", + "115", + "116", + "117", + "118", + "119", + "120", + "121", + "122", + "123", + "125", + "126", + "127", + "128", + "131", + "132", + "133", + "137", + "138", + "139", + "140", + "141", + "142", + "143", + "145", + "146", + "148", + "149", + "150", + "152", + "154", + "155", + "156", + "157", + "158", + "160", + "161", + "162", + "163", + "164", + "165", + "168", + "169", + "170", + "171", + "172", + "174", + "175", + "176", + "177", + "178", + "179", + "180", + "181", + "182", + "183", + "184", + "185", + "186", + "187", + "188", + "189", + "190", + "191", + "193", + "194", + "195", + "196", + "198", + "199", + "200", + "201", + "202", + "203", + "204", + "205", + "206", + "207", + "208", + "209", + "210" + ], + "deepnote_variable_select_type": "from-variable", + "deepnote_variable_selected_variable": "mids", + "deepnote_variable_value": "13", + "execution_context_id": "332de439-32ca-4300-9e80-0064a729953c", + "execution_millis": 1, + "execution_start": 1748455664254, + "source_hash": "b9c1d5fa" + }, + "outputs": [], + "source": [ + "microgrid_id = '13'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cell_id": "12d94c327cac4d41abbb4236206b0cea", + "deepnote_app_is_code_hidden": true, + "deepnote_cell_type": "code", + "execution_context_id": "332de439-32ca-4300-9e80-0064a729953c", + "execution_millis": 1, + "execution_start": 1748455666860, + "source_hash": "4638f079" + }, + "outputs": [], + "source": [ + "mid = int(microgrid_id)\n", + "ctypes = mdata.microgrid_configs[str(mid)].component_types()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cell_id": "49d0a1d74d6949988da627d4d5007ed8", + "deepnote_allow_multiple_values": true, + "deepnote_app_block_group_id": null, + "deepnote_app_block_order": 5, + "deepnote_app_block_visible": true, + "deepnote_cell_type": "input-select", + "deepnote_input_label": "", + "deepnote_variable_custom_options": [ + "grid", + "pv", + "bat", + "chp" + ], + "deepnote_variable_name": "component_types", + "deepnote_variable_options": [ + "grid", + "pv", + "chp", + "battery", + "consumption" + ], + "deepnote_variable_select_type": "from-variable", + "deepnote_variable_selected_variable": "ctypes", + "deepnote_variable_value": [ + "grid", + "pv", + "battery", + "consumption", + "chp" + ], + "execution_context_id": "332de439-32ca-4300-9e80-0064a729953c", + "execution_millis": 1, + "execution_start": 1748455669679, + "source_hash": "7b9ec5a" + }, + "outputs": [], + "source": [ + "component_types = ['grid','pv','battery','consumption','chp']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cell_id": "19bd8e9ab228459dbecc4a3a5c96584e", + "deepnote_app_block_group_id": "8ab0362605df4c9aae437b1cdf4cc2e7", + "deepnote_app_block_order": 6, + "deepnote_app_block_visible": true, + "deepnote_cell_type": "input-date", + "deepnote_input_date_version": 2, + "deepnote_input_label": "End Datum", + "deepnote_variable_name": "end_date", + "deepnote_variable_value": "2025-05-28T00:00:00.000Z", + "execution_context_id": "332de439-32ca-4300-9e80-0064a729953c", + "execution_millis": 1, + "execution_start": 1748455671345, + "source_hash": "fa146508" + }, + "outputs": [], + "source": [ + "\n", + "from dateutil.parser import parse as _deepnote_parse\n", + "end_date = _deepnote_parse('2025-05-28T00:00:00.000Z').date()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cell_id": "dd841bea93444a6faa2041baaa2d4992", + "deepnote_app_block_group_id": "8ab0362605df4c9aae437b1cdf4cc2e7", + "deepnote_app_block_order": 7, + "deepnote_app_block_visible": true, + "deepnote_cell_type": "input-text", + "deepnote_input_label": "", + "deepnote_variable_name": "days_back", + "deepnote_variable_value": "7", + "execution_context_id": "332de439-32ca-4300-9e80-0064a729953c", + "execution_millis": 1, + "execution_start": 1748455672474, + "source_hash": "6f708e28" + }, + "outputs": [], + "source": [ + "days_back = '7'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cell_id": "5e3a24193dd3421eaf217b65998b2846", + "deepnote_app_block_group_id": "8ab0362605df4c9aae437b1cdf4cc2e7", + "deepnote_app_block_order": 8, + "deepnote_app_block_visible": true, + "deepnote_cell_type": "input-select", + "deepnote_input_label": "Sampling period seconds", + "deepnote_variable_custom_options": [ + "10", + "60", + "900", + "1" + ], + "deepnote_variable_name": "resolution", + "deepnote_variable_options": [ + "10", + "60", + "900", + "1" + ], + "deepnote_variable_select_type": "from-options", + "deepnote_variable_selected_variable": "", + "deepnote_variable_value": "900", + "execution_context_id": "332de439-32ca-4300-9e80-0064a729953c", + "execution_millis": 1, + "execution_start": 1748455673418, + "source_hash": "ed3fae76" + }, + "outputs": [], + "source": [ + "resolution = '900'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cell_id": "d9cfecb790f243f486a9954c8c04c887", + "deepnote_app_block_visible": false, + "deepnote_app_is_code_hidden": true, + "deepnote_app_is_output_hidden": true, + "deepnote_cell_type": "code", + "execution_context_id": "332de439-32ca-4300-9e80-0064a729953c", + "execution_millis": 1, + "execution_start": 1748455675820, + "is_code_hidden": false, + "source_hash": "f374b47" + }, + "outputs": [], + "source": [ + "end_dt = datetime(end_date.year, end_date.month, end_date.day)\n", + "start_dt = end_dt - timedelta(days=int(days_back))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cell_id": "e001b98f5dc54835a3521512bf07f60a", + "deepnote_cell_type": "code", + "execution_context_id": "332de439-32ca-4300-9e80-0064a729953c", + "execution_millis": 1975, + "execution_start": 1748455677076, + "source_hash": "b99125fc" + }, + "outputs": [], + "source": [ + "data = await ao.fetch_data(\n", + " mdata=mdata,\n", + " component_types=component_types,\n", + " mid=mid,\n", + " start_time=start_dt,\n", + " end_time=end_dt,\n", + " resampling_period=timedelta(seconds=int(resolution)), \n", + " fetch_soc=True,\n", + ")\n", + "\n", + "df = data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cell_id": "4fd8e85dfa6b4fddb19439a08792b37a", + "deepnote_cell_type": "code", + "execution_context_id": "332de439-32ca-4300-9e80-0064a729953c", + "execution_millis": 394, + "execution_start": 1748455679248, + "source_hash": "c6343201" + }, + "outputs": [], + "source": [ + "ao.plot_power_flow(df)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cell_id": "1e8c70b35b044be3a116a9ba46b63eab", + "deepnote_cell_type": "code", + "execution_context_id": "332de439-32ca-4300-9e80-0064a729953c", + "execution_millis": 260, + "execution_start": 1748455681724, + "source_hash": "ef9b7fad" + }, + "outputs": [], + "source": [ + "ao.plot_battery_power(df)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cell_id": "21b8a7630bfa4efe8df398e25fde77cb", + "deepnote_cell_type": "code", + "execution_context_id": "332de439-32ca-4300-9e80-0064a729953c", + "execution_millis": 523, + "execution_start": 1748455682563, + "source_hash": "c4014e22" + }, + "outputs": [], + "source": [ + "ao.plot_power_flow_trade(df)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cell_id": "c80877b2b96a48389ddeb9b4ed0502cd", + "deepnote_app_block_group_id": null, + "deepnote_app_block_order": 13, + "deepnote_app_block_visible": true, + "deepnote_app_is_code_hidden": true, + "deepnote_app_is_output_hidden": false, + "deepnote_cell_type": "code", + "execution_context_id": "332de439-32ca-4300-9e80-0064a729953c", + "execution_millis": 242, + "execution_start": 1748455683334, + "source_hash": "eaef49c5" + }, + "outputs": [], + "source": [ + "df[[\"grid\", \"pv\", \"chp\", \"battery\"]].plot(color=[\"grey\", \"gold\", \"cornflowerblue\", \"red\"]);" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "cell_id": "61bcd4dfcc3a4844bef476bc047d5164", + "deepnote_app_block_group_id": null, + "deepnote_app_block_order": 15, + "deepnote_app_block_visible": true, + "deepnote_cell_type": "text-cell-h1", + "formattedRanges": [] + }, + "source": [ + "# Monthly aggregates" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "allow_embed": false, + "cell_id": "6ea5e39d0c7e43c7b124912556e8a839", + "deepnote_app_block_group_id": "2bafe889eda94a609bbef795b345c773", + "deepnote_app_block_order": 16, + "deepnote_app_block_visible": true, + "deepnote_cell_type": "input-date", + "deepnote_input_date_version": 2, + "deepnote_input_label": "Start Datum", + "deepnote_variable_name": "start_date_2", + "deepnote_variable_value": "2024-01-01T00:00:00.000Z", + "execution_context_id": "332de439-32ca-4300-9e80-0064a729953c", + "execution_millis": 0, + "execution_start": 1748455758828, + "source_hash": "d5b1cc3c" + }, + "outputs": [], + "source": [ + "\n", + "from dateutil.parser import parse as _deepnote_parse\n", + "start_date_2 = _deepnote_parse('2024-01-01T00:00:00.000Z').date()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "allow_embed": false, + "cell_id": "dbb18a2cf3844bfe91101c213307d4c8", + "deepnote_app_block_group_id": "2bafe889eda94a609bbef795b345c773", + "deepnote_app_block_order": 17, + "deepnote_app_block_visible": true, + "deepnote_cell_type": "input-date", + "deepnote_input_date_version": 2, + "deepnote_input_label": "End Datum", + "deepnote_variable_name": "end_date_2", + "deepnote_variable_value": "2025-06-01T00:00:00.000Z", + "execution_context_id": "332de439-32ca-4300-9e80-0064a729953c", + "execution_millis": 1, + "execution_start": 1748455759943, + "source_hash": "2a55aa8c" + }, + "outputs": [], + "source": [ + "\n", + "from dateutil.parser import parse as _deepnote_parse\n", + "end_date_2 = _deepnote_parse('2025-06-01T00:00:00.000Z').date()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "allow_embed": false, + "cell_id": "742679181e1d409a81a45fa2c4162261", + "deepnote_app_block_group_id": "2bafe889eda94a609bbef795b345c773", + "deepnote_app_block_order": 18, + "deepnote_app_block_visible": true, + "deepnote_cell_type": "input-select", + "deepnote_input_label": "Resampling period in seconds for raw data", + "deepnote_variable_custom_options": [ + "900", + "3600", + "86400" + ], + "deepnote_variable_name": "resolution2", + "deepnote_variable_options": [ + "900", + "3600", + "86400" + ], + "deepnote_variable_select_type": "from-options", + "deepnote_variable_selected_variable": "", + "deepnote_variable_value": "900", + "execution_context_id": "332de439-32ca-4300-9e80-0064a729953c", + "execution_millis": 1, + "execution_start": 1748455761505, + "source_hash": "98a617b3" + }, + "outputs": [], + "source": [ + "resolution2 = '900'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cell_id": "7b9230fb90a84a81b9369e509c2c32f2", + "deepnote_app_block_visible": false, + "deepnote_app_is_code_hidden": true, + "deepnote_app_is_output_hidden": true, + "deepnote_cell_type": "code", + "execution_context_id": "332de439-32ca-4300-9e80-0064a729953c", + "execution_millis": 0, + "execution_start": 1748455763980, + "source_hash": "c0cb4b3e" + }, + "outputs": [], + "source": [ + "start_dt2 = datetime(year=start_date_2.year, month=start_date_2.month, day=start_date_2.day)\n", + "end_dt2 = datetime(end_date_2.year, end_date_2.month, end_date_2.day)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cell_id": "ff9701fb932e42e6ab2eed123a910db1", + "deepnote_cell_type": "code", + "execution_context_id": "332de439-32ca-4300-9e80-0064a729953c", + "execution_millis": 15490, + "execution_start": 1748455765248, + "source_hash": "625b418" + }, + "outputs": [], + "source": [ + "data2 = await ao.fetch_data(\n", + " mdata=mdata,\n", + " component_types=component_types,\n", + " mid=mid,\n", + " start_time=start_dt2,\n", + " end_time=end_dt2,\n", + " resampling_period=timedelta(seconds=int(resolution2)),\n", + " splits=True,\n", + " fetch_soc=False,\n", + ")\n", + "\n", + "df2 = data2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cell_id": "0c77eb8cbbb04f92b28d31886f953645", + "deepnote_cell_type": "code", + "execution_context_id": "332de439-32ca-4300-9e80-0064a729953c", + "execution_millis": 2, + "execution_start": 1748455780788, + "source_hash": "caa55e2e" + }, + "outputs": [], + "source": [ + "df2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cell_id": "4574e87bad7c4153ba898e87e6128986", + "deepnote_cell_type": "code", + "execution_context_id": "332de439-32ca-4300-9e80-0064a729953c", + "execution_millis": 328, + "execution_start": 1748455780838, + "source_hash": "8a171a4c" + }, + "outputs": [], + "source": [ + "months = ao.plot_monthly(df2)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cell_id": "b7b932701bae486bad5c4e5b909c3573", + "deepnote_app_block_group_id": null, + "deepnote_app_block_order": 22, + "deepnote_app_block_visible": true, + "deepnote_app_is_code_hidden": true, + "deepnote_cell_type": "code", + "deepnote_table_loading": false, + "deepnote_table_state": { + "cellFormattingRules": [], + "columnDisplayNames": [], + "columnOrder": [ + "grid", + "pv", + "chp", + "battery", + "grid_pos", + "pv_pos", + "chp_pos", + "battery_pos", + "grid_neg", + "pv_neg", + "chp_neg", + "battery_neg", + "256", + "258", + "259", + "260", + "263", + "266", + "269", + "272", + "275", + "278", + "281", + "284", + "287", + "290", + "293", + "296" + ], + "conditionalFilters": [], + "filters": [], + "hiddenColumnIds": [], + "pageIndex": 0, + "pageSize": 50, + "sortBy": [], + "wrappedTextColumnIds": [] + }, + "execution_context_id": "332de439-32ca-4300-9e80-0064a729953c", + "execution_millis": 2, + "execution_start": 1748455781238, + "source_hash": "b09bd4ad" + }, + "outputs": [], + "source": [ + "months" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cell_id": "7113d4f46f5f4f7286c0ff83583a3693", + "deepnote_app_block_visible": false, + "deepnote_app_is_code_hidden": true, + "deepnote_app_is_output_hidden": true, + "deepnote_cell_type": "code", + "execution_context_id": "737a3927-eac0-43bc-9dac-bc83ffd1cad0", + "execution_millis": 189, + "execution_start": 1745910080929, + "source_hash": "6cf25205" + }, + "outputs": [], + "source": [ + "# Write the monthly data to a CSV file\n", + "# months.to_csv(\"monthly.csv\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "created_in_deepnote_cell": true, + "deepnote_cell_type": "markdown" + }, + "source": [ + "\n", + "Created in deepnote.com \n", + "Created in Deepnote" + ] + } + ], + "metadata": { + "deepnote_app_clear_outputs": false, + "deepnote_app_comments_enabled": false, + "deepnote_app_hide_all_code_blocks_enabled": true, + "deepnote_app_layout": "powerful-article", + "deepnote_app_reactivity_enabled": true, + "deepnote_app_run_on_input_enabled": false, + "deepnote_app_run_on_load_enabled": false, + "deepnote_app_table_of_contents_enabled": true, + "deepnote_app_width": "full-width", + "deepnote_full_width": true, + "deepnote_notebook_id": "bfa5ed90f2bc460db93c3580af16d789", + "kernelspec": { + "display_name": "python-3.11.0rc1", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.11.0" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} From 419d64d04841360ffc915651863984256cfb4194 Mon Sep 17 00:00:00 2001 From: cwasicki <126617870+cwasicki@users.noreply.github.com> Date: Tue, 10 Jun 2025 18:59:00 +0200 Subject: [PATCH 4/6] Support multiple config files in component data Signed-off-by: cwasicki <126617870+cwasicki@users.noreply.github.com> --- src/frequenz/data/microgrid/component_data.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/frequenz/data/microgrid/component_data.py b/src/frequenz/data/microgrid/component_data.py index c8f98677..fa8544ad 100644 --- a/src/frequenz/data/microgrid/component_data.py +++ b/src/frequenz/data/microgrid/component_data.py @@ -19,17 +19,19 @@ class MicrogridData: """Fetch power data for component types of a microgrid.""" - def __init__(self, server_url: str, key: str, microgrid_config_path: str) -> None: + def __init__(self, server_url: str, key: str, microgrid_config_path: str | list[str]) -> None: """Initialize microgrid data. Args: server_url: URL of the reporting service. key: Authentication key to the service. - microgrid_config_path: Path to the config file with microgrid components. + microgrid_config_path: Path(s) to the config file with microgrid components. """ self._client = ReportingApiClient(server_url=server_url, key=key) - - self._microgrid_configs = MicrogridConfig.load_configs(microgrid_config_path) + paths = [microgrid_config_path] if isinstance(microgrid_config_path, str) else microgrid_config_path + if len(paths) < 1: + raise ValueError("At least one microgrid config path must be provided") + self._microgrid_configs = MicrogridConfig.load_configs(*paths) @property def microgrid_ids(self) -> list[str]: From 71f574f02784fef4825c08312105e2cbbac28052 Mon Sep 17 00:00:00 2001 From: cwasicki <126617870+cwasicki@users.noreply.github.com> Date: Tue, 10 Jun 2025 21:24:42 +0200 Subject: [PATCH 5/6] Fix data fetching in asset optimization reporting A validation check for sites without configured consumption fails if SoC data is fetched too. --- .../notebooks/reporting/asset_optimization.py | 31 ++++++++++--------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/src/frequenz/lib/notebooks/reporting/asset_optimization.py b/src/frequenz/lib/notebooks/reporting/asset_optimization.py index f014609b..8624e665 100644 --- a/src/frequenz/lib/notebooks/reporting/asset_optimization.py +++ b/src/frequenz/lib/notebooks/reporting/asset_optimization.py @@ -66,6 +66,22 @@ async def fetch_data( print(f"Received {df.shape[0]} rows and {df.shape[1]} columns") + # For later vizualization we default to zero + df["battery"] = df.get("battery", 0) + df["chp"] = df["chp"].clip(upper=0) if "chp" in df.columns else 0 + df["pv"] = df["pv"].clip(upper=0) if "pv" in df.columns else 0 + + # Determine consumption if not present + if "consumption" not in df.columns: + if any( + ct not in ["grid", "pv", "battery", "chp", "consumption"] + for ct in df.columns + ): + raise ValueError( + f"Consumption not found in data and unexpected component types present: {df.columns.tolist()}" + ) + df["consumption"] = df["grid"] - (df["chp"] + df["pv"] + df["battery"]) + if fetch_soc: soc_df = await mdata.soc( microgrid_id=mid, @@ -81,21 +97,6 @@ async def fetch_data( df = pd.concat([df, soc_df.rename(columns={"battery": "soc"})[["soc"]]], axis=1) df["soc"] = df.get("soc", np.nan) - # For later vizualization we default to zero - df["battery"] = df.get("battery", 0) - df["chp"] = df["chp"].clip(upper=0) if "chp" in df.columns else 0 - df["pv"] = df["pv"].clip(upper=0) if "pv" in df.columns else 0 - - # Determine consumption if not present - if "consumption" not in df.columns: - if any( - ct not in ["grid", "pv", "battery", "chp", "consumption"] - for ct in df.columns - ): - raise ValueError( - "Consumption not found in data and unexpected component types present." - ) - df["consumption"] = df["grid"] - (df["chp"] + df["pv"] + df["battery"]) return df From 35dd4e64874eea4fb2cd9ca29693d9366b1d2e79 Mon Sep 17 00:00:00 2001 From: cwasicki <126617870+cwasicki@users.noreply.github.com> Date: Wed, 11 Jun 2025 18:54:28 +0200 Subject: [PATCH 6/6] Add plotly version of asset optimization plots --- .../notebooks/asset_optimization_plotly.py | 306 ++++++++++++++++++ 1 file changed, 306 insertions(+) create mode 100644 src/frequenz/lib/notebooks/asset_optimization_plotly.py diff --git a/src/frequenz/lib/notebooks/asset_optimization_plotly.py b/src/frequenz/lib/notebooks/asset_optimization_plotly.py new file mode 100644 index 00000000..ce59fa12 --- /dev/null +++ b/src/frequenz/lib/notebooks/asset_optimization_plotly.py @@ -0,0 +1,306 @@ +import numpy as np +import pandas as pd +import plotly.graph_objects as go + + +# ───────────────────────── helpers ────────────────────────── +def _split_segments(mask: np.ndarray) -> list[tuple[int, int]]: + """Return [(start, end), …] for each contiguous True-run in *mask*.""" + if not mask.any(): + return [] + diff = np.diff(mask.astype(int)) + starts = np.where(diff == 1)[0] + 1 + ends = np.where(diff == -1)[0] + if mask[0]: + starts = np.r_[0, starts] + if mask[-1]: + ends = np.r_[ends, mask.size - 1] + return list(zip(starts, ends)) + + +def _add_fill(fig, idx, upper, lower, segments, rgba, name): + """One closed polygon per (start,end) segment.""" + for s, e in segments: + xs = np.concatenate([idx[s : e + 1], idx[s : e + 1][::-1]]) + ys = np.concatenate([upper[s : e + 1], lower[s : e + 1][::-1]]) + fig.add_trace( + go.Scatter( + x=xs, + y=ys, + mode="lines", + line=dict(width=0), + line_shape="hv", # ← step edge + fill="toself", + fillcolor=rgba, + hoverinfo="skip", + showlegend=False, + ) + ) + fig.add_trace( + go.Scatter( + x=[None], + y=[None], + mode="markers", + marker=dict(size=10, color=rgba), + name=name, + hoverinfo="skip", + ) + ) + + +# ─────────────────────── power-flow plot ───────────────────── +def plot_power_flow2(df: pd.DataFrame, show: bool = True) -> go.Figure: + """Interactive Plotly clone of the Matplotlib power-flow plot (step-wise).""" + d = -df.copy() + idx = d.index + cons = -d["consumption"].to_numpy() + + chp = d["chp"].to_numpy() if "chp" in d else np.zeros_like(cons) + pv = d["pv"].clip(lower=0).to_numpy() if "pv" in d else np.zeros_like(cons) + bat = d["battery"].to_numpy() if "battery" in d else None + grid = -d["grid"].to_numpy() if "grid" in d else None + + fig = go.Figure() + + # CHP base + if "chp" in d: + fig.add_trace( + go.Scatter( + x=idx, + y=chp, + name="CHP", + mode="lines", + line=dict(width=0), + line_shape="hv", + fill="tozeroy", + fillcolor="rgba(100,149,237,0.50)", + hoverinfo="skip", + ) + ) + + # PV stacked on CHP + if "pv" in d: + fig.add_trace( + go.Scatter( + x=idx, + y=chp + pv, + name="PV (on CHP)" if "chp" in d else "PV", + mode="lines", + line=dict(width=0), + line_shape="hv", + fill="tonexty", + fillcolor="rgba(255,215,0,0.70)", + hoverinfo="skip", + ) + ) + + # Battery polygons + if bat is not None: + bat_cons = -(d["consumption"] + d["battery"]).to_numpy() + _add_fill( + fig, + idx, + bat_cons, + cons, + _split_segments(bat_cons > cons), + "rgba(0,128,0,0.30)", + "Charge", + ) + _add_fill( + fig, + idx, + cons, + bat_cons, + _split_segments(bat_cons < cons), + "rgba(240,128,128,0.55)", + "Discharge", + ) + + # Grid line + if grid is not None: + fig.add_trace( + go.Scatter( + x=idx, + y=grid, + name="Grid", + mode="lines", + line=dict(color="grey"), + line_shape="hv", + ) + ) + + # Consumption line + fig.add_trace( + go.Scatter( + x=idx, + y=cons, + name="Consumption", + mode="lines", + line=dict(color="black", width=2), + line_shape="hv", + ) + ) + + # Layout (unchanged) + fig.update_layout( + title="Microgrid Power Flow", + xaxis_title="Time", + yaxis_title="Power [kW]", + hovermode="x unified", + template="plotly_white", + margin=dict(l=40, r=40, t=40, b=40), + legend=dict(orientation="v", x=1.01, y=1), + height=800, + autosize=True, + width=None, + ) + fig.update_yaxes(range=[min(0, cons.min()), None]) + + if show: + fig.show() + return fig + + +# ───────────────────────── second helpers ────────────────────── +def _contiguous_runs(mask: np.ndarray) -> list[tuple[int, int]]: + if not mask.any(): + return [] + diff = np.diff(mask.astype(int)) + starts = np.where(diff == 1)[0] + 1 + ends = np.where(diff == -1)[0] + if mask[0]: + starts = np.r_[0, starts] + if mask[-1]: + ends = np.r_[ends, mask.size - 1] + return list(zip(starts, ends)) + + +def _add_polygon(fig, x, upper, lower, segments, rgba, name): + for s, e in segments: + xs = np.concatenate([x[s : e + 1], x[s : e + 1][::-1]]) + ys = np.concatenate([upper[s : e + 1], lower[s : e + 1][::-1]]) + fig.add_trace( + go.Scatter( + x=xs, + y=ys, + mode="lines", + line=dict(width=0), + line_shape="hv", # ← step edge + fill="toself", + fillcolor=rgba, + hoverinfo="skip", + showlegend=False, + ) + ) + fig.add_trace( + go.Scatter( + x=[None], + y=[None], + mode="markers", + marker=dict(size=10, color=rgba), + name=name, + hoverinfo="skip", + ) + ) + + +# ─────────────────── battery-power plot ──────────────────────── +def plot_battery_power2(df: pd.DataFrame, show: bool = True) -> go.Figure: + """Battery power & SoC with step-wise lines, all else identical.""" + if "soc" not in df.columns or df["soc"].nunique() <= 1: + raise ValueError("df must contain an 'soc' column that varies") + + idx = df.index + soc = df["soc"].iloc[:, 0] if df["soc"].ndim > 1 else df["soc"] + bat = df["battery"].to_numpy() + avail = (df["battery"] - df["grid"]).to_numpy() + + max_abs = max(map(abs, (bat.min(), bat.max(), avail.min(), avail.max()))) * 1.1 + + fig = go.Figure() + + # SoC grey band (secondary y-axis) + fig.add_trace( + go.Scatter( + x=idx, + y=soc, + mode="lines", + line=dict(width=0), + line_shape="hv", + fill="tozeroy", + fillcolor="rgba(128,128,128,0.40)", + yaxis="y2", + name="SOC", + ) + ) + + # Charge / Discharge polygons + _add_polygon( + fig, + idx, + bat, + np.zeros_like(bat), + _contiguous_runs(bat > 0), + "rgba(0,128,0,0.90)", + "Charge", + ) + _add_polygon( + fig, + idx, + np.zeros_like(bat), + bat, + _contiguous_runs(bat < 0), + "rgba(255,0,0,0.90)", + "Discharge", + ) + + # Available-power black line + fig.add_trace( + go.Scatter( + x=idx, + y=avail, + mode="lines", + line=dict(color="black"), + line_shape="hv", + name="Available power", + ) + ) + + # Zero reference + fig.add_shape( + type="line", + x0=idx.min(), + x1=idx.max(), + y0=0, + y1=0, + line=dict(color="grey", dash="dash"), + yref="y", + xref="x", + ) + + # Layout (unchanged except for title note) + fig.update_layout( + title="Battery Power & State of Charge", + xaxis=dict(title="Time"), + yaxis=dict(title="Battery Power [kW]", range=[-max_abs, max_abs]), + yaxis2=dict( + title="Battery SOC [%]", + range=[0, 100], + overlaying="y", + side="right", + showgrid=False, + ), + hovermode="x unified", + template="plotly_white", + legend=dict(orientation="h", yanchor="bottom", y=1.02, x=0), + margin=dict(l=60, r=60, t=60, b=40), + ) + + if show: + fig.show() + + fig.update_layout( + height=600, autosize=True, width=None + ) # Plotly fills the container + + return fig