Skip to content

Commit 1b8906f

Browse files
joshuaunityFlix6x
joshuaunity
and
Flix6x
authored
feat: extend DBFlexContextSchema validation to only allow valid units (#1364)
* feat: extend flexContext schema validation to only allow valid units Signed-off-by: Joshua Edward <[email protected]> * refactor: moved newly added unit validaitons to DBFlexContextSchema from FlexContextSchema Signed-off-by: Joshua Edward <[email protected]> * chore: added to changelog Signed-off-by: Joshua Edward <[email protected]> * chore: added test for DBFlexContextSchema unit validation feat Signed-off-by: Joshua Edward <[email protected]> * chore: modified changelog Signed-off-by: Joshua Edward <[email protected]> * chore: typo changes and modified changelog Signed-off-by: Joshua Edward <[email protected]> * chore: added accurate example units for flexContext fields in modal UI Signed-off-by: Joshua Edward <[email protected]> * chore: added new unit validation util funciton Signed-off-by: Joshua Edward <[email protected]> * refactor: refactored schema and tests to use the new validation util funciton Signed-off-by: Joshua Edward <[email protected]> * refactor: mapped schema keys to expected user facing keys Signed-off-by: Joshua Edward <[email protected]> * chore: set proper test DB address Signed-off-by: Joshua Edward <[email protected]> * chore: modified validation response contents for capacity price fields Signed-off-by: Joshua Edward <[email protected]> * chore: little modifications to schema validaiton Signed-off-by: Joshua Edward <[email protected]> * chore: more concise code Signed-off-by: Joshua Edward <[email protected]> * tests: added extra test case Signed-off-by: Joshua Edward <[email protected]> * feat: add test case for unsupported time series specs in DB flex-context Signed-off-by: F.N. Claessen <[email protected]> * fix: move comment to the correct class method Signed-off-by: F.N. Claessen <[email protected]> * fix: assign ValidationError to field name rather than to the general schema, and have the message refer to the user-facing field name rather than the internal variable name Signed-off-by: F.N. Claessen <[email protected]> * feat: add another test case for unsupported time series specs in DB flex-context Signed-off-by: F.N. Claessen <[email protected]> * fix: remove redundant validation case Signed-off-by: F.N. Claessen <[email protected]> * revert: more explicit check in relation to the error message Signed-off-by: F.N. Claessen <[email protected]> * fix: consumption-price and production-price are energy prices rather than capacity prices Signed-off-by: F.N. Claessen <[email protected]> * fix: more user-focused error message Signed-off-by: F.N. Claessen <[email protected]> * fix: more common units for Japanese prices Signed-off-by: F.N. Claessen <[email protected]> * fix: correct price examples for energy prices and capacity prices Signed-off-by: F.N. Claessen <[email protected]> --------- Signed-off-by: Joshua Edward <[email protected]> Signed-off-by: F.N. Claessen <[email protected]> Co-authored-by: F.N. Claessen <[email protected]>
1 parent a909665 commit 1b8906f

File tree

5 files changed

+301
-27
lines changed

5 files changed

+301
-27
lines changed

‎documentation/changelog.rst

+1-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ v0.25.0 | February XX, 2025
1111

1212
New features
1313
-------------
14-
* Added form modal to edit an asset's ``flex_context`` [see `PR #1320 <https://github.com/FlexMeasures/flexmeasures/pull/1365>`_ and `PR #1320 <https://github.com/FlexMeasures/flexmeasures/pull/1365>`]
14+
* Added form modal to edit an asset's ``flex_context`` [see `PR #1320 <https://github.com/FlexMeasures/flexmeasures/pull/1320>`_, `PR #1365 <https://github.com/FlexMeasures/flexmeasures/pull/1365>`_ and `PR #1364 <https://github.com/FlexMeasures/flexmeasures/pull/1364>`_]
1515
* Better y-axis titles for charts that show multiple sensors with a shared unit [see `PR #1346 <https://github.com/FlexMeasures/flexmeasures/pull/1346>`_]
1616
* Add CLI command ``flexmeasures jobs save-last-failed`` for saving the last failed jobs [see `PR #1342 <https://www.github.com/FlexMeasures/flexmeasures/pull/1342>`_ and `PR #1359 <https://github.com/FlexMeasures/flexmeasures/pull/1359>`_]
1717

‎flexmeasures/data/schemas/scheduling/__init__.py

+104-13
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
from __future__ import annotations
22

3-
import pint
43
from marshmallow import (
54
Schema,
65
fields,
@@ -19,7 +18,15 @@
1918
)
2019
from flexmeasures.data.schemas.utils import FMValidationError
2120
from flexmeasures.data.schemas.times import AwareDateTimeField, PlanningDurationField
22-
from flexmeasures.utils.unit_utils import ur, units_are_convertible
21+
from flexmeasures.utils.flexmeasures_inflection import p
22+
from flexmeasures.utils.unit_utils import (
23+
ur,
24+
units_are_convertible,
25+
is_capacity_price_unit,
26+
is_energy_price_unit,
27+
is_power_unit,
28+
is_energy_unit,
29+
)
2330

2431

2532
class FlexContextSchema(Schema):
@@ -153,6 +160,8 @@ def check_prices(self, data: dict, **kwargs):
153160
f"""Please switch to using `production-price: {{"sensor": {data[field_map["production-price-sensor"]].id}}}`."""
154161
)
155162

163+
# make sure that the prices fields are valid price units
164+
156165
# All prices must share the same unit
157166
data = self._try_to_convert_price_units(data)
158167

@@ -224,40 +233,122 @@ def _get_variable_quantity_unit(
224233

225234

226235
class DBFlexContextSchema(FlexContextSchema):
236+
mapped_schema_keys = {
237+
field: FlexContextSchema().declared_fields[field].data_key
238+
for field in FlexContextSchema().declared_fields
239+
}
227240

228241
@validates_schema
229242
def forbid_time_series_specs(self, data: dict, **kwargs):
230243
"""Do not allow time series specs for the flex-context fields saved in the db."""
231244

232-
keys_to_check = []
233245
# List of keys to check for time series specs
246+
keys_to_check = []
234247
# All the keys in this list are all fields of type VariableQuantity
235248
for field_var, field in self.declared_fields.items():
236249
if isinstance(field, VariableQuantityField):
237-
keys_to_check.append(field_var)
250+
keys_to_check.append((field_var, field))
238251

239252
# Check each key and raise a ValidationError if it's a list
240-
for key in keys_to_check:
241-
if key in data and isinstance(data[key], list):
253+
for field_var, field in keys_to_check:
254+
if field_var in data and isinstance(data[field_var], list):
242255
raise ValidationError(
243-
f"Time series specs are not allowed in flex-context fields in the DB for '{key}'."
256+
"A time series specification (listing segments) is not supported when storing flex-context fields. Use a fixed quantity or a sensor reference instead.",
257+
field_name=field.data_key,
244258
)
245259

246260
@validates_schema
247-
def forbid_fixed_prices(self, data: dict, **kwargs):
248-
"""Do not allow fixed consumption price or fixed production price in the flex-context fields saved in the db."""
261+
def validate_fields_unit(self, data: dict, **kwargs):
262+
"""Check that each field value has a valid unit."""
263+
264+
self._validate_price_fields(data)
265+
self._validate_power_fields(data)
266+
self._validate_inflexible_device_sensors(data)
267+
268+
def _validate_price_fields(self, data: dict):
269+
"""Validate price fields."""
270+
energy_price_fields = [
271+
"consumption_price",
272+
"production_price",
273+
]
274+
capacity_price_fields = [
275+
"ems_consumption_breach_price",
276+
"ems_production_breach_price",
277+
"ems_peak_consumption_price",
278+
"ems_peak_production_price",
279+
]
280+
281+
# Check that consumption and production prices are Sensors
282+
self._forbid_fixed_prices(data)
283+
284+
for field in energy_price_fields:
285+
if field in data:
286+
self._validate_field(data, "energy price", field, is_energy_price_unit)
287+
for field in capacity_price_fields:
288+
if field in data:
289+
self._validate_field(
290+
data, "capacity price", field, is_capacity_price_unit
291+
)
292+
293+
def _validate_power_fields(self, data: dict):
294+
"""Validate power fields."""
295+
power_fields = [
296+
"ems_power_capacity_in_mw",
297+
"ems_production_capacity_in_mw",
298+
"ems_consumption_capacity_in_mw",
299+
"ems_peak_consumption_in_mw",
300+
"ems_peak_production_in_mw",
301+
]
302+
303+
for field in power_fields:
304+
if field in data:
305+
self._validate_field(data, "power", field, is_power_unit)
306+
307+
def _validate_field(self, data: dict, field_type: str, field: str, unit_validator):
308+
"""Validate fields based on type and unit validator."""
309+
310+
if isinstance(data[field], ur.Quantity):
311+
if not unit_validator(str(data[field].units)):
312+
raise ValidationError(
313+
f"{field_type.capitalize()} field '{self.mapped_schema_keys[field]}' must have {p.a(field_type)} unit.",
314+
field_name=self.mapped_schema_keys[field],
315+
)
316+
elif isinstance(data[field], Sensor):
317+
if not unit_validator(data[field].unit):
318+
raise ValidationError(
319+
f"{field_type.capitalize()} field '{self.mapped_schema_keys[field]}' must have {p.a(field_type)} unit.",
320+
field_name=self.mapped_schema_keys[field],
321+
)
322+
323+
def _validate_inflexible_device_sensors(self, data: dict):
324+
"""Validate inflexible device sensors."""
325+
if "inflexible_device_sensors" in data:
326+
for sensor in data["inflexible_device_sensors"]:
327+
if not is_power_unit(sensor.unit) and not is_energy_unit(sensor.unit):
328+
raise ValidationError(
329+
f"Inflexible device sensor '{sensor.id}' must have a power or energy unit.",
330+
field_name="inflexible-device-sensors",
331+
)
332+
333+
def _forbid_fixed_prices(self, data: dict, **kwargs):
334+
"""Do not allow fixed consumption price or fixed production price in the flex-context fields saved in the db.
335+
336+
This is a temporary restriction as future iterations will allow fixed prices on these fields as well.
337+
"""
249338
if "consumption_price" in data and isinstance(
250-
data["consumption_price"], pint.Quantity
339+
data["consumption_price"], ur.Quantity
251340
):
252341
raise ValidationError(
253-
"Fixed prices are not currently supported for consumption_price in flex-context fields in the DB."
342+
"Fixed prices are not currently supported for consumption-price in flex-context fields in the DB.",
343+
field_name="consumption-price",
254344
)
255345

256346
if "production_price" in data and isinstance(
257-
data["production_price"], pint.Quantity
347+
data["production_price"], ur.Quantity
258348
):
259349
raise ValidationError(
260-
"Fixed prices are not currently supported for production_price in flex-context fields in the DB."
350+
"Fixed prices are not currently supported for production-price in flex-context fields in the DB.",
351+
field_name="production-price",
261352
)
262353

263354

‎flexmeasures/data/schemas/tests/test_scheduling.py

+168-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
from marshmallow.validate import ValidationError
66
import pandas as pd
77

8-
from flexmeasures.data.schemas.scheduling import FlexContextSchema
8+
from flexmeasures.data.schemas.scheduling import FlexContextSchema, DBFlexContextSchema
99
from flexmeasures.data.schemas.scheduling.process import (
1010
ProcessSchedulerFlexModelSchema,
1111
ProcessType,
@@ -284,3 +284,170 @@ def test_flex_context_schema(db, app, setup_site_capacity_sensor, flex_context,
284284
)
285285
else:
286286
schema.load(flex_context)
287+
288+
289+
# test DBFlexContextSchema
290+
@pytest.mark.parametrize(
291+
["flex_context", "fails"],
292+
[
293+
(
294+
{"consumption-price": "13000 kW"},
295+
{
296+
"consumption-price": "Fixed prices are not currently supported for consumption-price in flex-context fields in the DB.",
297+
},
298+
),
299+
(
300+
{
301+
"production-price": {
302+
"sensor": "placeholder for site-power-capacity sensor"
303+
}
304+
},
305+
{
306+
"production-price": "Energy price field 'production-price' must have an energy price unit."
307+
},
308+
),
309+
(
310+
{"production-price": {"sensor": "placeholder for price sensor"}},
311+
False,
312+
),
313+
(
314+
{"consumption-price": "100 EUR/MWh"},
315+
{
316+
"consumption-price": "Fixed prices are not currently supported for consumption-price in flex-context fields in the DB.",
317+
},
318+
),
319+
(
320+
{"production-price": "100 EUR/MW"},
321+
{
322+
"production-price": "Fixed prices are not currently supported for production-price in flex-context fields in the DB."
323+
},
324+
),
325+
(
326+
{"site-power-capacity": 100},
327+
{
328+
"site-power-capacity": f"Unsupported value type. `{type(100)}` was provided but only dict, list and str are supported."
329+
},
330+
),
331+
(
332+
{
333+
"site-power-capacity": [
334+
{
335+
"value": "100 kW",
336+
"start": "2025-03-18T00:00+01:00",
337+
"duration": "P2D",
338+
}
339+
]
340+
},
341+
{
342+
"site-power-capacity": "A time series specification (listing segments) is not supported when storing flex-context fields. Use a fixed quantity or a sensor reference instead."
343+
},
344+
),
345+
(
346+
{"site-power-capacity": "5 kWh"},
347+
{"site-power-capacity": "Cannot convert value `5 kWh` to 'MW'"},
348+
),
349+
(
350+
{"site-consumption-capacity": "6 kWh"},
351+
{"site-consumption-capacity": "Cannot convert value `6 kWh` to 'MW'"},
352+
),
353+
(
354+
{"site-consumption-capacity": "6000 kW"},
355+
False,
356+
),
357+
(
358+
{"site-production-capacity": "6 kWh"},
359+
{"site-production-capacity": "Cannot convert value `6 kWh` to 'MW'"},
360+
),
361+
(
362+
{"site-production-capacity": "7000 kW"},
363+
False,
364+
),
365+
(
366+
{"site-consumption-breach-price": "6 kWh"},
367+
{
368+
"site-consumption-breach-price": "Capacity price field 'site-consumption-breach-price' must have a capacity price unit."
369+
},
370+
),
371+
(
372+
{"site-consumption-breach-price": "450 EUR/MW"},
373+
False,
374+
),
375+
(
376+
{"site-production-breach-price": "550 EUR/MWh"},
377+
{
378+
"site-production-breach-price": "Capacity price field 'site-production-breach-price' must have a capacity price unit."
379+
},
380+
),
381+
(
382+
{"site-production-breach-price": "3500 EUR/MW"},
383+
False,
384+
),
385+
(
386+
{"site-peak-consumption": "60 EUR/MWh"},
387+
{"site-peak-consumption": "Cannot convert value `60 EUR/MWh` to 'MW'"},
388+
),
389+
(
390+
{"site-peak-consumption": "3500 kW"},
391+
False,
392+
),
393+
(
394+
{"site-peak-consumption-price": "6 orange/Mw"},
395+
{
396+
"site-peak-consumption-price": "Cannot convert value '6 orange/Mw' to a valid quantity. 'orange' is not defined in the unit registry"
397+
},
398+
),
399+
(
400+
{"site-peak-consumption-price": "100 EUR/MW"},
401+
False,
402+
),
403+
(
404+
{"site-peak-production": "75kWh"},
405+
{"site-peak-production": "Cannot convert value `75kWh` to 'MW'"},
406+
),
407+
(
408+
{"site-peak-production": "17000 kW"},
409+
False,
410+
),
411+
(
412+
{"site-peak-production-price": "4500 EUR/MWh"},
413+
{
414+
"site-peak-production-price": "Capacity price field 'site-peak-production-price' must have a capacity price unit."
415+
},
416+
),
417+
(
418+
{"site-peak-consumption-price": "700 EUR/MW"},
419+
False,
420+
),
421+
],
422+
)
423+
def test_db_flex_context_schema(
424+
db, app, setup_dummy_sensors, setup_site_capacity_sensor, flex_context, fails
425+
):
426+
schema = DBFlexContextSchema()
427+
428+
price_sensor = setup_dummy_sensors[1]
429+
capacity_sensor = setup_site_capacity_sensor["site-power-capacity"]
430+
431+
# Replace sensor name with sensor ID
432+
for field_name, field_value in flex_context.items():
433+
if isinstance(field_value, dict):
434+
if field_value["sensor"] == "placeholder for site-power-capacity sensor":
435+
flex_context[field_name]["sensor"] = capacity_sensor.id
436+
elif field_value["sensor"] == "placeholder for price sensor":
437+
flex_context[field_name]["sensor"] = price_sensor.id
438+
439+
if fails:
440+
with pytest.raises(ValidationError) as e_info:
441+
schema.load(flex_context)
442+
print(e_info.value.messages)
443+
for field_name, expected_message in fails.items():
444+
assert field_name in e_info.value.messages
445+
# Check all messages for the given field for the expected message
446+
assert any(
447+
[
448+
expected_message in message
449+
for message in e_info.value.messages[field_name]
450+
]
451+
)
452+
else:
453+
schema.load(flex_context)

0 commit comments

Comments
 (0)