From 7cd70a0525fbe883929dd03b65cd7e889664d652 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sat, 29 Jan 2022 15:57:22 +0100 Subject: [PATCH 1/6] Refactor util function for unit conversion Signed-off-by: F.N. Claessen --- .../api/common/schemas/sensor_data.py | 16 ++++++-------- flexmeasures/utils/unit_utils.py | 22 ++++++++++++++++++- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/flexmeasures/api/common/schemas/sensor_data.py b/flexmeasures/api/common/schemas/sensor_data.py index 803590181..d82c3da65 100644 --- a/flexmeasures/api/common/schemas/sensor_data.py +++ b/flexmeasures/api/common/schemas/sensor_data.py @@ -14,7 +14,7 @@ from flexmeasures.api.common.utils.api_utils import upsample_values from flexmeasures.data.schemas.times import AwareDateTimeField, DurationField from flexmeasures.utils.unit_utils import ( - determine_unit_conversion_multiplier, + convert_units, units_are_convertible, ) @@ -139,14 +139,12 @@ def possibly_convert_units(self, data, **kwargs): Convert values if needed, to fit the sensor's unit. Marshmallow runs this after validation. """ - posted_unit = data["unit"] - required_unit = data["sensor"].unit - - if posted_unit != required_unit: - multiplier = determine_unit_conversion_multiplier( - posted_unit, required_unit, data["sensor"].event_resolution - ) - data["values"] = [multiplier * value for value in data["values"]] + data["values"] = convert_units( + data["values"], + from_unit=data["unit"], + to_unit=data["sensor"].unit, + event_resolution=data["sensor"].event_resolution, + ) return data @post_load() diff --git a/flexmeasures/utils/unit_utils.py b/flexmeasures/utils/unit_utils.py index f119c2211..27b61a9f6 100644 --- a/flexmeasures/utils/unit_utils.py +++ b/flexmeasures/utils/unit_utils.py @@ -1,8 +1,9 @@ from datetime import timedelta -from typing import Optional +from typing import List, Optional, Union from moneyed import list_all_currencies import importlib.resources as pkg_resources +import pandas as pd import pint # Edit constants template to stop using h to represent planck_constant @@ -151,3 +152,22 @@ def is_energy_unit(unit: str) -> bool: if not is_valid_unit(unit): return False return ur.Quantity(unit).dimensionality == ur.Quantity("Wh").dimensionality + + +def convert_units( + data: Union[pd.Series, List[Union[int, float]]], + from_unit: str, + to_unit: str, + event_resolution: Optional[timedelta], +) -> List[Union[int, float]]: + """Updates data values to reflect the given unit conversion.""" + + if from_unit != to_unit: + multiplier = determine_unit_conversion_multiplier( + from_unit, to_unit, event_resolution + ) + if isinstance(data, pd.Series): + data = multiplier * data + else: + data = [multiplier * value for value in data] + return data From 1eeba617d958f2a85e240074780520854e339a6f Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sat, 29 Jan 2022 15:58:12 +0100 Subject: [PATCH 2/6] Add CLI option to pass unit when reading in beliefs Signed-off-by: F.N. Claessen --- flexmeasures/cli/data_add.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index f072de7b1..33c4653d7 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -34,6 +34,7 @@ get_source_or_none, ) from flexmeasures.utils.time_utils import server_now +from flexmeasures.utils.unit_utils import convert_units @click.group("add") @@ -390,6 +391,14 @@ def add_initial_structure(): type=str, help="Source of the beliefs (an existing source id or name, or a new name).", ) +@click.option( + "--unit", + required=False, + type=str, + help="Unit of the data, for conversion to the sensor unit, if possible (a string unit such as 'kW' or 'm³/h').\n" + "Hint: to switch the sign of the data, prepend a minus sign.\n" + "For example, when assigning kW consumption data to a kW production sensor, use '-kW'.", +) @click.option( "--horizon", required=False, @@ -471,6 +480,7 @@ def add_beliefs( file: str, sensor_id: int, source: str, + unit: Optional[str] = None, horizon: Optional[int] = None, cp: Optional[float] = None, resample: bool = True, @@ -541,6 +551,13 @@ def add_beliefs( parse_dates=True, **kwargs, ) + if unit is not None: + bdf["event_value"] = convert_units( + bdf["event_value"], + from_unit=unit, + to_unit=sensor.unit, + event_resolution=sensor.event_resolution, + ) try: TimedBelief.add( bdf, From fab84bb05b0e97839ff1cce8c9d2884b69cf5409 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sat, 29 Jan 2022 16:00:29 +0100 Subject: [PATCH 3/6] Changelog entry Signed-off-by: F.N. Claessen --- documentation/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 4d8cf1a3e..0fac3032f 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -8,6 +8,7 @@ v0.9.0 | February XX, 2022 New features ----------- * Three new CLI commands for cleaning up your database: delete 1) unchanged beliefs, 2) NaN values or 3) a sensor and all of its time series data [see `PR #328 `_] +* Add CLI option to pass a data unit when reading in time series data from CSV, so data can automatically be converted to the sensor unit [see `PR #341 `_] Bugfixes ----------- From 089ad896fb6f1a7003708d0df3c7e67a0df760a2 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 31 Jan 2022 12:18:25 +0100 Subject: [PATCH 4/6] Fix return type annotation Signed-off-by: F.N. Claessen --- flexmeasures/utils/unit_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/utils/unit_utils.py b/flexmeasures/utils/unit_utils.py index 27b61a9f6..1d65462fb 100644 --- a/flexmeasures/utils/unit_utils.py +++ b/flexmeasures/utils/unit_utils.py @@ -159,7 +159,7 @@ def convert_units( from_unit: str, to_unit: str, event_resolution: Optional[timedelta], -) -> List[Union[int, float]]: +) -> Union[pd.Series, List[Union[int, float]]]: """Updates data values to reflect the given unit conversion.""" if from_unit != to_unit: From 397e8ac058c5e79b40759bbe9c38bc421c4616fa Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 31 Jan 2022 13:14:01 +0100 Subject: [PATCH 5/6] Add unit tests Signed-off-by: F.N. Claessen --- flexmeasures/utils/tests/test_unit_utils.py | 42 +++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/flexmeasures/utils/tests/test_unit_utils.py b/flexmeasures/utils/tests/test_unit_utils.py index ed940ccc4..d954fca93 100644 --- a/flexmeasures/utils/tests/test_unit_utils.py +++ b/flexmeasures/utils/tests/test_unit_utils.py @@ -1,7 +1,12 @@ from datetime import timedelta + +import pint.errors import pytest +import pandas as pd + from flexmeasures.utils.unit_utils import ( + convert_units, determine_flow_unit, determine_stock_unit, determine_unit_conversion_multiplier, @@ -12,6 +17,43 @@ ) +@pytest.mark.parametrize( + "from_unit, to_unit, expected_multiplier, expected_values", + [ + ("m/s", "km/h", 3.6, None), + ("m³/h", "l/h", 1000, None), + ("m³", "m³/h", 4, None), + ("MW", "kW", 1000, None), + ("kWh", "kW", 4, None), + ("°C", "K", None, [273.15, 283.15, 284.15]), + ], +) +def test_convert_unit( + from_unit, + to_unit, + expected_multiplier, + expected_values, +): + """Check some common unit conversions. + + Note that for the above expectations: + - conversion from kWh to kW, and from m³ to m³/h, both depend on the event resolution set below + - conversion from °C to K depends on the data values set below + """ + data = pd.Series([0, 10.0, 11.0]) + converted_data: pd.Series = convert_units( + data=data, + from_unit=from_unit, + to_unit=to_unit, + event_resolution=timedelta(minutes=15), + ) + if expected_multiplier is not None: + expected_data = data * expected_multiplier + else: + expected_data = pd.Series(expected_values) + pd.testing.assert_series_equal(converted_data, expected_data) + + @pytest.mark.parametrize( "unit, time_unit, expected_unit", [ From f34704c1887e1025c8b9f65394bb8e7f5abdb7da Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 31 Jan 2022 13:17:06 +0100 Subject: [PATCH 6/6] Fix non-multiplicative temperature conversion and partly vectorize unit conversion Signed-off-by: F.N. Claessen --- flexmeasures/utils/tests/test_unit_utils.py | 4 ++- flexmeasures/utils/unit_utils.py | 32 ++++++++++++++------- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/flexmeasures/utils/tests/test_unit_utils.py b/flexmeasures/utils/tests/test_unit_utils.py index d954fca93..140a1fec2 100644 --- a/flexmeasures/utils/tests/test_unit_utils.py +++ b/flexmeasures/utils/tests/test_unit_utils.py @@ -99,7 +99,9 @@ def test_determine_unit_conversion_multiplier(): assert determine_unit_conversion_multiplier("J/s", "W") == 1 assert determine_unit_conversion_multiplier("Wh", "W", timedelta(minutes=10)) == 6 assert determine_unit_conversion_multiplier("kWh", "MJ") == 3.6 - assert determine_unit_conversion_multiplier("°C", "K") == 274.15 + with pytest.raises(pint.errors.OffsetUnitCalculusError): + # Not a conversion that can be specified as a multiplication + determine_unit_conversion_multiplier("°C", "K") def test_h_denotes_hour_and_not_planck_constant(): diff --git a/flexmeasures/utils/unit_utils.py b/flexmeasures/utils/unit_utils.py index 1d65462fb..3aed31249 100644 --- a/flexmeasures/utils/unit_utils.py +++ b/flexmeasures/utils/unit_utils.py @@ -78,9 +78,7 @@ def determine_unit_conversion_multiplier( """Determine the value multiplier for a given unit conversion. If needed, requires a duration to convert from units of stock change to units of flow. """ - scalar = ( - ur.Quantity(from_unit).to_base_units() / ur.Quantity(to_unit).to_base_units() - ) + scalar = ur.Quantity(from_unit) / ur.Quantity(to_unit) if scalar.dimensionality == ur.Quantity("h").dimensionality: if duration is None: raise ValueError( @@ -163,11 +161,25 @@ def convert_units( """Updates data values to reflect the given unit conversion.""" if from_unit != to_unit: - multiplier = determine_unit_conversion_multiplier( - from_unit, to_unit, event_resolution - ) - if isinstance(data, pd.Series): - data = multiplier * data - else: - data = [multiplier * value for value in data] + try: + if isinstance(data, pd.Series): + data = pd.Series( + pint.Quantity(data.values, from_unit) + .to(pint.Quantity(to_unit)) + .magnitude, + index=data.index, + name=data.name, + ) + else: + data = list( + pint.Quantity(data, from_unit).to(pint.Quantity(to_unit)).magnitude + ) + except pint.errors.DimensionalityError: + multiplier = determine_unit_conversion_multiplier( + from_unit, to_unit, event_resolution + ) + if isinstance(data, pd.Series): + data = multiplier * data + else: + data = [multiplier * value for value in data] return data