diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 4d8cf1a3e..0fac3032f 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -8,6 +8,7 @@ v0.9.0 | February XX, 2022 New features ----------- * Three new CLI commands for cleaning up your database: delete 1) unchanged beliefs, 2) NaN values or 3) a sensor and all of its time series data [see `PR #328 `_] +* Add CLI option to pass a data unit when reading in time series data from CSV, so data can automatically be converted to the sensor unit [see `PR #341 `_] Bugfixes ----------- diff --git a/flexmeasures/api/common/schemas/sensor_data.py b/flexmeasures/api/common/schemas/sensor_data.py index 803590181..d82c3da65 100644 --- a/flexmeasures/api/common/schemas/sensor_data.py +++ b/flexmeasures/api/common/schemas/sensor_data.py @@ -14,7 +14,7 @@ from flexmeasures.api.common.utils.api_utils import upsample_values from flexmeasures.data.schemas.times import AwareDateTimeField, DurationField from flexmeasures.utils.unit_utils import ( - determine_unit_conversion_multiplier, + convert_units, units_are_convertible, ) @@ -139,14 +139,12 @@ def possibly_convert_units(self, data, **kwargs): Convert values if needed, to fit the sensor's unit. Marshmallow runs this after validation. """ - posted_unit = data["unit"] - required_unit = data["sensor"].unit - - if posted_unit != required_unit: - multiplier = determine_unit_conversion_multiplier( - posted_unit, required_unit, data["sensor"].event_resolution - ) - data["values"] = [multiplier * value for value in data["values"]] + data["values"] = convert_units( + data["values"], + from_unit=data["unit"], + to_unit=data["sensor"].unit, + event_resolution=data["sensor"].event_resolution, + ) return data @post_load() diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index f072de7b1..33c4653d7 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -34,6 +34,7 @@ get_source_or_none, ) from flexmeasures.utils.time_utils import server_now +from flexmeasures.utils.unit_utils import convert_units @click.group("add") @@ -390,6 +391,14 @@ def add_initial_structure(): type=str, help="Source of the beliefs (an existing source id or name, or a new name).", ) +@click.option( + "--unit", + required=False, + type=str, + help="Unit of the data, for conversion to the sensor unit, if possible (a string unit such as 'kW' or 'm³/h').\n" + "Hint: to switch the sign of the data, prepend a minus sign.\n" + "For example, when assigning kW consumption data to a kW production sensor, use '-kW'.", +) @click.option( "--horizon", required=False, @@ -471,6 +480,7 @@ def add_beliefs( file: str, sensor_id: int, source: str, + unit: Optional[str] = None, horizon: Optional[int] = None, cp: Optional[float] = None, resample: bool = True, @@ -541,6 +551,13 @@ def add_beliefs( parse_dates=True, **kwargs, ) + if unit is not None: + bdf["event_value"] = convert_units( + bdf["event_value"], + from_unit=unit, + to_unit=sensor.unit, + event_resolution=sensor.event_resolution, + ) try: TimedBelief.add( bdf, diff --git a/flexmeasures/utils/tests/test_unit_utils.py b/flexmeasures/utils/tests/test_unit_utils.py index ed940ccc4..140a1fec2 100644 --- a/flexmeasures/utils/tests/test_unit_utils.py +++ b/flexmeasures/utils/tests/test_unit_utils.py @@ -1,7 +1,12 @@ from datetime import timedelta + +import pint.errors import pytest +import pandas as pd + from flexmeasures.utils.unit_utils import ( + convert_units, determine_flow_unit, determine_stock_unit, determine_unit_conversion_multiplier, @@ -12,6 +17,43 @@ ) +@pytest.mark.parametrize( + "from_unit, to_unit, expected_multiplier, expected_values", + [ + ("m/s", "km/h", 3.6, None), + ("m³/h", "l/h", 1000, None), + ("m³", "m³/h", 4, None), + ("MW", "kW", 1000, None), + ("kWh", "kW", 4, None), + ("°C", "K", None, [273.15, 283.15, 284.15]), + ], +) +def test_convert_unit( + from_unit, + to_unit, + expected_multiplier, + expected_values, +): + """Check some common unit conversions. + + Note that for the above expectations: + - conversion from kWh to kW, and from m³ to m³/h, both depend on the event resolution set below + - conversion from °C to K depends on the data values set below + """ + data = pd.Series([0, 10.0, 11.0]) + converted_data: pd.Series = convert_units( + data=data, + from_unit=from_unit, + to_unit=to_unit, + event_resolution=timedelta(minutes=15), + ) + if expected_multiplier is not None: + expected_data = data * expected_multiplier + else: + expected_data = pd.Series(expected_values) + pd.testing.assert_series_equal(converted_data, expected_data) + + @pytest.mark.parametrize( "unit, time_unit, expected_unit", [ @@ -57,7 +99,9 @@ def test_determine_unit_conversion_multiplier(): assert determine_unit_conversion_multiplier("J/s", "W") == 1 assert determine_unit_conversion_multiplier("Wh", "W", timedelta(minutes=10)) == 6 assert determine_unit_conversion_multiplier("kWh", "MJ") == 3.6 - assert determine_unit_conversion_multiplier("°C", "K") == 274.15 + with pytest.raises(pint.errors.OffsetUnitCalculusError): + # Not a conversion that can be specified as a multiplication + determine_unit_conversion_multiplier("°C", "K") def test_h_denotes_hour_and_not_planck_constant(): diff --git a/flexmeasures/utils/unit_utils.py b/flexmeasures/utils/unit_utils.py index f119c2211..3aed31249 100644 --- a/flexmeasures/utils/unit_utils.py +++ b/flexmeasures/utils/unit_utils.py @@ -1,8 +1,9 @@ from datetime import timedelta -from typing import Optional +from typing import List, Optional, Union from moneyed import list_all_currencies import importlib.resources as pkg_resources +import pandas as pd import pint # Edit constants template to stop using h to represent planck_constant @@ -77,9 +78,7 @@ def determine_unit_conversion_multiplier( """Determine the value multiplier for a given unit conversion. If needed, requires a duration to convert from units of stock change to units of flow. """ - scalar = ( - ur.Quantity(from_unit).to_base_units() / ur.Quantity(to_unit).to_base_units() - ) + scalar = ur.Quantity(from_unit) / ur.Quantity(to_unit) if scalar.dimensionality == ur.Quantity("h").dimensionality: if duration is None: raise ValueError( @@ -151,3 +150,36 @@ def is_energy_unit(unit: str) -> bool: if not is_valid_unit(unit): return False return ur.Quantity(unit).dimensionality == ur.Quantity("Wh").dimensionality + + +def convert_units( + data: Union[pd.Series, List[Union[int, float]]], + from_unit: str, + to_unit: str, + event_resolution: Optional[timedelta], +) -> Union[pd.Series, List[Union[int, float]]]: + """Updates data values to reflect the given unit conversion.""" + + if from_unit != to_unit: + try: + if isinstance(data, pd.Series): + data = pd.Series( + pint.Quantity(data.values, from_unit) + .to(pint.Quantity(to_unit)) + .magnitude, + index=data.index, + name=data.name, + ) + else: + data = list( + pint.Quantity(data, from_unit).to(pint.Quantity(to_unit)).magnitude + ) + except pint.errors.DimensionalityError: + multiplier = determine_unit_conversion_multiplier( + from_unit, to_unit, event_resolution + ) + if isinstance(data, pd.Series): + data = multiplier * data + else: + data = [multiplier * value for value in data] + return data