Cli unit conversion for adding data (#341)

Add a CLI option to pass a data unit when reading in time series data from CSV, so data can automatically be converted to the sensor unit. Also allows to swap signs.


* Refactor util function for unit conversion

Signed-off-by: F.N. Claessen <felix@seita.nl>

* Add CLI option to pass unit when reading in beliefs

Signed-off-by: F.N. Claessen <felix@seita.nl>

* Changelog entry

Signed-off-by: F.N. Claessen <felix@seita.nl>

* Fix return type annotation

Signed-off-by: F.N. Claessen <felix@seita.nl>

* Add unit tests

Signed-off-by: F.N. Claessen <felix@seita.nl>

* Fix non-multiplicative temperature conversion and partly vectorize unit conversion

Signed-off-by: F.N. Claessen <felix@seita.nl>

documentation/changelog.rst

@@ -8,6 +8,7 @@ v0.9.0 | February XX, 2022
 New features
 -----------
 * Three new CLI commands for cleaning up your database: delete 1) unchanged beliefs, 2) NaN values or 3) a sensor and all of its time series data [see `PR #328 <http://www.github.com/FlexMeasures/flexmeasures/pull/328>`_]
+* Add CLI option to pass a data unit when reading in time series data from CSV, so data can automatically be converted to the sensor unit [see `PR #341 <http://www.github.com/FlexMeasures/flexmeasures/pull/341>`_]
 
 * Add CLI-commands ``flexmeasures add sensor``, ``flexmeasures add asset-type``, ``flexmeasures add beliefs`` (which were experimental features before). [see `PR #337 <http://www.github.com/FlexMeasures/flexmeasures/pull/337>`_]
 

flexmeasures/api/common/schemas/sensor_data.py

@@ -14,7 +14,7 @@
 from flexmeasures.api.common.utils.api_utils import upsample_values
 from flexmeasures.data.schemas.times import AwareDateTimeField, DurationField
 from flexmeasures.utils.unit_utils import (
-    determine_unit_conversion_multiplier,
+    convert_units,
     units_are_convertible,
 )
 
@@ -139,14 +139,12 @@ def possibly_convert_units(self, data, **kwargs):
         Convert values if needed, to fit the sensor's unit.
         Marshmallow runs this after validation.
         """
-        posted_unit = data["unit"]
-        required_unit = data["sensor"].unit
-
-        if posted_unit != required_unit:
-            multiplier = determine_unit_conversion_multiplier(
-                posted_unit, required_unit, data["sensor"].event_resolution
-            )
-            data["values"] = [multiplier * value for value in data["values"]]
+        data["values"] = convert_units(
+            data["values"],
+            from_unit=data["unit"],
+            to_unit=data["sensor"].unit,
+            event_resolution=data["sensor"].event_resolution,
+        )
         return data
 
     @post_load()

flexmeasures/cli/data_add.py

@@ -31,6 +31,7 @@
     get_source_or_none,
 )
 from flexmeasures.utils.time_utils import server_now
+from flexmeasures.utils.unit_utils import convert_units
 
 
 @click.group("add")
@@ -313,6 +314,14 @@ def add_initial_structure():
     type=str,
     help="Source of the beliefs (an existing source id or name, or a new name).",
 )
+@click.option(
+    "--unit",
+    required=False,
+    type=str,
+    help="Unit of the data, for conversion to the sensor unit, if possible (a string unit such as 'kW' or 'm³/h').\n"
+    "Hint: to switch the sign of the data, prepend a minus sign.\n"
+    "For example, when assigning kW consumption data to a kW production sensor, use '-kW'.",
+)
 @click.option(
     "--horizon",
     required=False,
@@ -394,6 +403,7 @@ def add_beliefs(
     file: str,
     sensor_id: int,
     source: str,
+    unit: Optional[str] = None,
     horizon: Optional[int] = None,
     cp: Optional[float] = None,
     resample: bool = True,
@@ -464,6 +474,13 @@ def add_beliefs(
         parse_dates=True,
         **kwargs,
     )
+    if unit is not None:
+        bdf["event_value"] = convert_units(
+            bdf["event_value"],
+            from_unit=unit,
+            to_unit=sensor.unit,
+            event_resolution=sensor.event_resolution,
+        )
     try:
         TimedBelief.add(
             bdf,

flexmeasures/utils/tests/test_unit_utils.py

@@ -1,7 +1,12 @@
 from datetime import timedelta
+
+import pint.errors
 import pytest
 
+import pandas as pd
+
 from flexmeasures.utils.unit_utils import (
+    convert_units,
     determine_flow_unit,
     determine_stock_unit,
     determine_unit_conversion_multiplier,
@@ -12,6 +17,43 @@
 )
 
 
+@pytest.mark.parametrize(
+    "from_unit, to_unit, expected_multiplier, expected_values",
+    [
+        ("m/s", "km/h", 3.6, None),
+        ("m³/h", "l/h", 1000, None),
+        ("m³", "m³/h", 4, None),
+        ("MW", "kW", 1000, None),
+        ("kWh", "kW", 4, None),
+        ("°C", "K", None, [273.15, 283.15, 284.15]),
+    ],
+)
+def test_convert_unit(
+    from_unit,
+    to_unit,
+    expected_multiplier,
+    expected_values,
+):
+    """Check some common unit conversions.
+
+    Note that for the above expectations:
+    - conversion from kWh to kW, and from m³ to m³/h, both depend on the event resolution set below
+    - conversion from °C to K depends on the data values set below
+    """
+    data = pd.Series([0, 10.0, 11.0])
+    converted_data: pd.Series = convert_units(
+        data=data,
+        from_unit=from_unit,
+        to_unit=to_unit,
+        event_resolution=timedelta(minutes=15),
+    )
+    if expected_multiplier is not None:
+        expected_data = data * expected_multiplier
+    else:
+        expected_data = pd.Series(expected_values)
+    pd.testing.assert_series_equal(converted_data, expected_data)
+
+
 @pytest.mark.parametrize(
     "unit, time_unit, expected_unit",
     [
@@ -57,7 +99,9 @@ def test_determine_unit_conversion_multiplier():
     assert determine_unit_conversion_multiplier("J/s", "W") == 1
     assert determine_unit_conversion_multiplier("Wh", "W", timedelta(minutes=10)) == 6
     assert determine_unit_conversion_multiplier("kWh", "MJ") == 3.6
-    assert determine_unit_conversion_multiplier("°C", "K") == 274.15
+    with pytest.raises(pint.errors.OffsetUnitCalculusError):
+        # Not a conversion that can be specified as a multiplication
+        determine_unit_conversion_multiplier("°C", "K")
 
 
 def test_h_denotes_hour_and_not_planck_constant():

flexmeasures/utils/unit_utils.py

@@ -1,8 +1,9 @@
 from datetime import timedelta
-from typing import Optional
+from typing import List, Optional, Union
 
 from moneyed import list_all_currencies
 import importlib.resources as pkg_resources
+import pandas as pd
 import pint
 
 # Edit constants template to stop using h to represent planck_constant
@@ -77,9 +78,7 @@ def determine_unit_conversion_multiplier(
     """Determine the value multiplier for a given unit conversion.
     If needed, requires a duration to convert from units of stock change to units of flow.
     """
-    scalar = (
-        ur.Quantity(from_unit).to_base_units() / ur.Quantity(to_unit).to_base_units()
-    )
+    scalar = ur.Quantity(from_unit) / ur.Quantity(to_unit)
     if scalar.dimensionality == ur.Quantity("h").dimensionality:
         if duration is None:
             raise ValueError(
@@ -151,3 +150,36 @@ def is_energy_unit(unit: str) -> bool:
     if not is_valid_unit(unit):
         return False
     return ur.Quantity(unit).dimensionality == ur.Quantity("Wh").dimensionality
+
+
+def convert_units(
+    data: Union[pd.Series, List[Union[int, float]]],
+    from_unit: str,
+    to_unit: str,
+    event_resolution: Optional[timedelta],
+) -> Union[pd.Series, List[Union[int, float]]]:
+    """Updates data values to reflect the given unit conversion."""
+
+    if from_unit != to_unit:
+        try:
+            if isinstance(data, pd.Series):
+                data = pd.Series(
+                    pint.Quantity(data.values, from_unit)
+                    .to(pint.Quantity(to_unit))
+                    .magnitude,
+                    index=data.index,
+                    name=data.name,
+                )
+            else:
+                data = list(
+                    pint.Quantity(data, from_unit).to(pint.Quantity(to_unit)).magnitude
+                )
+        except pint.errors.DimensionalityError:
+            multiplier = determine_unit_conversion_multiplier(
+                from_unit, to_unit, event_resolution
+            )
+            if isinstance(data, pd.Series):
+                data = multiplier * data
+            else:
+                data = [multiplier * value for value in data]
+    return data