Issue 282 use pint to check and convert units (#283)

Prepare for future user functionality to auto-convert data to desired units when POSTing and GETting data, and hook up unit conversion to our dev endpoint for posting sensor data.


* Use pint for unit_utils, using h to denote hour, adding the world's currencies, and simplifying units according to preference; also add tests

Signed-off-by: F.N. Claessen <felix@seita.nl>

* Add new dependencies to requirements

Signed-off-by: F.N. Claessen <felix@seita.nl>

* Fix docstring: cubic unit with unicode

Signed-off-by: F.N. Claessen <felix@seita.nl>

* Fix docstring examples

Signed-off-by: F.N. Claessen <felix@seita.nl>

* Convert units and data in SensorDataSchema

Signed-off-by: F.N. Claessen <felix@seita.nl>

* Fix case of simple multiplier

Signed-off-by: F.N. Claessen <felix@seita.nl>

* Test more unit conversions

Signed-off-by: F.N. Claessen <felix@seita.nl>

* Refactor unit utils

Signed-off-by: F.N. Claessen <felix@seita.nl>

* Fix converting offset units (such as degrees Celsius)

Signed-off-by: F.N. Claessen <felix@seita.nl>

* Extra testing of unit util functions, incl. conversion of offset units

Signed-off-by: F.N. Claessen <felix@seita.nl>

* Add missing docstring

Signed-off-by: F.N. Claessen <felix@seita.nl>

* Black

Signed-off-by: F.N. Claessen <felix@seita.nl>

* Add inline note

Signed-off-by: F.N. Claessen <felix@seita.nl>

* Clarify what test util function does

Signed-off-by: F.N. Claessen <felix@seita.nl>

* Add missing type annotation

Signed-off-by: F.N. Claessen <felix@seita.nl>

flexmeasures/api/common/schemas/sensor_data.py

@@ -13,6 +13,10 @@
 from flexmeasures.api.common.schemas.sensors import SensorField
 from flexmeasures.api.common.utils.api_utils import upsample_values
 from flexmeasures.data.schemas.times import AwareDateTimeField, DurationField
+from flexmeasures.utils.unit_utils import (
+    determine_unit_conversion_multiplier,
+    units_are_convertible,
+)
 
 
 class SingleValueField(fields.Float):
@@ -81,12 +85,21 @@ def check_user_rights_against_sensor(self, data, **kwargs):
 
     @validates_schema
     def check_schema_unit_against_sensor_unit(self, data, **kwargs):
-        # TODO: technically, there are compatible units, like kWh and kW.
-        #       They could be allowed here, and the SensorDataSchema could
-        #       even convert values to the sensor's unit if possible.
-        if data["unit"] != data["sensor"].unit:
+        """Allows units compatible with that of the sensor.
+        For example, a sensor with W units allows data to be posted with units:
+        - W, kW, MW, etc. (i.e. units with different prefixes)
+        - J/s, Nm/s, etc. (i.e. units that can be converted using some multiplier)
+        - Wh, kWh, etc. (i.e. units that represent a stock delta, which knowing the duration can be converted to a flow)
+        For compatible units, the SensorDataSchema converts values to the sensor's unit.
+        """
+        posted_unit = data["unit"]
+        required_unit = data["sensor"].unit
+
+        if posted_unit != required_unit and not units_are_convertible(
+            posted_unit, required_unit
+        ):
             raise ValidationError(
-                f"Required unit for this sensor is {data['sensor'].unit}, got: {data['unit']}"
+                f"Required unit for this sensor is {data['sensor'].unit}, got incompatible unit: {data['unit']}"
             )
 
 
@@ -120,6 +133,22 @@ def check_resolution_compatibility_of_values(self, data, **kwargs):
                 f"Resolution of {inferred_resolution} is incompatible with the sensor's required resolution of {required_resolution}."
             )
 
+    @post_load()
+    def possibly_convert_units(self, data, **kwargs):
+        """
+        Convert values if needed, to fit the sensor's unit.
+        Marshmallow runs this after validation.
+        """
+        posted_unit = data["unit"]
+        required_unit = data["sensor"].unit
+
+        if posted_unit != required_unit:
+            multiplier = determine_unit_conversion_multiplier(
+                posted_unit, required_unit, data["sensor"].event_resolution
+            )
+            data["values"] = [multiplier * value for value in data["values"]]
+        return data
+
     @post_load()
     def possibly_upsample_values(self, data, **kwargs):
         """

flexmeasures/api/dev/tests/test_sensor_data.py

@@ -2,7 +2,7 @@
 import pytest
 
 from flexmeasures.api.tests.utils import get_auth_token
-from flexmeasures.api.dev.tests.utils import make_sensor_data_request
+from flexmeasures.api.dev.tests.utils import make_sensor_data_request_for_gas_sensor
 
 
 @pytest.mark.parametrize("use_auth", [False, True])
@@ -48,7 +48,7 @@ def test_post_sensor_data_bad_auth(client, setup_api_test_data, use_auth):
 def test_post_invalid_sensor_data(
     client, setup_api_test_data, request_field, new_value, error_field, error_text
 ):
-    post_data = make_sensor_data_request()
+    post_data = make_sensor_data_request_for_gas_sensor()
     post_data[request_field] = new_value
     # this guy is allowed to post sensorData
     auth_token = get_auth_token(client, "test_prosumer_user@seita.nl", "testtest")
@@ -64,7 +64,7 @@ def test_post_invalid_sensor_data(
 
 def test_post_sensor_data_twice(client, setup_api_test_data):
     auth_token = get_auth_token(client, "test_prosumer_user@seita.nl", "testtest")
-    post_data = make_sensor_data_request()
+    post_data = make_sensor_data_request_for_gas_sensor()
     response = client.post(
         url_for("post_sensor_data"),
         json=post_data,

flexmeasures/api/dev/tests/test_sensor_data_fresh_db.py

@@ -3,22 +3,36 @@
 from flask import url_for
 
 from flexmeasures.api.tests.utils import get_auth_token
-from flexmeasures.api.dev.tests.utils import make_sensor_data_request
+from flexmeasures.api.dev.tests.utils import make_sensor_data_request_for_gas_sensor
 from flexmeasures.data.models.time_series import TimedBelief, Sensor
 
 
 @pytest.mark.parametrize(
-    "num_values, expected_num_values",
+    "num_values, expected_num_values, unit, expected_value",
     [
-        (6, 6),
-        (3, 6),  # upsample
-        (1, 6),  # upsample single value sent as float rather than list of floats
+        (6, 6, "m³/h", -11.28),
+        (6, 6, "m³", 6 * -11.28),  # 6 * 10-min intervals per hour
+        (6, 6, "l/h", -11.28 / 1000),  # 1 m³ = 1000 l
+        (3, 6, "m³/h", -11.28),  # upsample to 20-min intervals
+        (
+            1,
+            6,
+            "m³/h",
+            -11.28,
+        ),  # upsample to single value for 1-hour interval, sent as float rather than list of floats
     ],
 )
 def test_post_sensor_data(
-    client, setup_api_fresh_test_data, num_values, expected_num_values
+    client,
+    setup_api_fresh_test_data,
+    num_values,
+    expected_num_values,
+    unit,
+    expected_value,
 ):
-    post_data = make_sensor_data_request(num_values=num_values)
+    post_data = make_sensor_data_request_for_gas_sensor(
+        num_values=num_values, unit=unit
+    )
     sensor = Sensor.query.filter(Sensor.name == "some gas sensor").one_or_none()
     beliefs_before = TimedBelief.query.filter(TimedBelief.sensor_id == sensor.id).all()
     print(f"BELIEFS BEFORE: {beliefs_before}")
@@ -35,4 +49,5 @@ def test_post_sensor_data(
     beliefs = TimedBelief.query.filter(TimedBelief.sensor_id == sensor.id).all()
     print(f"BELIEFS AFTER: {beliefs}")
     assert len(beliefs) == expected_num_values
-    assert beliefs[0].event_value == -11.28
+    # check that values are scaled to the sensor unit correctly
+    assert pytest.approx(beliefs[0].event_value - expected_value) == 0

flexmeasures/api/dev/tests/utils.py

@@ -1,15 +1,20 @@
 from flexmeasures.data.models.time_series import Sensor
 
 
-def make_sensor_data_request(num_values: int = 6, duration: str = "PT1H") -> dict:
+def make_sensor_data_request_for_gas_sensor(
+    num_values: int = 6, duration: str = "PT1H", unit: str = "m³"
+) -> dict:
+    """Creates request to post sensor data for a gas sensor.
+    This particular gas sensor measures units of m³/h with a 10-minute resolution.
+    """
     sensor = Sensor.query.filter(Sensor.name == "some gas sensor").one_or_none()
     message: dict = {
         "type": "PostSensorDataRequest",
         "sensor": f"ea1.2021-01.io.flexmeasures:fm1.{sensor.id}",
         "values": num_values * [-11.28],
         "start": "2021-06-07T00:00:00+02:00",
         "duration": duration,
-        "unit": "m³/h",
+        "unit": unit,
     }
     if num_values == 1:
         # flatten [<float>] to <float>

flexmeasures/utils/tests/test_unit_utils.py

@@ -0,0 +1,112 @@
+from datetime import timedelta
+import pytest
+
+from flexmeasures.utils.unit_utils import (
+    determine_flow_unit,
+    determine_stock_unit,
+    determine_unit_conversion_multiplier,
+    units_are_convertible,
+    is_energy_unit,
+    is_power_unit,
+    u,
+)
+
+
+@pytest.mark.parametrize(
+    "unit, time_unit, expected_unit",
+    [
+        ("m³", None, "m³/h"),
+        ("kWh", None, "kW"),
+        ("km", "h", "km/h"),
+        ("m", "s", "km/h"),
+    ],
+)
+def test_determine_flow_unit(
+    unit,
+    time_unit,
+    expected_unit,
+):
+    if time_unit is None:
+        assert determine_flow_unit(unit) == expected_unit
+    else:
+        assert determine_flow_unit(unit, time_unit) == expected_unit
+
+
+@pytest.mark.parametrize(
+    "unit, time_unit, expected_unit",
+    [
+        ("m³/h", None, "m³"),
+        ("kW", None, "kWh"),
+        ("m/s", "s", "m"),
+        ("m/s", "h", "km"),
+    ],
+)
+def test_determine_stock_unit(
+    unit,
+    time_unit,
+    expected_unit,
+):
+    if time_unit is None:
+        assert determine_stock_unit(unit) == expected_unit
+    else:
+        assert determine_stock_unit(unit, time_unit) == expected_unit
+
+
+def test_determine_unit_conversion_multiplier():
+    assert determine_unit_conversion_multiplier("kW", "W") == 1000
+    assert determine_unit_conversion_multiplier("J/s", "W") == 1
+    assert determine_unit_conversion_multiplier("Wh", "W", timedelta(minutes=10)) == 6
+    assert determine_unit_conversion_multiplier("kWh", "MJ") == 3.6
+    assert determine_unit_conversion_multiplier("°C", "K") == 274.15
+
+
+def test_h_denotes_hour_and_not_planck_constant():
+    assert u.Quantity("h").dimensionality == u.Quantity("hour").dimensionality
+    assert (
+        u.Quantity("hbar").dimensionality
+        == u.Quantity("planck_constant").dimensionality
+    )
+
+
+def test_units_are_convertible():
+    assert units_are_convertible("kW", "W")  # units just have different prefixes
+    assert units_are_convertible(
+        "J/s", "W"
+    )  # units can be converted using some multiplier
+    assert units_are_convertible(
+        "Wh", "W"
+    )  # units that represent a stock delta can, knowing the duration, be converted to a flow
+    assert units_are_convertible("toe", "W")  # tonne of oil equivalent
+    assert units_are_convertible("°C", "K")  # offset unit to absolute unit
+    assert not units_are_convertible("°C", "W")
+    assert not units_are_convertible("EUR/MWh", "W")
+
+
+@pytest.mark.parametrize(
+    "unit, power_unit",
+    [
+        ("EUR/MWh", False),
+        ("KRW/kWh", False),
+        ("kWh", False),
+        ("kW", True),
+        ("watt", True),
+        ("°C", False),
+    ],
+)
+def test_is_power_unit(unit: str, power_unit: bool):
+    assert is_power_unit(unit) is power_unit
+
+
+@pytest.mark.parametrize(
+    "unit, energy_unit",
+    [
+        ("EUR/MWh", False),
+        ("KRW/kWh", False),
+        ("kWh", True),
+        ("kW", False),
+        ("watthour", True),
+        ("°C", False),
+    ],
+)
+def test_is_energy_unit(unit: str, energy_unit: bool):
+    assert is_energy_unit(unit) is energy_unit