548 allow sending null values when posting data

documentation/api/change_log.rst

@@ -5,9 +5,10 @@ API change log
 
 .. note:: The FlexMeasures API follows its own versioning scheme. This is also reflected in the URL, allowing developers to upgrade at their own pace.
 
-v3.0-4 | 2022-11-29
+v3.0-4 | 2022-12-08
 """""""""""""""""""
 
+- Allow posting ``null`` values to `/sensors/data` (POST) to correctly space time series that include missing values (the missing values are not stored).
 - Introduced the ``source`` field to `/sensors/data` (GET) to obtain data for a given source (ID).
 - Fixed the JSON wrapping of the return message for `/sensors/data` (GET).
 - Changed the Notation section:

documentation/changelog.rst

@@ -17,6 +17,7 @@ New features
 * The CLI command ``flexmeasures show beliefs`` supports showing beliefs data in a custom resolution and/or timezone, and also saving the shown beliefs data to a CSV file [see `PR #519 <http://www.github.com/FlexMeasures/flexmeasures/pull/519>`_]
 * Improved import of time series data from CSV file: 1) drop duplicate records with warning, 2) allow configuring which column contains explicit recording times for each data point (use case: import forecasts) [see `PR #501 <http://www.github.com/FlexMeasures/flexmeasures/pull/501>`_], 3) localize timezone naive data, 4) support reading in datetime and timedelta values, 5) remove rows with NaN values, and 6) filter by values in specific columns [see `PR #521 <http://www.github.com/FlexMeasures/flexmeasures/pull/521>`_]
 * Filter data by source in the API endpoint `/sensors/data` (GET) [see `PR #543 <http://www.github.com/FlexMeasures/flexmeasures/pull/543>`_]
+* Allow posting ``null`` values to `/sensors/data` (POST) to correctly space time series that include missing values (the missing values are not stored) [see `PR #549 <http://www.github.com/FlexMeasures/flexmeasures/pull/549>`_]
 
 Bugfixes
 -----------

flexmeasures/api/common/schemas/sensor_data.py

@@ -59,7 +59,7 @@ def select_schema_to_ensure_list_of_floats(
     This ensures that we are not requiring the same flexibility from users who are retrieving data.
     """
     if isinstance(values, list):
-        return fields.List(fields.Float)
+        return fields.List(fields.Float(allow_none=True))
     else:
         return SingleValueField()
 
@@ -356,4 +356,4 @@ def load_bdf(sensor_data: dict) -> BeliefsDataFrame:
             source=source,
             sensor=sensor_data["sensor"],
             **belief_timing,
-        )
+        ).dropna()

flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py

@@ -57,6 +57,14 @@ def test_resolution_field_deserialization(
             [2.7],
             [2.7],
         ),
+        (
+            [1, None, 3],  # sending a None/null value as part of a list is allowed
+            [1, None, 3],
+        ),
+        (
+            [None],  # sending a None/null value as part of a list is allowed
+            [None],
+        ),
     ],
 )
 def test_value_field_deserialization(
@@ -103,6 +111,10 @@ def test_value_field_serialization(
             "3, 4",
             "Not a valid number",
         ),
+        (
+            None,
+            "may not be null",  # sending a single None/null value is not allowed
+        ),
     ],
 )
 def test_value_field_invalid(deserialization_input, error_msg):

flexmeasures/api/v3_0/sensors.py

@@ -138,12 +138,13 @@ def post_data(self, bdf: BeliefsDataFrame):
             }
 
         The above request posts four values for a duration of one hour, where the first
-        event start is at the given start time, and subsequent values start in 15 minute intervals throughout the one hour duration.
+        event start is at the given start time, and subsequent events start in 15 minute intervals throughout the one hour duration.
 
         The sensor is the one with ID=1.
         The unit has to be convertible to the sensor's unit.
         The resolution of the data has to match the sensor's required resolution, but
         FlexMeasures will attempt to upsample lower resolutions.
+        The list of values may include null values.
 
         :reqheader Authorization: The authentication token
         :reqheader Content-Type: application/json

flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py

@@ -4,6 +4,7 @@
 
 import pytest
 from flask import url_for
+from timely_beliefs.tests.utils import equal_lists
 
 from flexmeasures import Sensor, Source, User
 from flexmeasures.api.tests.utils import get_auth_token
@@ -12,18 +13,20 @@
 
 
 @pytest.mark.parametrize(
-    "num_values, expected_num_values, unit, expected_value",
+    "num_values, expected_num_values, unit, include_a_null, expected_value",
     [
-        (6, 6, "m³/h", -11.28),
-        (6, 6, "m³", 6 * -11.28),  # 6 * 10-min intervals per hour
-        (6, 6, "l/h", -11.28 / 1000),  # 1 m³ = 1000 l
-        (3, 6, "m³/h", -11.28),  # upsample to 20-min intervals
+        (6, 6, "m³/h", False, -11.28),
+        (6, 5, "m³/h", True, -11.28),  # NaN value does not enter database
+        (6, 6, "m³", False, 6 * -11.28),  # 6 * 10-min intervals per hour
+        (6, 6, "l/h", False, -11.28 / 1000),  # 1 m³ = 1000 l
+        (3, 6, "m³/h", False, -11.28),  # upsample from 20-min intervals
         (
             1,
             6,
             "m³/h",
+            False,
             -11.28,
-        ),  # upsample to single value for 1-hour interval, sent as float rather than list of floats
+        ),  # upsample from single value for 1-hour interval, sent as float rather than list of floats
     ],
 )
 def test_post_sensor_data(
@@ -32,10 +35,11 @@ def test_post_sensor_data(
     num_values,
     expected_num_values,
     unit,
+    include_a_null,
     expected_value,
 ):
     post_data = make_sensor_data_request_for_gas_sensor(
-        num_values=num_values, unit=unit
+        num_values=num_values, unit=unit, include_a_null=include_a_null
     )
     sensor = Sensor.query.filter(Sensor.name == "some gas sensor").one_or_none()
     beliefs_before = TimedBelief.query.filter(TimedBelief.sensor_id == sensor.id).all()
@@ -54,7 +58,9 @@ def test_post_sensor_data(
     print(f"BELIEFS AFTER: {beliefs}")
     assert len(beliefs) == expected_num_values
     # check that values are scaled to the sensor unit correctly
-    assert pytest.approx(beliefs[0].event_value - expected_value) == 0
+    assert equal_lists(
+        [b.event_value for b in beliefs], [expected_value] * expected_num_values
+    )
 
 
 def test_get_sensor_data(

flexmeasures/api/v3_0/tests/utils.py

@@ -2,16 +2,22 @@
 
 
 def make_sensor_data_request_for_gas_sensor(
-    num_values: int = 6, duration: str = "PT1H", unit: str = "m³"
+    num_values: int = 6,
+    duration: str = "PT1H",
+    unit: str = "m³",
+    include_a_null: bool = False,
 ) -> dict:
     """Creates request to post sensor data for a gas sensor.
     This particular gas sensor measures units of m³/h with a 10-minute resolution.
     """
     sensor = Sensor.query.filter(Sensor.name == "some gas sensor").one_or_none()
+    values = num_values * [-11.28]
+    if include_a_null:
+        values[0] = None
     message: dict = {
         "type": "PostSensorDataRequest",
         "sensor": f"ea1.2021-01.io.flexmeasures:fm1.{sensor.id}",
-        "values": num_values * [-11.28],
+        "values": values,
         "start": "2021-06-07T00:00:00+02:00",
         "duration": duration,
         "horizon": "PT0H",

requirements/app.in

@@ -28,7 +28,7 @@ tldextract
 pyomo>=5.6
 tabulate
 timetomodel>=0.7.1
-timely-beliefs[forecast]>=1.14
+timely-beliefs[forecast]>=1.16
 python-dotenv
 # a backport, not needed in Python3.8
 importlib_metadata

requirements/app.txt

@@ -314,7 +314,7 @@ tabulate==0.8.10
     # via -r requirements/app.in
 threadpoolctl==3.1.0
     # via scikit-learn
-timely-beliefs[forecast]==1.14.0
+timely-beliefs[forecast]==1.16.0
     # via -r requirements/app.in
 timetomodel==0.7.1
     # via -r requirements/app.in