Users may send null values in order to correctly 'space' their time s…

…eries data, but we then throw them out before saving. 548 allow sending null values when posting data (#549) * Allow POSTing null values Signed-off-by: F.N. Claessen <felix@seita.nl> * Update endpoint documentation Signed-off-by: F.N. Claessen <felix@seita.nl> * Extend schema tests Signed-off-by: F.N. Claessen <felix@seita.nl> * Extend endpoint tests Signed-off-by: F.N. Claessen <felix@seita.nl> * Prevent null values forming NaN values in the database: drop them instead Signed-off-by: F.N. Claessen <felix@seita.nl> * API changelog entry Signed-off-by: F.N. Claessen <felix@seita.nl> * Changelog entry Signed-off-by: F.N. Claessen <felix@seita.nl> * Upgrade timely-beliefs for enhanced test util Signed-off-by: F.N. Claessen <felix@seita.nl> * flake8 Signed-off-by: F.N. Claessen <felix@seita.nl> Signed-off-by: F.N. Claessen <felix@seita.nl>
FlexMeasures · Dec 9, 2022 · 8d8461d · 8d8461d
1 parent 3ea5152
commit 8d8461d
Show file tree

Hide file tree

Showing 9 changed files with 43 additions and 16 deletions.
diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst
@@ -5,9 +5,10 @@ API change log
 
 .. note:: The FlexMeasures API follows its own versioning scheme. This is also reflected in the URL, allowing developers to upgrade at their own pace.
 
-v3.0-4 | 2022-11-29
+v3.0-4 | 2022-12-08
 """""""""""""""""""
 
+- Allow posting ``null`` values to `/sensors/data` (POST) to correctly space time series that include missing values (the missing values are not stored).
 - Introduced the ``source`` field to `/sensors/data` (GET) to obtain data for a given source (ID).
 - Fixed the JSON wrapping of the return message for `/sensors/data` (GET).
 - Changed the Notation section:

diff --git a/documentation/changelog.rst b/documentation/changelog.rst
@@ -17,6 +17,7 @@ New features
 * The CLI command ``flexmeasures show beliefs`` supports showing beliefs data in a custom resolution and/or timezone, and also saving the shown beliefs data to a CSV file [see `PR #519 <http://www.github.com/FlexMeasures/flexmeasures/pull/519>`_]
 * Improved import of time series data from CSV file: 1) drop duplicate records with warning, 2) allow configuring which column contains explicit recording times for each data point (use case: import forecasts) [see `PR #501 <http://www.github.com/FlexMeasures/flexmeasures/pull/501>`_], 3) localize timezone naive data, 4) support reading in datetime and timedelta values, 5) remove rows with NaN values, and 6) filter by values in specific columns [see `PR #521 <http://www.github.com/FlexMeasures/flexmeasures/pull/521>`_]
 * Filter data by source in the API endpoint `/sensors/data` (GET) [see `PR #543 <http://www.github.com/FlexMeasures/flexmeasures/pull/543>`_]
+* Allow posting ``null`` values to `/sensors/data` (POST) to correctly space time series that include missing values (the missing values are not stored) [see `PR #549 <http://www.github.com/FlexMeasures/flexmeasures/pull/549>`_]
 
 Bugfixes
 -----------

diff --git a/flexmeasures/api/common/schemas/sensor_data.py b/flexmeasures/api/common/schemas/sensor_data.py
@@ -59,7 +59,7 @@ def select_schema_to_ensure_list_of_floats(
     This ensures that we are not requiring the same flexibility from users who are retrieving data.
     """
     if isinstance(values, list):
-        return fields.List(fields.Float)
+        return fields.List(fields.Float(allow_none=True))
     else:
         return SingleValueField()
 
@@ -356,4 +356,4 @@ def load_bdf(sensor_data: dict) -> BeliefsDataFrame:
             source=source,
             sensor=sensor_data["sensor"],
             **belief_timing,
-        )
+        ).dropna()
diff --git a/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py b/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py
@@ -57,6 +57,14 @@ def test_resolution_field_deserialization(
             [2.7],
             [2.7],
         ),
+        (
+            [1, None, 3],  # sending a None/null value as part of a list is allowed
+            [1, None, 3],
+        ),
+        (
+            [None],  # sending a None/null value as part of a list is allowed
+            [None],
+        ),
     ],
 )
 def test_value_field_deserialization(
@@ -103,6 +111,10 @@ def test_value_field_serialization(
             "3, 4",
             "Not a valid number",
         ),
+        (
+            None,
+            "may not be null",  # sending a single None/null value is not allowed
+        ),
     ],
 )
 def test_value_field_invalid(deserialization_input, error_msg):

diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py
@@ -138,12 +138,13 @@ def post_data(self, bdf: BeliefsDataFrame):
             }
 
         The above request posts four values for a duration of one hour, where the first
-        event start is at the given start time, and subsequent values start in 15 minute intervals throughout the one hour duration.
+        event start is at the given start time, and subsequent events start in 15 minute intervals throughout the one hour duration.
 
         The sensor is the one with ID=1.
         The unit has to be convertible to the sensor's unit.
         The resolution of the data has to match the sensor's required resolution, but
         FlexMeasures will attempt to upsample lower resolutions.
+        The list of values may include null values.
 
         :reqheader Authorization: The authentication token
         :reqheader Content-Type: application/json

diff --git a/flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py b/flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py
@@ -4,6 +4,7 @@
 
 import pytest
 from flask import url_for
+from timely_beliefs.tests.utils import equal_lists
 
 from flexmeasures import Sensor, Source, User
 from flexmeasures.api.tests.utils import get_auth_token
@@ -12,18 +13,20 @@
 
 
 @pytest.mark.parametrize(
-    "num_values, expected_num_values, unit, expected_value",
+    "num_values, expected_num_values, unit, include_a_null, expected_value",
     [
-        (6, 6, "m³/h", -11.28),
-        (6, 6, "m³", 6 * -11.28),  # 6 * 10-min intervals per hour
-        (6, 6, "l/h", -11.28 / 1000),  # 1 m³ = 1000 l
-        (3, 6, "m³/h", -11.28),  # upsample to 20-min intervals
+        (6, 6, "m³/h", False, -11.28),
+        (6, 5, "m³/h", True, -11.28),  # NaN value does not enter database
+        (6, 6, "m³", False, 6 * -11.28),  # 6 * 10-min intervals per hour
+        (6, 6, "l/h", False, -11.28 / 1000),  # 1 m³ = 1000 l
+        (3, 6, "m³/h", False, -11.28),  # upsample from 20-min intervals
         (
             1,
             6,
             "m³/h",
+            False,
             -11.28,
-        ),  # upsample to single value for 1-hour interval, sent as float rather than list of floats
+        ),  # upsample from single value for 1-hour interval, sent as float rather than list of floats
     ],
 )
 def test_post_sensor_data(
@@ -32,10 +35,11 @@ def test_post_sensor_data(
     num_values,
     expected_num_values,
     unit,
+    include_a_null,
     expected_value,
 ):
     post_data = make_sensor_data_request_for_gas_sensor(
-        num_values=num_values, unit=unit
+        num_values=num_values, unit=unit, include_a_null=include_a_null
     )
     sensor = Sensor.query.filter(Sensor.name == "some gas sensor").one_or_none()
     beliefs_before = TimedBelief.query.filter(TimedBelief.sensor_id == sensor.id).all()
@@ -54,7 +58,9 @@ def test_post_sensor_data(
     print(f"BELIEFS AFTER: {beliefs}")
     assert len(beliefs) == expected_num_values
     # check that values are scaled to the sensor unit correctly
-    assert pytest.approx(beliefs[0].event_value - expected_value) == 0
+    assert equal_lists(
+        [b.event_value for b in beliefs], [expected_value] * expected_num_values
+    )
 
 
 def test_get_sensor_data(

diff --git a/flexmeasures/api/v3_0/tests/utils.py b/flexmeasures/api/v3_0/tests/utils.py
@@ -2,16 +2,22 @@
 
 
 def make_sensor_data_request_for_gas_sensor(
-    num_values: int = 6, duration: str = "PT1H", unit: str = "m³"
+    num_values: int = 6,
+    duration: str = "PT1H",
+    unit: str = "m³",
+    include_a_null: bool = False,
 ) -> dict:
     """Creates request to post sensor data for a gas sensor.
     This particular gas sensor measures units of m³/h with a 10-minute resolution.
     """
     sensor = Sensor.query.filter(Sensor.name == "some gas sensor").one_or_none()
+    values = num_values * [-11.28]
+    if include_a_null:
+        values[0] = None
     message: dict = {
         "type": "PostSensorDataRequest",
         "sensor": f"ea1.2021-01.io.flexmeasures:fm1.{sensor.id}",
-        "values": num_values * [-11.28],
+        "values": values,
         "start": "2021-06-07T00:00:00+02:00",
         "duration": duration,
         "horizon": "PT0H",

diff --git a/requirements/app.in b/requirements/app.in
@@ -28,7 +28,7 @@ tldextract
 pyomo>=5.6
 tabulate
 timetomodel>=0.7.1
-timely-beliefs[forecast]>=1.14
+timely-beliefs[forecast]>=1.16
 python-dotenv
 # a backport, not needed in Python3.8
 importlib_metadata

diff --git a/requirements/app.txt b/requirements/app.txt
@@ -314,7 +314,7 @@ tabulate==0.8.10
     # via -r requirements/app.in
 threadpoolctl==3.1.0
     # via scikit-learn
-timely-beliefs[forecast]==1.14.0
+timely-beliefs[forecast]==1.16.0
     # via -r requirements/app.in
 timetomodel==0.7.1
     # via -r requirements/app.in