Polyfield event values (#156)

Improve user experience for our upcoming postSensorData API endpoint by not needing to make a list out of a single item. * Create draft PR for #145 * endpoint receives data and makes a BeliefsDataFrame * save beliefs, first test running * validation and upsampling in schema * changelog entry * test with upsampling, fix test data freshness * more tests * get test with non-existant sensor to work * test sending in the wrong unit * move unit check to SensorDataDescriptionSchema and comment on future work * move BeliefsDataFrame creation to Schema * clarity about type field, simpler date_range creation * some endpoint documentation * add some tests as Felix requested * save beliefs with api_utils.save_to_db, test sending data multiple times * add optional horizon field * Reverse if-else statement * Add placeholder for ownership check * Allow posting a single event value as a float rather than as a list of floats * Add dependency * PR fixes * Add some form of version management to dependencies installed by our Makefile Co-authored-by: nhoening <nhoening@users.noreply.github.com> Co-authored-by: Nicolas Höning <nicolas@seita.nl> Co-authored-by: Nicolas Höning <iam@nicolashoening.de>
FlexMeasures · Aug 25, 2021 · 43dbe80 · 43dbe80
1 parent 3f709df
commit 43dbe80
Show file tree

Hide file tree

Showing 9 changed files with 188 additions and 48 deletions.
diff --git a/Makefile b/Makefile
@@ -15,13 +15,13 @@ test:
 # ---- Documentation ---
 
 update-docs:
-	pip3 install sphinx>=4.0.3 sphinxcontrib.httpdomain sphinx-rtd-theme
+	make install-sphinx-tools
 	cd documentation; make clean; make html; cd ..
 
 update-docs-pdf:
 	@echo "NOTE: PDF documentation requires packages (on Debian: latexmk texlive-latex-recommended texlive-latex-extra texlive-fonts-recommended)"
 	@echo "NOTE: Currently, the docs require some pictures which are not in the git repo atm. Ask the devs."
-	pip3 install sphinx sphinxcontrib.httpdomain sphinx-rtd-theme
+	make install-sphinx-tools
 
 	cd documentation; make clean; make latexpdf; make latexpdf; cd ..  # make latexpdf can require two passes
 
@@ -30,27 +30,32 @@ update-docs-pdf:
 install: install-deps install-flexmeasures
 
 install-for-dev:
-	pip3 install -q pip-tools
 	make freeze-deps
 	pip-sync requirements/app.txt requirements/dev.txt requirements/test.txt
 	make install-flexmeasures
 
 install-deps:
-	pip3 install -q pip-tools
+	make install-pip-tools
 	make freeze-deps
 	pip-sync requirements/app.txt
 
 install-flexmeasures:
 	python setup.py develop
 
+install-pip-tools:
+	pip3 install -q pip-tools>=6.2
+
+install-sphinx-tools:
+	pip3 install sphinx>=4.0.3 sphinxcontrib.httpdomain sphinx-rtd-theme
+
 freeze-deps:
-	pip3 install -q pip-tools
+	make install-pip-tools
 	pip-compile -o requirements/app.txt requirements/app.in
 	pip-compile -o requirements/dev.txt requirements/dev.in
 	pip-compile -o requirements/test.txt requirements/test.in
 
 upgrade-deps:
-	pip3 install -q pip-tools
+	make install-pip-tools
 	pip-compile --upgrade -o requirements/app.txt requirements/app.in
 	pip-compile --upgrade -o requirements/test.txt requirements/test.in
 	pip-compile --upgrade -o requirements/dev.txt requirements/dev.in

diff --git a/flexmeasures/api/common/schemas/sensor_data.py b/flexmeasures/api/common/schemas/sensor_data.py
@@ -1,8 +1,10 @@
 from datetime import timedelta
+from typing import List, Union
 
 from flask_login import current_user
 from marshmallow import fields, post_load, validates_schema, ValidationError
 from marshmallow.validate import Equal, OneOf
+from marshmallow_polyfield import PolyField
 from timely_beliefs import BeliefsDataFrame
 import pandas as pd
 
@@ -13,6 +15,44 @@
 from flexmeasures.data.schemas.times import AwareDateTimeField, DurationField
 
 
+class SingleValueField(fields.Float):
+    """Field that both deserializes and serializes a single value to a list of floats (length 1)."""
+
+    def _deserialize(self, value, attr, obj, **kwargs) -> List[float]:
+        return [self._validated(value)]
+
+    def _serialize(self, value, attr, data, **kwargs) -> List[float]:
+        return [self._validated(value)]
+
+
+def select_schema_to_ensure_list_of_floats(
+    values: Union[List[float], float], _
+) -> Union[fields.List, SingleValueField]:
+    """Allows both a single float and a list of floats. Always returns a list of floats.
+
+    Meant to improve user experience by not needing to make a list out of a single item, such that:
+
+        {
+            "values": [3.7]
+        }
+
+    can be written as:
+
+        {
+            "values": 3.7
+        }
+
+    Either will be deserialized to [3.7].
+
+    Note that serialization always results in a list of floats.
+    This ensures that we are not requiring the same flexibility from users who are retrieving data.
+    """
+    if isinstance(values, list):
+        return fields.List(fields.Float)
+    else:
+        return SingleValueField()
+
+
 class SensorDataDescriptionSchema(ma.Schema):
     """
     Describing sensor data (i.e. in a GET request).
@@ -62,7 +102,11 @@ class SensorDataSchema(SensorDataDescriptionSchema):
     type = fields.Str(
         validate=OneOf(["PostSensorDataRequest", "GetSensorDataResponse"])
     )
-    values = fields.List(fields.Float())
+    values = PolyField(
+        deserialization_schema_selector=select_schema_to_ensure_list_of_floats,
+        serialization_schema_selector=select_schema_to_ensure_list_of_floats,
+        many=False,
+    )
 
     @validates_schema
     def check_resolution_compatibility_of_values(self, data, **kwargs):

diff --git a/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py b/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py
@@ -0,0 +1,82 @@
+import pytest
+
+from marshmallow import ValidationError
+
+from flexmeasures.api.common.schemas.sensor_data import (
+    SingleValueField,
+    SensorDataSchema,
+)
+
+
+@pytest.mark.parametrize(
+    "deserialization_input, exp_deserialization_output",
+    [
+        (
+            1,
+            [1],
+        ),
+        (
+            2.7,
+            [2.7],
+        ),
+        (
+            [1],
+            [1],
+        ),
+        (
+            [2.7],
+            [2.7],
+        ),
+    ],
+)
+def test_value_field_deserialization(
+    deserialization_input,
+    exp_deserialization_output,
+):
+    """Testing straightforward cases"""
+    vf = SensorDataSchema.values
+    deser = vf.deserialize(deserialization_input)
+    assert deser == exp_deserialization_output
+
+
+@pytest.mark.parametrize(
+    "serialization_input, exp_serialization_output",
+    [
+        (
+            1,
+            [1],
+        ),
+        (
+            2.7,
+            [2.7],
+        ),
+    ],
+)
+def test_value_field_serialization(
+    serialization_input,
+    exp_serialization_output,
+):
+    """Testing straightforward cases"""
+    vf = SingleValueField()
+    ser = vf.serialize("values", {"values": serialization_input})
+    assert ser == exp_serialization_output
+
+
+@pytest.mark.parametrize(
+    "deserialization_input, error_msg",
+    [
+        (
+            ["three", 4],
+            "Not a valid number",
+        ),
+        (
+            "3, 4",
+            "Not a valid number",
+        ),
+    ],
+)
+def test_value_field_invalid(deserialization_input, error_msg):
+    sf = SingleValueField()
+    with pytest.raises(ValidationError) as ve:
+        sf.deserialize(deserialization_input)
+    assert error_msg in str(ve)
diff --git a/flexmeasures/api/dev/tests/test_sensor_data_fresh_db.py b/flexmeasures/api/dev/tests/test_sensor_data_fresh_db.py
@@ -12,6 +12,7 @@
     [
         (6, 6),
         (3, 6),  # upsample
+        (1, 6),  # upsample single value sent as float rather than list of floats
     ],
 )
 def test_post_sensor_data(

diff --git a/flexmeasures/api/dev/tests/utils.py b/flexmeasures/api/dev/tests/utils.py
@@ -3,11 +3,15 @@
 
 def make_sensor_data_request(num_values: int = 6, duration: str = "PT1H") -> dict:
     sensor = Sensor.query.filter(Sensor.name == "some gas sensor").one_or_none()
-    return {
+    message: dict = {
         "type": "PostSensorDataRequest",
         "sensor": f"ea1.2021-01.io.flexmeasures:fm1.{sensor.id}",
         "values": num_values * [-11.28],
         "start": "2021-06-07T00:00:00+02:00",
         "duration": duration,
         "unit": "m³/h",
     }
+    if num_values == 1:
+        # flatten [<float>] to <float>
+        message["values"] = message["values"][0]
+    return message
diff --git a/requirements/app.in b/requirements/app.in
@@ -49,6 +49,7 @@ Flask-Classful
 Flask-Marshmallow
 Flask-Cors
 sentry-sdk[flask]
+marshmallow-polyfield
 marshmallow-sqlalchemy>=0.23.1
 webargs
 # flask should be after all the flask plugins, because setup might find they ARE flask

diff --git a/requirements/app.txt b/requirements/app.txt
@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile
+# This file is autogenerated by pip-compile with python 3.9
 # To update, run:
 #
 #    pip-compile --output-file=requirements/app.txt requirements/app.in
@@ -57,6 +57,23 @@ entrypoints==0.3
     # via altair
 filelock==3.0.12
     # via tldextract
+flask==2.0.1
+    # via
+    #   -r requirements/app.in
+    #   flask-classful
+    #   flask-cors
+    #   flask-json
+    #   flask-login
+    #   flask-mail
+    #   flask-marshmallow
+    #   flask-migrate
+    #   flask-principal
+    #   flask-security-too
+    #   flask-sqlalchemy
+    #   flask-sslify
+    #   flask-wtf
+    #   rq-dashboard
+    #   sentry-sdk
 flask-classful==0.14.2
     # via -r requirements/app.in
 flask-cors==3.0.10
@@ -87,23 +104,6 @@ flask-wtf==0.15.1
     # via
     #   -r requirements/app.in
     #   flask-security-too
-flask==2.0.1
-    # via
-    #   -r requirements/app.in
-    #   flask-classful
-    #   flask-cors
-    #   flask-json
-    #   flask-login
-    #   flask-mail
-    #   flask-marshmallow
-    #   flask-migrate
-    #   flask-principal
-    #   flask-security-too
-    #   flask-sqlalchemy
-    #   flask-sslify
-    #   flask-wtf
-    #   rq-dashboard
-    #   sentry-sdk
 greenlet==1.1.0
     # via sqlalchemy
 humanize==3.9.0
@@ -148,13 +148,16 @@ markupsafe==2.0.1
     #   jinja2
     #   mako
     #   wtforms
-marshmallow-sqlalchemy==0.26.1
-    # via -r requirements/app.in
 marshmallow==3.12.1
     # via
     #   flask-marshmallow
+    #   marshmallow-polyfield
     #   marshmallow-sqlalchemy
     #   webargs
+marshmallow-polyfield==5.10
+    # via -r requirements/app.in
+marshmallow-sqlalchemy==0.26.1
+    # via -r requirements/app.in
 matplotlib==3.4.2
     # via timetomodel
 netcdf4==1.5.6
@@ -185,8 +188,6 @@ openturns==1.16
     # via timely-beliefs
 packaging==20.9
     # via bokeh
-pandas-bokeh==0.4.3
-    # via -r requirements/app.in
 pandas==1.2.4
     # via
     #   -r requirements/app.in
@@ -197,6 +198,8 @@ pandas==1.2.4
     #   statsmodels
     #   timely-beliefs
     #   timetomodel
+pandas-bokeh==0.4.3
+    # via -r requirements/app.in
 passlib==1.7.4
     # via flask-security-too
 patsy==0.5.1
@@ -255,20 +258,20 @@ redis==3.5.3
     # via
     #   rq
     #   rq-dashboard
-requests-file==1.5.1
-    # via tldextract
 requests==2.25.1
     # via
     #   pvlib
     #   requests-file
     #   siphon
     #   tldextract
-rq-dashboard==0.6.1
-    # via -r requirements/app.in
+requests-file==1.5.1
+    # via tldextract
 rq==1.8.1
     # via
     #   -r requirements/app.in
     #   rq-dashboard
+rq-dashboard==0.6.1
+    # via -r requirements/app.in
 scikit-learn==0.24.2
     # via sklearn
 scipy==1.7.0

diff --git a/requirements/dev.txt b/requirements/dev.txt
@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile
+# This file is autogenerated by pip-compile with python 3.9
 # To update, run:
 #
 #    pip-compile --output-file=requirements/dev.txt requirements/dev.in
@@ -23,20 +23,20 @@ filelock==3.0.12
     # via
     #   -c requirements/app.txt
     #   virtualenv
-flake8-blind-except==0.2.0
-    # via -r requirements/dev.in
 flake8==3.9.2
     # via -r requirements/dev.in
+flake8-blind-except==0.2.0
+    # via -r requirements/dev.in
 identify==2.2.10
     # via pre-commit
 mccabe==0.6.1
     # via flake8
+mypy==0.902
+    # via -r requirements/dev.in
 mypy-extensions==0.4.3
     # via
     #   black
     #   mypy
-mypy==0.902
-    # via -r requirements/dev.in
 nodeenv==1.6.0
     # via pre-commit
 pathspec==0.8.1