Sensor view (#99)

Add a new UI view for individual Sensors, initially showing Altair graphs of sensor data but meant to be expanded with crud capabilities. The ability to graph sensor data is implemented as a new Sensor method, which is accessible via the API. Chart specs are part of the data package (#104).

API functions relating to Sensors are not part of the official documentation until the new data model is properly integrated (https://github.com/SeitaBV/flexmeasures/projects/3).


* Add CLI command to add beliefs from csv

* Refactor if-else block

* Add API functions to chart sensor data

* Move chart specs to dedicated module

* Docstring correction

* Fix data loading

* Rename variables in sync with version 1.4.1 of timely-beliefs, and expand docstrings

* Clarify what belief_charts_mapping does

* Make mapping chart types to specs a function

* Adjust query parameters to renamed tb variables

* Separate endpoint for loading chart data and clean up boolean chart options

* Upgrade tb dependency

* Dedicated endpoint for get_chart

* Split SensorView into UI and API parts

* Remove the need for jquery in sensors.html

* Add FlexMeasures menu (integrate page) and move js util functions

* Better Altair js dependency management

* Merge class methods (chart_data into search_beliefs)

* Missing indentation

* Move SensorAPI to API package under dev version

* Update API path in frontend

* flexmeasures.js should be module when exporting functions

* Factor out as_html

* Move chart specs from the UI package into the data package (#104)

* Move code to data package

Co-authored-by: F.N. Claessen <felix@seita.nl>

* Fix regression

Co-authored-by: Nicolas Höning <iam@nicolashoening.de>

Co-authored-by: F.N. Claessen <felix@seita.nl>
Co-authored-by: Nicolas Höning <iam@nicolashoening.de>

flexmeasures/api/__init__.py

@@ -105,9 +105,11 @@ def register_at(app: Flask):
     from flexmeasures.api.v1_2 import register_at as v1_2_register_at
     from flexmeasures.api.v1_3 import register_at as v1_3_register_at
     from flexmeasures.api.v2_0 import register_at as v2_0_register_at
+    from flexmeasures.api.dev import register_at as dev_register_at
 
     v1_register_at(app)
     v1_1_register_at(app)
     v1_2_register_at(app)
     v1_3_register_at(app)
     v2_0_register_at(app)
+    dev_register_at(app)

flexmeasures/api/common/schemas/times.py

@@ -62,3 +62,16 @@ def ground_from(
             )
             return (pd.Timestamp(start) + offset).to_pydatetime() - start
         return duration
+
+
+class AwareDateTimeField(fields.AwareDateTime):
+    """Field that deserializes to a timezone aware datetime
+    and serializes back to a string."""
+
+    def _deserialize(self, value: str, attr, obj, **kwargs) -> datetime:
+        """
+        Work-around until this PR lands:
+        https://github.com/marshmallow-code/marshmallow/pull/1787
+        """
+        value = value.replace(" ", "+")
+        return fields.AwareDateTime._deserialize(self, value, attr, obj, **kwargs)

flexmeasures/api/dev/__init__.py

@@ -0,0 +1,9 @@
+from flask import Flask
+
+
+def register_at(app: Flask):
+    """This can be used to register FlaskViews."""
+
+    from flexmeasures.api.dev.sensors import SensorAPI
+
+    SensorAPI.register(app, route_prefix="/api/dev")

flexmeasures/api/dev/sensors.py

@@ -0,0 +1,74 @@
+import json
+
+from flask_classful import FlaskView, route
+from flask_login import login_required
+from flask_security import roles_required
+from marshmallow import fields
+from webargs.flaskparser import use_kwargs
+from werkzeug.exceptions import abort
+
+from flexmeasures.api.common.schemas.times import AwareDateTimeField
+from flexmeasures.data.models.time_series import Sensor
+
+
+class SensorAPI(FlaskView):
+    """
+    This view exposes sensor attributes through API endpoints under development.
+    These endpoints are not yet part of our official API, but support the FlexMeasures UI.
+    """
+
+    route_base = "/sensor"
+
+    @login_required
+    @roles_required("admin")  # todo: remove after we check for sensor ownership
+    @route("/<id>/chart/")
+    @use_kwargs(
+        {
+            "event_starts_after": AwareDateTimeField(format="iso", required=False),
+            "event_ends_before": AwareDateTimeField(format="iso", required=False),
+            "beliefs_after": AwareDateTimeField(format="iso", required=False),
+            "beliefs_before": AwareDateTimeField(format="iso", required=False),
+            "include_data": fields.Boolean(required=False),
+            "dataset_name": fields.Str(required=False),
+        },
+        location="query",
+    )
+    def get_chart(self, id, **kwargs):
+        """GET from /sensor/<id>/chart"""
+        sensor = get_sensor_or_abort(id)
+        return json.dumps(sensor.chart(**kwargs))
+
+    @login_required
+    @roles_required("admin")  # todo: remove after we check for sensor ownership
+    @route("/<id>/chart_data/")
+    @use_kwargs(
+        {
+            "event_starts_after": AwareDateTimeField(format="iso", required=False),
+            "event_ends_before": AwareDateTimeField(format="iso", required=False),
+            "beliefs_after": AwareDateTimeField(format="iso", required=False),
+            "beliefs_before": AwareDateTimeField(format="iso", required=False),
+        },
+        location="query",
+    )
+    def get_chart_data(self, id, **kwargs):
+        """GET from /sensor/<id>/chart_data
+
+        Data for use in charts (in case you have the chart specs already).
+        """
+        sensor = get_sensor_or_abort(id)
+        return sensor.search_beliefs(as_json=True, **kwargs)
+
+    @login_required
+    @roles_required("admin")  # todo: remove after we check for sensor ownership
+    def get(self, id: int):
+        """GET from /sensor/<id>"""
+        sensor = get_sensor_or_abort(id)
+        attributes = ["name", "timezone", "timerange"]
+        return {attr: getattr(sensor, attr) for attr in attributes}
+
+
+def get_sensor_or_abort(id: int) -> Sensor:
+    sensor = Sensor.query.filter(Sensor.id == id).one_or_none()
+    if sensor is None:
+        raise abort(404, f"Sensor {id} not found")
+    return sensor

flexmeasures/api/v2_0/implementations/assets.py

@@ -66,7 +66,7 @@ def load_asset(admins_only: bool = False):
     should be allowed.
 
         @app.route('/asset/<id>')
-        @check_asset
+        @load_asset
         def get_asset(asset):
             return asset_schema.dump(asset), 200
 

flexmeasures/data/models/charts/__init__.py

@@ -0,0 +1,26 @@
+from inspect import getmembers, isfunction
+
+from . import belief_charts
+from .defaults import apply_chart_defaults
+
+
+def chart_type_to_chart_specs(chart_type: str, **kwargs) -> dict:
+    """Create chart specs of a given chart type, using FlexMeasures defaults for settings like width and height.
+
+    :param chart_type:  Name of a variable defining chart specs or a function returning chart specs.
+                        The chart specs can be a dictionary or an Altair chart specification.
+                        - In case of a dictionary, the creator needs to ensure that the dictionary contains valid specs
+                        - In case of an Altair chart specification, Altair validates for you
+    :returns:           A dictionary containing a vega-lite chart specification
+    """
+    # Create a dictionary mapping chart types to chart specs, and apply defaults to the chart specs, too.
+    belief_charts_mapping = {
+        chart_type: apply_chart_defaults(chart_specs)
+        for chart_type, chart_specs in getmembers(belief_charts)
+        if isfunction(chart_specs) or isinstance(chart_specs, dict)
+    }
+    # Create chart specs
+    chart_specs_or_fnc = belief_charts_mapping[chart_type]
+    if isfunction(chart_specs_or_fnc):
+        return chart_specs_or_fnc(**kwargs)
+    return chart_specs_or_fnc

flexmeasures/data/models/charts/belief_charts.py

@@ -0,0 +1,27 @@
+from flexmeasures.data.models.charts.defaults import TIME_TITLE, TIME_TOOLTIP_TITLE
+
+
+def bar_chart(title: str, quantity: str = "unknown quantity", unit: str = "a.u."):
+    if not unit:
+        unit = "a.u."
+    return {
+        "description": "A simple bar chart.",
+        "title": title,
+        "mark": "bar",
+        "encoding": {
+            "x": {"field": "event_start", "type": "T", "title": TIME_TITLE},
+            "y": {
+                "field": "event_value",
+                "type": "quantitative",
+                "title": quantity + " (" + unit + ")",
+            },
+            "tooltip": [
+                {"field": "full_date", "title": TIME_TOOLTIP_TITLE, "type": "nominal"},
+                {
+                    "field": "event_value",
+                    "title": quantity + " (" + unit + ")",
+                    "type": "quantitative",
+                },
+            ],
+        },
+    }

flexmeasures/data/models/charts/defaults.py

@@ -0,0 +1,42 @@
+from functools import wraps
+from typing import Callable, Union
+
+import altair as alt
+
+
+HEIGHT = 300
+WIDTH = 600
+REDUCED_HEIGHT = REDUCED_WIDTH = 60
+SELECTOR_COLOR = "darkred"
+TIME_FORMAT = "%I:%M %p on %A %b %e, %Y"
+TIME_TOOLTIP_TITLE = "Time and date"
+TIME_TITLE = None
+TIME_SELECTION_TOOLTIP = "Click and drag to select a time window"
+
+
+def apply_chart_defaults(fn):
+    @wraps(fn)
+    def decorated_chart_specs(*args, **kwargs):
+        dataset_name = kwargs.pop("dataset_name", None)
+        if isinstance(fn, Callable):
+            # function that returns a chart specification
+            chart_specs: Union[dict, alt.TopLevelMixin] = fn(*args, **kwargs)
+        else:
+            # not a function, but a direct chart specification
+            chart_specs: Union[dict, alt.TopLevelMixin] = fn
+        if isinstance(chart_specs, alt.TopLevelMixin):
+            chart_specs = chart_specs.to_dict()
+            chart_specs.pop("$schema")
+        if dataset_name:
+            chart_specs["data"] = {"name": dataset_name}
+        chart_specs["height"] = HEIGHT
+        chart_specs["width"] = WIDTH
+        chart_specs["transform"] = [
+            {
+                "as": "full_date",
+                "calculate": f"timeFormat(datum.event_start, '{TIME_FORMAT}')",
+            }
+        ]
+        return chart_specs
+
+    return decorated_chart_specs

flexmeasures/data/models/charts/readme.md

@@ -0,0 +1,7 @@
+# Developer docs for adding chart specs
+
+Chart specs can be specified as a dictionary with a vega-lite specification or as an altair chart.
+Alternatively, they can be specified as a function that returns a dict (with vega-lite specs) or an altair chart.
+This approach is useful if you need to parameterize the specification with kwargs.
+
+Todo: support a plug-in architecture, see https://packaging.python.org/guides/creating-and-discovering-plugins/