Issue 427 chart option to include annotations

flexmeasures/api/dev/sensors.py

@@ -1,12 +1,16 @@
+from itertools import chain
 import json
+import warnings
 
 from flask_classful import FlaskView, route
 from flask_security import current_user, auth_required
 from marshmallow import fields
+from textwrap import wrap
 from webargs.flaskparser import use_kwargs
 from werkzeug.exceptions import abort
 
 from flexmeasures.auth.policy import ADMIN_ROLE, ADMIN_READER_ROLE
+from flexmeasures.data.schemas.sensors import SensorIdField
 from flexmeasures.data.schemas.times import AwareDateTimeField
 from flexmeasures.data.models.time_series import Sensor
 
@@ -15,36 +19,44 @@ class SensorAPI(FlaskView):
     """
     This view exposes sensor attributes through API endpoints under development.
     These endpoints are not yet part of our official API, but support the FlexMeasures UI.
-
-    TODO: use permission-based auth and marshmallow (SensorIDField).
     """
 
     route_base = "/sensor"
     decorators = [auth_required()]
 
     @route("/<id>/chart/")
+    @use_kwargs(
+        {"sensor": SensorIdField(data_key="id")},
+        location="path",
+    )
     @use_kwargs(
         {
             "event_starts_after": AwareDateTimeField(format="iso", required=False),
             "event_ends_before": AwareDateTimeField(format="iso", required=False),
             "beliefs_after": AwareDateTimeField(format="iso", required=False),
             "beliefs_before": AwareDateTimeField(format="iso", required=False),
             "include_data": fields.Boolean(required=False),
+            "include_sensor_annotations": fields.Boolean(required=False),
+            "include_asset_annotations": fields.Boolean(required=False),
+            "include_account_annotations": fields.Boolean(required=False),
             "dataset_name": fields.Str(required=False),
             "height": fields.Str(required=False),
             "width": fields.Str(required=False),
         },
         location="query",
     )
-    def get_chart(self, id: int, **kwargs):
+    def get_chart(self, id: int, sensor: Sensor, **kwargs):
         """GET from /sensor/<id>/chart
 
         .. :quickref: Chart; Download a chart with time series
         """
-        sensor = get_sensor_or_abort(id)
         return json.dumps(sensor.chart(**kwargs))
 
     @route("/<id>/chart_data/")
+    @use_kwargs(
+        {"sensor": SensorIdField(data_key="id")},
+        location="path",
+    )
     @use_kwargs(
         {
             "event_starts_after": AwareDateTimeField(format="iso", required=False),
@@ -54,30 +66,76 @@ def get_chart(self, id: int, **kwargs):
         },
         location="query",
     )
-    def get_chart_data(self, id: int, **kwargs):
+    def get_chart_data(self, id: int, sensor: Sensor, **kwargs):
         """GET from /sensor/<id>/chart_data
 
         .. :quickref: Chart; Download time series for use in charts
 
         Data for use in charts (in case you have the chart specs already).
         """
-        sensor = get_sensor_or_abort(id)
         return sensor.search_beliefs(as_json=True, **kwargs)
 
-    def get(self, id: int):
+    @route("/<id>/chart_annotations/")
+    @use_kwargs(
+        {"sensor": SensorIdField(data_key="id")},
+        location="path",
+    )
+    @use_kwargs(
+        {
+            "event_starts_after": AwareDateTimeField(format="iso", required=False),
+            "event_ends_before": AwareDateTimeField(format="iso", required=False),
+            "beliefs_after": AwareDateTimeField(format="iso", required=False),
+            "beliefs_before": AwareDateTimeField(format="iso", required=False),
+        },
+        location="query",
+    )
+    def get_chart_annotations(self, id: int, sensor: Sensor, **kwargs):
+        """GET from /sensor/<id>/chart_annotations
+
+        .. :quickref: Chart; Download annotations for use in charts
+
+        Annotations for use in charts (in case you have the chart specs already).
+        """
+        df = sensor.generic_asset.search_annotations(
+            annotation_starts_after=kwargs.get("event_starts_after", None),
+            annotation_ends_before=kwargs.get("event_ends_before", None),
+            as_frame=True,
+        )
+
+        # Wrap on whitespace with some max line length
+        df["content"] = df["content"].apply(wrap, args=[60])
+
+        # Stack annotations for the same event
+        if not df.empty:
+            df = df.groupby("end", group_keys=False).apply(stack_annotations)
+
+        # Return JSON records
+        df = df.reset_index()
+        df["source"] = df["source"].astype(str)
+        return df.to_json(orient="records")
+
+    @route("/<id>/")
+    @use_kwargs(
+        {"sensor": SensorIdField(data_key="id")},
+        location="path",
+    )
+    def get(self, id: int, sensor: Sensor):
         """GET from /sensor/<id>
 
         .. :quickref: Chart; Download sensor attributes for use in charts
         """
-        sensor = get_sensor_or_abort(id)
         attributes = ["name", "timezone", "timerange"]
         return {attr: getattr(sensor, attr) for attr in attributes}
 
 
 def get_sensor_or_abort(id: int) -> Sensor:
     """
-    Util function to help the GET requests. Will be obsolete, see TODO above.
+    Util function to help the GET requests. Will be obsolete..
     """
+    warnings.warn(
+        "Util function will be deprecated. Switch to using SensorIdField to suppress this warning.",
+        FutureWarning,
+    )
     sensor = Sensor.query.filter(Sensor.id == id).one_or_none()
     if sensor is None:
         raise abort(404, f"Sensor {id} not found")
@@ -89,3 +147,13 @@ def get_sensor_or_abort(id: int) -> Sensor:
     ):
         raise abort(403)
     return sensor
+
+
+def stack_annotations(x):
+    """Select earliest start, and include all annotations as a list.
+
+    The list of strings results in a multi-line text encoding in the chart.
+    """
+    x = x.sort_values(["start", "belief_time"], ascending=True)
+    x["content"].iloc[0] = list(chain(*(x["content"].tolist())))
+    return x.head(1)

flexmeasures/data/models/annotations.py

@@ -201,3 +201,14 @@ def get_or_create_annotation(
     if annotation in db.session:
         db.session.expunge(annotation)
     return existing_annotation
+
+
+def to_annotation_frame(annotations: List[Annotation]) -> pd.DataFrame:
+    """Transform a list of annotations into a DataFrame."""
+    return pd.DataFrame(
+        [
+            [a.start, a.end, a.belief_time, a.source, a.type, a.content]
+            for a in annotations
+        ],
+        columns=["start", "end", "belief_time", "source", "type", "content"],
+    )

flexmeasures/data/models/charts/defaults.py

@@ -5,8 +5,9 @@
 
 
 FONT_SIZE = 16
+ANNOTATION_MARGIN = 16
 HEIGHT = 300
-WIDTH = 600
+WIDTH = "container"
 REDUCED_HEIGHT = REDUCED_WIDTH = 60
 SELECTOR_COLOR = "darkred"
 TIME_FORMAT = "%I:%M %p on %A %b %e, %Y"
@@ -37,6 +38,67 @@
         title="Time and date",
     ),
 }
+SHADE_LAYER = {
+    "mark": {
+        "type": "bar",
+        "color": "#bbbbbb",
+        "opacity": 0.3,
+        "size": HEIGHT,
+    },
+    "encoding": {
+        "x": dict(
+            field="start",
+            type="temporal",
+            title=None,
+        ),
+        "x2": dict(
+            field="end",
+            type="temporal",
+            title=None,
+        ),
+    },
+    "params": [
+        {
+            "name": "highlight",
+            "select": {"type": "point", "on": "mouseover"},
+        },
+        {"name": "select", "select": "point"},
+    ],
+}
+TEXT_LAYER = {
+    "mark": {
+        "type": "text",
+        "y": HEIGHT,
+        "dy": FONT_SIZE + ANNOTATION_MARGIN,
+        "baseline": "top",
+        "align": "left",
+        "fontSize": FONT_SIZE,
+        "fontStyle": "italic",
+    },
+    "encoding": {
+        "x": dict(
+            field="start",
+            type="temporal",
+            title=None,
+        ),
+        "text": {"type": "nominal", "field": "content"},
+        "opacity": {
+            "condition": [
+                {
+                    "param": "select",
+                    "empty": False,
+                    "value": 1,
+                },
+                {
+                    "param": "highlight",
+                    "empty": False,
+                    "value": 1,
+                },
+            ],
+            "value": 0,
+        },
+    },
+}
 LEGIBILITY_DEFAULTS = dict(
     config=dict(
         axis=dict(
@@ -66,6 +128,7 @@ def apply_chart_defaults(fn):
     @wraps(fn)
     def decorated_chart_specs(*args, **kwargs):
         dataset_name = kwargs.pop("dataset_name", None)
+        include_annotations = kwargs.pop("include_annotations", None)
         if isinstance(fn, Callable):
             # function that returns a chart specification
             chart_specs: Union[dict, alt.TopLevelMixin] = fn(*args, **kwargs)
@@ -75,8 +138,33 @@ def decorated_chart_specs(*args, **kwargs):
         if isinstance(chart_specs, alt.TopLevelMixin):
             chart_specs = chart_specs.to_dict()
             chart_specs.pop("$schema")
+
+        # Add transform function to calculate full date
+        if "transform" not in chart_specs:
+            chart_specs["transform"] = []
+        chart_specs["transform"].append(
+            {
+                "as": "full_date",
+                "calculate": f"timeFormat(datum.event_start, '{TIME_FORMAT}')",
+            }
+        )
+
         if dataset_name:
             chart_specs["data"] = {"name": dataset_name}
+            if include_annotations:
+                annotation_shades_layer = SHADE_LAYER
+                annotation_text_layer = TEXT_LAYER
+                annotation_shades_layer["data"] = {
+                    "name": dataset_name + "_annotations"
+                }
+                annotation_text_layer["data"] = {"name": dataset_name + "_annotations"}
+                chart_specs = {
+                    "layer": [
+                        annotation_shades_layer,
+                        chart_specs,
+                        annotation_text_layer,
+                    ]
+                }
 
         # Fall back to default height and width, if needed
         if "height" not in chart_specs:
@@ -90,15 +178,6 @@ def decorated_chart_specs(*args, **kwargs):
             chart_specs,
         )
 
-        # Add transform function to calculate full date
-        if "transform" not in chart_specs:
-            chart_specs["transform"] = []
-        chart_specs["transform"].append(
-            {
-                "as": "full_date",
-                "calculate": f"timeFormat(datum.event_start, '{TIME_FORMAT}')",
-            }
-        )
         return chart_specs
 
     return decorated_chart_specs

flexmeasures/data/models/generic_assets.py

@@ -10,7 +10,7 @@
 from sqlalchemy.schema import UniqueConstraint
 
 from flexmeasures.data import db
-from flexmeasures.data.models.annotations import Annotation
+from flexmeasures.data.models.annotations import Annotation, to_annotation_frame
 from flexmeasures.data.models.data_sources import DataSource
 from flexmeasures.data.models.parsing_utils import parse_source_arg
 from flexmeasures.data.models.user import User
@@ -216,15 +216,7 @@ def search_annotations(
                 source=source,
             )
 
-        if as_frame:
-            return pd.DataFrame(
-                [
-                    [a.start, a.end, a.belief_time, a.source, a.type, a.content]
-                    for a in annotations
-                ],
-                columns=["start", "end", "belief_time", "source", "type", "content"],
-            )
-        return annotations
+        return to_annotation_frame(annotations) if as_frame else annotations
 
     def count_annotations(
         self,