From 1ebc0089f47b056a66518966200124476514df3e Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 6 May 2022 12:28:56 +0200 Subject: [PATCH 01/38] Use SensorIdField Signed-off-by: F.N. Claessen --- flexmeasures/api/dev/sensors.py | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/flexmeasures/api/dev/sensors.py b/flexmeasures/api/dev/sensors.py index 8e269a6d7..3528751ee 100644 --- a/flexmeasures/api/dev/sensors.py +++ b/flexmeasures/api/dev/sensors.py @@ -1,4 +1,5 @@ import json +import warnings from flask_classful import FlaskView, route from flask_security import current_user, auth_required @@ -7,6 +8,7 @@ from werkzeug.exceptions import abort from flexmeasures.auth.policy import ADMIN_ROLE, ADMIN_READER_ROLE +from flexmeasures.data.schemas.sensors import SensorIdField from flexmeasures.data.schemas.times import AwareDateTimeField from flexmeasures.data.models.time_series import Sensor @@ -15,14 +17,16 @@ class SensorAPI(FlaskView): """ This view exposes sensor attributes through API endpoints under development. These endpoints are not yet part of our official API, but support the FlexMeasures UI. - - TODO: use permission-based auth and marshmallow (SensorIDField). """ route_base = "/sensor" decorators = [auth_required()] @route("//chart/") + @use_kwargs( + {"sensor": SensorIdField(data_key="id")}, + location="path", + ) @use_kwargs( { "event_starts_after": AwareDateTimeField(format="iso", required=False), @@ -36,15 +40,18 @@ class SensorAPI(FlaskView): }, location="query", ) - def get_chart(self, id: int, **kwargs): + def get_chart(self, id: int, sensor: Sensor, **kwargs): """GET from /sensor//chart .. :quickref: Chart; Download a chart with time series """ - sensor = get_sensor_or_abort(id) return json.dumps(sensor.chart(**kwargs)) @route("//chart_data/") + @use_kwargs( + {"sensor": SensorIdField(data_key="id")}, + location="path", + ) @use_kwargs( { "event_starts_after": AwareDateTimeField(format="iso", required=False), @@ -54,30 +61,37 @@ def get_chart(self, id: int, **kwargs): }, location="query", ) - def get_chart_data(self, id: int, **kwargs): + def get_chart_data(self, id: int, sensor: Sensor, **kwargs): """GET from /sensor//chart_data .. :quickref: Chart; Download time series for use in charts Data for use in charts (in case you have the chart specs already). """ - sensor = get_sensor_or_abort(id) return sensor.search_beliefs(as_json=True, **kwargs) - def get(self, id: int): + @route("//") + @use_kwargs( + {"sensor": SensorIdField(data_key="id")}, + location="path", + ) + def get(self, id: int, sensor: Sensor): """GET from /sensor/ .. :quickref: Chart; Download sensor attributes for use in charts """ - sensor = get_sensor_or_abort(id) attributes = ["name", "timezone", "timerange"] return {attr: getattr(sensor, attr) for attr in attributes} def get_sensor_or_abort(id: int) -> Sensor: """ - Util function to help the GET requests. Will be obsolete, see TODO above. + Util function to help the GET requests. Will be obsolete.. """ + warnings.warn( + "Util function will be deprecated. Switch to using SensorIdField to suppress this warning.", + FutureWarning, + ) sensor = Sensor.query.filter(Sensor.id == id).one_or_none() if sensor is None: raise abort(404, f"Sensor {id} not found") From b7d949d24842d6bc9c220e32fe30354bdf63fd78 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 6 May 2022 16:10:34 +0200 Subject: [PATCH 02/38] Fix function call Signed-off-by: F.N. Claessen --- flexmeasures/data/models/time_series.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flexmeasures/data/models/time_series.py b/flexmeasures/data/models/time_series.py index 813ea36cd..68319a4bf 100644 --- a/flexmeasures/data/models/time_series.py +++ b/flexmeasures/data/models/time_series.py @@ -224,13 +224,13 @@ def search_annotations( annotations = query.all() if include_asset_annotations: annotations += self.generic_asset.search_annotations( - annotation_starts_before=annotation_starts_after, + annotation_starts_after=annotation_starts_after, annotation_ends_before=annotation_ends_before, source=source, ) if include_account_annotations: annotations += self.generic_asset.owner.search_annotations( - annotation_starts_before=annotation_starts_after, + annotation_starts_after=annotation_starts_after, annotation_ends_before=annotation_ends_before, source=source, ) From 42dba0b6e6fab0406ee79722c598d1be41962430 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 6 May 2022 16:15:55 +0200 Subject: [PATCH 03/38] Layer annotations charts on top Signed-off-by: F.N. Claessen --- flexmeasures/data/models/charts/defaults.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/flexmeasures/data/models/charts/defaults.py b/flexmeasures/data/models/charts/defaults.py index bb2a32e17..dcaf6d108 100644 --- a/flexmeasures/data/models/charts/defaults.py +++ b/flexmeasures/data/models/charts/defaults.py @@ -77,6 +77,26 @@ def decorated_chart_specs(*args, **kwargs): chart_specs.pop("$schema") if dataset_name: chart_specs["data"] = {"name": dataset_name} + chart_specs = {"layer": [ + chart_specs, + { + "data": {"name": dataset_name + "_annotations"}, + "mark": { + "type": "text", + "y": 0, + "baseline": "top", + "align": "left", + }, + "encoding": { + "x": dict( + field="start", + type="temporal", + title=None, + ), + "text": {'type': 'nominal', 'field': 'content'}, + }, + } + ]} # Fall back to default height and width, if needed if "height" not in chart_specs: From c444d877ba42e78a53b3e18e8e0a221e7e48a040 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 6 May 2022 16:17:32 +0200 Subject: [PATCH 04/38] Add API endpoint to fetch chart annotations Signed-off-by: F.N. Claessen --- flexmeasures/api/dev/sensors.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/flexmeasures/api/dev/sensors.py b/flexmeasures/api/dev/sensors.py index 3528751ee..f63995578 100644 --- a/flexmeasures/api/dev/sensors.py +++ b/flexmeasures/api/dev/sensors.py @@ -70,6 +70,38 @@ def get_chart_data(self, id: int, sensor: Sensor, **kwargs): """ return sensor.search_beliefs(as_json=True, **kwargs) + @route("//chart_annotations/") + @use_kwargs( + {"sensor": SensorIdField(data_key="id")}, + location="path", + ) + @use_kwargs( + { + "event_starts_after": AwareDateTimeField(format="iso", required=False), + "event_ends_before": AwareDateTimeField(format="iso", required=False), + "beliefs_after": AwareDateTimeField(format="iso", required=False), + "beliefs_before": AwareDateTimeField(format="iso", required=False), + }, + location="query", + ) + def get_chart_annotations(self, id: int, sensor: Sensor, **kwargs): + """GET from /sensor//chart_annotations + + .. :quickref: Chart; Download annotations for use in charts + + Annotations for use in charts (in case you have the chart specs already). + """ + df = sensor.generic_asset.search_annotations( + annotation_starts_after=kwargs.get("event_starts_after", None), + annotation_ends_before=kwargs.get("event_ends_before", None), + as_frame=True, + ) + + # Return JSON records + df = df.reset_index() + df["source"] = df["source"].astype(str) + return df.to_json(orient="records") + @route("//") @use_kwargs( {"sensor": SensorIdField(data_key="id")}, From 1642cf1ed266e93cb2c62b9f80bfa5943fa29ac0 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 6 May 2022 16:18:00 +0200 Subject: [PATCH 05/38] Wrap long annotations Signed-off-by: F.N. Claessen --- flexmeasures/api/dev/sensors.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/flexmeasures/api/dev/sensors.py b/flexmeasures/api/dev/sensors.py index f63995578..d4f7e6b01 100644 --- a/flexmeasures/api/dev/sensors.py +++ b/flexmeasures/api/dev/sensors.py @@ -4,6 +4,7 @@ from flask_classful import FlaskView, route from flask_security import current_user, auth_required from marshmallow import fields +from textwrap import wrap from webargs.flaskparser import use_kwargs from werkzeug.exceptions import abort @@ -97,6 +98,9 @@ def get_chart_annotations(self, id: int, sensor: Sensor, **kwargs): as_frame=True, ) + # Wrap on whitespace with some max line length + df['content'] = df['content'].apply(wrap, args=[60]) + # Return JSON records df = df.reset_index() df["source"] = df["source"].astype(str) From 39e2fb3ff90ac8e52e0d36eeb037b6176cba4afb Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 6 May 2022 16:23:12 +0200 Subject: [PATCH 06/38] Stack multiple annotations for the same event Signed-off-by: F.N. Claessen --- flexmeasures/api/dev/sensors.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/flexmeasures/api/dev/sensors.py b/flexmeasures/api/dev/sensors.py index d4f7e6b01..73a3eb47c 100644 --- a/flexmeasures/api/dev/sensors.py +++ b/flexmeasures/api/dev/sensors.py @@ -1,3 +1,4 @@ +from itertools import chain import json import warnings @@ -101,6 +102,9 @@ def get_chart_annotations(self, id: int, sensor: Sensor, **kwargs): # Wrap on whitespace with some max line length df['content'] = df['content'].apply(wrap, args=[60]) + # Stack annotations for the same event + df = df.groupby('end', group_keys=False).apply(stack_annotations) + # Return JSON records df = df.reset_index() df["source"] = df["source"].astype(str) @@ -139,3 +143,13 @@ def get_sensor_or_abort(id: int) -> Sensor: ): raise abort(403) return sensor + + +def stack_annotations(x): + """Select earliest start, and include all annotations as a list. + + The list of strings results in a multi-line text encoding in the chart. + """ + x = x.sort_values("start", ascending=True) + x["content"][0] = list(chain(*(x["content"].tolist()))) + return x.head(1) From eaca9b2714e3e5a447d3d8d40920eb8ed74b6549 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 6 May 2022 16:23:52 +0200 Subject: [PATCH 07/38] Fetch annotations in sensors.html Signed-off-by: F.N. Claessen --- flexmeasures/ui/templates/views/sensors.html | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/flexmeasures/ui/templates/views/sensors.html b/flexmeasures/ui/templates/views/sensors.html index af814a544..92afd0de5 100644 --- a/flexmeasures/ui/templates/views/sensors.html +++ b/flexmeasures/ui/templates/views/sensors.html @@ -85,6 +85,8 @@ endDate.setDate(endDate.getDate() + 1); var queryStartDate = (startDate != null) ? (startDate.toISOString()) : (null) var queryEndDate = (endDate != null) ? (endDate.toISOString()) : (null) + + // Fetch time series data fetch(sensorPath + '/chart_data/?event_starts_after=' + queryStartDate + '&event_ends_before=' + queryEndDate, { method: "GET", headers: {"Content-Type": "application/json"}, @@ -92,6 +94,16 @@ .then(function(response) { return response.json(); }) .then(function(data) { vegaView.change(datasetName, vega.changeset().remove(vega.truthy).insert(data)).resize().run(); + }) + + // Fetch annotations + fetch(sensorPath + '/chart_annotations/?event_starts_after=' + queryStartDate + '&event_ends_before=' + queryEndDate, { + method: "GET", + headers: {"Content-Type": "application/json"}, + }) + .then(function(response) { return response.json(); }) + .then(function(data) { + vegaView.change(datasetName + '_annotations', vega.changeset().remove(vega.truthy).insert(data)).resize().run(); }); }); From 349619c98fcb4fcce9add3a049eec8ff74c17983 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 6 May 2022 17:11:41 +0200 Subject: [PATCH 08/38] Allow returning annotations as DataFrame Signed-off-by: F.N. Claessen --- flexmeasures/data/models/time_series.py | 12 +++++++++++- flexmeasures/data/models/user.py | 12 +++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/flexmeasures/data/models/time_series.py b/flexmeasures/data/models/time_series.py index 68319a4bf..96b98c48b 100644 --- a/flexmeasures/data/models/time_series.py +++ b/flexmeasures/data/models/time_series.py @@ -203,7 +203,8 @@ def search_annotations( ] = None, include_asset_annotations: bool = False, include_account_annotations: bool = False, - ): + as_frame: bool = False, + ) -> Union[List[Annotation], pd.DataFrame]: parsed_sources = parse_source_arg(source) query = Annotation.query.join(SensorAnnotationRelationship).filter( SensorAnnotationRelationship.sensor_id == self.id, @@ -234,6 +235,15 @@ def search_annotations( annotation_ends_before=annotation_ends_before, source=source, ) + + if as_frame: + return pd.DataFrame( + [ + [a.start, a.end, a.belief_time, a.source, a.type, a.content] + for a in annotations + ], + columns=["start", "end", "belief_time", "source", "type", "content"], + ) return annotations def search_beliefs( diff --git a/flexmeasures/data/models/user.py b/flexmeasures/data/models/user.py index eb9243f57..e570d1ea5 100644 --- a/flexmeasures/data/models/user.py +++ b/flexmeasures/data/models/user.py @@ -94,7 +94,8 @@ def search_annotations( source: Optional[ Union[DataSource, List[DataSource], int, List[int], str, List[str]] ] = None, - ): + as_frame: bool = False, + ) -> Union[List[Annotation], pd.DataFrame]: parsed_sources = parse_source_arg(source) query = Annotation.query.join(AccountAnnotationRelationship).filter( AccountAnnotationRelationship.account_id == self.id, @@ -113,6 +114,15 @@ def search_annotations( Annotation.source.in_(parsed_sources), ) annotations = query.all() + + if as_frame: + return pd.DataFrame( + [ + [a.start, a.end, a.belief_time, a.source, a.type, a.content] + for a in annotations + ], + columns=["start", "end", "belief_time", "source", "type", "content"], + ) return annotations From 9ee07ec4914f926c23ec0cc0d397c1e94c5c2697 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 6 May 2022 17:16:18 +0200 Subject: [PATCH 09/38] Refactor into util function Signed-off-by: F.N. Claessen --- flexmeasures/data/models/annotations.py | 11 +++++++++++ flexmeasures/data/models/generic_assets.py | 12 ++---------- flexmeasures/data/models/time_series.py | 11 ++--------- flexmeasures/data/models/user.py | 11 ++--------- 4 files changed, 17 insertions(+), 28 deletions(-) diff --git a/flexmeasures/data/models/annotations.py b/flexmeasures/data/models/annotations.py index 39aeb2346..5aac0ea1e 100644 --- a/flexmeasures/data/models/annotations.py +++ b/flexmeasures/data/models/annotations.py @@ -201,3 +201,14 @@ def get_or_create_annotation( if annotation in db.session: db.session.expunge(annotation) return existing_annotation + + +def to_annotation_frame(annotations: List[Annotation]) -> pd.DataFrame: + """Transform a list of annotations into a DataFrame.""" + return pd.DataFrame( + [ + [a.start, a.end, a.belief_time, a.source, a.type, a.content] + for a in annotations + ], + columns=["start", "end", "belief_time", "source", "type", "content"], + ) diff --git a/flexmeasures/data/models/generic_assets.py b/flexmeasures/data/models/generic_assets.py index 4f7ae95b1..46cc62d4c 100644 --- a/flexmeasures/data/models/generic_assets.py +++ b/flexmeasures/data/models/generic_assets.py @@ -10,7 +10,7 @@ from sqlalchemy.schema import UniqueConstraint from flexmeasures.data import db -from flexmeasures.data.models.annotations import Annotation +from flexmeasures.data.models.annotations import Annotation, to_annotation_frame from flexmeasures.data.models.data_sources import DataSource from flexmeasures.data.models.parsing_utils import parse_source_arg from flexmeasures.data.models.user import User @@ -216,15 +216,7 @@ def search_annotations( source=source, ) - if as_frame: - return pd.DataFrame( - [ - [a.start, a.end, a.belief_time, a.source, a.type, a.content] - for a in annotations - ], - columns=["start", "end", "belief_time", "source", "type", "content"], - ) - return annotations + return to_annotation_frame(annotations) if as_frame else annotations def count_annotations( self, diff --git a/flexmeasures/data/models/time_series.py b/flexmeasures/data/models/time_series.py index 96b98c48b..09dbcc18b 100644 --- a/flexmeasures/data/models/time_series.py +++ b/flexmeasures/data/models/time_series.py @@ -27,6 +27,7 @@ from flexmeasures.data.models.annotations import ( Annotation, SensorAnnotationRelationship, + to_annotation_frame, ) from flexmeasures.data.models.charts import chart_type_to_chart_specs from flexmeasures.data.models.data_sources import DataSource @@ -236,15 +237,7 @@ def search_annotations( source=source, ) - if as_frame: - return pd.DataFrame( - [ - [a.start, a.end, a.belief_time, a.source, a.type, a.content] - for a in annotations - ], - columns=["start", "end", "belief_time", "source", "type", "content"], - ) - return annotations + return to_annotation_frame(annotations) if as_frame else annotations def search_beliefs( self, diff --git a/flexmeasures/data/models/user.py b/flexmeasures/data/models/user.py index e570d1ea5..5cf8ec5f8 100644 --- a/flexmeasures/data/models/user.py +++ b/flexmeasures/data/models/user.py @@ -11,6 +11,7 @@ from flexmeasures.data.models.annotations import ( Annotation, AccountAnnotationRelationship, + to_annotation_frame, ) from flexmeasures.data.models.parsing_utils import parse_source_arg from flexmeasures.auth.policy import AuthModelMixin @@ -115,15 +116,7 @@ def search_annotations( ) annotations = query.all() - if as_frame: - return pd.DataFrame( - [ - [a.start, a.end, a.belief_time, a.source, a.type, a.content] - for a in annotations - ], - columns=["start", "end", "belief_time", "source", "type", "content"], - ) - return annotations + return to_annotation_frame(annotations) if as_frame else annotations class RolesUsers(db.Model): From 1f5ec6436f774341783ad549cd3858263558a734 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 6 May 2022 17:18:15 +0200 Subject: [PATCH 10/38] Fix import Signed-off-by: F.N. Claessen --- flexmeasures/data/models/time_series.py | 1 + flexmeasures/data/models/user.py | 1 + 2 files changed, 2 insertions(+) diff --git a/flexmeasures/data/models/time_series.py b/flexmeasures/data/models/time_series.py index 09dbcc18b..52139be84 100644 --- a/flexmeasures/data/models/time_series.py +++ b/flexmeasures/data/models/time_series.py @@ -2,6 +2,7 @@ from datetime import datetime as datetime_type, timedelta import json +import pandas as pd from sqlalchemy.ext.declarative import declared_attr from sqlalchemy.ext.mutable import MutableDict from sqlalchemy.orm import Query, Session diff --git a/flexmeasures/data/models/user.py b/flexmeasures/data/models/user.py index 5cf8ec5f8..52772749c 100644 --- a/flexmeasures/data/models/user.py +++ b/flexmeasures/data/models/user.py @@ -3,6 +3,7 @@ from datetime import datetime from flask_security import UserMixin, RoleMixin +import pandas as pd from sqlalchemy.orm import relationship, backref from sqlalchemy import Boolean, DateTime, Column, Integer, String, ForeignKey from sqlalchemy.ext.hybrid import hybrid_property From 051a68f29146fafbb968782e9b818569b7253586 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 6 May 2022 17:19:46 +0200 Subject: [PATCH 11/38] Allow including annotations in /chart endpoint Signed-off-by: F.N. Claessen --- flexmeasures/data/models/time_series.py | 47 +++++++++++++++++++++++-- flexmeasures/ui/views/sensors.py | 3 ++ 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/flexmeasures/data/models/time_series.py b/flexmeasures/data/models/time_series.py index 52139be84..b8732ef21 100644 --- a/flexmeasures/data/models/time_series.py +++ b/flexmeasures/data/models/time_series.py @@ -313,6 +313,9 @@ def chart( ] = None, most_recent_beliefs_only: bool = True, include_data: bool = False, + include_sensor_annotations: bool = False, + include_asset_annotations: bool = False, + include_account_annotations: bool = False, dataset_name: Optional[str] = None, **kwargs, ) -> dict: @@ -326,6 +329,9 @@ def chart( :param source: search only beliefs by this source (pass the DataSource, or its name or id) or list of sources :param most_recent_beliefs_only: only return the most recent beliefs for each event from each source (minimum belief horizon) :param include_data: if True, include data in the chart, or if False, exclude data + :param include_sensor_annotations: if True and include_data is True, include sensor annotations in the chart, or if False, exclude these + :param include_asset_annotations: if True and include_data is True, include asset annotations in the chart, or if False, exclude them + :param include_account_annotations: if True and include_data is True, include account annotations in the chart, or if False, exclude them :param dataset_name: optionally name the dataset used in the chart (the default name is sensor_) """ @@ -343,7 +349,7 @@ def chart( ) if include_data: - # Set up data + # Get data data = self.search_beliefs( as_json=True, event_starts_after=event_starts_after, @@ -353,8 +359,43 @@ def chart( most_recent_beliefs_only=most_recent_beliefs_only, source=source, ) - # Combine chart specs and data - chart_specs["datasets"] = {dataset_name: json.loads(data)} + + # Get annotations + if include_sensor_annotations: + annotations_df = self.search_annotations( + annotation_starts_after=event_starts_after, + annotation_ends_before=event_ends_before, + include_asset_annotations=include_asset_annotations, + include_account_annotations=include_account_annotations, + as_frame=True, + ) + elif include_asset_annotations: + annotations_df = self.generic_asset.search_annotations( + annotation_starts_after=event_starts_after, + annotation_ends_before=event_ends_before, + include_account_annotations=include_account_annotations, + as_frame=True, + ) + elif include_account_annotations: + annotations_df = self.generic_asset.owner.search_annotations( + annotation_starts_after=event_starts_after, + annotation_ends_before=event_ends_before, + as_frame=True, + ) + else: + annotations_df = to_annotation_frame([]) + + # Annotations to JSON records + annotations_df = annotations_df.reset_index() + annotations_df["source"] = annotations_df["source"].astype(str) + annotations_data = annotations_df.to_json(orient="records") + + # Combine chart specs, data and annotations + chart_specs["datasets"] = { + dataset_name: json.loads(data), + dataset_name + "_annotations": json.loads(annotations_data), + } + return chart_specs @property diff --git a/flexmeasures/ui/views/sensors.py b/flexmeasures/ui/views/sensors.py index e8cd4836d..cea3487f7 100644 --- a/flexmeasures/ui/views/sensors.py +++ b/flexmeasures/ui/views/sensors.py @@ -30,6 +30,9 @@ class SensorUI(FlaskView): "event_ends_before": AwareDateTimeField(format="iso", required=False), "beliefs_after": AwareDateTimeField(format="iso", required=False), "beliefs_before": AwareDateTimeField(format="iso", required=False), + "include_sensor_annotations": fields.Bool(required=False), + "include_asset_annotations": fields.Bool(required=False), + "include_account_annotations": fields.Bool(required=False), "dataset_name": fields.Str(required=False), "chart_theme": fields.Str(required=False), }, From daa53e6d05d94418a209f6bfb07982470b4a7ed2 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 6 May 2022 17:21:42 +0200 Subject: [PATCH 12/38] black Signed-off-by: F.N. Claessen --- flexmeasures/api/dev/sensors.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flexmeasures/api/dev/sensors.py b/flexmeasures/api/dev/sensors.py index 73a3eb47c..11fcead78 100644 --- a/flexmeasures/api/dev/sensors.py +++ b/flexmeasures/api/dev/sensors.py @@ -100,10 +100,10 @@ def get_chart_annotations(self, id: int, sensor: Sensor, **kwargs): ) # Wrap on whitespace with some max line length - df['content'] = df['content'].apply(wrap, args=[60]) + df["content"] = df["content"].apply(wrap, args=[60]) # Stack annotations for the same event - df = df.groupby('end', group_keys=False).apply(stack_annotations) + df = df.groupby("end", group_keys=False).apply(stack_annotations) # Return JSON records df = df.reset_index() From 7fc6785e6be663fa32823c0ba0bc203b04d710fb Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 6 May 2022 17:30:58 +0200 Subject: [PATCH 13/38] black Signed-off-by: F.N. Claessen --- flexmeasures/data/models/charts/defaults.py | 40 +++++++++++---------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/flexmeasures/data/models/charts/defaults.py b/flexmeasures/data/models/charts/defaults.py index dcaf6d108..f5403b83a 100644 --- a/flexmeasures/data/models/charts/defaults.py +++ b/flexmeasures/data/models/charts/defaults.py @@ -77,26 +77,28 @@ def decorated_chart_specs(*args, **kwargs): chart_specs.pop("$schema") if dataset_name: chart_specs["data"] = {"name": dataset_name} - chart_specs = {"layer": [ - chart_specs, - { - "data": {"name": dataset_name + "_annotations"}, - "mark": { - "type": "text", - "y": 0, - "baseline": "top", - "align": "left", + chart_specs = { + "layer": [ + chart_specs, + { + "data": {"name": dataset_name + "_annotations"}, + "mark": { + "type": "text", + "y": 0, + "baseline": "top", + "align": "left", + }, + "encoding": { + "x": dict( + field="start", + type="temporal", + title=None, + ), + "text": {"type": "nominal", "field": "content"}, + }, }, - "encoding": { - "x": dict( - field="start", - type="temporal", - title=None, - ), - "text": {'type': 'nominal', 'field': 'content'}, - }, - } - ]} + ] + } # Fall back to default height and width, if needed if "height" not in chart_specs: From a12a4a71725c46585d948f563bf300d77ad0d77a Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 6 May 2022 17:47:48 +0200 Subject: [PATCH 14/38] Update vega dependencies Signed-off-by: F.N. Claessen --- flexmeasures/utils/config_defaults.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flexmeasures/utils/config_defaults.py b/flexmeasures/utils/config_defaults.py index 43d833778..a04611f09 100644 --- a/flexmeasures/utils/config_defaults.py +++ b/flexmeasures/utils/config_defaults.py @@ -122,9 +122,9 @@ class Config(object): FLEXMEASURES_REDIS_DB_NR: int = 0 # Redis per default has 16 databases, [0-15] FLEXMEASURES_REDIS_PASSWORD: Optional[str] = None FLEXMEASURES_JS_VERSIONS: dict = dict( - vega="5", - vegaembed="6.17.0", - vegalite="5.0.0", + vega="5.22.1", + vegaembed="6.20.8", + vegalite="5.2.0", # todo: expand with other js versions used in FlexMeasures ) From 03de437501e8728b06806b160d7fb682a44c5330 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 6 May 2022 17:53:13 +0200 Subject: [PATCH 15/38] Do not stack annotations if there are none Signed-off-by: F.N. Claessen --- flexmeasures/api/dev/sensors.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/flexmeasures/api/dev/sensors.py b/flexmeasures/api/dev/sensors.py index 11fcead78..21b29d5fa 100644 --- a/flexmeasures/api/dev/sensors.py +++ b/flexmeasures/api/dev/sensors.py @@ -103,7 +103,8 @@ def get_chart_annotations(self, id: int, sensor: Sensor, **kwargs): df["content"] = df["content"].apply(wrap, args=[60]) # Stack annotations for the same event - df = df.groupby("end", group_keys=False).apply(stack_annotations) + if not df.empty: + df = df.groupby("end", group_keys=False).apply(stack_annotations) # Return JSON records df = df.reset_index() From 7efd6f925dfc2c46f0cbc8dbee987034a02d7e28 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 6 May 2022 17:56:30 +0200 Subject: [PATCH 16/38] Only add annotation chart specs if a dataset_name is given Signed-off-by: F.N. Claessen --- flexmeasures/data/models/charts/defaults.py | 42 ++++++++++----------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/flexmeasures/data/models/charts/defaults.py b/flexmeasures/data/models/charts/defaults.py index f5403b83a..f69dfdff1 100644 --- a/flexmeasures/data/models/charts/defaults.py +++ b/flexmeasures/data/models/charts/defaults.py @@ -77,28 +77,28 @@ def decorated_chart_specs(*args, **kwargs): chart_specs.pop("$schema") if dataset_name: chart_specs["data"] = {"name": dataset_name} - chart_specs = { - "layer": [ - chart_specs, - { - "data": {"name": dataset_name + "_annotations"}, - "mark": { - "type": "text", - "y": 0, - "baseline": "top", - "align": "left", + chart_specs = { + "layer": [ + chart_specs, + { + "data": {"name": dataset_name + "_annotations"}, + "mark": { + "type": "text", + "y": 0, + "baseline": "top", + "align": "left", + }, + "encoding": { + "x": dict( + field="start", + type="temporal", + title=None, + ), + "text": {"type": "nominal", "field": "content"}, + }, }, - "encoding": { - "x": dict( - field="start", - type="temporal", - title=None, - ), - "text": {"type": "nominal", "field": "content"}, - }, - }, - ] - } + ] + } # Fall back to default height and width, if needed if "height" not in chart_specs: From 0a49e3b69acc12c07ec1e67426ac28bc19fb80b2 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 12 May 2022 12:38:12 +0200 Subject: [PATCH 17/38] Visualize annotated data with shading Signed-off-by: F.N. Claessen --- flexmeasures/data/models/charts/defaults.py | 26 +++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/flexmeasures/data/models/charts/defaults.py b/flexmeasures/data/models/charts/defaults.py index f69dfdff1..65798bf52 100644 --- a/flexmeasures/data/models/charts/defaults.py +++ b/flexmeasures/data/models/charts/defaults.py @@ -6,7 +6,7 @@ FONT_SIZE = 16 HEIGHT = 300 -WIDTH = 600 +WIDTH = "container" REDUCED_HEIGHT = REDUCED_WIDTH = 60 SELECTOR_COLOR = "darkred" TIME_FORMAT = "%I:%M %p on %A %b %e, %Y" @@ -79,12 +79,34 @@ def decorated_chart_specs(*args, **kwargs): chart_specs["data"] = {"name": dataset_name} chart_specs = { "layer": [ + { + "data": {"name": dataset_name + "_annotations"}, + "mark": { + "type": "bar", + "color": "#bbbbbb", + "opacity": 0.3, + "size": HEIGHT, + }, + "encoding": { + "x": dict( + field="start", + type="temporal", + title=None, + ), + "x2": dict( + field="end", + type="temporal", + title=None, + ), + }, + }, chart_specs, { "data": {"name": dataset_name + "_annotations"}, "mark": { "type": "text", - "y": 0, + "y": HEIGHT, + "dy": 30, "baseline": "top", "align": "left", }, From 9565545f58281116ddc296ab36f5fb4e39039e37 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 12 May 2022 12:40:51 +0200 Subject: [PATCH 18/38] Parameterize spacing Signed-off-by: F.N. Claessen --- flexmeasures/data/models/charts/defaults.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/flexmeasures/data/models/charts/defaults.py b/flexmeasures/data/models/charts/defaults.py index 65798bf52..3a1994e84 100644 --- a/flexmeasures/data/models/charts/defaults.py +++ b/flexmeasures/data/models/charts/defaults.py @@ -5,6 +5,7 @@ FONT_SIZE = 16 +ANNOTATION_MARGIN = 16 HEIGHT = 300 WIDTH = "container" REDUCED_HEIGHT = REDUCED_WIDTH = 60 @@ -106,7 +107,7 @@ def decorated_chart_specs(*args, **kwargs): "mark": { "type": "text", "y": HEIGHT, - "dy": 30, + "dy": FONT_SIZE + ANNOTATION_MARGIN, "baseline": "top", "align": "left", }, From f470260aa25bb686abd58580c0bc1071a716877a Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 12 May 2022 14:42:19 +0200 Subject: [PATCH 19/38] Annotation font size and style Signed-off-by: F.N. Claessen --- flexmeasures/data/models/charts/defaults.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flexmeasures/data/models/charts/defaults.py b/flexmeasures/data/models/charts/defaults.py index 3a1994e84..695cb55f1 100644 --- a/flexmeasures/data/models/charts/defaults.py +++ b/flexmeasures/data/models/charts/defaults.py @@ -110,6 +110,8 @@ def decorated_chart_specs(*args, **kwargs): "dy": FONT_SIZE + ANNOTATION_MARGIN, "baseline": "top", "align": "left", + "fontSize": FONT_SIZE, + "fontStyle": "italic", }, "encoding": { "x": dict( From 7620f2b946a7e1b770bdd8f21fbe65c9db728864 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 12 May 2022 16:44:09 +0200 Subject: [PATCH 20/38] Add tooltip Signed-off-by: F.N. Claessen --- flexmeasures/data/models/charts/defaults.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/flexmeasures/data/models/charts/defaults.py b/flexmeasures/data/models/charts/defaults.py index 695cb55f1..912920b38 100644 --- a/flexmeasures/data/models/charts/defaults.py +++ b/flexmeasures/data/models/charts/defaults.py @@ -99,7 +99,13 @@ def decorated_chart_specs(*args, **kwargs): type="temporal", title=None, ), + "tooltip": [ + {"field": "content_str", "title": "Annotations"}, + ], }, + "transform": [ + {"calculate": "join(datum.content, ' ')", "as": "content_str"}, + ], }, chart_specs, { From c3a55aca0e342be3753efd95c40df83036d4b1a6 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 12 May 2022 22:13:08 +0200 Subject: [PATCH 21/38] Increase tooltip font size Signed-off-by: F.N. Claessen --- flexmeasures/ui/static/css/flexmeasures.css | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/flexmeasures/ui/static/css/flexmeasures.css b/flexmeasures/ui/static/css/flexmeasures.css index 8c318ca09..1f06d6ef3 100644 --- a/flexmeasures/ui/static/css/flexmeasures.css +++ b/flexmeasures/ui/static/css/flexmeasures.css @@ -1062,6 +1062,10 @@ body .dataTables_wrapper .dataTables_paginate .paginate_button.current, body .da margin-bottom: 40px; } +#vg-tooltip-element { + font-size: 16px; +} + /* --- End Sensor Data --- */ From da92df0ab56bc4459023bdb1f243d100c95f591c Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 12 May 2022 22:16:57 +0200 Subject: [PATCH 22/38] Fix full_date transform Signed-off-by: F.N. Claessen --- flexmeasures/data/models/charts/defaults.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/flexmeasures/data/models/charts/defaults.py b/flexmeasures/data/models/charts/defaults.py index 912920b38..801d5c896 100644 --- a/flexmeasures/data/models/charts/defaults.py +++ b/flexmeasures/data/models/charts/defaults.py @@ -76,6 +76,17 @@ def decorated_chart_specs(*args, **kwargs): if isinstance(chart_specs, alt.TopLevelMixin): chart_specs = chart_specs.to_dict() chart_specs.pop("$schema") + + # Add transform function to calculate full date + if "transform" not in chart_specs: + chart_specs["transform"] = [] + chart_specs["transform"].append( + { + "as": "full_date", + "calculate": f"timeFormat(datum.event_start, '{TIME_FORMAT}')", + } + ) + if dataset_name: chart_specs["data"] = {"name": dataset_name} chart_specs = { @@ -143,15 +154,6 @@ def decorated_chart_specs(*args, **kwargs): chart_specs, ) - # Add transform function to calculate full date - if "transform" not in chart_specs: - chart_specs["transform"] = [] - chart_specs["transform"].append( - { - "as": "full_date", - "calculate": f"timeFormat(datum.event_start, '{TIME_FORMAT}')", - } - ) return chart_specs return decorated_chart_specs From 54842a3089d41188e89137c44cd8b957acb22e80 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 12 May 2022 22:33:01 +0200 Subject: [PATCH 23/38] Refactor: move layer definitions Signed-off-by: F.N. Claessen --- flexmeasures/data/models/charts/defaults.py | 98 +++++++++++---------- 1 file changed, 51 insertions(+), 47 deletions(-) diff --git a/flexmeasures/data/models/charts/defaults.py b/flexmeasures/data/models/charts/defaults.py index 801d5c896..35b67c4e8 100644 --- a/flexmeasures/data/models/charts/defaults.py +++ b/flexmeasures/data/models/charts/defaults.py @@ -38,6 +38,51 @@ title="Time and date", ), } +SHADE_LAYER = { + "mark": { + "type": "bar", + "color": "#bbbbbb", + "opacity": 0.3, + "size": HEIGHT, + }, + "encoding": { + "x": dict( + field="start", + type="temporal", + title=None, + ), + "x2": dict( + field="end", + type="temporal", + title=None, + ), + "tooltip": [ + {"field": "content_str", "title": "Annotations"}, + ], + }, + "transform": [ + {"calculate": "join(datum.content, ' ')", "as": "content_str"}, + ], +} +TEXT_LAYER = { + "mark": { + "type": "text", + "y": HEIGHT, + "dy": FONT_SIZE + ANNOTATION_MARGIN, + "baseline": "top", + "align": "left", + "fontSize": FONT_SIZE, + "fontStyle": "italic", + }, + "encoding": { + "x": dict( + field="start", + type="temporal", + title=None, + ), + "text": {"type": "nominal", "field": "content"}, + }, +} LEGIBILITY_DEFAULTS = dict( config=dict( axis=dict( @@ -89,56 +134,15 @@ def decorated_chart_specs(*args, **kwargs): if dataset_name: chart_specs["data"] = {"name": dataset_name} + annotation_shades_layer = SHADE_LAYER + annotation_text_layer = TEXT_LAYER + annotation_shades_layer["data"] = {"name": dataset_name + "_annotations"} + annotation_text_layer["data"] = {"name": dataset_name + "_annotations"} chart_specs = { "layer": [ - { - "data": {"name": dataset_name + "_annotations"}, - "mark": { - "type": "bar", - "color": "#bbbbbb", - "opacity": 0.3, - "size": HEIGHT, - }, - "encoding": { - "x": dict( - field="start", - type="temporal", - title=None, - ), - "x2": dict( - field="end", - type="temporal", - title=None, - ), - "tooltip": [ - {"field": "content_str", "title": "Annotations"}, - ], - }, - "transform": [ - {"calculate": "join(datum.content, ' ')", "as": "content_str"}, - ], - }, + annotation_shades_layer, chart_specs, - { - "data": {"name": dataset_name + "_annotations"}, - "mark": { - "type": "text", - "y": HEIGHT, - "dy": FONT_SIZE + ANNOTATION_MARGIN, - "baseline": "top", - "align": "left", - "fontSize": FONT_SIZE, - "fontStyle": "italic", - }, - "encoding": { - "x": dict( - field="start", - type="temporal", - title=None, - ), - "text": {"type": "nominal", "field": "content"}, - }, - }, + annotation_text_layer, ] } From 137a34bd553972fad07b78f7e90d97e7d076f1e6 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 12 May 2022 22:47:57 +0200 Subject: [PATCH 24/38] Don't cut off tooltip annotations at 7em (which is the default vega-tooltip css) Signed-off-by: F.N. Claessen --- flexmeasures/ui/static/css/flexmeasures.css | 3 +++ 1 file changed, 3 insertions(+) diff --git a/flexmeasures/ui/static/css/flexmeasures.css b/flexmeasures/ui/static/css/flexmeasures.css index 1f06d6ef3..e46295b37 100644 --- a/flexmeasures/ui/static/css/flexmeasures.css +++ b/flexmeasures/ui/static/css/flexmeasures.css @@ -1065,6 +1065,9 @@ body .dataTables_wrapper .dataTables_paginate .paginate_button.current, body .da #vg-tooltip-element { font-size: 16px; } +#vg-tooltip-element table tr td.value { + max-height: inherit !important; +} /* --- End Sensor Data --- */ From 83377f4adaeadd0a10e625d8826261b980c6dcf8 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 13 May 2022 09:57:31 +0200 Subject: [PATCH 25/38] Remove tooltip in favour of showing annotations only underneath the graph Signed-off-by: F.N. Claessen --- flexmeasures/data/models/charts/defaults.py | 26 +++++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/flexmeasures/data/models/charts/defaults.py b/flexmeasures/data/models/charts/defaults.py index 35b67c4e8..435475a88 100644 --- a/flexmeasures/data/models/charts/defaults.py +++ b/flexmeasures/data/models/charts/defaults.py @@ -56,12 +56,13 @@ type="temporal", title=None, ), - "tooltip": [ - {"field": "content_str", "title": "Annotations"}, - ], }, - "transform": [ - {"calculate": "join(datum.content, ' ')", "as": "content_str"}, + "params": [ + { + "name": "highlight", + "select": {"type": "point", "on": "mouseover"}, + }, + {"name": "select", "select": "point"}, ], } TEXT_LAYER = { @@ -81,6 +82,21 @@ title=None, ), "text": {"type": "nominal", "field": "content"}, + "opacity": { + "condition": [ + { + "param": "select", + "empty": False, + "value": 1, + }, + { + "param": "highlight", + "empty": False, + "value": 1, + }, + ], + "value": 0, + }, }, } LEGIBILITY_DEFAULTS = dict( From 29c9fca52695c0f6257845d49cbf60aecc4fa484 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 13 May 2022 10:08:55 +0200 Subject: [PATCH 26/38] Optional inclusion of annotations in charts, and in sensor charts include sensor annotations and asset annotations by default Signed-off-by: F.N. Claessen --- flexmeasures/api/dev/sensors.py | 3 +++ flexmeasures/data/models/charts/defaults.py | 26 +++++++++++--------- flexmeasures/data/models/time_series.py | 3 +++ flexmeasures/ui/templates/views/sensors.html | 2 +- 4 files changed, 22 insertions(+), 12 deletions(-) diff --git a/flexmeasures/api/dev/sensors.py b/flexmeasures/api/dev/sensors.py index 21b29d5fa..277ace7b5 100644 --- a/flexmeasures/api/dev/sensors.py +++ b/flexmeasures/api/dev/sensors.py @@ -36,6 +36,9 @@ class SensorAPI(FlaskView): "beliefs_after": AwareDateTimeField(format="iso", required=False), "beliefs_before": AwareDateTimeField(format="iso", required=False), "include_data": fields.Boolean(required=False), + "include_sensor_annotations": fields.Boolean(required=False), + "include_asset_annotations": fields.Boolean(required=False), + "include_account_annotations": fields.Boolean(required=False), "dataset_name": fields.Str(required=False), "height": fields.Str(required=False), "width": fields.Str(required=False), diff --git a/flexmeasures/data/models/charts/defaults.py b/flexmeasures/data/models/charts/defaults.py index 435475a88..64136bb0d 100644 --- a/flexmeasures/data/models/charts/defaults.py +++ b/flexmeasures/data/models/charts/defaults.py @@ -128,6 +128,7 @@ def apply_chart_defaults(fn): @wraps(fn) def decorated_chart_specs(*args, **kwargs): dataset_name = kwargs.pop("dataset_name", None) + include_annotations = kwargs.pop("include_annotations", None) if isinstance(fn, Callable): # function that returns a chart specification chart_specs: Union[dict, alt.TopLevelMixin] = fn(*args, **kwargs) @@ -150,17 +151,20 @@ def decorated_chart_specs(*args, **kwargs): if dataset_name: chart_specs["data"] = {"name": dataset_name} - annotation_shades_layer = SHADE_LAYER - annotation_text_layer = TEXT_LAYER - annotation_shades_layer["data"] = {"name": dataset_name + "_annotations"} - annotation_text_layer["data"] = {"name": dataset_name + "_annotations"} - chart_specs = { - "layer": [ - annotation_shades_layer, - chart_specs, - annotation_text_layer, - ] - } + if include_annotations: + annotation_shades_layer = SHADE_LAYER + annotation_text_layer = TEXT_LAYER + annotation_shades_layer["data"] = { + "name": dataset_name + "_annotations" + } + annotation_text_layer["data"] = {"name": dataset_name + "_annotations"} + chart_specs = { + "layer": [ + annotation_shades_layer, + chart_specs, + annotation_text_layer, + ] + } # Fall back to default height and width, if needed if "height" not in chart_specs: diff --git a/flexmeasures/data/models/time_series.py b/flexmeasures/data/models/time_series.py index b8732ef21..7cff1dc36 100644 --- a/flexmeasures/data/models/time_series.py +++ b/flexmeasures/data/models/time_series.py @@ -345,6 +345,9 @@ def chart( chart_type, sensor=self, dataset_name=dataset_name, + include_annotations=include_sensor_annotations + or include_asset_annotations + or include_account_annotations, **kwargs, ) diff --git a/flexmeasures/ui/templates/views/sensors.html b/flexmeasures/ui/templates/views/sensors.html index fe64049e4..3238e55f5 100644 --- a/flexmeasures/ui/templates/views/sensors.html +++ b/flexmeasures/ui/templates/views/sensors.html @@ -36,7 +36,7 @@ async function embedAndLoad(chartSpecsPath, elementId, datasetName) { - await vegaEmbed('#'+elementId, chartSpecsPath + '?dataset_name=' + datasetName + '&width=container', {{ chart_options | safe }}) + await vegaEmbed('#'+elementId, chartSpecsPath + '?dataset_name=' + datasetName + '&width=container&include_sensor_annotations=true&include_asset_annotations=true', {{ chart_options | safe }}) .then(function (result) { // result.view is the Vega View, chartSpecsPath is the original Vega-Lite specification vegaView = result.view; From 11f17ae1207906cbad1115ca56730f16175e6428 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 13 May 2022 11:01:08 +0200 Subject: [PATCH 27/38] Fix annotation stacking Signed-off-by: F.N. Claessen --- flexmeasures/api/dev/sensors.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flexmeasures/api/dev/sensors.py b/flexmeasures/api/dev/sensors.py index 277ace7b5..1fd0fa416 100644 --- a/flexmeasures/api/dev/sensors.py +++ b/flexmeasures/api/dev/sensors.py @@ -154,6 +154,6 @@ def stack_annotations(x): The list of strings results in a multi-line text encoding in the chart. """ - x = x.sort_values("start", ascending=True) - x["content"][0] = list(chain(*(x["content"].tolist()))) + x = x.sort_values(["start", "belief_time"], ascending=True) + x["content"].iloc[0] = list(chain(*(x["content"].tolist()))) return x.head(1) From fc2e70dce7a72b1b118c060d2b553d95b16bd0e1 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 20 May 2022 14:41:27 +0200 Subject: [PATCH 28/38] Include annotations that only fall partly within the queried time window Signed-off-by: F.N. Claessen --- flexmeasures/api/dev/sensors.py | 12 ++++- flexmeasures/data/models/generic_assets.py | 48 ++++++++++++++---- flexmeasures/data/models/time_series.py | 58 ++++++++++++++++------ flexmeasures/data/models/user.py | 39 ++++++++++++--- flexmeasures/data/queries/annotations.py | 12 ++--- 5 files changed, 128 insertions(+), 41 deletions(-) diff --git a/flexmeasures/api/dev/sensors.py b/flexmeasures/api/dev/sensors.py index 1fd0fa416..542fca91a 100644 --- a/flexmeasures/api/dev/sensors.py +++ b/flexmeasures/api/dev/sensors.py @@ -96,12 +96,20 @@ def get_chart_annotations(self, id: int, sensor: Sensor, **kwargs): Annotations for use in charts (in case you have the chart specs already). """ + event_starts_after = kwargs.get("event_starts_after", None) + event_ends_before = kwargs.get("event_ends_before", None) df = sensor.generic_asset.search_annotations( - annotation_starts_after=kwargs.get("event_starts_after", None), - annotation_ends_before=kwargs.get("event_ends_before", None), + annotations_after=event_starts_after, + annotations_before=event_ends_before, as_frame=True, ) + # Clip annotations outside the requested time window + if event_starts_after is not None: + df.loc[df["start"] < event_starts_after, "start"] = event_starts_after + if event_ends_before is not None: + df.loc[df["end"] > event_ends_before, "end"] = event_ends_before + # Wrap on whitespace with some max line length df["content"] = df["content"].apply(wrap, args=[60]) diff --git a/flexmeasures/data/models/generic_assets.py b/flexmeasures/data/models/generic_assets.py index 46cc62d4c..39d5ea8dc 100644 --- a/flexmeasures/data/models/generic_assets.py +++ b/flexmeasures/data/models/generic_assets.py @@ -8,6 +8,7 @@ from sqlalchemy.sql.expression import func from sqlalchemy.ext.mutable import MutableDict from sqlalchemy.schema import UniqueConstraint +from timely_beliefs import utils as tb_utils from flexmeasures.data import db from flexmeasures.data.models.annotations import Annotation, to_annotation_frame @@ -191,8 +192,8 @@ def add_annotations( def search_annotations( self, - annotation_starts_after: Optional[datetime] = None, - annotation_ends_before: Optional[datetime] = None, + annotations_after: Optional[datetime] = None, + annotations_before: Optional[datetime] = None, source: Optional[ Union[DataSource, List[DataSource], int, List[int], str, List[str]] ] = None, @@ -200,19 +201,23 @@ def search_annotations( include_account_annotations: bool = False, as_frame: bool = False, ) -> Union[List[Annotation], pd.DataFrame]: - """Return annotations assigned to this asset, and optionally, also those assigned to the asset's account.""" + """Return annotations assigned to this asset, and optionally, also those assigned to the asset's account. + + :param annotations_after: only return annotations that end after this datetime (exclusive) + :param annotations_before: only return annotations that start before this datetime (exclusive) + """ parsed_sources = parse_source_arg(source) annotations = query_asset_annotations( asset_id=self.id, - annotation_starts_after=annotation_starts_after, - annotation_ends_before=annotation_ends_before, + annotations_after=annotations_after, + annotations_before=annotations_before, sources=parsed_sources, annotation_type=annotation_type, ).all() if include_account_annotations: annotations += self.owner.search_annotations( - annotation_starts_before=annotation_starts_after, - annotation_ends_before=annotation_ends_before, + annotations_after=annotations_after, + annotations_before=annotations_before, source=source, ) @@ -220,19 +225,40 @@ def search_annotations( def count_annotations( self, - annotation_starts_after: Optional[datetime] = None, - annotation_ends_before: Optional[datetime] = None, + annotation_starts_after: Optional[datetime] = None, # deprecated + annotations_after: Optional[datetime] = None, + annotation_ends_before: Optional[datetime] = None, # deprecated + annotations_before: Optional[datetime] = None, source: Optional[ Union[DataSource, List[DataSource], int, List[int], str, List[str]] ] = None, annotation_type: str = None, ) -> int: """Count the number of annotations assigned to this asset.""" + + # todo: deprecate the 'annotation_starts_after' argument in favor of 'annotations_after' (announced v0.11.0) + annotations_after = tb_utils.replace_deprecated_argument( + "annotation_starts_after", + annotation_starts_after, + "annotations_after", + annotations_after, + required_argument=False, + ) + + # todo: deprecate the 'annotation_ends_before' argument in favor of 'annotations_before' (announced v0.11.0) + annotations_before = tb_utils.replace_deprecated_argument( + "annotation_ends_before", + annotation_ends_before, + "annotations_before", + annotations_before, + required_argument=False, + ) + parsed_sources = parse_source_arg(source) return query_asset_annotations( asset_id=self.id, - annotation_starts_after=annotation_starts_after, - annotation_ends_before=annotation_ends_before, + annotations_after=annotations_after, + annotations_before=annotations_before, sources=parsed_sources, annotation_type=annotation_type, ).count() diff --git a/flexmeasures/data/models/time_series.py b/flexmeasures/data/models/time_series.py index 7cff1dc36..9f6a111a9 100644 --- a/flexmeasures/data/models/time_series.py +++ b/flexmeasures/data/models/time_series.py @@ -198,8 +198,10 @@ def latest_state( def search_annotations( self, - annotation_starts_after: Optional[datetime_type] = None, - annotation_ends_before: Optional[datetime_type] = None, + annotation_starts_after: Optional[datetime_type] = None, # deprecated + annotations_after: Optional[datetime_type] = None, + annotation_ends_before: Optional[datetime_type] = None, # deprecated + annotations_before: Optional[datetime_type] = None, source: Optional[ Union[DataSource, List[DataSource], int, List[int], str, List[str]] ] = None, @@ -207,18 +209,42 @@ def search_annotations( include_account_annotations: bool = False, as_frame: bool = False, ) -> Union[List[Annotation], pd.DataFrame]: + """Return annotations assigned to this sensor, and optionally, also those assigned to the sensor's asset and the asset's account. + + :param annotations_after: only return annotations that end after this datetime (exclusive) + :param annotations_before: only return annotations that start before this datetime (exclusive) + """ + + # todo: deprecate the 'annotation_starts_after' argument in favor of 'annotations_after' (announced v0.11.0) + annotations_after = tb_utils.replace_deprecated_argument( + "annotation_starts_after", + annotation_starts_after, + "annotations_after", + annotations_after, + required_argument=False, + ) + + # todo: deprecate the 'annotation_ends_before' argument in favor of 'annotations_before' (announced v0.11.0) + annotations_before = tb_utils.replace_deprecated_argument( + "annotation_ends_before", + annotation_ends_before, + "annotations_before", + annotations_before, + required_argument=False, + ) + parsed_sources = parse_source_arg(source) query = Annotation.query.join(SensorAnnotationRelationship).filter( SensorAnnotationRelationship.sensor_id == self.id, SensorAnnotationRelationship.annotation_id == Annotation.id, ) - if annotation_starts_after is not None: + if annotations_after is not None: query = query.filter( - Annotation.start >= annotation_starts_after, + Annotation.end > annotations_after, ) - if annotation_ends_before is not None: + if annotations_before is not None: query = query.filter( - Annotation.end <= annotation_ends_before, + Annotation.start < annotations_before, ) if parsed_sources: query = query.filter( @@ -227,14 +253,14 @@ def search_annotations( annotations = query.all() if include_asset_annotations: annotations += self.generic_asset.search_annotations( - annotation_starts_after=annotation_starts_after, - annotation_ends_before=annotation_ends_before, + annotations_after=annotations_after, + annotations_before=annotations_before, source=source, ) if include_account_annotations: annotations += self.generic_asset.owner.search_annotations( - annotation_starts_after=annotation_starts_after, - annotation_ends_before=annotation_ends_before, + annotations_after=annotations_after, + annotations_before=annotations_before, source=source, ) @@ -366,23 +392,23 @@ def chart( # Get annotations if include_sensor_annotations: annotations_df = self.search_annotations( - annotation_starts_after=event_starts_after, - annotation_ends_before=event_ends_before, + annotations_after=event_starts_after, + annotations_before=event_ends_before, include_asset_annotations=include_asset_annotations, include_account_annotations=include_account_annotations, as_frame=True, ) elif include_asset_annotations: annotations_df = self.generic_asset.search_annotations( - annotation_starts_after=event_starts_after, - annotation_ends_before=event_ends_before, + annotations_after=event_starts_after, + annotations_before=event_ends_before, include_account_annotations=include_account_annotations, as_frame=True, ) elif include_account_annotations: annotations_df = self.generic_asset.owner.search_annotations( - annotation_starts_after=event_starts_after, - annotation_ends_before=event_ends_before, + annotations_after=event_starts_after, + annotations_before=event_ends_before, as_frame=True, ) else: diff --git a/flexmeasures/data/models/user.py b/flexmeasures/data/models/user.py index 52772749c..ebbbaaa40 100644 --- a/flexmeasures/data/models/user.py +++ b/flexmeasures/data/models/user.py @@ -7,6 +7,7 @@ from sqlalchemy.orm import relationship, backref from sqlalchemy import Boolean, DateTime, Column, Integer, String, ForeignKey from sqlalchemy.ext.hybrid import hybrid_property +from timely_beliefs import utils as tb_utils from flexmeasures.data import db from flexmeasures.data.models.annotations import ( @@ -91,25 +92,51 @@ def has_role(self, role: Union[str, AccountRole]) -> bool: def search_annotations( self, - annotation_starts_after: Optional[datetime] = None, - annotation_ends_before: Optional[datetime] = None, + annotation_starts_after: Optional[datetime] = None, # deprecated + annotations_after: Optional[datetime] = None, + annotation_ends_before: Optional[datetime] = None, # deprecated + annotations_before: Optional[datetime] = None, source: Optional[ Union[DataSource, List[DataSource], int, List[int], str, List[str]] ] = None, as_frame: bool = False, ) -> Union[List[Annotation], pd.DataFrame]: + """Return annotations assigned to this account. + + :param annotations_after: only return annotations that end after this datetime (exclusive) + :param annotations_before: only return annotations that start before this datetime (exclusive) + """ + + # todo: deprecate the 'annotation_starts_after' argument in favor of 'annotations_after' (announced v0.11.0) + annotations_after = tb_utils.replace_deprecated_argument( + "annotation_starts_after", + annotation_starts_after, + "annotations_after", + annotations_after, + required_argument=False, + ) + + # todo: deprecate the 'annotation_ends_before' argument in favor of 'annotations_before' (announced v0.11.0) + annotations_before = tb_utils.replace_deprecated_argument( + "annotation_ends_before", + annotation_ends_before, + "annotations_before", + annotations_before, + required_argument=False, + ) + parsed_sources = parse_source_arg(source) query = Annotation.query.join(AccountAnnotationRelationship).filter( AccountAnnotationRelationship.account_id == self.id, AccountAnnotationRelationship.annotation_id == Annotation.id, ) - if annotation_starts_after is not None: + if annotations_after is not None: query = query.filter( - Annotation.start >= annotation_starts_after, + Annotation.end > annotations_after, ) - if annotation_ends_before is not None: + if annotations_before is not None: query = query.filter( - Annotation.end <= annotation_ends_before, + Annotation.start < annotations_before, ) if parsed_sources: query = query.filter( diff --git a/flexmeasures/data/queries/annotations.py b/flexmeasures/data/queries/annotations.py index daf53ad4a..a9b344a8b 100644 --- a/flexmeasures/data/queries/annotations.py +++ b/flexmeasures/data/queries/annotations.py @@ -12,8 +12,8 @@ def query_asset_annotations( asset_id: int, - annotation_starts_after: Optional[datetime] = None, - annotation_ends_before: Optional[datetime] = None, + annotations_after: Optional[datetime] = None, + annotations_before: Optional[datetime] = None, sources: Optional[List[DataSource]] = None, annotation_type: str = None, ) -> Query: @@ -22,13 +22,13 @@ def query_asset_annotations( GenericAssetAnnotationRelationship.generic_asset_id == asset_id, GenericAssetAnnotationRelationship.annotation_id == Annotation.id, ) - if annotation_starts_after is not None: + if annotations_after is not None: query = query.filter( - Annotation.start >= annotation_starts_after, + Annotation.end > annotations_after, ) - if annotation_ends_before is not None: + if annotations_before is not None: query = query.filter( - Annotation.end <= annotation_ends_before, + Annotation.start < annotations_before, ) if sources: query = query.filter( From 503f6bf5292dddf4628367cfb7e6b864d1154393 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 20 May 2022 14:56:07 +0200 Subject: [PATCH 29/38] Changelog entry Signed-off-by: F.N. Claessen --- documentation/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 2c715e64e..62d6036c9 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -7,6 +7,7 @@ v0.11.0 | June XX, 2022 New features ------------- +* Individual sensor charts show available annotations [see `PR #428 `_] Bugfixes ----------- From 9bb9ffd28b5028b65e152216874a35359c0983a8 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 23 May 2022 11:22:54 +0200 Subject: [PATCH 30/38] Deprecate util function that is no longer used Signed-off-by: F.N. Claessen --- flexmeasures/api/dev/sensors.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/flexmeasures/api/dev/sensors.py b/flexmeasures/api/dev/sensors.py index 542fca91a..53e2e1bf0 100644 --- a/flexmeasures/api/dev/sensors.py +++ b/flexmeasures/api/dev/sensors.py @@ -136,27 +136,6 @@ def get(self, id: int, sensor: Sensor): return {attr: getattr(sensor, attr) for attr in attributes} -def get_sensor_or_abort(id: int) -> Sensor: - """ - Util function to help the GET requests. Will be obsolete.. - """ - warnings.warn( - "Util function will be deprecated. Switch to using SensorIdField to suppress this warning.", - FutureWarning, - ) - sensor = Sensor.query.filter(Sensor.id == id).one_or_none() - if sensor is None: - raise abort(404, f"Sensor {id} not found") - if not ( - current_user.has_role(ADMIN_ROLE) - or current_user.has_role(ADMIN_READER_ROLE) - or sensor.generic_asset.owner is None # public - or sensor.generic_asset.owner == current_user.account # private but authorized - ): - raise abort(403) - return sensor - - def stack_annotations(x): """Select earliest start, and include all annotations as a list. From 528a7238fd9e299853ee50516b56467a9ae517fc Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 23 May 2022 11:25:18 +0200 Subject: [PATCH 31/38] Revert "Deprecate util function that is no longer used" This reverts commit 9bb9ffd28b5028b65e152216874a35359c0983a8. --- flexmeasures/api/dev/sensors.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/flexmeasures/api/dev/sensors.py b/flexmeasures/api/dev/sensors.py index 53e2e1bf0..542fca91a 100644 --- a/flexmeasures/api/dev/sensors.py +++ b/flexmeasures/api/dev/sensors.py @@ -136,6 +136,27 @@ def get(self, id: int, sensor: Sensor): return {attr: getattr(sensor, attr) for attr in attributes} +def get_sensor_or_abort(id: int) -> Sensor: + """ + Util function to help the GET requests. Will be obsolete.. + """ + warnings.warn( + "Util function will be deprecated. Switch to using SensorIdField to suppress this warning.", + FutureWarning, + ) + sensor = Sensor.query.filter(Sensor.id == id).one_or_none() + if sensor is None: + raise abort(404, f"Sensor {id} not found") + if not ( + current_user.has_role(ADMIN_ROLE) + or current_user.has_role(ADMIN_READER_ROLE) + or sensor.generic_asset.owner is None # public + or sensor.generic_asset.owner == current_user.account # private but authorized + ): + raise abort(403) + return sensor + + def stack_annotations(x): """Select earliest start, and include all annotations as a list. From 8c9e89b1b30e67c8ebd40a228e235bdf39505463 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 23 May 2022 12:04:22 +0200 Subject: [PATCH 32/38] Fix individual sensor chart UI page Signed-off-by: F.N. Claessen --- flexmeasures/ui/views/sensors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/ui/views/sensors.py b/flexmeasures/ui/views/sensors.py index cea3487f7..016fd168c 100644 --- a/flexmeasures/ui/views/sensors.py +++ b/flexmeasures/ui/views/sensors.py @@ -61,7 +61,7 @@ def get_chart(self, id, **kwargs): "vegalite" ], embed_options=embed_options, - ) + ).replace('
', '
') @login_required def get(self, id: int): From 76724e319d7893f7d986c239a725fa03889fbc1f Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 23 May 2022 12:06:08 +0200 Subject: [PATCH 33/38] Use auth decorator for read permissions on the sensor Signed-off-by: F.N. Claessen --- flexmeasures/api/dev/sensors.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/flexmeasures/api/dev/sensors.py b/flexmeasures/api/dev/sensors.py index 542fca91a..1bfa388a0 100644 --- a/flexmeasures/api/dev/sensors.py +++ b/flexmeasures/api/dev/sensors.py @@ -3,13 +3,14 @@ import warnings from flask_classful import FlaskView, route -from flask_security import current_user, auth_required +from flask_security import current_user from marshmallow import fields from textwrap import wrap from webargs.flaskparser import use_kwargs from werkzeug.exceptions import abort from flexmeasures.auth.policy import ADMIN_ROLE, ADMIN_READER_ROLE +from flexmeasures.auth.decorators import permission_required_for_context from flexmeasures.data.schemas.sensors import SensorIdField from flexmeasures.data.schemas.times import AwareDateTimeField from flexmeasures.data.models.time_series import Sensor @@ -22,7 +23,6 @@ class SensorAPI(FlaskView): """ route_base = "/sensor" - decorators = [auth_required()] @route("//chart/") @use_kwargs( @@ -45,6 +45,7 @@ class SensorAPI(FlaskView): }, location="query", ) + @permission_required_for_context("read", arg_name="sensor") def get_chart(self, id: int, sensor: Sensor, **kwargs): """GET from /sensor//chart @@ -66,6 +67,7 @@ def get_chart(self, id: int, sensor: Sensor, **kwargs): }, location="query", ) + @permission_required_for_context("read", arg_name="sensor") def get_chart_data(self, id: int, sensor: Sensor, **kwargs): """GET from /sensor//chart_data @@ -89,6 +91,7 @@ def get_chart_data(self, id: int, sensor: Sensor, **kwargs): }, location="query", ) + @permission_required_for_context("read", arg_name="sensor") def get_chart_annotations(self, id: int, sensor: Sensor, **kwargs): """GET from /sensor//chart_annotations @@ -127,6 +130,7 @@ def get_chart_annotations(self, id: int, sensor: Sensor, **kwargs): {"sensor": SensorIdField(data_key="id")}, location="path", ) + @permission_required_for_context("read", arg_name="sensor") def get(self, id: int, sensor: Sensor): """GET from /sensor/ From 40f9b0a8ab36c1aed9e7d7a4affbbe359f5622f0 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 23 May 2022 12:22:54 +0200 Subject: [PATCH 34/38] Fix default title font size for layered charts Signed-off-by: F.N. Claessen --- flexmeasures/data/models/charts/defaults.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/flexmeasures/data/models/charts/defaults.py b/flexmeasures/data/models/charts/defaults.py index 64136bb0d..1f4bb3274 100644 --- a/flexmeasures/data/models/charts/defaults.py +++ b/flexmeasures/data/models/charts/defaults.py @@ -104,9 +104,11 @@ axis=dict( titleFontSize=FONT_SIZE, labelFontSize=FONT_SIZE, - ) + ), + title=dict( + fontSize=FONT_SIZE, + ), ), - title=dict(fontSize=FONT_SIZE), encoding=dict( color=dict( dict( From 21be38936a872251964142555a5679fe3ec3950e Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 23 May 2022 12:25:03 +0200 Subject: [PATCH 35/38] Wrap and stack annotations also in the UI endpoint for standalone sensor charts Signed-off-by: F.N. Claessen --- flexmeasures/api/dev/sensors.py | 27 ++------------ flexmeasures/data/models/time_series.py | 4 ++ flexmeasures/data/services/annotations.py | 45 +++++++++++++++++++++++ 3 files changed, 52 insertions(+), 24 deletions(-) create mode 100644 flexmeasures/data/services/annotations.py diff --git a/flexmeasures/api/dev/sensors.py b/flexmeasures/api/dev/sensors.py index 1bfa388a0..c95685544 100644 --- a/flexmeasures/api/dev/sensors.py +++ b/flexmeasures/api/dev/sensors.py @@ -1,11 +1,9 @@ -from itertools import chain import json import warnings from flask_classful import FlaskView, route from flask_security import current_user from marshmallow import fields -from textwrap import wrap from webargs.flaskparser import use_kwargs from werkzeug.exceptions import abort @@ -14,6 +12,7 @@ from flexmeasures.data.schemas.sensors import SensorIdField from flexmeasures.data.schemas.times import AwareDateTimeField from flexmeasures.data.models.time_series import Sensor +from flexmeasures.data.services.annotations import prepare_annotations_for_chart class SensorAPI(FlaskView): @@ -107,18 +106,8 @@ def get_chart_annotations(self, id: int, sensor: Sensor, **kwargs): as_frame=True, ) - # Clip annotations outside the requested time window - if event_starts_after is not None: - df.loc[df["start"] < event_starts_after, "start"] = event_starts_after - if event_ends_before is not None: - df.loc[df["end"] > event_ends_before, "end"] = event_ends_before - - # Wrap on whitespace with some max line length - df["content"] = df["content"].apply(wrap, args=[60]) - - # Stack annotations for the same event - if not df.empty: - df = df.groupby("end", group_keys=False).apply(stack_annotations) + # Wrap and stack annotations + df = prepare_annotations_for_chart(df) # Return JSON records df = df.reset_index() @@ -159,13 +148,3 @@ def get_sensor_or_abort(id: int) -> Sensor: ): raise abort(403) return sensor - - -def stack_annotations(x): - """Select earliest start, and include all annotations as a list. - - The list of strings results in a multi-line text encoding in the chart. - """ - x = x.sort_values(["start", "belief_time"], ascending=True) - x["content"].iloc[0] = list(chain(*(x["content"].tolist()))) - return x.head(1) diff --git a/flexmeasures/data/models/time_series.py b/flexmeasures/data/models/time_series.py index 9f6a111a9..ef304b0ff 100644 --- a/flexmeasures/data/models/time_series.py +++ b/flexmeasures/data/models/time_series.py @@ -14,6 +14,7 @@ from flexmeasures.auth.policy import AuthModelMixin, EVERY_LOGGED_IN_USER from flexmeasures.data import db from flexmeasures.data.models.parsing_utils import parse_source_arg +from flexmeasures.data.services.annotations import prepare_annotations_for_chart from flexmeasures.data.queries.utils import ( create_beliefs_query, get_belief_timing_criteria, @@ -414,6 +415,9 @@ def chart( else: annotations_df = to_annotation_frame([]) + # Wrap and stack annotations + annotations_df = prepare_annotations_for_chart(annotations_df) + # Annotations to JSON records annotations_df = annotations_df.reset_index() annotations_df["source"] = annotations_df["source"].astype(str) diff --git a/flexmeasures/data/services/annotations.py b/flexmeasures/data/services/annotations.py new file mode 100644 index 000000000..751f7817f --- /dev/null +++ b/flexmeasures/data/services/annotations.py @@ -0,0 +1,45 @@ +from datetime import datetime +from itertools import chain +from textwrap import wrap +from typing import Optional + +import pandas as pd + + +def stack_annotations(x): + """Select earliest start, and include all annotations as a list. + + The list of strings results in a multi-line text encoding in the chart. + """ + x = x.sort_values(["start", "belief_time"], ascending=True) + x["content"].iloc[0] = list(chain(*(x["content"].tolist()))) + return x.head(1) + + +def prepare_annotations_for_chart( + df: pd.DataFrame, + event_starts_after: Optional[datetime] = None, + event_ends_before: Optional[datetime] = None, + max_line_length: int = 60, +) -> pd.DataFrame: + """Prepare a DataFrame with annotations for use in a chart. + + - Clips annotations outside the requested time window. + - Wraps on whitespace with a given max line length + - Stacks annotations for the same event + """ + + # Clip annotations outside the requested time window + if event_starts_after is not None: + df.loc[df["start"] < event_starts_after, "start"] = event_starts_after + if event_ends_before is not None: + df.loc[df["end"] > event_ends_before, "end"] = event_ends_before + + # Wrap on whitespace with some max line length + df["content"] = df["content"].apply(wrap, args=[max_line_length]) + + # Stack annotations for the same event + if not df.empty: + df = df.groupby("end", group_keys=False).apply(stack_annotations) + + return df From e9bf0dba8846d0b6bd6897bcd81d82c38c90b686 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 23 May 2022 12:31:39 +0200 Subject: [PATCH 36/38] Clarify return type of sensor.chart() Signed-off-by: F.N. Claessen --- flexmeasures/data/models/time_series.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/flexmeasures/data/models/time_series.py b/flexmeasures/data/models/time_series.py index ef304b0ff..210a1dbaf 100644 --- a/flexmeasures/data/models/time_series.py +++ b/flexmeasures/data/models/time_series.py @@ -346,7 +346,7 @@ def chart( dataset_name: Optional[str] = None, **kwargs, ) -> dict: - """Create a chart showing sensor data. + """Create a vega-lite chart showing sensor data. :param chart_type: currently only "bar_chart" # todo: where can we properly list the available chart types? :param event_starts_after: only return beliefs about events that start after this datetime (inclusive) @@ -360,6 +360,7 @@ def chart( :param include_asset_annotations: if True and include_data is True, include asset annotations in the chart, or if False, exclude them :param include_account_annotations: if True and include_data is True, include account annotations in the chart, or if False, exclude them :param dataset_name: optionally name the dataset used in the chart (the default name is sensor_) + :returns: JSON string defining vega-lite chart specs """ # Set up chart specification From 1b73ed4aa0abe1d8a0367a1e24fb670294f997e2 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 23 May 2022 12:34:21 +0200 Subject: [PATCH 37/38] Add missing type annotation Signed-off-by: F.N. Claessen --- flexmeasures/data/services/annotations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/services/annotations.py b/flexmeasures/data/services/annotations.py index 751f7817f..d13db3608 100644 --- a/flexmeasures/data/services/annotations.py +++ b/flexmeasures/data/services/annotations.py @@ -6,7 +6,7 @@ import pandas as pd -def stack_annotations(x): +def stack_annotations(x: pd.DataFrame) -> pd.DataFrame: """Select earliest start, and include all annotations as a list. The list of strings results in a multi-line text encoding in the chart. From c6bbb3125ce006c4a76c8cd92524fa16b19252e9 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 23 May 2022 12:35:42 +0200 Subject: [PATCH 38/38] Add note from feedback on PR #428 Signed-off-by: F.N. Claessen --- flexmeasures/data/models/annotations.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/flexmeasures/data/models/annotations.py b/flexmeasures/data/models/annotations.py index 5aac0ea1e..cb3fd5454 100644 --- a/flexmeasures/data/models/annotations.py +++ b/flexmeasures/data/models/annotations.py @@ -204,7 +204,10 @@ def get_or_create_annotation( def to_annotation_frame(annotations: List[Annotation]) -> pd.DataFrame: - """Transform a list of annotations into a DataFrame.""" + """Transform a list of annotations into a DataFrame. + + We don't use a BeliefsDataFrame here, because they are designed for quantitative data only. + """ return pd.DataFrame( [ [a.start, a.end, a.belief_time, a.source, a.type, a.content]