diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 2c715e64e..62d6036c9 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -7,6 +7,7 @@ v0.11.0 | June XX, 2022 New features ------------- +* Individual sensor charts show available annotations [see `PR #428 `_] Bugfixes ----------- diff --git a/flexmeasures/api/dev/sensors.py b/flexmeasures/api/dev/sensors.py index 8e269a6d7..c95685544 100644 --- a/flexmeasures/api/dev/sensors.py +++ b/flexmeasures/api/dev/sensors.py @@ -1,28 +1,33 @@ import json +import warnings from flask_classful import FlaskView, route -from flask_security import current_user, auth_required +from flask_security import current_user from marshmallow import fields from webargs.flaskparser import use_kwargs from werkzeug.exceptions import abort from flexmeasures.auth.policy import ADMIN_ROLE, ADMIN_READER_ROLE +from flexmeasures.auth.decorators import permission_required_for_context +from flexmeasures.data.schemas.sensors import SensorIdField from flexmeasures.data.schemas.times import AwareDateTimeField from flexmeasures.data.models.time_series import Sensor +from flexmeasures.data.services.annotations import prepare_annotations_for_chart class SensorAPI(FlaskView): """ This view exposes sensor attributes through API endpoints under development. These endpoints are not yet part of our official API, but support the FlexMeasures UI. - - TODO: use permission-based auth and marshmallow (SensorIDField). """ route_base = "/sensor" - decorators = [auth_required()] @route("//chart/") + @use_kwargs( + {"sensor": SensorIdField(data_key="id")}, + location="path", + ) @use_kwargs( { "event_starts_after": AwareDateTimeField(format="iso", required=False), @@ -30,21 +35,28 @@ class SensorAPI(FlaskView): "beliefs_after": AwareDateTimeField(format="iso", required=False), "beliefs_before": AwareDateTimeField(format="iso", required=False), "include_data": fields.Boolean(required=False), + "include_sensor_annotations": fields.Boolean(required=False), + "include_asset_annotations": fields.Boolean(required=False), + "include_account_annotations": fields.Boolean(required=False), "dataset_name": fields.Str(required=False), "height": fields.Str(required=False), "width": fields.Str(required=False), }, location="query", ) - def get_chart(self, id: int, **kwargs): + @permission_required_for_context("read", arg_name="sensor") + def get_chart(self, id: int, sensor: Sensor, **kwargs): """GET from /sensor//chart .. :quickref: Chart; Download a chart with time series """ - sensor = get_sensor_or_abort(id) return json.dumps(sensor.chart(**kwargs)) @route("//chart_data/") + @use_kwargs( + {"sensor": SensorIdField(data_key="id")}, + location="path", + ) @use_kwargs( { "event_starts_after": AwareDateTimeField(format="iso", required=False), @@ -54,30 +66,77 @@ def get_chart(self, id: int, **kwargs): }, location="query", ) - def get_chart_data(self, id: int, **kwargs): + @permission_required_for_context("read", arg_name="sensor") + def get_chart_data(self, id: int, sensor: Sensor, **kwargs): """GET from /sensor//chart_data .. :quickref: Chart; Download time series for use in charts Data for use in charts (in case you have the chart specs already). """ - sensor = get_sensor_or_abort(id) return sensor.search_beliefs(as_json=True, **kwargs) - def get(self, id: int): + @route("//chart_annotations/") + @use_kwargs( + {"sensor": SensorIdField(data_key="id")}, + location="path", + ) + @use_kwargs( + { + "event_starts_after": AwareDateTimeField(format="iso", required=False), + "event_ends_before": AwareDateTimeField(format="iso", required=False), + "beliefs_after": AwareDateTimeField(format="iso", required=False), + "beliefs_before": AwareDateTimeField(format="iso", required=False), + }, + location="query", + ) + @permission_required_for_context("read", arg_name="sensor") + def get_chart_annotations(self, id: int, sensor: Sensor, **kwargs): + """GET from /sensor//chart_annotations + + .. :quickref: Chart; Download annotations for use in charts + + Annotations for use in charts (in case you have the chart specs already). + """ + event_starts_after = kwargs.get("event_starts_after", None) + event_ends_before = kwargs.get("event_ends_before", None) + df = sensor.generic_asset.search_annotations( + annotations_after=event_starts_after, + annotations_before=event_ends_before, + as_frame=True, + ) + + # Wrap and stack annotations + df = prepare_annotations_for_chart(df) + + # Return JSON records + df = df.reset_index() + df["source"] = df["source"].astype(str) + return df.to_json(orient="records") + + @route("//") + @use_kwargs( + {"sensor": SensorIdField(data_key="id")}, + location="path", + ) + @permission_required_for_context("read", arg_name="sensor") + def get(self, id: int, sensor: Sensor): """GET from /sensor/ .. :quickref: Chart; Download sensor attributes for use in charts """ - sensor = get_sensor_or_abort(id) attributes = ["name", "timezone", "timerange"] return {attr: getattr(sensor, attr) for attr in attributes} def get_sensor_or_abort(id: int) -> Sensor: """ - Util function to help the GET requests. Will be obsolete, see TODO above. + Util function to help the GET requests. Will be obsolete.. """ + warnings.warn( + "Util function will be deprecated. Switch to using SensorIdField to suppress this warning.", + FutureWarning, + ) sensor = Sensor.query.filter(Sensor.id == id).one_or_none() if sensor is None: raise abort(404, f"Sensor {id} not found") diff --git a/flexmeasures/data/models/annotations.py b/flexmeasures/data/models/annotations.py index 39aeb2346..cb3fd5454 100644 --- a/flexmeasures/data/models/annotations.py +++ b/flexmeasures/data/models/annotations.py @@ -201,3 +201,17 @@ def get_or_create_annotation( if annotation in db.session: db.session.expunge(annotation) return existing_annotation + + +def to_annotation_frame(annotations: List[Annotation]) -> pd.DataFrame: + """Transform a list of annotations into a DataFrame. + + We don't use a BeliefsDataFrame here, because they are designed for quantitative data only. + """ + return pd.DataFrame( + [ + [a.start, a.end, a.belief_time, a.source, a.type, a.content] + for a in annotations + ], + columns=["start", "end", "belief_time", "source", "type", "content"], + ) diff --git a/flexmeasures/data/models/charts/defaults.py b/flexmeasures/data/models/charts/defaults.py index bb2a32e17..1f4bb3274 100644 --- a/flexmeasures/data/models/charts/defaults.py +++ b/flexmeasures/data/models/charts/defaults.py @@ -5,8 +5,9 @@ FONT_SIZE = 16 +ANNOTATION_MARGIN = 16 HEIGHT = 300 -WIDTH = 600 +WIDTH = "container" REDUCED_HEIGHT = REDUCED_WIDTH = 60 SELECTOR_COLOR = "darkred" TIME_FORMAT = "%I:%M %p on %A %b %e, %Y" @@ -37,14 +38,77 @@ title="Time and date", ), } +SHADE_LAYER = { + "mark": { + "type": "bar", + "color": "#bbbbbb", + "opacity": 0.3, + "size": HEIGHT, + }, + "encoding": { + "x": dict( + field="start", + type="temporal", + title=None, + ), + "x2": dict( + field="end", + type="temporal", + title=None, + ), + }, + "params": [ + { + "name": "highlight", + "select": {"type": "point", "on": "mouseover"}, + }, + {"name": "select", "select": "point"}, + ], +} +TEXT_LAYER = { + "mark": { + "type": "text", + "y": HEIGHT, + "dy": FONT_SIZE + ANNOTATION_MARGIN, + "baseline": "top", + "align": "left", + "fontSize": FONT_SIZE, + "fontStyle": "italic", + }, + "encoding": { + "x": dict( + field="start", + type="temporal", + title=None, + ), + "text": {"type": "nominal", "field": "content"}, + "opacity": { + "condition": [ + { + "param": "select", + "empty": False, + "value": 1, + }, + { + "param": "highlight", + "empty": False, + "value": 1, + }, + ], + "value": 0, + }, + }, +} LEGIBILITY_DEFAULTS = dict( config=dict( axis=dict( titleFontSize=FONT_SIZE, labelFontSize=FONT_SIZE, - ) + ), + title=dict( + fontSize=FONT_SIZE, + ), ), - title=dict(fontSize=FONT_SIZE), encoding=dict( color=dict( dict( @@ -66,6 +130,7 @@ def apply_chart_defaults(fn): @wraps(fn) def decorated_chart_specs(*args, **kwargs): dataset_name = kwargs.pop("dataset_name", None) + include_annotations = kwargs.pop("include_annotations", None) if isinstance(fn, Callable): # function that returns a chart specification chart_specs: Union[dict, alt.TopLevelMixin] = fn(*args, **kwargs) @@ -75,8 +140,33 @@ def decorated_chart_specs(*args, **kwargs): if isinstance(chart_specs, alt.TopLevelMixin): chart_specs = chart_specs.to_dict() chart_specs.pop("$schema") + + # Add transform function to calculate full date + if "transform" not in chart_specs: + chart_specs["transform"] = [] + chart_specs["transform"].append( + { + "as": "full_date", + "calculate": f"timeFormat(datum.event_start, '{TIME_FORMAT}')", + } + ) + if dataset_name: chart_specs["data"] = {"name": dataset_name} + if include_annotations: + annotation_shades_layer = SHADE_LAYER + annotation_text_layer = TEXT_LAYER + annotation_shades_layer["data"] = { + "name": dataset_name + "_annotations" + } + annotation_text_layer["data"] = {"name": dataset_name + "_annotations"} + chart_specs = { + "layer": [ + annotation_shades_layer, + chart_specs, + annotation_text_layer, + ] + } # Fall back to default height and width, if needed if "height" not in chart_specs: @@ -90,15 +180,6 @@ def decorated_chart_specs(*args, **kwargs): chart_specs, ) - # Add transform function to calculate full date - if "transform" not in chart_specs: - chart_specs["transform"] = [] - chart_specs["transform"].append( - { - "as": "full_date", - "calculate": f"timeFormat(datum.event_start, '{TIME_FORMAT}')", - } - ) return chart_specs return decorated_chart_specs diff --git a/flexmeasures/data/models/generic_assets.py b/flexmeasures/data/models/generic_assets.py index 4f7ae95b1..39d5ea8dc 100644 --- a/flexmeasures/data/models/generic_assets.py +++ b/flexmeasures/data/models/generic_assets.py @@ -8,9 +8,10 @@ from sqlalchemy.sql.expression import func from sqlalchemy.ext.mutable import MutableDict from sqlalchemy.schema import UniqueConstraint +from timely_beliefs import utils as tb_utils from flexmeasures.data import db -from flexmeasures.data.models.annotations import Annotation +from flexmeasures.data.models.annotations import Annotation, to_annotation_frame from flexmeasures.data.models.data_sources import DataSource from flexmeasures.data.models.parsing_utils import parse_source_arg from flexmeasures.data.models.user import User @@ -191,8 +192,8 @@ def add_annotations( def search_annotations( self, - annotation_starts_after: Optional[datetime] = None, - annotation_ends_before: Optional[datetime] = None, + annotations_after: Optional[datetime] = None, + annotations_before: Optional[datetime] = None, source: Optional[ Union[DataSource, List[DataSource], int, List[int], str, List[str]] ] = None, @@ -200,47 +201,64 @@ def search_annotations( include_account_annotations: bool = False, as_frame: bool = False, ) -> Union[List[Annotation], pd.DataFrame]: - """Return annotations assigned to this asset, and optionally, also those assigned to the asset's account.""" + """Return annotations assigned to this asset, and optionally, also those assigned to the asset's account. + + :param annotations_after: only return annotations that end after this datetime (exclusive) + :param annotations_before: only return annotations that start before this datetime (exclusive) + """ parsed_sources = parse_source_arg(source) annotations = query_asset_annotations( asset_id=self.id, - annotation_starts_after=annotation_starts_after, - annotation_ends_before=annotation_ends_before, + annotations_after=annotations_after, + annotations_before=annotations_before, sources=parsed_sources, annotation_type=annotation_type, ).all() if include_account_annotations: annotations += self.owner.search_annotations( - annotation_starts_before=annotation_starts_after, - annotation_ends_before=annotation_ends_before, + annotations_after=annotations_after, + annotations_before=annotations_before, source=source, ) - if as_frame: - return pd.DataFrame( - [ - [a.start, a.end, a.belief_time, a.source, a.type, a.content] - for a in annotations - ], - columns=["start", "end", "belief_time", "source", "type", "content"], - ) - return annotations + return to_annotation_frame(annotations) if as_frame else annotations def count_annotations( self, - annotation_starts_after: Optional[datetime] = None, - annotation_ends_before: Optional[datetime] = None, + annotation_starts_after: Optional[datetime] = None, # deprecated + annotations_after: Optional[datetime] = None, + annotation_ends_before: Optional[datetime] = None, # deprecated + annotations_before: Optional[datetime] = None, source: Optional[ Union[DataSource, List[DataSource], int, List[int], str, List[str]] ] = None, annotation_type: str = None, ) -> int: """Count the number of annotations assigned to this asset.""" + + # todo: deprecate the 'annotation_starts_after' argument in favor of 'annotations_after' (announced v0.11.0) + annotations_after = tb_utils.replace_deprecated_argument( + "annotation_starts_after", + annotation_starts_after, + "annotations_after", + annotations_after, + required_argument=False, + ) + + # todo: deprecate the 'annotation_ends_before' argument in favor of 'annotations_before' (announced v0.11.0) + annotations_before = tb_utils.replace_deprecated_argument( + "annotation_ends_before", + annotation_ends_before, + "annotations_before", + annotations_before, + required_argument=False, + ) + parsed_sources = parse_source_arg(source) return query_asset_annotations( asset_id=self.id, - annotation_starts_after=annotation_starts_after, - annotation_ends_before=annotation_ends_before, + annotations_after=annotations_after, + annotations_before=annotations_before, sources=parsed_sources, annotation_type=annotation_type, ).count() diff --git a/flexmeasures/data/models/time_series.py b/flexmeasures/data/models/time_series.py index 813ea36cd..210a1dbaf 100644 --- a/flexmeasures/data/models/time_series.py +++ b/flexmeasures/data/models/time_series.py @@ -2,6 +2,7 @@ from datetime import datetime as datetime_type, timedelta import json +import pandas as pd from sqlalchemy.ext.declarative import declared_attr from sqlalchemy.ext.mutable import MutableDict from sqlalchemy.orm import Query, Session @@ -13,6 +14,7 @@ from flexmeasures.auth.policy import AuthModelMixin, EVERY_LOGGED_IN_USER from flexmeasures.data import db from flexmeasures.data.models.parsing_utils import parse_source_arg +from flexmeasures.data.services.annotations import prepare_annotations_for_chart from flexmeasures.data.queries.utils import ( create_beliefs_query, get_belief_timing_criteria, @@ -27,6 +29,7 @@ from flexmeasures.data.models.annotations import ( Annotation, SensorAnnotationRelationship, + to_annotation_frame, ) from flexmeasures.data.models.charts import chart_type_to_chart_specs from flexmeasures.data.models.data_sources import DataSource @@ -196,26 +199,53 @@ def latest_state( def search_annotations( self, - annotation_starts_after: Optional[datetime_type] = None, - annotation_ends_before: Optional[datetime_type] = None, + annotation_starts_after: Optional[datetime_type] = None, # deprecated + annotations_after: Optional[datetime_type] = None, + annotation_ends_before: Optional[datetime_type] = None, # deprecated + annotations_before: Optional[datetime_type] = None, source: Optional[ Union[DataSource, List[DataSource], int, List[int], str, List[str]] ] = None, include_asset_annotations: bool = False, include_account_annotations: bool = False, - ): + as_frame: bool = False, + ) -> Union[List[Annotation], pd.DataFrame]: + """Return annotations assigned to this sensor, and optionally, also those assigned to the sensor's asset and the asset's account. + + :param annotations_after: only return annotations that end after this datetime (exclusive) + :param annotations_before: only return annotations that start before this datetime (exclusive) + """ + + # todo: deprecate the 'annotation_starts_after' argument in favor of 'annotations_after' (announced v0.11.0) + annotations_after = tb_utils.replace_deprecated_argument( + "annotation_starts_after", + annotation_starts_after, + "annotations_after", + annotations_after, + required_argument=False, + ) + + # todo: deprecate the 'annotation_ends_before' argument in favor of 'annotations_before' (announced v0.11.0) + annotations_before = tb_utils.replace_deprecated_argument( + "annotation_ends_before", + annotation_ends_before, + "annotations_before", + annotations_before, + required_argument=False, + ) + parsed_sources = parse_source_arg(source) query = Annotation.query.join(SensorAnnotationRelationship).filter( SensorAnnotationRelationship.sensor_id == self.id, SensorAnnotationRelationship.annotation_id == Annotation.id, ) - if annotation_starts_after is not None: + if annotations_after is not None: query = query.filter( - Annotation.start >= annotation_starts_after, + Annotation.end > annotations_after, ) - if annotation_ends_before is not None: + if annotations_before is not None: query = query.filter( - Annotation.end <= annotation_ends_before, + Annotation.start < annotations_before, ) if parsed_sources: query = query.filter( @@ -224,17 +254,18 @@ def search_annotations( annotations = query.all() if include_asset_annotations: annotations += self.generic_asset.search_annotations( - annotation_starts_before=annotation_starts_after, - annotation_ends_before=annotation_ends_before, + annotations_after=annotations_after, + annotations_before=annotations_before, source=source, ) if include_account_annotations: annotations += self.generic_asset.owner.search_annotations( - annotation_starts_before=annotation_starts_after, - annotation_ends_before=annotation_ends_before, + annotations_after=annotations_after, + annotations_before=annotations_before, source=source, ) - return annotations + + return to_annotation_frame(annotations) if as_frame else annotations def search_beliefs( self, @@ -309,10 +340,13 @@ def chart( ] = None, most_recent_beliefs_only: bool = True, include_data: bool = False, + include_sensor_annotations: bool = False, + include_asset_annotations: bool = False, + include_account_annotations: bool = False, dataset_name: Optional[str] = None, **kwargs, ) -> dict: - """Create a chart showing sensor data. + """Create a vega-lite chart showing sensor data. :param chart_type: currently only "bar_chart" # todo: where can we properly list the available chart types? :param event_starts_after: only return beliefs about events that start after this datetime (inclusive) @@ -322,7 +356,11 @@ def chart( :param source: search only beliefs by this source (pass the DataSource, or its name or id) or list of sources :param most_recent_beliefs_only: only return the most recent beliefs for each event from each source (minimum belief horizon) :param include_data: if True, include data in the chart, or if False, exclude data + :param include_sensor_annotations: if True and include_data is True, include sensor annotations in the chart, or if False, exclude these + :param include_asset_annotations: if True and include_data is True, include asset annotations in the chart, or if False, exclude them + :param include_account_annotations: if True and include_data is True, include account annotations in the chart, or if False, exclude them :param dataset_name: optionally name the dataset used in the chart (the default name is sensor_) + :returns: JSON string defining vega-lite chart specs """ # Set up chart specification @@ -335,11 +373,14 @@ def chart( chart_type, sensor=self, dataset_name=dataset_name, + include_annotations=include_sensor_annotations + or include_asset_annotations + or include_account_annotations, **kwargs, ) if include_data: - # Set up data + # Get data data = self.search_beliefs( as_json=True, event_starts_after=event_starts_after, @@ -349,8 +390,46 @@ def chart( most_recent_beliefs_only=most_recent_beliefs_only, source=source, ) - # Combine chart specs and data - chart_specs["datasets"] = {dataset_name: json.loads(data)} + + # Get annotations + if include_sensor_annotations: + annotations_df = self.search_annotations( + annotations_after=event_starts_after, + annotations_before=event_ends_before, + include_asset_annotations=include_asset_annotations, + include_account_annotations=include_account_annotations, + as_frame=True, + ) + elif include_asset_annotations: + annotations_df = self.generic_asset.search_annotations( + annotations_after=event_starts_after, + annotations_before=event_ends_before, + include_account_annotations=include_account_annotations, + as_frame=True, + ) + elif include_account_annotations: + annotations_df = self.generic_asset.owner.search_annotations( + annotations_after=event_starts_after, + annotations_before=event_ends_before, + as_frame=True, + ) + else: + annotations_df = to_annotation_frame([]) + + # Wrap and stack annotations + annotations_df = prepare_annotations_for_chart(annotations_df) + + # Annotations to JSON records + annotations_df = annotations_df.reset_index() + annotations_df["source"] = annotations_df["source"].astype(str) + annotations_data = annotations_df.to_json(orient="records") + + # Combine chart specs, data and annotations + chart_specs["datasets"] = { + dataset_name: json.loads(data), + dataset_name + "_annotations": json.loads(annotations_data), + } + return chart_specs @property diff --git a/flexmeasures/data/models/user.py b/flexmeasures/data/models/user.py index eb9243f57..ebbbaaa40 100644 --- a/flexmeasures/data/models/user.py +++ b/flexmeasures/data/models/user.py @@ -3,14 +3,17 @@ from datetime import datetime from flask_security import UserMixin, RoleMixin +import pandas as pd from sqlalchemy.orm import relationship, backref from sqlalchemy import Boolean, DateTime, Column, Integer, String, ForeignKey from sqlalchemy.ext.hybrid import hybrid_property +from timely_beliefs import utils as tb_utils from flexmeasures.data import db from flexmeasures.data.models.annotations import ( Annotation, AccountAnnotationRelationship, + to_annotation_frame, ) from flexmeasures.data.models.parsing_utils import parse_source_arg from flexmeasures.auth.policy import AuthModelMixin @@ -89,31 +92,59 @@ def has_role(self, role: Union[str, AccountRole]) -> bool: def search_annotations( self, - annotation_starts_after: Optional[datetime] = None, - annotation_ends_before: Optional[datetime] = None, + annotation_starts_after: Optional[datetime] = None, # deprecated + annotations_after: Optional[datetime] = None, + annotation_ends_before: Optional[datetime] = None, # deprecated + annotations_before: Optional[datetime] = None, source: Optional[ Union[DataSource, List[DataSource], int, List[int], str, List[str]] ] = None, - ): + as_frame: bool = False, + ) -> Union[List[Annotation], pd.DataFrame]: + """Return annotations assigned to this account. + + :param annotations_after: only return annotations that end after this datetime (exclusive) + :param annotations_before: only return annotations that start before this datetime (exclusive) + """ + + # todo: deprecate the 'annotation_starts_after' argument in favor of 'annotations_after' (announced v0.11.0) + annotations_after = tb_utils.replace_deprecated_argument( + "annotation_starts_after", + annotation_starts_after, + "annotations_after", + annotations_after, + required_argument=False, + ) + + # todo: deprecate the 'annotation_ends_before' argument in favor of 'annotations_before' (announced v0.11.0) + annotations_before = tb_utils.replace_deprecated_argument( + "annotation_ends_before", + annotation_ends_before, + "annotations_before", + annotations_before, + required_argument=False, + ) + parsed_sources = parse_source_arg(source) query = Annotation.query.join(AccountAnnotationRelationship).filter( AccountAnnotationRelationship.account_id == self.id, AccountAnnotationRelationship.annotation_id == Annotation.id, ) - if annotation_starts_after is not None: + if annotations_after is not None: query = query.filter( - Annotation.start >= annotation_starts_after, + Annotation.end > annotations_after, ) - if annotation_ends_before is not None: + if annotations_before is not None: query = query.filter( - Annotation.end <= annotation_ends_before, + Annotation.start < annotations_before, ) if parsed_sources: query = query.filter( Annotation.source.in_(parsed_sources), ) annotations = query.all() - return annotations + + return to_annotation_frame(annotations) if as_frame else annotations class RolesUsers(db.Model): diff --git a/flexmeasures/data/queries/annotations.py b/flexmeasures/data/queries/annotations.py index daf53ad4a..a9b344a8b 100644 --- a/flexmeasures/data/queries/annotations.py +++ b/flexmeasures/data/queries/annotations.py @@ -12,8 +12,8 @@ def query_asset_annotations( asset_id: int, - annotation_starts_after: Optional[datetime] = None, - annotation_ends_before: Optional[datetime] = None, + annotations_after: Optional[datetime] = None, + annotations_before: Optional[datetime] = None, sources: Optional[List[DataSource]] = None, annotation_type: str = None, ) -> Query: @@ -22,13 +22,13 @@ def query_asset_annotations( GenericAssetAnnotationRelationship.generic_asset_id == asset_id, GenericAssetAnnotationRelationship.annotation_id == Annotation.id, ) - if annotation_starts_after is not None: + if annotations_after is not None: query = query.filter( - Annotation.start >= annotation_starts_after, + Annotation.end > annotations_after, ) - if annotation_ends_before is not None: + if annotations_before is not None: query = query.filter( - Annotation.end <= annotation_ends_before, + Annotation.start < annotations_before, ) if sources: query = query.filter( diff --git a/flexmeasures/data/services/annotations.py b/flexmeasures/data/services/annotations.py new file mode 100644 index 000000000..d13db3608 --- /dev/null +++ b/flexmeasures/data/services/annotations.py @@ -0,0 +1,45 @@ +from datetime import datetime +from itertools import chain +from textwrap import wrap +from typing import Optional + +import pandas as pd + + +def stack_annotations(x: pd.DataFrame) -> pd.DataFrame: + """Select earliest start, and include all annotations as a list. + + The list of strings results in a multi-line text encoding in the chart. + """ + x = x.sort_values(["start", "belief_time"], ascending=True) + x["content"].iloc[0] = list(chain(*(x["content"].tolist()))) + return x.head(1) + + +def prepare_annotations_for_chart( + df: pd.DataFrame, + event_starts_after: Optional[datetime] = None, + event_ends_before: Optional[datetime] = None, + max_line_length: int = 60, +) -> pd.DataFrame: + """Prepare a DataFrame with annotations for use in a chart. + + - Clips annotations outside the requested time window. + - Wraps on whitespace with a given max line length + - Stacks annotations for the same event + """ + + # Clip annotations outside the requested time window + if event_starts_after is not None: + df.loc[df["start"] < event_starts_after, "start"] = event_starts_after + if event_ends_before is not None: + df.loc[df["end"] > event_ends_before, "end"] = event_ends_before + + # Wrap on whitespace with some max line length + df["content"] = df["content"].apply(wrap, args=[max_line_length]) + + # Stack annotations for the same event + if not df.empty: + df = df.groupby("end", group_keys=False).apply(stack_annotations) + + return df diff --git a/flexmeasures/ui/static/css/flexmeasures.css b/flexmeasures/ui/static/css/flexmeasures.css index 8c318ca09..e46295b37 100644 --- a/flexmeasures/ui/static/css/flexmeasures.css +++ b/flexmeasures/ui/static/css/flexmeasures.css @@ -1062,6 +1062,13 @@ body .dataTables_wrapper .dataTables_paginate .paginate_button.current, body .da margin-bottom: 40px; } +#vg-tooltip-element { + font-size: 16px; +} +#vg-tooltip-element table tr td.value { + max-height: inherit !important; +} + /* --- End Sensor Data --- */ diff --git a/flexmeasures/ui/templates/views/sensors.html b/flexmeasures/ui/templates/views/sensors.html index 88cbdf3dd..3238e55f5 100644 --- a/flexmeasures/ui/templates/views/sensors.html +++ b/flexmeasures/ui/templates/views/sensors.html @@ -36,7 +36,7 @@ async function embedAndLoad(chartSpecsPath, elementId, datasetName) { - await vegaEmbed('#'+elementId, chartSpecsPath + '?dataset_name=' + datasetName + '&width=container', {{ chart_options | safe }}) + await vegaEmbed('#'+elementId, chartSpecsPath + '?dataset_name=' + datasetName + '&width=container&include_sensor_annotations=true&include_asset_annotations=true', {{ chart_options | safe }}) .then(function (result) { // result.view is the Vega View, chartSpecsPath is the original Vega-Lite specification vegaView = result.view; @@ -85,6 +85,8 @@ endDate.setDate(endDate.getDate() + 1); var queryStartDate = (startDate != null) ? (startDate.toISOString()) : (null) var queryEndDate = (endDate != null) ? (endDate.toISOString()) : (null) + + // Fetch time series data fetch(sensorPath + '/chart_data/?event_starts_after=' + queryStartDate + '&event_ends_before=' + queryEndDate, { method: "GET", headers: {"Content-Type": "application/json"}, @@ -92,6 +94,16 @@ .then(function(response) { return response.json(); }) .then(function(data) { vegaView.change(datasetName, vega.changeset().remove(vega.truthy).insert(data)).resize().run(); + }) + + // Fetch annotations + fetch(sensorPath + '/chart_annotations/?event_starts_after=' + queryStartDate + '&event_ends_before=' + queryEndDate, { + method: "GET", + headers: {"Content-Type": "application/json"}, + }) + .then(function(response) { return response.json(); }) + .then(function(data) { + vegaView.change(datasetName + '_annotations', vega.changeset().remove(vega.truthy).insert(data)).resize().run(); }); }); diff --git a/flexmeasures/ui/views/sensors.py b/flexmeasures/ui/views/sensors.py index e8cd4836d..016fd168c 100644 --- a/flexmeasures/ui/views/sensors.py +++ b/flexmeasures/ui/views/sensors.py @@ -30,6 +30,9 @@ class SensorUI(FlaskView): "event_ends_before": AwareDateTimeField(format="iso", required=False), "beliefs_after": AwareDateTimeField(format="iso", required=False), "beliefs_before": AwareDateTimeField(format="iso", required=False), + "include_sensor_annotations": fields.Bool(required=False), + "include_asset_annotations": fields.Bool(required=False), + "include_account_annotations": fields.Bool(required=False), "dataset_name": fields.Str(required=False), "chart_theme": fields.Str(required=False), }, @@ -58,7 +61,7 @@ def get_chart(self, id, **kwargs): "vegalite" ], embed_options=embed_options, - ) + ).replace('
', '
') @login_required def get(self, id: int): diff --git a/flexmeasures/utils/config_defaults.py b/flexmeasures/utils/config_defaults.py index 43d833778..a04611f09 100644 --- a/flexmeasures/utils/config_defaults.py +++ b/flexmeasures/utils/config_defaults.py @@ -122,9 +122,9 @@ class Config(object): FLEXMEASURES_REDIS_DB_NR: int = 0 # Redis per default has 16 databases, [0-15] FLEXMEASURES_REDIS_PASSWORD: Optional[str] = None FLEXMEASURES_JS_VERSIONS: dict = dict( - vega="5", - vegaembed="6.17.0", - vegalite="5.0.0", + vega="5.22.1", + vegaembed="6.20.8", + vegalite="5.2.0", # todo: expand with other js versions used in FlexMeasures )