diff --git a/flexmeasures/api/__init__.py b/flexmeasures/api/__init__.py index ef838271c..32b008f51 100644 --- a/flexmeasures/api/__init__.py +++ b/flexmeasures/api/__init__.py @@ -105,9 +105,11 @@ def register_at(app: Flask): from flexmeasures.api.v1_2 import register_at as v1_2_register_at from flexmeasures.api.v1_3 import register_at as v1_3_register_at from flexmeasures.api.v2_0 import register_at as v2_0_register_at + from flexmeasures.api.dev import register_at as dev_register_at v1_register_at(app) v1_1_register_at(app) v1_2_register_at(app) v1_3_register_at(app) v2_0_register_at(app) + dev_register_at(app) diff --git a/flexmeasures/api/common/schemas/times.py b/flexmeasures/api/common/schemas/times.py index 8f2108ee8..a9bff0d07 100644 --- a/flexmeasures/api/common/schemas/times.py +++ b/flexmeasures/api/common/schemas/times.py @@ -62,3 +62,16 @@ def ground_from( ) return (pd.Timestamp(start) + offset).to_pydatetime() - start return duration + + +class AwareDateTimeField(fields.AwareDateTime): + """Field that deserializes to a timezone aware datetime + and serializes back to a string.""" + + def _deserialize(self, value: str, attr, obj, **kwargs) -> datetime: + """ + Work-around until this PR lands: + https://github.com/marshmallow-code/marshmallow/pull/1787 + """ + value = value.replace(" ", "+") + return fields.AwareDateTime._deserialize(self, value, attr, obj, **kwargs) diff --git a/flexmeasures/api/dev/__init__.py b/flexmeasures/api/dev/__init__.py new file mode 100644 index 000000000..c175be741 --- /dev/null +++ b/flexmeasures/api/dev/__init__.py @@ -0,0 +1,9 @@ +from flask import Flask + + +def register_at(app: Flask): + """This can be used to register FlaskViews.""" + + from flexmeasures.api.dev.sensors import SensorAPI + + SensorAPI.register(app, route_prefix="/api/dev") diff --git a/flexmeasures/api/dev/sensors.py b/flexmeasures/api/dev/sensors.py new file mode 100644 index 000000000..ca6794159 --- /dev/null +++ b/flexmeasures/api/dev/sensors.py @@ -0,0 +1,74 @@ +import json + +from flask_classful import FlaskView, route +from flask_login import login_required +from flask_security import roles_required +from marshmallow import fields +from webargs.flaskparser import use_kwargs +from werkzeug.exceptions import abort + +from flexmeasures.api.common.schemas.times import AwareDateTimeField +from flexmeasures.data.models.time_series import Sensor + + +class SensorAPI(FlaskView): + """ + This view exposes sensor attributes through API endpoints under development. + These endpoints are not yet part of our official API, but support the FlexMeasures UI. + """ + + route_base = "/sensor" + + @login_required + @roles_required("admin") # todo: remove after we check for sensor ownership + @route("//chart/") + @use_kwargs( + { + "event_starts_after": AwareDateTimeField(format="iso", required=False), + "event_ends_before": AwareDateTimeField(format="iso", required=False), + "beliefs_after": AwareDateTimeField(format="iso", required=False), + "beliefs_before": AwareDateTimeField(format="iso", required=False), + "include_data": fields.Boolean(required=False), + "dataset_name": fields.Str(required=False), + }, + location="query", + ) + def get_chart(self, id, **kwargs): + """GET from /sensor//chart""" + sensor = get_sensor_or_abort(id) + return json.dumps(sensor.chart(**kwargs)) + + @login_required + @roles_required("admin") # todo: remove after we check for sensor ownership + @route("//chart_data/") + @use_kwargs( + { + "event_starts_after": AwareDateTimeField(format="iso", required=False), + "event_ends_before": AwareDateTimeField(format="iso", required=False), + "beliefs_after": AwareDateTimeField(format="iso", required=False), + "beliefs_before": AwareDateTimeField(format="iso", required=False), + }, + location="query", + ) + def get_chart_data(self, id, **kwargs): + """GET from /sensor//chart_data + + Data for use in charts (in case you have the chart specs already). + """ + sensor = get_sensor_or_abort(id) + return sensor.search_beliefs(as_json=True, **kwargs) + + @login_required + @roles_required("admin") # todo: remove after we check for sensor ownership + def get(self, id: int): + """GET from /sensor/""" + sensor = get_sensor_or_abort(id) + attributes = ["name", "timezone", "timerange"] + return {attr: getattr(sensor, attr) for attr in attributes} + + +def get_sensor_or_abort(id: int) -> Sensor: + sensor = Sensor.query.filter(Sensor.id == id).one_or_none() + if sensor is None: + raise abort(404, f"Sensor {id} not found") + return sensor diff --git a/flexmeasures/api/v2_0/implementations/assets.py b/flexmeasures/api/v2_0/implementations/assets.py index d4fb8f79e..0d05533f0 100644 --- a/flexmeasures/api/v2_0/implementations/assets.py +++ b/flexmeasures/api/v2_0/implementations/assets.py @@ -66,7 +66,7 @@ def load_asset(admins_only: bool = False): should be allowed. @app.route('/asset/') - @check_asset + @load_asset def get_asset(asset): return asset_schema.dump(asset), 200 diff --git a/flexmeasures/data/models/charts/__init__.py b/flexmeasures/data/models/charts/__init__.py new file mode 100644 index 000000000..a25bd3cf3 --- /dev/null +++ b/flexmeasures/data/models/charts/__init__.py @@ -0,0 +1,26 @@ +from inspect import getmembers, isfunction + +from . import belief_charts +from .defaults import apply_chart_defaults + + +def chart_type_to_chart_specs(chart_type: str, **kwargs) -> dict: + """Create chart specs of a given chart type, using FlexMeasures defaults for settings like width and height. + + :param chart_type: Name of a variable defining chart specs or a function returning chart specs. + The chart specs can be a dictionary or an Altair chart specification. + - In case of a dictionary, the creator needs to ensure that the dictionary contains valid specs + - In case of an Altair chart specification, Altair validates for you + :returns: A dictionary containing a vega-lite chart specification + """ + # Create a dictionary mapping chart types to chart specs, and apply defaults to the chart specs, too. + belief_charts_mapping = { + chart_type: apply_chart_defaults(chart_specs) + for chart_type, chart_specs in getmembers(belief_charts) + if isfunction(chart_specs) or isinstance(chart_specs, dict) + } + # Create chart specs + chart_specs_or_fnc = belief_charts_mapping[chart_type] + if isfunction(chart_specs_or_fnc): + return chart_specs_or_fnc(**kwargs) + return chart_specs_or_fnc diff --git a/flexmeasures/data/models/charts/belief_charts.py b/flexmeasures/data/models/charts/belief_charts.py new file mode 100644 index 000000000..e9c810de0 --- /dev/null +++ b/flexmeasures/data/models/charts/belief_charts.py @@ -0,0 +1,27 @@ +from flexmeasures.data.models.charts.defaults import TIME_TITLE, TIME_TOOLTIP_TITLE + + +def bar_chart(title: str, quantity: str = "unknown quantity", unit: str = "a.u."): + if not unit: + unit = "a.u." + return { + "description": "A simple bar chart.", + "title": title, + "mark": "bar", + "encoding": { + "x": {"field": "event_start", "type": "T", "title": TIME_TITLE}, + "y": { + "field": "event_value", + "type": "quantitative", + "title": quantity + " (" + unit + ")", + }, + "tooltip": [ + {"field": "full_date", "title": TIME_TOOLTIP_TITLE, "type": "nominal"}, + { + "field": "event_value", + "title": quantity + " (" + unit + ")", + "type": "quantitative", + }, + ], + }, + } diff --git a/flexmeasures/data/models/charts/defaults.py b/flexmeasures/data/models/charts/defaults.py new file mode 100644 index 000000000..4d979c5a6 --- /dev/null +++ b/flexmeasures/data/models/charts/defaults.py @@ -0,0 +1,42 @@ +from functools import wraps +from typing import Callable, Union + +import altair as alt + + +HEIGHT = 300 +WIDTH = 600 +REDUCED_HEIGHT = REDUCED_WIDTH = 60 +SELECTOR_COLOR = "darkred" +TIME_FORMAT = "%I:%M %p on %A %b %e, %Y" +TIME_TOOLTIP_TITLE = "Time and date" +TIME_TITLE = None +TIME_SELECTION_TOOLTIP = "Click and drag to select a time window" + + +def apply_chart_defaults(fn): + @wraps(fn) + def decorated_chart_specs(*args, **kwargs): + dataset_name = kwargs.pop("dataset_name", None) + if isinstance(fn, Callable): + # function that returns a chart specification + chart_specs: Union[dict, alt.TopLevelMixin] = fn(*args, **kwargs) + else: + # not a function, but a direct chart specification + chart_specs: Union[dict, alt.TopLevelMixin] = fn + if isinstance(chart_specs, alt.TopLevelMixin): + chart_specs = chart_specs.to_dict() + chart_specs.pop("$schema") + if dataset_name: + chart_specs["data"] = {"name": dataset_name} + chart_specs["height"] = HEIGHT + chart_specs["width"] = WIDTH + chart_specs["transform"] = [ + { + "as": "full_date", + "calculate": f"timeFormat(datum.event_start, '{TIME_FORMAT}')", + } + ] + return chart_specs + + return decorated_chart_specs diff --git a/flexmeasures/data/models/charts/readme.md b/flexmeasures/data/models/charts/readme.md new file mode 100644 index 000000000..f918ff80f --- /dev/null +++ b/flexmeasures/data/models/charts/readme.md @@ -0,0 +1,7 @@ +# Developer docs for adding chart specs + +Chart specs can be specified as a dictionary with a vega-lite specification or as an altair chart. +Alternatively, they can be specified as a function that returns a dict (with vega-lite specs) or an altair chart. +This approach is useful if you need to parameterize the specification with kwargs. + +Todo: support a plug-in architecture, see https://packaging.python.org/guides/creating-and-discovering-plugins/ diff --git a/flexmeasures/data/models/time_series.py b/flexmeasures/data/models/time_series.py index 873e7088b..aafdc5542 100644 --- a/flexmeasures/data/models/time_series.py +++ b/flexmeasures/data/models/time_series.py @@ -1,5 +1,6 @@ from typing import List, Dict, Optional, Union, Tuple from datetime import datetime as datetime_type, timedelta +import json from sqlalchemy.ext.declarative import declared_attr from sqlalchemy.orm import Query, Session @@ -17,6 +18,9 @@ exclude_source_type_filter, ) from flexmeasures.data.services.time_series import collect_time_series_data +from flexmeasures.data.models.charts import chart_type_to_chart_specs +from flexmeasures.utils.time_utils import server_now +from flexmeasures.utils.flexmeasures_inflection import capitalize class Sensor(db.Model, tb.SensorDBMixin): @@ -29,27 +33,116 @@ def __init__(self, name: str, **kwargs): def search_beliefs( self, - event_time_window: Tuple[Optional[datetime_type], Optional[datetime_type]] = ( - None, - None, - ), - belief_time_window: Tuple[Optional[datetime_type], Optional[datetime_type]] = ( - None, - None, - ), + event_starts_after: Optional[datetime_type] = None, + event_ends_before: Optional[datetime_type] = None, + beliefs_after: Optional[datetime_type] = None, + beliefs_before: Optional[datetime_type] = None, source: Optional[Union[int, List[int], str, List[str]]] = None, + as_json: bool = False, ): """Search all beliefs about events for this sensor. - :param event_time_window: search only events within this time window - :param belief_time_window: search only beliefs within this time window - :param source: search only beliefs by this source (pass its name or id) or list of sources""" - return TimedBelief.search( + :param event_starts_after: only return beliefs about events that start after this datetime (inclusive) + :param event_ends_before: only return beliefs about events that end before this datetime (inclusive) + :param beliefs_after: only return beliefs formed after this datetime (inclusive) + :param beliefs_before: only return beliefs formed before this datetime (inclusive) + :param source: search only beliefs by this source (pass its name or id) or list of sources + :param as_json: return beliefs in JSON format (e.g. for use in charts) rather than as BeliefsDataFrame + """ + bdf = TimedBelief.search( sensor=self, - event_time_window=event_time_window, - belief_time_window=belief_time_window, + event_starts_after=event_starts_after, + event_ends_before=event_ends_before, + beliefs_after=beliefs_after, + beliefs_before=beliefs_before, source=source, ) + if as_json: + df = bdf.reset_index() + df["source"] = df["source"].apply(lambda x: x.name) + return df.to_json(orient="records") + return bdf + + def chart( + self, + chart_type: str = "bar_chart", + event_starts_after: Optional[datetime_type] = None, + event_ends_before: Optional[datetime_type] = None, + beliefs_after: Optional[datetime_type] = None, + beliefs_before: Optional[datetime_type] = None, + source: Optional[Union[int, List[int], str, List[str]]] = None, + include_data: bool = False, + dataset_name: Optional[str] = None, + **kwargs, + ) -> dict: + """Create a chart showing sensor data. + + :param chart_type: currently only "bar_chart" # todo: where can we properly list the available chart types? + :param event_starts_after: only return beliefs about events that start after this datetime (inclusive) + :param event_ends_before: only return beliefs about events that end before this datetime (inclusive) + :param beliefs_after: only return beliefs formed after this datetime (inclusive) + :param beliefs_before: only return beliefs formed before this datetime (inclusive) + :param source: search only beliefs by this source (pass its name or id) or list of sources + :param include_data: if True, include data in the chart, or if False, exclude data + :param dataset_name: optionally name the dataset used in the chart (the default name is sensor_) + """ + + # Set up chart specification + if dataset_name is None: + dataset_name = "sensor_" + str(self.id) + self.sensor_type = ( + self.name + ) # todo remove this placeholder when sensor types are modelled + chart_specs = chart_type_to_chart_specs( + chart_type, + title=capitalize(self.name), + quantity=capitalize(self.sensor_type), + unit=self.unit, + dataset_name=dataset_name, + **kwargs, + ) + + if include_data: + # Set up data + data = self.search_beliefs( + as_json=True, + event_starts_after=event_starts_after, + event_ends_before=event_ends_before, + beliefs_after=beliefs_after, + beliefs_before=beliefs_before, + source=source, + ) + # Combine chart specs and data + chart_specs["datasets"] = {dataset_name: json.loads(data)} + return chart_specs + + @property + def timerange(self) -> Dict[str, datetime_type]: + """Timerange for which sensor data exists. + + :returns: dictionary with start and end, for example: + { + 'start': datetime.datetime(2020, 12, 3, 14, 0, tzinfo=pytz.utc), + 'end': datetime.datetime(2020, 12, 3, 14, 30, tzinfo=pytz.utc) + } + """ + least_recent_query = ( + TimedBelief.query.filter(TimedBelief.sensor == self) + .order_by(TimedBelief.event_start.asc()) + .limit(1) + ) + most_recent_query = ( + TimedBelief.query.filter(TimedBelief.sensor == self) + .order_by(TimedBelief.event_start.desc()) + .limit(1) + ) + results = least_recent_query.union_all(most_recent_query).all() + if not results: + # return now in case there is no data for the sensor + now = server_now() + return dict(start=now, end=now) + least_recent, most_recent = results + return dict(start=least_recent.event_start, end=most_recent.event_end) def __repr__(self) -> str: return f"" @@ -82,30 +175,28 @@ def __init__( def search( cls, sensor: Sensor, - event_time_window: Tuple[Optional[datetime_type], Optional[datetime_type]] = ( - None, - None, - ), - belief_time_window: Tuple[Optional[datetime_type], Optional[datetime_type]] = ( - None, - None, - ), + event_starts_after: Optional[datetime_type] = None, + event_ends_before: Optional[datetime_type] = None, + beliefs_after: Optional[datetime_type] = None, + beliefs_before: Optional[datetime_type] = None, source: Optional[Union[int, List[int], str, List[str]]] = None, ) -> tb.BeliefsDataFrame: """Search all beliefs about events for a given sensor. :param sensor: search only this sensor - :param event_time_window: search only events within this time window - :param belief_time_window: search only beliefs within this time window + :param event_starts_after: only return beliefs about events that start after this datetime (inclusive) + :param event_ends_before: only return beliefs about events that end before this datetime (inclusive) + :param beliefs_after: only return beliefs formed after this datetime (inclusive) + :param beliefs_before: only return beliefs formed before this datetime (inclusive) :param source: search only beliefs by this source (pass its name or id) or list of sources """ return cls.search_session( session=db.session, sensor=sensor, - event_before=event_time_window[1], - event_not_before=event_time_window[0], - belief_before=belief_time_window[1], - belief_not_before=belief_time_window[0], + event_starts_after=event_starts_after, + event_ends_before=event_ends_before, + beliefs_after=beliefs_after, + beliefs_before=beliefs_before, source=source, ) diff --git a/flexmeasures/ui/__init__.py b/flexmeasures/ui/__init__.py index 7e4d2681c..6348aa578 100644 --- a/flexmeasures/ui/__init__.py +++ b/flexmeasures/ui/__init__.py @@ -33,9 +33,11 @@ def register_at(app: Flask): from flexmeasures.ui.crud.assets import AssetCrudUI from flexmeasures.ui.crud.users import UserCrudUI + from flexmeasures.ui.views.sensors import SensorUI AssetCrudUI.register(app) UserCrudUI.register(app) + SensorUI.register(app) import flexmeasures.ui.views # noqa: F401 this is necessary to load the views diff --git a/flexmeasures/ui/static/js/flexmeasures.js b/flexmeasures/ui/static/js/flexmeasures.js index 8dd0becba..66e00404c 100644 --- a/flexmeasures/ui/static/js/flexmeasures.js +++ b/flexmeasures/ui/static/js/flexmeasures.js @@ -310,3 +310,24 @@ function submit_market() { function submit_sensor_type() { $("#sensor_type-form").attr("action", empty_location).submit(); } + + +// Date range utils +export function subtract(oldDate, nDays) { + var newDate = new Date(oldDate) + newDate.setDate(newDate.getDate() - nDays); + return newDate; +} +export function thisMonth(oldDate) { + var d1 = new Date(oldDate) + d1.setDate(1); + var d2 = new Date(d1.getFullYear(), d1.getMonth() + 1, 0); + return [d1, d2]; +}; +export function lastNMonths(oldDate, nMonths) { + var d0 = new Date(oldDate) + var d1 = new Date(d0.getFullYear(), d0.getMonth() - nMonths + 2, 0); + d1.setDate(1); + var d2 = new Date(d0.getFullYear(), d0.getMonth() + 1, 0); + return [d1, d2]; +}; diff --git a/flexmeasures/ui/templates/base.html b/flexmeasures/ui/templates/base.html index 2bfd0be51..328f35830 100644 --- a/flexmeasures/ui/templates/base.html +++ b/flexmeasures/ui/templates/base.html @@ -209,7 +209,7 @@ - + {% if show_datepicker %} {% endif %} diff --git a/flexmeasures/ui/templates/views/sensors.html b/flexmeasures/ui/templates/views/sensors.html new file mode 100644 index 000000000..2800d5f8c --- /dev/null +++ b/flexmeasures/ui/templates/views/sensors.html @@ -0,0 +1,137 @@ +{% extends "base.html" %} + +{% set active_page = "assets" %} + +{% block title %} Assets {% endblock %} + +{% block divs %} + +
+
+
+
+
+ + + + + + + + + + + + + + +{% endblock %} \ No newline at end of file diff --git a/flexmeasures/ui/utils/view_utils.py b/flexmeasures/ui/utils/view_utils.py index 1b50a9caa..50f9674f2 100644 --- a/flexmeasures/ui/utils/view_utils.py +++ b/flexmeasures/ui/utils/view_utils.py @@ -84,6 +84,7 @@ def render_flexmeasures_template(html_filename: str, **variables): variables["user_name"] = ( current_user.is_authenticated and current_user.username or "" ) + variables["js_versions"] = current_app.config.get("FLEXMEASURES_JS_VERSIONS") return render_template(html_filename, **variables) diff --git a/flexmeasures/ui/views/sensors.py b/flexmeasures/ui/views/sensors.py new file mode 100644 index 000000000..c9e4eafd0 --- /dev/null +++ b/flexmeasures/ui/views/sensors.py @@ -0,0 +1,60 @@ +from altair.utils.html import spec_to_html +from flask import current_app +from flask_classful import FlaskView, route +from flask_security import login_required, roles_required +from marshmallow import fields +from webargs.flaskparser import use_kwargs + +from flexmeasures.api.common.schemas.times import AwareDateTimeField +from flexmeasures.api.dev.sensors import SensorAPI +from flexmeasures.ui.utils.view_utils import render_flexmeasures_template + + +class SensorUI(FlaskView): + """ + This view creates several new UI endpoints for viewing sensors. + + todo: consider extending this view for crud purposes + """ + + route_base = "/sensors" + + @login_required + @roles_required("admin") # todo: remove after we check for sensor ownership + @route("//chart/") + @use_kwargs( + { + "event_starts_after": AwareDateTimeField(format="iso", required=False), + "event_ends_before": AwareDateTimeField(format="iso", required=False), + "beliefs_after": AwareDateTimeField(format="iso", required=False), + "beliefs_before": AwareDateTimeField(format="iso", required=False), + "dataset_name": fields.Str(required=False), + }, + location="query", + ) + def get_chart(self, id, **kwargs): + """GET from /sensors//chart""" + chart_specs = SensorAPI().get_chart( + id, include_data=True, as_html=True, **kwargs + ) + return spec_to_html( + chart_specs, + "vega-lite", + vega_version=current_app.config.get("FLEXMEASURES_JS_VERSIONS").vega, + vegaembed_version=current_app.config.get( + "FLEXMEASURES_JS_VERSIONS" + ).vegaembed, + vegalite_version=current_app.config.get( + "FLEXMEASURES_JS_VERSIONS" + ).vegalite, + ) + + @login_required + @roles_required("admin") # todo: remove after we check for sensor ownership + def get(self, id: int): + """GET from /sensors/""" + return render_flexmeasures_template( + "views/sensors.html", + sensor_id=id, + msg="", + ) diff --git a/flexmeasures/utils/config_defaults.py b/flexmeasures/utils/config_defaults.py index 995bf9d7f..685736d0f 100644 --- a/flexmeasures/utils/config_defaults.py +++ b/flexmeasures/utils/config_defaults.py @@ -105,6 +105,12 @@ class Config(object): FLEXMEASURES_REDIS_PORT: int = 6379 FLEXMEASURES_REDIS_DB_NR: int = 0 # Redis per default has 16 databases, [0-15] FLEXMEASURES_REDIS_PASSWORD: Optional[str] = None + FLEXMEASURES_JS_VERSIONS: dict = dict( + vega="5", + vegaembed="6.17.0", + vegalite="5.0.0", + # todo: expand with other js versions used in FlexMeasures + ) # names of settings which cannot be None diff --git a/requirements/app.in b/requirements/app.in index bd0450c63..82dd94e72 100644 --- a/requirements/app.in +++ b/requirements/app.in @@ -1,4 +1,5 @@ # see ui/utils/plotting_utils: separate_legend() and create_hover_tool() +altair bokeh==1.0.4 colour pscript