From a8a028ae57d7d59b2a45492d371e05520d296381 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sun, 4 Apr 2021 14:36:15 +0200 Subject: [PATCH 01/25] Add CLI command to add beliefs from csv --- flexmeasures/data/models/data_sources.py | 2 +- .../data/scripts/cli_tasks/data_add.py | 83 ++++++++++++++++++- requirements/app.in | 2 +- 3 files changed, 83 insertions(+), 4 deletions(-) diff --git a/flexmeasures/data/models/data_sources.py b/flexmeasures/data/models/data_sources.py index 430dc4de9..0ebed3eb5 100644 --- a/flexmeasures/data/models/data_sources.py +++ b/flexmeasures/data/models/data_sources.py @@ -46,7 +46,7 @@ def label(self): return f"schedule by {self.name}" elif self.type == "crawling script": return f"data retrieved from {self.name}" - elif self.type == "demo script": + elif self.type in ("demo script", "CLI script"): return f"demo data entered by {self.name}" else: return f"data from {self.name}" diff --git a/flexmeasures/data/scripts/cli_tasks/data_add.py b/flexmeasures/data/scripts/cli_tasks/data_add.py index ff8b7e4ed..bbf88063e 100644 --- a/flexmeasures/data/scripts/cli_tasks/data_add.py +++ b/flexmeasures/data/scripts/cli_tasks/data_add.py @@ -1,7 +1,7 @@ """CLI Tasks for (de)populating the database - most useful in development""" from datetime import timedelta -from typing import List +from typing import List, Optional import pandas as pd import pytz @@ -10,13 +10,17 @@ from flask_security.utils import hash_password import click import getpass +import timely_beliefs as tb +from flexmeasures.data import db from flexmeasures.data.services.forecasting import create_forecasting_jobs from flexmeasures.data.services.users import create_user -from flexmeasures.data.models.time_series import Sensor, SensorSchema +from flexmeasures.data.models.time_series import Sensor, SensorSchema, TimedBelief from flexmeasures.data.models.assets import Asset, AssetSchema from flexmeasures.data.models.markets import Market from flexmeasures.data.models.weather import WeatherSensor, WeatherSensorSchema +from flexmeasures.data.models.data_sources import DataSource +from flexmeasures.utils.time_utils import server_now @click.group("add") @@ -201,6 +205,81 @@ def add_initial_structure(): populate_structure(app.db) +@fm_add_data.command("beliefs") +@with_appcontext +@click.argument("file", type=click.Path(exists=True)) +@click.option( + "--sensor-id", + required=True, + type=click.IntRange(min=1), + help="Sensor to which the beliefs pertain.", +) +@click.option( + "--horizon", + required=False, + type=click.IntRange(), + help="Belief horizon in minutes (use postive horizon for ex-ante beliefs or negative horizon for ex-post beliefs).", +) +@click.option( + "--cp", + required=False, + type=click.FloatRange(0, 1), + help="Cumulative probability in the range [0, 1].", +) +def add_beliefs( + file: str, sensor_id: int, horizon: Optional[int] = None, cp: Optional[float] = None +): + """Add sensor data from a csv file. + + Structure your csv file as follows: + + - One header line (will be ignored!) + - UTC datetimes in 1st column + - values in 2nd column + + For example: + + Date,Inflow (cubic meter) + 2020-12-03 14:00,212 + 2020-12-03 14:10,215.6 + 2020-12-03 14:20,203.8 + + """ + sensor = Sensor.query.filter(Sensor.id == sensor_id).one_or_none() + source = ( + DataSource.query.filter(DataSource.name == "Seita") + .filter(DataSource.type == "CLI script") + .one_or_none() + ) + if not source: + print("SETTING UP CLI SCRIPT AS NEW DATA SOURCE...") + source = DataSource(name="Seita", type="CLI script") + db.session.add(source) + if horizon is not None: + bdf = tb.read_csv( + file, + sensor, + source, + belief_horizon=timedelta(minutes=horizon), + cumulative_probability=cp, + parse_dates=True, + infer_datetime_format=True, + ) + else: + bdf = tb.read_csv( + file, + sensor, + source, + belief_time=server_now().astimezone(pytz.timezone(sensor.timezone)), + cumulative_probability=cp, + parse_dates=True, + infer_datetime_format=True, + ) + TimedBelief.add(bdf, commit_transaction=False) + db.session.commit() + print(f"Successfully created beliefs\n{bdf}") + + @fm_add_data.command("forecasts") @with_appcontext @click.option( diff --git a/requirements/app.in b/requirements/app.in index 09d7da32f..6a858c250 100644 --- a/requirements/app.in +++ b/requirements/app.in @@ -32,7 +32,7 @@ netCDF4 siphon tables timetomodel>=0.6.8 -timely-beliefs>=1.3.0 +timely-beliefs>=1.3.1 python-dotenv # a backport, not needed in Python3.8 importlib_metadata From 1eb8f5c6b3b2715090d9ce5590c7a9140470d9da Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sun, 4 Apr 2021 14:51:23 +0200 Subject: [PATCH 02/25] Refactor if-else block --- .../data/scripts/cli_tasks/data_add.py | 35 ++++++++----------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/flexmeasures/data/scripts/cli_tasks/data_add.py b/flexmeasures/data/scripts/cli_tasks/data_add.py index bbf88063e..51cc7653c 100644 --- a/flexmeasures/data/scripts/cli_tasks/data_add.py +++ b/flexmeasures/data/scripts/cli_tasks/data_add.py @@ -255,26 +255,21 @@ def add_beliefs( print("SETTING UP CLI SCRIPT AS NEW DATA SOURCE...") source = DataSource(name="Seita", type="CLI script") db.session.add(source) - if horizon is not None: - bdf = tb.read_csv( - file, - sensor, - source, - belief_horizon=timedelta(minutes=horizon), - cumulative_probability=cp, - parse_dates=True, - infer_datetime_format=True, - ) - else: - bdf = tb.read_csv( - file, - sensor, - source, - belief_time=server_now().astimezone(pytz.timezone(sensor.timezone)), - cumulative_probability=cp, - parse_dates=True, - infer_datetime_format=True, - ) + bdf = tb.read_csv( + file, + sensor, + source=source, + cumulative_probability=cp, + parse_dates=True, + infer_datetime_format=True, + **( + dict(belief_horizon=timedelta(minutes=horizon)) + if horizon is not None + else dict( + belief_time=server_now().astimezone(pytz.timezone(sensor.timezone)) + ) + ), + ) TimedBelief.add(bdf, commit_transaction=False) db.session.commit() print(f"Successfully created beliefs\n{bdf}") From f9548a0dfb815efa951667c2cdd478d64dd80033 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 9 Apr 2021 17:43:43 +0200 Subject: [PATCH 03/25] Add API functions to chart sensor data --- flexmeasures/api/common/schemas/times.py | 13 ++ flexmeasures/data/models/time_series.py | 79 +++++++++ flexmeasures/ui/__init__.py | 2 + flexmeasures/ui/templates/views/sensors.html | 163 +++++++++++++++++++ flexmeasures/ui/views/sensors.py | 54 ++++++ 5 files changed, 311 insertions(+) create mode 100644 flexmeasures/ui/templates/views/sensors.html create mode 100644 flexmeasures/ui/views/sensors.py diff --git a/flexmeasures/api/common/schemas/times.py b/flexmeasures/api/common/schemas/times.py index 8f2108ee8..a9bff0d07 100644 --- a/flexmeasures/api/common/schemas/times.py +++ b/flexmeasures/api/common/schemas/times.py @@ -62,3 +62,16 @@ def ground_from( ) return (pd.Timestamp(start) + offset).to_pydatetime() - start return duration + + +class AwareDateTimeField(fields.AwareDateTime): + """Field that deserializes to a timezone aware datetime + and serializes back to a string.""" + + def _deserialize(self, value: str, attr, obj, **kwargs) -> datetime: + """ + Work-around until this PR lands: + https://github.com/marshmallow-code/marshmallow/pull/1787 + """ + value = value.replace(" ", "+") + return fields.AwareDateTime._deserialize(self, value, attr, obj, **kwargs) diff --git a/flexmeasures/data/models/time_series.py b/flexmeasures/data/models/time_series.py index f77d3285f..8fb206c1d 100644 --- a/flexmeasures/data/models/time_series.py +++ b/flexmeasures/data/models/time_series.py @@ -1,5 +1,6 @@ from typing import List, Dict, Optional, Union, Tuple from datetime import datetime as datetime_type, timedelta +import json from sqlalchemy.ext.declarative import declared_attr from sqlalchemy.orm import Query, Session @@ -17,6 +18,7 @@ exclude_source_type_filter, ) from flexmeasures.data.services.time_series import collect_time_series_data +from flexmeasures.utils.time_utils import server_now class Sensor(db.Model, tb.SensorDBMixin): @@ -51,6 +53,83 @@ def search_beliefs( source=source, ) + def chart( + self, + events_not_before: Optional[datetime_type] = None, + events_before: Optional[datetime_type] = None, + belief_start: Optional[datetime_type] = None, + belief_end: Optional[datetime_type] = None, + source: Optional[Union[int, List[int], str, List[str]]] = None, + data_only: bool = False, + chart_only: bool = True, + as_html: bool = False, + ) -> str: + """ + + :param data_only: return just the data (in case you have the chart specs already) + :param as_html: todo: allow returning as standalone html + """ + bdf = self.search_beliefs( + (events_not_before, events_before), (belief_start, belief_end), source + ) + + bar_chart = { + "description": "A simple bar chart.", + "data": {"name": "my_dataset"}, + "mark": "bar", + "encoding": { + "x": {"field": "event_start", "type": "T"}, + "y": {"field": "event_value", "type": "quantitative"}, + "tooltip": [ + # {"field": "full_date", "title": "Time and date", "type": "nominal"}, + { + "field": "event_value", + "title": "Consumption rate", + "type": "quantitative", + }, + ], + }, + } + if chart_only: + return json.dumps(bar_chart) + + if data_only: + df = bdf.reset_index() + df["source"] = df["source"].apply(lambda x: x.name) + return df.to_json(orient="records") + df = bdf.reset_index() + df["source"] = df["source"].apply(lambda x: x.name) + bar_chart["datasets"] = dict(my_dataset=json.loads(self.chart(data_only=True))) + return json.dumps(bar_chart) + + @property + def timerange(self) -> Dict[str, datetime_type]: + """Timerange for which sensor data exists. + + :returns: dictionary with start and end, for example: + { + 'start': datetime.datetime(2020, 12, 3, 14, 0, tzinfo=pytz.utc), + 'end': datetime.datetime(2020, 12, 3, 14, 30, tzinfo=pytz.utc) + } + """ + least_recent_query = ( + TimedBelief.query.filter(TimedBelief.sensor == self) + .order_by(TimedBelief.event_start.asc()) + .limit(1) + ) + most_recent_query = ( + TimedBelief.query.filter(TimedBelief.sensor == self) + .order_by(TimedBelief.event_start.desc()) + .limit(1) + ) + results = least_recent_query.union_all(most_recent_query).all() + if not results: + # return now in case there is no data for the sensor + now = server_now() + return dict(start=now, end=now) + least_recent, most_recent = results + return dict(start=least_recent.event_start, end=most_recent.event_end) + def __repr__(self) -> str: return f"" diff --git a/flexmeasures/ui/__init__.py b/flexmeasures/ui/__init__.py index c3df28dff..2d69a962b 100644 --- a/flexmeasures/ui/__init__.py +++ b/flexmeasures/ui/__init__.py @@ -33,9 +33,11 @@ def register_at(app: Flask): from flexmeasures.ui.crud.assets import AssetCrudUI from flexmeasures.ui.crud.users import UserCrudUI + from flexmeasures.ui.views.sensors import SensorView AssetCrudUI.register(app) UserCrudUI.register(app) + SensorView.register(app) import flexmeasures.ui.views # noqa: F401 this is necessary to load the views diff --git a/flexmeasures/ui/templates/views/sensors.html b/flexmeasures/ui/templates/views/sensors.html new file mode 100644 index 000000000..093b84b39 --- /dev/null +++ b/flexmeasures/ui/templates/views/sensors.html @@ -0,0 +1,163 @@ + + + + FlexMeasures + + + + + + + + + + + + + + + + + + + +
+
+
+
+
+ + + + + + diff --git a/flexmeasures/ui/views/sensors.py b/flexmeasures/ui/views/sensors.py new file mode 100644 index 000000000..4374e12a7 --- /dev/null +++ b/flexmeasures/ui/views/sensors.py @@ -0,0 +1,54 @@ +from flask_classful import FlaskView, route +from flask_security import login_required, roles_required +from marshmallow import fields +from webargs.flaskparser import use_kwargs + +from flexmeasures.api.common.schemas.times import AwareDateTimeField +from flexmeasures.data.models.time_series import Sensor +from flexmeasures.ui.utils.view_utils import render_flexmeasures_template + + +class SensorView(FlaskView): + """ + This view exposes sensor attributes through the API. + + todo: consider extending this view for crud purposes + """ + + route_base = "/sensors" + + @login_required + @roles_required("admin") # todo: remove after we check for sensor ownership + @route("///") + @use_kwargs( + { + "events_not_before": AwareDateTimeField(format="iso", required=False), + "events_before": AwareDateTimeField(format="iso", required=False), + "beliefs_not_before": AwareDateTimeField(format="iso", required=False), + "beliefs_before": AwareDateTimeField(format="iso", required=False), + "data_only": fields.Boolean(required=False), + "chart_only": fields.Boolean(required=False), + "as_html": fields.Boolean(required=False), + }, + location="query", + ) + def get_attr(self, id, attr, **kwargs): + """GET from /sensors//""" + sensor = Sensor.query.filter(Sensor.id == id).one_or_none() + sensor_attr = getattr(sensor, attr) + if not callable(sensor_attr): + # property + return {attr: sensor_attr} + else: + # method + return sensor_attr(**kwargs) + + @login_required + @roles_required("admin") # todo: remove after we check for sensor ownership + def get(self, id: str): + """GET from /sensors/""" + return render_flexmeasures_template( + ("views/sensors.html"), + sensor_id=id, + msg="", + ) From d938edab5283612df074d27bb6aa4a4646b8e666 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 9 Apr 2021 22:25:05 +0200 Subject: [PATCH 04/25] Move chart specs to dedicated module --- flexmeasures/data/models/time_series.py | 56 ++++++++++++-------- flexmeasures/ui/charts/__init__.py | 10 ++++ flexmeasures/ui/charts/belief_charts.py | 27 ++++++++++ flexmeasures/ui/charts/defaults.py | 42 +++++++++++++++ flexmeasures/ui/charts/readme.md | 7 +++ flexmeasures/ui/templates/views/sensors.html | 10 ++-- flexmeasures/ui/views/sensors.py | 1 + requirements/app.in | 1 + 8 files changed, 127 insertions(+), 27 deletions(-) create mode 100644 flexmeasures/ui/charts/__init__.py create mode 100644 flexmeasures/ui/charts/belief_charts.py create mode 100644 flexmeasures/ui/charts/defaults.py create mode 100644 flexmeasures/ui/charts/readme.md diff --git a/flexmeasures/data/models/time_series.py b/flexmeasures/data/models/time_series.py index 8fb206c1d..4713def74 100644 --- a/flexmeasures/data/models/time_series.py +++ b/flexmeasures/data/models/time_series.py @@ -2,6 +2,7 @@ from datetime import datetime as datetime_type, timedelta import json +from altair.utils.html import spec_to_html from sqlalchemy.ext.declarative import declared_attr from sqlalchemy.orm import Query, Session import timely_beliefs as tb @@ -18,7 +19,9 @@ exclude_source_type_filter, ) from flexmeasures.data.services.time_series import collect_time_series_data +from flexmeasures.ui.charts import belief_charts_mapping from flexmeasures.utils.time_utils import server_now +from flexmeasures.utils.flexmeasures_inflection import capitalize class Sensor(db.Model, tb.SensorDBMixin): @@ -60,47 +63,54 @@ def chart( belief_start: Optional[datetime_type] = None, belief_end: Optional[datetime_type] = None, source: Optional[Union[int, List[int], str, List[str]]] = None, + chart_type: str = "bar_chart", data_only: bool = False, chart_only: bool = True, as_html: bool = False, + dataset_name: Optional[str] = None, + **kwargs, ) -> str: """ :param data_only: return just the data (in case you have the chart specs already) - :param as_html: todo: allow returning as standalone html + :param as_html: return the chart with data as a standalone html """ + if dataset_name is None: + dataset_name = "sensor_" + str(self.id) bdf = self.search_beliefs( (events_not_before, events_before), (belief_start, belief_end), source ) + self.sensor_type = ( + self.name + ) # todo remove this placeholder when sensor types are modelled + chart = belief_charts_mapping[chart_type]( + title=capitalize(self.name), + quantity=capitalize(self.sensor_type), + unit=self.unit, + dataset_name=dataset_name, + **kwargs, + ) - bar_chart = { - "description": "A simple bar chart.", - "data": {"name": "my_dataset"}, - "mark": "bar", - "encoding": { - "x": {"field": "event_start", "type": "T"}, - "y": {"field": "event_value", "type": "quantitative"}, - "tooltip": [ - # {"field": "full_date", "title": "Time and date", "type": "nominal"}, - { - "field": "event_value", - "title": "Consumption rate", - "type": "quantitative", - }, - ], - }, - } if chart_only: - return json.dumps(bar_chart) - - if data_only: + return json.dumps(chart) + elif data_only: df = bdf.reset_index() df["source"] = df["source"].apply(lambda x: x.name) return df.to_json(orient="records") df = bdf.reset_index() df["source"] = df["source"].apply(lambda x: x.name) - bar_chart["datasets"] = dict(my_dataset=json.loads(self.chart(data_only=True))) - return json.dumps(bar_chart) + chart["datasets"] = { + dataset_name: json.loads(self.chart(data_only=True, chart_only=False)) + } + if as_html: + return spec_to_html( + chart, + "vega-lite", + vega_version="5", + vegaembed_version="6.17.0", + vegalite_version="5.0.0", + ) + return json.dumps(chart) @property def timerange(self) -> Dict[str, datetime_type]: diff --git a/flexmeasures/ui/charts/__init__.py b/flexmeasures/ui/charts/__init__.py new file mode 100644 index 000000000..75805e1a7 --- /dev/null +++ b/flexmeasures/ui/charts/__init__.py @@ -0,0 +1,10 @@ +from inspect import getmembers, isfunction + +from . import belief_charts +from .defaults import apply_chart_defaults + +belief_charts_mapping = { + k: apply_chart_defaults(v) + for k, v in getmembers(belief_charts) + if isfunction(v) or isinstance(v, dict) +} diff --git a/flexmeasures/ui/charts/belief_charts.py b/flexmeasures/ui/charts/belief_charts.py new file mode 100644 index 000000000..3692deb6b --- /dev/null +++ b/flexmeasures/ui/charts/belief_charts.py @@ -0,0 +1,27 @@ +from flexmeasures.ui.charts.defaults import TIME_TITLE, TIME_TOOLTIP_TITLE + + +def bar_chart(title: str, quantity: str = "unknown quantity", unit: str = "a.u."): + if not unit: + unit = "a.u." + return { + "description": "A simple bar chart.", + "title": title, + "mark": "bar", + "encoding": { + "x": {"field": "event_start", "type": "T", "title": TIME_TITLE}, + "y": { + "field": "event_value", + "type": "quantitative", + "title": quantity + " (" + unit + ")", + }, + "tooltip": [ + {"field": "full_date", "title": TIME_TOOLTIP_TITLE, "type": "nominal"}, + { + "field": "event_value", + "title": quantity + " (" + unit + ")", + "type": "quantitative", + }, + ], + }, + } diff --git a/flexmeasures/ui/charts/defaults.py b/flexmeasures/ui/charts/defaults.py new file mode 100644 index 000000000..4d979c5a6 --- /dev/null +++ b/flexmeasures/ui/charts/defaults.py @@ -0,0 +1,42 @@ +from functools import wraps +from typing import Callable, Union + +import altair as alt + + +HEIGHT = 300 +WIDTH = 600 +REDUCED_HEIGHT = REDUCED_WIDTH = 60 +SELECTOR_COLOR = "darkred" +TIME_FORMAT = "%I:%M %p on %A %b %e, %Y" +TIME_TOOLTIP_TITLE = "Time and date" +TIME_TITLE = None +TIME_SELECTION_TOOLTIP = "Click and drag to select a time window" + + +def apply_chart_defaults(fn): + @wraps(fn) + def decorated_chart_specs(*args, **kwargs): + dataset_name = kwargs.pop("dataset_name", None) + if isinstance(fn, Callable): + # function that returns a chart specification + chart_specs: Union[dict, alt.TopLevelMixin] = fn(*args, **kwargs) + else: + # not a function, but a direct chart specification + chart_specs: Union[dict, alt.TopLevelMixin] = fn + if isinstance(chart_specs, alt.TopLevelMixin): + chart_specs = chart_specs.to_dict() + chart_specs.pop("$schema") + if dataset_name: + chart_specs["data"] = {"name": dataset_name} + chart_specs["height"] = HEIGHT + chart_specs["width"] = WIDTH + chart_specs["transform"] = [ + { + "as": "full_date", + "calculate": f"timeFormat(datum.event_start, '{TIME_FORMAT}')", + } + ] + return chart_specs + + return decorated_chart_specs diff --git a/flexmeasures/ui/charts/readme.md b/flexmeasures/ui/charts/readme.md new file mode 100644 index 000000000..f918ff80f --- /dev/null +++ b/flexmeasures/ui/charts/readme.md @@ -0,0 +1,7 @@ +# Developer docs for adding chart specs + +Chart specs can be specified as a dictionary with a vega-lite specification or as an altair chart. +Alternatively, they can be specified as a function that returns a dict (with vega-lite specs) or an altair chart. +This approach is useful if you need to parameterize the specification with kwargs. + +Todo: support a plug-in architecture, see https://packaging.python.org/guides/creating-and-discovering-plugins/ diff --git a/flexmeasures/ui/templates/views/sensors.html b/flexmeasures/ui/templates/views/sensors.html index 093b84b39..f22670f09 100644 --- a/flexmeasures/ui/templates/views/sensors.html +++ b/flexmeasures/ui/templates/views/sensors.html @@ -32,14 +32,14 @@ let vegaView; - async function embedAndLoad(chartSpecsPath, elementId) { + async function embedAndLoad(chartSpecsPath, elementId, datasetName) { var opt = { mode: 'vega-lite', renderer: 'svg', actions: {export: true, source: true, editor: true}, }; - await vegaEmbed('#'+elementId, chartSpecsPath, opt) + await vegaEmbed('#'+elementId, chartSpecsPath + '?dataset_name=' + datasetName, opt) .then(function (result) { // result.view is the Vega View, chartSpecsPath is the original Vega-Lite specification vegaView = result.view; @@ -71,7 +71,8 @@ sensorPath = '/sensors/' + sensorId chartSpecsPath = sensorPath + '/chart' elementId = 'sensorchart' - embedAndLoad(chartSpecsPath, elementId); + datasetName = 'sensor_' + sensorId + embedAndLoad(chartSpecsPath, elementId, datasetName); const date = Date(); const picker = new Litepicker({ @@ -113,8 +114,9 @@ chart_only: false, events_not_before: queryStartDate, events_before: queryEndDate, + dataset_name: datasetName }, function( data ) { - vegaView.change('my_dataset', vega.changeset().remove(vega.truthy).insert(data)).resize().run(); + vegaView.change(datasetName, vega.changeset().remove(vega.truthy).insert(data)).resize().run(); }); }); diff --git a/flexmeasures/ui/views/sensors.py b/flexmeasures/ui/views/sensors.py index 4374e12a7..2b97b31f4 100644 --- a/flexmeasures/ui/views/sensors.py +++ b/flexmeasures/ui/views/sensors.py @@ -29,6 +29,7 @@ class SensorView(FlaskView): "data_only": fields.Boolean(required=False), "chart_only": fields.Boolean(required=False), "as_html": fields.Boolean(required=False), + "dataset_name": fields.Str(required=False), }, location="query", ) diff --git a/requirements/app.in b/requirements/app.in index 6a858c250..8974000f5 100644 --- a/requirements/app.in +++ b/requirements/app.in @@ -1,4 +1,5 @@ # see ui/utils/plotting_utils: separate_legend() and create_hover_tool() +altair bokeh==1.0.4 colour pscript From 56d839715d6b7f70eff0a7e737247554e652dff5 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 12 Apr 2021 09:27:51 +0200 Subject: [PATCH 05/25] Docstring correction --- flexmeasures/api/v2_0/implementations/assets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/api/v2_0/implementations/assets.py b/flexmeasures/api/v2_0/implementations/assets.py index d4fb8f79e..0d05533f0 100644 --- a/flexmeasures/api/v2_0/implementations/assets.py +++ b/flexmeasures/api/v2_0/implementations/assets.py @@ -66,7 +66,7 @@ def load_asset(admins_only: bool = False): should be allowed. @app.route('/asset/') - @check_asset + @load_asset def get_asset(asset): return asset_schema.dump(asset), 200 From 19dc9e3f18349e49117e148ffa9ef6ae51ea5f11 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 12 Apr 2021 11:51:52 +0200 Subject: [PATCH 06/25] Fix data loading --- flexmeasures/data/models/time_series.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/flexmeasures/data/models/time_series.py b/flexmeasures/data/models/time_series.py index 4713def74..207d06a3d 100644 --- a/flexmeasures/data/models/time_series.py +++ b/flexmeasures/data/models/time_series.py @@ -75,11 +75,10 @@ def chart( :param data_only: return just the data (in case you have the chart specs already) :param as_html: return the chart with data as a standalone html """ + + # Set up chart specification if dataset_name is None: dataset_name = "sensor_" + str(self.id) - bdf = self.search_beliefs( - (events_not_before, events_before), (belief_start, belief_end), source - ) self.sensor_type = ( self.name ) # todo remove this placeholder when sensor types are modelled @@ -90,18 +89,21 @@ def chart( dataset_name=dataset_name, **kwargs, ) - if chart_only: return json.dumps(chart) - elif data_only: - df = bdf.reset_index() - df["source"] = df["source"].apply(lambda x: x.name) - return df.to_json(orient="records") + + # Set up data + bdf = self.search_beliefs( + (events_not_before, events_before), (belief_start, belief_end), source + ) df = bdf.reset_index() df["source"] = df["source"].apply(lambda x: x.name) - chart["datasets"] = { - dataset_name: json.loads(self.chart(data_only=True, chart_only=False)) - } + data = df.to_json(orient="records") + if data_only: + return data + + # Combine chart specs and data + chart["datasets"] = {dataset_name: json.loads(data)} if as_html: return spec_to_html( chart, From 8ed1d025943eb4246ce6d0b70bae81619e7bb7f2 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 15 Apr 2021 11:29:12 +0200 Subject: [PATCH 07/25] Rename variables in sync with version 1.4.1 of timely-beliefs, and expand docstrings --- flexmeasures/data/models/time_series.py | 78 ++++++++++++++----------- requirements/app.in | 2 +- 2 files changed, 45 insertions(+), 35 deletions(-) diff --git a/flexmeasures/data/models/time_series.py b/flexmeasures/data/models/time_series.py index 207d06a3d..594cb90ef 100644 --- a/flexmeasures/data/models/time_series.py +++ b/flexmeasures/data/models/time_series.py @@ -34,46 +34,54 @@ def __init__(self, name: str, **kwargs): def search_beliefs( self, - event_time_window: Tuple[Optional[datetime_type], Optional[datetime_type]] = ( - None, - None, - ), - belief_time_window: Tuple[Optional[datetime_type], Optional[datetime_type]] = ( - None, - None, - ), + event_starts_after: Optional[datetime_type] = None, + event_ends_before: Optional[datetime_type] = None, + beliefs_after: Optional[datetime_type] = None, + beliefs_before: Optional[datetime_type] = None, source: Optional[Union[int, List[int], str, List[str]]] = None, ): """Search all beliefs about events for this sensor. - :param event_time_window: search only events within this time window - :param belief_time_window: search only beliefs within this time window - :param source: search only beliefs by this source (pass its name or id) or list of sources""" + :param event_starts_after: only return beliefs about events that start after this datetime (inclusive) + :param event_ends_before: only return beliefs about events that end before this datetime (inclusive) + :param beliefs_after: only return beliefs formed after this datetime (inclusive) + :param beliefs_before: only return beliefs formed before this datetime (inclusive) + :param source: search only beliefs by this source (pass its name or id) or list of sources + """ return TimedBelief.search( sensor=self, - event_time_window=event_time_window, - belief_time_window=belief_time_window, + event_starts_after=event_starts_after, + event_ends_before=event_ends_before, + beliefs_after=beliefs_after, + beliefs_before=beliefs_before, source=source, ) def chart( self, - events_not_before: Optional[datetime_type] = None, - events_before: Optional[datetime_type] = None, - belief_start: Optional[datetime_type] = None, - belief_end: Optional[datetime_type] = None, - source: Optional[Union[int, List[int], str, List[str]]] = None, chart_type: str = "bar_chart", + event_starts_after: Optional[datetime_type] = None, + event_ends_before: Optional[datetime_type] = None, + beliefs_after: Optional[datetime_type] = None, + beliefs_before: Optional[datetime_type] = None, + source: Optional[Union[int, List[int], str, List[str]]] = None, data_only: bool = False, chart_only: bool = True, as_html: bool = False, dataset_name: Optional[str] = None, **kwargs, ) -> str: - """ + """Create a chart showing sensor data. + :param chart_type: currently only "bar_chart" # todo: where can we properly list the available chart types? + :param event_starts_after: only return beliefs about events that start after this datetime (inclusive) + :param event_ends_before: only return beliefs about events that end before this datetime (inclusive) + :param beliefs_after: only return beliefs formed after this datetime (inclusive) + :param beliefs_before: only return beliefs formed before this datetime (inclusive) + :param source: search only beliefs by this source (pass its name or id) or list of sources :param data_only: return just the data (in case you have the chart specs already) :param as_html: return the chart with data as a standalone html + :param dataset_name: optionally name the dataset used in the chart (the default name is sensor_) """ # Set up chart specification @@ -94,7 +102,11 @@ def chart( # Set up data bdf = self.search_beliefs( - (events_not_before, events_before), (belief_start, belief_end), source + event_starts_after=event_starts_after, + event_ends_before=event_ends_before, + beliefs_after=beliefs_after, + beliefs_before=beliefs_before, + source=source, ) df = bdf.reset_index() df["source"] = df["source"].apply(lambda x: x.name) @@ -173,30 +185,28 @@ def __init__( def search( cls, sensor: Sensor, - event_time_window: Tuple[Optional[datetime_type], Optional[datetime_type]] = ( - None, - None, - ), - belief_time_window: Tuple[Optional[datetime_type], Optional[datetime_type]] = ( - None, - None, - ), + event_starts_after: Optional[datetime_type] = None, + event_ends_before: Optional[datetime_type] = None, + beliefs_after: Optional[datetime_type] = None, + beliefs_before: Optional[datetime_type] = None, source: Optional[Union[int, List[int], str, List[str]]] = None, ) -> tb.BeliefsDataFrame: """Search all beliefs about events for a given sensor. :param sensor: search only this sensor - :param event_time_window: search only events within this time window - :param belief_time_window: search only beliefs within this time window + :param event_starts_after: only return beliefs about events that start after this datetime (inclusive) + :param event_ends_before: only return beliefs about events that end before this datetime (inclusive) + :param beliefs_after: only return beliefs formed after this datetime (inclusive) + :param beliefs_before: only return beliefs formed before this datetime (inclusive) :param source: search only beliefs by this source (pass its name or id) or list of sources """ return cls.search_session( session=db.session, sensor=sensor, - event_before=event_time_window[1], - event_not_before=event_time_window[0], - belief_before=belief_time_window[1], - belief_not_before=belief_time_window[0], + event_starts_after=event_starts_after, + event_ends_before=event_ends_before, + beliefs_after=beliefs_after, + beliefs_before=beliefs_before, source=source, ) diff --git a/requirements/app.in b/requirements/app.in index 8974000f5..31c8d0de6 100644 --- a/requirements/app.in +++ b/requirements/app.in @@ -33,7 +33,7 @@ netCDF4 siphon tables timetomodel>=0.6.8 -timely-beliefs>=1.3.1 +timely-beliefs>=1.4.1 python-dotenv # a backport, not needed in Python3.8 importlib_metadata From 461409106217f0f635e73babd12a7efae6374028 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 15 Apr 2021 11:37:50 +0200 Subject: [PATCH 08/25] Clarify what belief_charts_mapping does --- flexmeasures/ui/charts/__init__.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/flexmeasures/ui/charts/__init__.py b/flexmeasures/ui/charts/__init__.py index 75805e1a7..36a95f9e3 100644 --- a/flexmeasures/ui/charts/__init__.py +++ b/flexmeasures/ui/charts/__init__.py @@ -3,8 +3,13 @@ from . import belief_charts from .defaults import apply_chart_defaults +# Create a dictionary mapping chart types to chart specs, and apply defaults to the chart specs, too. +# chart types: Name of a variable defining chart specs or a function returning chart specs. +# chart specs: A dictionary or Altair chart specification. +# In case of a dictionary, the creator needs to ensure that the dictionary contains valid vega-lite specs +# In case of an Altair chart specification, Altair checks for you belief_charts_mapping = { - k: apply_chart_defaults(v) - for k, v in getmembers(belief_charts) - if isfunction(v) or isinstance(v, dict) + chart_type: apply_chart_defaults(chart_specs) + for chart_type, chart_specs in getmembers(belief_charts) + if isfunction(chart_specs) or isinstance(chart_specs, dict) } From 503e5ab8bf0b9b0dde98dce5de7f45039b5f27b2 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 15 Apr 2021 14:40:04 +0200 Subject: [PATCH 09/25] Make mapping chart types to specs a function --- flexmeasures/data/models/time_series.py | 13 ++++++----- flexmeasures/ui/charts/__init__.py | 31 +++++++++++++++++-------- 2 files changed, 28 insertions(+), 16 deletions(-) diff --git a/flexmeasures/data/models/time_series.py b/flexmeasures/data/models/time_series.py index 594cb90ef..9a738e8f4 100644 --- a/flexmeasures/data/models/time_series.py +++ b/flexmeasures/data/models/time_series.py @@ -19,7 +19,7 @@ exclude_source_type_filter, ) from flexmeasures.data.services.time_series import collect_time_series_data -from flexmeasures.ui.charts import belief_charts_mapping +from flexmeasures.ui.charts import chart_type_to_chart_specs from flexmeasures.utils.time_utils import server_now from flexmeasures.utils.flexmeasures_inflection import capitalize @@ -90,7 +90,8 @@ def chart( self.sensor_type = ( self.name ) # todo remove this placeholder when sensor types are modelled - chart = belief_charts_mapping[chart_type]( + chart_specs = chart_type_to_chart_specs( + chart_type, title=capitalize(self.name), quantity=capitalize(self.sensor_type), unit=self.unit, @@ -98,7 +99,7 @@ def chart( **kwargs, ) if chart_only: - return json.dumps(chart) + return json.dumps(chart_specs) # Set up data bdf = self.search_beliefs( @@ -115,16 +116,16 @@ def chart( return data # Combine chart specs and data - chart["datasets"] = {dataset_name: json.loads(data)} + chart_specs["datasets"] = {dataset_name: json.loads(data)} if as_html: return spec_to_html( - chart, + chart_specs, "vega-lite", vega_version="5", vegaembed_version="6.17.0", vegalite_version="5.0.0", ) - return json.dumps(chart) + return json.dumps(chart_specs) @property def timerange(self) -> Dict[str, datetime_type]: diff --git a/flexmeasures/ui/charts/__init__.py b/flexmeasures/ui/charts/__init__.py index 36a95f9e3..a25bd3cf3 100644 --- a/flexmeasures/ui/charts/__init__.py +++ b/flexmeasures/ui/charts/__init__.py @@ -3,13 +3,24 @@ from . import belief_charts from .defaults import apply_chart_defaults -# Create a dictionary mapping chart types to chart specs, and apply defaults to the chart specs, too. -# chart types: Name of a variable defining chart specs or a function returning chart specs. -# chart specs: A dictionary or Altair chart specification. -# In case of a dictionary, the creator needs to ensure that the dictionary contains valid vega-lite specs -# In case of an Altair chart specification, Altair checks for you -belief_charts_mapping = { - chart_type: apply_chart_defaults(chart_specs) - for chart_type, chart_specs in getmembers(belief_charts) - if isfunction(chart_specs) or isinstance(chart_specs, dict) -} + +def chart_type_to_chart_specs(chart_type: str, **kwargs) -> dict: + """Create chart specs of a given chart type, using FlexMeasures defaults for settings like width and height. + + :param chart_type: Name of a variable defining chart specs or a function returning chart specs. + The chart specs can be a dictionary or an Altair chart specification. + - In case of a dictionary, the creator needs to ensure that the dictionary contains valid specs + - In case of an Altair chart specification, Altair validates for you + :returns: A dictionary containing a vega-lite chart specification + """ + # Create a dictionary mapping chart types to chart specs, and apply defaults to the chart specs, too. + belief_charts_mapping = { + chart_type: apply_chart_defaults(chart_specs) + for chart_type, chart_specs in getmembers(belief_charts) + if isfunction(chart_specs) or isinstance(chart_specs, dict) + } + # Create chart specs + chart_specs_or_fnc = belief_charts_mapping[chart_type] + if isfunction(chart_specs_or_fnc): + return chart_specs_or_fnc(**kwargs) + return chart_specs_or_fnc From 3fdeed145acd7ffb09dd67378f82a938ab3c4f80 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 19 Apr 2021 12:44:55 +0200 Subject: [PATCH 10/25] Adjust query parameters to renamed tb variables --- flexmeasures/ui/templates/views/sensors.html | 4 ++-- flexmeasures/ui/views/sensors.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/flexmeasures/ui/templates/views/sensors.html b/flexmeasures/ui/templates/views/sensors.html index f22670f09..271bdb3b3 100644 --- a/flexmeasures/ui/templates/views/sensors.html +++ b/flexmeasures/ui/templates/views/sensors.html @@ -112,8 +112,8 @@ $.getJSON( sensorPath + '/chart', { data_only: true, chart_only: false, - events_not_before: queryStartDate, - events_before: queryEndDate, + event_starts_after: queryStartDate, + event_ends_before: queryEndDate, dataset_name: datasetName }, function( data ) { vegaView.change(datasetName, vega.changeset().remove(vega.truthy).insert(data)).resize().run(); diff --git a/flexmeasures/ui/views/sensors.py b/flexmeasures/ui/views/sensors.py index 2b97b31f4..50ca9cd34 100644 --- a/flexmeasures/ui/views/sensors.py +++ b/flexmeasures/ui/views/sensors.py @@ -22,9 +22,9 @@ class SensorView(FlaskView): @route("///") @use_kwargs( { - "events_not_before": AwareDateTimeField(format="iso", required=False), - "events_before": AwareDateTimeField(format="iso", required=False), - "beliefs_not_before": AwareDateTimeField(format="iso", required=False), + "event_starts_after": AwareDateTimeField(format="iso", required=False), + "event_ends_before": AwareDateTimeField(format="iso", required=False), + "beliefs_after": AwareDateTimeField(format="iso", required=False), "beliefs_before": AwareDateTimeField(format="iso", required=False), "data_only": fields.Boolean(required=False), "chart_only": fields.Boolean(required=False), From 44c9b1b109b60e3436d78c8be8db6f670a5d5a08 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 19 Apr 2021 12:49:01 +0200 Subject: [PATCH 11/25] Separate endpoint for loading chart data and clean up boolean chart options --- flexmeasures/data/models/time_series.py | 63 +++++++++++++------- flexmeasures/ui/templates/views/sensors.html | 5 +- flexmeasures/ui/views/sensors.py | 20 ++++++- 3 files changed, 60 insertions(+), 28 deletions(-) diff --git a/flexmeasures/data/models/time_series.py b/flexmeasures/data/models/time_series.py index 9a738e8f4..1fb19dc1b 100644 --- a/flexmeasures/data/models/time_series.py +++ b/flexmeasures/data/models/time_series.py @@ -57,6 +57,33 @@ def search_beliefs( source=source, ) + def chart_data( + self, + event_starts_after: Optional[datetime_type] = None, + event_ends_before: Optional[datetime_type] = None, + beliefs_after: Optional[datetime_type] = None, + beliefs_before: Optional[datetime_type] = None, + source: Optional[Union[int, List[int], str, List[str]]] = None, + ): + """Get sensor data for use in charts. + + :param event_starts_after: only return beliefs about events that start after this datetime (inclusive) + :param event_ends_before: only return beliefs about events that end before this datetime (inclusive) + :param beliefs_after: only return beliefs formed after this datetime (inclusive) + :param beliefs_before: only return beliefs formed before this datetime (inclusive) + :param source: search only beliefs by this source (pass its name or id) or list of sources + """ + bdf = self.search_beliefs( + event_starts_after=event_starts_after, + event_ends_before=event_ends_before, + beliefs_after=beliefs_after, + beliefs_before=beliefs_before, + source=source, + ) + df = bdf.reset_index() + df["source"] = df["source"].apply(lambda x: x.name) + return df.to_json(orient="records") + def chart( self, chart_type: str = "bar_chart", @@ -65,8 +92,7 @@ def chart( beliefs_after: Optional[datetime_type] = None, beliefs_before: Optional[datetime_type] = None, source: Optional[Union[int, List[int], str, List[str]]] = None, - data_only: bool = False, - chart_only: bool = True, + include_data: bool = False, as_html: bool = False, dataset_name: Optional[str] = None, **kwargs, @@ -79,8 +105,8 @@ def chart( :param beliefs_after: only return beliefs formed after this datetime (inclusive) :param beliefs_before: only return beliefs formed before this datetime (inclusive) :param source: search only beliefs by this source (pass its name or id) or list of sources - :param data_only: return just the data (in case you have the chart specs already) - :param as_html: return the chart with data as a standalone html + :param include_data: if True, include data in the chart, or if False, exclude data + :param as_html: if True, return the chart as a standalone html, or if False, return as JSON :param dataset_name: optionally name the dataset used in the chart (the default name is sensor_) """ @@ -98,25 +124,18 @@ def chart( dataset_name=dataset_name, **kwargs, ) - if chart_only: - return json.dumps(chart_specs) - # Set up data - bdf = self.search_beliefs( - event_starts_after=event_starts_after, - event_ends_before=event_ends_before, - beliefs_after=beliefs_after, - beliefs_before=beliefs_before, - source=source, - ) - df = bdf.reset_index() - df["source"] = df["source"].apply(lambda x: x.name) - data = df.to_json(orient="records") - if data_only: - return data - - # Combine chart specs and data - chart_specs["datasets"] = {dataset_name: json.loads(data)} + if include_data: + # Set up data + data = self.chart_data( + event_starts_after=event_starts_after, + event_ends_before=event_ends_before, + beliefs_after=beliefs_after, + beliefs_before=beliefs_before, + source=source, + ) + # Combine chart specs and data + chart_specs["datasets"] = {dataset_name: json.loads(data)} if as_html: return spec_to_html( chart_specs, diff --git a/flexmeasures/ui/templates/views/sensors.html b/flexmeasures/ui/templates/views/sensors.html index 271bdb3b3..6756bf701 100644 --- a/flexmeasures/ui/templates/views/sensors.html +++ b/flexmeasures/ui/templates/views/sensors.html @@ -109,12 +109,9 @@ endDate.setDate(endDate.getDate() + 1); queryStartDate = (startDate != null) ? (startDate.toISOString()) : (null) queryEndDate = (endDate != null) ? (endDate.toISOString()) : (null) - $.getJSON( sensorPath + '/chart', { - data_only: true, - chart_only: false, + $.getJSON( sensorPath + '/chart_data', { event_starts_after: queryStartDate, event_ends_before: queryEndDate, - dataset_name: datasetName }, function( data ) { vegaView.change(datasetName, vega.changeset().remove(vega.truthy).insert(data)).resize().run(); }); diff --git a/flexmeasures/ui/views/sensors.py b/flexmeasures/ui/views/sensors.py index 50ca9cd34..b3da30668 100644 --- a/flexmeasures/ui/views/sensors.py +++ b/flexmeasures/ui/views/sensors.py @@ -26,8 +26,7 @@ class SensorView(FlaskView): "event_ends_before": AwareDateTimeField(format="iso", required=False), "beliefs_after": AwareDateTimeField(format="iso", required=False), "beliefs_before": AwareDateTimeField(format="iso", required=False), - "data_only": fields.Boolean(required=False), - "chart_only": fields.Boolean(required=False), + "include_data": fields.Boolean(required=False), "as_html": fields.Boolean(required=False), "dataset_name": fields.Str(required=False), }, @@ -44,6 +43,23 @@ def get_attr(self, id, attr, **kwargs): # method return sensor_attr(**kwargs) + @login_required + @roles_required("admin") # todo: remove after we check for sensor ownership + @route("//chart_data/") + @use_kwargs( + { + "event_starts_after": AwareDateTimeField(format="iso", required=False), + "event_ends_before": AwareDateTimeField(format="iso", required=False), + "beliefs_after": AwareDateTimeField(format="iso", required=False), + "beliefs_before": AwareDateTimeField(format="iso", required=False), + }, + location="query", + ) + def get_chart_data(self, id, **kwargs): + """GET data for use in charts (in case you have the chart specs already).""" + sensor = Sensor.query.filter(Sensor.id == id).one_or_none() + return sensor.chart_data(**kwargs) + @login_required @roles_required("admin") # todo: remove after we check for sensor ownership def get(self, id: str): From 324f4e22e9194bfdf070942b6901d94a31dbbc10 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 19 Apr 2021 12:52:17 +0200 Subject: [PATCH 12/25] Upgrade tb dependency --- requirements/app.in | 2 +- requirements/app.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements/app.in b/requirements/app.in index 31c8d0de6..9264d4555 100644 --- a/requirements/app.in +++ b/requirements/app.in @@ -33,7 +33,7 @@ netCDF4 siphon tables timetomodel>=0.6.8 -timely-beliefs>=1.4.1 +timely-beliefs>=1.4.2 python-dotenv # a backport, not needed in Python3.8 importlib_metadata diff --git a/requirements/app.txt b/requirements/app.txt index 35570ffd7..d2b22980f 100644 --- a/requirements/app.txt +++ b/requirements/app.txt @@ -317,7 +317,7 @@ tables==3.6.1 # via -r requirements/app.in threadpoolctl==2.1.0 # via scikit-learn -timely-beliefs==1.3.0 +timely-beliefs==1.4.2 # via -r requirements/app.in timetomodel==0.6.9 # via -r requirements/app.in From 13f23708907f46c63cf2e7ff0140de84267968d1 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 19 Apr 2021 15:31:07 +0200 Subject: [PATCH 13/25] Dedicated endpoint for get_chart --- flexmeasures/ui/views/sensors.py | 39 ++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/flexmeasures/ui/views/sensors.py b/flexmeasures/ui/views/sensors.py index b3da30668..53bdf6b31 100644 --- a/flexmeasures/ui/views/sensors.py +++ b/flexmeasures/ui/views/sensors.py @@ -1,3 +1,4 @@ +from flask import abort from flask_classful import FlaskView, route from flask_security import login_required, roles_required from marshmallow import fields @@ -20,6 +21,16 @@ class SensorView(FlaskView): @login_required @roles_required("admin") # todo: remove after we check for sensor ownership @route("///") + def get_attr(self, id, attr): + """GET from /sensors//""" + sensor = get_sensor_or_abort(id) + if not hasattr(sensor, attr): + raise abort(404, f"Sensor attribute {attr} not found") + return {attr: getattr(sensor, attr)} + + @login_required + @roles_required("admin") # todo: remove after we check for sensor ownership + @route("//chart/") @use_kwargs( { "event_starts_after": AwareDateTimeField(format="iso", required=False), @@ -32,16 +43,10 @@ class SensorView(FlaskView): }, location="query", ) - def get_attr(self, id, attr, **kwargs): - """GET from /sensors//""" - sensor = Sensor.query.filter(Sensor.id == id).one_or_none() - sensor_attr = getattr(sensor, attr) - if not callable(sensor_attr): - # property - return {attr: sensor_attr} - else: - # method - return sensor_attr(**kwargs) + def get_chart(self, id, **kwargs): + """GET from /sensors//chart""" + sensor = get_sensor_or_abort(id) + return sensor.chart(**kwargs) @login_required @roles_required("admin") # todo: remove after we check for sensor ownership @@ -56,8 +61,11 @@ def get_attr(self, id, attr, **kwargs): location="query", ) def get_chart_data(self, id, **kwargs): - """GET data for use in charts (in case you have the chart specs already).""" - sensor = Sensor.query.filter(Sensor.id == id).one_or_none() + """GET from /sensors//chart_data + + Data for use in charts (in case you have the chart specs already). + """ + sensor = get_sensor_or_abort(id) return sensor.chart_data(**kwargs) @login_required @@ -69,3 +77,10 @@ def get(self, id: str): sensor_id=id, msg="", ) + + +def get_sensor_or_abort(id: int) -> Sensor: + sensor = Sensor.query.filter(Sensor.id == id).one_or_none() + if sensor is None: + raise abort(404, f"Sensor {id} not found") + return sensor From 6894988f85192ceade1d4585d76d157da15ba51b Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 19 Apr 2021 19:47:22 +0200 Subject: [PATCH 14/25] Split SensorView into UI and API parts --- flexmeasures/ui/__init__.py | 5 +- flexmeasures/ui/templates/views/sensors.html | 7 +-- flexmeasures/ui/views/sensors.py | 60 ++++++++++++++------ 3 files changed, 47 insertions(+), 25 deletions(-) diff --git a/flexmeasures/ui/__init__.py b/flexmeasures/ui/__init__.py index 2d69a962b..262503fb1 100644 --- a/flexmeasures/ui/__init__.py +++ b/flexmeasures/ui/__init__.py @@ -33,11 +33,12 @@ def register_at(app: Flask): from flexmeasures.ui.crud.assets import AssetCrudUI from flexmeasures.ui.crud.users import UserCrudUI - from flexmeasures.ui.views.sensors import SensorView + from flexmeasures.ui.views.sensors import SensorUI, SensorAPI AssetCrudUI.register(app) UserCrudUI.register(app) - SensorView.register(app) + SensorUI.register(app) + SensorAPI.register(app) import flexmeasures.ui.views # noqa: F401 this is necessary to load the views diff --git a/flexmeasures/ui/templates/views/sensors.html b/flexmeasures/ui/templates/views/sensors.html index 6756bf701..bfd2ca073 100644 --- a/flexmeasures/ui/templates/views/sensors.html +++ b/flexmeasures/ui/templates/views/sensors.html @@ -68,7 +68,7 @@ }; var sensorId = {{ sensor_id }} - sensorPath = '/sensors/' + sensorId + sensorPath = '/sensor/' + sensorId chartSpecsPath = sensorPath + '/chart' elementId = 'sensorchart' datasetName = 'sensor_' + sensorId @@ -124,14 +124,11 @@ } $(document).ready(function () { - $.getJSON( sensorPath + '/timezone', { + $.getJSON( sensorPath, { }, function( data ) { if (data.timezone != jstz.determine().name()) { $('#tzwarn').show().multiline('Please note that the sensor data you are viewing is located in a different timezone.\nTo view the data from a local perspective, set your locale timezone to ' + data.timezone + '.'); } - }); - $.getJSON( sensorPath + '/timerange', { - }, function( data ) { start = new Date(data.timerange.start); end = new Date(data.timerange.end) end.setSeconds(end.getSeconds() - 1); // -1 second in case most recent event ends at midnight diff --git a/flexmeasures/ui/views/sensors.py b/flexmeasures/ui/views/sensors.py index 53bdf6b31..945710b49 100644 --- a/flexmeasures/ui/views/sensors.py +++ b/flexmeasures/ui/views/sensors.py @@ -9,9 +9,9 @@ from flexmeasures.ui.utils.view_utils import render_flexmeasures_template -class SensorView(FlaskView): +class SensorUI(FlaskView): """ - This view exposes sensor attributes through the API. + This view creates several new UI endpoints for viewing sensors. todo: consider extending this view for crud purposes """ @@ -20,13 +20,39 @@ class SensorView(FlaskView): @login_required @roles_required("admin") # todo: remove after we check for sensor ownership - @route("///") - def get_attr(self, id, attr): - """GET from /sensors//""" - sensor = get_sensor_or_abort(id) - if not hasattr(sensor, attr): - raise abort(404, f"Sensor attribute {attr} not found") - return {attr: getattr(sensor, attr)} + @route("//chart/") + @use_kwargs( + { + "event_starts_after": AwareDateTimeField(format="iso", required=False), + "event_ends_before": AwareDateTimeField(format="iso", required=False), + "beliefs_after": AwareDateTimeField(format="iso", required=False), + "beliefs_before": AwareDateTimeField(format="iso", required=False), + "dataset_name": fields.Str(required=False), + }, + location="query", + ) + def get_chart(self, id, **kwargs): + """GET from /sensors//chart""" + return SensorAPI().get_chart(id, include_data=True, as_html=True, **kwargs) + + @login_required + @roles_required("admin") # todo: remove after we check for sensor ownership + def get(self, id: int): + """GET from /sensors/""" + return render_flexmeasures_template( + "views/sensors.html", + sensor_id=id, + msg="", + ) + + +class SensorAPI(FlaskView): + """ + This view exposes sensor attributes through API endpoints under development. + These endpoints are not yet part of our official API, but support the FlexMeasures UI. + """ + + route_base = "/sensor" @login_required @roles_required("admin") # todo: remove after we check for sensor ownership @@ -44,7 +70,7 @@ def get_attr(self, id, attr): location="query", ) def get_chart(self, id, **kwargs): - """GET from /sensors//chart""" + """GET from /sensor//chart""" sensor = get_sensor_or_abort(id) return sensor.chart(**kwargs) @@ -61,7 +87,7 @@ def get_chart(self, id, **kwargs): location="query", ) def get_chart_data(self, id, **kwargs): - """GET from /sensors//chart_data + """GET from /sensor//chart_data Data for use in charts (in case you have the chart specs already). """ @@ -70,13 +96,11 @@ def get_chart_data(self, id, **kwargs): @login_required @roles_required("admin") # todo: remove after we check for sensor ownership - def get(self, id: str): - """GET from /sensors/""" - return render_flexmeasures_template( - ("views/sensors.html"), - sensor_id=id, - msg="", - ) + def get(self, id: int): + """GET from /sensor/""" + sensor = get_sensor_or_abort(id) + attributes = ["name", "timezone", "timerange"] + return {attr: getattr(sensor, attr) for attr in attributes} def get_sensor_or_abort(id: int) -> Sensor: From 9bf9e571a1b38781519274867ff3a177178d5c28 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 20 Apr 2021 09:40:27 +0200 Subject: [PATCH 15/25] Remove the need for jquery in sensors.html --- flexmeasures/ui/templates/views/sensors.html | 81 ++++++++++---------- 1 file changed, 41 insertions(+), 40 deletions(-) diff --git a/flexmeasures/ui/templates/views/sensors.html b/flexmeasures/ui/templates/views/sensors.html index bfd2ca073..5275dff92 100644 --- a/flexmeasures/ui/templates/views/sensors.html +++ b/flexmeasures/ui/templates/views/sensors.html @@ -4,10 +4,8 @@ FlexMeasures - - @@ -109,49 +107,52 @@ endDate.setDate(endDate.getDate() + 1); queryStartDate = (startDate != null) ? (startDate.toISOString()) : (null) queryEndDate = (endDate != null) ? (endDate.toISOString()) : (null) - $.getJSON( sensorPath + '/chart_data', { - event_starts_after: queryStartDate, - event_ends_before: queryEndDate, - }, function( data ) { + fetch(sensorPath + '/chart_data/?event_starts_after=' + queryStartDate + '&event_ends_before=' + queryEndDate, { + method: "GET", + headers: {"Content-Type": "application/json"}, + }) + .then(function(response) { return response.json(); }) + .then(function(data) { vegaView.change(datasetName, vega.changeset().remove(vega.truthy).insert(data)).resize().run(); }); }); - $.fn.multiline = function(text){ - this.text(text); - this.html(this.html().replace(/\n/g,'
')); - return this; - } - - $(document).ready(function () { - $.getJSON( sensorPath, { - }, function( data ) { - if (data.timezone != jstz.determine().name()) { - $('#tzwarn').show().multiline('Please note that the sensor data you are viewing is located in a different timezone.\nTo view the data from a local perspective, set your locale timezone to ' + data.timezone + '.'); - } - start = new Date(data.timerange.start); - end = new Date(data.timerange.end) - end.setSeconds(end.getSeconds() - 1); // -1 second in case most recent event ends at midnight - start.setHours(0,0,0,0) // get start of first day - end.setHours(0,0,0,0) // get start of last day - - // Initialize picker to the last 2 days of sensor data - nearEnd = new Date(end)//.setDate(end.getDate() - 1) - nearEnd.setDate(nearEnd.getDate() - 1) - picker.setDateRange( - nearEnd, - end, - ) - - // No use looking for data in years outside timerange of sensor data - picker.setOptions({ - dropdowns: { - minYear: start.getFullYear(), - maxYear: end.getFullYear(), - }, + document.onreadystatechange = () => { + if (document.readyState === 'complete') { + fetch(sensorPath, { + method: "GET", + headers: {"Content-Type": "application/json"}, + }) + .then(function(response) { return response.json(); }) + .then(function(data) { + if (data.timezone != jstz.determine().name()) { + document.getElementById('tzwarn').style.display = 'block'; + document.getElementById('tzwarn').innerHTML = 'Please note that the sensor data you are viewing is located in a different timezone.
To view the data from a local perspective, set your locale timezone to ' + data.timezone + '.' + } + start = new Date(data.timerange.start); + end = new Date(data.timerange.end) + end.setSeconds(end.getSeconds() - 1); // -1 second in case most recent event ends at midnight + start.setHours(0,0,0,0) // get start of first day + end.setHours(0,0,0,0) // get start of last day + + // Initialize picker to the last 2 days of sensor data + nearEnd = new Date(end)//.setDate(end.getDate() - 1) + nearEnd.setDate(nearEnd.getDate() - 1) + picker.setDateRange( + nearEnd, + end, + ) + + // No use looking for data in years outside timerange of sensor data + picker.setOptions({ + dropdowns: { + minYear: start.getFullYear(), + maxYear: end.getFullYear(), + }, + }); }); - }); - }); + } + }; From 1a238ef172085a3f4c1aeac9bd6cfbe87c86a855 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 20 Apr 2021 10:14:54 +0200 Subject: [PATCH 16/25] Add FlexMeasures menu (integrate page) and move js util functions --- flexmeasures/ui/static/js/flexmeasures.js | 21 ++++++ flexmeasures/ui/templates/views/sensors.html | 73 +++++++------------- 2 files changed, 46 insertions(+), 48 deletions(-) diff --git a/flexmeasures/ui/static/js/flexmeasures.js b/flexmeasures/ui/static/js/flexmeasures.js index 8dd0becba..eaa045800 100644 --- a/flexmeasures/ui/static/js/flexmeasures.js +++ b/flexmeasures/ui/static/js/flexmeasures.js @@ -310,3 +310,24 @@ function submit_market() { function submit_sensor_type() { $("#sensor_type-form").attr("action", empty_location).submit(); } + + +// Date range utils +export function subtract(oldDate, nDays) { + var newDate = new Date(oldDate) + newDate.setDate(newDate.getDate() - nDays); +return newDate; +} +export function thisMonth(oldDate) { + var d1 = new Date(oldDate) + d1.setDate(1); + var d2 = new Date(d1.getFullYear(), d1.getMonth() + 1, 0); + return [d1, d2]; +}; +export function lastNMonths(oldDate, nMonths) { + var d0 = new Date(oldDate) + var d1 = new Date(d0.getFullYear(), d0.getMonth() - nMonths + 2, 0); + d1.setDate(1); + var d2 = new Date(d0.getFullYear(), d0.getMonth() + 1, 0); + return [d1, d2]; +}; diff --git a/flexmeasures/ui/templates/views/sensors.html b/flexmeasures/ui/templates/views/sensors.html index 5275dff92..d9f5d00c6 100644 --- a/flexmeasures/ui/templates/views/sensors.html +++ b/flexmeasures/ui/templates/views/sensors.html @@ -1,11 +1,16 @@ - - - - FlexMeasures - +{% extends "base.html" %} - - +{% set active_page = "assets" %} + +{% block title %} Assets {% endblock %} + +{% block divs %} + +
+
+
+
+
@@ -16,17 +21,11 @@ - - - -
-
-
-
-
- - - +{% endblock %} \ No newline at end of file From fdcd2f704278a1ddf4749d2af3ac5f794a56ddd6 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 20 Apr 2021 11:13:46 +0200 Subject: [PATCH 17/25] Better Altair js dependency management --- flexmeasures/data/models/time_series.py | 11 ++++++++--- flexmeasures/ui/templates/views/sensors.html | 6 +++--- flexmeasures/ui/utils/view_utils.py | 1 + flexmeasures/utils/config_defaults.py | 6 ++++++ 4 files changed, 18 insertions(+), 6 deletions(-) diff --git a/flexmeasures/data/models/time_series.py b/flexmeasures/data/models/time_series.py index 1fb19dc1b..15c81ead3 100644 --- a/flexmeasures/data/models/time_series.py +++ b/flexmeasures/data/models/time_series.py @@ -3,6 +3,7 @@ import json from altair.utils.html import spec_to_html +from flask import current_app from sqlalchemy.ext.declarative import declared_attr from sqlalchemy.orm import Query, Session import timely_beliefs as tb @@ -140,9 +141,13 @@ def chart( return spec_to_html( chart_specs, "vega-lite", - vega_version="5", - vegaembed_version="6.17.0", - vegalite_version="5.0.0", + vega_version=current_app.config.get("FLEXMEASURES_JS_VERSIONS").vega, + vegaembed_version=current_app.config.get( + "FLEXMEASURES_JS_VERSIONS" + ).vegaembed, + vegalite_version=current_app.config.get( + "FLEXMEASURES_JS_VERSIONS" + ).vegalite, ) return json.dumps(chart_specs) diff --git a/flexmeasures/ui/templates/views/sensors.html b/flexmeasures/ui/templates/views/sensors.html index d9f5d00c6..e395c4cf2 100644 --- a/flexmeasures/ui/templates/views/sensors.html +++ b/flexmeasures/ui/templates/views/sensors.html @@ -13,9 +13,9 @@ - - - + + + diff --git a/flexmeasures/ui/utils/view_utils.py b/flexmeasures/ui/utils/view_utils.py index 6ee924e06..c14ad5f13 100644 --- a/flexmeasures/ui/utils/view_utils.py +++ b/flexmeasures/ui/utils/view_utils.py @@ -84,6 +84,7 @@ def render_flexmeasures_template(html_filename: str, **variables): variables["user_name"] = ( current_user.is_authenticated and current_user.username or "" ) + variables["js_versions"] = current_app.config.get("FLEXMEASURES_JS_VERSIONS") return render_template(html_filename, **variables) diff --git a/flexmeasures/utils/config_defaults.py b/flexmeasures/utils/config_defaults.py index c84ebe111..ad4bbfd5c 100644 --- a/flexmeasures/utils/config_defaults.py +++ b/flexmeasures/utils/config_defaults.py @@ -98,6 +98,12 @@ class Config(object): FLEXMEASURES_REDIS_PORT: int = 6379 FLEXMEASURES_REDIS_DB_NR: int = 0 # Redis per default has 16 databases, [0-15] FLEXMEASURES_REDIS_PASSWORD: Optional[str] = None + FLEXMEASURES_JS_VERSIONS: dict = dict( + vega="5", + vegaembed="6.17.0", + vegalite="5.0.0", + # todo: expand with other js versions used in FlexMeasures + ) # names of settings which cannot be None From 1f404b7e334f90c01a5a748fa446deae6769d1a8 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 22 Apr 2021 09:34:20 +0200 Subject: [PATCH 18/25] Merge class methods (chart_data into search_beliefs) --- flexmeasures/data/models/time_series.py | 36 ++++++------------------- flexmeasures/ui/views/sensors.py | 2 +- 2 files changed, 9 insertions(+), 29 deletions(-) diff --git a/flexmeasures/data/models/time_series.py b/flexmeasures/data/models/time_series.py index 5cee92a36..988bf65fb 100644 --- a/flexmeasures/data/models/time_series.py +++ b/flexmeasures/data/models/time_series.py @@ -40,6 +40,7 @@ def search_beliefs( beliefs_after: Optional[datetime_type] = None, beliefs_before: Optional[datetime_type] = None, source: Optional[Union[int, List[int], str, List[str]]] = None, + as_json: bool = False, ): """Search all beliefs about events for this sensor. @@ -48,8 +49,9 @@ def search_beliefs( :param beliefs_after: only return beliefs formed after this datetime (inclusive) :param beliefs_before: only return beliefs formed before this datetime (inclusive) :param source: search only beliefs by this source (pass its name or id) or list of sources + :param as_json: return beliefs in JSON format (e.g. for use in charts) rather than as BeliefsDataFrame """ - return TimedBelief.search( + bdf = TimedBelief.search( sensor=self, event_starts_after=event_starts_after, event_ends_before=event_ends_before, @@ -57,33 +59,11 @@ def search_beliefs( beliefs_before=beliefs_before, source=source, ) - - def chart_data( - self, - event_starts_after: Optional[datetime_type] = None, - event_ends_before: Optional[datetime_type] = None, - beliefs_after: Optional[datetime_type] = None, - beliefs_before: Optional[datetime_type] = None, - source: Optional[Union[int, List[int], str, List[str]]] = None, - ): - """Get sensor data for use in charts. - - :param event_starts_after: only return beliefs about events that start after this datetime (inclusive) - :param event_ends_before: only return beliefs about events that end before this datetime (inclusive) - :param beliefs_after: only return beliefs formed after this datetime (inclusive) - :param beliefs_before: only return beliefs formed before this datetime (inclusive) - :param source: search only beliefs by this source (pass its name or id) or list of sources - """ - bdf = self.search_beliefs( - event_starts_after=event_starts_after, - event_ends_before=event_ends_before, - beliefs_after=beliefs_after, - beliefs_before=beliefs_before, - source=source, - ) - df = bdf.reset_index() - df["source"] = df["source"].apply(lambda x: x.name) - return df.to_json(orient="records") + if as_json: + df = bdf.reset_index() + df["source"] = df["source"].apply(lambda x: x.name) + return df.to_json(orient="records") + return bdf def chart( self, diff --git a/flexmeasures/ui/views/sensors.py b/flexmeasures/ui/views/sensors.py index 945710b49..ffaa28f90 100644 --- a/flexmeasures/ui/views/sensors.py +++ b/flexmeasures/ui/views/sensors.py @@ -92,7 +92,7 @@ def get_chart_data(self, id, **kwargs): Data for use in charts (in case you have the chart specs already). """ sensor = get_sensor_or_abort(id) - return sensor.chart_data(**kwargs) + return sensor.search_beliefs(as_json=True, **kwargs) @login_required @roles_required("admin") # todo: remove after we check for sensor ownership From 9bd99f93df077b2374e57233ead55ae81b36c395 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 22 Apr 2021 09:35:27 +0200 Subject: [PATCH 19/25] Missing indentation --- flexmeasures/ui/static/js/flexmeasures.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/ui/static/js/flexmeasures.js b/flexmeasures/ui/static/js/flexmeasures.js index eaa045800..66e00404c 100644 --- a/flexmeasures/ui/static/js/flexmeasures.js +++ b/flexmeasures/ui/static/js/flexmeasures.js @@ -316,7 +316,7 @@ function submit_sensor_type() { export function subtract(oldDate, nDays) { var newDate = new Date(oldDate) newDate.setDate(newDate.getDate() - nDays); -return newDate; + return newDate; } export function thisMonth(oldDate) { var d1 = new Date(oldDate) From 65122e4f3cbeb44c9ccbe5b2a48d0cda500c5209 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 22 Apr 2021 09:55:14 +0200 Subject: [PATCH 20/25] Move SensorAPI to API package under dev version --- flexmeasures/api/__init__.py | 2 + flexmeasures/api/dev/__init__.py | 9 ++++ flexmeasures/api/dev/sensors.py | 73 ++++++++++++++++++++++++++++++++ flexmeasures/ui/__init__.py | 3 +- flexmeasures/ui/views/sensors.py | 67 +---------------------------- 5 files changed, 86 insertions(+), 68 deletions(-) create mode 100644 flexmeasures/api/dev/__init__.py create mode 100644 flexmeasures/api/dev/sensors.py diff --git a/flexmeasures/api/__init__.py b/flexmeasures/api/__init__.py index ef838271c..32b008f51 100644 --- a/flexmeasures/api/__init__.py +++ b/flexmeasures/api/__init__.py @@ -105,9 +105,11 @@ def register_at(app: Flask): from flexmeasures.api.v1_2 import register_at as v1_2_register_at from flexmeasures.api.v1_3 import register_at as v1_3_register_at from flexmeasures.api.v2_0 import register_at as v2_0_register_at + from flexmeasures.api.dev import register_at as dev_register_at v1_register_at(app) v1_1_register_at(app) v1_2_register_at(app) v1_3_register_at(app) v2_0_register_at(app) + dev_register_at(app) diff --git a/flexmeasures/api/dev/__init__.py b/flexmeasures/api/dev/__init__.py new file mode 100644 index 000000000..c175be741 --- /dev/null +++ b/flexmeasures/api/dev/__init__.py @@ -0,0 +1,9 @@ +from flask import Flask + + +def register_at(app: Flask): + """This can be used to register FlaskViews.""" + + from flexmeasures.api.dev.sensors import SensorAPI + + SensorAPI.register(app, route_prefix="/api/dev") diff --git a/flexmeasures/api/dev/sensors.py b/flexmeasures/api/dev/sensors.py new file mode 100644 index 000000000..a28e44e1c --- /dev/null +++ b/flexmeasures/api/dev/sensors.py @@ -0,0 +1,73 @@ +from flask_classful import FlaskView, route +from flask_login import login_required +from flask_security import roles_required +from marshmallow import fields +from webargs.flaskparser import use_kwargs +from werkzeug.exceptions import abort + +from flexmeasures.api.common.schemas.times import AwareDateTimeField +from flexmeasures.data.models.time_series import Sensor + + +class SensorAPI(FlaskView): + """ + This view exposes sensor attributes through API endpoints under development. + These endpoints are not yet part of our official API, but support the FlexMeasures UI. + """ + + route_base = "/sensor" + + @login_required + @roles_required("admin") # todo: remove after we check for sensor ownership + @route("//chart/") + @use_kwargs( + { + "event_starts_after": AwareDateTimeField(format="iso", required=False), + "event_ends_before": AwareDateTimeField(format="iso", required=False), + "beliefs_after": AwareDateTimeField(format="iso", required=False), + "beliefs_before": AwareDateTimeField(format="iso", required=False), + "include_data": fields.Boolean(required=False), + "as_html": fields.Boolean(required=False), + "dataset_name": fields.Str(required=False), + }, + location="query", + ) + def get_chart(self, id, **kwargs): + """GET from /sensor//chart""" + sensor = get_sensor_or_abort(id) + return sensor.chart(**kwargs) + + @login_required + @roles_required("admin") # todo: remove after we check for sensor ownership + @route("//chart_data/") + @use_kwargs( + { + "event_starts_after": AwareDateTimeField(format="iso", required=False), + "event_ends_before": AwareDateTimeField(format="iso", required=False), + "beliefs_after": AwareDateTimeField(format="iso", required=False), + "beliefs_before": AwareDateTimeField(format="iso", required=False), + }, + location="query", + ) + def get_chart_data(self, id, **kwargs): + """GET from /sensor//chart_data + + Data for use in charts (in case you have the chart specs already). + """ + sensor = get_sensor_or_abort(id) + return sensor.search_beliefs(as_json=True, **kwargs) + + @login_required + @roles_required("admin") # todo: remove after we check for sensor ownership + def get(self, id: int): + """GET from /sensor/""" + sensor = get_sensor_or_abort(id) + attributes = ["name", "timezone", "timerange"] + return {attr: getattr(sensor, attr) for attr in attributes} + + +def get_sensor_or_abort(id: int) -> Sensor: + sensor = Sensor.query.filter(Sensor.id == id).one_or_none() + if sensor is None: + raise abort(404, f"Sensor {id} not found") + return sensor diff --git a/flexmeasures/ui/__init__.py b/flexmeasures/ui/__init__.py index 2107e5755..6348aa578 100644 --- a/flexmeasures/ui/__init__.py +++ b/flexmeasures/ui/__init__.py @@ -33,12 +33,11 @@ def register_at(app: Flask): from flexmeasures.ui.crud.assets import AssetCrudUI from flexmeasures.ui.crud.users import UserCrudUI - from flexmeasures.ui.views.sensors import SensorUI, SensorAPI + from flexmeasures.ui.views.sensors import SensorUI AssetCrudUI.register(app) UserCrudUI.register(app) SensorUI.register(app) - SensorAPI.register(app) import flexmeasures.ui.views # noqa: F401 this is necessary to load the views diff --git a/flexmeasures/ui/views/sensors.py b/flexmeasures/ui/views/sensors.py index ffaa28f90..5be5e1763 100644 --- a/flexmeasures/ui/views/sensors.py +++ b/flexmeasures/ui/views/sensors.py @@ -1,11 +1,10 @@ -from flask import abort from flask_classful import FlaskView, route from flask_security import login_required, roles_required from marshmallow import fields from webargs.flaskparser import use_kwargs from flexmeasures.api.common.schemas.times import AwareDateTimeField -from flexmeasures.data.models.time_series import Sensor +from flexmeasures.api.dev.sensors import SensorAPI from flexmeasures.ui.utils.view_utils import render_flexmeasures_template @@ -44,67 +43,3 @@ def get(self, id: int): sensor_id=id, msg="", ) - - -class SensorAPI(FlaskView): - """ - This view exposes sensor attributes through API endpoints under development. - These endpoints are not yet part of our official API, but support the FlexMeasures UI. - """ - - route_base = "/sensor" - - @login_required - @roles_required("admin") # todo: remove after we check for sensor ownership - @route("//chart/") - @use_kwargs( - { - "event_starts_after": AwareDateTimeField(format="iso", required=False), - "event_ends_before": AwareDateTimeField(format="iso", required=False), - "beliefs_after": AwareDateTimeField(format="iso", required=False), - "beliefs_before": AwareDateTimeField(format="iso", required=False), - "include_data": fields.Boolean(required=False), - "as_html": fields.Boolean(required=False), - "dataset_name": fields.Str(required=False), - }, - location="query", - ) - def get_chart(self, id, **kwargs): - """GET from /sensor//chart""" - sensor = get_sensor_or_abort(id) - return sensor.chart(**kwargs) - - @login_required - @roles_required("admin") # todo: remove after we check for sensor ownership - @route("//chart_data/") - @use_kwargs( - { - "event_starts_after": AwareDateTimeField(format="iso", required=False), - "event_ends_before": AwareDateTimeField(format="iso", required=False), - "beliefs_after": AwareDateTimeField(format="iso", required=False), - "beliefs_before": AwareDateTimeField(format="iso", required=False), - }, - location="query", - ) - def get_chart_data(self, id, **kwargs): - """GET from /sensor//chart_data - - Data for use in charts (in case you have the chart specs already). - """ - sensor = get_sensor_or_abort(id) - return sensor.search_beliefs(as_json=True, **kwargs) - - @login_required - @roles_required("admin") # todo: remove after we check for sensor ownership - def get(self, id: int): - """GET from /sensor/""" - sensor = get_sensor_or_abort(id) - attributes = ["name", "timezone", "timerange"] - return {attr: getattr(sensor, attr) for attr in attributes} - - -def get_sensor_or_abort(id: int) -> Sensor: - sensor = Sensor.query.filter(Sensor.id == id).one_or_none() - if sensor is None: - raise abort(404, f"Sensor {id} not found") - return sensor From d0ee4311c2d867b1a5fcc9a1726ffa9c22433546 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 22 Apr 2021 10:07:58 +0200 Subject: [PATCH 21/25] Update API path in frontend --- flexmeasures/ui/templates/views/sensors.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/ui/templates/views/sensors.html b/flexmeasures/ui/templates/views/sensors.html index e395c4cf2..2800d5f8c 100644 --- a/flexmeasures/ui/templates/views/sensors.html +++ b/flexmeasures/ui/templates/views/sensors.html @@ -44,7 +44,7 @@ } var sensorId = {{ sensor_id }} - var sensorPath = '/sensor/' + sensorId + var sensorPath = '/api/dev/sensor/' + sensorId var chartSpecsPath = sensorPath + '/chart' var elementId = 'sensorchart' var datasetName = 'sensor_' + sensorId From 85780181603281aea70549afac21bbd2dc536a98 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 22 Apr 2021 12:01:46 +0200 Subject: [PATCH 22/25] flexmeasures.js should be module when exporting functions --- flexmeasures/ui/templates/base.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/ui/templates/base.html b/flexmeasures/ui/templates/base.html index 2bfd0be51..328f35830 100644 --- a/flexmeasures/ui/templates/base.html +++ b/flexmeasures/ui/templates/base.html @@ -209,7 +209,7 @@ - + {% if show_datepicker %} {% endif %} From 8fc6c1a4e8bf3963a2353dd5f253651a65fb7f78 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 22 Apr 2021 16:28:30 +0200 Subject: [PATCH 23/25] Factor out as_html --- flexmeasures/api/dev/sensors.py | 5 +++-- flexmeasures/data/models/time_series.py | 20 ++------------------ flexmeasures/ui/views/sensors.py | 17 ++++++++++++++++- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/flexmeasures/api/dev/sensors.py b/flexmeasures/api/dev/sensors.py index a28e44e1c..ca6794159 100644 --- a/flexmeasures/api/dev/sensors.py +++ b/flexmeasures/api/dev/sensors.py @@ -1,3 +1,5 @@ +import json + from flask_classful import FlaskView, route from flask_login import login_required from flask_security import roles_required @@ -27,7 +29,6 @@ class SensorAPI(FlaskView): "beliefs_after": AwareDateTimeField(format="iso", required=False), "beliefs_before": AwareDateTimeField(format="iso", required=False), "include_data": fields.Boolean(required=False), - "as_html": fields.Boolean(required=False), "dataset_name": fields.Str(required=False), }, location="query", @@ -35,7 +36,7 @@ class SensorAPI(FlaskView): def get_chart(self, id, **kwargs): """GET from /sensor//chart""" sensor = get_sensor_or_abort(id) - return sensor.chart(**kwargs) + return json.dumps(sensor.chart(**kwargs)) @login_required @roles_required("admin") # todo: remove after we check for sensor ownership diff --git a/flexmeasures/data/models/time_series.py b/flexmeasures/data/models/time_series.py index 988bf65fb..4d4f43a81 100644 --- a/flexmeasures/data/models/time_series.py +++ b/flexmeasures/data/models/time_series.py @@ -2,8 +2,6 @@ from datetime import datetime as datetime_type, timedelta import json -from altair.utils.html import spec_to_html -from flask import current_app from sqlalchemy.ext.declarative import declared_attr from sqlalchemy.orm import Query, Session import timely_beliefs as tb @@ -74,10 +72,9 @@ def chart( beliefs_before: Optional[datetime_type] = None, source: Optional[Union[int, List[int], str, List[str]]] = None, include_data: bool = False, - as_html: bool = False, dataset_name: Optional[str] = None, **kwargs, - ) -> str: + ) -> dict: """Create a chart showing sensor data. :param chart_type: currently only "bar_chart" # todo: where can we properly list the available chart types? @@ -87,7 +84,6 @@ def chart( :param beliefs_before: only return beliefs formed before this datetime (inclusive) :param source: search only beliefs by this source (pass its name or id) or list of sources :param include_data: if True, include data in the chart, or if False, exclude data - :param as_html: if True, return the chart as a standalone html, or if False, return as JSON :param dataset_name: optionally name the dataset used in the chart (the default name is sensor_) """ @@ -117,19 +113,7 @@ def chart( ) # Combine chart specs and data chart_specs["datasets"] = {dataset_name: json.loads(data)} - if as_html: - return spec_to_html( - chart_specs, - "vega-lite", - vega_version=current_app.config.get("FLEXMEASURES_JS_VERSIONS").vega, - vegaembed_version=current_app.config.get( - "FLEXMEASURES_JS_VERSIONS" - ).vegaembed, - vegalite_version=current_app.config.get( - "FLEXMEASURES_JS_VERSIONS" - ).vegalite, - ) - return json.dumps(chart_specs) + return chart_specs @property def timerange(self) -> Dict[str, datetime_type]: diff --git a/flexmeasures/ui/views/sensors.py b/flexmeasures/ui/views/sensors.py index 5be5e1763..c9e4eafd0 100644 --- a/flexmeasures/ui/views/sensors.py +++ b/flexmeasures/ui/views/sensors.py @@ -1,3 +1,5 @@ +from altair.utils.html import spec_to_html +from flask import current_app from flask_classful import FlaskView, route from flask_security import login_required, roles_required from marshmallow import fields @@ -32,7 +34,20 @@ class SensorUI(FlaskView): ) def get_chart(self, id, **kwargs): """GET from /sensors//chart""" - return SensorAPI().get_chart(id, include_data=True, as_html=True, **kwargs) + chart_specs = SensorAPI().get_chart( + id, include_data=True, as_html=True, **kwargs + ) + return spec_to_html( + chart_specs, + "vega-lite", + vega_version=current_app.config.get("FLEXMEASURES_JS_VERSIONS").vega, + vegaembed_version=current_app.config.get( + "FLEXMEASURES_JS_VERSIONS" + ).vegaembed, + vegalite_version=current_app.config.get( + "FLEXMEASURES_JS_VERSIONS" + ).vegalite, + ) @login_required @roles_required("admin") # todo: remove after we check for sensor ownership From 48255a5fabd6b6743a17ae5ed711a88e15a476eb Mon Sep 17 00:00:00 2001 From: Felix Claessen <30658763+Flix6x@users.noreply.github.com> Date: Thu, 29 Apr 2021 10:48:45 +0200 Subject: [PATCH 24/25] Move chart specs from the UI package into the data package (#104) * Move code to data package Co-authored-by: F.N. Claessen --- flexmeasures/{ui => data/models}/charts/__init__.py | 0 flexmeasures/{ui => data/models}/charts/belief_charts.py | 2 +- flexmeasures/{ui => data/models}/charts/defaults.py | 0 flexmeasures/{ui => data/models}/charts/readme.md | 0 flexmeasures/data/models/time_series.py | 2 +- 5 files changed, 2 insertions(+), 2 deletions(-) rename flexmeasures/{ui => data/models}/charts/__init__.py (100%) rename flexmeasures/{ui => data/models}/charts/belief_charts.py (90%) rename flexmeasures/{ui => data/models}/charts/defaults.py (100%) rename flexmeasures/{ui => data/models}/charts/readme.md (100%) diff --git a/flexmeasures/ui/charts/__init__.py b/flexmeasures/data/models/charts/__init__.py similarity index 100% rename from flexmeasures/ui/charts/__init__.py rename to flexmeasures/data/models/charts/__init__.py diff --git a/flexmeasures/ui/charts/belief_charts.py b/flexmeasures/data/models/charts/belief_charts.py similarity index 90% rename from flexmeasures/ui/charts/belief_charts.py rename to flexmeasures/data/models/charts/belief_charts.py index 3692deb6b..e9c810de0 100644 --- a/flexmeasures/ui/charts/belief_charts.py +++ b/flexmeasures/data/models/charts/belief_charts.py @@ -1,4 +1,4 @@ -from flexmeasures.ui.charts.defaults import TIME_TITLE, TIME_TOOLTIP_TITLE +from flexmeasures.data.models.charts.defaults import TIME_TITLE, TIME_TOOLTIP_TITLE def bar_chart(title: str, quantity: str = "unknown quantity", unit: str = "a.u."): diff --git a/flexmeasures/ui/charts/defaults.py b/flexmeasures/data/models/charts/defaults.py similarity index 100% rename from flexmeasures/ui/charts/defaults.py rename to flexmeasures/data/models/charts/defaults.py diff --git a/flexmeasures/ui/charts/readme.md b/flexmeasures/data/models/charts/readme.md similarity index 100% rename from flexmeasures/ui/charts/readme.md rename to flexmeasures/data/models/charts/readme.md diff --git a/flexmeasures/data/models/time_series.py b/flexmeasures/data/models/time_series.py index 4d4f43a81..de5770d48 100644 --- a/flexmeasures/data/models/time_series.py +++ b/flexmeasures/data/models/time_series.py @@ -18,7 +18,7 @@ exclude_source_type_filter, ) from flexmeasures.data.services.time_series import collect_time_series_data -from flexmeasures.ui.charts import chart_type_to_chart_specs +from flexmeasures.data.models.charts import chart_type_to_chart_specs from flexmeasures.utils.time_utils import server_now from flexmeasures.utils.flexmeasures_inflection import capitalize From c921ba912840e44f40e4134a28838b17ff0c567a Mon Sep 17 00:00:00 2001 From: Felix Claessen <30658763+Flix6x@users.noreply.github.com> Date: Thu, 29 Apr 2021 10:51:11 +0200 Subject: [PATCH 25/25] Fix regression MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Nicolas Höning --- flexmeasures/data/models/time_series.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/flexmeasures/data/models/time_series.py b/flexmeasures/data/models/time_series.py index de5770d48..aafdc5542 100644 --- a/flexmeasures/data/models/time_series.py +++ b/flexmeasures/data/models/time_series.py @@ -104,7 +104,8 @@ def chart( if include_data: # Set up data - data = self.chart_data( + data = self.search_beliefs( + as_json=True, event_starts_after=event_starts_after, event_ends_before=event_ends_before, beliefs_after=beliefs_after,