Sensor view

flexmeasures/api/common/schemas/times.py

@@ -62,3 +62,16 @@ def ground_from(
             )
             return (pd.Timestamp(start) + offset).to_pydatetime() - start
         return duration
+
+
+class AwareDateTimeField(fields.AwareDateTime):
+    """Field that deserializes to a timezone aware datetime
+    and serializes back to a string."""
+
+    def _deserialize(self, value: str, attr, obj, **kwargs) -> datetime:
+        """
+        Work-around until this PR lands:
+        https://github.com/marshmallow-code/marshmallow/pull/1787
+        """
+        value = value.replace(" ", "+")
+        return fields.AwareDateTime._deserialize(self, value, attr, obj, **kwargs)

flexmeasures/api/v2_0/implementations/assets.py

@@ -66,7 +66,7 @@ def load_asset(admins_only: bool = False):
     should be allowed.
 
         @app.route('/asset/<id>')
-        @check_asset
+        @load_asset
         def get_asset(asset):
             return asset_schema.dump(asset), 200
 

flexmeasures/data/models/data_sources.py

@@ -46,7 +46,7 @@ def label(self):
             return f"schedule by {self.name}"
         elif self.type == "crawling script":
             return f"data retrieved from {self.name}"
-        elif self.type == "demo script":
+        elif self.type in ("demo script", "CLI script"):
             return f"demo data entered by {self.name}"
         else:
             return f"data from {self.name}"

flexmeasures/data/models/time_series.py

@@ -1,6 +1,8 @@
 from typing import List, Dict, Optional, Union, Tuple
 from datetime import datetime as datetime_type, timedelta
+import json
 
+from altair.utils.html import spec_to_html
 from sqlalchemy.ext.declarative import declared_attr
 from sqlalchemy.orm import Query, Session
 import timely_beliefs as tb
@@ -17,6 +19,9 @@
     exclude_source_type_filter,
 )
 from flexmeasures.data.services.time_series import collect_time_series_data
+from flexmeasures.ui.charts import belief_charts_mapping
+from flexmeasures.utils.time_utils import server_now
+from flexmeasures.utils.flexmeasures_inflection import capitalize
 
 
 class Sensor(db.Model, tb.SensorDBMixin):
@@ -51,6 +56,92 @@ def search_beliefs(
             source=source,
         )
 
+    def chart(
+        self,
+        events_not_before: Optional[datetime_type] = None,
+        events_before: Optional[datetime_type] = None,
+        belief_start: Optional[datetime_type] = None,
+        belief_end: Optional[datetime_type] = None,
+        source: Optional[Union[int, List[int], str, List[str]]] = None,
+        chart_type: str = "bar_chart",
+        data_only: bool = False,
+        chart_only: bool = True,
+        as_html: bool = False,
+        dataset_name: Optional[str] = None,
+        **kwargs,
+    ) -> str:
+        """
+
+        :param data_only: return just the data (in case you have the chart specs already)
+        :param as_html: return the chart with data as a standalone html
+        """
+
+        # Set up chart specification
+        if dataset_name is None:
+            dataset_name = "sensor_" + str(self.id)
+        self.sensor_type = (
+            self.name
+        )  # todo remove this placeholder when sensor types are modelled
+        chart = belief_charts_mapping[chart_type](
+            title=capitalize(self.name),
+            quantity=capitalize(self.sensor_type),
+            unit=self.unit,
+            dataset_name=dataset_name,
+            **kwargs,
+        )
+        if chart_only:
+            return json.dumps(chart)
+
+        # Set up data
+        bdf = self.search_beliefs(
+            (events_not_before, events_before), (belief_start, belief_end), source
+        )
+        df = bdf.reset_index()
+        df["source"] = df["source"].apply(lambda x: x.name)
+        data = df.to_json(orient="records")
+        if data_only:
+            return data
+
+        # Combine chart specs and data
+        chart["datasets"] = {dataset_name: json.loads(data)}
+        if as_html:
+            return spec_to_html(
+                chart,
+                "vega-lite",
+                vega_version="5",
+                vegaembed_version="6.17.0",
+                vegalite_version="5.0.0",
+            )
+        return json.dumps(chart)
+
+    @property
+    def timerange(self) -> Dict[str, datetime_type]:
+        """Timerange for which sensor data exists.
+
+        :returns: dictionary with start and end, for example:
+                  {
+                      'start': datetime.datetime(2020, 12, 3, 14, 0, tzinfo=pytz.utc),
+                      'end': datetime.datetime(2020, 12, 3, 14, 30, tzinfo=pytz.utc)
+                  }
+        """
+        least_recent_query = (
+            TimedBelief.query.filter(TimedBelief.sensor == self)
+            .order_by(TimedBelief.event_start.asc())
+            .limit(1)
+        )
+        most_recent_query = (
+            TimedBelief.query.filter(TimedBelief.sensor == self)
+            .order_by(TimedBelief.event_start.desc())
+            .limit(1)
+        )
+        results = least_recent_query.union_all(most_recent_query).all()
+        if not results:
+            # return now in case there is no data for the sensor
+            now = server_now()
+            return dict(start=now, end=now)
+        least_recent, most_recent = results
+        return dict(start=least_recent.event_start, end=most_recent.event_end)
+
     def __repr__(self) -> str:
         return f"<Sensor {self.id}: {self.name}>"
 

flexmeasures/data/scripts/cli_tasks/data_add.py

@@ -1,7 +1,7 @@
 """CLI Tasks for (de)populating the database - most useful in development"""
 
 from datetime import timedelta
-from typing import List
+from typing import List, Optional
 
 import pandas as pd
 import pytz
@@ -10,13 +10,17 @@
 from flask_security.utils import hash_password
 import click
 import getpass
+import timely_beliefs as tb
 
+from flexmeasures.data import db
 from flexmeasures.data.services.forecasting import create_forecasting_jobs
 from flexmeasures.data.services.users import create_user
-from flexmeasures.data.models.time_series import Sensor, SensorSchema
+from flexmeasures.data.models.time_series import Sensor, SensorSchema, TimedBelief
 from flexmeasures.data.models.assets import Asset, AssetSchema
 from flexmeasures.data.models.markets import Market
 from flexmeasures.data.models.weather import WeatherSensor, WeatherSensorSchema
+from flexmeasures.data.models.data_sources import DataSource
+from flexmeasures.utils.time_utils import server_now
 
 
 @click.group("add")
@@ -201,6 +205,76 @@ def add_initial_structure():
     populate_structure(app.db)
 
 
+@fm_add_data.command("beliefs")
+@with_appcontext
+@click.argument("file", type=click.Path(exists=True))
+@click.option(
+    "--sensor-id",
+    required=True,
+    type=click.IntRange(min=1),
+    help="Sensor to which the beliefs pertain.",
+)
+@click.option(
+    "--horizon",
+    required=False,
+    type=click.IntRange(),
+    help="Belief horizon in minutes (use postive horizon for ex-ante beliefs or negative horizon for ex-post beliefs).",
+)
+@click.option(
+    "--cp",
+    required=False,
+    type=click.FloatRange(0, 1),
+    help="Cumulative probability in the range [0, 1].",
+)
+def add_beliefs(
+    file: str, sensor_id: int, horizon: Optional[int] = None, cp: Optional[float] = None
+):
+    """Add sensor data from a csv file.
+
+    Structure your csv file as follows:
+
+        - One header line (will be ignored!)
+        - UTC datetimes in 1st column
+        - values in 2nd column
+
+    For example:
+
+        Date,Inflow (cubic meter)
+        2020-12-03 14:00,212
+        2020-12-03 14:10,215.6
+        2020-12-03 14:20,203.8
+
+    """
+    sensor = Sensor.query.filter(Sensor.id == sensor_id).one_or_none()
+    source = (
+        DataSource.query.filter(DataSource.name == "Seita")
+        .filter(DataSource.type == "CLI script")
+        .one_or_none()
+    )
+    if not source:
+        print("SETTING UP CLI SCRIPT AS NEW DATA SOURCE...")
+        source = DataSource(name="Seita", type="CLI script")
+        db.session.add(source)
+    bdf = tb.read_csv(
+        file,
+        sensor,
+        source=source,
+        cumulative_probability=cp,
+        parse_dates=True,
+        infer_datetime_format=True,
+        **(
+            dict(belief_horizon=timedelta(minutes=horizon))
+            if horizon is not None
+            else dict(
+                belief_time=server_now().astimezone(pytz.timezone(sensor.timezone))
+            )
+        ),
+    )
+    TimedBelief.add(bdf, commit_transaction=False)
+    db.session.commit()
+    print(f"Successfully created beliefs\n{bdf}")
+
+
 @fm_add_data.command("forecasts")
 @with_appcontext
 @click.option(

flexmeasures/ui/__init__.py

@@ -33,9 +33,11 @@ def register_at(app: Flask):
 
     from flexmeasures.ui.crud.assets import AssetCrudUI
     from flexmeasures.ui.crud.users import UserCrudUI
+    from flexmeasures.ui.views.sensors import SensorView
 
     AssetCrudUI.register(app)
     UserCrudUI.register(app)
+    SensorView.register(app)
 
     import flexmeasures.ui.views  # noqa: F401 this is necessary to load the views
 

flexmeasures/ui/charts/__init__.py

@@ -0,0 +1,10 @@
+from inspect import getmembers, isfunction
+
+from . import belief_charts
+from .defaults import apply_chart_defaults
+
+belief_charts_mapping = {
+    k: apply_chart_defaults(v)
+    for k, v in getmembers(belief_charts)
+    if isfunction(v) or isinstance(v, dict)
+}

flexmeasures/ui/charts/belief_charts.py

@@ -0,0 +1,27 @@
+from flexmeasures.ui.charts.defaults import TIME_TITLE, TIME_TOOLTIP_TITLE
+
+
+def bar_chart(title: str, quantity: str = "unknown quantity", unit: str = "a.u."):
+    if not unit:
+        unit = "a.u."
+    return {
+        "description": "A simple bar chart.",
+        "title": title,
+        "mark": "bar",
+        "encoding": {
+            "x": {"field": "event_start", "type": "T", "title": TIME_TITLE},
+            "y": {
+                "field": "event_value",
+                "type": "quantitative",
+                "title": quantity + " (" + unit + ")",
+            },
+            "tooltip": [
+                {"field": "full_date", "title": TIME_TOOLTIP_TITLE, "type": "nominal"},
+                {
+                    "field": "event_value",
+                    "title": quantity + " (" + unit + ")",
+                    "type": "quantitative",
+                },
+            ],
+        },
+    }

flexmeasures/ui/charts/defaults.py

@@ -0,0 +1,42 @@
+from functools import wraps
+from typing import Callable, Union
+
+import altair as alt
+
+
+HEIGHT = 300
+WIDTH = 600
+REDUCED_HEIGHT = REDUCED_WIDTH = 60
+SELECTOR_COLOR = "darkred"
+TIME_FORMAT = "%I:%M %p on %A %b %e, %Y"
+TIME_TOOLTIP_TITLE = "Time and date"
+TIME_TITLE = None
+TIME_SELECTION_TOOLTIP = "Click and drag to select a time window"
+
+
+def apply_chart_defaults(fn):
+    @wraps(fn)
+    def decorated_chart_specs(*args, **kwargs):
+        dataset_name = kwargs.pop("dataset_name", None)
+        if isinstance(fn, Callable):
+            # function that returns a chart specification
+            chart_specs: Union[dict, alt.TopLevelMixin] = fn(*args, **kwargs)
+        else:
+            # not a function, but a direct chart specification
+            chart_specs: Union[dict, alt.TopLevelMixin] = fn
+        if isinstance(chart_specs, alt.TopLevelMixin):
+            chart_specs = chart_specs.to_dict()
+            chart_specs.pop("$schema")
+        if dataset_name:
+            chart_specs["data"] = {"name": dataset_name}
+        chart_specs["height"] = HEIGHT
+        chart_specs["width"] = WIDTH
+        chart_specs["transform"] = [
+            {
+                "as": "full_date",
+                "calculate": f"timeFormat(datum.event_start, '{TIME_FORMAT}')",
+            }
+        ]
+        return chart_specs
+
+    return decorated_chart_specs

flexmeasures/ui/charts/readme.md

@@ -0,0 +1,7 @@
+# Developer docs for adding chart specs
+
+Chart specs can be specified as a dictionary with a vega-lite specification or as an altair chart.
+Alternatively, they can be specified as a function that returns a dict (with vega-lite specs) or an altair chart.
+This approach is useful if you need to parameterize the specification with kwargs.
+
+Todo: support a plug-in architecture, see https://packaging.python.org/guides/creating-and-discovering-plugins/