Issue 385 CLI: show beliefs does not handle NaN well (#516)

The CLI command ``flexmeasures show beliefs`` now supports plotting time series data that includes NaN values, and provides better support for plotting multiple sensors that do not share the same unit.


* Fix showing beliefs for multiple sensors with missing data, and with different units

Signed-off-by: F.N. Claessen <felix@seita.nl>

* Refactor: util function to determine the minimum resolution for resampling

Signed-off-by: F.N. Claessen <felix@seita.nl>

* More explicit function signature

Signed-off-by: F.N. Claessen <felix@seita.nl>

* flake8

Signed-off-by: F.N. Claessen <felix@seita.nl>

* changelog entry

Signed-off-by: F.N. Claessen <felix@seita.nl>

* Add note as to why we set a minimum version requirement

Signed-off-by: F.N. Claessen <felix@seita.nl>

Signed-off-by: F.N. Claessen <felix@seita.nl>

documentation/changelog.rst

@@ -16,7 +16,7 @@ New features
 
 Bugfixes
 -----------
-
+* The CLI command ``flexmeasures show beliefs`` now supports plotting time series data that includes NaN values, and provides better support for plotting multiple sensors that do not share the same unit [see `PR #516 <http://www.github.com/FlexMeasures/flexmeasures/pull/516>`_]
 
 Infrastructure / Support
 ----------------------

flexmeasures/cli/data_show.py

@@ -8,6 +8,7 @@
 from flask.cli import with_appcontext
 from tabulate import tabulate
 from humanize import naturaldelta, naturaltime
+import pandas as pd
 import uniplot
 
 from flexmeasures.data.models.user import Account, AccountRole, User, Role
@@ -19,6 +20,8 @@
 from flexmeasures.data.schemas.account import AccountIdField
 from flexmeasures.data.schemas.sources import DataSourceIdField
 from flexmeasures.data.schemas.times import AwareDateTimeField, DurationField
+from flexmeasures.data.services.time_series import simplify_index
+from flexmeasures.utils.time_utils import determine_minimum_resampling_resolution
 
 
 @click.group("show")
@@ -266,7 +269,9 @@ def plot_beliefs(
     Show a simple plot of belief data directly in the terminal.
     """
     sensors = list(sensors)
-    min_resolution = min([s.event_resolution for s in sensors])
+    minimum_resampling_resolution = determine_minimum_resampling_resolution(
+        [sensor.event_resolution for sensor in sensors]
+    )
 
     # query data
     beliefs_by_sensor = TimedBelief.search(
@@ -276,7 +281,7 @@ def plot_beliefs(
         beliefs_before=belief_time_before,
         source=source,
         one_deterministic_belief_per_event=True,
-        resolution=min_resolution,
+        resolution=minimum_resampling_resolution,
         sum_multiple=False,
     )
     # only keep non-empty
@@ -288,33 +293,37 @@ def plot_beliefs(
     if len(beliefs_by_sensor.keys()) == 0:
         click.echo("No data found!")
         raise click.Abort()
-    first_df = beliefs_by_sensor[sensors[0].name]
+    if all(sensor.unit == sensors[0].unit for sensor in sensors):
+        shared_unit = sensors[0].unit
+    else:
+        shared_unit = ""
+        click.echo(
+            "The y-axis shows no unit, because the selected sensors do not share the same unit."
+        )
+    df = pd.concat([simplify_index(df) for df in beliefs_by_sensor.values()], axis=1)
+    df.columns = beliefs_by_sensor.keys()
 
     # Build title
     if len(sensors) == 1:
         title = f"Beliefs for Sensor '{sensors[0].name}' (Id {sensors[0].id}).\n"
     else:
-        title = f"Beliefs for Sensor(s) [{','.join([s.name for s in sensors])}], (Id(s): [{','.join([str(s.id) for s in sensors])}]).\n"
+        title = f"Beliefs for Sensor(s) [{', '.join([s.name for s in sensors])}], (Id(s): [{', '.join([str(s.id) for s in sensors])}]).\n"
     title += f"Data spans {naturaldelta(duration)} and starts at {start}."
     if belief_time_before:
         title += f"\nOnly beliefs made before: {belief_time_before}."
     if source:
         title += f"\nSource: {source.description}"
-    title += f"\nThe time resolution (x-axis) is {naturaldelta(min_resolution)}."
+    title += f"\nThe time resolution (x-axis) is {naturaldelta(minimum_resampling_resolution)}."
 
     uniplot.plot(
-        [
-            beliefs.event_value
-            for beliefs in [beliefs_by_sensor[sn] for sn in [s.name for s in sensors]]
-        ],
+        [df[col] for col in df.columns],
         title=title,
         color=True,
         lines=True,
-        y_unit=first_df.sensor.unit
-        if len(beliefs_by_sensor) == 1
-        or all(sensor.unit == first_df.sensor.unit for sensor in sensors)
-        else "",
-        legend_labels=[s.name for s in sensors],
+        y_unit=shared_unit,
+        legend_labels=[s.name for s in sensors]
+        if shared_unit
+        else [s.name + f" (in {s.unit})" for s in sensors],
     )
 
 

flexmeasures/data/models/generic_assets.py

@@ -20,7 +20,10 @@
 from flexmeasures.data.queries.annotations import query_asset_annotations
 from flexmeasures.auth.policy import AuthModelMixin, EVERY_LOGGED_IN_USER
 from flexmeasures.utils import geo_utils
-from flexmeasures.utils.time_utils import server_now
+from flexmeasures.utils.time_utils import (
+    determine_minimum_resampling_resolution,
+    server_now,
+)
 
 
 class GenericAssetType(db.Model):
@@ -382,18 +385,13 @@ def search_beliefs(
             from flexmeasures.data.services.time_series import simplify_index
 
             if sensors:
-                condition = list(
-                    bdf.event_resolution
-                    for bdf in bdf_dict.values()
-                    if bdf.event_resolution > timedelta(0)
-                )
-                minimum_non_zero_resolution = (
-                    min(condition) if any(condition) else timedelta(0)
+                minimum_resampling_resolution = determine_minimum_resampling_resolution(
+                    [bdf.event_resolution for bdf in bdf_dict.values()]
                 )
                 df_dict = {}
                 for sensor, bdf in bdf_dict.items():
                     if bdf.event_resolution > timedelta(0):
-                        bdf = bdf.resample_events(minimum_non_zero_resolution)
+                        bdf = bdf.resample_events(minimum_resampling_resolution)
                     bdf["belief_horizon"] = bdf.belief_horizons.to_numpy()
                     df = simplify_index(
                         bdf,
@@ -406,7 +404,7 @@ def search_beliefs(
                         else ["belief_time", "source"],
                         append=True,
                     )
-                    df["sensor"] = sensor  # or some JSONfiable representation
+                    df["sensor"] = sensor  # or some JSONifiable representation
                     df = df.set_index(["sensor"], append=True)
                     df_dict[sensor.id] = df
                 df = pd.concat(df_dict.values())

flexmeasures/utils/time_utils.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import re
 from datetime import datetime, timedelta
 from typing import List, Union, Tuple, Optional
@@ -333,3 +335,15 @@ def duration_isoformat(duration: timedelta):
     # at least one component has to be there.
     repl = ret and "".join(ret) or "T0H"
     return re.sub("%P", repl, "P%P")
+
+
+def determine_minimum_resampling_resolution(
+    event_resolutions: list[timedelta],
+) -> timedelta:
+    """Return minimum non-zero event resolution, or zero resolution if none of the event resolutions is non-zero."""
+    condition = list(
+        event_resolution
+        for event_resolution in event_resolutions
+        if event_resolution > timedelta(0)
+    )
+    return min(condition) if any(condition) else timedelta(0)

requirements/app.in

@@ -50,7 +50,8 @@ marshmallow>=3
 marshmallow-polyfield
 marshmallow-sqlalchemy>=0.23.1
 webargs
-uniplot
+# Minimum version that correctly aligns time series that include NaN values
+uniplot>=0.7.0
 # flask should be after all the flask plugins, because setup might find they ARE flask
 # Minimum here due to Flask-Classful not supporting Werkzeug 2.2.0 yet, see https://github.com/teracyhq/flask-classful/pull/145
 flask>=1.0, <=2.1.2

requirements/app.txt

@@ -339,7 +339,7 @@ typing-extensions==4.3.0
     # via
     #   py-moneyed
     #   pydantic
-uniplot==0.5.0
+uniplot==0.7.0
     # via -r requirements/app.in
 urllib3[socks]==1.26.12
     # via