Issue 502 visually distinguish forecasts from measurements

documentation/changelog.rst

@@ -9,6 +9,7 @@ New features
 -------------
 
 * Hit the replay button to replay what happened, available on the sensor and asset pages [see `PR #463 <http://www.github.com/FlexMeasures/flexmeasures/pull/463>`_]
+* Visually distinguish forecasts/schedules (dashed lines) from measurements (solid lines), and expand the tooltip with timing info regarding the forecast/schedule horizon or measurement lag [see `PR #503 <http://www.github.com/FlexMeasures/flexmeasures/pull/503>`_]
 * The asset page also allows to show sensor data from other assets that belong to the same account [see `PR #500 <http://www.github.com/FlexMeasures/flexmeasures/pull/500>`_]
 * Improved import of time series data from CSV file: 1) drop duplicate records with warning, and 2) allow configuring which column contains explicit recording times for each data point (use case: import forecasts) [see `PR #501 <http://www.github.com/FlexMeasures/flexmeasures/pull/501>`_]
 

flexmeasures/data/models/charts/belief_charts.py

@@ -99,34 +99,68 @@ def chart_for_multiple_sensors(
             }
         shared_tooltip = [
             FIELD_DEFINITIONS["full_date"],
+            dict(
+                field="belief_horizon",
+                type="quantitative",
+                title="Horizon",
+                format=["d", 4],
+                formatType="timedeltaFormat",
+            ),
             {
                 **event_value_field_definition,
                 **dict(title=f"{capitalize(sensor.sensor_type)}"),
             },
             FIELD_DEFINITIONS["source_name"],
             FIELD_DEFINITIONS["source_model"],
         ]
+        line_layer = {
+            "mark": {
+                "type": "line",
+                "interpolate": "step-after"
+                if sensor.event_resolution != timedelta(0)
+                else "linear",
+                "clip": True,
+            },
+            "encoding": {
+                "x": event_start_field_definition,
+                "y": event_value_field_definition,
+                "color": FIELD_DEFINITIONS["source_name"],
+                "strokeDash": {
+                    "field": "belief_horizon",
+                    "type": "quantitative",
+                    "bin": {
+                        # Divide belief horizons into 2 bins by setting a very large bin size.
+                        # The bins should be defined as follows: ex ante (>0) and ex post (<=0),
+                        # but because the bin anchor is included in the ex-ante bin,
+                        # and 0 belief horizons should be attributed to the ex-post bin,
+                        # (and belief horizons are given with 1 ms precision,)
+                        # the bin anchor is set at 1 ms before knowledge time to obtain: ex ante (>=1) and ex post (<1).
+                        "anchor": 1,
+                        "step": 8640000000000000,  # JS max ms for a Date object (NB 10 times less than Python max ms)
+                        # "step": timedelta.max.total_seconds() * 10**2,
+                    },
+                    "legend": {
+                        # Belief horizons binned as 1 ms contain ex-ante beliefs; the other bin contains ex-post beliefs
+                        "labelExpr": "datum.label > 0 ? 'ex ante' : 'ex post'",
+                        "title": "Recorded",
+                    },
+                    "scale": {
+                        # Positive belief horizons are clamped to 1, negative belief horizons are clamped to 0
+                        "domain": [1, 0],
+                        # belief horizons >= 1 ms get a dashed line, belief horizons < 1 ms get a solid line
+                        "range": [[1, 2], [1, 0]],
+                    },
+                },
+                "detail": FIELD_DEFINITIONS["source"],
+            },
+        }
         sensor_specs = {
             "title": capitalize(sensor.name)
             if sensor.name != sensor.sensor_type
             else None,
             "transform": [{"filter": f"datum.sensor.id == {sensor.id}"}],
             "layer": [
-                {
-                    "mark": {
-                        "type": "line",
-                        "interpolate": "step-after"
-                        if sensor.event_resolution != timedelta(0)
-                        else "linear",
-                        "clip": True,
-                    },
-                    "encoding": {
-                        "x": event_start_field_definition,
-                        "y": event_value_field_definition,
-                        "color": FIELD_DEFINITIONS["source_name"],
-                        "detail": FIELD_DEFINITIONS["source"],
-                    },
-                },
+                line_layer,
                 {
                     "mark": {
                         "type": "rect",

flexmeasures/data/models/generic_assets.py

@@ -394,6 +394,7 @@ def search_beliefs(
                 for sensor, bdf in bdf_dict.items():
                     if bdf.event_resolution > timedelta(0):
                         bdf = bdf.resample_events(minimum_non_zero_resolution)
+                    bdf["belief_horizon"] = bdf.belief_horizons.to_numpy()
                     df = simplify_index(
                         bdf,
                         index_levels_to_columns=["source"]

flexmeasures/ui/static/js/flexmeasures.js

@@ -311,13 +311,33 @@ function submit_sensor_type() {
     $("#sensor_type-form").attr("action", empty_location).submit();
 }
 
-/** Tooltips: Register custom formatter for quantities incl. units
-              Usage:
-                  {
-                      'format': [<d3-format>, <sensor unit>],
-                      'formatType': 'quantityWithUnitFormat'
-                  }
-*/
+/** Tooltips: Register custom formatters
+ *
+ *          - Quantities incl. units
+ *            Usage:
+ *                 {
+ *                     'format': [<d3-format>, <sensor unit>],
+ *                     'formatType': 'quantityWithUnitFormat'
+ *                 }
+ *          - Timedeltas measured in human-readable quantities (usually not milliseconds)
+ *            Usage:
+ *                 {
+ *                     'format': [<d3-format>, <breakpoint>],
+ *                     'formatType': 'timedeltaFormat'
+ *                 }
+ *            <d3-format>  is a d3 format identifier, e.g. 'd' for decimal notation, rounded to integer.
+ *                         See https://github.com/d3/d3-format for more details.
+ *            <breakpoint> is a scalar that decides the breakpoint from one duration unit to the next larger unit.
+  *                        For example, a breakpoint of 4 means we format 4 days as '4 days', but 3.96 days as '95 hours'.
+ */
 vega.expressionFunction('quantityWithUnitFormat', function(datum, params) {
     return d3.format(params[0])(datum) + " " + params[1];
 });
+vega.expressionFunction('timedeltaFormat', function(timedelta, params) {
+    return (Math.abs(timedelta) > 1000 * 60 * 60 * 24 * 365.2425 * params[1] ? d3.format(params[0])(timedelta / (1000 * 60 * 60 * 24 * 365.2425)) + " years"
+        : Math.abs(timedelta) > 1000 * 60 * 60 * 24 * params[1] ? d3.format(params[0])(timedelta / (1000 * 60 * 60 * 24)) + " days"
+        : Math.abs(timedelta) > 1000 * 60 * 60 * params[1] ? d3.format(params[0])(timedelta / (1000 * 60 * 60)) + " hours"
+        : Math.abs(timedelta) > 1000 * 60 * params[1] ? d3.format(params[0])(timedelta / (1000 * 60)) + " minutes"
+        : Math.abs(timedelta) > 1000 * params[1] ? d3.format(params[0])(timedelta / 1000) + " seconds"
+        : d3.format(params[0])(timedelta) + " milliseconds");
+});