From 5b6e0e3c3eae5567d72a2cb1ff71672ea536a6b0 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Tue, 6 Jun 2023 22:16:21 +0200
Subject: [PATCH 01/34] Introduce new sensor chart type

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/api/dev/sensors.py               |  1 +
 .../data/models/charts/belief_charts.py       | 98 +++++++++++++++++++
 flexmeasures/ui/templates/base.html           |  2 +-
 3 files changed, 100 insertions(+), 1 deletion(-)

diff --git a/flexmeasures/api/dev/sensors.py b/flexmeasures/api/dev/sensors.py
index 346ebfca9..834d2d5c4 100644
--- a/flexmeasures/api/dev/sensors.py
+++ b/flexmeasures/api/dev/sensors.py
@@ -45,6 +45,7 @@ class SensorAPI(FlaskView):
             "include_asset_annotations": fields.Boolean(required=False),
             "include_account_annotations": fields.Boolean(required=False),
             "dataset_name": fields.Str(required=False),
+            "chart_type": fields.Str(required=False),
             "height": fields.Str(required=False),
             "width": fields.Str(required=False),
         },
diff --git a/flexmeasures/data/models/charts/belief_charts.py b/flexmeasures/data/models/charts/belief_charts.py
index 09782470e..a9f12d340 100644
--- a/flexmeasures/data/models/charts/belief_charts.py
+++ b/flexmeasures/data/models/charts/belief_charts.py
@@ -83,6 +83,104 @@ def bar_chart(
     return chart_specs
 
 
+def matrix_chart(
+    sensor: "Sensor",  # noqa F821
+    event_starts_after: datetime | None = None,
+    event_ends_before: datetime | None = None,
+    **override_chart_specs: dict,
+):
+    unit = sensor.unit if sensor.unit else "a.u."
+    event_value_field_definition = dict(
+        title=f"{capitalize(sensor.sensor_type)} ({unit})",
+        format=[".3~r", unit],
+        formatType="quantityWithUnitFormat",
+        stack=None,
+        **FIELD_DEFINITIONS["event_value"],
+        scale={"scheme": "purplegreen", "domainMid": 0},
+    )
+    event_start_field_definition = dict(
+        field="event_start",
+        type="temporal",
+        title=None,
+        axis={
+            "labelExpr": "timeFormat(datum.value, '%H:%M')",
+            "labelOverlap": True,
+            "labelSeparation": 1,
+        },
+    )
+    event_start_date_field_definition = event_start_field_definition.copy()
+    event_start_field_definition["timeUnit"] = {
+        "unit": "hoursminutesseconds",
+        "step": sensor.event_resolution.total_seconds(),
+    }
+    event_start_date_field_definition["timeUnit"] = {
+        "unit": "yearmonthdate",
+    }
+    if event_starts_after and event_ends_before:
+        event_start_date_field_definition["scale"] = {
+            "domain": [
+                event_starts_after.timestamp() * 10**3,
+                event_ends_before.timestamp() * 10**3
+                - 1,  # prevent showing next date outside selected daterange
+            ],
+        }
+    event_start_date_field_definition["axis"] = {
+        "tickCount": "day",
+        # it's not trivial to center align the labels (vega-lite is missing timeband functionality)
+        "labelBaseline": "line-bottom",
+    }
+    event_start_field_definition["scale"] = {
+        "domain": [
+            {"hours": 0},
+            {"hours": 24},
+        ]
+    }
+    chart_specs = {
+        "description": "A simple heatmap chart showing sensor data.",
+        # the sensor type is already shown as the y-axis title (avoid redundant info)
+        "title": capitalize(sensor.name) if sensor.name != sensor.sensor_type else None,
+        "layer": [
+            {
+                "mark": {
+                    "type": "rect",
+                    "clip": True,
+                },
+                "encoding": {
+                    "x": event_start_field_definition,
+                    "y": event_start_date_field_definition,
+                    # "color": FIELD_DEFINITIONS["source_name"],
+                    "color": event_value_field_definition,
+                    "detail": FIELD_DEFINITIONS["source"],
+                    "opacity": {"value": 0.7},
+                    "tooltip": [
+                        FIELD_DEFINITIONS["full_date"],
+                        {
+                            **event_value_field_definition,
+                            **dict(title=f"{capitalize(sensor.sensor_type)}"),
+                        },
+                        FIELD_DEFINITIONS["source_name_and_id"],
+                        FIELD_DEFINITIONS["source_model"],
+                    ],
+                },
+                "transform": [
+                    {
+                        "calculate": "datum.source.name + ' (ID: ' + datum.source.id + ')'",
+                        "as": "source_name_and_id",
+                    },
+                ],
+            },
+            REPLAY_RULER,
+        ],
+    }
+    for k, v in override_chart_specs.items():
+        chart_specs[k] = v
+    chart_specs["config"] = {
+        "legend": {"orient": "right"},
+        # "legend": {"direction": "horizontal"},
+    }
+    return chart_specs
+
+
 def chart_for_multiple_sensors(
     sensors_to_show: list["Sensor", list["Sensor"]],  # noqa F821
     event_starts_after: datetime | None = None,
diff --git a/flexmeasures/ui/templates/base.html b/flexmeasures/ui/templates/base.html
index edbf456ee..bdfc8c99b 100644
--- a/flexmeasures/ui/templates/base.html
+++ b/flexmeasures/ui/templates/base.html
@@ -247,7 +247,7 @@
 
     async function embedAndLoad(chartSpecsPath, elementId, datasetName, previousResult, startDate, endDate) {
 
-        await vegaEmbed('#'+elementId, chartSpecsPath + 'dataset_name=' + datasetName + '&width=container&include_sensor_annotations=false&include_asset_annotations=false', {{ chart_options | safe }})
+        await vegaEmbed('#'+elementId, chartSpecsPath + 'dataset_name=' + datasetName + '&width=container&include_sensor_annotations=false&include_asset_annotations=false&chart_type=matrix_chart', {{ chart_options | safe }})
         .then(function (result) {
             // result.view is the Vega View, chartSpecsPath is the original Vega-Lite specification
             vegaView = result.view;

From 2239ff7a36ebb3a66289f656a0ef27b9b9566114 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Wed, 7 Jun 2023 11:25:24 +0200
Subject: [PATCH 02/34] Fix alignment of y-axis labels

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/charts/belief_charts.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/flexmeasures/data/models/charts/belief_charts.py b/flexmeasures/data/models/charts/belief_charts.py
index a9f12d340..a34a4011a 100644
--- a/flexmeasures/data/models/charts/belief_charts.py
+++ b/flexmeasures/data/models/charts/belief_charts.py
@@ -126,8 +126,10 @@ def matrix_chart(
         }
     event_start_date_field_definition["axis"] = {
         "tickCount": "day",
-        # it's not trivial to center align the labels (vega-lite is missing timeband functionality)
-        "labelBaseline": "line-bottom",
+        # Center align the date labels
+        "labelOffset": {
+            "expr": "(scale('y', 24 * 60 * 60 * 1000) - scale('y', 0)) / 2"
+        },
     }
     event_start_field_definition["scale"] = {
         "domain": [

From bfa52abe207bf11ea13c8dd8a39a5171c8163b3a Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Wed, 7 Jun 2023 13:25:13 +0200
Subject: [PATCH 03/34] Add button to switch between sensor chart types

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 .../data/models/charts/belief_charts.py       |  2 +-
 flexmeasures/ui/static/css/flexmeasures.css   | 24 +++++++++----
 flexmeasures/ui/templates/base.html           | 34 ++++++++++++++++++-
 flexmeasures/ui/templates/views/sensors.html  |  9 +++++
 4 files changed, 60 insertions(+), 9 deletions(-)

diff --git a/flexmeasures/data/models/charts/belief_charts.py b/flexmeasures/data/models/charts/belief_charts.py
index a34a4011a..024330b3d 100644
--- a/flexmeasures/data/models/charts/belief_charts.py
+++ b/flexmeasures/data/models/charts/belief_charts.py
@@ -96,7 +96,7 @@ def matrix_chart(
         formatType="quantityWithUnitFormat",
         stack=None,
         **FIELD_DEFINITIONS["event_value"],
-        scale={"scheme": "purplegreen", "domainMid": 0},
+        scale={"scheme": "blueorange", "domainMid": 0},
     )
     event_start_field_definition = dict(
         field="event_start",
diff --git a/flexmeasures/ui/static/css/flexmeasures.css b/flexmeasures/ui/static/css/flexmeasures.css
index cd42d220a..3dddde6fe 100644
--- a/flexmeasures/ui/static/css/flexmeasures.css
+++ b/flexmeasures/ui/static/css/flexmeasures.css
@@ -238,7 +238,7 @@ p.error {
     border: none;
 }
 
-.navbar-default .dropdown-menu {
+.dropdown-menu {
     border: none !important;
 }
 
@@ -304,7 +304,7 @@ p.error {
         padding-bottom: 20px;
     }
 
-    .navbar-default .dropdown-menu a {
+    .dropdown-menu a {
         transition: .3s;
         -webkit-transition: .3s;
         -moz-transition: .3s;
@@ -315,7 +315,7 @@ p.error {
 }
 
 @media (max-width: 767px) {
-    .navbar-default .navbar-nav .open .dropdown-menu>li>a {
+    .dropdown-menu>li>a {
         color: var(--nav-default-color);
         transition: .4s;
         -webkit-transition: .4s;
@@ -325,13 +325,23 @@ p.error {
     }
 }
 
-.navbar-default .navbar-nav .open .dropdown-menu,
-.navbar-default .navbar-nav .open .dropdown-menu>li>a {
+.dropdown-menu.center-aligned {
+    right: auto;
+    left: 50%;
+    transform: translateX(-50%);
+}
+
+.dropdown-menu,
+.dropdown-menu>li>a:not(.active) {
     color: var(--nav-default-color);
     background-color: var(--nav-default-background-color);
 }
-.navbar-default .navbar-nav .open .dropdown-menu>li>a:focus,
-.navbar-default .navbar-nav .open .dropdown-menu>li>a:hover {
+.dropdown-menu>li>a.active {
+    color: var(--nav-hover-color);
+    background-color: var(--nav-hover-background-color);
+}
+.dropdown-menu>li>a:focus,
+.dropdown-menu>li>a:hover {
     color: var(--nav-hover-color);
     background-color: var(--nav-hover-background-color);
 }
diff --git a/flexmeasures/ui/templates/base.html b/flexmeasures/ui/templates/base.html
index bdfc8c99b..ca83638d0 100644
--- a/flexmeasures/ui/templates/base.html
+++ b/flexmeasures/ui/templates/base.html
@@ -244,10 +244,42 @@
         storeEndDate = new Date('{{ event_ends_before }}');
     {% endif %}
     let replaySpeed = 100
+    let chartType = 'bar_chart';  // initial chart type // todo: get from session?
+
+    // Update chart type
+    document.addEventListener('DOMContentLoaded', function() {
+        var dropdownItems = document.querySelectorAll('.dropdown-item');
+        for (var i = 0; i < dropdownItems.length; i++) {
+
+            // Set initial chart type
+            if (dropdownItems[i].getAttribute('data-chart-type') === chartType) {
+                dropdownItems[i].classList.add('active');
+            }
+
+            // Add event listener
+            dropdownItems[i].addEventListener('click', function(e) {
+                e.preventDefault();
+                chartType = this.getAttribute('data-chart-type');
+
+                // Update the active state of the dropdown items
+                var dropdownItems = document.querySelectorAll('.dropdown-item');
+                dropdownItems.forEach(item => {
+                    if (item === this) {
+                        item.classList.add('active');
+                    } else {
+                        item.classList.remove('active');
+                    }
+                });
+
+                // Reload daterange
+                picker.setDateRange(picker.getStartDate(), picker.getEndDate());
+            });
+        }
+    });
 
     async function embedAndLoad(chartSpecsPath, elementId, datasetName, previousResult, startDate, endDate) {
 
-        await vegaEmbed('#'+elementId, chartSpecsPath + 'dataset_name=' + datasetName + '&width=container&include_sensor_annotations=false&include_asset_annotations=false&chart_type=matrix_chart', {{ chart_options | safe }})
+        await vegaEmbed('#'+elementId, chartSpecsPath + 'dataset_name=' + datasetName + '&width=container&include_sensor_annotations=false&include_asset_annotations=false&chart_type=' + chartType, {{ chart_options | safe }})
         .then(function (result) {
             // result.view is the Vega View, chartSpecsPath is the original Vega-Lite specification
             vegaView = result.view;
diff --git a/flexmeasures/ui/templates/views/sensors.html b/flexmeasures/ui/templates/views/sensors.html
index a031ca4ef..945f04c82 100644
--- a/flexmeasures/ui/templates/views/sensors.html
+++ b/flexmeasures/ui/templates/views/sensors.html
@@ -14,6 +14,15 @@
                       <div class="left-sidepanel-label">Select dates</div>
                       <div class="sidepanel left-sidepanel">
                           <div id="datepicker"></div>
+                          <div class="dropdown header-action-button">
+                              <button class="btn dropdown-toggle" type="button" id="chartTypeDropdown" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
+                                  Select Chart Type
+                              </button>
+                              <ul class="dropdown-menu center-aligned" aria-labelledby="chartTypeDropdown">
+                                  <li><a class="dropdown-item" href="#" data-chart-type="bar_chart">Time series</a></li>
+                                  <li><a class="dropdown-item" href="#" data-chart-type="matrix_chart">Time of day</a></li>
+                              </ul>
+                          </div>
                       </div>
                   </div>
               </div>

From 0a859830c4a2c6d5978d488ea521b6db0a729e27 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Wed, 7 Jun 2023 13:36:08 +0200
Subject: [PATCH 04/34] Streamline button title with others

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/ui/templates/views/sensors.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flexmeasures/ui/templates/views/sensors.html b/flexmeasures/ui/templates/views/sensors.html
index 945f04c82..ef8533ce6 100644
--- a/flexmeasures/ui/templates/views/sensors.html
+++ b/flexmeasures/ui/templates/views/sensors.html
@@ -16,7 +16,7 @@
                           <div id="datepicker"></div>
                           <div class="dropdown header-action-button">
                               <button class="btn dropdown-toggle" type="button" id="chartTypeDropdown" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
-                                  Select Chart Type
+                                  Select chart
                               </button>
                               <ul class="dropdown-menu center-aligned" aria-labelledby="chartTypeDropdown">
                                   <li><a class="dropdown-item" href="#" data-chart-type="bar_chart">Time series</a></li>

From 4098e76d5b0b006cf82d899914d57f89736c7e9d Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Wed, 7 Jun 2023 13:40:01 +0200
Subject: [PATCH 05/34] Improve alignment of first and last time-of-day labels

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/charts/belief_charts.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/flexmeasures/data/models/charts/belief_charts.py b/flexmeasures/data/models/charts/belief_charts.py
index 024330b3d..adae46c85 100644
--- a/flexmeasures/data/models/charts/belief_charts.py
+++ b/flexmeasures/data/models/charts/belief_charts.py
@@ -104,6 +104,7 @@ def matrix_chart(
         title=None,
         axis={
             "labelExpr": "timeFormat(datum.value, '%H:%M')",
+            "labelFlush": False,
             "labelOverlap": True,
             "labelSeparation": 1,
         },

From 2c98c9de809e5a44d910bd44aeb0038139c33e76 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Wed, 7 Jun 2023 14:25:46 +0200
Subject: [PATCH 06/34] Cleaner solution to prevent showing next date outside
 selected daterange

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/charts/belief_charts.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/flexmeasures/data/models/charts/belief_charts.py b/flexmeasures/data/models/charts/belief_charts.py
index adae46c85..ec7a7e31e 100644
--- a/flexmeasures/data/models/charts/belief_charts.py
+++ b/flexmeasures/data/models/charts/belief_charts.py
@@ -121,8 +121,7 @@ def matrix_chart(
         event_start_date_field_definition["scale"] = {
             "domain": [
                 event_starts_after.timestamp() * 10**3,
-                event_ends_before.timestamp() * 10**3
-                - 1,  # prevent showing next date outside selected daterange
+                event_ends_before.timestamp() * 10**3,
             ],
         }
     event_start_date_field_definition["axis"] = {
@@ -131,6 +130,8 @@ def matrix_chart(
         "labelOffset": {
             "expr": "(scale('y', 24 * 60 * 60 * 1000) - scale('y', 0)) / 2"
         },
+        "labelFlush": False,
+        "labelBound": True,
     }
     event_start_field_definition["scale"] = {
         "domain": [

From 04d0e284f8e396f69cc76555c7f79057a0ab87df Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Wed, 7 Jun 2023 14:29:51 +0200
Subject: [PATCH 07/34] refactor: field definitions

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 .../data/models/charts/belief_charts.py       | 50 ++++++++++---------
 1 file changed, 27 insertions(+), 23 deletions(-)

diff --git a/flexmeasures/data/models/charts/belief_charts.py b/flexmeasures/data/models/charts/belief_charts.py
index ec7a7e31e..6562443ce 100644
--- a/flexmeasures/data/models/charts/belief_charts.py
+++ b/flexmeasures/data/models/charts/belief_charts.py
@@ -102,21 +102,40 @@ def matrix_chart(
         field="event_start",
         type="temporal",
         title=None,
+        timeUnit={
+            "unit": "hoursminutesseconds",
+            "step": sensor.event_resolution.total_seconds(),
+        },
         axis={
             "labelExpr": "timeFormat(datum.value, '%H:%M')",
             "labelFlush": False,
             "labelOverlap": True,
             "labelSeparation": 1,
         },
+        scale={
+            "domain": [
+                {"hours": 0},
+                {"hours": 24},
+            ]
+        }
+    )
+    event_start_date_field_definition = dict(
+        field="event_start",
+        type="temporal",
+        title=None,
+        timeUnit={
+            "unit": "yearmonthdate",
+        },
+        axis={
+            "tickCount": "day",
+            # Center align the date labels
+            "labelOffset": {
+                "expr": "(scale('y', 24 * 60 * 60 * 1000) - scale('y', 0)) / 2"
+            },
+            "labelFlush": False,
+            "labelBound": True,
+        },
     )
-    event_start_date_field_definition = event_start_field_definition.copy()
-    event_start_field_definition["timeUnit"] = {
-        "unit": "hoursminutesseconds",
-        "step": sensor.event_resolution.total_seconds(),
-    }
-    event_start_date_field_definition["timeUnit"] = {
-        "unit": "yearmonthdate",
-    }
     if event_starts_after and event_ends_before:
         event_start_date_field_definition["scale"] = {
             "domain": [
@@ -124,21 +143,6 @@ def matrix_chart(
                 event_ends_before.timestamp() * 10**3,
             ],
         }
-    event_start_date_field_definition["axis"] = {
-        "tickCount": "day",
-        # Center align the date labels
-        "labelOffset": {
-            "expr": "(scale('y', 24 * 60 * 60 * 1000) - scale('y', 0)) / 2"
-        },
-        "labelFlush": False,
-        "labelBound": True,
-    }
-    event_start_field_definition["scale"] = {
-        "domain": [
-            {"hours": 0},
-            {"hours": 24},
-        ]
-    }
     chart_specs = {
         "description": "A simple heatmap chart showing sensor data.",
         # the sensor type is already shown as the y-axis title (avoid redundant info)

From 402ae9e591cf5d9876d8d49e19e8dce448160046 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Wed, 7 Jun 2023 14:31:15 +0200
Subject: [PATCH 08/34] remove obsolete line

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/charts/belief_charts.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/flexmeasures/data/models/charts/belief_charts.py b/flexmeasures/data/models/charts/belief_charts.py
index 6562443ce..00dc8df24 100644
--- a/flexmeasures/data/models/charts/belief_charts.py
+++ b/flexmeasures/data/models/charts/belief_charts.py
@@ -156,7 +156,6 @@ def matrix_chart(
                 "encoding": {
                     "x": event_start_field_definition,
                     "y": event_start_date_field_definition,
-                    # "color": FIELD_DEFINITIONS["source_name"],
                     "color": event_value_field_definition,
                     "detail": FIELD_DEFINITIONS["source"],
                     "opacity": {"value": 0.7},

From ac7430c90bc8d49008c55bde59aaadb2bd16ed58 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Wed, 7 Jun 2023 14:34:38 +0200
Subject: [PATCH 09/34] fix: replay ruler for new chart type

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/charts/belief_charts.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/flexmeasures/data/models/charts/belief_charts.py b/flexmeasures/data/models/charts/belief_charts.py
index 00dc8df24..3e685fe4b 100644
--- a/flexmeasures/data/models/charts/belief_charts.py
+++ b/flexmeasures/data/models/charts/belief_charts.py
@@ -176,7 +176,19 @@ def matrix_chart(
                     },
                 ],
             },
-            REPLAY_RULER,
+            {
+                "data": {"name": "replay"},
+                "mark": {
+                    "type": "rule",
+                },
+                "encoding": {
+                    "x": {
+                        "field": "belief_time",
+                        "type": "temporal",
+                        "timeUnit": "hoursminutesseconds",
+                    },
+                },
+            },
         ],
     }
     for k, v in override_chart_specs.items():

From 16a1a79f794a9e3d04d5a48f3ff1fda9ac6fbf77 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Wed, 7 Jun 2023 15:18:31 +0200
Subject: [PATCH 10/34] black

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/charts/belief_charts.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flexmeasures/data/models/charts/belief_charts.py b/flexmeasures/data/models/charts/belief_charts.py
index 3e685fe4b..008faacc0 100644
--- a/flexmeasures/data/models/charts/belief_charts.py
+++ b/flexmeasures/data/models/charts/belief_charts.py
@@ -117,7 +117,7 @@ def matrix_chart(
                 {"hours": 0},
                 {"hours": 24},
             ]
-        }
+        },
     )
     event_start_date_field_definition = dict(
         field="event_start",

From dfb2a4aac2e865a52bd4d5daf34976a2c3824410 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Tue, 13 Jun 2023 14:57:24 +0200
Subject: [PATCH 11/34] fix: separate overlapping data during the fall DST
 transition

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 .../data/models/charts/belief_charts.py       | 55 +++++++++++++++++++
 1 file changed, 55 insertions(+)

diff --git a/flexmeasures/data/models/charts/belief_charts.py b/flexmeasures/data/models/charts/belief_charts.py
index 008faacc0..bbeb02c45 100644
--- a/flexmeasures/data/models/charts/belief_charts.py
+++ b/flexmeasures/data/models/charts/belief_charts.py
@@ -170,6 +170,10 @@ def matrix_chart(
                     ],
                 },
                 "transform": [
+                    {
+                        # Mask overlapping data during the fall DST transition, which we show later with a special layer
+                        "filter": "timezoneoffset(datum.event_start) >= timezoneoffset(datum.event_start + 60 * 60 * 1000) && timezoneoffset(datum.event_start) <= timezoneoffset(datum.event_start - 60 * 60 * 1000)"
+                    },
                     {
                         "calculate": "datum.source.name + ' (ID: ' + datum.source.id + ')'",
                         "as": "source_name_and_id",
@@ -189,6 +193,7 @@ def matrix_chart(
                     },
                 },
             },
+            create_fall_dst_transition_layer(sensor, event_value_field_definition),
         ],
     }
     for k, v in override_chart_specs.items():
@@ -200,6 +205,56 @@ def matrix_chart(
     return chart_specs
 
 
+def create_fall_dst_transition_layer(sensor, event_value_field_definition) -> dict:
+    """Special layer for showing data during the daylight savings time transition in fall."""
+    return {
+        "mark": {"type": "rect", "clip": True},
+        "encoding": {
+            "x": {
+                "field": "event_start",
+                "type": "temporal",
+                "title": None,
+                "timeUnit": {
+                    "unit": "hoursminutesseconds",
+                    "step": sensor.event_resolution.total_seconds(),
+                },
+            },
+            "y": {
+                "field": "dst_transition_event_start",
+                "type": "temporal",
+                "title": None,
+                "timeUnit": {"unit": "yearmonthdatehours", "step": 12},
+            },
+            "color": event_value_field_definition,
+            "detail": FIELD_DEFINITIONS["source"],
+            "opacity": {"value": 0.7},
+            "tooltip": [
+                FIELD_DEFINITIONS["full_date"],
+                {
+                    **event_value_field_definition,
+                    **dict(title=f"{capitalize(sensor.sensor_type)}"),
+                },
+                FIELD_DEFINITIONS["source_name_and_id"],
+                FIELD_DEFINITIONS["source_model"],
+            ],
+        },
+        "transform": [
+            {
+                "filter": "timezoneoffset(datum.event_start) < timezoneoffset(datum.event_start + 60 * 60 * 1000) || timezoneoffset(datum.event_start) > timezoneoffset(datum.event_start - 60 * 60 * 1000)"
+            },
+            {
+                # Push the more recent hour into the second 12-hour bin
+                "calculate": "timezoneoffset(datum.event_start + 60 * 60 * 1000) > timezoneoffset(datum.event_start) ? datum.event_start : datum.event_start + 12 * 60 * 60 * 1000",
+                "as": "dst_transition_event_start",
+            },
+            {
+                "calculate": "datum.event_start + ' (ID: ' + datum.source.id + ')'",
+                "as": "source_name_and_id",
+            },
+        ],
+    }
+
+
 def chart_for_multiple_sensors(
     sensors_to_show: list["Sensor", list["Sensor"]],  # noqa F821
     event_starts_after: datetime | None = None,

From efed90ec78aea5e8783ee1257130c6a3958661d9 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Tue, 13 Jun 2023 15:08:24 +0200
Subject: [PATCH 12/34] refactor: reuse code

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 .../data/models/charts/belief_charts.py       | 60 ++++++++-----------
 1 file changed, 24 insertions(+), 36 deletions(-)

diff --git a/flexmeasures/data/models/charts/belief_charts.py b/flexmeasures/data/models/charts/belief_charts.py
index bbeb02c45..ab8f6b56e 100644
--- a/flexmeasures/data/models/charts/belief_charts.py
+++ b/flexmeasures/data/models/charts/belief_charts.py
@@ -143,31 +143,29 @@ def matrix_chart(
                 event_ends_before.timestamp() * 10**3,
             ],
         }
+    mark = {"type": "rect", "clip": True, "opacity": 0.7}
+    tooltip = [
+        FIELD_DEFINITIONS["full_date"],
+        {
+            **event_value_field_definition,
+            **dict(title=f"{capitalize(sensor.sensor_type)}"),
+        },
+        FIELD_DEFINITIONS["source_name_and_id"],
+        FIELD_DEFINITIONS["source_model"],
+    ]
     chart_specs = {
         "description": "A simple heatmap chart showing sensor data.",
         # the sensor type is already shown as the y-axis title (avoid redundant info)
         "title": capitalize(sensor.name) if sensor.name != sensor.sensor_type else None,
         "layer": [
             {
-                "mark": {
-                    "type": "rect",
-                    "clip": True,
-                },
+                "mark": mark,
                 "encoding": {
                     "x": event_start_field_definition,
                     "y": event_start_date_field_definition,
                     "color": event_value_field_definition,
                     "detail": FIELD_DEFINITIONS["source"],
-                    "opacity": {"value": 0.7},
-                    "tooltip": [
-                        FIELD_DEFINITIONS["full_date"],
-                        {
-                            **event_value_field_definition,
-                            **dict(title=f"{capitalize(sensor.sensor_type)}"),
-                        },
-                        FIELD_DEFINITIONS["source_name_and_id"],
-                        FIELD_DEFINITIONS["source_model"],
-                    ],
+                    "tooltip": tooltip,
                 },
                 "transform": [
                     {
@@ -193,7 +191,12 @@ def matrix_chart(
                     },
                 },
             },
-            create_fall_dst_transition_layer(sensor, event_value_field_definition),
+            create_fall_dst_transition_layer(
+                mark,
+                event_value_field_definition,
+                event_start_field_definition,
+                tooltip,
+            ),
         ],
     }
     for k, v in override_chart_specs.items():
@@ -205,20 +208,14 @@ def matrix_chart(
     return chart_specs
 
 
-def create_fall_dst_transition_layer(sensor, event_value_field_definition) -> dict:
+def create_fall_dst_transition_layer(
+    mark, event_value_field_definition, event_start_field_definition, tooltip
+) -> dict:
     """Special layer for showing data during the daylight savings time transition in fall."""
     return {
-        "mark": {"type": "rect", "clip": True},
+        "mark": mark,
         "encoding": {
-            "x": {
-                "field": "event_start",
-                "type": "temporal",
-                "title": None,
-                "timeUnit": {
-                    "unit": "hoursminutesseconds",
-                    "step": sensor.event_resolution.total_seconds(),
-                },
-            },
+            "x": event_start_field_definition,
             "y": {
                 "field": "dst_transition_event_start",
                 "type": "temporal",
@@ -227,16 +224,7 @@ def create_fall_dst_transition_layer(sensor, event_value_field_definition) -> di
             },
             "color": event_value_field_definition,
             "detail": FIELD_DEFINITIONS["source"],
-            "opacity": {"value": 0.7},
-            "tooltip": [
-                FIELD_DEFINITIONS["full_date"],
-                {
-                    **event_value_field_definition,
-                    **dict(title=f"{capitalize(sensor.sensor_type)}"),
-                },
-                FIELD_DEFINITIONS["source_name_and_id"],
-                FIELD_DEFINITIONS["source_model"],
-            ],
+            "tooltip": tooltip,
         },
         "transform": [
             {

From cef688e30a0f4993b0619c9109d2dc8757849ec9 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Tue, 13 Jun 2023 20:36:14 +0200
Subject: [PATCH 13/34] fix: tooltip

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/charts/belief_charts.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flexmeasures/data/models/charts/belief_charts.py b/flexmeasures/data/models/charts/belief_charts.py
index ab8f6b56e..493a71ca1 100644
--- a/flexmeasures/data/models/charts/belief_charts.py
+++ b/flexmeasures/data/models/charts/belief_charts.py
@@ -236,7 +236,7 @@ def create_fall_dst_transition_layer(
                 "as": "dst_transition_event_start",
             },
             {
-                "calculate": "datum.event_start + ' (ID: ' + datum.source.id + ')'",
+                "calculate": "datum.source.name + ' (ID: ' + datum.source.id + ')'",
                 "as": "source_name_and_id",
             },
         ],

From fc049a19577d034c47cba851fb172cd7c672a41d Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Tue, 13 Jun 2023 20:46:30 +0200
Subject: [PATCH 14/34] fix: mind the gap

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/charts/belief_charts.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/flexmeasures/data/models/charts/belief_charts.py b/flexmeasures/data/models/charts/belief_charts.py
index 493a71ca1..3357a5ee8 100644
--- a/flexmeasures/data/models/charts/belief_charts.py
+++ b/flexmeasures/data/models/charts/belief_charts.py
@@ -222,6 +222,10 @@ def create_fall_dst_transition_layer(
                 "title": None,
                 "timeUnit": {"unit": "yearmonthdatehours", "step": 12},
             },
+            "y2": {
+                "field": "dst_transition_event_start_next",
+                "timeUnit": {"unit": "yearmonthdatehours", "step": 12},
+            },
             "color": event_value_field_definition,
             "detail": FIELD_DEFINITIONS["source"],
             "tooltip": tooltip,
@@ -235,6 +239,11 @@ def create_fall_dst_transition_layer(
                 "calculate": "timezoneoffset(datum.event_start + 60 * 60 * 1000) > timezoneoffset(datum.event_start) ? datum.event_start : datum.event_start + 12 * 60 * 60 * 1000",
                 "as": "dst_transition_event_start",
             },
+            {
+                # Calculate a time point in the next 12-hour bin
+                "calculate": "datum.dst_transition_event_start + 12 * 60 * 60 * 1000 - 60 * 60 * 1000",
+                "as": "dst_transition_event_start_next",
+            },
             {
                 "calculate": "datum.source.name + ' (ID: ' + datum.source.id + ')'",
                 "as": "source_name_and_id",

From 9571b6442ecd018babcc838d09c7efa5960f9f72 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Wed, 19 Jul 2023 12:32:37 +0200
Subject: [PATCH 15/34] add: warn users when viewing sensor data that includes
 a DST transition

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/ui/static/js/daterange-utils.js | 31 +++++++++++++++++++-
 flexmeasures/ui/templates/base.html          | 23 +++++++++++++--
 flexmeasures/ui/templates/views/sensors.html |  1 +
 3 files changed, 51 insertions(+), 4 deletions(-)

diff --git a/flexmeasures/ui/static/js/daterange-utils.js b/flexmeasures/ui/static/js/daterange-utils.js
index ebf9fde97..1f43467fb 100644
--- a/flexmeasures/ui/static/js/daterange-utils.js
+++ b/flexmeasures/ui/static/js/daterange-utils.js
@@ -38,4 +38,33 @@ function getTimeZoneOffset(date, timeZone) {
     // Positive values are West of GMT, opposite of ISO 8601
     // this matches the output of `Date.getTimeZoneOffset`
     return -(lie - date) / 60 / 1000;
-}
\ No newline at end of file
+}
+
+/**
+ * Count the number of Daylight Saving Time (DST) transitions within a given datetime range.
+ * @param {Date} startDate - The start date of the datetime range.
+ * @param {Date} endDate - The end date of the datetime range.
+ * @param {number} increment - The number of days to increment between iterations.
+ * @returns {number} The count of DST transitions within the specified range.
+ */
+export function countDSTTransitions(startDate, endDate, increment) {
+    let transitions = 0;
+    let currentDate = new Date(startDate);
+    let nextDate = new Date(startDate);
+
+    while (currentDate <= endDate) {
+        const currentOffset = currentDate.getTimezoneOffset();
+        nextDate.setDate(currentDate.getDate() + increment);
+        if (nextDate > endDate) {
+            nextDate = endDate;
+        }
+        const nextOffset = nextDate.getTimezoneOffset();
+
+        if (currentOffset !== nextOffset) {
+            transitions++;
+        }
+        currentDate.setDate(currentDate.getDate() + increment);
+    }
+
+    return transitions;
+}
diff --git a/flexmeasures/ui/templates/base.html b/flexmeasures/ui/templates/base.html
index ca83638d0..2793c1558 100644
--- a/flexmeasures/ui/templates/base.html
+++ b/flexmeasures/ui/templates/base.html
@@ -230,7 +230,7 @@
     <!-- Render Charts -->
     <script type="module" type="text/javascript">
 
-    import { subtract, thisMonth, lastNMonths, getOffsetBetweenTimezonesForDate } from "{{ url_for('flexmeasures_ui.static', filename='js/daterange-utils.js') }}";
+    import { subtract, thisMonth, lastNMonths, countDSTTransitions, getOffsetBetweenTimezonesForDate } from "{{ url_for('flexmeasures_ui.static', filename='js/daterange-utils.js') }}";
     import { partition, updateBeliefs, beliefTimedelta, setAbortableTimeout} from "{{ url_for('flexmeasures_ui.static', filename='js/replay-utils.js') }}";
 
     let vegaView;
@@ -242,6 +242,7 @@
     {% if event_starts_after and event_ends_before %}
         storeStartDate = new Date('{{ event_starts_after }}');
         storeEndDate = new Date('{{ event_ends_before }}');
+        checkDSTTransitions(storeStartDate, storeEndDate);
     {% endif %}
     let replaySpeed = 100
     let chartType = 'bar_chart';  // initial chart type // todo: get from session?
@@ -277,6 +278,21 @@
         }
     });
 
+    function checkDSTTransitions(startDate, endDate) {
+        var numDSTTransitions = countDSTTransitions(startDate, endDate, 90)
+        if (numDSTTransitions != 0) {
+            document.getElementById('dstwarn').style.display = 'block';
+            if (numDSTTransitions == 1) {
+                document.getElementById('dstwarn').innerHTML = 'Please note that the sensor data you are viewing includes a daylight saving time (DST) transition.';
+            } else {
+                document.getElementById('dstwarn').innerHTML = 'Please note that the sensor data you are viewing includes ' + numDSTTransitions + ' daylight saving time (DST) transitions.';
+            }
+        }
+        else {
+            document.getElementById('dstwarn').style.display = 'none';
+        }
+    }
+
     async function embedAndLoad(chartSpecsPath, elementId, datasetName, previousResult, startDate, endDate) {
 
         await vegaEmbed('#'+elementId, chartSpecsPath + 'dataset_name=' + datasetName + '&width=container&include_sensor_annotations=false&include_asset_annotations=false&chart_type=' + chartType, {{ chart_options | safe }})
@@ -369,6 +385,7 @@
         stopReplay()
 
         $("#spinner").show();
+        checkDSTTransitions(startDate, endDate)
         Promise.all([
             // Fetch time series data
             fetch(dataPath + '/chart_data/?event_starts_after=' + queryStartDate + '&event_ends_before=' + queryEndDate, {
@@ -428,8 +445,8 @@
                 var offsetDifference = getOffsetBetweenTimezonesForDate(new Date(), data.timezone, jstz.determine().name());
                 if (offsetDifference != 0) {
                     document.getElementById('tzwarn').style.display = 'block';
-                    var offsetNotice = (offsetDifference > 0) ? 'which is currently ahead by ' + offsetDifference + 'minutes' : 'which is currently behind by ' + offsetDifference + ' minutes'
-                    document.getElementById('tzwarn').innerHTML = 'Please note that the sensor data you are viewing is located in a different timezone (' + offsetNotice + ').<br/>To view the data from a local perspective, set your locale timezone to ' + data.timezone + '.'
+                    var offsetNotice = (offsetDifference > 0) ? 'which is currently ahead by ' + offsetDifference + 'minutes' : 'which is currently behind by ' + offsetDifference + ' minutes';
+                    document.getElementById('tzwarn').innerHTML = 'Please note that the sensor data you are viewing is located in a different timezone (' + offsetNotice + ').<br/>To view the data from a local perspective, set your locale timezone to ' + data.timezone + '.';
                 }
                 {% if active_page == "assets" %}
                     var timerangeVar = 'timerange_of_sensors_to_show';
diff --git a/flexmeasures/ui/templates/views/sensors.html b/flexmeasures/ui/templates/views/sensors.html
index ef8533ce6..35cea85f2 100644
--- a/flexmeasures/ui/templates/views/sensors.html
+++ b/flexmeasures/ui/templates/views/sensors.html
@@ -8,6 +8,7 @@
 
       <div class="sensor-data charts text-center">
           <div class="row"><div class="alert alert-info" id="tzwarn" style="display:none;"></div></div>
+          <div class="row"><div class="alert alert-info" id="dstwarn" style="display:none;"></div></div>
           <div class="row on-top-md">
               <div class="col-sm-2">
                   <div class="sidepanel-container">

From abb09980c10d72e4b07413289a8b033ca5ee5628 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Wed, 19 Jul 2023 12:34:11 +0200
Subject: [PATCH 16/34] add: also warn users when viewing asset data that
 includes a DST transition

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/ui/templates/crud/asset.html | 1 +
 1 file changed, 1 insertion(+)

diff --git a/flexmeasures/ui/templates/crud/asset.html b/flexmeasures/ui/templates/crud/asset.html
index a128a07c0..82ee6846d 100644
--- a/flexmeasures/ui/templates/crud/asset.html
+++ b/flexmeasures/ui/templates/crud/asset.html
@@ -8,6 +8,7 @@
 
 <div class="container-fluid">
     <div class="row"><div class="alert alert-info" id="tzwarn" style="display:none;"></div></div>
+    <div class="row"><div class="alert alert-info" id="dstwarn" style="display:none;"></div></div>
     <div class="row">
         <div class="col-sm-2 on-top-md">
             <div class="header-action-button">

From d6df4a6cbf5fb97976de4282a2bb23692302a57b Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Wed, 19 Jul 2023 12:46:40 +0200
Subject: [PATCH 17/34] add: UI styling

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/ui/static/css/flexmeasures.css  | 2 +-
 flexmeasures/ui/templates/crud/asset.html    | 6 ++++--
 flexmeasures/ui/templates/views/sensors.html | 6 ++++--
 3 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/flexmeasures/ui/static/css/flexmeasures.css b/flexmeasures/ui/static/css/flexmeasures.css
index 3dddde6fe..fdcf711f3 100644
--- a/flexmeasures/ui/static/css/flexmeasures.css
+++ b/flexmeasures/ui/static/css/flexmeasures.css
@@ -189,7 +189,7 @@ p.error {
     font-weight: 700;
 }
 
-#tzwarn {
+#tzwarn,#dstwarn {
     margin-top: 20px;
     margin-bottom: 20px;
     color: var(--secondary-color);
diff --git a/flexmeasures/ui/templates/crud/asset.html b/flexmeasures/ui/templates/crud/asset.html
index 82ee6846d..2298a4f42 100644
--- a/flexmeasures/ui/templates/crud/asset.html
+++ b/flexmeasures/ui/templates/crud/asset.html
@@ -7,8 +7,10 @@
 {% block divs %}
 
 <div class="container-fluid">
-    <div class="row"><div class="alert alert-info" id="tzwarn" style="display:none;"></div></div>
-    <div class="row"><div class="alert alert-info" id="dstwarn" style="display:none;"></div></div>
+    <div class="row">
+        <div class="alert alert-info" id="tzwarn" style="display:none;"></div>
+        <div class="alert alert-info" id="dstwarn" style="display:none;"></div>
+    </div>
     <div class="row">
         <div class="col-sm-2 on-top-md">
             <div class="header-action-button">
diff --git a/flexmeasures/ui/templates/views/sensors.html b/flexmeasures/ui/templates/views/sensors.html
index 35cea85f2..d8be4b1a8 100644
--- a/flexmeasures/ui/templates/views/sensors.html
+++ b/flexmeasures/ui/templates/views/sensors.html
@@ -7,8 +7,10 @@
 {% block divs %}
 
       <div class="sensor-data charts text-center">
-          <div class="row"><div class="alert alert-info" id="tzwarn" style="display:none;"></div></div>
-          <div class="row"><div class="alert alert-info" id="dstwarn" style="display:none;"></div></div>
+          <div class="row">
+              <div class="alert alert-info" id="tzwarn" style="display:none;"></div>
+              <div class="alert alert-info" id="dstwarn" style="display:none;"></div>
+          </div>
           <div class="row on-top-md">
               <div class="col-sm-2">
                   <div class="sidepanel-container">

From dc8f7490e03eefd47d8378d6a81f4a063d2ffbb1 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Thu, 20 Jul 2023 22:03:52 +0200
Subject: [PATCH 18/34] add: add timezone to tooltip during fall DST transition

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 .../data/models/charts/belief_charts.py       |  8 ++-
 flexmeasures/ui/static/js/flexmeasures.js     | 54 ++++++++++++-------
 2 files changed, 42 insertions(+), 20 deletions(-)

diff --git a/flexmeasures/data/models/charts/belief_charts.py b/flexmeasures/data/models/charts/belief_charts.py
index 3357a5ee8..f94055cab 100644
--- a/flexmeasures/data/models/charts/belief_charts.py
+++ b/flexmeasures/data/models/charts/belief_charts.py
@@ -192,6 +192,7 @@ def matrix_chart(
                 },
             },
             create_fall_dst_transition_layer(
+                sensor.timezone,
                 mark,
                 event_value_field_definition,
                 event_start_field_definition,
@@ -209,7 +210,7 @@ def matrix_chart(
 
 
 def create_fall_dst_transition_layer(
-    mark, event_value_field_definition, event_start_field_definition, tooltip
+    timezone, mark, event_value_field_definition, event_start_field_definition, tooltip
 ) -> dict:
     """Special layer for showing data during the daylight savings time transition in fall."""
     return {
@@ -228,7 +229,10 @@ def create_fall_dst_transition_layer(
             },
             "color": event_value_field_definition,
             "detail": FIELD_DEFINITIONS["source"],
-            "tooltip": tooltip,
+            "tooltip": [
+                {"field": "event_start", "type": "temporal", "title": "Timezone", "timeUnit": "utc", "format": [timezone], "formatType": "timezoneFormat"},
+                *tooltip,
+            ],
         },
         "transform": [
             {
diff --git a/flexmeasures/ui/static/js/flexmeasures.js b/flexmeasures/ui/static/js/flexmeasures.js
index 24a7ba4b6..73374c964 100644
--- a/flexmeasures/ui/static/js/flexmeasures.js
+++ b/flexmeasures/ui/static/js/flexmeasures.js
@@ -311,28 +311,31 @@ function submit_sensor_type() {
     $("#sensor_type-form").attr("action", empty_location).submit();
 }
 
-/** Tooltips: Register custom formatters
- *
- *          - Quantities incl. units
- *            Usage:
- *                 {
- *                     'format': [<d3-format>, <sensor unit>],
- *                     'formatType': 'quantityWithUnitFormat'
- *                 }
- *          - Timedeltas measured in human-readable quantities (usually not milliseconds)
- *            Usage:
- *                 {
- *                     'format': [<d3-format>, <breakpoint>],
- *                     'formatType': 'timedeltaFormat'
- *                 }
- *            <d3-format>  is a d3 format identifier, e.g. 'd' for decimal notation, rounded to integer.
- *                         See https://github.com/d3/d3-format for more details.
- *            <breakpoint> is a scalar that decides the breakpoint from one duration unit to the next larger unit.
-  *                        For example, a breakpoint of 4 means we format 4 days as '4 days', but 3.96 days as '95 hours'.
+/** Tooltips: Register custom formatters */
+
+/* Quantities incl. units
+ * Usage:
+ *     {
+ *         'format': [<d3-format>, <sensor unit>],
+ *         'formatType': 'quantityWithUnitFormat'
+ *     }
  */
 vega.expressionFunction('quantityWithUnitFormat', function(datum, params) {
     return d3.format(params[0])(datum) + " " + params[1];
 });
+
+/*
+ * Timedeltas measured in human-readable quantities (usually not milliseconds)
+ * Usage:
+ *     {
+ *         'format': [<d3-format>, <breakpoint>],
+ *         'formatType': 'timedeltaFormat'
+ *     }
+ * <d3-format>  is a d3 format identifier, e.g. 'd' for decimal notation, rounded to integer.
+ *              See https://github.com/d3/d3-format for more details.
+ * <breakpoint> is a scalar that decides the breakpoint from one duration unit to the next larger unit.
+ *              For example, a breakpoint of 4 means we format 4 days as '4 days', but 3.96 days as '95 hours'.
+ */
 vega.expressionFunction('timedeltaFormat', function(timedelta, params) {
     return (Math.abs(timedelta) > 1000 * 60 * 60 * 24 * 365.2425 * params[1] ? d3.format(params[0])(timedelta / (1000 * 60 * 60 * 24 * 365.2425)) + " years"
         : Math.abs(timedelta) > 1000 * 60 * 60 * 24 * params[1] ? d3.format(params[0])(timedelta / (1000 * 60 * 60 * 24)) + " days"
@@ -341,3 +344,18 @@ vega.expressionFunction('timedeltaFormat', function(timedelta, params) {
         : Math.abs(timedelta) > 1000 * params[1] ? d3.format(params[0])(timedelta / 1000) + " seconds"
         : d3.format(params[0])(timedelta) + " milliseconds");
 });
+
+/*
+ * Timezone offset including IANA timezone name
+ * Usage:
+ *     {
+ *         'format': [<IANA timezone name, e.g. 'Europe/Amsterdam'>],
+ *         'formatType': 'timezoneFormat'
+ *     }
+ */
+vega.expressionFunction('timezoneFormat', function(date, params) {
+    const timezoneString = params[0];
+    const tzOffsetNumber = date.getTimezoneOffset();
+    const tzDate = new Date(0,0,0,0,Math.abs(tzOffsetNumber));
+    return `${ tzOffsetNumber > 0 ? '-' : '+'}${("" + tzDate.getHours()).padStart(2, '0')}:${("" + tzDate.getMinutes()).padStart(2, '0')}` + ' (' + timezoneString + ')';
+});

From bf29897cb21afb26b3306285a5d309884e5423aa Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Thu, 20 Jul 2023 22:08:39 +0200
Subject: [PATCH 19/34] style: black

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/charts/belief_charts.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/flexmeasures/data/models/charts/belief_charts.py b/flexmeasures/data/models/charts/belief_charts.py
index f94055cab..c958c0d83 100644
--- a/flexmeasures/data/models/charts/belief_charts.py
+++ b/flexmeasures/data/models/charts/belief_charts.py
@@ -230,7 +230,14 @@ def create_fall_dst_transition_layer(
             "color": event_value_field_definition,
             "detail": FIELD_DEFINITIONS["source"],
             "tooltip": [
-                {"field": "event_start", "type": "temporal", "title": "Timezone", "timeUnit": "utc", "format": [timezone], "formatType": "timezoneFormat"},
+                {
+                    "field": "event_start",
+                    "type": "temporal",
+                    "title": "Timezone",
+                    "timeUnit": "utc",
+                    "format": [timezone],
+                    "formatType": "timezoneFormat",
+                },
                 *tooltip,
             ],
         },

From b2a71a1976e03a7908a67d6efdcc95374c965a61 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Wed, 26 Jul 2023 10:43:08 +0200
Subject: [PATCH 20/34] feat: remember selected chart type within session

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/api/dev/sensors.py     | 8 +++++++-
 flexmeasures/ui/templates/base.html | 2 +-
 flexmeasures/ui/utils/view_utils.py | 4 ++++
 flexmeasures/ui/views/sensors.py    | 3 ++-
 4 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/flexmeasures/api/dev/sensors.py b/flexmeasures/api/dev/sensors.py
index 834d2d5c4..4c00f36a9 100644
--- a/flexmeasures/api/dev/sensors.py
+++ b/flexmeasures/api/dev/sensors.py
@@ -18,7 +18,10 @@
 from flexmeasures.data.models.generic_assets import GenericAsset
 from flexmeasures.data.models.time_series import Sensor
 from flexmeasures.data.services.annotations import prepare_annotations_for_chart
-from flexmeasures.ui.utils.view_utils import set_time_range_for_session
+from flexmeasures.ui.utils.view_utils import (
+    set_chart_type_for_session,
+    set_time_range_for_session,
+)
 
 
 class SensorAPI(FlaskView):
@@ -64,9 +67,12 @@ def get_chart(self, id: int, sensor: Sensor, **kwargs):
         - "beliefs_after" (see the `timely-beliefs documentation <https://github.com/SeitaBV/timely-beliefs/blob/main/timely_beliefs/docs/timing.md/#events-and-sensors>`_)
         - "beliefs_before" (see the `timely-beliefs documentation <https://github.com/SeitaBV/timely-beliefs/blob/main/timely_beliefs/docs/timing.md/#events-and-sensors>`_)
         - "include_data" (if true, chart specs include the data; if false, use the `GET /api/dev/sensor/(id)/chart_data/ <../api/dev.html#get--api-dev-sensor-(id)-chart_data->`_ endpoint to fetch data)
+        - "chart_type" (currently 'bar_chart' and 'matrix_chart' are supported types)
         - "width" (an integer number of pixels; without it, the chart will be scaled to the full width of the container (hint: use ``<div style="width: 100%;">`` to set a div width to 100%)
         - "height" (an integer number of pixels; without it, FlexMeasures sets a default, currently 300)
         """
+        # Store selected chart type and time range as session variables, for a consistent UX across UI page loads
+        set_chart_type_for_session()
         set_time_range_for_session()
         return json.dumps(sensor.chart(**kwargs))
 
diff --git a/flexmeasures/ui/templates/base.html b/flexmeasures/ui/templates/base.html
index 8bf723885..cbb477685 100644
--- a/flexmeasures/ui/templates/base.html
+++ b/flexmeasures/ui/templates/base.html
@@ -247,7 +247,7 @@
         storeEndDate = new Date('{{ event_ends_before }}');
     {% endif %}
     let replaySpeed = 100
-    let chartType = 'bar_chart';  // initial chart type // todo: get from session?
+    let chartType = '{{ chart_type }}';  // initial chart type from session variable
 
     // Update chart type
     document.addEventListener('DOMContentLoaded', function() {
diff --git a/flexmeasures/ui/utils/view_utils.py b/flexmeasures/ui/utils/view_utils.py
index 0c9bf07da..717d95f7e 100644
--- a/flexmeasures/ui/utils/view_utils.py
+++ b/flexmeasures/ui/utils/view_utils.py
@@ -110,6 +110,10 @@ def clear_session():
         del session[skey]
 
 
+def set_chart_type_for_session():
+    session["chart_type"] = request.values.get("chart_type")
+
+
 def set_time_range_for_session():
     """Set period on session if they are not yet set.
     The daterangepicker sends times as tz-aware UTC strings.
diff --git a/flexmeasures/ui/views/sensors.py b/flexmeasures/ui/views/sensors.py
index 016fd168c..7f6ae5892 100644
--- a/flexmeasures/ui/views/sensors.py
+++ b/flexmeasures/ui/views/sensors.py
@@ -1,7 +1,7 @@
 import json
 
 from altair.utils.html import spec_to_html
-from flask import current_app
+from flask import current_app, session
 from flask_classful import FlaskView, route
 from flask_security import auth_required, login_required
 from marshmallow import fields
@@ -69,5 +69,6 @@ def get(self, id: int):
         return render_flexmeasures_template(
             "views/sensors.html",
             sensor_id=id,
+            chart_type=session.get("chart_type", "bar_chart"),
             msg="",
         )

From 2d09c7c1a00eb21ef72cfcc7e2981def9ebc232f Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Wed, 26 Jul 2023 11:03:00 +0200
Subject: [PATCH 21/34] refactor: move towards a single function to set session
 variables from request variables

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/api/dev/sensors.py     | 10 +++-------
 flexmeasures/api/v3_0/assets.py     |  5 +++--
 flexmeasures/ui/utils/view_utils.py |  9 +++++++--
 3 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/flexmeasures/api/dev/sensors.py b/flexmeasures/api/dev/sensors.py
index 4c00f36a9..19f22300e 100644
--- a/flexmeasures/api/dev/sensors.py
+++ b/flexmeasures/api/dev/sensors.py
@@ -18,10 +18,7 @@
 from flexmeasures.data.models.generic_assets import GenericAsset
 from flexmeasures.data.models.time_series import Sensor
 from flexmeasures.data.services.annotations import prepare_annotations_for_chart
-from flexmeasures.ui.utils.view_utils import (
-    set_chart_type_for_session,
-    set_time_range_for_session,
-)
+from flexmeasures.ui.utils.view_utils import set_session_variables
 
 
 class SensorAPI(FlaskView):
@@ -71,9 +68,8 @@ def get_chart(self, id: int, sensor: Sensor, **kwargs):
         - "width" (an integer number of pixels; without it, the chart will be scaled to the full width of the container (hint: use ``<div style="width: 100%;">`` to set a div width to 100%)
         - "height" (an integer number of pixels; without it, FlexMeasures sets a default, currently 300)
         """
-        # Store selected chart type and time range as session variables, for a consistent UX across UI page loads
-        set_chart_type_for_session()
-        set_time_range_for_session()
+        # Store selected time range and chart type as session variables, for a consistent UX across UI page loads
+        set_session_variables("event_starts_after", "event_ends_before", "chart_type")
         return json.dumps(sensor.chart(**kwargs))
 
     @route("/<id>/chart_data/")
diff --git a/flexmeasures/api/v3_0/assets.py b/flexmeasures/api/v3_0/assets.py
index db505e34a..1f6fbdde5 100644
--- a/flexmeasures/api/v3_0/assets.py
+++ b/flexmeasures/api/v3_0/assets.py
@@ -16,7 +16,7 @@
 from flexmeasures.api.common.schemas.generic_assets import AssetIdField
 from flexmeasures.api.common.schemas.users import AccountIdField
 from flexmeasures.utils.coding_utils import flatten_unique
-from flexmeasures.ui.utils.view_utils import set_time_range_for_session
+from flexmeasures.ui.utils.view_utils import set_session_variables
 
 
 asset_schema = AssetSchema()
@@ -284,7 +284,8 @@ def get_chart(self, id: int, asset: GenericAsset, **kwargs):
 
         .. :quickref: Chart; Download a chart with time series
         """
-        set_time_range_for_session()
+        # Store selected time range as session variables, for a consistent UX across UI page loads
+        set_session_variables("event_starts_after", "event_ends_before")
         return json.dumps(asset.chart(**kwargs))
 
     @route("/<id>/chart_data/")
diff --git a/flexmeasures/ui/utils/view_utils.py b/flexmeasures/ui/utils/view_utils.py
index 717d95f7e..40755dc85 100644
--- a/flexmeasures/ui/utils/view_utils.py
+++ b/flexmeasures/ui/utils/view_utils.py
@@ -110,8 +110,13 @@ def clear_session():
         del session[skey]
 
 
-def set_chart_type_for_session():
-    session["chart_type"] = request.values.get("chart_type")
+def set_session_variables(*var_names: str):
+    """Store request values as session variables, for a consistent UX across UI page loads.
+
+    >>> set_session_variables("event_starts_after", "event_ends_before", "chart_type")
+    """
+    for var_name in var_names:
+        session[var_name] = request.values.get(var_name)
 
 
 def set_time_range_for_session():

From 01759dadca7978758955e6c90b3dc40bb9de021d Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Thu, 27 Jul 2023 10:26:36 +0200
Subject: [PATCH 22/34] refactor: move template variables

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/ui/utils/view_utils.py | 1 +
 flexmeasures/ui/views/sensors.py    | 1 -
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/flexmeasures/ui/utils/view_utils.py b/flexmeasures/ui/utils/view_utils.py
index 40755dc85..dbda2876c 100644
--- a/flexmeasures/ui/utils/view_utils.py
+++ b/flexmeasures/ui/utils/view_utils.py
@@ -50,6 +50,7 @@ def render_flexmeasures_template(html_filename: str, **variables):
 
     variables["event_starts_after"] = session.get("event_starts_after")
     variables["event_ends_before"] = session.get("event_ends_before")
+    variables["chart_type"] = session.get("chart_type", "bar_chart")
 
     variables["page"] = html_filename.split("/")[-1].replace(".html", "")
     if "show_datepicker" not in variables:
diff --git a/flexmeasures/ui/views/sensors.py b/flexmeasures/ui/views/sensors.py
index 7f6ae5892..42e3e3acd 100644
--- a/flexmeasures/ui/views/sensors.py
+++ b/flexmeasures/ui/views/sensors.py
@@ -69,6 +69,5 @@ def get(self, id: int):
         return render_flexmeasures_template(
             "views/sensors.html",
             sensor_id=id,
-            chart_type=session.get("chart_type", "bar_chart"),
             msg="",
         )

From 129b5d92fd49020cf408ce34b5ce5e0d0b52e0ff Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Thu, 27 Jul 2023 10:28:49 +0200
Subject: [PATCH 23/34] only store non-None session variables

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/ui/utils/view_utils.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/flexmeasures/ui/utils/view_utils.py b/flexmeasures/ui/utils/view_utils.py
index dbda2876c..1123f108e 100644
--- a/flexmeasures/ui/utils/view_utils.py
+++ b/flexmeasures/ui/utils/view_utils.py
@@ -117,7 +117,9 @@ def set_session_variables(*var_names: str):
     >>> set_session_variables("event_starts_after", "event_ends_before", "chart_type")
     """
     for var_name in var_names:
-        session[var_name] = request.values.get(var_name)
+        var = request.values.get(var_name)
+        if var is not None:
+            session[var_name] = var
 
 
 def set_time_range_for_session():

From 90808fa8c784bf5a7162661e490fd2599b4c9084 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Thu, 27 Jul 2023 10:35:14 +0200
Subject: [PATCH 24/34] docs: changelog entry

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 documentation/changelog.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/documentation/changelog.rst b/documentation/changelog.rst
index 31e7af3b2..6d2d1a137 100644
--- a/documentation/changelog.rst
+++ b/documentation/changelog.rst
@@ -11,6 +11,7 @@ v0.15.0 | July XX, 2023
 New features
 -------------
 
+* Users can select a new chart type on the sensor page of the UI, to show how sensor values are distributed over the time of day [see `PR #715 <https://www.github.com/FlexMeasures/flexmeasures/pull/715>`_]
 * Allow deleting multiple sensors with a single call to ``flexmeasures delete sensor`` by passing the ``--id`` option multiple times [see `PR #734 <https://www.github.com/FlexMeasures/flexmeasures/pull/734>`_]
 * Make it a lot easier to read off the color legend on the asset page, especially when showing many sensors, as they will now be ordered from top to bottom in the same order as they appear in the chart (as defined in the ``sensors_to_show`` attribute), rather than alphabetically [see `PR #742 <https://www.github.com/FlexMeasures/flexmeasures/pull/742>`_]
 * Having percentages within the [0, 100] domain is such a common use case that we now always include it in sensor charts with % units, making it easier to read off individual charts and also to compare across charts [see `PR #739 <https://www.github.com/FlexMeasures/flexmeasures/pull/739>`_]

From 0ebb4af4b218d92dfd0bec447ef841b8af2d80bf Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Thu, 27 Jul 2023 10:38:33 +0200
Subject: [PATCH 25/34] flake8

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/ui/views/sensors.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flexmeasures/ui/views/sensors.py b/flexmeasures/ui/views/sensors.py
index 42e3e3acd..016fd168c 100644
--- a/flexmeasures/ui/views/sensors.py
+++ b/flexmeasures/ui/views/sensors.py
@@ -1,7 +1,7 @@
 import json
 
 from altair.utils.html import spec_to_html
-from flask import current_app, session
+from flask import current_app
 from flask_classful import FlaskView, route
 from flask_security import auth_required, login_required
 from marshmallow import fields

From 9b92b6e6c7fac27900f9edd8a8f2fb25ac2b8981 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Fri, 28 Jul 2023 11:17:34 +0200
Subject: [PATCH 26/34] feat: top y-coordinate of ruler moves with belief time,
 too

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/charts/belief_charts.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/flexmeasures/data/models/charts/belief_charts.py b/flexmeasures/data/models/charts/belief_charts.py
index cbfc8eb04..25ba61e8f 100644
--- a/flexmeasures/data/models/charts/belief_charts.py
+++ b/flexmeasures/data/models/charts/belief_charts.py
@@ -191,6 +191,16 @@ def matrix_chart(
                         "type": "temporal",
                         "timeUnit": "hoursminutesseconds",
                     },
+                    "y": {
+                        "field": "belief_time",
+                        "type": "temporal",
+                        "timeUnit": "yearmonthdate",
+                    },
+                    "yOffset": {
+                        "value": {
+                            "expr": "(scale('y', 24 * 60 * 60 * 1000) - scale('y', 0))"
+                        }
+                    },
                 },
             },
         ],

From 8222cf8dc487cddea427b0221698a942834d86a9 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Fri, 28 Jul 2023 11:25:16 +0200
Subject: [PATCH 27/34] rename matrix chart to heatmap

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/api/dev/sensors.py                  | 2 +-
 flexmeasures/data/models/charts/belief_charts.py | 4 ++--
 flexmeasures/ui/templates/views/sensors.html     | 4 ++--
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/flexmeasures/api/dev/sensors.py b/flexmeasures/api/dev/sensors.py
index 19f22300e..836eb5433 100644
--- a/flexmeasures/api/dev/sensors.py
+++ b/flexmeasures/api/dev/sensors.py
@@ -64,7 +64,7 @@ def get_chart(self, id: int, sensor: Sensor, **kwargs):
         - "beliefs_after" (see the `timely-beliefs documentation <https://github.com/SeitaBV/timely-beliefs/blob/main/timely_beliefs/docs/timing.md/#events-and-sensors>`_)
         - "beliefs_before" (see the `timely-beliefs documentation <https://github.com/SeitaBV/timely-beliefs/blob/main/timely_beliefs/docs/timing.md/#events-and-sensors>`_)
         - "include_data" (if true, chart specs include the data; if false, use the `GET /api/dev/sensor/(id)/chart_data/ <../api/dev.html#get--api-dev-sensor-(id)-chart_data->`_ endpoint to fetch data)
-        - "chart_type" (currently 'bar_chart' and 'matrix_chart' are supported types)
+        - "chart_type" (currently 'bar_chart' and 'daily_heatmap' are supported types)
         - "width" (an integer number of pixels; without it, the chart will be scaled to the full width of the container (hint: use ``<div style="width: 100%;">`` to set a div width to 100%)
         - "height" (an integer number of pixels; without it, FlexMeasures sets a default, currently 300)
         """
diff --git a/flexmeasures/data/models/charts/belief_charts.py b/flexmeasures/data/models/charts/belief_charts.py
index 25ba61e8f..d129352e1 100644
--- a/flexmeasures/data/models/charts/belief_charts.py
+++ b/flexmeasures/data/models/charts/belief_charts.py
@@ -87,7 +87,7 @@ def bar_chart(
     return chart_specs
 
 
-def matrix_chart(
+def daily_heatmap(
     sensor: "Sensor",  # noqa F821
     event_starts_after: datetime | None = None,
     event_ends_before: datetime | None = None,
@@ -148,7 +148,7 @@ def matrix_chart(
             ],
         }
     chart_specs = {
-        "description": "A simple heatmap chart showing sensor data.",
+        "description": "A daily heatmap showing sensor data.",
         # the sensor type is already shown as the y-axis title (avoid redundant info)
         "title": capitalize(sensor.name) if sensor.name != sensor.sensor_type else None,
         "layer": [
diff --git a/flexmeasures/ui/templates/views/sensors.html b/flexmeasures/ui/templates/views/sensors.html
index ef8533ce6..00c46efec 100644
--- a/flexmeasures/ui/templates/views/sensors.html
+++ b/flexmeasures/ui/templates/views/sensors.html
@@ -19,8 +19,8 @@
                                   Select chart
                               </button>
                               <ul class="dropdown-menu center-aligned" aria-labelledby="chartTypeDropdown">
-                                  <li><a class="dropdown-item" href="#" data-chart-type="bar_chart">Time series</a></li>
-                                  <li><a class="dropdown-item" href="#" data-chart-type="matrix_chart">Time of day</a></li>
+                                  <li><a class="dropdown-item" href="#" data-chart-type="bar_chart">Bar chart</a></li>
+                                  <li><a class="dropdown-item" href="#" data-chart-type="daily_heatmap">Daily heatmap</a></li>
                               </ul>
                           </div>
                       </div>

From ef4563e28f73d3187df7c029fafde4071c65fa4c Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Mon, 31 Jul 2023 17:23:14 +0200
Subject: [PATCH 28/34] feat: heatmap shows only data from a single source

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/charts/belief_charts.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/flexmeasures/data/models/charts/belief_charts.py b/flexmeasures/data/models/charts/belief_charts.py
index d129352e1..49fecbe6b 100644
--- a/flexmeasures/data/models/charts/belief_charts.py
+++ b/flexmeasures/data/models/charts/belief_charts.py
@@ -178,6 +178,12 @@ def daily_heatmap(
                         "calculate": "datum.source.name + ' (ID: ' + datum.source.id + ')'",
                         "as": "source_name_and_id",
                     },
+                    # In case of multiple sources, arbitrarily choose the first one occurring in the data
+                    {
+                        "window": [{"op": "first_value", "field": "source.id", "as": "first_source_id"}],
+                        "groupby": [],
+                    },
+                    {"filter": "datum.source.id == datum.first_source_id"},
                 ],
             },
             {

From 57d6bea41704d7624a76fc2bb382549d216e5d2d Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Mon, 31 Jul 2023 21:31:12 +0200
Subject: [PATCH 29/34] feat: selected source is the one with the most visible
 data

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/charts/belief_charts.py | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/flexmeasures/data/models/charts/belief_charts.py b/flexmeasures/data/models/charts/belief_charts.py
index 49fecbe6b..19cf6917e 100644
--- a/flexmeasures/data/models/charts/belief_charts.py
+++ b/flexmeasures/data/models/charts/belief_charts.py
@@ -178,12 +178,19 @@ def daily_heatmap(
                         "calculate": "datum.source.name + ' (ID: ' + datum.source.id + ')'",
                         "as": "source_name_and_id",
                     },
-                    # In case of multiple sources, arbitrarily choose the first one occurring in the data
+                    # In case of multiple sources, show the one with the most visible data
                     {
-                        "window": [{"op": "first_value", "field": "source.id", "as": "first_source_id"}],
-                        "groupby": [],
+                        "joinaggregate": [{"op": "count", "as": "source_count"}],
+                        "groupby": ["source.id"],
                     },
-                    {"filter": "datum.source.id == datum.first_source_id"},
+                    {
+                        "window": [
+                            {"op": "rank", "field": "source_count", "as": "source_rank"}
+                        ],
+                        "sort": [{"field": "source_count", "order": "descending"}],
+                        "frame": [None, None],
+                    },
+                    {"filter": "datum.source_rank == 1"},
                 ],
             },
             {

From 7d13ee3ae462de27611a02ccea361eb18c71ecaa Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Mon, 31 Jul 2023 22:35:37 +0200
Subject: [PATCH 30/34] docs: add name of new chart type to changelog entry

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 documentation/changelog.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/documentation/changelog.rst b/documentation/changelog.rst
index 3ec5f942f..bf3b178d2 100644
--- a/documentation/changelog.rst
+++ b/documentation/changelog.rst
@@ -11,7 +11,7 @@ v0.15.0 | July XX, 2023
 New features
 -------------
 
-* Users can select a new chart type on the sensor page of the UI, to show how sensor values are distributed over the time of day [see `PR #715 <https://www.github.com/FlexMeasures/flexmeasures/pull/715>`_]
+* Users can select a new chart type (daily heatmap) on the sensor page of the UI, showing how sensor values are distributed over the time of day [see `PR #715 <https://www.github.com/FlexMeasures/flexmeasures/pull/715>`_]
 * Allow deleting multiple sensors with a single call to ``flexmeasures delete sensor`` by passing the ``--id`` option multiple times [see `PR #734 <https://www.github.com/FlexMeasures/flexmeasures/pull/734>`_]
 * Make it a lot easier to read off the color legend on the asset page, especially when showing many sensors, as they will now be ordered from top to bottom in the same order as they appear in the chart (as defined in the ``sensors_to_show`` attribute), rather than alphabetically [see `PR #742 <https://www.github.com/FlexMeasures/flexmeasures/pull/742>`_]
 * Having percentages within the [0, 100] domain is such a common use case that we now always include it in sensor charts with % units, making it easier to read off individual charts and also to compare across charts [see `PR #739 <https://www.github.com/FlexMeasures/flexmeasures/pull/739>`_]

From 3d23cd9c6e16b17fd672a78d8ca24bbe8e47704f Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Tue, 1 Aug 2023 13:05:30 +0200
Subject: [PATCH 31/34] fix: workaround for
 https://github.com/vega/vega-lite/issues/6544

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/charts/belief_charts.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flexmeasures/data/models/charts/belief_charts.py b/flexmeasures/data/models/charts/belief_charts.py
index 2ce0edaae..674c212ff 100644
--- a/flexmeasures/data/models/charts/belief_charts.py
+++ b/flexmeasures/data/models/charts/belief_charts.py
@@ -100,7 +100,7 @@ def daily_heatmap(
         formatType="quantityWithUnitFormat",
         stack=None,
         **FIELD_DEFINITIONS["event_value"],
-        scale={"scheme": "blueorange", "domainMid": 0},
+        scale={"scheme": "blueorange", "domainMid": 0, "domain": {"unionWith": [0]}},
     )
     event_start_field_definition = dict(
         field="event_start",

From ccb39ca0c0a5307278ff9b9ca400fc4e78b0456a Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Tue, 1 Aug 2023 13:10:10 +0200
Subject: [PATCH 32/34] docs: changelog entry

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 documentation/changelog.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/documentation/changelog.rst b/documentation/changelog.rst
index bf3b178d2..da7cc3516 100644
--- a/documentation/changelog.rst
+++ b/documentation/changelog.rst
@@ -12,6 +12,7 @@ New features
 -------------
 
 * Users can select a new chart type (daily heatmap) on the sensor page of the UI, showing how sensor values are distributed over the time of day [see `PR #715 <https://www.github.com/FlexMeasures/flexmeasures/pull/715>`_]
+* Users are warned in the UI on when the data they are seeing includes one or more Daylight Saving Time (DST) transitions, and heatmaps (see previous feature) visualize these transitions intuitively [see `PR #723 <https://www.github.com/FlexMeasures/flexmeasures/pull/723>`_]
 * Allow deleting multiple sensors with a single call to ``flexmeasures delete sensor`` by passing the ``--id`` option multiple times [see `PR #734 <https://www.github.com/FlexMeasures/flexmeasures/pull/734>`_]
 * Make it a lot easier to read off the color legend on the asset page, especially when showing many sensors, as they will now be ordered from top to bottom in the same order as they appear in the chart (as defined in the ``sensors_to_show`` attribute), rather than alphabetically [see `PR #742 <https://www.github.com/FlexMeasures/flexmeasures/pull/742>`_]
 * Having percentages within the [0, 100] domain is such a common use case that we now always include it in sensor charts with % units, making it easier to read off individual charts and also to compare across charts [see `PR #739 <https://www.github.com/FlexMeasures/flexmeasures/pull/739>`_]

From 12bac09e48360454a30b7f20bc0520eb3bb51732 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Tue, 1 Aug 2023 17:42:24 +0200
Subject: [PATCH 33/34] fix: in case of a tied rank, where multiple sources
 have exactly the same number of data points, arbitrarily pick the first one
 in the data

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/charts/belief_charts.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/flexmeasures/data/models/charts/belief_charts.py b/flexmeasures/data/models/charts/belief_charts.py
index 19cf6917e..195a4d628 100644
--- a/flexmeasures/data/models/charts/belief_charts.py
+++ b/flexmeasures/data/models/charts/belief_charts.py
@@ -191,6 +191,17 @@ def daily_heatmap(
                         "frame": [None, None],
                     },
                     {"filter": "datum.source_rank == 1"},
+                    # In case of a tied rank, arbitrarily choose the first one occurring in the data
+                    {
+                        "window": [
+                            {
+                                "op": "first_value",
+                                "field": "source.id",
+                                "as": "first_source_id",
+                            }
+                        ],
+                    },
+                    {"filter": "datum.source.id == datum.first_source_id"},
                 ],
             },
             {

From 60d81464473ea61f9beb7daf9497bf172b31ac5e Mon Sep 17 00:00:00 2001
From: Felix Claessen <30658763+Flix6x@users.noreply.github.com>
Date: Tue, 1 Aug 2023 17:43:22 +0200
Subject: [PATCH 34/34] feature: show a warning message whenever a heatmap is
 shown that masks sources (#780)

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/ui/static/css/flexmeasures.css  |  2 +-
 flexmeasures/ui/static/js/data-utils.js      | 63 ++++++++++++++++++++
 flexmeasures/ui/templates/base.html          | 15 +++++
 flexmeasures/ui/templates/views/sensors.html |  5 +-
 4 files changed, 83 insertions(+), 2 deletions(-)
 create mode 100644 flexmeasures/ui/static/js/data-utils.js

diff --git a/flexmeasures/ui/static/css/flexmeasures.css b/flexmeasures/ui/static/css/flexmeasures.css
index 3dddde6fe..517fe3f57 100644
--- a/flexmeasures/ui/static/css/flexmeasures.css
+++ b/flexmeasures/ui/static/css/flexmeasures.css
@@ -189,7 +189,7 @@ p.error {
     font-weight: 700;
 }
 
-#tzwarn {
+#tzwarn,#sourcewarn {
     margin-top: 20px;
     margin-bottom: 20px;
     color: var(--secondary-color);
diff --git a/flexmeasures/ui/static/js/data-utils.js b/flexmeasures/ui/static/js/data-utils.js
new file mode 100644
index 000000000..5677d90f2
--- /dev/null
+++ b/flexmeasures/ui/static/js/data-utils.js
@@ -0,0 +1,63 @@
+// Data source utils
+
+/**
+ * Extracts unique values of a specified nested property from an array of JSON objects.
+ *
+ * @param {Array} data - An array of JSON objects from which to extract unique values.
+ * @param {string} key - The dot-separated key representing the nested property (e.g., 'source.id').
+ * @returns {Array} An array containing the unique values of the specified nested property.
+ *
+ * @example
+ * const data = [
+ *   {"id": 1, "name": "foo", "source": {"id": 4, "name": "bar"}},
+ *   {"id": 2, "name": "baz", "source": {"id": 4, "name": "qux"}},
+ *   {"id": 3, "name": "quux", "source": {"id": 5, "name": "corge"}}
+ * ];
+ *
+ * const key = 'source.id';
+ * const uniqueSourceIds = getUniqueValues(data, key);
+ * console.log(uniqueSourceIds); // Output: [4, 5]
+ */
+export function getUniqueValues(data, key) {
+    var lookup = {};
+    var items = data;
+    var results = [];
+
+    for (var item, i = 0; item = items[i++];) {
+        var val = getValueByNestedKey(item, key);
+
+        if (!(val in lookup)) {
+            lookup[val] = 1;
+            results.push(val);
+        }
+    }
+    return results;
+}
+
+/**
+ * Retrieves the value of a nested property in an object using a dot-separated key.
+ *
+ * @param {Object} obj - The input JavaScript object from which to retrieve the nested value.
+ * @param {string} key - The dot-separated key representing the nested property (e.g., 'source.id').
+ * @returns {*} The value of the nested property if found, otherwise, returns undefined.
+ *
+ * @example
+ * const jsonString = '{"id":11,"name":"ajax","subject":"OR","mark":63,"source":{"id":4,"name":"foo"}}';
+ * const jsonObject = JSON.parse(jsonString);
+ *
+ * const key = 'source.id';
+ * const sourceId = getValueByNestedKey(jsonObject, key);
+ * console.log(sourceId); // Output: 4
+ */
+function getValueByNestedKey(obj, key) {
+    const keys = key.split('.');
+    let value = obj;
+    for (const k of keys) {
+        if (value[k] === undefined) {
+            return undefined; // Property not found
+        }
+        value = value[k];
+    }
+    return value;
+}
+
diff --git a/flexmeasures/ui/templates/base.html b/flexmeasures/ui/templates/base.html
index cbb477685..f19237016 100644
--- a/flexmeasures/ui/templates/base.html
+++ b/flexmeasures/ui/templates/base.html
@@ -233,6 +233,7 @@
     <!-- Render Charts -->
     <script type="module" type="text/javascript">
 
+    import { getUniqueValues } from "{{ url_for('flexmeasures_ui.static', filename='js/data-utils.js') }}";
     import { subtract, thisMonth, lastNMonths, getOffsetBetweenTimezonesForDate } from "{{ url_for('flexmeasures_ui.static', filename='js/daterange-utils.js') }}";
     import { partition, updateBeliefs, beliefTimedelta, setAbortableTimeout} from "{{ url_for('flexmeasures_ui.static', filename='js/replay-utils.js') }}";
 
@@ -280,6 +281,17 @@
         }
     });
 
+    function checkSourceMasking(data) {
+        var uniqueSourceIds = getUniqueValues(data, 'source.id');
+        if (chartType == 'daily_heatmap' && uniqueSourceIds.length > 1) {
+            document.getElementById('sourcewarn').style.display = 'block';
+            document.getElementById('sourcewarn').innerHTML = 'Please note that only data from the most prevalent source is shown.';
+        }
+        else {
+            document.getElementById('sourcewarn').style.display = 'none';
+        }
+    }
+
     async function embedAndLoad(chartSpecsPath, elementId, datasetName, previousResult, startDate, endDate) {
 
         await vegaEmbed('#'+elementId, chartSpecsPath + 'dataset_name=' + datasetName + '&width=container&include_sensor_annotations=false&include_asset_annotations=false&chart_type=' + chartType, {{ chart_options | safe }})
@@ -287,6 +299,7 @@
             // result.view is the Vega View, chartSpecsPath is the original Vega-Lite specification
             vegaView = result.view;
             if (previousResult) {
+                checkSourceMasking(previousResult);
                 var slicedPreviousResult = previousResult.filter(item => {
                     return item.event_start >= startDate.getTime() && item.event_start < endDate.getTime();
                 })
@@ -397,6 +410,7 @@
             $("#spinner").hide();
             vegaView.change(datasetName, vega.changeset().remove(vega.truthy).insert(result[0])).resize().run();
             previousResult = result[0];
+            checkSourceMasking(previousResult);
             /**
             vegaView.change(datasetName + '_annotations', vega.changeset().remove(vega.truthy).insert(result[1])).resize().run();
             */
@@ -418,6 +432,7 @@
                 $("#spinner").hide();
                 vegaView.change(datasetName, vega.changeset().remove(vega.truthy).insert(fetchedInitialData)).resize().run();
                 previousResult = fetchedInitialData;
+                checkSourceMasking(previousResult);
                 var timerangeNotSetYet = false
             {% else %}
                 var timerangeNotSetYet = true
diff --git a/flexmeasures/ui/templates/views/sensors.html b/flexmeasures/ui/templates/views/sensors.html
index 00c46efec..239af8f45 100644
--- a/flexmeasures/ui/templates/views/sensors.html
+++ b/flexmeasures/ui/templates/views/sensors.html
@@ -7,7 +7,10 @@
 {% block divs %}
 
       <div class="sensor-data charts text-center">
-          <div class="row"><div class="alert alert-info" id="tzwarn" style="display:none;"></div></div>
+          <div class="row">
+              <div class="alert alert-info" id="tzwarn" style="display:none;"></div>
+              <div class="alert alert-info" id="sourcewarn" style="display:none;"></div>
+          </div>
           <div class="row on-top-md">
               <div class="col-sm-2">
                   <div class="sidepanel-container">