feat: show split data during fall DST transition in daily heatmap

flexmeasures/api/dev/sensors.py

@@ -45,6 +45,7 @@ class SensorAPI(FlaskView):
             "include_asset_annotations": fields.Boolean(required=False),
             "include_account_annotations": fields.Boolean(required=False),
             "dataset_name": fields.Str(required=False),
+            "chart_type": fields.Str(required=False),
             "height": fields.Str(required=False),
             "width": fields.Str(required=False),
         },

flexmeasures/data/models/charts/belief_charts.py

@@ -83,6 +83,186 @@ def bar_chart(
     return chart_specs
 
 
+def matrix_chart(
+    sensor: "Sensor",  # noqa F821
+    event_starts_after: datetime | None = None,
+    event_ends_before: datetime | None = None,
+    **override_chart_specs: dict,
+):
+    unit = sensor.unit if sensor.unit else "a.u."
+    event_value_field_definition = dict(
+        title=f"{capitalize(sensor.sensor_type)} ({unit})",
+        format=[".3~r", unit],
+        formatType="quantityWithUnitFormat",
+        stack=None,
+        **FIELD_DEFINITIONS["event_value"],
+        scale={"scheme": "blueorange", "domainMid": 0},
+    )
+    event_start_field_definition = dict(
+        field="event_start",
+        type="temporal",
+        title=None,
+        timeUnit={
+            "unit": "hoursminutesseconds",
+            "step": sensor.event_resolution.total_seconds(),
+        },
+        axis={
+            "labelExpr": "timeFormat(datum.value, '%H:%M')",
+            "labelFlush": False,
+            "labelOverlap": True,
+            "labelSeparation": 1,
+        },
+        scale={
+            "domain": [
+                {"hours": 0},
+                {"hours": 24},
+            ]
+        },
+    )
+    event_start_date_field_definition = dict(
+        field="event_start",
+        type="temporal",
+        title=None,
+        timeUnit={
+            "unit": "yearmonthdate",
+        },
+        axis={
+            "tickCount": "day",
+            # Center align the date labels
+            "labelOffset": {
+                "expr": "(scale('y', 24 * 60 * 60 * 1000) - scale('y', 0)) / 2"
+            },
+            "labelFlush": False,
+            "labelBound": True,
+        },
+    )
+    if event_starts_after and event_ends_before:
+        event_start_date_field_definition["scale"] = {
+            "domain": [
+                event_starts_after.timestamp() * 10**3,
+                event_ends_before.timestamp() * 10**3,
+            ],
+        }
+    mark = {"type": "rect", "clip": True, "opacity": 0.7}
+    tooltip = [
+        FIELD_DEFINITIONS["full_date"],
+        {
+            **event_value_field_definition,
+            **dict(title=f"{capitalize(sensor.sensor_type)}"),
+        },
+        FIELD_DEFINITIONS["source_name_and_id"],
+        FIELD_DEFINITIONS["source_model"],
+    ]
+    chart_specs = {
+        "description": "A simple heatmap chart showing sensor data.",
+        # the sensor type is already shown as the y-axis title (avoid redundant info)
+        "title": capitalize(sensor.name) if sensor.name != sensor.sensor_type else None,
+        "layer": [
+            {
+                "mark": mark,
+                "encoding": {
+                    "x": event_start_field_definition,
+                    "y": event_start_date_field_definition,
+                    "color": event_value_field_definition,
+                    "detail": FIELD_DEFINITIONS["source"],
+                    "tooltip": tooltip,
+                },
+                "transform": [
+                    {
+                        # Mask overlapping data during the fall DST transition, which we show later with a special layer
+                        "filter": "timezoneoffset(datum.event_start) >= timezoneoffset(datum.event_start + 60 * 60 * 1000) && timezoneoffset(datum.event_start) <= timezoneoffset(datum.event_start - 60 * 60 * 1000)"
+                    },
+                    {
+                        "calculate": "datum.source.name + ' (ID: ' + datum.source.id + ')'",
+                        "as": "source_name_and_id",
+                    },
+                ],
+            },
+            {
+                "data": {"name": "replay"},
+                "mark": {
+                    "type": "rule",
+                },
+                "encoding": {
+                    "x": {
+                        "field": "belief_time",
+                        "type": "temporal",
+                        "timeUnit": "hoursminutesseconds",
+                    },
+                },
+            },
+            create_fall_dst_transition_layer(
+                sensor.timezone,
+                mark,
+                event_value_field_definition,
+                event_start_field_definition,
+                tooltip,
+            ),
+        ],
+    }
+    for k, v in override_chart_specs.items():
+        chart_specs[k] = v
+    chart_specs["config"] = {
+        "legend": {"orient": "right"},
+        # "legend": {"direction": "horizontal"},
+    }
+    return chart_specs
+
+
+def create_fall_dst_transition_layer(
+    timezone, mark, event_value_field_definition, event_start_field_definition, tooltip
+) -> dict:
+    """Special layer for showing data during the daylight savings time transition in fall."""
+    return {
+        "mark": mark,
+        "encoding": {
+            "x": event_start_field_definition,
+            "y": {
+                "field": "dst_transition_event_start",
+                "type": "temporal",
+                "title": None,
+                "timeUnit": {"unit": "yearmonthdatehours", "step": 12},
+            },
+            "y2": {
+                "field": "dst_transition_event_start_next",
+                "timeUnit": {"unit": "yearmonthdatehours", "step": 12},
+            },
+            "color": event_value_field_definition,
+            "detail": FIELD_DEFINITIONS["source"],
+            "tooltip": [
+                {
+                    "field": "event_start",
+                    "type": "temporal",
+                    "title": "Timezone",
+                    "timeUnit": "utc",
+                    "format": [timezone],
+                    "formatType": "timezoneFormat",
+                },
+                *tooltip,
+            ],
+        },
+        "transform": [
+            {
+                "filter": "timezoneoffset(datum.event_start) < timezoneoffset(datum.event_start + 60 * 60 * 1000) || timezoneoffset(datum.event_start) > timezoneoffset(datum.event_start - 60 * 60 * 1000)"
+            },
+            {
+                # Push the more recent hour into the second 12-hour bin
+                "calculate": "timezoneoffset(datum.event_start + 60 * 60 * 1000) > timezoneoffset(datum.event_start) ? datum.event_start : datum.event_start + 12 * 60 * 60 * 1000",
+                "as": "dst_transition_event_start",
+            },
+            {
+                # Calculate a time point in the next 12-hour bin
+                "calculate": "datum.dst_transition_event_start + 12 * 60 * 60 * 1000 - 60 * 60 * 1000",
+                "as": "dst_transition_event_start_next",
+            },
+            {
+                "calculate": "datum.source.name + ' (ID: ' + datum.source.id + ')'",
+                "as": "source_name_and_id",
+            },
+        ],
+    }
+
+
 def chart_for_multiple_sensors(
     sensors_to_show: list["Sensor", list["Sensor"]],  # noqa F821
     event_starts_after: datetime | None = None,

flexmeasures/ui/static/css/flexmeasures.css

@@ -189,7 +189,7 @@ p.error {
     font-weight: 700;
 }
 
-#tzwarn {
+#tzwarn,#dstwarn {
     margin-top: 20px;
     margin-bottom: 20px;
     color: var(--secondary-color);
@@ -238,7 +238,7 @@ p.error {
     border: none;
 }
 
-.navbar-default .dropdown-menu {
+.dropdown-menu {
     border: none !important;
 }
 
@@ -304,7 +304,7 @@ p.error {
         padding-bottom: 20px;
     }
 
-    .navbar-default .dropdown-menu a {
+    .dropdown-menu a {
         transition: .3s;
         -webkit-transition: .3s;
         -moz-transition: .3s;
@@ -315,7 +315,7 @@ p.error {
 }
 
 @media (max-width: 767px) {
-    .navbar-default .navbar-nav .open .dropdown-menu>li>a {
+    .dropdown-menu>li>a {
         color: var(--nav-default-color);
         transition: .4s;
         -webkit-transition: .4s;
@@ -325,13 +325,23 @@ p.error {
     }
 }
 
-.navbar-default .navbar-nav .open .dropdown-menu,
-.navbar-default .navbar-nav .open .dropdown-menu>li>a {
+.dropdown-menu.center-aligned {
+    right: auto;
+    left: 50%;
+    transform: translateX(-50%);
+}
+
+.dropdown-menu,
+.dropdown-menu>li>a:not(.active) {
     color: var(--nav-default-color);
     background-color: var(--nav-default-background-color);
 }
-.navbar-default .navbar-nav .open .dropdown-menu>li>a:focus,
-.navbar-default .navbar-nav .open .dropdown-menu>li>a:hover {
+.dropdown-menu>li>a.active {
+    color: var(--nav-hover-color);
+    background-color: var(--nav-hover-background-color);
+}
+.dropdown-menu>li>a:focus,
+.dropdown-menu>li>a:hover {
     color: var(--nav-hover-color);
     background-color: var(--nav-hover-background-color);
 }

flexmeasures/ui/static/js/daterange-utils.js

@@ -38,4 +38,33 @@ function getTimeZoneOffset(date, timeZone) {
     // Positive values are West of GMT, opposite of ISO 8601
     // this matches the output of `Date.getTimeZoneOffset`
     return -(lie - date) / 60 / 1000;
-}
+}
+
+/**
+ * Count the number of Daylight Saving Time (DST) transitions within a given datetime range.
+ * @param {Date} startDate - The start date of the datetime range.
+ * @param {Date} endDate - The end date of the datetime range.
+ * @param {number} increment - The number of days to increment between iterations.
+ * @returns {number} The count of DST transitions within the specified range.
+ */
+export function countDSTTransitions(startDate, endDate, increment) {
+    let transitions = 0;
+    let currentDate = new Date(startDate);
+    let nextDate = new Date(startDate);
+
+    while (currentDate <= endDate) {
+        const currentOffset = currentDate.getTimezoneOffset();
+        nextDate.setDate(currentDate.getDate() + increment);
+        if (nextDate > endDate) {
+            nextDate = endDate;
+        }
+        const nextOffset = nextDate.getTimezoneOffset();
+
+        if (currentOffset !== nextOffset) {
+            transitions++;
+        }
+        currentDate.setDate(currentDate.getDate() + increment);
+    }
+
+    return transitions;
+}

flexmeasures/ui/static/js/flexmeasures.js

@@ -311,28 +311,31 @@ function submit_sensor_type() {
     $("#sensor_type-form").attr("action", empty_location).submit();
 }
 
-/** Tooltips: Register custom formatters
- *
- *          - Quantities incl. units
- *            Usage:
- *                 {
- *                     'format': [<d3-format>, <sensor unit>],
- *                     'formatType': 'quantityWithUnitFormat'
- *                 }
- *          - Timedeltas measured in human-readable quantities (usually not milliseconds)
- *            Usage:
- *                 {
- *                     'format': [<d3-format>, <breakpoint>],
- *                     'formatType': 'timedeltaFormat'
- *                 }
- *            <d3-format>  is a d3 format identifier, e.g. 'd' for decimal notation, rounded to integer.
- *                         See https://github.com/d3/d3-format for more details.
- *            <breakpoint> is a scalar that decides the breakpoint from one duration unit to the next larger unit.
-  *                        For example, a breakpoint of 4 means we format 4 days as '4 days', but 3.96 days as '95 hours'.
+/** Tooltips: Register custom formatters */
+
+/* Quantities incl. units
+ * Usage:
+ *     {
+ *         'format': [<d3-format>, <sensor unit>],
+ *         'formatType': 'quantityWithUnitFormat'
+ *     }
  */
 vega.expressionFunction('quantityWithUnitFormat', function(datum, params) {
     return d3.format(params[0])(datum) + " " + params[1];
 });
+
+/*
+ * Timedeltas measured in human-readable quantities (usually not milliseconds)
+ * Usage:
+ *     {
+ *         'format': [<d3-format>, <breakpoint>],
+ *         'formatType': 'timedeltaFormat'
+ *     }
+ * <d3-format>  is a d3 format identifier, e.g. 'd' for decimal notation, rounded to integer.
+ *              See https://github.com/d3/d3-format for more details.
+ * <breakpoint> is a scalar that decides the breakpoint from one duration unit to the next larger unit.
+ *              For example, a breakpoint of 4 means we format 4 days as '4 days', but 3.96 days as '95 hours'.
+ */
 vega.expressionFunction('timedeltaFormat', function(timedelta, params) {
     return (Math.abs(timedelta) > 1000 * 60 * 60 * 24 * 365.2425 * params[1] ? d3.format(params[0])(timedelta / (1000 * 60 * 60 * 24 * 365.2425)) + " years"
         : Math.abs(timedelta) > 1000 * 60 * 60 * 24 * params[1] ? d3.format(params[0])(timedelta / (1000 * 60 * 60 * 24)) + " days"
@@ -341,3 +344,18 @@ vega.expressionFunction('timedeltaFormat', function(timedelta, params) {
         : Math.abs(timedelta) > 1000 * params[1] ? d3.format(params[0])(timedelta / 1000) + " seconds"
         : d3.format(params[0])(timedelta) + " milliseconds");
 });
+
+/*
+ * Timezone offset including IANA timezone name
+ * Usage:
+ *     {
+ *         'format': [<IANA timezone name, e.g. 'Europe/Amsterdam'>],
+ *         'formatType': 'timezoneFormat'
+ *     }
+ */
+vega.expressionFunction('timezoneFormat', function(date, params) {
+    const timezoneString = params[0];
+    const tzOffsetNumber = date.getTimezoneOffset();
+    const tzDate = new Date(0,0,0,0,Math.abs(tzOffsetNumber));
+    return `${ tzOffsetNumber > 0 ? '-' : '+'}${("" + tzDate.getHours()).padStart(2, '0')}:${("" + tzDate.getMinutes()).padStart(2, '0')}` + ' (' + timezoneString + ')';
+});