Adapt formatter for ISO durations (#459)

Implement a workaround for incorrect formatting of nominal days in isodate.


* Adapt formatter for ISO durations

Signed-off-by: F.N. Claessen <felix@seita.nl>

* Add tests

Signed-off-by: F.N. Claessen <felix@seita.nl>

* Changelog entry

Signed-off-by: F.N. Claessen <felix@seita.nl>

documentation/changelog.rst

@@ -21,6 +21,7 @@ Bugfixes
 * Time scale axes in sensor data charts now match the requested date range, rather than stopping at the edge of the available data [see `PR #449 <http://www.github.com/FlexMeasures/flexmeasures/pull/449>`_]
 * The docker-based tutorial now works with UI on all platforms (port 5000 did not expose on MacOS) [see `PR #465 <http://www.github.com/FlexMeasures/flexmeasures/pull/465>`_]
 * Fix interpretation of scheduling results in toy tutorial [see `PR #466 <http://www.github.com/FlexMeasures/flexmeasures/pull/466>`_]
+* Avoid formatting datetime.timedelta durations as nominal ISO durations [see `PR #459 <http://www.github.com/FlexMeasures/flexmeasures/pull/459>`_]
 
 Infrastructure / Support
 ----------------------

flexmeasures/api/common/schemas/sensor_data.py

@@ -2,7 +2,7 @@
 from typing import List, Union
 
 from flask_login import current_user
-from isodate import datetime_isoformat, duration_isoformat
+from isodate import datetime_isoformat
 from marshmallow import fields, post_load, validates_schema, ValidationError
 from marshmallow.validate import OneOf
 from marshmallow_polyfield import PolyField
@@ -16,7 +16,7 @@
 from flexmeasures.api.common.utils.api_utils import upsample_values
 from flexmeasures.data.schemas.times import AwareDateTimeField, DurationField
 from flexmeasures.data.services.time_series import simplify_index
-from flexmeasures.utils.time_utils import server_now
+from flexmeasures.utils.time_utils import duration_isoformat, server_now
 from flexmeasures.utils.unit_utils import (
     convert_units,
     units_are_convertible,

flexmeasures/api/v1_1/tests/utils.py

@@ -1,7 +1,7 @@
 """Useful test messages"""
 from typing import Optional, Dict, Any, List, Union
 from datetime import timedelta
-from isodate import duration_isoformat, parse_duration, parse_datetime
+from isodate import parse_datetime, parse_duration
 
 import pandas as pd
 from numpy import tile
@@ -10,6 +10,7 @@
 
 from flexmeasures.api.common.schemas.sensors import SensorField
 from flexmeasures.data.models.time_series import Sensor, TimedBelief
+from flexmeasures.utils.time_utils import duration_isoformat
 
 
 def message_for_get_prognosis(

flexmeasures/api/v1_2/implementations.py

@@ -39,6 +39,7 @@
 )
 from flexmeasures.data.models.time_series import Sensor
 from flexmeasures.data.services.resources import has_assets, can_access_asset
+from flexmeasures.utils.time_utils import duration_isoformat
 
 
 @type_accepted("GetDeviceMessageRequest")
@@ -120,7 +121,7 @@ def get_device_message_response(generic_asset_name_groups, duration):
         new_event_groups, value_groups, generic_asset_type_name="event"
     )
     response["start"] = isodate.datetime_isoformat(start)
-    response["duration"] = isodate.duration_isoformat(duration)
+    response["duration"] = duration_isoformat(duration)
     response["unit"] = unit
 
     d, s = request_processed()

flexmeasures/api/v1_3/implementations.py

@@ -44,6 +44,7 @@
 from flexmeasures.data.queries.utils import simplify_index
 from flexmeasures.data.services.resources import has_assets, can_access_asset
 from flexmeasures.data.services.scheduling import create_scheduling_job
+from flexmeasures.utils.time_utils import duration_isoformat
 
 
 p = inflect.engine()
@@ -182,7 +183,7 @@ def get_device_message_response(generic_asset_name_groups, duration):
         new_event_groups, value_groups, generic_asset_type_name="event"
     )
     response["start"] = isodate.datetime_isoformat(start)
-    response["duration"] = isodate.duration_isoformat(duration)
+    response["duration"] = duration_isoformat(duration)
     response["unit"] = unit
 
     d, s = request_processed()

flexmeasures/api/v2_0/tests/utils.py

@@ -1,6 +1,6 @@
 from typing import Optional
 from datetime import timedelta
-from isodate import duration_isoformat, parse_duration, parse_datetime
+from isodate import parse_datetime, parse_duration
 
 import pandas as pd
 import timely_beliefs as tb
@@ -11,6 +11,7 @@
 from flexmeasures.api.v1_1.tests.utils import (
     message_for_post_price_data as v1_1_message_for_post_price_data,
 )
+from flexmeasures.utils.time_utils import duration_isoformat
 
 
 def get_asset_post_data() -> dict:

flexmeasures/api/v3_0/sensors.py

@@ -43,6 +43,7 @@
 from flexmeasures.data.schemas import AwareDateTimeField
 from flexmeasures.data.services.sensors import get_sensors
 from flexmeasures.data.services.scheduling import create_scheduling_job
+from flexmeasures.utils.time_utils import duration_isoformat
 from flexmeasures.utils.unit_utils import ur
 
 
@@ -527,7 +528,7 @@ def get_schedule(self, sensor: Sensor, job_id: str, duration: timedelta, **kwarg
         response = dict(
             values=consumption_schedule.tolist(),
             start=isodate.datetime_isoformat(start),
-            duration=isodate.duration_isoformat(duration),
+            duration=duration_isoformat(duration),
             unit=sensor.unit,
         )
 

flexmeasures/utils/tests/test_time_utils.py

@@ -1,16 +1,49 @@
 from datetime import datetime, timedelta
 
+from isodate import duration_isoformat as original_duration_isoformat
 import pandas as pd
 import pytz
 import pytest
 
 from flexmeasures.utils.time_utils import (
+    duration_isoformat,
     server_now,
     naturalized_datetime_str,
     get_most_recent_clocktime_window,
 )
 
 
+@pytest.mark.parametrize(
+    "td, iso",
+    [
+        (timedelta(hours=1), "PT1H"),
+        (timedelta(hours=14), "PT14H"),
+        (timedelta(hours=24), "PT24H"),
+        (timedelta(days=1), "PT24H"),
+        (timedelta(days=1, seconds=22), "PT24H22S"),
+        (timedelta(days=1, seconds=122), "PT24H2M2S"),
+    ],
+)
+def test_duration_isoformat(td: timedelta, iso: str):
+    assert duration_isoformat(td) == iso
+
+
+@pytest.mark.parametrize(
+    "td, iso",
+    [
+        (timedelta(hours=1), "PT1H"),
+        (timedelta(hours=14), "PT14H"),
+        # todo: if the following test cases fail, we can start using isodate.duration_isoformat again (see #459)
+        (timedelta(hours=24), "P1D"),
+        (timedelta(days=1), "P1D"),
+        (timedelta(days=1, seconds=22), "P1DT22S"),
+        (timedelta(days=1, seconds=122), "P1DT2M2S"),
+    ],
+)
+def test_original_duration_isoformat(td: timedelta, iso: str):
+    assert original_duration_isoformat(td) == iso
+
+
 @pytest.mark.parametrize(
     "dt_tz, now, server_tz, delta_in_h, exp_result",
     # there can be two results depending of today's date, due to humanize.

flexmeasures/utils/time_utils.py

@@ -1,3 +1,4 @@
+import re
 from datetime import datetime, timedelta
 from typing import List, Union, Tuple, Optional
 
@@ -290,3 +291,34 @@ def supported_horizons() -> List[timedelta]:
 
 def timedelta_to_pandas_freq_str(resolution: timedelta) -> str:
     return to_offset(resolution).freqstr
+
+
+def duration_isoformat(duration: timedelta):
+    """Adapted version of isodate.duration_isoformat for formatting a datetime.timedelta.
+
+    The difference is that absolute days are not formatted as nominal days.
+    Workaround for https://github.com/gweis/isodate/issues/74.
+    """
+    ret = []
+    usecs = abs(
+        (duration.days * 24 * 60 * 60 + duration.seconds) * 1000000
+        + duration.microseconds
+    )
+    seconds, usecs = divmod(usecs, 1000000)
+    minutes, seconds = divmod(seconds, 60)
+    hours, minutes = divmod(minutes, 60)
+    if hours or minutes or seconds or usecs:
+        ret.append("T")
+        if hours:
+            ret.append("%sH" % hours)
+        if minutes:
+            ret.append("%sM" % minutes)
+        if seconds or usecs:
+            if usecs:
+                ret.append(("%d.%06d" % (seconds, usecs)).rstrip("0"))
+            else:
+                ret.append("%d" % seconds)
+            ret.append("S")
+    # at least one component has to be there.
+    repl = ret and "".join(ret) or "T0H"
+    return re.sub("%P", repl, "P%P")