From ed890c2ef2879668004632206c53894f7f76abff Mon Sep 17 00:00:00 2001 From: Felix Claessen <30658763+Flix6x@users.noreply.github.com> Date: Fri, 22 Jul 2022 00:45:34 +0200 Subject: [PATCH] Adapt formatter for ISO durations (#459) Implement a workaround for incorrect formatting of nominal days in isodate. * Adapt formatter for ISO durations Signed-off-by: F.N. Claessen * Add tests Signed-off-by: F.N. Claessen * Changelog entry Signed-off-by: F.N. Claessen --- documentation/changelog.rst | 1 + .../api/common/schemas/sensor_data.py | 4 +-- flexmeasures/api/v1_1/tests/utils.py | 3 +- flexmeasures/api/v1_2/implementations.py | 3 +- flexmeasures/api/v1_3/implementations.py | 3 +- flexmeasures/api/v2_0/tests/utils.py | 3 +- flexmeasures/api/v3_0/sensors.py | 3 +- flexmeasures/utils/tests/test_time_utils.py | 33 +++++++++++++++++++ flexmeasures/utils/time_utils.py | 32 ++++++++++++++++++ 9 files changed, 78 insertions(+), 7 deletions(-) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index e27ba1edf..a1a095c08 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -21,6 +21,7 @@ Bugfixes * Time scale axes in sensor data charts now match the requested date range, rather than stopping at the edge of the available data [see `PR #449 `_] * The docker-based tutorial now works with UI on all platforms (port 5000 did not expose on MacOS) [see `PR #465 `_] * Fix interpretation of scheduling results in toy tutorial [see `PR #466 `_] +* Avoid formatting datetime.timedelta durations as nominal ISO durations [see `PR #459 `_] Infrastructure / Support ---------------------- diff --git a/flexmeasures/api/common/schemas/sensor_data.py b/flexmeasures/api/common/schemas/sensor_data.py index 8e40e114b..de0fc0ae4 100644 --- a/flexmeasures/api/common/schemas/sensor_data.py +++ b/flexmeasures/api/common/schemas/sensor_data.py @@ -2,7 +2,7 @@ from typing import List, Union from flask_login import current_user -from isodate import datetime_isoformat, duration_isoformat +from isodate import datetime_isoformat from marshmallow import fields, post_load, validates_schema, ValidationError from marshmallow.validate import OneOf from marshmallow_polyfield import PolyField @@ -16,7 +16,7 @@ from flexmeasures.api.common.utils.api_utils import upsample_values from flexmeasures.data.schemas.times import AwareDateTimeField, DurationField from flexmeasures.data.services.time_series import simplify_index -from flexmeasures.utils.time_utils import server_now +from flexmeasures.utils.time_utils import duration_isoformat, server_now from flexmeasures.utils.unit_utils import ( convert_units, units_are_convertible, diff --git a/flexmeasures/api/v1_1/tests/utils.py b/flexmeasures/api/v1_1/tests/utils.py index 064f8a9b6..bb21cb982 100644 --- a/flexmeasures/api/v1_1/tests/utils.py +++ b/flexmeasures/api/v1_1/tests/utils.py @@ -1,7 +1,7 @@ """Useful test messages""" from typing import Optional, Dict, Any, List, Union from datetime import timedelta -from isodate import duration_isoformat, parse_duration, parse_datetime +from isodate import parse_datetime, parse_duration import pandas as pd from numpy import tile @@ -10,6 +10,7 @@ from flexmeasures.api.common.schemas.sensors import SensorField from flexmeasures.data.models.time_series import Sensor, TimedBelief +from flexmeasures.utils.time_utils import duration_isoformat def message_for_get_prognosis( diff --git a/flexmeasures/api/v1_2/implementations.py b/flexmeasures/api/v1_2/implementations.py index ea71ab638..5f5a9fef3 100644 --- a/flexmeasures/api/v1_2/implementations.py +++ b/flexmeasures/api/v1_2/implementations.py @@ -39,6 +39,7 @@ ) from flexmeasures.data.models.time_series import Sensor from flexmeasures.data.services.resources import has_assets, can_access_asset +from flexmeasures.utils.time_utils import duration_isoformat @type_accepted("GetDeviceMessageRequest") @@ -120,7 +121,7 @@ def get_device_message_response(generic_asset_name_groups, duration): new_event_groups, value_groups, generic_asset_type_name="event" ) response["start"] = isodate.datetime_isoformat(start) - response["duration"] = isodate.duration_isoformat(duration) + response["duration"] = duration_isoformat(duration) response["unit"] = unit d, s = request_processed() diff --git a/flexmeasures/api/v1_3/implementations.py b/flexmeasures/api/v1_3/implementations.py index 7540e2a39..c62442e00 100644 --- a/flexmeasures/api/v1_3/implementations.py +++ b/flexmeasures/api/v1_3/implementations.py @@ -44,6 +44,7 @@ from flexmeasures.data.queries.utils import simplify_index from flexmeasures.data.services.resources import has_assets, can_access_asset from flexmeasures.data.services.scheduling import create_scheduling_job +from flexmeasures.utils.time_utils import duration_isoformat p = inflect.engine() @@ -182,7 +183,7 @@ def get_device_message_response(generic_asset_name_groups, duration): new_event_groups, value_groups, generic_asset_type_name="event" ) response["start"] = isodate.datetime_isoformat(start) - response["duration"] = isodate.duration_isoformat(duration) + response["duration"] = duration_isoformat(duration) response["unit"] = unit d, s = request_processed() diff --git a/flexmeasures/api/v2_0/tests/utils.py b/flexmeasures/api/v2_0/tests/utils.py index 9d51bd748..31e237b2e 100644 --- a/flexmeasures/api/v2_0/tests/utils.py +++ b/flexmeasures/api/v2_0/tests/utils.py @@ -1,6 +1,6 @@ from typing import Optional from datetime import timedelta -from isodate import duration_isoformat, parse_duration, parse_datetime +from isodate import parse_datetime, parse_duration import pandas as pd import timely_beliefs as tb @@ -11,6 +11,7 @@ from flexmeasures.api.v1_1.tests.utils import ( message_for_post_price_data as v1_1_message_for_post_price_data, ) +from flexmeasures.utils.time_utils import duration_isoformat def get_asset_post_data() -> dict: diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index 82f2c5608..5b7fb0f1f 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -43,6 +43,7 @@ from flexmeasures.data.schemas import AwareDateTimeField from flexmeasures.data.services.sensors import get_sensors from flexmeasures.data.services.scheduling import create_scheduling_job +from flexmeasures.utils.time_utils import duration_isoformat from flexmeasures.utils.unit_utils import ur @@ -527,7 +528,7 @@ def get_schedule(self, sensor: Sensor, job_id: str, duration: timedelta, **kwarg response = dict( values=consumption_schedule.tolist(), start=isodate.datetime_isoformat(start), - duration=isodate.duration_isoformat(duration), + duration=duration_isoformat(duration), unit=sensor.unit, ) diff --git a/flexmeasures/utils/tests/test_time_utils.py b/flexmeasures/utils/tests/test_time_utils.py index 55d79adab..196c7ae6c 100644 --- a/flexmeasures/utils/tests/test_time_utils.py +++ b/flexmeasures/utils/tests/test_time_utils.py @@ -1,16 +1,49 @@ from datetime import datetime, timedelta +from isodate import duration_isoformat as original_duration_isoformat import pandas as pd import pytz import pytest from flexmeasures.utils.time_utils import ( + duration_isoformat, server_now, naturalized_datetime_str, get_most_recent_clocktime_window, ) +@pytest.mark.parametrize( + "td, iso", + [ + (timedelta(hours=1), "PT1H"), + (timedelta(hours=14), "PT14H"), + (timedelta(hours=24), "PT24H"), + (timedelta(days=1), "PT24H"), + (timedelta(days=1, seconds=22), "PT24H22S"), + (timedelta(days=1, seconds=122), "PT24H2M2S"), + ], +) +def test_duration_isoformat(td: timedelta, iso: str): + assert duration_isoformat(td) == iso + + +@pytest.mark.parametrize( + "td, iso", + [ + (timedelta(hours=1), "PT1H"), + (timedelta(hours=14), "PT14H"), + # todo: if the following test cases fail, we can start using isodate.duration_isoformat again (see #459) + (timedelta(hours=24), "P1D"), + (timedelta(days=1), "P1D"), + (timedelta(days=1, seconds=22), "P1DT22S"), + (timedelta(days=1, seconds=122), "P1DT2M2S"), + ], +) +def test_original_duration_isoformat(td: timedelta, iso: str): + assert original_duration_isoformat(td) == iso + + @pytest.mark.parametrize( "dt_tz, now, server_tz, delta_in_h, exp_result", # there can be two results depending of today's date, due to humanize. diff --git a/flexmeasures/utils/time_utils.py b/flexmeasures/utils/time_utils.py index 65f117404..129e20998 100644 --- a/flexmeasures/utils/time_utils.py +++ b/flexmeasures/utils/time_utils.py @@ -1,3 +1,4 @@ +import re from datetime import datetime, timedelta from typing import List, Union, Tuple, Optional @@ -290,3 +291,34 @@ def supported_horizons() -> List[timedelta]: def timedelta_to_pandas_freq_str(resolution: timedelta) -> str: return to_offset(resolution).freqstr + + +def duration_isoformat(duration: timedelta): + """Adapted version of isodate.duration_isoformat for formatting a datetime.timedelta. + + The difference is that absolute days are not formatted as nominal days. + Workaround for https://github.com/gweis/isodate/issues/74. + """ + ret = [] + usecs = abs( + (duration.days * 24 * 60 * 60 + duration.seconds) * 1000000 + + duration.microseconds + ) + seconds, usecs = divmod(usecs, 1000000) + minutes, seconds = divmod(seconds, 60) + hours, minutes = divmod(minutes, 60) + if hours or minutes or seconds or usecs: + ret.append("T") + if hours: + ret.append("%sH" % hours) + if minutes: + ret.append("%sM" % minutes) + if seconds or usecs: + if usecs: + ret.append(("%d.%06d" % (seconds, usecs)).rstrip("0")) + else: + ret.append("%d" % seconds) + ret.append("S") + # at least one component has to be there. + repl = ret and "".join(ret) or "T0H" + return re.sub("%P", repl, "P%P")