From b64089c7215b507b7bcaccb633d7aeba0b79e721 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 23 Feb 2022 14:13:31 +0100 Subject: [PATCH 01/41] Rename variable Signed-off-by: F.N. Claessen --- flexmeasures/data/services/scheduling.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py index 4793e38bc..d0f8e82cd 100644 --- a/flexmeasures/data/services/scheduling.py +++ b/flexmeasures/data/services/scheduling.py @@ -29,7 +29,7 @@ def create_scheduling_job( - asset_id: int, + sensor_id: int, start_of_schedule: datetime, end_of_schedule: datetime, belief_time: datetime, @@ -57,7 +57,7 @@ def create_scheduling_job( job = Job.create( make_schedule, kwargs=dict( - asset_id=asset_id, + sensor_id=sensor_id, start=start_of_schedule, end=end_of_schedule, belief_time=belief_time, @@ -87,7 +87,7 @@ def create_scheduling_job( def make_schedule( - asset_id: int, + sensor_id: int, start: datetime, end: datetime, belief_time: datetime, @@ -109,7 +109,7 @@ def make_schedule( rq_job = get_current_job() # find sensor - sensor = Sensor.query.filter_by(id=asset_id).one_or_none() + sensor = Sensor.query.filter_by(id=sensor_id).one_or_none() click.echo( "Running Scheduling Job %s: %s, from %s to %s" % (rq_job.id, sensor, start, end) From 05a534fd35e28ab60dcb8b2246b60becf0d04bbd Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 23 Feb 2022 14:27:34 +0100 Subject: [PATCH 02/41] Allow passing an explicit sensor id to find prices Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/battery.py | 7 ++++++- .../data/models/planning/charging_station.py | 7 ++++++- flexmeasures/data/models/planning/utils.py | 14 ++++++++++---- flexmeasures/data/services/scheduling.py | 3 +++ 4 files changed, 25 insertions(+), 6 deletions(-) diff --git a/flexmeasures/data/models/planning/battery.py b/flexmeasures/data/models/planning/battery.py index e0365e1ec..4329d9503 100644 --- a/flexmeasures/data/models/planning/battery.py +++ b/flexmeasures/data/models/planning/battery.py @@ -25,6 +25,7 @@ def schedule_battery( soc_max: Optional[float] = None, roundtrip_efficiency: Optional[float] = None, prefer_charging_sooner: bool = True, + price_sensor: Optional[Sensor] = None, ) -> Union[pd.Series, None]: """Schedule a battery asset based directly on the latest beliefs regarding market prices within the specified time window. @@ -55,7 +56,11 @@ def schedule_battery( # Check for known prices or price forecasts, trimming planning window accordingly prices, (start, end) = get_prices( - sensor, (start, end), resolution, allow_trimmed_query_window=True + (start, end), + resolution, + price_sensor=price_sensor, + sensor=sensor, + allow_trimmed_query_window=True, ) if soc_targets is not None: # soc targets are at the end of each time slot, while prices are indexed by the start of each time slot diff --git a/flexmeasures/data/models/planning/charging_station.py b/flexmeasures/data/models/planning/charging_station.py index f1ff011bf..915a0d79c 100644 --- a/flexmeasures/data/models/planning/charging_station.py +++ b/flexmeasures/data/models/planning/charging_station.py @@ -25,6 +25,7 @@ def schedule_charging_station( soc_max: Optional[float] = None, roundtrip_efficiency: Optional[float] = None, prefer_charging_sooner: bool = True, + price_sensor: Optional[Sensor] = None, ) -> Union[Series, None]: """Schedule a charging station asset based directly on the latest beliefs regarding market prices within the specified time window. @@ -52,7 +53,11 @@ def schedule_charging_station( # Check for known prices or price forecasts, trimming planning window accordingly prices, (start, end) = get_prices( - sensor, (start, end), resolution, allow_trimmed_query_window=True + (start, end), + resolution, + price_sensor=price_sensor, + sensor=sensor, + allow_trimmed_query_window=True, ) # soc targets are at the end of each time slot, while prices are indexed by the start of each time slot soc_targets = soc_targets.tz_convert("UTC") diff --git a/flexmeasures/data/models/planning/utils.py b/flexmeasures/data/models/planning/utils.py index 8cdf26394..e11d50810 100644 --- a/flexmeasures/data/models/planning/utils.py +++ b/flexmeasures/data/models/planning/utils.py @@ -68,9 +68,10 @@ def get_market(sensor: Sensor) -> Sensor: def get_prices( - sensor: Sensor, query_window: Tuple[datetime, datetime], resolution: timedelta, + price_sensor: Optional[Sensor] = None, + sensor: Optional[Sensor] = None, allow_trimmed_query_window: bool = True, ) -> Tuple[pd.DataFrame, Tuple[datetime, datetime]]: """Check for known prices or price forecasts, trimming query window accordingly if allowed. @@ -78,11 +79,16 @@ def get_prices( (this may require implementing a belief time for scheduling jobs). """ - # Look for the applicable market sensor - sensor = get_market(sensor) + # Look for the applicable price sensor + if price_sensor is None: + if sensor is None: + raise UnknownMarketException( + "Missing price sensor cannot be derived from a missing sensor" + ) + price_sensor = get_market(sensor) price_bdf: tb.BeliefsDataFrame = TimedBelief.search( - sensor.name, + price_sensor.name, event_starts_after=query_window[0], event_ends_before=query_window[1], resolution=to_offset(resolution).freqstr, diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py index d0f8e82cd..33f1ab5e8 100644 --- a/flexmeasures/data/services/scheduling.py +++ b/flexmeasures/data/services/scheduling.py @@ -97,6 +97,7 @@ def make_schedule( soc_min: Optional[float] = None, soc_max: Optional[float] = None, roundtrip_efficiency: Optional[float] = None, + price_sensor: Optional[Sensor] = None, ) -> bool: """Preferably, a starting soc is given. Otherwise, we try to retrieve the current state of charge from the asset (if that is the valid one at the start). @@ -140,6 +141,7 @@ def make_schedule( soc_min, soc_max, roundtrip_efficiency, + price_sensor=price_sensor, ) elif sensor.generic_asset.generic_asset_type.name in ( "one-way_evse", @@ -155,6 +157,7 @@ def make_schedule( soc_min, soc_max, roundtrip_efficiency, + price_sensor=price_sensor, ) else: raise ValueError( From 36325607316a94f298c6674f2c8a05754289fb2e Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 23 Feb 2022 14:29:01 +0100 Subject: [PATCH 03/41] Add CLI command to create a schedule Signed-off-by: F.N. Claessen --- flexmeasures/cli/data_add.py | 59 ++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index bce22938a..ca0e4a72f 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -16,6 +16,7 @@ from flexmeasures.data import db from flexmeasures.data.services.forecasting import create_forecasting_jobs +from flexmeasures.data.services.scheduling import make_schedule from flexmeasures.data.services.users import create_user from flexmeasures.data.models.user import Account, AccountRole, RolesAccounts from flexmeasures.data.models.time_series import ( @@ -799,6 +800,64 @@ def create_forecasts( ) +@fm_add_data.command("schedule") +@with_appcontext +@click.option( + "--sensor-id", + "power_sensor_id", + required=True, + help="Create schedule for this sensor. Follow up with the sensor's ID.", +) +@click.option( + "--factor-id", + "factor_sensor_id", + required=True, + help="Optimize against this sensor, which measures a price factor or CO₂ intensity factor. Follow up with the sensor's ID.", +) +@click.option( + "--from", + "start_str", + required=True, + help="Schedule starts at this datetime. Follow up with a timezone-aware datetime in ISO 6801 format.", +) +@click.option( + "--until", + "end_str", + required=True, + help="Schedule ends at this datetime. Follow up with a timezone-aware datetime in ISO 6801 format.", +) +@click.option( + "--soc-at-start", + "soc_at_start", + type=float, + required=True, + help="State of charge at the start of the schedule.", +) +def create_schedule( + power_sensor_id: int, + factor_sensor_id: int, + start_str: str, + end_str: str, + soc_at_start: float, +): + # Parse input + factor_sensor = Sensor.query.filter(Sensor.id == factor_sensor_id).one_or_none() + start = pd.Timestamp(start_str) + end = pd.Timestamp(end_str) + + success = make_schedule( + sensor_id=power_sensor_id, + start=start, + end=end, + belief_time=server_now(), + resolution=factor_sensor.resolution, + soc_at_start=soc_at_start, + price_sensor=factor_sensor, + ) + if success: + print("New schedule is stored.") + + @fm_add_data.command("external-weather-forecasts") @with_appcontext @click.option( From e71fd31411a76a01b9081b7b9168c4f2f9ec00e7 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 23 Feb 2022 14:38:00 +0100 Subject: [PATCH 04/41] Add docstring Signed-off-by: F.N. Claessen --- flexmeasures/cli/data_add.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index ca0e4a72f..e0a806cdd 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -840,6 +840,11 @@ def create_schedule( end_str: str, soc_at_start: float, ): + """Create a new schedule for a given power sensor. + + Currently supports batteries and Charge Points. + """ + # Parse input factor_sensor = Sensor.query.filter(Sensor.id == factor_sensor_id).one_or_none() start = pd.Timestamp(start_str) From 2e46702938c470625c2c8f2405717c7920b95b46 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 23 Feb 2022 14:44:13 +0100 Subject: [PATCH 05/41] Expose soc_min, soc_max and roundtrip_efficiency parameters as CLI options, too Signed-off-by: F.N. Claessen --- flexmeasures/cli/data_add.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index e0a806cdd..2678cfdaa 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -833,12 +833,36 @@ def create_forecasts( required=True, help="State of charge at the start of the schedule.", ) +@click.option( + "--soc-min", + "soc_min", + type=float, + required=False, + help="Minimum state of charge for the schedule.", +) +@click.option( + "--soc-max", + "soc_max", + type=float, + required=False, + help="Maximum state of charge for the schedule.", +) +@click.option( + "--roundtrip-efficiency", + "roundtrip_efficiency", + type=float, + required=False, + help="Round-trip efficiency to use for the schedule.", +) def create_schedule( power_sensor_id: int, factor_sensor_id: int, start_str: str, end_str: str, soc_at_start: float, + soc_min: float, + soc_max: float, + roundtrip_efficiency: float, ): """Create a new schedule for a given power sensor. @@ -857,6 +881,9 @@ def create_schedule( belief_time=server_now(), resolution=factor_sensor.resolution, soc_at_start=soc_at_start, + soc_min=soc_min, + soc_max=soc_max, + roundtrip_efficiency=roundtrip_efficiency, price_sensor=factor_sensor, ) if success: From c213f94b2877c50ebb47846ba08087ba82c1dedd Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 23 Feb 2022 14:57:55 +0100 Subject: [PATCH 06/41] Allow setting SoC targets Signed-off-by: F.N. Claessen --- flexmeasures/cli/data_add.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index 2678cfdaa..242a6af4a 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -4,6 +4,7 @@ from typing import Dict, List, Optional import json +import numpy as np import pandas as pd import pytz from flask import current_app as app @@ -833,6 +834,16 @@ def create_forecasts( required=True, help="State of charge at the start of the schedule.", ) +@click.option( + "--soc-target", + "soc_target_strings", + type=float, + multiple=True, + required=False, + help="Target state of charge at some datetime. Follow up with a string in the form '(, )'." + " This argument can be given multiple times." + " For example: --soc-target '(32.8, 2022-02-23T13:40:52+00:00)'", +) @click.option( "--soc-min", "soc_min", @@ -860,6 +871,7 @@ def create_schedule( start_str: str, end_str: str, soc_at_start: float, + soc_target_strings: List[str], soc_min: float, soc_max: float, roundtrip_efficiency: float, @@ -873,6 +885,17 @@ def create_schedule( factor_sensor = Sensor.query.filter(Sensor.id == factor_sensor_id).one_or_none() start = pd.Timestamp(start_str) end = pd.Timestamp(end_str) + soc_targets = pd.Series( + np.nan, + index=pd.date_range( + start, end, freq=factor_sensor.resolution, closed="right" + ), # note that target values are indexed by their due date (i.e. closed="right") + ) + for soc_target_str in soc_target_strings: + soc_target_value_str, soc_target_dt_str = soc_target_str.split(", ") + soc_target_value = float(soc_target_value_str) + soc_target_datetime = pd.Timestamp(soc_target_dt_str) + soc_targets.loc[soc_target_datetime] = soc_target_value success = make_schedule( sensor_id=power_sensor_id, @@ -881,6 +904,7 @@ def create_schedule( belief_time=server_now(), resolution=factor_sensor.resolution, soc_at_start=soc_at_start, + soc_targets=soc_targets, soc_min=soc_min, soc_max=soc_max, roundtrip_efficiency=roundtrip_efficiency, From de85f30e634961d4109e40a9254448acd2d43896 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 23 Feb 2022 15:01:36 +0100 Subject: [PATCH 07/41] Fix attribute Signed-off-by: F.N. Claessen --- flexmeasures/cli/data_add.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index 242a6af4a..1c2821cc5 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -888,7 +888,7 @@ def create_schedule( soc_targets = pd.Series( np.nan, index=pd.date_range( - start, end, freq=factor_sensor.resolution, closed="right" + start, end, freq=factor_sensor.event_resolution, closed="right" ), # note that target values are indexed by their due date (i.e. closed="right") ) for soc_target_str in soc_target_strings: @@ -896,13 +896,14 @@ def create_schedule( soc_target_value = float(soc_target_value_str) soc_target_datetime = pd.Timestamp(soc_target_dt_str) soc_targets.loc[soc_target_datetime] = soc_target_value + print(soc_targets) success = make_schedule( sensor_id=power_sensor_id, start=start, end=end, belief_time=server_now(), - resolution=factor_sensor.resolution, + resolution=factor_sensor.event_resolution, soc_at_start=soc_at_start, soc_targets=soc_targets, soc_min=soc_min, From 2b427848a91e12602d2a4371cba092b838531775 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 23 Feb 2022 15:03:49 +0100 Subject: [PATCH 08/41] Allow use of make_schedule outside of job context Signed-off-by: F.N. Claessen --- flexmeasures/data/services/scheduling.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py index 33f1ab5e8..d7694d0bd 100644 --- a/flexmeasures/data/services/scheduling.py +++ b/flexmeasures/data/services/scheduling.py @@ -112,9 +112,11 @@ def make_schedule( # find sensor sensor = Sensor.query.filter_by(id=sensor_id).one_or_none() - click.echo( - "Running Scheduling Job %s: %s, from %s to %s" % (rq_job.id, sensor, start, end) - ) + if rq_job: + click.echo( + "Running Scheduling Job %s: %s, from %s to %s" + % (rq_job.id, sensor, start, end) + ) if soc_at_start is None: if ( @@ -169,7 +171,8 @@ def make_schedule( data_source_name="Seita", data_source_type="scheduling script", ) - click.echo("Job %s made schedule." % rq_job.id) + if rq_job: + click.echo("Job %s made schedule." % rq_job.id) ts_value_schedule = [ TimedBelief( From 95d1208b0744cbaf324ab0dfb459e3fd58cdadbd Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 23 Feb 2022 15:11:42 +0100 Subject: [PATCH 09/41] Fix SoC target input and corresponding type annotation Signed-off-by: F.N. Claessen --- flexmeasures/cli/data_add.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index 1c2821cc5..5807ddb60 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -1,7 +1,7 @@ """CLI Tasks for populating the database - most useful in development""" from datetime import timedelta -from typing import Dict, List, Optional +from typing import Dict, List, Optional, Tuple import json import numpy as np @@ -837,12 +837,12 @@ def create_forecasts( @click.option( "--soc-target", "soc_target_strings", - type=float, + type=click.Tuple(types=[float, str]), multiple=True, required=False, - help="Target state of charge at some datetime. Follow up with a string in the form '(, )'." + help="Target state of charge at some datetime. Follow up with a float value and a timezone-aware datetime in ISO 6081 format." " This argument can be given multiple times." - " For example: --soc-target '(32.8, 2022-02-23T13:40:52+00:00)'", + " For example: --soc-target 32.8 2022-02-23T13:40:52+00:00", ) @click.option( "--soc-min", @@ -871,7 +871,7 @@ def create_schedule( start_str: str, end_str: str, soc_at_start: float, - soc_target_strings: List[str], + soc_target_strings: List[Tuple[float, str]], soc_min: float, soc_max: float, roundtrip_efficiency: float, @@ -891,8 +891,8 @@ def create_schedule( start, end, freq=factor_sensor.event_resolution, closed="right" ), # note that target values are indexed by their due date (i.e. closed="right") ) - for soc_target_str in soc_target_strings: - soc_target_value_str, soc_target_dt_str = soc_target_str.split(", ") + for soc_target_tuple in soc_target_strings: + soc_target_value_str, soc_target_dt_str = soc_target_tuple soc_target_value = float(soc_target_value_str) soc_target_datetime = pd.Timestamp(soc_target_dt_str) soc_targets.loc[soc_target_datetime] = soc_target_value From 6d1e5ee35f8229ab8873e412edfa3e73a6be7f3c Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 23 Feb 2022 15:16:37 +0100 Subject: [PATCH 10/41] Add note about current limitations Signed-off-by: F.N. Claessen --- flexmeasures/cli/data_add.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index 5807ddb60..92a66f187 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -878,7 +878,9 @@ def create_schedule( ): """Create a new schedule for a given power sensor. - Currently supports batteries and Charge Points. + Current limitations: + - only supports battery assets and Charge Points + - only supports datetimes on the hour or a multiple of the sensor resolution thereafter """ # Parse input From becf30403e7fe793c5a74bab36509b876bafc061 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 23 Feb 2022 15:50:20 +0100 Subject: [PATCH 11/41] Add notes about units Signed-off-by: F.N. Claessen --- flexmeasures/cli/data_add.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index 92a66f187..0a59da33a 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -832,7 +832,7 @@ def create_forecasts( "soc_at_start", type=float, required=True, - help="State of charge at the start of the schedule.", + help="State of charge (in MWh) at the start of the schedule.", ) @click.option( "--soc-target", @@ -840,7 +840,7 @@ def create_forecasts( type=click.Tuple(types=[float, str]), multiple=True, required=False, - help="Target state of charge at some datetime. Follow up with a float value and a timezone-aware datetime in ISO 6081 format." + help="Target state of charge (in MWh) at some datetime. Follow up with a float value and a timezone-aware datetime in ISO 6081 format." " This argument can be given multiple times." " For example: --soc-target 32.8 2022-02-23T13:40:52+00:00", ) @@ -849,21 +849,21 @@ def create_forecasts( "soc_min", type=float, required=False, - help="Minimum state of charge for the schedule.", + help="Minimum state of charge (in MWh) for the schedule.", ) @click.option( "--soc-max", "soc_max", type=float, required=False, - help="Maximum state of charge for the schedule.", + help="Maximum state of charge (in MWh) for the schedule.", ) @click.option( "--roundtrip-efficiency", "roundtrip_efficiency", type=float, required=False, - help="Round-trip efficiency to use for the schedule.", + help="Round-trip efficiency (e.g. 0.85) to use for the schedule.", ) def create_schedule( power_sensor_id: int, From 1794c678d40785fcd04ce93f30265c5b8143aa99 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 23 Feb 2022 15:53:17 +0100 Subject: [PATCH 12/41] Remove print statement Signed-off-by: F.N. Claessen --- flexmeasures/cli/data_add.py | 1 - 1 file changed, 1 deletion(-) diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index 0a59da33a..e9825ddbe 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -898,7 +898,6 @@ def create_schedule( soc_target_value = float(soc_target_value_str) soc_target_datetime = pd.Timestamp(soc_target_dt_str) soc_targets.loc[soc_target_datetime] = soc_target_value - print(soc_targets) success = make_schedule( sensor_id=power_sensor_id, From bf8cfe22545cf1d94b7b7334ab2485c55e7a0de1 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 23 Feb 2022 15:56:00 +0100 Subject: [PATCH 13/41] Switch the schedule to the resolution of the power sensor Signed-off-by: F.N. Claessen --- flexmeasures/cli/data_add.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index e9825ddbe..c450eea8e 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -884,13 +884,14 @@ def create_schedule( """ # Parse input + power_sensor = Sensor.query.filter(Sensor.id == power_sensor_id).one_or_none() factor_sensor = Sensor.query.filter(Sensor.id == factor_sensor_id).one_or_none() start = pd.Timestamp(start_str) end = pd.Timestamp(end_str) soc_targets = pd.Series( np.nan, index=pd.date_range( - start, end, freq=factor_sensor.event_resolution, closed="right" + start, end, freq=power_sensor.event_resolution, closed="right" ), # note that target values are indexed by their due date (i.e. closed="right") ) for soc_target_tuple in soc_target_strings: @@ -904,7 +905,7 @@ def create_schedule( start=start, end=end, belief_time=server_now(), - resolution=factor_sensor.event_resolution, + resolution=power_sensor.event_resolution, soc_at_start=soc_at_start, soc_targets=soc_targets, soc_min=soc_min, From e2b12e0d6437353216127447399d8f18940742fe Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 23 Feb 2022 16:13:34 +0100 Subject: [PATCH 14/41] Rename variable in tests, too Signed-off-by: F.N. Claessen --- flexmeasures/api/v1_3/tests/test_api_v1_3.py | 2 +- flexmeasures/api/v1_3/tests/test_api_v1_3_fresh_db.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/flexmeasures/api/v1_3/tests/test_api_v1_3.py b/flexmeasures/api/v1_3/tests/test_api_v1_3.py index 33f508610..54429774a 100644 --- a/flexmeasures/api/v1_3/tests/test_api_v1_3.py +++ b/flexmeasures/api/v1_3/tests/test_api_v1_3.py @@ -74,7 +74,7 @@ def test_post_udi_event_and_get_device_message( len(app.queues["scheduling"]) == 1 ) # only 1 schedule should be made for 1 asset job = app.queues["scheduling"].jobs[0] - assert job.kwargs["asset_id"] == sensor.id + assert job.kwargs["sensor_id"] == sensor.id assert job.kwargs["start"] == parse_datetime(message["datetime"]) assert job.id == message["event"] diff --git a/flexmeasures/api/v1_3/tests/test_api_v1_3_fresh_db.py b/flexmeasures/api/v1_3/tests/test_api_v1_3_fresh_db.py index 317288cec..7039532ce 100644 --- a/flexmeasures/api/v1_3/tests/test_api_v1_3_fresh_db.py +++ b/flexmeasures/api/v1_3/tests/test_api_v1_3_fresh_db.py @@ -38,7 +38,7 @@ def test_post_udi_event_and_get_device_message_with_unknown_prices( len(app.queues["scheduling"]) == 1 ) # only 1 schedule should be made for 1 asset job = app.queues["scheduling"].jobs[0] - assert job.kwargs["asset_id"] == sensor.id + assert job.kwargs["sensor_id"] == sensor.id assert job.kwargs["start"] == parse_datetime(message["datetime"]) assert job.id == message["event"] assert ( From a103b401e1554da99ccbcb79154344f48a0c2bb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Fri, 25 Feb 2022 21:26:47 +0100 Subject: [PATCH 15/41] check parameters: do sensors exist, do we have soc-min and soc-max information available? MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- flexmeasures/cli/data_add.py | 41 ++++++++++++++++++++++++++++++------ 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index c450eea8e..b1122dd77 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -863,7 +863,7 @@ def create_forecasts( "roundtrip_efficiency", type=float, required=False, - help="Round-trip efficiency (e.g. 0.85) to use for the schedule.", + help="Round-trip efficiency (e.g. 0.85) to use for the schedule. Defaults to 1 (no losses).", ) def create_schedule( power_sensor_id: int, @@ -872,9 +872,9 @@ def create_schedule( end_str: str, soc_at_start: float, soc_target_strings: List[Tuple[float, str]], - soc_min: float, - soc_max: float, - roundtrip_efficiency: float, + soc_min: Optional[float], + soc_max: Optional[float], + roundtrip_efficiency: Optional[float], ): """Create a new schedule for a given power sensor. @@ -884,10 +884,39 @@ def create_schedule( """ # Parse input - power_sensor = Sensor.query.filter(Sensor.id == power_sensor_id).one_or_none() - factor_sensor = Sensor.query.filter(Sensor.id == factor_sensor_id).one_or_none() + power_sensor: Sensor = Sensor.query.filter( + Sensor.id == power_sensor_id + ).one_or_none() + if power_sensor is None: + click.echo(f"No sensor found with Id {power_sensor_id}.") + raise click.Abort() + if not power_sensor.measures_power: + click.echo(f"Sensor with Id {power_sensor_id} is not a power sensor.") + raise click.Abort() + factor_sensor: Sensor = Sensor.query.filter( + Sensor.id == factor_sensor_id + ).one_or_none() + if factor_sensor is None: + click.echo(f"No sensor found with Id {factor_sensor_id}.") + raise click.Abort() start = pd.Timestamp(start_str) end = pd.Timestamp(end_str) + if ( + soc_min is None + and power_sensor.generic_asset.get_attribute("min_soc_in_mwh") is None + ): + click.echo( + f"No --soc-min given and sensor {power_sensor.generic_asset} has no 'min_soc_in_mwh' attribute." + ) + raise click.Abort() + if ( + soc_max is None + and power_sensor.generic_asset.get_attribute("max_soc_in_mwh") is None + ): + click.echo( + f"No --soc-max given and sensor {power_sensor.generic_asset} has no 'max_soc_in_mwh' attribute." + ) + raise click.Abort() soc_targets = pd.Series( np.nan, index=pd.date_range( From 32db57ad37417d5d7d42734ed0c4f5b7bdfbca51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Fri, 25 Feb 2022 21:34:03 +0100 Subject: [PATCH 16/41] Actually the code expects these soc attributes to be on the sensor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- flexmeasures/cli/data_add.py | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index b1122dd77..3f44b7e2b 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -901,21 +901,12 @@ def create_schedule( raise click.Abort() start = pd.Timestamp(start_str) end = pd.Timestamp(end_str) - if ( - soc_min is None - and power_sensor.generic_asset.get_attribute("min_soc_in_mwh") is None - ): - click.echo( - f"No --soc-min given and sensor {power_sensor.generic_asset} has no 'min_soc_in_mwh' attribute." - ) + # check required attributes + if power_sensor.get_attribute("min_soc_in_mwh") is None: + click.echo(f"{power_sensor} has no 'min_soc_in_mwh' attribute.") raise click.Abort() - if ( - soc_max is None - and power_sensor.generic_asset.get_attribute("max_soc_in_mwh") is None - ): - click.echo( - f"No --soc-max given and sensor {power_sensor.generic_asset} has no 'max_soc_in_mwh' attribute." - ) + if power_sensor.get_attribute("max_soc_in_mwh") is None: + click.echo(f"{power_sensor} has no 'max_soc_in_mwh' attribute.") raise click.Abort() soc_targets = pd.Series( np.nan, From 82221cf00919f64788615b887ae822a1bde1de8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Fri, 25 Feb 2022 22:58:40 +0100 Subject: [PATCH 17/41] use utility function to check attribute MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- flexmeasures/cli/data_add.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index 3f44b7e2b..a9cb0992d 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -24,6 +24,10 @@ Sensor, TimedBelief, ) +from flexmeasures.data.models.validation_utils import ( + check_required_attributes, + MissingAttributeException, +) from flexmeasures.data.models.annotations import Annotation, get_or_create_annotation from flexmeasures.data.schemas.sensors import SensorSchema from flexmeasures.data.schemas.generic_assets import ( @@ -901,13 +905,12 @@ def create_schedule( raise click.Abort() start = pd.Timestamp(start_str) end = pd.Timestamp(end_str) - # check required attributes - if power_sensor.get_attribute("min_soc_in_mwh") is None: - click.echo(f"{power_sensor} has no 'min_soc_in_mwh' attribute.") - raise click.Abort() - if power_sensor.get_attribute("max_soc_in_mwh") is None: - click.echo(f"{power_sensor} has no 'max_soc_in_mwh' attribute.") - raise click.Abort() + for attribute in ("min_soc_in_mwh", "max_soc_in_mwh"): + try: + check_required_attributes(power_sensor, [(attribute, float)]) + except MissingAttributeException: + click.echo(f"{power_sensor} has no {attribute} attribute.") + raise click.Abort() soc_targets = pd.Series( np.nan, index=pd.date_range( From c5a0ac78e2e20e36b48f6b1a09751f4eb845634a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Fri, 25 Feb 2022 22:59:32 +0100 Subject: [PATCH 18/41] identify sensor by instance, as two may have the same name across assets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- flexmeasures/data/models/planning/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/models/planning/utils.py b/flexmeasures/data/models/planning/utils.py index e11d50810..cfc71c048 100644 --- a/flexmeasures/data/models/planning/utils.py +++ b/flexmeasures/data/models/planning/utils.py @@ -88,7 +88,7 @@ def get_prices( price_sensor = get_market(sensor) price_bdf: tb.BeliefsDataFrame = TimedBelief.search( - price_sensor.name, + price_sensor, event_starts_after=query_window[0], event_ends_before=query_window[1], resolution=to_offset(resolution).freqstr, From b7acb6820c8b2234bb1d8131807e569f3d9b56e8 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 25 Feb 2022 23:11:09 +0100 Subject: [PATCH 19/41] Consistent capitalization Signed-off-by: F.N. Claessen --- flexmeasures/cli/data_add.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index 6bc2b43e0..1bcb399fc 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -140,7 +140,7 @@ def new_user( raise click.Abort account = db.session.query(Account).get(account_id) if account is None: - print(f"No account with id {account_id} found!") + print(f"No account with ID {account_id} found!") raise click.Abort pwd1 = getpass.getpass(prompt="Please enter the password:") pwd2 = getpass.getpass(prompt="Please repeat the password:") @@ -462,7 +462,7 @@ def add_beliefs( """ sensor = Sensor.query.filter(Sensor.id == sensor_id).one_or_none() if sensor is None: - print(f"Failed to create beliefs: no sensor found with id {sensor_id}.") + print(f"Failed to create beliefs: no sensor found with ID {sensor_id}.") return if source.isdigit(): _source = get_source_or_none(int(source), source_type="CLI script") @@ -895,16 +895,16 @@ def create_schedule( Sensor.id == power_sensor_id ).one_or_none() if power_sensor is None: - click.echo(f"No sensor found with Id {power_sensor_id}.") + click.echo(f"No sensor found with ID {power_sensor_id}.") raise click.Abort() if not power_sensor.measures_power: - click.echo(f"Sensor with Id {power_sensor_id} is not a power sensor.") + click.echo(f"Sensor with ID {power_sensor_id} is not a power sensor.") raise click.Abort() factor_sensor: Sensor = Sensor.query.filter( Sensor.id == factor_sensor_id ).one_or_none() if factor_sensor is None: - click.echo(f"No sensor found with Id {factor_sensor_id}.") + click.echo(f"No sensor found with ID {factor_sensor_id}.") raise click.Abort() start = pd.Timestamp(start_str) end = pd.Timestamp(end_str) From 15913f5b8f4aee93243b0670c7086ca8e184d119 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 28 Feb 2022 15:38:38 +0100 Subject: [PATCH 20/41] Allow unit conversion for individual int/float values Signed-off-by: F.N. Claessen --- flexmeasures/utils/unit_utils.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/flexmeasures/utils/unit_utils.py b/flexmeasures/utils/unit_utils.py index d634bc53a..094b87197 100644 --- a/flexmeasures/utils/unit_utils.py +++ b/flexmeasures/utils/unit_utils.py @@ -174,16 +174,20 @@ def is_energy_unit(unit: str) -> bool: def convert_units( - data: Union[pd.Series, List[Union[int, float]]], + data: Union[pd.Series, List[Union[int, float]], int, float], from_unit: str, to_unit: str, event_resolution: Optional[timedelta], -) -> Union[pd.Series, List[Union[int, float]]]: +) -> Union[pd.Series, List[Union[int, float]], int, float]: """Updates data values to reflect the given unit conversion.""" if from_unit != to_unit: from_magnitudes = ( - data.to_numpy() if isinstance(data, pd.Series) else np.asarray(data) + data.to_numpy() + if isinstance(data, pd.Series) + else np.asarray(data) + if isinstance(data, list) + else np.array([data]) ) try: from_quantities = ur.Quantity(from_magnitudes, from_unit) @@ -201,12 +205,19 @@ def convert_units( from_unit, to_unit, event_resolution ) to_magnitudes = from_magnitudes * multiplier + + # Output type should match input type if isinstance(data, pd.Series): + # Pandas Series data = pd.Series( to_magnitudes, index=data.index, name=data.name, ) - else: + elif isinstance(data, list): + # list data = list(to_magnitudes) + else: + # int or float + data = to_magnitudes[0] return data From 0211ecbbb93c8c905a3a0413c0c7d74b949103bc Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 28 Feb 2022 15:40:26 +0100 Subject: [PATCH 21/41] Allow unit conversion to and from a percentage of some capacity Signed-off-by: F.N. Claessen --- flexmeasures/utils/unit_utils.py | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/flexmeasures/utils/unit_utils.py b/flexmeasures/utils/unit_utils.py index 094b87197..96e69608c 100644 --- a/flexmeasures/utils/unit_utils.py +++ b/flexmeasures/utils/unit_utils.py @@ -177,7 +177,8 @@ def convert_units( data: Union[pd.Series, List[Union[int, float]], int, float], from_unit: str, to_unit: str, - event_resolution: Optional[timedelta], + event_resolution: Optional[timedelta] = None, + capacity: Optional[str] = None, ) -> Union[pd.Series, List[Union[int, float]], int, float]: """Updates data values to reflect the given unit conversion.""" @@ -199,12 +200,26 @@ def convert_units( raise e # reraise try: to_magnitudes = from_quantities.to(ur.Quantity(to_unit)).magnitude - except pint.errors.DimensionalityError: - # Catch multiplicative conversions that use the resolution, like "kWh/15min" to "kW" - multiplier = determine_unit_conversion_multiplier( - from_unit, to_unit, event_resolution - ) - to_magnitudes = from_magnitudes * multiplier + except pint.errors.DimensionalityError as e: + # Catch multiplicative conversions that rely on a capacity, like "%" to "kWh" and vice versa + if "from 'percent'" in str(e): + to_magnitudes = ( + (from_quantities * ur.Quantity(capacity)) + .to(ur.Quantity(to_unit)) + .magnitude + ) + elif "to 'percent'" in str(e): + to_magnitudes = ( + (from_quantities / ur.Quantity(capacity)) + .to(ur.Quantity(to_unit)) + .magnitude + ) + else: + # Catch multiplicative conversions that use the resolution, like "kWh/15min" to "kW" + multiplier = determine_unit_conversion_multiplier( + from_unit, to_unit, event_resolution + ) + to_magnitudes = from_magnitudes * multiplier # Output type should match input type if isinstance(data, pd.Series): From 4e925fa08695039b7af77a08799a24f9941118e9 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 28 Feb 2022 15:40:35 +0100 Subject: [PATCH 22/41] Test unit conversion to and from a percentage of some capacity Signed-off-by: F.N. Claessen --- flexmeasures/utils/tests/test_unit_utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/flexmeasures/utils/tests/test_unit_utils.py b/flexmeasures/utils/tests/test_unit_utils.py index eaa6c97ed..726cfdde1 100644 --- a/flexmeasures/utils/tests/test_unit_utils.py +++ b/flexmeasures/utils/tests/test_unit_utils.py @@ -25,6 +25,8 @@ ("m³/h", "l/h", 1000, None), ("m³", "m³/h", 4, None), ("MW", "kW", 1000, None), + ("%", "kWh", 0.5, None), # i.e. 1% of 50 kWh (the capacity used in the test) + ("kWh", "%", 2, None), # i.e. 1 kWh = 2% of 50 kWh ("kWh", "kW", 4, None), ("kW", "kWh", 1 / 4, None), ("-W", "W", -1, None), @@ -53,6 +55,7 @@ def test_convert_unit( from_unit=from_unit, to_unit=to_unit, event_resolution=timedelta(minutes=15), + capacity="50 kWh", ) if expected_multiplier is not None: expected_data = data * expected_multiplier From 7b42db0c576cda354492f87301d747cf3a9bb16b Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 28 Feb 2022 15:42:11 +0100 Subject: [PATCH 23/41] CLI scheduling command uses % SoC units by default Signed-off-by: F.N. Claessen --- flexmeasures/cli/data_add.py | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index 1bcb399fc..709c4926d 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -839,7 +839,7 @@ def create_forecasts( "soc_at_start", type=float, required=True, - help="State of charge (in MWh) at the start of the schedule.", + help="State of charge (in %) at the start of the schedule. Use --soc-unit to set a different unit.", ) @click.option( "--soc-target", @@ -847,7 +847,8 @@ def create_forecasts( type=click.Tuple(types=[float, str]), multiple=True, required=False, - help="Target state of charge (in MWh) at some datetime. Follow up with a float value and a timezone-aware datetime in ISO 6081 format." + help="Target state of charge (in %) at some datetime. Follow up with a float value and a timezone-aware datetime in ISO 6081 format." + " Use --soc-unit to set a different unit." " This argument can be given multiple times." " For example: --soc-target 32.8 2022-02-23T13:40:52+00:00", ) @@ -856,14 +857,14 @@ def create_forecasts( "soc_min", type=float, required=False, - help="Minimum state of charge (in MWh) for the schedule.", + help="Minimum state of charge (in %) for the schedule. Use --soc-unit to set a different unit.", ) @click.option( "--soc-max", "soc_max", type=float, required=False, - help="Maximum state of charge (in MWh) for the schedule.", + help="Maximum state of charge (in %) for the schedule. Use --soc-unit to set a different unit.", ) @click.option( "--roundtrip-efficiency", @@ -872,6 +873,12 @@ def create_forecasts( required=False, help="Round-trip efficiency (e.g. 0.85) to use for the schedule. Defaults to 1 (no losses).", ) +@click.option( + "--soc-unit", + "soc_unit", + default="%", + help="Unit of the passed SoC values, such as 'MWh', 'kWh' or '%' (the default).", +) def create_schedule( power_sensor_id: int, factor_sensor_id: int, @@ -882,6 +889,7 @@ def create_schedule( soc_min: Optional[float], soc_max: Optional[float], roundtrip_efficiency: Optional[float], + soc_unit: str = "%", ): """Create a new schedule for a given power sensor. @@ -926,6 +934,16 @@ def create_schedule( soc_target_datetime = pd.Timestamp(soc_target_dt_str) soc_targets.loc[soc_target_datetime] = soc_target_value + # Convert SoC units if needed + if soc_unit != "MWh": + capacity_str = f"{power_sensor.get_attribute('max_soc_in_mwh')} MWh" + soc_at_start = convert_units( + soc_at_start, soc_unit, "MWh", capacity=capacity_str + ) + soc_targets = convert_units(soc_targets, soc_unit, "MWh", capacity=capacity_str) + soc_min = convert_units(soc_min, soc_unit, "MWh", capacity=capacity_str) + soc_max = convert_units(soc_max, soc_unit, "MWh", capacity=capacity_str) + success = make_schedule( sensor_id=power_sensor_id, start=start, From 47b0bef5c6f029213f7b650b8052303eb8ef6e5d Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 28 Feb 2022 15:52:23 +0100 Subject: [PATCH 24/41] Round charging schedules Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/battery.py | 5 +++++ flexmeasures/data/models/planning/charging_station.py | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/flexmeasures/data/models/planning/battery.py b/flexmeasures/data/models/planning/battery.py index 4329d9503..76426b89f 100644 --- a/flexmeasures/data/models/planning/battery.py +++ b/flexmeasures/data/models/planning/battery.py @@ -26,6 +26,7 @@ def schedule_battery( roundtrip_efficiency: Optional[float] = None, prefer_charging_sooner: bool = True, price_sensor: Optional[Sensor] = None, + round_to_decimals: Optional[int] = 6, ) -> Union[pd.Series, None]: """Schedule a battery asset based directly on the latest beliefs regarding market prices within the specified time window. @@ -136,4 +137,8 @@ def schedule_battery( else: battery_schedule = ems_schedule[0] + # Round schedule + if round_to_decimals: + battery_schedule = battery_schedule.round(round_to_decimals) + return battery_schedule diff --git a/flexmeasures/data/models/planning/charging_station.py b/flexmeasures/data/models/planning/charging_station.py index 915a0d79c..4effd311d 100644 --- a/flexmeasures/data/models/planning/charging_station.py +++ b/flexmeasures/data/models/planning/charging_station.py @@ -26,6 +26,7 @@ def schedule_charging_station( roundtrip_efficiency: Optional[float] = None, prefer_charging_sooner: bool = True, price_sensor: Optional[Sensor] = None, + round_to_decimals: Optional[int] = 6, ) -> Union[Series, None]: """Schedule a charging station asset based directly on the latest beliefs regarding market prices within the specified time window. @@ -139,4 +140,8 @@ def schedule_charging_station( else: charging_station_schedule = ems_schedule[0] + # Round schedule + if round_to_decimals: + charging_station_schedule = charging_station_schedule.round(round_to_decimals) + return charging_station_schedule From b6603de31f284dba491fa9b7460f5bf37bf7de9e Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 28 Feb 2022 15:59:25 +0100 Subject: [PATCH 25/41] Fix calculation of belief time (by handing the problem over to timely beliefs) Signed-off-by: F.N. Claessen --- flexmeasures/data/services/scheduling.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py index d7694d0bd..46f95f11a 100644 --- a/flexmeasures/data/services/scheduling.py +++ b/flexmeasures/data/services/scheduling.py @@ -177,7 +177,7 @@ def make_schedule( ts_value_schedule = [ TimedBelief( event_start=dt, - belief_horizon=dt.astimezone(pytz.utc) - belief_time.astimezone(pytz.utc), + belief_time=belief_time, event_value=-value, sensor=sensor, source=data_source, From 135c0d6e78f3e905b082a6b9c09abe7b2e2fe48f Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 28 Feb 2022 16:11:37 +0100 Subject: [PATCH 26/41] flake8 Signed-off-by: F.N. Claessen --- flexmeasures/data/services/scheduling.py | 1 - 1 file changed, 1 deletion(-) diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py index 46f95f11a..b58a6ca5a 100644 --- a/flexmeasures/data/services/scheduling.py +++ b/flexmeasures/data/services/scheduling.py @@ -5,7 +5,6 @@ import click import numpy as np import pandas as pd -import pytz from rq import get_current_job from rq.job import Job import timely_beliefs as tb From c998fe1518d564e3a8e8c1b42b960b7c40000ea4 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 28 Feb 2022 16:23:27 +0100 Subject: [PATCH 27/41] fix (thanks mypy) Signed-off-by: F.N. Claessen --- flexmeasures/cli/data_add.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index 709c4926d..528c58ddd 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -937,12 +937,12 @@ def create_schedule( # Convert SoC units if needed if soc_unit != "MWh": capacity_str = f"{power_sensor.get_attribute('max_soc_in_mwh')} MWh" - soc_at_start = convert_units( - soc_at_start, soc_unit, "MWh", capacity=capacity_str - ) + soc_at_start = convert_units(soc_at_start, soc_unit, "MWh", capacity=capacity_str) # type: ignore soc_targets = convert_units(soc_targets, soc_unit, "MWh", capacity=capacity_str) - soc_min = convert_units(soc_min, soc_unit, "MWh", capacity=capacity_str) - soc_max = convert_units(soc_max, soc_unit, "MWh", capacity=capacity_str) + if soc_min is not None: + soc_min = convert_units(soc_min, soc_unit, "MWh", capacity=capacity_str) # type: ignore + if soc_max is not None: + soc_max = convert_units(soc_max, soc_unit, "MWh", capacity=capacity_str) # type: ignore success = make_schedule( sensor_id=power_sensor_id, From 95d44e51e1b6bfa2e1a8bd06fdbdc93c8076a268 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 28 Feb 2022 16:52:09 +0100 Subject: [PATCH 28/41] Fix timezone issue for trimming planning window Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/battery.py | 3 +++ flexmeasures/data/models/planning/charging_station.py | 10 +++++----- flexmeasures/data/models/planning/utils.py | 2 +- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/flexmeasures/data/models/planning/battery.py b/flexmeasures/data/models/planning/battery.py index 76426b89f..25a90fca8 100644 --- a/flexmeasures/data/models/planning/battery.py +++ b/flexmeasures/data/models/planning/battery.py @@ -63,8 +63,11 @@ def schedule_battery( sensor=sensor, allow_trimmed_query_window=True, ) + start = pd.Timestamp(start).tz_convert("UTC") + end = pd.Timestamp(end).tz_convert("UTC") if soc_targets is not None: # soc targets are at the end of each time slot, while prices are indexed by the start of each time slot + soc_targets = soc_targets.tz_convert("UTC") soc_targets = soc_targets[start + resolution : end] # Add tiny price slope to prefer charging now rather than later, and discharging later rather than now. diff --git a/flexmeasures/data/models/planning/charging_station.py b/flexmeasures/data/models/planning/charging_station.py index 4effd311d..0329cc92e 100644 --- a/flexmeasures/data/models/planning/charging_station.py +++ b/flexmeasures/data/models/planning/charging_station.py @@ -1,7 +1,7 @@ from typing import Optional, Union from datetime import datetime, timedelta -from pandas import Series, Timestamp +import pandas as pd from flexmeasures.data.models.time_series import Sensor from flexmeasures.data.models.planning.solver import device_scheduler @@ -20,14 +20,14 @@ def schedule_charging_station( end: datetime, resolution: timedelta, soc_at_start: float, - soc_targets: Series, + soc_targets: pd.Series, soc_min: Optional[float] = None, soc_max: Optional[float] = None, roundtrip_efficiency: Optional[float] = None, prefer_charging_sooner: bool = True, price_sensor: Optional[Sensor] = None, round_to_decimals: Optional[int] = 6, -) -> Union[Series, None]: +) -> Union[pd.Series, None]: """Schedule a charging station asset based directly on the latest beliefs regarding market prices within the specified time window. For the resulting consumption schedule, consumption is defined as positive values. @@ -62,8 +62,8 @@ def schedule_charging_station( ) # soc targets are at the end of each time slot, while prices are indexed by the start of each time slot soc_targets = soc_targets.tz_convert("UTC") - start = Timestamp(start).tz_convert("UTC") - end = Timestamp(end).tz_convert("UTC") + start = pd.Timestamp(start).tz_convert("UTC") + end = pd.Timestamp(end).tz_convert("UTC") soc_targets = soc_targets[start + resolution : end] # Add tiny price slope to prefer charging now rather than later, and discharging later rather than now. diff --git a/flexmeasures/data/models/planning/utils.py b/flexmeasures/data/models/planning/utils.py index cfc71c048..47a066d30 100644 --- a/flexmeasures/data/models/planning/utils.py +++ b/flexmeasures/data/models/planning/utils.py @@ -103,7 +103,7 @@ def get_prices( nan_prices.any() or pd.Timestamp(price_df.index[0]).tz_convert("UTC") != pd.Timestamp(query_window[0]).tz_convert("UTC") - or pd.Timestamp(price_df.index[-1]).tz_convert("UTC") + or pd.Timestamp(price_df.index[-1]).tz_convert("UTC") + resolution != pd.Timestamp(query_window[-1]).tz_convert("UTC") ): if allow_trimmed_query_window: From 93519446603c01254cec4391ad136a28e3c135c0 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 28 Feb 2022 17:25:38 +0100 Subject: [PATCH 29/41] Fix timezone issue for setting SoC targets for schedules crossing DST transitions Signed-off-by: F.N. Claessen --- flexmeasures/cli/data_add.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index 528c58ddd..c6cdf2dda 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -925,7 +925,10 @@ def create_schedule( soc_targets = pd.Series( np.nan, index=pd.date_range( - start, end, freq=power_sensor.event_resolution, closed="right" + pd.Timestamp(start).tz_convert(power_sensor.timezone), + pd.Timestamp(end).tz_convert(power_sensor.timezone), + freq=power_sensor.event_resolution, + closed="right", ), # note that target values are indexed by their due date (i.e. closed="right") ) for soc_target_tuple in soc_target_strings: From 0a42036fbd677a5151913c518c1b211911987622 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Tue, 1 Mar 2022 11:53:25 +0100 Subject: [PATCH 30/41] turn --end parameter into --duration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- flexmeasures/cli/data_add.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index c6cdf2dda..97d7bb067 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -14,6 +14,7 @@ from sqlalchemy.exc import IntegrityError import timely_beliefs as tb from workalendar.registry import registry as workalendar_registry +import isodate from flexmeasures.data import db from flexmeasures.data.scripts.data_gen import ( @@ -829,10 +830,10 @@ def create_forecasts( help="Schedule starts at this datetime. Follow up with a timezone-aware datetime in ISO 6801 format.", ) @click.option( - "--until", - "end_str", + "--duration", + "duration_str", required=True, - help="Schedule ends at this datetime. Follow up with a timezone-aware datetime in ISO 6801 format.", + help="Duration of schedule, after --from. Follow up with a duration in ISO 6801 format, e.g. PT1H (1 hour) or PT45M (45 minutes).", ) @click.option( "--soc-at-start", @@ -883,7 +884,7 @@ def create_schedule( power_sensor_id: int, factor_sensor_id: int, start_str: str, - end_str: str, + duration_str: str, soc_at_start: float, soc_target_strings: List[Tuple[float, str]], soc_min: Optional[float], @@ -915,7 +916,7 @@ def create_schedule( click.echo(f"No sensor found with ID {factor_sensor_id}.") raise click.Abort() start = pd.Timestamp(start_str) - end = pd.Timestamp(end_str) + end = start + isodate.parse_duration(duration_str) for attribute in ("min_soc_in_mwh", "max_soc_in_mwh"): try: check_required_attributes(power_sensor, [(attribute, float)]) From 525a303a906f2c8ef1236a2abdcaff016170c50d Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 3 Mar 2022 17:02:07 +0100 Subject: [PATCH 31/41] Switch efficiency input to % values and add input validation using marshmallow Signed-off-by: F.N. Claessen --- flexmeasures/cli/data_add.py | 30 ++++++++++++++++-------------- flexmeasures/data/schemas/units.py | 19 +++++++++++++++++++ 2 files changed, 35 insertions(+), 14 deletions(-) create mode 100644 flexmeasures/data/schemas/units.py diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index 97d7bb067..a4c510e11 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -36,6 +36,7 @@ ) from flexmeasures.data.models.annotations import Annotation, get_or_create_annotation from flexmeasures.data.schemas.sensors import SensorSchema +from flexmeasures.data.schemas.units import NonNegativeFloat, PercentageFloat from flexmeasures.data.schemas.generic_assets import ( GenericAssetSchema, GenericAssetTypeSchema, @@ -838,41 +839,42 @@ def create_forecasts( @click.option( "--soc-at-start", "soc_at_start", - type=float, + type=NonNegativeFloat(), required=True, - help="State of charge (in %) at the start of the schedule. Use --soc-unit to set a different unit.", + help="State of charge (in %, e.g. 32.8) at the start of the schedule. Use --soc-unit to set a different unit.", ) @click.option( "--soc-target", "soc_target_strings", - type=click.Tuple(types=[float, str]), + type=click.Tuple(types=[NonNegativeFloat(), str]), multiple=True, required=False, - help="Target state of charge (in %) at some datetime. Follow up with a float value and a timezone-aware datetime in ISO 6081 format." + help="Target state of charge (in %, e.g. 100) at some datetime. Follow up with a float value and a timezone-aware datetime in ISO 6081 format." " Use --soc-unit to set a different unit." " This argument can be given multiple times." - " For example: --soc-target 32.8 2022-02-23T13:40:52+00:00", + " For example: --soc-target 100 2022-02-23T13:40:52+00:00", ) @click.option( "--soc-min", "soc_min", - type=float, + type=NonNegativeFloat(), required=False, - help="Minimum state of charge (in %) for the schedule. Use --soc-unit to set a different unit.", + help="Minimum state of charge (in %, e.g. 20) for the schedule. Use --soc-unit to set a different unit.", ) @click.option( "--soc-max", "soc_max", - type=float, + type=NonNegativeFloat(), required=False, - help="Maximum state of charge (in %) for the schedule. Use --soc-unit to set a different unit.", + help="Maximum state of charge (in %, e.g. 80) for the schedule. Use --soc-unit to set a different unit.", ) @click.option( "--roundtrip-efficiency", - "roundtrip_efficiency", - type=float, + "roundtrip_efficiency_as_percentage", + type=PercentageFloat(), required=False, - help="Round-trip efficiency (e.g. 0.85) to use for the schedule. Defaults to 1 (no losses).", + default=100, + help="Round-trip efficiency (in %, e.g. 85) to use for the schedule. Defaults to 100 (no losses).", ) @click.option( "--soc-unit", @@ -889,7 +891,7 @@ def create_schedule( soc_target_strings: List[Tuple[float, str]], soc_min: Optional[float], soc_max: Optional[float], - roundtrip_efficiency: Optional[float], + roundtrip_efficiency_as_percentage: Optional[float] = 100, soc_unit: str = "%", ): """Create a new schedule for a given power sensor. @@ -958,7 +960,7 @@ def create_schedule( soc_targets=soc_targets, soc_min=soc_min, soc_max=soc_max, - roundtrip_efficiency=roundtrip_efficiency, + roundtrip_efficiency=roundtrip_efficiency_as_percentage / 100, price_sensor=factor_sensor, ) if success: diff --git a/flexmeasures/data/schemas/units.py b/flexmeasures/data/schemas/units.py new file mode 100644 index 000000000..b8fdd6a2d --- /dev/null +++ b/flexmeasures/data/schemas/units.py @@ -0,0 +1,19 @@ +from marshmallow import fields, validate + +from flexmeasures.data.schemas.utils import MarshmallowClickMixin + + +class PercentageFloat(fields.Float, MarshmallowClickMixin): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # Insert validation into self.validators so that multiple errors can be stored. + validator = validate.Range(min=0, max=100) + self.validators.insert(0, validator) + + +class NonNegativeFloat(fields.Float, MarshmallowClickMixin): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # Insert validation into self.validators so that multiple errors can be stored. + validator = validate.Range(min=0) + self.validators.insert(0, validator) From c3c6a813bd88ef05628a398dc092ddb454a0b8fb Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 4 Mar 2022 15:31:28 +0100 Subject: [PATCH 32/41] Force users to be specific upon providing input of percentage values or ratios for the round-trip efficiency Signed-off-by: F.N. Claessen --- flexmeasures/cli/data_add.py | 18 +++++++---- flexmeasures/data/schemas/units.py | 52 ++++++++++++++++++++++++++---- 2 files changed, 57 insertions(+), 13 deletions(-) diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index a4c510e11..5d473a9fe 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -4,6 +4,7 @@ from typing import Dict, List, Optional, Tuple import json +from marshmallow import validate import numpy as np import pandas as pd import pytz @@ -36,7 +37,10 @@ ) from flexmeasures.data.models.annotations import Annotation, get_or_create_annotation from flexmeasures.data.schemas.sensors import SensorSchema -from flexmeasures.data.schemas.units import NonNegativeFloat, PercentageFloat +from flexmeasures.data.schemas.units import ( + NonNegativeFloat, + QuantityField, +) from flexmeasures.data.schemas.generic_assets import ( GenericAssetSchema, GenericAssetTypeSchema, @@ -870,11 +874,11 @@ def create_forecasts( ) @click.option( "--roundtrip-efficiency", - "roundtrip_efficiency_as_percentage", - type=PercentageFloat(), + "roundtrip_efficiency", + type=QuantityField("dimensionless", validate=validate.Range(min=0, max=1)), required=False, - default=100, - help="Round-trip efficiency (in %, e.g. 85) to use for the schedule. Defaults to 100 (no losses).", + default=1, + help="Round-trip efficiency (e.g. 85% or 0.85) to use for the schedule. Defaults to 100% (no losses).", ) @click.option( "--soc-unit", @@ -891,7 +895,7 @@ def create_schedule( soc_target_strings: List[Tuple[float, str]], soc_min: Optional[float], soc_max: Optional[float], - roundtrip_efficiency_as_percentage: Optional[float] = 100, + roundtrip_efficiency: Optional[float] = 1, soc_unit: str = "%", ): """Create a new schedule for a given power sensor. @@ -960,7 +964,7 @@ def create_schedule( soc_targets=soc_targets, soc_min=soc_min, soc_max=soc_max, - roundtrip_efficiency=roundtrip_efficiency_as_percentage / 100, + roundtrip_efficiency=roundtrip_efficiency, price_sensor=factor_sensor, ) if success: diff --git a/flexmeasures/data/schemas/units.py b/flexmeasures/data/schemas/units.py index b8fdd6a2d..15ac0771f 100644 --- a/flexmeasures/data/schemas/units.py +++ b/flexmeasures/data/schemas/units.py @@ -1,19 +1,59 @@ -from marshmallow import fields, validate +from typing import Optional + +from marshmallow import fields, validate, ValidationError +from marshmallow.validate import Validator from flexmeasures.data.schemas.utils import MarshmallowClickMixin +from flexmeasures.utils.unit_utils import is_valid_unit, ur -class PercentageFloat(fields.Float, MarshmallowClickMixin): +class NonNegativeFloat(fields.Float, MarshmallowClickMixin): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) # Insert validation into self.validators so that multiple errors can be stored. - validator = validate.Range(min=0, max=100) + validator = validate.Range(min=0) self.validators.insert(0, validator) -class NonNegativeFloat(fields.Float, MarshmallowClickMixin): - def __init__(self, *args, **kwargs): +class Quantity(Validator): + """Validator which succeeds if the value passed to it is a valid quantity.""" + + def __init__(self, *, error: Optional[str] = None): + self.error = error + + def __call__(self, value): + if not is_valid_unit(value): + raise ValidationError("Not a valid quantity") + return value + + +class QuantityField(fields.Str, MarshmallowClickMixin): + """Marshmallow/Click field for validating quantities against a unit registry. + + The FlexMeasures unit registry is based on the pint library. + + For example: + >>> percentage_field = QuantityField("dimensionless", validate=validate.Range(min=0, max=1)) + >>> percentage_field.deserialize("10%") + 0.1 + >>> percentage_field.deserialize(0.1) + 0.1 + >>> power_field = QuantityField("kW") + >>> power_field.deserialize("120 W") + 0.12 + """ + + def __init__(self, to_unit: str, *args, **kwargs): super().__init__(*args, **kwargs) # Insert validation into self.validators so that multiple errors can be stored. - validator = validate.Range(min=0) + validator = Quantity() self.validators.insert(0, validator) + self.to_unit = ur.Quantity(to_unit) + + def _deserialize(self, value, attr, obj, **kwargs) -> ur.Quantity: + """Turn a quantity describing string into a float magnitude.""" + return ur.Quantity(value).to(self.to_unit).magnitude + + def _serialize(self, value, attr, data, **kwargs): + """Turn a Quantity into a string in scientific format.""" + return "{:~P}".format(value.to(self.to_unit)) From efea48c288bfb8570f150087b1dc0821f15251cd Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 4 Mar 2022 16:09:20 +0100 Subject: [PATCH 33/41] Fix percentage range and add range example for power quantity Signed-off-by: F.N. Claessen --- flexmeasures/cli/data_add.py | 2 +- flexmeasures/data/schemas/units.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index 5d473a9fe..83dfcaf8e 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -875,7 +875,7 @@ def create_forecasts( @click.option( "--roundtrip-efficiency", "roundtrip_efficiency", - type=QuantityField("dimensionless", validate=validate.Range(min=0, max=1)), + type=QuantityField("%", validate=validate.Range(min=0, max=1)), required=False, default=1, help="Round-trip efficiency (e.g. 85% or 0.85) to use for the schedule. Defaults to 100% (no losses).", diff --git a/flexmeasures/data/schemas/units.py b/flexmeasures/data/schemas/units.py index 15ac0771f..ef6436a94 100644 --- a/flexmeasures/data/schemas/units.py +++ b/flexmeasures/data/schemas/units.py @@ -33,12 +33,12 @@ class QuantityField(fields.Str, MarshmallowClickMixin): The FlexMeasures unit registry is based on the pint library. For example: - >>> percentage_field = QuantityField("dimensionless", validate=validate.Range(min=0, max=1)) - >>> percentage_field.deserialize("10%") - 0.1 - >>> percentage_field.deserialize(0.1) - 0.1 - >>> power_field = QuantityField("kW") + >>> percentage_field = QuantityField("%", validate=validate.Range(min=0, max=1)) + >>> percentage_field.deserialize("2.5%") + 2.5 + >>> percentage_field.deserialize(0.025) + 2.5 + >>> power_field = QuantityField("kW", validate=validate.Range(max=ur.Quantity("1 kW"))) >>> power_field.deserialize("120 W") 0.12 """ From dc8f1a9657d7e98e9e2d82cb08c5ff3094a776d3 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 4 Mar 2022 17:04:00 +0100 Subject: [PATCH 34/41] Switch validation to return Quantity objects instead of magnitudes Signed-off-by: F.N. Claessen --- flexmeasures/cli/data_add.py | 8 +++++++- flexmeasures/data/schemas/units.py | 10 +++++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index 83dfcaf8e..e9d8d5be8 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -55,7 +55,7 @@ ) from flexmeasures.utils import flexmeasures_inflection from flexmeasures.utils.time_utils import server_now -from flexmeasures.utils.unit_utils import convert_units +from flexmeasures.utils.unit_utils import convert_units, ur @click.group("add") @@ -938,6 +938,12 @@ def create_schedule( closed="right", ), # note that target values are indexed by their due date (i.e. closed="right") ) + + # Convert round-trip efficiency to dimensionless + roundtrip_efficiency = roundtrip_efficiency.to( + ur.Quantity("dimensionless") + ).magnitude + for soc_target_tuple in soc_target_strings: soc_target_value_str, soc_target_dt_str = soc_target_tuple soc_target_value = float(soc_target_value_str) diff --git a/flexmeasures/data/schemas/units.py b/flexmeasures/data/schemas/units.py index ef6436a94..d3d851005 100644 --- a/flexmeasures/data/schemas/units.py +++ b/flexmeasures/data/schemas/units.py @@ -35,12 +35,12 @@ class QuantityField(fields.Str, MarshmallowClickMixin): For example: >>> percentage_field = QuantityField("%", validate=validate.Range(min=0, max=1)) >>> percentage_field.deserialize("2.5%") - 2.5 + >>> percentage_field.deserialize(0.025) - 2.5 + >>> power_field = QuantityField("kW", validate=validate.Range(max=ur.Quantity("1 kW"))) >>> power_field.deserialize("120 W") - 0.12 + """ def __init__(self, to_unit: str, *args, **kwargs): @@ -51,8 +51,8 @@ def __init__(self, to_unit: str, *args, **kwargs): self.to_unit = ur.Quantity(to_unit) def _deserialize(self, value, attr, obj, **kwargs) -> ur.Quantity: - """Turn a quantity describing string into a float magnitude.""" - return ur.Quantity(value).to(self.to_unit).magnitude + """Turn a quantity describing string into a Quantity.""" + return ur.Quantity(value).to(self.to_unit) def _serialize(self, value, attr, data, **kwargs): """Turn a Quantity into a string in scientific format.""" From ec3035aef8c1d75f7b5ce5751536e2a0bf6d3f23 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 4 Mar 2022 17:06:42 +0100 Subject: [PATCH 35/41] Switch SOC input to percentages only Signed-off-by: F.N. Claessen --- flexmeasures/cli/data_add.py | 66 ++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 36 deletions(-) diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index e9d8d5be8..f87b43f57 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -37,10 +37,7 @@ ) from flexmeasures.data.models.annotations import Annotation, get_or_create_annotation from flexmeasures.data.schemas.sensors import SensorSchema -from flexmeasures.data.schemas.units import ( - NonNegativeFloat, - QuantityField, -) +from flexmeasures.data.schemas.units import QuantityField from flexmeasures.data.schemas.generic_assets import ( GenericAssetSchema, GenericAssetTypeSchema, @@ -843,34 +840,36 @@ def create_forecasts( @click.option( "--soc-at-start", "soc_at_start", - type=NonNegativeFloat(), + type=QuantityField("%", validate=validate.Range(min=0, max=1)), required=True, - help="State of charge (in %, e.g. 32.8) at the start of the schedule. Use --soc-unit to set a different unit.", + help="State of charge (e.g 32.8%, or 0.328) at the start of the schedule. Use --soc-unit to set a different unit.", ) @click.option( "--soc-target", "soc_target_strings", - type=click.Tuple(types=[NonNegativeFloat(), str]), + type=click.Tuple( + types=[QuantityField("%", validate=validate.Range(min=0, max=1)), str] + ), multiple=True, required=False, - help="Target state of charge (in %, e.g. 100) at some datetime. Follow up with a float value and a timezone-aware datetime in ISO 6081 format." + help="Target state of charge (e.g 100%, or 1) at some datetime. Follow up with a float value and a timezone-aware datetime in ISO 6081 format." " Use --soc-unit to set a different unit." " This argument can be given multiple times." - " For example: --soc-target 100 2022-02-23T13:40:52+00:00", + " For example: --soc-target 100% 2022-02-23T13:40:52+00:00", ) @click.option( "--soc-min", "soc_min", - type=NonNegativeFloat(), + type=QuantityField("%", validate=validate.Range(min=0, max=1)), required=False, - help="Minimum state of charge (in %, e.g. 20) for the schedule. Use --soc-unit to set a different unit.", + help="Minimum state of charge (e.g 20%, or 0.2) for the schedule. Use --soc-unit to set a different unit.", ) @click.option( "--soc-max", "soc_max", - type=NonNegativeFloat(), + type=QuantityField("%", validate=validate.Range(min=0, max=100)), required=False, - help="Maximum state of charge (in %, e.g. 80) for the schedule. Use --soc-unit to set a different unit.", + help="Maximum state of charge (e.g 80%, or 0.8) for the schedule. Use --soc-unit to set a different unit.", ) @click.option( "--roundtrip-efficiency", @@ -880,23 +879,16 @@ def create_forecasts( default=1, help="Round-trip efficiency (e.g. 85% or 0.85) to use for the schedule. Defaults to 100% (no losses).", ) -@click.option( - "--soc-unit", - "soc_unit", - default="%", - help="Unit of the passed SoC values, such as 'MWh', 'kWh' or '%' (the default).", -) def create_schedule( power_sensor_id: int, factor_sensor_id: int, start_str: str, duration_str: str, - soc_at_start: float, - soc_target_strings: List[Tuple[float, str]], - soc_min: Optional[float], - soc_max: Optional[float], - roundtrip_efficiency: Optional[float] = 1, - soc_unit: str = "%", + soc_at_start: ur.Quantity, + soc_target_strings: List[Tuple[ur.Quantity, str]], + soc_min: Optional[ur.Quantity] = None, + soc_max: Optional[ur.Quantity] = None, + roundtrip_efficiency: Optional[ur.Quantity] = None, ): """Create a new schedule for a given power sensor. @@ -944,21 +936,23 @@ def create_schedule( ur.Quantity("dimensionless") ).magnitude + # Convert SoC units to MWh, given the storage capacity + capacity_str = f"{power_sensor.get_attribute('max_soc_in_mwh')} MWh" + soc_at_start = convert_units(soc_at_start.magnitude, soc_at_start.units, "MWh", capacity=capacity_str) # type: ignore for soc_target_tuple in soc_target_strings: soc_target_value_str, soc_target_dt_str = soc_target_tuple - soc_target_value = float(soc_target_value_str) + soc_target_value = convert_units( + soc_target_value_str.magnitude, + str(soc_target_value_str.units), + "MWh", + capacity=capacity_str, + ) soc_target_datetime = pd.Timestamp(soc_target_dt_str) soc_targets.loc[soc_target_datetime] = soc_target_value - - # Convert SoC units if needed - if soc_unit != "MWh": - capacity_str = f"{power_sensor.get_attribute('max_soc_in_mwh')} MWh" - soc_at_start = convert_units(soc_at_start, soc_unit, "MWh", capacity=capacity_str) # type: ignore - soc_targets = convert_units(soc_targets, soc_unit, "MWh", capacity=capacity_str) - if soc_min is not None: - soc_min = convert_units(soc_min, soc_unit, "MWh", capacity=capacity_str) # type: ignore - if soc_max is not None: - soc_max = convert_units(soc_max, soc_unit, "MWh", capacity=capacity_str) # type: ignore + if soc_min is not None: + soc_min = convert_units(soc_min.magnitude, str(soc_min.units), "MWh", capacity=capacity_str) # type: ignore + if soc_max is not None: + soc_max = convert_units(soc_max.magnitude, str(soc_max.units), "MWh", capacity=capacity_str) # type: ignore success = make_schedule( sensor_id=power_sensor_id, From abc7ac6af2a8b549b8b85ee407ac1caab2c8036b Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 4 Mar 2022 17:09:02 +0100 Subject: [PATCH 36/41] Rename factor_id to optimization_context_id Signed-off-by: F.N. Claessen --- flexmeasures/cli/data_add.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index f87b43f57..87471ae1e 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -820,8 +820,8 @@ def create_forecasts( help="Create schedule for this sensor. Follow up with the sensor's ID.", ) @click.option( - "--factor-id", - "factor_sensor_id", + "--optimization-context-id", + "optimization_context_sensor_id", required=True, help="Optimize against this sensor, which measures a price factor or CO₂ intensity factor. Follow up with the sensor's ID.", ) @@ -881,7 +881,7 @@ def create_forecasts( ) def create_schedule( power_sensor_id: int, - factor_sensor_id: int, + optimization_context_sensor_id: int, start_str: str, duration_str: str, soc_at_start: ur.Quantity, @@ -907,11 +907,11 @@ def create_schedule( if not power_sensor.measures_power: click.echo(f"Sensor with ID {power_sensor_id} is not a power sensor.") raise click.Abort() - factor_sensor: Sensor = Sensor.query.filter( - Sensor.id == factor_sensor_id + optimization_context_sensor: Sensor = Sensor.query.filter( + Sensor.id == optimization_context_sensor_id ).one_or_none() - if factor_sensor is None: - click.echo(f"No sensor found with ID {factor_sensor_id}.") + if optimization_context_sensor is None: + click.echo(f"No sensor found with ID {optimization_context_sensor_id}.") raise click.Abort() start = pd.Timestamp(start_str) end = start + isodate.parse_duration(duration_str) @@ -965,7 +965,7 @@ def create_schedule( soc_min=soc_min, soc_max=soc_max, roundtrip_efficiency=roundtrip_efficiency, - price_sensor=factor_sensor, + price_sensor=optimization_context_sensor, ) if success: print("New schedule is stored.") From a86ab94bd7fa4ed8e0c0ffa4ccd83ab9f627230c Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 4 Mar 2022 17:10:12 +0100 Subject: [PATCH 37/41] Delete obsolete class Signed-off-by: F.N. Claessen --- flexmeasures/data/schemas/units.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/flexmeasures/data/schemas/units.py b/flexmeasures/data/schemas/units.py index d3d851005..63ea58284 100644 --- a/flexmeasures/data/schemas/units.py +++ b/flexmeasures/data/schemas/units.py @@ -7,14 +7,6 @@ from flexmeasures.utils.unit_utils import is_valid_unit, ur -class NonNegativeFloat(fields.Float, MarshmallowClickMixin): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - # Insert validation into self.validators so that multiple errors can be stored. - validator = validate.Range(min=0) - self.validators.insert(0, validator) - - class Quantity(Validator): """Validator which succeeds if the value passed to it is a valid quantity.""" From a12ecefb29b123de8b95c0d9cd96cd09f539c41c Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 4 Mar 2022 17:11:27 +0100 Subject: [PATCH 38/41] Rename validator class Signed-off-by: F.N. Claessen --- flexmeasures/data/schemas/units.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flexmeasures/data/schemas/units.py b/flexmeasures/data/schemas/units.py index 63ea58284..9161ebbf3 100644 --- a/flexmeasures/data/schemas/units.py +++ b/flexmeasures/data/schemas/units.py @@ -7,7 +7,7 @@ from flexmeasures.utils.unit_utils import is_valid_unit, ur -class Quantity(Validator): +class QuantityValidator(Validator): """Validator which succeeds if the value passed to it is a valid quantity.""" def __init__(self, *, error: Optional[str] = None): @@ -38,7 +38,7 @@ class QuantityField(fields.Str, MarshmallowClickMixin): def __init__(self, to_unit: str, *args, **kwargs): super().__init__(*args, **kwargs) # Insert validation into self.validators so that multiple errors can be stored. - validator = Quantity() + validator = QuantityValidator() self.validators.insert(0, validator) self.to_unit = ur.Quantity(to_unit) From 6d0e9b875169b8a05439d587608654a4ad0ffe03 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 4 Mar 2022 17:15:43 +0100 Subject: [PATCH 39/41] flake8 and mypy Signed-off-by: F.N. Claessen --- flexmeasures/cli/data_add.py | 7 ++++--- flexmeasures/data/schemas/units.py | 3 +-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index 87471ae1e..a075bf0ac 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -932,9 +932,10 @@ def create_schedule( ) # Convert round-trip efficiency to dimensionless - roundtrip_efficiency = roundtrip_efficiency.to( - ur.Quantity("dimensionless") - ).magnitude + if roundtrip_efficiency is not None: + roundtrip_efficiency = roundtrip_efficiency.to( + ur.Quantity("dimensionless") + ).magnitude # Convert SoC units to MWh, given the storage capacity capacity_str = f"{power_sensor.get_attribute('max_soc_in_mwh')} MWh" diff --git a/flexmeasures/data/schemas/units.py b/flexmeasures/data/schemas/units.py index 9161ebbf3..7ede76633 100644 --- a/flexmeasures/data/schemas/units.py +++ b/flexmeasures/data/schemas/units.py @@ -1,13 +1,12 @@ from typing import Optional from marshmallow import fields, validate, ValidationError -from marshmallow.validate import Validator from flexmeasures.data.schemas.utils import MarshmallowClickMixin from flexmeasures.utils.unit_utils import is_valid_unit, ur -class QuantityValidator(Validator): +class QuantityValidator(validate.Validator): """Validator which succeeds if the value passed to it is a valid quantity.""" def __init__(self, *, error: Optional[str] = None): From 56aab7747278338c3b817874f762bb65e19ae0e2 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 4 Mar 2022 21:50:28 +0100 Subject: [PATCH 40/41] Changelog entries Signed-off-by: F.N. Claessen --- documentation/changelog.rst | 5 +++-- documentation/cli/change_log.rst | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 6832fc0e4..8f01062f7 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -12,12 +12,13 @@ New features * Three new CLI commands for cleaning up your database: delete 1) unchanged beliefs, 2) NaN values or 3) a sensor and all of its time series data [see `PR #328 `_] * Add CLI option to pass a data unit when reading in time series data from CSV, so data can automatically be converted to the sensor unit [see `PR #341 `_] * Add CLI option to specify custom strings that should be interpreted as NaN values when reading in time series data from CSV [see `PR #357 `_] -* Add CLI commands ``flexmeasures add sensor``, ``flexmeasures add asset-type``, ``flexmeasures add beliefs`` (which were experimental features before). [see `PR #337 `_] +* Add CLI commands ``flexmeasures add sensor``, ``flexmeasures add asset-type``, ``flexmeasures add beliefs`` (which were experimental features before) [see `PR #337 `_] * Add CLI commands for showing data [see `PR #339 `_] * Add CLI command for attaching annotations to assets: ``flexmeasures add holidays`` adds public holidays [see `PR #343 `_] * Add CLI command for resampling existing sensor data to new resolution [see `PR #360 `_] -* Add CLI command to add a toy account for tutorials and trying things [see `PR #368 `_]. +* Add CLI command to add a toy account for tutorials and trying things [see `PR #368 `_] +* Add CLI command to create a charging schedule [see `PR #372 `_] * Support for percent (%) and permille (‰) sensor units [see `PR #359 `_] Bugfixes diff --git a/documentation/cli/change_log.rst b/documentation/cli/change_log.rst index 2be1bd5ac..94a9f860f 100644 --- a/documentation/cli/change_log.rst +++ b/documentation/cli/change_log.rst @@ -10,6 +10,7 @@ since v0.9.0 | January 26, 2022 * Add CLI commands for showing data ``flexmeasures show accounts``, ``flexmeasures show account``, ``flexmeasures show roles``, ``flexmeasures show asset-types``, ``flexmeasures show asset`` and ``flexmeasures show data-sources``. * Add ``flexmeasures db-ops resample-data`` CLI command to resample sensor data to a different resolution. * Add ``flexmeasures add toy-account`` for tutorials and trying things. +* Add ``flexmeasures add schedule`` to create a new schedule for a given power sensor. * Rename ``flexmeasures add structure`` to ``flexmeasures add initial-structure``. From 96c23645f848c77c9f41da92da8c3be4ec3fae27 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 4 Mar 2022 21:54:37 +0100 Subject: [PATCH 41/41] Update CLI commands listing Signed-off-by: F.N. Claessen --- documentation/cli/commands.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/documentation/cli/commands.rst b/documentation/cli/commands.rst index 3d02fb315..556e71570 100644 --- a/documentation/cli/commands.rst +++ b/documentation/cli/commands.rst @@ -34,6 +34,7 @@ of which some are referred to in this documentation. ``flexmeasures add external-weather-forecasts`` Collect weather forecasts from the DarkSky API. ``flexmeasures add beliefs`` Load beliefs from file. ``flexmeasures add forecasts`` Create forecasts. +``flexmeasures add schedule`` Create a charging schedule. ``flexmeasures add toy-account`` Create a toy account, for tutorials and trying things. ================================================= =======================================