From 31afb2ea88d0788afa6ddb5532036df42117e709 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 15 Nov 2022 14:16:54 +0100 Subject: [PATCH 01/73] fix charging sensor name in tutorial Signed-off-by: F.N. Claessen --- documentation/tut/toy-example-from-scratch.rst | 6 +++--- flexmeasures/cli/data_add.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/documentation/tut/toy-example-from-scratch.rst b/documentation/tut/toy-example-from-scratch.rst index bc656d214..52ea1942c 100644 --- a/documentation/tut/toy-example-from-scratch.rst +++ b/documentation/tut/toy-example-from-scratch.rst @@ -108,7 +108,7 @@ FlexMeasures offers a command to create a toy account with a battery: $ flexmeasures add toy-account --kind battery Toy account Toy Account with user toy-user@flexmeasures.io created successfully. You might want to run `flexmeasures show account --id 1` - The sensor for battery charging is . + The sensor for battery (dis)charging is . The sensor for Day ahead prices is . And with that, we're done with the structural data for this tutorial! @@ -278,7 +278,7 @@ Great. Let's see what we made: .. code-block:: console $ flexmeasures show beliefs --sensor-id 2 --start ${TOMORROW}T07:00:00+01:00 --duration PT12H - Beliefs for Sensor 'charging' (Id 2). + Beliefs for Sensor 'discharging' (Id 2). Data spans 12 hours and starts at 2022-03-04 07:00:00+01:00. The time resolution (x-axis) is 15 minutes. ┌────────────────────────────────────────────────────────────┐ @@ -301,7 +301,7 @@ Great. Let's see what we made: │ ▙▄▄▌ ▐▄▄▞ │ └────────────────────────────────────────────────────────────┘ 10 20 30 40 - ██ charging + ██ discharging Here, negative values denote output from the grid, so that's when the battery gets charged. diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index 15d00dfe5..d044f4218 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -1110,7 +1110,7 @@ def add_toy_account(kind: str, name: str): click.echo( f"Toy account {name} with user {user.email} created successfully. You might want to run `flexmeasures show account --id {user.account.id}`" ) - click.echo(f"The sensor for battery charging is {charging_sensor}.") + click.echo(f"The sensor for battery discharging is {charging_sensor}.") click.echo(f"The sensor for Day ahead prices is {day_ahead_sensor}.") From e60f97dce8af011453f35d63da8e330df0d025b1 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 15 Nov 2022 14:18:38 +0100 Subject: [PATCH 02/73] update price sensor name in tutorial (prefer non-capitalized sensor names) Signed-off-by: F.N. Claessen --- documentation/tut/toy-example-from-scratch.rst | 8 ++++---- flexmeasures/cli/data_add.py | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/documentation/tut/toy-example-from-scratch.rst b/documentation/tut/toy-example-from-scratch.rst index 52ea1942c..f4f824852 100644 --- a/documentation/tut/toy-example-from-scratch.rst +++ b/documentation/tut/toy-example-from-scratch.rst @@ -108,8 +108,8 @@ FlexMeasures offers a command to create a toy account with a battery: $ flexmeasures add toy-account --kind battery Toy account Toy Account with user toy-user@flexmeasures.io created successfully. You might want to run `flexmeasures show account --id 1` - The sensor for battery (dis)charging is . - The sensor for Day ahead prices is . + The sensor recording battery power is . + The sensor recording day-ahead prices is . And with that, we're done with the structural data for this tutorial! @@ -223,7 +223,7 @@ Let's look at the price data we just loaded: .. code-block:: console $ flexmeasures show beliefs --sensor-id 3 --start ${TOMORROW}T01:00:00+01:00 --duration PT24H - Beliefs for Sensor 'Day ahead prices' (Id 3). + Beliefs for Sensor 'day-ahead prices' (Id 3). Data spans a day and starts at 2022-03-03 01:00:00+01:00. The time resolution (x-axis) is an hour. ┌────────────────────────────────────────────────────────────┐ @@ -246,7 +246,7 @@ Let's look at the price data we just loaded: │ ▝▚▄▄▄▄▘ │ └────────────────────────────────────────────────────────────┘ 5 10 15 20 - ██ Day ahead prices + ██ day-ahead prices diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index d044f4218..fe9e30f33 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -1089,11 +1089,11 @@ def add_toy_account(kind: str, name: str): # add public day-ahead market (as sensor of transmission zone asset) nl_zone = add_transmission_zone_asset("NL", db=db) day_ahead_sensor = Sensor.query.filter( - Sensor.generic_asset == nl_zone, Sensor.name == "Day ahead prices" + Sensor.generic_asset == nl_zone, Sensor.name == "day-ahead prices" ).one_or_none() if not day_ahead_sensor: day_ahead_sensor = Sensor( - name="Day ahead prices", + name="day-ahead prices", generic_asset=nl_zone, unit="EUR/MWh", timezone="Europe/Amsterdam", @@ -1110,8 +1110,8 @@ def add_toy_account(kind: str, name: str): click.echo( f"Toy account {name} with user {user.email} created successfully. You might want to run `flexmeasures show account --id {user.account.id}`" ) - click.echo(f"The sensor for battery discharging is {charging_sensor}.") - click.echo(f"The sensor for Day ahead prices is {day_ahead_sensor}.") + click.echo(f"The sensor recording battery discharging is {charging_sensor}.") + click.echo(f"The sensor recording day-ahead prices is {day_ahead_sensor}.") app.cli.add_command(fm_add_data) From 625bc2b77e7bf1a4bd5a11b6420282e78057cf91 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 15 Nov 2022 14:22:41 +0100 Subject: [PATCH 03/73] Print out account ID if toy account already exists Signed-off-by: F.N. Claessen --- flexmeasures/cli/data_add.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index fe9e30f33..be72be289 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -1045,7 +1045,7 @@ def add_toy_account(kind: str, name: str): # make an account (if not exist) account = Account.query.filter(Account.name == name).one_or_none() if account: - click.echo(f"Account {name} already exists.") + click.echo(f"Account already exists: {account}") return # make an account user (account-admin?) email = "toy-user@flexmeasures.io" From fd1938147f6ef50eecf6791faee5c95f9d2ea733 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 15 Nov 2022 14:35:14 +0100 Subject: [PATCH 04/73] Fix for changed default when reading in timezone naive data: we no longer assume utc, but raise instead (see PR #521) Signed-off-by: F.N. Claessen --- .../tut/toy-example-from-scratch.rst | 80 +++++++++---------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/documentation/tut/toy-example-from-scratch.rst b/documentation/tut/toy-example-from-scratch.rst index f4f824852..1e777cbce 100644 --- a/documentation/tut/toy-example-from-scratch.rst +++ b/documentation/tut/toy-example-from-scratch.rst @@ -211,42 +211,42 @@ This is time series data, in FlexMeasures we call "beliefs". Beliefs can also be .. code-block:: console - $ flexmeasures add beliefs --sensor-id 3 --source toy-user prices-tomorrow.csv + $ flexmeasures add beliefs --sensor-id 3 --source toy-user prices-tomorrow.csv --timezone Europe/Amsterdam Successfully created beliefs In FlexMeasures, all beliefs have a data source. Here, we use the username of the user we created earlier. We could also pass a user ID, or the name of a new data source we want to use for CLI scripts. -.. note:: Attention: We created and imported prices where the times have no time zone component! That happens a lot. FlexMeasures will then interpret them as UTC time. So if you are in Amsterdam time, the start time for the first price, when expressed in your time zone, is actually `2022-03-03 01:00:00+01:00`. +.. note:: Attention: We created and imported prices where the times have no time zone component! That happens a lot. FlexMeasures can localize them for you to a given timezone. Here, we localized the data to the timezone of the price sensor - ``Europe/Amsterdam`` - so the start time for the first price is `2022-03-03 00:00:00+01:00` (midnight in Amsterdam). Let's look at the price data we just loaded: .. code-block:: console - $ flexmeasures show beliefs --sensor-id 3 --start ${TOMORROW}T01:00:00+01:00 --duration PT24H + $ flexmeasures show beliefs --sensor-id 3 --start ${TOMORROW}T00:00:00+01:00 --duration PT24H Beliefs for Sensor 'day-ahead prices' (Id 3). - Data spans a day and starts at 2022-03-03 01:00:00+01:00. + Data spans a day and starts at 2022-03-03 00:00:00+01:00. The time resolution (x-axis) is an hour. ┌────────────────────────────────────────────────────────────┐ │ ▗▀▚▖ │ 18EUR/MWh - │ ▞ ▝▌ │ - │ ▐ ▚ │ - │ ▗▘ ▐ │ - │ ▌ ▌ ▖ │ - │ ▞ ▚ ▗▄▀▝▄ │ + │ ▞ ▝▌ │ + │ ▐ ▚ │ + │ ▗▘ ▐ │ + │ ▌ ▌ ▖ │ + │ ▞ ▚ ▗▄▀▝▄ │ │ ▗▘ ▐ ▗▞▀ ▚ │ 13EUR/MWh - │ ▗▄▘ ▌ ▐▘ ▚ │ - │ ▗▞▘ ▚ ▌ ▚ │ - │▞▘ ▝▄ ▗ ▐ ▝▖ │ - │ ▚▄▄▀▚▄▄ ▞▘▚ ▌ ▝▖ │ - │ ▀▀▛ ▚ ▐ ▚ │ + │ ▗▄▘ ▌ ▐▘ ▚ │ + │ ▗▞▘ ▚ ▌ ▚ │ + │▞▘ ▝▄ ▗ ▐ ▝▖ │ + │ ▚▄▄▀▚▄▄ ▞▘▚ ▌ ▝▖ │ + │ ▀▀▛ ▚ ▐ ▚ │ │ ▚ ▗▘ ▚│ 8EUR/MWh - │ ▌ ▗▘ ▝│ - │ ▝▖ ▞ │ - │ ▐▖ ▗▀ │ - │ ▝▚▄▄▄▄▘ │ + │ ▌ ▗▘ ▝│ + │ ▝▖ ▞ │ + │ ▐▖ ▗▀ │ + │ ▝▚▄▄▄▄▘ │ └────────────────────────────────────────────────────────────┘ - 5 10 15 20 - ██ day-ahead prices + 5 10 15 20 + ██ day-ahead prices @@ -282,26 +282,26 @@ Great. Let's see what we made: Data spans 12 hours and starts at 2022-03-04 07:00:00+01:00. The time resolution (x-axis) is 15 minutes. ┌────────────────────────────────────────────────────────────┐ - │ ▐ ▐▀▀▌ ▛▀▀│ - │ ▞▌ ▞ ▐ ▌ │ 0.4MW - │ ▌▌ ▌ ▐ ▐ │ - │ ▗▘▌ ▌ ▐ ▐ │ - │ ▐ ▐ ▗▘ ▝▖ ▐ │ - │ ▞ ▐ ▐ ▌ ▌ │ 0.2MW - │ ▗▘ ▐ ▐ ▌ ▌ │ - │ ▐ ▝▖ ▌ ▚ ▞ │ - │▀▘───▀▀▀▀▀▀▀▀▀▀▀▀▀▀▌────▐─────▝▀▀▀▀▀▀▀▀▜─────▐▀▀▀▀▀▀▀▀▀─────│ 0MW - │ ▌ ▞ ▐ ▗▘ │ - │ ▚ ▌ ▐ ▐ │ - │ ▐ ▗▘ ▝▖ ▌ │ -0.2MW - │ ▐ ▐ ▌ ▌ │ - │ ▐ ▐ ▌ ▗▘ │ - │ ▌ ▞ ▌ ▐ │ - │ ▌ ▌ ▐ ▐ │ -0.4MW - │ ▙▄▄▌ ▐▄▄▞ │ + │ ▐▌ ▐▀▀▌ ▛▀▀│ + │ ▐▌ ▞ ▚ ▌ │ 0.4MW + │ ▌▌ ▌ ▐ ▗▘ │ + │ ▌▚ ▌ ▐ ▐ │ + │ ▗▘▐ ▗▘ ▐ ▐ │ + │ ▐ ▐ ▐ ▌ ▞ │ 0.2MW + │ ▌ ▐ ▐ ▌ ▌ │ + │ ▗▘ ▌ ▐ ▌ ▌ │ + │▀▀▀▀▀▀▀───▀▀▀▀▌─────▌────▝▀▀▀▀▀▀▀▀▌─────▐▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▘───│ 0MW + │ ▌ ▗▘ ▐ ▞ │ + │ ▌ ▐ ▐ ▗▘ │ + │ ▚ ▌ ▐ ▐ │ -0.2MW + │ ▐ ▗▘ ▌ ▌ │ + │ ▐ ▐ ▌ ▌ │ + │ ▝▖ ▞ ▌ ▐ │ + │ ▌ ▌ ▚ ▐ │ -0.4MW + │ ▙▄▄▘ ▐▄▄▌ │ └────────────────────────────────────────────────────────────┘ - 10 20 30 40 - ██ discharging + 10 20 30 40 + ██ discharging Here, negative values denote output from the grid, so that's when the battery gets charged. @@ -311,7 +311,7 @@ We can also look at the charging schedule in the `FlexMeasures UI Date: Tue, 15 Nov 2022 15:08:39 +0100 Subject: [PATCH 05/73] Add solar power sensor to toy account, and allow existing toy accounts to be updated by simply running `flexmeasures add toy-account` again Signed-off-by: F.N. Claessen --- flexmeasures/cli/data_add.py | 49 ++++++++++++++++++++++++++++-------- 1 file changed, 39 insertions(+), 10 deletions(-) diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index be72be289..19e94a55f 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -2,7 +2,7 @@ from __future__ import annotations from datetime import datetime, timedelta -from typing import Dict, List, Optional, Tuple +from typing import Dict, List, Optional, Tuple, Type import json from marshmallow import validate @@ -1046,7 +1046,6 @@ def add_toy_account(kind: str, name: str): account = Account.query.filter(Account.name == name).one_or_none() if account: click.echo(f"Account already exists: {account}") - return # make an account user (account-admin?) email = "toy-user@flexmeasures.io" user = User.query.filter_by(email=email).one_or_none() @@ -1064,27 +1063,37 @@ def add_toy_account(kind: str, name: str): ) # make assets for asset_type in ("solar", "building", "battery"): - asset = GenericAsset( + asset = get_or_create_model( + GenericAsset, name=f"toy-{asset_type}", generic_asset_type=asset_types[asset_type], owner=user.account, latitude=location[0], longitude=location[1], ) - db.session.add(asset) + power_sensor_specs = dict( + generic_asset=asset, + unit="MW", + timezone="Europe/Amsterdam", + event_resolution=timedelta(minutes=15), + ) if asset_type == "battery": asset.attributes = dict( capacity_in_mw=0.5, min_soc_in_mwh=0.05, max_soc_in_mwh=0.45 ) # add charging sensor to battery - charging_sensor = Sensor( + charging_sensor = get_or_create_model( + Sensor, name="discharging", - generic_asset=asset, - unit="MW", - timezone="Europe/Amsterdam", - event_resolution=timedelta(minutes=15), + **power_sensor_specs, + ) + elif asset_type == "solar": + # add production sensor to solar asset + production_sensor = get_or_create_model( + Sensor, + name="production", + **power_sensor_specs, ) - db.session.add(charging_sensor) # add public day-ahead market (as sensor of transmission zone asset) nl_zone = add_transmission_zone_asset("NL", db=db) @@ -1142,3 +1151,23 @@ def parse_source(source): else: _source = get_or_create_source(source, source_type="CLI script") return _source + + +def get_or_create_model( + model_class: Type[GenericAsset | GenericAssetType | Sensor], **kwargs +) -> GenericAsset | GenericAssetType | Sensor: + """Get a model from the database or add it if it's missing. + + For example: + >>> weather_station_type = get_or_create_model( + >>> GenericAssetType, + >>> name="weather station", + >>> description="A weather station with various sensors.", + >>> ) + """ + model = model_class.query.filter_by(**kwargs).one_or_none() + if model is None: + model = model_class(**kwargs) + click.echo(f"Created {model}") + db.session.add(model) + return model From 4220d922366e64f2fd73333162482978cc8a530b Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 15 Nov 2022 16:06:26 +0100 Subject: [PATCH 06/73] Support JSON and Callable arguments Signed-off-by: F.N. Claessen --- flexmeasures/cli/data_add.py | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index 19e94a55f..8d383c89d 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -13,6 +13,7 @@ from flask.cli import with_appcontext import click import getpass +from sqlalchemy import cast, literal, JSON, String from sqlalchemy.exc import IntegrityError from timely_beliefs.sensors.func_store.knowledge_horizons import x_days_ago_at_y_oclock import timely_beliefs as tb @@ -1165,9 +1166,39 @@ def get_or_create_model( >>> description="A weather station with various sensors.", >>> ) """ - model = model_class.query.filter_by(**kwargs).one_or_none() + + # unpack custom initialization parameters that map to multiple database columns + init_kwargs = kwargs.copy() + lookup_kwargs = kwargs.copy() + if "knowledge_horizon" in kwargs: + ( + lookup_kwargs["knowledge_horizon_fnc"], + lookup_kwargs["knowledge_horizon_par"], + ) = lookup_kwargs.pop("knowledge_horizon") + + # Find out which attributes are dictionaries mapped to JSON database columns, + # or callables mapped to string database columns (by their name) + filter_json_kwargs = {} + filter_by_kwargs = lookup_kwargs.copy() + for kw, arg in lookup_kwargs.items(): + model_attribute = getattr(model_class, kw) + print(f"argument {arg} is {callable(arg)} callable") + if hasattr(model_attribute, "type") and isinstance(model_attribute.type, JSON): + filter_json_kwargs[kw] = filter_by_kwargs.pop(kw) + elif callable(arg) and isinstance(model_attribute.type, String): + filter_by_kwargs[kw] = filter_by_kwargs[kw].__name__ + + # See if the model already exists as a db row + model_query = model_class.query.filter_by(**filter_by_kwargs) + for kw, arg in filter_json_kwargs.items(): + model_query = model_query.filter( + cast(getattr(model_class, kw), String) == cast(literal(arg, JSON()), String) + ) + model = model_query.one_or_none() + + # Create the model and add it to the database if it didn't already exist if model is None: - model = model_class(**kwargs) + model = model_class(**init_kwargs) click.echo(f"Created {model}") db.session.add(model) return model From 07f8dc8b52d36e7118d14077a606d71af6b8b2fa Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 15 Nov 2022 16:07:03 +0100 Subject: [PATCH 07/73] get or create price sensor Signed-off-by: F.N. Claessen --- flexmeasures/cli/data_add.py | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index 8d383c89d..2b530f833 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -1098,22 +1098,18 @@ def add_toy_account(kind: str, name: str): # add public day-ahead market (as sensor of transmission zone asset) nl_zone = add_transmission_zone_asset("NL", db=db) - day_ahead_sensor = Sensor.query.filter( - Sensor.generic_asset == nl_zone, Sensor.name == "day-ahead prices" - ).one_or_none() - if not day_ahead_sensor: - day_ahead_sensor = Sensor( - name="day-ahead prices", - generic_asset=nl_zone, - unit="EUR/MWh", - timezone="Europe/Amsterdam", - event_resolution=timedelta(minutes=60), - knowledge_horizon=( - x_days_ago_at_y_oclock, - {"x": 1, "y": 12, "z": "Europe/Paris"}, - ), - ) - db.session.add(day_ahead_sensor) + day_ahead_sensor = get_or_create_model( + Sensor, + name="day-ahead prices", + generic_asset=nl_zone, + unit="EUR/MWh", + timezone="Europe/Amsterdam", + event_resolution=timedelta(minutes=60), + knowledge_horizon=( + x_days_ago_at_y_oclock, + {"x": 1, "y": 12, "z": "Europe/Paris"}, + ), + ) db.session.commit() From bd977b2fd1422fd106f53d7d306e41535a9f2f58 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 15 Nov 2022 16:18:50 +0100 Subject: [PATCH 08/73] Refactor to keep original order of sensor ids Signed-off-by: F.N. Claessen --- .../tut/toy-example-from-scratch.rst | 1 + flexmeasures/cli/data_add.py | 46 +++++++++++-------- 2 files changed, 27 insertions(+), 20 deletions(-) diff --git a/documentation/tut/toy-example-from-scratch.rst b/documentation/tut/toy-example-from-scratch.rst index 1e777cbce..6da8dcc21 100644 --- a/documentation/tut/toy-example-from-scratch.rst +++ b/documentation/tut/toy-example-from-scratch.rst @@ -110,6 +110,7 @@ FlexMeasures offers a command to create a toy account with a battery: Toy account Toy Account with user toy-user@flexmeasures.io created successfully. You might want to run `flexmeasures show account --id 1` The sensor recording battery power is . The sensor recording day-ahead prices is . + The sensor recording solar forecasts is . And with that, we're done with the structural data for this tutorial! diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index 2b530f833..409b40163 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -1062,8 +1062,8 @@ def add_toy_account(kind: str, name: str): user_roles=["account-admin"], account_name=name, ) - # make assets - for asset_type in ("solar", "building", "battery"): + + def create_power_asset(asset_type: str, sensor_name: str, **attributes): asset = get_or_create_model( GenericAsset, name=f"toy-{asset_type}", @@ -1072,29 +1072,28 @@ def add_toy_account(kind: str, name: str): latitude=location[0], longitude=location[1], ) + asset.attributes = attributes power_sensor_specs = dict( generic_asset=asset, unit="MW", timezone="Europe/Amsterdam", event_resolution=timedelta(minutes=15), ) - if asset_type == "battery": - asset.attributes = dict( - capacity_in_mw=0.5, min_soc_in_mwh=0.05, max_soc_in_mwh=0.45 - ) - # add charging sensor to battery - charging_sensor = get_or_create_model( - Sensor, - name="discharging", - **power_sensor_specs, - ) - elif asset_type == "solar": - # add production sensor to solar asset - production_sensor = get_or_create_model( - Sensor, - name="production", - **power_sensor_specs, - ) + power_sensor = get_or_create_model( + Sensor, + name=sensor_name, + **power_sensor_specs, + ) + return power_sensor + + # create battery + discharging_sensor = create_power_asset( + "battery", + "discharging", + capacity_in_mw=0.5, + min_soc_in_mwh=0.05, + max_soc_in_mwh=0.45, + ) # add public day-ahead market (as sensor of transmission zone asset) nl_zone = add_transmission_zone_asset("NL", db=db) @@ -1111,13 +1110,20 @@ def add_toy_account(kind: str, name: str): ), ) + # create solar + production_sensor = create_power_asset( + "solar", + "production", + ) + db.session.commit() click.echo( f"Toy account {name} with user {user.email} created successfully. You might want to run `flexmeasures show account --id {user.account.id}`" ) - click.echo(f"The sensor recording battery discharging is {charging_sensor}.") + click.echo(f"The sensor recording battery discharging is {discharging_sensor}.") click.echo(f"The sensor recording day-ahead prices is {day_ahead_sensor}.") + click.echo(f"The sensor recording solar forecasts is {production_sensor}.") app.cli.add_command(fm_add_data) From be49c470af5705e524b6b9471ea2aa252a16927b Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 15 Nov 2022 17:05:05 +0100 Subject: [PATCH 09/73] Add CLI option to pass inflexible device sensors Signed-off-by: F.N. Claessen --- flexmeasures/cli/data_add.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index 409b40163..ec48209ba 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -853,6 +853,14 @@ def create_forecasts( required=False, help="To be deprecated. Use consumption-price-sensor instead.", ) +@click.option( + "--inflexible-device-sensor", + "inflexible_device_sensors", + type=SensorIdField(), + multiple=True, + help="Take into account the power flow of inflexible devices. Follow up with the sensor's ID." + " This argument can be given multiple times." +) @click.option( "--start", "start", @@ -919,6 +927,7 @@ def create_schedule( consumption_price_sensor: Sensor, production_price_sensor: Sensor, optimization_context_sensor: Sensor, + inflexible_device_sensors: list[Sensor], start: datetime, duration: timedelta, soc_at_start: ur.Quantity, @@ -1005,6 +1014,7 @@ def create_schedule( roundtrip_efficiency=roundtrip_efficiency, consumption_price_sensor=consumption_price_sensor, production_price_sensor=production_price_sensor, + inflexible_device_sensors=inflexible_device_sensors, ) if job: print(f"New scheduling job {job.id} has been added to the queue.") @@ -1022,6 +1032,7 @@ def create_schedule( roundtrip_efficiency=roundtrip_efficiency, consumption_price_sensor=consumption_price_sensor, production_price_sensor=production_price_sensor, + inflexible_device_sensors=inflexible_device_sensors, ) if success: print("New schedule is stored.") From 9320a6d6782c75c886b95c9d75b95a00a67032fb Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 15 Nov 2022 17:05:52 +0100 Subject: [PATCH 10/73] Expand tutorial with scheduling against solar, which limits the available headroom for the battery Signed-off-by: F.N. Claessen --- .../tut/toy-example-from-scratch.rst | 55 ++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/documentation/tut/toy-example-from-scratch.rst b/documentation/tut/toy-example-from-scratch.rst index 6da8dcc21..309cf4261 100644 --- a/documentation/tut/toy-example-from-scratch.rst +++ b/documentation/tut/toy-example-from-scratch.rst @@ -315,4 +315,57 @@ We can also look at the charging schedule in the `FlexMeasures UI solar-tomorrow.csv + +Then, we read in the created CSV file as beliefs data: + +.. code-block:: console + + $ flexmeasures add beliefs --sensor-id 4 --source toy-user solar-tomorrow.csv --timezone Europe/Amsterdam + Successfully created beliefs + +Now, we'll reschedule the battery while taking into account the solar production. This will have an effect on the available headroom for the battery. + +.. code-block:: console + + $ flexmeasures add schedule --sensor-id 2 --consumption-price-sensor 3 \ + --inflexible-device-sensor 4 \ + --start ${TOMORROW}T07:00+01:00 --duration PT12H \ + --soc-at-start 50% --roundtrip-efficiency 90% + New schedule is stored. + From 639dd25af06924ab3da632b4cb2cd373cdab66d2 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 15 Nov 2022 17:10:37 +0100 Subject: [PATCH 11/73] Add note about resampling Signed-off-by: F.N. Claessen --- documentation/tut/toy-example-from-scratch.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/documentation/tut/toy-example-from-scratch.rst b/documentation/tut/toy-example-from-scratch.rst index 309cf4261..468a35e42 100644 --- a/documentation/tut/toy-example-from-scratch.rst +++ b/documentation/tut/toy-example-from-scratch.rst @@ -359,6 +359,10 @@ Then, we read in the created CSV file as beliefs data: $ flexmeasures add beliefs --sensor-id 4 --source toy-user solar-tomorrow.csv --timezone Europe/Amsterdam Successfully created beliefs +Notice that, by default, the one-hour CSV data is automatically resampled to the 15-minute resolution of the sensor that is recording solar production. + +.. note:: The ``flexmeasures add beliefs`` command has many options to make sure the read-in data is correctly interpreted (unit, timezone, delimiter, etc). But that is not the point of this tutorial. See ``flexmeasures add beliefs --help``. + Now, we'll reschedule the battery while taking into account the solar production. This will have an effect on the available headroom for the battery. .. code-block:: console From e48e055d1ca4a84dffd898153657ddc8a9bda2f3 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 15 Nov 2022 17:23:55 +0100 Subject: [PATCH 12/73] black Signed-off-by: F.N. Claessen --- flexmeasures/cli/data_add.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index ec48209ba..a990b9b44 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -859,7 +859,7 @@ def create_forecasts( type=SensorIdField(), multiple=True, help="Take into account the power flow of inflexible devices. Follow up with the sensor's ID." - " This argument can be given multiple times." + " This argument can be given multiple times.", ) @click.option( "--start", From ba4ca7021b422dd5d8f25127f17ca9840cd6900f Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 30 Nov 2022 17:48:22 +0100 Subject: [PATCH 13/73] Allow nesting sensors_to_show to support layering multiple sensors in a single row within the multi-row asset chart Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/assets.py | 3 +- .../data/models/charts/belief_charts.py | 364 ++++++++++++------ flexmeasures/data/models/charts/defaults.py | 16 + flexmeasures/data/models/data_sources.py | 7 +- flexmeasures/data/models/generic_assets.py | 39 +- flexmeasures/utils/coding_utils.py | 20 + flexmeasures/utils/flexmeasures_inflection.py | 6 +- 7 files changed, 321 insertions(+), 134 deletions(-) diff --git a/flexmeasures/api/v3_0/assets.py b/flexmeasures/api/v3_0/assets.py index 919f1f4c3..5f6ff730e 100644 --- a/flexmeasures/api/v3_0/assets.py +++ b/flexmeasures/api/v3_0/assets.py @@ -15,6 +15,7 @@ from flexmeasures.data.schemas.generic_assets import GenericAssetSchema as AssetSchema from flexmeasures.api.common.schemas.generic_assets import AssetIdField from flexmeasures.api.common.schemas.users import AccountIdField +from flexmeasures.utils.coding_utils import flatten_unique from flexmeasures.ui.utils.view_utils import set_time_range_for_session @@ -309,5 +310,5 @@ def get_chart_data(self, id: int, asset: GenericAsset, **kwargs): Data for use in charts (in case you have the chart specs already). """ - sensors = asset.sensors_to_show + sensors = flatten_unique(asset.sensors_to_show) return asset.search_beliefs(sensors=sensors, as_json=True, **kwargs) diff --git a/flexmeasures/data/models/charts/belief_charts.py b/flexmeasures/data/models/charts/belief_charts.py index 3d064f9bd..52cec8634 100644 --- a/flexmeasures/data/models/charts/belief_charts.py +++ b/flexmeasures/data/models/charts/belief_charts.py @@ -3,7 +3,11 @@ from datetime import datetime, timedelta from flexmeasures.data.models.charts.defaults import FIELD_DEFINITIONS -from flexmeasures.utils.flexmeasures_inflection import capitalize +from flexmeasures.utils.flexmeasures_inflection import ( + capitalize, + join_words_into_a_list, +) +from flexmeasures.utils.coding_utils import flatten_unique def bar_chart( @@ -66,12 +70,15 @@ def bar_chart( def chart_for_multiple_sensors( - sensors: list["Sensor"], # noqa F821 + sensors_to_show: list["Sensor", list["Sensor"]], # noqa F821 event_starts_after: datetime | None = None, event_ends_before: datetime | None = None, **override_chart_specs: dict, ): - sensors_specs = [] + # Unpack nested sensors + sensors = flatten_unique(sensors_to_show) + + # Determine the shared data resolution condition = list( sensor.event_resolution for sensor in sensors @@ -80,24 +87,49 @@ def chart_for_multiple_sensors( minimum_non_zero_resolution_in_ms = ( min(condition).total_seconds() * 1000 if any(condition) else 0 ) - for sensor in sensors: - unit = sensor.unit if sensor.unit else "a.u." + + # Set up field definition for event starts + event_start_field_definition = FIELD_DEFINITIONS["event_start"] + if event_starts_after and event_ends_before: + event_start_field_definition["scale"] = { + "domain": [ + event_starts_after.timestamp() * 10**3, + event_ends_before.timestamp() * 10**3, + ] + } + + sensors_specs = [] + for s in sensors_to_show: + # List the sensors that go into one row + if isinstance(s, list): + row_sensors: list["Sensor"] = s + else: + row_sensors: list["Sensor"] = [s] + + # Derive the unit that should be shown + unit = determine_shared_unit(row_sensors) + sensor_type = determine_shared_sensor_type(row_sensors) + + # Set up field definition for event values event_value_field_definition = dict( - title=f"{capitalize(sensor.sensor_type)} ({unit})", + title=f"{capitalize(sensor_type)} ({unit})", format=[".3~r", unit], formatType="quantityWithUnitFormat", stack=None, **FIELD_DEFINITIONS["event_value"], ) - event_start_field_definition = FIELD_DEFINITIONS["event_start"] - if event_starts_after and event_ends_before: - event_start_field_definition["scale"] = { - "domain": [ - event_starts_after.timestamp() * 10**3, - event_ends_before.timestamp() * 10**3, - ] - } + + # Set up shared tooltip shared_tooltip = [ + dict( + field="sensor.name", + type="nominal", + title="Sensor", + ), + { + **event_value_field_definition, + **dict(title=f"{capitalize(sensor_type)}"), + }, FIELD_DEFINITIONS["full_date"], dict( field="belief_horizon", @@ -106,123 +138,64 @@ def chart_for_multiple_sensors( format=["d", 4], formatType="timedeltaFormat", ), - { - **event_value_field_definition, - **dict(title=f"{capitalize(sensor.sensor_type)}"), - }, FIELD_DEFINITIONS["source_name"], + FIELD_DEFINITIONS["source_type"], FIELD_DEFINITIONS["source_model"], ] - line_layer = { - "mark": { - "type": "line", - "interpolate": "step-after" - if sensor.event_resolution != timedelta(0) - else "linear", - "clip": True, - }, - "encoding": { - "x": event_start_field_definition, - "y": event_value_field_definition, - "color": FIELD_DEFINITIONS["source_name"], - "strokeDash": { - "field": "belief_horizon", - "type": "quantitative", - "bin": { - # Divide belief horizons into 2 bins by setting a very large bin size. - # The bins should be defined as follows: ex ante (>0) and ex post (<=0), - # but because the bin anchor is included in the ex-ante bin, - # and 0 belief horizons should be attributed to the ex-post bin, - # (and belief horizons are given with 1 ms precision,) - # the bin anchor is set at 1 ms before knowledge time to obtain: ex ante (>=1) and ex post (<1). - "anchor": 1, - "step": 8640000000000000, # JS max ms for a Date object (NB 10 times less than Python max ms) - # "step": timedelta.max.total_seconds() * 10**2, - }, - "legend": { - # Belief horizons binned as 1 ms contain ex-ante beliefs; the other bin contains ex-post beliefs - "labelExpr": "datum.label > 0 ? 'ex ante' : 'ex post'", - "title": "Recorded", - }, - "scale": { - # Positive belief horizons are clamped to 1, negative belief horizons are clamped to 0 - "domain": [1, 0], - # belief horizons >= 1 ms get a dashed line, belief horizons < 1 ms get a solid line - "range": [[1, 2], [1, 0]], - }, - }, - "detail": FIELD_DEFINITIONS["source"], - }, - } + + # Draw a line for each sensor (and each source) + layers = [ + create_line_layer( + row_sensors, event_start_field_definition, event_value_field_definition + ) + ] + + # Optionally, draw transparent full-height rectangles that activate the tooltip anywhere in the graph + # (to be precise, only at points on the x-axis where there is data) + if len(row_sensors) == 1: + # With multiple sensors, we cannot do this, because it is ambiguous which tooltip to activate (instead, we use a different brush in the circle layer) + layers.append( + create_rect_layer( + event_start_field_definition, + event_value_field_definition, + minimum_non_zero_resolution_in_ms, + shared_tooltip, + ) + ) + + # Draw circle markers that are shown on hover + layers.append( + create_circle_layer( + row_sensors, + event_start_field_definition, + event_value_field_definition, + shared_tooltip, + ) + ) + + # Layer the lines, rectangles and circles within one row, and filter by which sensors are represented in the row sensor_specs = { - "title": capitalize(sensor.name) - if sensor.name != sensor.sensor_type - else None, - "transform": [{"filter": f"datum.sensor.id == {sensor.id}"}], - "layer": [ - line_layer, - { - "mark": { - "type": "rect", - "y2": "height", - "opacity": 0, - }, - "encoding": { - "x": event_start_field_definition, - "x2": FIELD_DEFINITIONS["event_end"], - "y": { - "condition": { - "test": "isNaN(datum['event_value'])", - **event_value_field_definition, - }, - "value": 0, - }, - "detail": FIELD_DEFINITIONS["source"], - "tooltip": shared_tooltip, - }, - "transform": [ - { - "calculate": f"datum.event_start + {minimum_non_zero_resolution_in_ms}", - "as": "event_end", - }, - ], - }, + "title": join_words_into_a_list( + [ + f"{capitalize(sensor.name)}" + for sensor in row_sensors + if sensor.name != sensor.sensor_type + ] + ), + "transform": [ { - "mark": { - "type": "circle", - "opacity": 1, - "clip": True, - }, - "encoding": { - "x": event_start_field_definition, - "y": event_value_field_definition, - "color": FIELD_DEFINITIONS["source_name"], - "detail": FIELD_DEFINITIONS["source"], - "size": { - "condition": { - "value": "200", - "test": {"param": "paintbrush", "empty": False}, - }, - "value": "0", - }, - "tooltip": shared_tooltip, - }, - "params": [ - { - "name": "paintbrush", - "select": { - "type": "point", - "encodings": ["x"], - "on": "mouseover", - "nearest": False, - }, - }, - ], - }, + "filter": { + "field": "sensor.id", + "oneOf": [sensor.id for sensor in row_sensors], + } + } ], + "layer": layers, "width": "container", } sensors_specs.append(sensor_specs) + + # Vertically concatenate the rows chart_specs = dict( description="A vertically concatenated chart showing sensor data.", vconcat=[*sensors_specs], @@ -237,3 +210,150 @@ def chart_for_multiple_sensors( for k, v in override_chart_specs.items(): chart_specs[k] = v return chart_specs + + +def determine_shared_unit(sensors: list["Sensor"]) -> str: # noqa F821 + units = list(set([sensor.unit for sensor in sensors if sensor.unit])) + + # Replace with 'a.u.' in case of mixing units + shared_unit = units[0] if len(units) == 1 else "a.u." + + # Replace with 'dimensionless' in case of empty unit + return shared_unit if shared_unit else "dimensionless" + + +def determine_shared_sensor_type(sensors: list["Sensor"]) -> str: # noqa F821 + sensor_types = list(set([sensor.sensor_type for sensor in sensors])) + shared_sensor_type = sensor_types[0] if len(sensor_types) == 1 else "value" + return shared_sensor_type + + +def create_line_layer( + sensors: list["Sensor"], # noqa F821 + event_start_field_definition: dict, + event_value_field_definition: dict, +): + event_resolutions = list(set([sensor.event_resolution for sensor in sensors])) + assert ( + len(event_resolutions) == 1 + ), "Sensors shown within one row must share the same event resolution." + event_resolution = event_resolutions[0] + line_layer = { + "mark": { + "type": "line", + "interpolate": "step-after" + if event_resolution != timedelta(0) + else "linear", + "clip": True, + }, + "encoding": { + "x": event_start_field_definition, + "y": event_value_field_definition, + "color": FIELD_DEFINITIONS["sensor_name"], + "strokeDash": { + "scale": { + # Distinguish forecasters and schedulers by line stroke + "domain": ["forecaster", "scheduler", "other"], + # Schedulers get a dashed line, forecasters get a dotted line, the rest gets a solid line + "range": [[2, 2], [4, 4], [1, 0]], + }, + "field": "source.type", + "legend": { + "title": "Source", + }, + }, + "detail": [FIELD_DEFINITIONS["source"], FIELD_DEFINITIONS["sensor"]], + }, + } + return line_layer + + +def create_circle_layer( + sensors: list["Sensor"], # noqa F821 + event_start_field_definition: dict, + event_value_field_definition: dict, + shared_tooltip: list, +): + params = [ + { + "name": "hover_x_brush", + "select": { + "type": "point", + "encodings": ["x"], + "on": "mouseover", + "nearest": False, + "clear": "mouseout", + }, + } + ] + if len(sensors) > 1: + # extra brush for showing the tooltip of the closest sensor + params.append( + { + "name": "hover_nearest_brush", + "select": { + "type": "point", + "on": "mouseover", + "nearest": True, + "clear": "mouseout", + }, + } + ) + or_conditions = [{"param": "hover_x_brush", "empty": False}] + if len(sensors) > 1: + or_conditions.append({"param": "hover_nearest_brush", "empty": False}) + circle_layer = { + "mark": { + "type": "circle", + "opacity": 1, + "clip": True, + }, + "encoding": { + "x": event_start_field_definition, + "y": event_value_field_definition, + "color": FIELD_DEFINITIONS["sensor_name"], + # "detail": [FIELD_DEFINITIONS["source"], FIELD_DEFINITIONS["sensor"]], # todo: may be redundant for circle markers + "size": { + "condition": {"value": "200", "test": {"or": or_conditions}}, + "value": "0", + }, + "tooltip": shared_tooltip, + }, + "params": params, + } + return circle_layer + + +def create_rect_layer( + event_start_field_definition: dict, + event_value_field_definition: dict, + minimum_non_zero_resolution_in_ms: int, + shared_tooltip: list, +): + rect_layer = { + "mark": { + "type": "rect", + "y2": "height", + "opacity": 0, + }, + "encoding": { + "x": event_start_field_definition, + "x2": FIELD_DEFINITIONS["event_end"], + "y": { + "condition": { + "test": "isNaN(datum['event_value'])", + **event_value_field_definition, + }, + "value": 0, + }, + # "detail": FIELD_DEFINITIONS["source"], # todo: may be redundant for rect markers + "tooltip": shared_tooltip, + }, + "transform": [ + { + "calculate": f"datum.event_start + {minimum_non_zero_resolution_in_ms}", + "as": "event_end", + }, + ], + } + return rect_layer diff --git a/flexmeasures/data/models/charts/defaults.py b/flexmeasures/data/models/charts/defaults.py index f1ff22c2a..b4b0dd0ff 100644 --- a/flexmeasures/data/models/charts/defaults.py +++ b/flexmeasures/data/models/charts/defaults.py @@ -31,11 +31,27 @@ field="event_value", type="quantitative", ), + "sensor": dict( + field="sensor.id", + type="nominal", + title=None, + legend=None, + ), + "sensor_name": dict( + field="sensor.name", + type="nominal", + title="Sensor", + ), "source": dict( field="source.id", type="nominal", title=None, ), + "source_type": dict( + field="source.type", + type="nominal", + title="Type", + ), "source_name": dict( field="source.name", type="nominal", diff --git a/flexmeasures/data/models/data_sources.py b/flexmeasures/data/models/data_sources.py index 7705400c0..10cfe823d 100644 --- a/flexmeasures/data/models/data_sources.py +++ b/flexmeasures/data/models/data_sources.py @@ -50,7 +50,7 @@ def __init__( @property def label(self): - """Human-readable label (preferably not starting with a capital letter so it can be used in a sentence).""" + """Human-readable label (preferably not starting with a capital letter, so it can be used in a sentence).""" if self.type == "user": return f"data entered by user {self.user.username}" # todo: give users a display name elif self.type == "forecasting script": @@ -95,5 +95,10 @@ def to_dict(self) -> dict: id=self.id, name=self.name, model=model_incl_version, + type="forecaster" + if self.type == "forecasting script" + else "scheduler" + if self.type == "scheduling script" + else "other", description=self.description, ) diff --git a/flexmeasures/data/models/generic_assets.py b/flexmeasures/data/models/generic_assets.py index 2d8117210..92c5b7ab6 100644 --- a/flexmeasures/data/models/generic_assets.py +++ b/flexmeasures/data/models/generic_assets.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from datetime import datetime, timedelta from typing import Any, Dict, Optional, Tuple, List, Union import json @@ -20,6 +22,7 @@ from flexmeasures.data.queries.annotations import query_asset_annotations from flexmeasures.auth.policy import AuthModelMixin, EVERY_LOGGED_IN_USER from flexmeasures.utils import geo_utils +from flexmeasures.utils.coding_utils import flatten_unique from flexmeasures.utils.time_utils import ( determine_minimum_resampling_resolution, server_now, @@ -296,7 +299,8 @@ def chart( :param dataset_name: optionally name the dataset used in the chart (the default name is sensor_) :returns: JSON string defining vega-lite chart specs """ - sensors = self.sensors_to_show + sensors_to_show = self.sensors_to_show + sensors = flatten_unique(sensors_to_show) for sensor in sensors: sensor.sensor_type = sensor.get_attribute("sensor_type", sensor.name) @@ -309,7 +313,7 @@ def chart( kwargs["event_ends_before"] = event_ends_before chart_specs = chart_type_to_chart_specs( chart_type, - sensors=sensors, + sensors_to_show=sensors_to_show, dataset_name=dataset_name, **kwargs, ) @@ -427,34 +431,51 @@ def search_beliefs( return bdf_dict @property - def sensors_to_show(self) -> List["Sensor"]: # noqa F821 + def sensors_to_show(self) -> list["Sensor" | list["Sensor"]]: # noqa F821 """Sensors to show, as defined by the sensors_to_show attribute. Sensors to show are defined as a list of sensor ids, which is set by the "sensors_to_show" field of the asset's "attributes" column. Valid sensors either belong to the asset itself, to other assets in the same account, or to public assets. + In case the field is missing, defaults to two of the asset's sensors. + + Sensor ids can be nested to denote that sensors should be 'shown together', + for example, layered rather than vertically concatenated. + How to interpret 'shown together' is technically left up to the function returning chart specs, + as are any restrictions regarding what sensors can be shown together, such as: + - whether they should share the same unit + - whether they should share the same name + - whether they should belong to different assets + For example, this denotes showing sensors 42 and 44 together: + + sensors_to_show = [40, 35, 41, [42, 44], 43, 45] - Defaults to two of the asset's sensors. """ if not self.has_attribute("sensors_to_show"): return self.sensors[:2] from flexmeasures.data.services.sensors import get_sensors - sensor_ids = self.get_attribute("sensors_to_show") + sensor_ids_to_show = self.get_attribute("sensors_to_show") sensor_map = { sensor.id: sensor for sensor in get_sensors( account=self.owner, include_public_assets=True, - sensor_id_allowlist=sensor_ids, + sensor_id_allowlist=flatten_unique(sensor_ids_to_show), ) } - # Return sensors in the order given by the sensors_to_show attribute - return [sensor_map[sensor_id] for sensor_id in sensor_ids] + # Return sensors in the order given by the sensors_to_show attribute, and with the same nesting + sensors_to_show = [] + for s in sensor_ids_to_show: + if isinstance(s, list): + sensors_to_show.append([sensor_map[sensor_id] for sensor_id in s]) + else: + sensors_to_show.append(sensor_map[s]) + return sensors_to_show @property def timezone( @@ -507,7 +528,7 @@ def get_timerange(cls, sensors: List["Sensor"]) -> Dict[str, datetime]: # noqa """ from flexmeasures.data.models.time_series import TimedBelief - sensor_ids = [sensor.id for sensor in sensors] + sensor_ids = [s.id for s in flatten_unique(sensors)] least_recent_query = ( TimedBelief.query.filter(TimedBelief.sensor_id.in_(sensor_ids)) .order_by(TimedBelief.event_start.asc()) diff --git a/flexmeasures/utils/coding_utils.py b/flexmeasures/utils/coding_utils.py index 4b4d763f8..61237924d 100644 --- a/flexmeasures/utils/coding_utils.py +++ b/flexmeasures/utils/coding_utils.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import functools import time @@ -117,6 +119,24 @@ def sort_dict(unsorted_dict: dict) -> dict: return sorted_dict +def flatten_unique( + nested_list_of_objects: list[int | list[int]] | list["Sensor" | list["Sensor"]], +) -> list[int] | list["Sensor"]: + """Returns unique objects in a possibly nested (one level) list of objects. + + For example: + >>> flatten_unique([1, [2, 3, 4], 3, 5]) + <<< [1, 2, 3, 4, 5] + """ + all_objects = [] + for s in nested_list_of_objects: + if isinstance(s, list): + all_objects.extend(s) + else: + all_objects.append(s) + return list(set(all_objects)) + + def timeit(func): """Decorator for printing the time it took to execute the decorated function.""" diff --git a/flexmeasures/utils/flexmeasures_inflection.py b/flexmeasures/utils/flexmeasures_inflection.py index 3c3d07cef..eaa825cad 100644 --- a/flexmeasures/utils/flexmeasures_inflection.py +++ b/flexmeasures/utils/flexmeasures_inflection.py @@ -25,7 +25,7 @@ def humanize(word): def parameterize(word): - """Parameterize the word so it can be used as a python or javascript variable name. + """Parameterize the word, so it can be used as a python or javascript variable name. For example: >>> word = "Acme® EV-Charger™" "acme_ev_chargertm" @@ -55,3 +55,7 @@ def titleize(word): for ac in ACRONYMS: word = re.sub(inflection.titleize(ac), ac, word) return word + + +def join_words_into_a_list(words: list[str]) -> str: + return p.join(words, final_sep="") From 1b95b9dd304f461649a1b9f780f8b6628f237dc4 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 30 Nov 2022 20:01:48 +0100 Subject: [PATCH 14/73] Textual changes Signed-off-by: F.N. Claessen --- documentation/tut/toy-example-from-scratch.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation/tut/toy-example-from-scratch.rst b/documentation/tut/toy-example-from-scratch.rst index 468a35e42..1c30200b2 100644 --- a/documentation/tut/toy-example-from-scratch.rst +++ b/documentation/tut/toy-example-from-scratch.rst @@ -321,7 +321,7 @@ Recall that we only asked for a 12 hour schedule here. We started our schedule * Take into account solar production --------------------------------------- -First, we'll create a new csv file with solar forecast (MW, see the setup for sensor 4 above) for tomorrow. +First, we'll create a new csv file with solar forecasts (MW, see the setup for sensor 4 above) for tomorrow. .. code-block:: console @@ -359,7 +359,7 @@ Then, we read in the created CSV file as beliefs data: $ flexmeasures add beliefs --sensor-id 4 --source toy-user solar-tomorrow.csv --timezone Europe/Amsterdam Successfully created beliefs -Notice that, by default, the one-hour CSV data is automatically resampled to the 15-minute resolution of the sensor that is recording solar production. +The one-hour CSV data is automatically resampled to the 15-minute resolution of the sensor that is recording solar production. .. note:: The ``flexmeasures add beliefs`` command has many options to make sure the read-in data is correctly interpreted (unit, timezone, delimiter, etc). But that is not the point of this tutorial. See ``flexmeasures add beliefs --help``. From bd919676b357f0456b1e2d5f2a3cfbb4f464f40d Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 30 Nov 2022 20:07:57 +0100 Subject: [PATCH 15/73] Remove print statement Signed-off-by: F.N. Claessen --- flexmeasures/cli/data_add.py | 1 - 1 file changed, 1 deletion(-) diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index 1d498de50..6ef34db09 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -1196,7 +1196,6 @@ def get_or_create_model( filter_by_kwargs = lookup_kwargs.copy() for kw, arg in lookup_kwargs.items(): model_attribute = getattr(model_class, kw) - print(f"argument {arg} is {callable(arg)} callable") if hasattr(model_attribute, "type") and isinstance(model_attribute.type, JSON): filter_json_kwargs[kw] = filter_by_kwargs.pop(kw) elif callable(arg) and isinstance(model_attribute.type, String): From cf16ecff59aa102d39d621b37ee6fedc22f8be1f Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 30 Nov 2022 20:12:06 +0100 Subject: [PATCH 16/73] Add missing timezone Signed-off-by: F.N. Claessen --- documentation/tut/toy-example-from-scratch.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/tut/toy-example-from-scratch.rst b/documentation/tut/toy-example-from-scratch.rst index 1c30200b2..1cef6be13 100644 --- a/documentation/tut/toy-example-from-scratch.rst +++ b/documentation/tut/toy-example-from-scratch.rst @@ -19,7 +19,7 @@ Below are the ``flexmeasures`` CLI commands we'll run, and which we'll explain s # setup an account with a user, a battery (Id 2) and a market (Id 3) $ flexmeasures add toy-account --kind battery # load prices to optimise the schedule against - $ flexmeasures add beliefs --sensor-id 3 --source toy-user prices-tomorrow.csv + $ flexmeasures add beliefs --sensor-id 3 --source toy-user prices-tomorrow.csv --timezone Europe/Amsterdam # make the schedule $ flexmeasures add schedule --sensor-id 2 --consumption-price-sensor 3 \ --start ${TOMORROW}T07:00+01:00 --duration PT12H \ From 7dcb001182a56e9a613b1af21d534de27c627d3c Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 30 Nov 2022 20:40:49 +0100 Subject: [PATCH 17/73] Fix call to make_schedule Signed-off-by: F.N. Claessen --- flexmeasures/cli/data_add.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index 6ef34db09..40776d4c9 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -1026,11 +1026,13 @@ def create_schedule( end=end, belief_time=server_now(), resolution=power_sensor.event_resolution, - soc_at_start=soc_at_start, - soc_targets=soc_targets, - soc_min=soc_min, - soc_max=soc_max, - roundtrip_efficiency=roundtrip_efficiency, + storage_specs=dict( + soc_at_start=soc_at_start, + soc_targets=soc_targets, + soc_min=soc_min, + soc_max=soc_max, + roundtrip_efficiency=roundtrip_efficiency, + ), consumption_price_sensor=consumption_price_sensor, production_price_sensor=production_price_sensor, inflexible_device_sensors=inflexible_device_sensors, From 141a79484833c8213577b8b03a500d2d53f2e4a6 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 30 Nov 2022 20:42:20 +0100 Subject: [PATCH 18/73] Ensure storage specs, also when make_schedule is called directly (i.e. no job) Signed-off-by: F.N. Claessen --- flexmeasures/data/services/scheduling.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py index 510c59b4a..497bf7ba5 100644 --- a/flexmeasures/data/services/scheduling.py +++ b/flexmeasures/data/services/scheduling.py @@ -144,6 +144,8 @@ def make_schedule( % sensor.generic_asset.generic_asset_type ) + storage_specs = ensure_storage_specs(storage_specs, sensor, start, end, resolution) + consumption_schedule = scheduler().schedule( sensor, start, From fec7cbbf49220bfa4ccab359c6caf8a75468cecd Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 30 Nov 2022 20:52:05 +0100 Subject: [PATCH 19/73] Legend shows which asset a sensor belongs to, too, and sensors that share a name don't share a color (note that sensors belonging to the same asset aren't allowed to have the same name) Signed-off-by: F.N. Claessen --- flexmeasures/data/models/charts/belief_charts.py | 4 ++-- flexmeasures/data/models/charts/defaults.py | 5 +++++ flexmeasures/data/models/time_series.py | 1 + 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/flexmeasures/data/models/charts/belief_charts.py b/flexmeasures/data/models/charts/belief_charts.py index 52cec8634..5e09c3324 100644 --- a/flexmeasures/data/models/charts/belief_charts.py +++ b/flexmeasures/data/models/charts/belief_charts.py @@ -249,7 +249,7 @@ def create_line_layer( "encoding": { "x": event_start_field_definition, "y": event_value_field_definition, - "color": FIELD_DEFINITIONS["sensor_name"], + "color": FIELD_DEFINITIONS["sensor_description"], "strokeDash": { "scale": { # Distinguish forecasters and schedulers by line stroke @@ -311,7 +311,7 @@ def create_circle_layer( "encoding": { "x": event_start_field_definition, "y": event_value_field_definition, - "color": FIELD_DEFINITIONS["sensor_name"], + "color": FIELD_DEFINITIONS["sensor_description"], # "detail": [FIELD_DEFINITIONS["source"], FIELD_DEFINITIONS["sensor"]], # todo: may be redundant for circle markers "size": { "condition": {"value": "200", "test": {"or": or_conditions}}, diff --git a/flexmeasures/data/models/charts/defaults.py b/flexmeasures/data/models/charts/defaults.py index b4b0dd0ff..0fd61e8fc 100644 --- a/flexmeasures/data/models/charts/defaults.py +++ b/flexmeasures/data/models/charts/defaults.py @@ -42,6 +42,11 @@ type="nominal", title="Sensor", ), + "sensor_description": dict( + field="sensor.description", + type="nominal", + title="Sensor", + ), "source": dict( field="source.id", type="nominal", diff --git a/flexmeasures/data/models/time_series.py b/flexmeasures/data/models/time_series.py index d1acf04f6..d9cb1900d 100644 --- a/flexmeasures/data/models/time_series.py +++ b/flexmeasures/data/models/time_series.py @@ -489,6 +489,7 @@ def to_dict(self) -> dict: return dict( id=self.id, name=self.name, + description=f"{self.name} ({self.generic_asset.name})", ) @classmethod From 15a59a22e0b6336808420e8259617e6c7615e7ff Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 30 Nov 2022 21:39:28 +0100 Subject: [PATCH 20/73] Add CLI command for adding a data source Signed-off-by: F.N. Claessen --- flexmeasures/cli/data_add.py | 38 +++++++++++++++++++++++ flexmeasures/data/queries/data_sources.py | 9 ++++-- 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index 40776d4c9..c6e9caa26 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -283,6 +283,44 @@ def add_initial_structure(): populate_initial_structure(db) +@fm_add_data.command("source") +@with_appcontext +@click.option( + "--name", + required=True, + type=str, + help="Name of the source (usually an organisation)", +) +@click.option( + "--model", + required=False, + type=str, + help="Optionally, specify a model (for example, a class name, function name or url).", +) +@click.option( + "--version", + required=False, + type=str, + help="Optionally, specify a version (for example, '1.0'.", +) +@click.option( + "--type", + "source_type", + required=True, + type=str, + help="Type of source (for example, 'forecaster' or 'scheduler').", +) +def add_source(name: str, model: str, version: str, source_type: str): + source = get_or_create_source( + source=name, + model=model, + version=version, + source_type=source_type, + ) + db.session.commit() + print(f"Added source {source.__repr__()}") + + @fm_add_data.command("beliefs") @with_appcontext @click.argument("file", type=click.Path(exists=True)) diff --git a/flexmeasures/data/queries/data_sources.py b/flexmeasures/data/queries/data_sources.py index 178891a34..cccb891eb 100644 --- a/flexmeasures/data/queries/data_sources.py +++ b/flexmeasures/data/queries/data_sources.py @@ -13,6 +13,7 @@ def get_or_create_source( source: Union[User, str], source_type: Optional[str] = None, model: Optional[str] = None, + version: Optional[str] = None, flush: bool = True, ) -> DataSource: if is_user(source): @@ -20,6 +21,8 @@ def get_or_create_source( query = DataSource.query.filter(DataSource.type == source_type) if model is not None: query = query.filter(DataSource.model == model) + if version is not None: + query = query.filter(DataSource.version == version) if is_user(source): query = query.filter(DataSource.user == source) elif isinstance(source, str): @@ -29,11 +32,13 @@ def get_or_create_source( _source = query.one_or_none() if not _source: if is_user(source): - _source = DataSource(user=source, model=model) + _source = DataSource(user=source, model=model, version=version) else: if source_type is None: raise TypeError("Please specify a source type") - _source = DataSource(name=source, model=model, type=source_type) + _source = DataSource( + name=source, model=model, version=version, type=source_type + ) current_app.logger.info(f"Setting up {_source} as new data source...") db.session.add(_source) if flush: From 42523e4e90e564ac120bc9b0a682b71a70e27cd2 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 30 Nov 2022 21:42:25 +0100 Subject: [PATCH 21/73] DB migration for source types 'forecaster' and 'scheduler' Signed-off-by: F.N. Claessen --- ...rce_type_for_forecasters_and_schedulers.py | 33 +++++++++++++++++++ flexmeasures/data/models/data_sources.py | 6 +--- 2 files changed, 34 insertions(+), 5 deletions(-) create mode 100644 flexmeasures/data/migrations/versions/c41beee0c904_rename_DataSource_type_for_forecasters_and_schedulers.py diff --git a/flexmeasures/data/migrations/versions/c41beee0c904_rename_DataSource_type_for_forecasters_and_schedulers.py b/flexmeasures/data/migrations/versions/c41beee0c904_rename_DataSource_type_for_forecasters_and_schedulers.py new file mode 100644 index 000000000..4469cc12e --- /dev/null +++ b/flexmeasures/data/migrations/versions/c41beee0c904_rename_DataSource_type_for_forecasters_and_schedulers.py @@ -0,0 +1,33 @@ +"""Rename DataSource types for forecasters and schedulers + +Revision ID: c41beee0c904 +Revises: 650b085c0ad3 +Create Date: 2022-11-30 21:33:09.046751 + +""" +from alembic import op + + +# revision identifiers, used by Alembic. +revision = 'c41beee0c904' +down_revision = '650b085c0ad3' +branch_labels = None +depends_on = None + + +def upgrade(): + op.execute( + "update data_source set type='scheduler' where type='scheduling script';" + ) + op.execute( + "update data_source set type='forecaster' where type='forecasting script';" + ) + + +def downgrade(): + op.execute( + "update data_source set type='scheduling script' where type='scheduler';" + ) + op.execute( + "update data_source set type='forecasting script' where type='forecaster';" + ) diff --git a/flexmeasures/data/models/data_sources.py b/flexmeasures/data/models/data_sources.py index 10cfe823d..893c6f36b 100644 --- a/flexmeasures/data/models/data_sources.py +++ b/flexmeasures/data/models/data_sources.py @@ -95,10 +95,6 @@ def to_dict(self) -> dict: id=self.id, name=self.name, model=model_incl_version, - type="forecaster" - if self.type == "forecasting script" - else "scheduler" - if self.type == "scheduling script" - else "other", + type=self.type if self.type in ("forecaster", "scheduler") else "other", description=self.description, ) From 8d283d5c82dc09a604435a660de54da90869efe9 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 30 Nov 2022 21:43:09 +0100 Subject: [PATCH 22/73] Attribute toy solar forecasts to a new DataSource of type 'forecaster' Signed-off-by: F.N. Claessen --- documentation/tut/toy-example-from-scratch.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/documentation/tut/toy-example-from-scratch.rst b/documentation/tut/toy-example-from-scratch.rst index 1cef6be13..e4d09bdb4 100644 --- a/documentation/tut/toy-example-from-scratch.rst +++ b/documentation/tut/toy-example-from-scratch.rst @@ -352,11 +352,13 @@ First, we'll create a new csv file with solar forecasts (MW, see the setup for s $ ${TOMORROW}T22:00:00,0.0 $ ${TOMORROW}T23:00:00,0.0" > solar-tomorrow.csv -Then, we read in the created CSV file as beliefs data: +Then, we register a new forecaster and read in the created CSV file as beliefs data: .. code-block:: console - $ flexmeasures add beliefs --sensor-id 4 --source toy-user solar-tomorrow.csv --timezone Europe/Amsterdam + $ flexmeasures add source --name "toy-forecaster" --type forecaster + Added source + $ flexmeasures add beliefs --sensor-id 4 --source 2 solar-tomorrow.csv --timezone Europe/Amsterdam Successfully created beliefs The one-hour CSV data is automatically resampled to the 15-minute resolution of the sensor that is recording solar production. From 517061681e58347e3bba68a6e8d4663544f20507 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 30 Nov 2022 21:47:53 +0100 Subject: [PATCH 23/73] Check common cases of shared sensor types Signed-off-by: F.N. Claessen --- .../data/models/charts/belief_charts.py | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/flexmeasures/data/models/charts/belief_charts.py b/flexmeasures/data/models/charts/belief_charts.py index 5e09c3324..198f4ffac 100644 --- a/flexmeasures/data/models/charts/belief_charts.py +++ b/flexmeasures/data/models/charts/belief_charts.py @@ -8,6 +8,11 @@ join_words_into_a_list, ) from flexmeasures.utils.coding_utils import flatten_unique +from flexmeasures.utils.unit_utils import ( + is_power_unit, + is_energy_unit, + is_energy_price_unit, +) def bar_chart( @@ -224,8 +229,20 @@ def determine_shared_unit(sensors: list["Sensor"]) -> str: # noqa F821 def determine_shared_sensor_type(sensors: list["Sensor"]) -> str: # noqa F821 sensor_types = list(set([sensor.sensor_type for sensor in sensors])) - shared_sensor_type = sensor_types[0] if len(sensor_types) == 1 else "value" - return shared_sensor_type + + # Return the sole sensor type + if len(sensor_types) == 1: + return sensor_types[0] + + # Check the units for common cases + shared_unit = determine_shared_unit(sensors) + if is_power_unit(shared_unit): + return "power" + elif is_energy_unit(shared_unit): + return "energy" + elif is_energy_price_unit(shared_unit): + return "energy price" + return "value" def create_line_layer( From daf0273c4cf0e6de2eeba560f1595e4a8ce1d501 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 30 Nov 2022 21:51:44 +0100 Subject: [PATCH 24/73] Add version identifier and missing parenthesis Signed-off-by: F.N. Claessen --- flexmeasures/data/models/data_sources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/models/data_sources.py b/flexmeasures/data/models/data_sources.py index 893c6f36b..e51c4fc19 100644 --- a/flexmeasures/data/models/data_sources.py +++ b/flexmeasures/data/models/data_sources.py @@ -90,7 +90,7 @@ def __str__(self) -> str: def to_dict(self) -> dict: model_incl_version = self.model if self.model else "" if self.model and self.version: - model_incl_version += f" ({self.version}" + model_incl_version += f" (v{self.version})" return dict( id=self.id, name=self.name, From cd4f41b666b66a95970596ac2515a8febf583b78 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 30 Nov 2022 22:50:18 +0100 Subject: [PATCH 25/73] black and flake8 Signed-off-by: F.N. Claessen --- ..._rename_DataSource_type_for_forecasters_and_schedulers.py | 4 ++-- flexmeasures/data/models/charts/belief_charts.py | 4 ++-- flexmeasures/utils/coding_utils.py | 5 +++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/flexmeasures/data/migrations/versions/c41beee0c904_rename_DataSource_type_for_forecasters_and_schedulers.py b/flexmeasures/data/migrations/versions/c41beee0c904_rename_DataSource_type_for_forecasters_and_schedulers.py index 4469cc12e..4510f7294 100644 --- a/flexmeasures/data/migrations/versions/c41beee0c904_rename_DataSource_type_for_forecasters_and_schedulers.py +++ b/flexmeasures/data/migrations/versions/c41beee0c904_rename_DataSource_type_for_forecasters_and_schedulers.py @@ -9,8 +9,8 @@ # revision identifiers, used by Alembic. -revision = 'c41beee0c904' -down_revision = '650b085c0ad3' +revision = "c41beee0c904" +down_revision = "650b085c0ad3" branch_labels = None depends_on = None diff --git a/flexmeasures/data/models/charts/belief_charts.py b/flexmeasures/data/models/charts/belief_charts.py index 198f4ffac..768e88d78 100644 --- a/flexmeasures/data/models/charts/belief_charts.py +++ b/flexmeasures/data/models/charts/belief_charts.py @@ -107,9 +107,9 @@ def chart_for_multiple_sensors( for s in sensors_to_show: # List the sensors that go into one row if isinstance(s, list): - row_sensors: list["Sensor"] = s + row_sensors: list["Sensor"] = s # noqa F821 else: - row_sensors: list["Sensor"] = [s] + row_sensors: list["Sensor"] = [s] # noqa F821 # Derive the unit that should be shown unit = determine_shared_unit(row_sensors) diff --git a/flexmeasures/utils/coding_utils.py b/flexmeasures/utils/coding_utils.py index 61237924d..3e60455cb 100644 --- a/flexmeasures/utils/coding_utils.py +++ b/flexmeasures/utils/coding_utils.py @@ -120,8 +120,9 @@ def sort_dict(unsorted_dict: dict) -> dict: def flatten_unique( - nested_list_of_objects: list[int | list[int]] | list["Sensor" | list["Sensor"]], -) -> list[int] | list["Sensor"]: + nested_list_of_objects: list[int | list[int]] + | list["Sensor" | list["Sensor"]], # noqa F821 +) -> list[int] | list["Sensor"]: # noqa F821 """Returns unique objects in a possibly nested (one level) list of objects. For example: From 9eb668d62330e268995b0c1a540fda400030e8e4 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 30 Nov 2022 23:04:56 +0100 Subject: [PATCH 26/73] Fix test (no legend in PositionFieldDef) Signed-off-by: F.N. Claessen --- flexmeasures/data/models/charts/defaults.py | 1 - 1 file changed, 1 deletion(-) diff --git a/flexmeasures/data/models/charts/defaults.py b/flexmeasures/data/models/charts/defaults.py index 0fd61e8fc..566043b50 100644 --- a/flexmeasures/data/models/charts/defaults.py +++ b/flexmeasures/data/models/charts/defaults.py @@ -35,7 +35,6 @@ field="sensor.id", type="nominal", title=None, - legend=None, ), "sensor_name": dict( field="sensor.name", From 71e983f7344fed45f710099d7e3e220967115f2f Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 30 Nov 2022 23:07:29 +0100 Subject: [PATCH 27/73] Remove redundant detail encodings Signed-off-by: F.N. Claessen --- flexmeasures/data/models/charts/belief_charts.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/flexmeasures/data/models/charts/belief_charts.py b/flexmeasures/data/models/charts/belief_charts.py index 768e88d78..deb7e132a 100644 --- a/flexmeasures/data/models/charts/belief_charts.py +++ b/flexmeasures/data/models/charts/belief_charts.py @@ -279,7 +279,7 @@ def create_line_layer( "title": "Source", }, }, - "detail": [FIELD_DEFINITIONS["source"], FIELD_DEFINITIONS["sensor"]], + "detail": [FIELD_DEFINITIONS["source"]], }, } return line_layer @@ -329,7 +329,6 @@ def create_circle_layer( "x": event_start_field_definition, "y": event_value_field_definition, "color": FIELD_DEFINITIONS["sensor_description"], - # "detail": [FIELD_DEFINITIONS["source"], FIELD_DEFINITIONS["sensor"]], # todo: may be redundant for circle markers "size": { "condition": {"value": "200", "test": {"or": or_conditions}}, "value": "0", @@ -363,7 +362,6 @@ def create_rect_layer( }, "value": 0, }, - # "detail": FIELD_DEFINITIONS["source"], # todo: may be redundant for rect markers "tooltip": shared_tooltip, }, "transform": [ From f174c350f0ffc2cb34167ddc1efb833ef7e4264b Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 2 Dec 2022 12:09:52 +0100 Subject: [PATCH 28/73] Upgrade timely-beliefs for required feature from https://github.com/SeitaBV/timely-beliefs/pull/122 Signed-off-by: F.N. Claessen --- requirements/app.in | 2 +- requirements/app.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements/app.in b/requirements/app.in index df6a9a220..32744fa5f 100644 --- a/requirements/app.in +++ b/requirements/app.in @@ -28,7 +28,7 @@ tldextract pyomo>=5.6 tabulate timetomodel>=0.7.1 -timely-beliefs[forecast]>=1.14 +timely-beliefs[forecast]>=1.16 python-dotenv # a backport, not needed in Python3.8 importlib_metadata diff --git a/requirements/app.txt b/requirements/app.txt index 480874aa5..a04ed8704 100644 --- a/requirements/app.txt +++ b/requirements/app.txt @@ -314,7 +314,7 @@ tabulate==0.8.10 # via -r requirements/app.in threadpoolctl==3.1.0 # via scikit-learn -timely-beliefs[forecast]==1.14.0 +timely-beliefs[forecast]==1.16.0 # via -r requirements/app.in timetomodel==0.7.1 # via -r requirements/app.in From a495bd117f8fce704c1e7f5b37f04a7183e6decc Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 12 Dec 2022 15:40:00 +0100 Subject: [PATCH 29/73] Merge db revisions Signed-off-by: F.N. Claessen --- .../migrations/versions/d814c0688ae0_merge.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 flexmeasures/data/migrations/versions/d814c0688ae0_merge.py diff --git a/flexmeasures/data/migrations/versions/d814c0688ae0_merge.py b/flexmeasures/data/migrations/versions/d814c0688ae0_merge.py new file mode 100644 index 000000000..ae0a5abc9 --- /dev/null +++ b/flexmeasures/data/migrations/versions/d814c0688ae0_merge.py @@ -0,0 +1,22 @@ +"""merge + +Revision ID: d814c0688ae0 +Revises: 75f53d2dbfae, c41beee0c904 +Create Date: 2022-12-12 15:31:41.509921 + +""" + + +# revision identifiers, used by Alembic. +revision = 'd814c0688ae0' +down_revision = ("75f53d2dbfae", "c41beee0c904") +branch_labels = None +depends_on = None + + +def upgrade(): + pass + + +def downgrade(): + pass From 401cd3e96fdaf1ac5a4cb3108820120acf38fcb6 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 12 Dec 2022 15:41:46 +0100 Subject: [PATCH 30/73] black Signed-off-by: F.N. Claessen --- flexmeasures/data/migrations/versions/d814c0688ae0_merge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/migrations/versions/d814c0688ae0_merge.py b/flexmeasures/data/migrations/versions/d814c0688ae0_merge.py index ae0a5abc9..ff3204329 100644 --- a/flexmeasures/data/migrations/versions/d814c0688ae0_merge.py +++ b/flexmeasures/data/migrations/versions/d814c0688ae0_merge.py @@ -8,7 +8,7 @@ # revision identifiers, used by Alembic. -revision = 'd814c0688ae0' +revision = "d814c0688ae0" down_revision = ("75f53d2dbfae", "c41beee0c904") branch_labels = None depends_on = None From a337269b1a772a93d3d9daf7dc2ac8da648410fc Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 20 Dec 2022 17:03:45 +0100 Subject: [PATCH 31/73] Speed up viz by avoiding redundant client-side transformation Signed-off-by: F.N. Claessen --- .../data/models/charts/belief_charts.py | 27 +++++++------------ flexmeasures/data/models/charts/defaults.py | 6 ----- 2 files changed, 10 insertions(+), 23 deletions(-) diff --git a/flexmeasures/data/models/charts/belief_charts.py b/flexmeasures/data/models/charts/belief_charts.py index 1b3e872d8..cf6d0cc7c 100644 --- a/flexmeasures/data/models/charts/belief_charts.py +++ b/flexmeasures/data/models/charts/belief_charts.py @@ -30,6 +30,10 @@ def bar_chart( **FIELD_DEFINITIONS["event_value"], ) event_start_field_definition = FIELD_DEFINITIONS["event_start"] + event_start_field_definition["timeUnit"] = { + "unit": "yearmonthdatehoursminutesseconds", + "step": sensor.event_resolution.total_seconds(), + } if event_starts_after and event_ends_before: event_start_field_definition["scale"] = { "domain": [ @@ -37,7 +41,6 @@ def bar_chart( event_ends_before.timestamp() * 10**3, ] } - resolution_in_ms = sensor.event_resolution.total_seconds() * 1000 chart_specs = { "description": "A simple bar chart showing sensor data.", "title": capitalize(sensor.name) if sensor.name != sensor.sensor_type else None, @@ -47,7 +50,6 @@ def bar_chart( }, "encoding": { "x": event_start_field_definition, - "x2": FIELD_DEFINITIONS["event_end"], "y": event_value_field_definition, "color": FIELD_DEFINITIONS["source_name"], "detail": FIELD_DEFINITIONS["source"], @@ -63,10 +65,6 @@ def bar_chart( ], }, "transform": [ - { - "calculate": f"datum.event_start + {resolution_in_ms}", - "as": "event_end", - }, { "calculate": "datum.source.name + ' (ID: ' + datum.source.id + ')'", "as": "source_name_and_id", @@ -93,12 +91,16 @@ def chart_for_multiple_sensors( for sensor in sensors if sensor.event_resolution > timedelta(0) ) - minimum_non_zero_resolution_in_ms = ( - min(condition).total_seconds() * 1000 if any(condition) else 0 + minimum_non_zero_resolution = ( + min(condition) if any(condition) else timedelta(0) ) # Set up field definition for event starts event_start_field_definition = FIELD_DEFINITIONS["event_start"] + event_start_field_definition["timeUnit"] = { + "unit": "yearmonthdatehoursminutesseconds", + "step": minimum_non_zero_resolution.total_seconds(), + } if event_starts_after and event_ends_before: event_start_field_definition["scale"] = { "domain": [ @@ -171,7 +173,6 @@ def chart_for_multiple_sensors( create_rect_layer( event_start_field_definition, event_value_field_definition, - minimum_non_zero_resolution_in_ms, shared_tooltip, ) ) @@ -357,7 +358,6 @@ def create_circle_layer( def create_rect_layer( event_start_field_definition: dict, event_value_field_definition: dict, - minimum_non_zero_resolution_in_ms: int, shared_tooltip: list, ): rect_layer = { @@ -368,7 +368,6 @@ def create_rect_layer( }, "encoding": { "x": event_start_field_definition, - "x2": FIELD_DEFINITIONS["event_end"], "y": { "condition": { "test": "isNaN(datum['event_value'])", @@ -378,11 +377,5 @@ def create_rect_layer( }, "tooltip": shared_tooltip, }, - "transform": [ - { - "calculate": f"datum.event_start + {minimum_non_zero_resolution_in_ms}", - "as": "event_end", - }, - ], } return rect_layer diff --git a/flexmeasures/data/models/charts/defaults.py b/flexmeasures/data/models/charts/defaults.py index 8c701c6d3..e857b9e0f 100644 --- a/flexmeasures/data/models/charts/defaults.py +++ b/flexmeasures/data/models/charts/defaults.py @@ -21,12 +21,6 @@ title=None, axis={"labelExpr": FORMAT_24H, "labelOverlap": True, "labelSeparation": 1}, ), - "event_end": dict( - field="event_end", - type="temporal", - title=None, - axis={"labelExpr": FORMAT_24H, "labelOverlap": True, "labelSeparation": 1}, - ), "event_value": dict( field="event_value", type="quantitative", From 8992ea474210766e6ca7d97bcfdce71828f46dbd Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 20 Dec 2022 17:17:23 +0100 Subject: [PATCH 32/73] Bump vega-lite and vegaembed Signed-off-by: F.N. Claessen --- flexmeasures/utils/config_defaults.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flexmeasures/utils/config_defaults.py b/flexmeasures/utils/config_defaults.py index a04611f09..48b3be5a3 100644 --- a/flexmeasures/utils/config_defaults.py +++ b/flexmeasures/utils/config_defaults.py @@ -123,8 +123,8 @@ class Config(object): FLEXMEASURES_REDIS_PASSWORD: Optional[str] = None FLEXMEASURES_JS_VERSIONS: dict = dict( vega="5.22.1", - vegaembed="6.20.8", - vegalite="5.2.0", + vegaembed="6.21.0", + vegalite="5.5.0", # "5.6.0" has a problematic bar chart: see our sensor page and https://github.com/vega/vega-lite/issues/8496 # todo: expand with other js versions used in FlexMeasures ) From 4001debb71df0497209fd7003d1ca150832ba3b0 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 20 Dec 2022 21:05:55 +0100 Subject: [PATCH 33/73] Ensure full-width bar width Signed-off-by: F.N. Claessen --- flexmeasures/data/models/charts/belief_charts.py | 1 + 1 file changed, 1 insertion(+) diff --git a/flexmeasures/data/models/charts/belief_charts.py b/flexmeasures/data/models/charts/belief_charts.py index bd97f82ee..8e45d798f 100644 --- a/flexmeasures/data/models/charts/belief_charts.py +++ b/flexmeasures/data/models/charts/belief_charts.py @@ -49,6 +49,7 @@ def bar_chart( "mark": { "type": "bar", "clip": True, + "width": {"band": 0.999}, }, "encoding": { "x": event_start_field_definition, From b2746ae6636bb866107f0ba54c7d3e5c3fbad8f2 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 21 Dec 2022 13:00:29 +0100 Subject: [PATCH 34/73] black Signed-off-by: F.N. Claessen --- flexmeasures/data/models/charts/belief_charts.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/flexmeasures/data/models/charts/belief_charts.py b/flexmeasures/data/models/charts/belief_charts.py index 8e45d798f..253fa8bce 100644 --- a/flexmeasures/data/models/charts/belief_charts.py +++ b/flexmeasures/data/models/charts/belief_charts.py @@ -97,9 +97,7 @@ def chart_for_multiple_sensors( for sensor in sensors if sensor.event_resolution > timedelta(0) ) - minimum_non_zero_resolution = ( - min(condition) if any(condition) else timedelta(0) - ) + minimum_non_zero_resolution = min(condition) if any(condition) else timedelta(0) # Set up field definition for event starts event_start_field_definition = FIELD_DEFINITIONS["event_start"] From 061c153b8f2c60a04f358872b28a1b46880e6a3b Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 9 Mar 2023 10:01:01 +0100 Subject: [PATCH 35/73] Update CLI scheduling command by adding `for-storage` Signed-off-by: F.N. Claessen --- documentation/tut/toy-example-from-scratch.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/tut/toy-example-from-scratch.rst b/documentation/tut/toy-example-from-scratch.rst index ef730ae57..70f8d5a09 100644 --- a/documentation/tut/toy-example-from-scratch.rst +++ b/documentation/tut/toy-example-from-scratch.rst @@ -378,7 +378,7 @@ Now, we'll reschedule the battery while taking into account the solar production .. code-block:: console - $ flexmeasures add schedule --sensor-id 2 --consumption-price-sensor 3 \ + $ flexmeasures add schedule for-storage --sensor-id 2 --consumption-price-sensor 3 \ --inflexible-device-sensor 4 \ --start ${TOMORROW}T07:00+01:00 --duration PT12H \ --soc-at-start 50% --roundtrip-efficiency 90% From f1dc65d0e0f2fcab85711ce71b386e3a16e66291 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 9 Mar 2023 10:13:07 +0100 Subject: [PATCH 36/73] Improve introduction to using `flexmeasures add source` Signed-off-by: F.N. Claessen --- documentation/tut/toy-example-from-scratch.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/documentation/tut/toy-example-from-scratch.rst b/documentation/tut/toy-example-from-scratch.rst index 70f8d5a09..8dd4a064e 100644 --- a/documentation/tut/toy-example-from-scratch.rst +++ b/documentation/tut/toy-example-from-scratch.rst @@ -361,7 +361,12 @@ First, we'll create a new csv file with solar forecasts (MW, see the setup for s $ ${TOMORROW}T22:00:00,0.0 $ ${TOMORROW}T23:00:00,0.0" > solar-tomorrow.csv -Then, we register a new forecaster and read in the created CSV file as beliefs data: +Then, we read in the created CSV file as beliefs data. +This time, different to above, we want to use a new data source (not the user) ― it represents whoever is making these solar production forecasts. +We create that data source first, so we can tell `flexmeasures add beliefs` to use it. +Setting the data source type to "forecaster" helps FlexMeasures to visualize distinguish its data from e.g. schedules and measurements. + +.. note:: The ``flexmeasures add source`` command also allows to set a model and version, so sources can be distinguished in more detail. But that is not the point of this tutorial. See ``flexmeasures add source --help``. .. code-block:: console From 5f99e631e5a779a615a12da27a92e5c831e4784b Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 9 Mar 2023 10:18:38 +0100 Subject: [PATCH 37/73] Introduce follow-up solar example Signed-off-by: F.N. Claessen --- documentation/tut/toy-example-from-scratch.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/documentation/tut/toy-example-from-scratch.rst b/documentation/tut/toy-example-from-scratch.rst index 8dd4a064e..dead6d1cd 100644 --- a/documentation/tut/toy-example-from-scratch.rst +++ b/documentation/tut/toy-example-from-scratch.rst @@ -330,6 +330,8 @@ Recall that we only asked for a 12 hour schedule here. We started our schedule * Take into account solar production --------------------------------------- +So far we haven't taken into account any other devices that consume or produce electricity. We'll now add solar production forecasts and reschedule, to see the effect of solar on the available headroom for the battery. + First, we'll create a new csv file with solar forecasts (MW, see the setup for sensor 4 above) for tomorrow. .. code-block:: console From 1c04f5f9042bfaf1bc0063a859604eceb31c8980 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 9 Mar 2023 11:50:42 +0100 Subject: [PATCH 38/73] Refactor: rename "scheduling script" to "scheduler" and "forecasting script" to "forecaster" Signed-off-by: F.N. Claessen --- documentation/tut/forecasting_scheduling.rst | 2 +- flexmeasures/api/v1_1/implementations.py | 2 +- flexmeasures/api/v1_3/tests/test_api_v1_3.py | 2 +- flexmeasures/api/v1_3/tests/test_api_v1_3_fresh_db.py | 2 +- flexmeasures/data/models/data_sources.py | 8 ++++---- flexmeasures/data/queries/analytics.py | 10 +++++----- flexmeasures/data/scripts/data_gen.py | 4 ++-- flexmeasures/data/services/scheduling.py | 4 ++-- flexmeasures/data/tests/test_scheduling_jobs.py | 6 +++--- .../data/tests/test_scheduling_jobs_fresh_db.py | 4 ++-- 10 files changed, 22 insertions(+), 22 deletions(-) diff --git a/documentation/tut/forecasting_scheduling.rst b/documentation/tut/forecasting_scheduling.rst index 9c0453ced..d2c545f5b 100644 --- a/documentation/tut/forecasting_scheduling.rst +++ b/documentation/tut/forecasting_scheduling.rst @@ -58,7 +58,7 @@ In FlexMeasures, the usual way of creating forecasting jobs would be right in th So technically, you don't have to do anything to keep fresh forecasts. The decision which horizons to forecast is currently also taken by FlexMeasures. For power data, FlexMeasures makes this decision depending on the asset resolution. For instance, a resolution of 15 minutes leads to forecast horizons of 1, 6, 24 and 48 hours. For price data, FlexMeasures chooses to forecast prices forward 24 and 48 hours -These are decent defaults, and fixing them has the advantage that scheduling scripts (see below) will know what to expect. However, horizons will probably become more configurable in the near future of FlexMeasures. +These are decent defaults, and fixing them has the advantage that schedulers (see below) will know what to expect. However, horizons will probably become more configurable in the near future of FlexMeasures. You can also add forecasting jobs directly via the CLI. We explain this practice in the next section. diff --git a/flexmeasures/api/v1_1/implementations.py b/flexmeasures/api/v1_1/implementations.py index 37a59f270..912f8d8f6 100644 --- a/flexmeasures/api/v1_1/implementations.py +++ b/flexmeasures/api/v1_1/implementations.py @@ -255,7 +255,7 @@ def get_prognosis_response( belief_time_window = (None, prior) # Check the user's intention first, fall back to schedules, then forecasts, then other data from script - source_types = ["user", "scheduling script", "forecasting script", "script"] + source_types = ["user", "scheduler", "forecaster", "script"] return collect_connection_and_value_groups( unit, diff --git a/flexmeasures/api/v1_3/tests/test_api_v1_3.py b/flexmeasures/api/v1_3/tests/test_api_v1_3.py index 12887972a..3ac080ac0 100644 --- a/flexmeasures/api/v1_3/tests/test_api_v1_3.py +++ b/flexmeasures/api/v1_3/tests/test_api_v1_3.py @@ -93,7 +93,7 @@ def test_post_udi_event_and_get_device_message( job.refresh() # catch meta info that was added on this very instance data_source_info = job.meta.get("data_source_info") scheduler_source = DataSource.query.filter_by( - type="scheduling script", **data_source_info + type="scheduler", **data_source_info ).one_or_none() assert ( scheduler_source is not None diff --git a/flexmeasures/api/v1_3/tests/test_api_v1_3_fresh_db.py b/flexmeasures/api/v1_3/tests/test_api_v1_3_fresh_db.py index 707d7ab6e..ea53f0b47 100644 --- a/flexmeasures/api/v1_3/tests/test_api_v1_3_fresh_db.py +++ b/flexmeasures/api/v1_3/tests/test_api_v1_3_fresh_db.py @@ -56,7 +56,7 @@ def test_post_udi_event_and_get_device_message_with_unknown_prices( # check results are not in the database scheduler_source = DataSource.query.filter_by( - name="Seita", type="scheduling script" + name="Seita", type="scheduler" ).one_or_none() assert ( scheduler_source is None diff --git a/flexmeasures/data/models/data_sources.py b/flexmeasures/data/models/data_sources.py index e51c4fc19..042195ab9 100644 --- a/flexmeasures/data/models/data_sources.py +++ b/flexmeasures/data/models/data_sources.py @@ -15,7 +15,7 @@ class DataSource(db.Model, tb.BeliefSourceDBMixin): __tablename__ = "data_source" __table_args__ = (db.UniqueConstraint("name", "user_id", "model", "version"),) - # The type of data source (e.g. user, forecasting script or scheduling script) + # The type of data source (e.g. user, forecaster or scheduler) type = db.Column(db.String(80), default="") # The id of the user source (can link e.g. to fm_user table) @@ -53,9 +53,9 @@ def label(self): """Human-readable label (preferably not starting with a capital letter, so it can be used in a sentence).""" if self.type == "user": return f"data entered by user {self.user.username}" # todo: give users a display name - elif self.type == "forecasting script": + elif self.type == "forecaster": return f"forecast by {self.name}" # todo: give DataSource an optional db column to persist versioned models separately to the name of the data source? - elif self.type == "scheduling script": + elif self.type == "scheduler": return f"schedule by {self.name}" elif self.type == "crawling script": return f"data retrieved from {self.name}" @@ -70,7 +70,7 @@ def description(self): For example: - >>> DataSource("Seita", type="forecasting script", model="naive", version="1.2").description + >>> DataSource("Seita", type="forecaster", model="naive", version="1.2").description <<< "Seita's naive model v1.2.0" """ diff --git a/flexmeasures/data/queries/analytics.py b/flexmeasures/data/queries/analytics.py index 3183678b9..08967be27 100644 --- a/flexmeasures/data/queries/analytics.py +++ b/flexmeasures/data/queries/analytics.py @@ -55,7 +55,7 @@ def get_power_data( end=query_window[-1], resolution=resolution, belief_horizon_window=(None, timedelta(hours=0)), - exclude_source_types=["scheduling script"], + exclude_source_types=["scheduler"], ) if showing_individual_traces_for == "power": power_bdf = resource.power_data @@ -87,7 +87,7 @@ def get_power_data( end=query_window[-1], resolution=resolution, belief_horizon_window=(forecast_horizon, None), - exclude_source_types=["scheduling script"], + exclude_source_types=["scheduler"], ).aggregate_power_data power_forecast_df: pd.DataFrame = simplify_index( power_forecast_bdf, index_levels_to_columns=["belief_horizon", "source"] @@ -103,7 +103,7 @@ def get_power_data( end=query_window[-1], resolution=resolution, belief_horizon_window=(None, None), - source_types=["scheduling script"], + source_types=["scheduler"], ) if showing_individual_traces_for == "schedules": power_schedule_bdf = resource.power_data @@ -205,7 +205,7 @@ def get_prices_data( resolution=resolution, horizons_at_least=forecast_horizon, horizons_at_most=None, - source_types=["user", "forecasting script", "script"], + source_types=["user", "forecaster", "script"], ) price_forecast_df: pd.DataFrame = simplify_index( price_forecast_bdf, index_levels_to_columns=["belief_horizon", "source"] @@ -297,7 +297,7 @@ def get_weather_data( resolution=resolution, horizons_at_least=forecast_horizon, horizons_at_most=None, - source_types=["user", "forecasting script", "script"], + source_types=["user", "forecaster", "script"], sum_multiple=False, ) weather_forecast_df_dict: Dict[str, pd.DataFrame] = {} diff --git a/flexmeasures/data/scripts/data_gen.py b/flexmeasures/data/scripts/data_gen.py index 9da2b336c..e527dc803 100644 --- a/flexmeasures/data/scripts/data_gen.py +++ b/flexmeasures/data/scripts/data_gen.py @@ -37,8 +37,8 @@ def add_default_data_sources(db: SQLAlchemy): for source_name, source_type in ( ("Seita", "demo script"), - ("Seita", "forecasting script"), - ("Seita", "scheduling script"), + ("Seita", "forecaster"), + ("Seita", "scheduler"), ): source = DataSource.query.filter( and_(DataSource.name == source_name, DataSource.type == source_type) diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py index f88dc3c23..4118ecaf7 100644 --- a/flexmeasures/data/services/scheduling.py +++ b/flexmeasures/data/services/scheduling.py @@ -125,7 +125,7 @@ def make_schedule( data_source_name=data_source_info["name"], data_source_model=data_source_info["model"], data_source_version=data_source_info["version"], - data_source_type="scheduling script", + data_source_type="scheduler", ) # saving info on the job, so the API for a job can look the data up @@ -273,7 +273,7 @@ def get_data_source_for_job(job: Job | None) -> DataSource | None: ) scheduler_sources = ( DataSource.query.filter_by( - type="scheduling script", + type="scheduler", **data_source_info, ) .order_by(DataSource.version.desc()) diff --git a/flexmeasures/data/tests/test_scheduling_jobs.py b/flexmeasures/data/tests/test_scheduling_jobs.py index 517273c39..3470f878a 100644 --- a/flexmeasures/data/tests/test_scheduling_jobs.py +++ b/flexmeasures/data/tests/test_scheduling_jobs.py @@ -29,7 +29,7 @@ def test_scheduling_a_battery(db, app, add_battery_assets, setup_test_data): assert ( DataSource.query.filter_by( - name="FlexMeasures", type="scheduling script" + name="FlexMeasures", type="scheduler" ).one_or_none() is None ) # Make sure the scheduler data source isn't there @@ -47,7 +47,7 @@ def test_scheduling_a_battery(db, app, add_battery_assets, setup_test_data): work_on_rq(app.queues["scheduling"], exc_handler=exception_reporter) scheduler_source = DataSource.query.filter_by( - name="Seita", type="scheduling script" + name="Seita", type="scheduler" ).one_or_none() assert ( scheduler_source is not None @@ -125,7 +125,7 @@ def test_assigning_custom_scheduler(db, app, add_battery_assets, is_path: bool): assert finished_job.meta["data_source_info"]["model"] == scheduler_specs["class"] scheduler_source = DataSource.query.filter_by( - type="scheduling script", + type="scheduler", **finished_job.meta["data_source_info"], ).one_or_none() assert ( diff --git a/flexmeasures/data/tests/test_scheduling_jobs_fresh_db.py b/flexmeasures/data/tests/test_scheduling_jobs_fresh_db.py index 5b8e8dfcd..c54add758 100644 --- a/flexmeasures/data/tests/test_scheduling_jobs_fresh_db.py +++ b/flexmeasures/data/tests/test_scheduling_jobs_fresh_db.py @@ -33,7 +33,7 @@ def test_scheduling_a_charging_station( soc_targets = [dict(datetime=target_datetime.isoformat(), value=target_soc)] assert ( - DataSource.query.filter_by(name="Seita", type="scheduling script").one_or_none() + DataSource.query.filter_by(name="Seita", type="scheduler").one_or_none() is None ) # Make sure the scheduler data source isn't there @@ -51,7 +51,7 @@ def test_scheduling_a_charging_station( work_on_rq(app.queues["scheduling"], exc_handler=exception_reporter) scheduler_source = DataSource.query.filter_by( - name="Seita", type="scheduling script" + name="Seita", type="scheduler" ).one_or_none() assert ( scheduler_source is not None From 9340613efd2fb08d31b8915445dd2e37ec9e7bb5 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 9 Mar 2023 12:34:55 +0100 Subject: [PATCH 39/73] Clarify event start domain setting Signed-off-by: F.N. Claessen --- flexmeasures/data/models/charts/belief_charts.py | 1 + 1 file changed, 1 insertion(+) diff --git a/flexmeasures/data/models/charts/belief_charts.py b/flexmeasures/data/models/charts/belief_charts.py index 253fa8bce..2c090aa43 100644 --- a/flexmeasures/data/models/charts/belief_charts.py +++ b/flexmeasures/data/models/charts/belief_charts.py @@ -105,6 +105,7 @@ def chart_for_multiple_sensors( "unit": "yearmonthdatehoursminutesseconds", "step": minimum_non_zero_resolution.total_seconds(), } + # If a time window was set explicitly, adjust the domain to show the full window regardless of available data if event_starts_after and event_ends_before: event_start_field_definition["scale"] = { "domain": [ From 67ad86c0e122ce291a2f3e866debc87415f93ed2 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 9 Mar 2023 12:47:04 +0100 Subject: [PATCH 40/73] Raise instead of warn in case toy account already exists Signed-off-by: F.N. Claessen --- flexmeasures/cli/data_add.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index 57e901c9b..8d1ac2910 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -1095,7 +1095,10 @@ def add_toy_account(kind: str, name: str): # make an account (if not exist) account = Account.query.filter(Account.name == name).one_or_none() if account: - click.echo(f"Account already exists: {account}") + click.echo( + f"Account '{account}' already exists. Use `flexmeasures delete account --id {account.id}` to remove it first." + ) + raise click.Abort # make an account user (account-admin?) email = "toy-user@flexmeasures.io" user = User.query.filter_by(email=email).one_or_none() From 7c969499bea4c05b28e778efb68156ca36a46704 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 21 Mar 2023 17:35:30 +0100 Subject: [PATCH 41/73] Clarify loop over kwargs Signed-off-by: F.N. Claessen --- flexmeasures/cli/data_add.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index 8d1ac2910..f1f44e88a 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -1255,7 +1255,14 @@ def get_or_create_model( if hasattr(model_attribute, "type") and isinstance(model_attribute.type, JSON): filter_json_kwargs[kw] = filter_by_kwargs.pop(kw) elif callable(arg) and isinstance(model_attribute.type, String): + # Callables are stored in the database by their name + # e.g. knowledge_horizon_fnc = x_days_ago_at_y_oclock + # is stored as "x_days_ago_at_y_oclock" filter_by_kwargs[kw] = filter_by_kwargs[kw].__name__ + else: + # The kw is already present in filter_by_kwargs and doesn't need to be adapted + # i.e. it can be used as an argument to .filter_by() + pass # See if the model already exists as a db row model_query = model_class.query.filter_by(**filter_by_kwargs) From 6c25137e75e089af3a8015aef2e763b3aaca4b86 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 21 Mar 2023 17:37:39 +0100 Subject: [PATCH 42/73] black Signed-off-by: F.N. Claessen --- flexmeasures/data/tests/test_scheduling_jobs.py | 4 +--- flexmeasures/data/tests/test_scheduling_jobs_fresh_db.py | 3 +-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/flexmeasures/data/tests/test_scheduling_jobs.py b/flexmeasures/data/tests/test_scheduling_jobs.py index 3470f878a..c7fc313c6 100644 --- a/flexmeasures/data/tests/test_scheduling_jobs.py +++ b/flexmeasures/data/tests/test_scheduling_jobs.py @@ -28,9 +28,7 @@ def test_scheduling_a_battery(db, app, add_battery_assets, setup_test_data): resolution = timedelta(minutes=15) assert ( - DataSource.query.filter_by( - name="FlexMeasures", type="scheduler" - ).one_or_none() + DataSource.query.filter_by(name="FlexMeasures", type="scheduler").one_or_none() is None ) # Make sure the scheduler data source isn't there diff --git a/flexmeasures/data/tests/test_scheduling_jobs_fresh_db.py b/flexmeasures/data/tests/test_scheduling_jobs_fresh_db.py index c54add758..ca125614f 100644 --- a/flexmeasures/data/tests/test_scheduling_jobs_fresh_db.py +++ b/flexmeasures/data/tests/test_scheduling_jobs_fresh_db.py @@ -33,8 +33,7 @@ def test_scheduling_a_charging_station( soc_targets = [dict(datetime=target_datetime.isoformat(), value=target_soc)] assert ( - DataSource.query.filter_by(name="Seita", type="scheduler").one_or_none() - is None + DataSource.query.filter_by(name="Seita", type="scheduler").one_or_none() is None ) # Make sure the scheduler data source isn't there job = create_scheduling_job( From ab000f2b135a0b21bc062cd99c64028906aad626 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 21 Mar 2023 17:40:34 +0100 Subject: [PATCH 43/73] Merge steps Signed-off-by: F.N. Claessen --- flexmeasures/data/models/charts/belief_charts.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/flexmeasures/data/models/charts/belief_charts.py b/flexmeasures/data/models/charts/belief_charts.py index 2c090aa43..6c11ef587 100644 --- a/flexmeasures/data/models/charts/belief_charts.py +++ b/flexmeasures/data/models/charts/belief_charts.py @@ -88,13 +88,10 @@ def chart_for_multiple_sensors( event_ends_before: datetime | None = None, **override_chart_specs: dict, ): - # Unpack nested sensors - sensors = flatten_unique(sensors_to_show) - # Determine the shared data resolution condition = list( sensor.event_resolution - for sensor in sensors + for sensor in flatten_unique(sensors_to_show) if sensor.event_resolution > timedelta(0) ) minimum_non_zero_resolution = min(condition) if any(condition) else timedelta(0) From 8b6c2c45b35a539580014c9e9dd0abbc8b2470ba Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 21 Mar 2023 17:50:03 +0100 Subject: [PATCH 44/73] Clarify when we choose not to show the sensor name Signed-off-by: F.N. Claessen --- flexmeasures/data/models/charts/belief_charts.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flexmeasures/data/models/charts/belief_charts.py b/flexmeasures/data/models/charts/belief_charts.py index 6c11ef587..09782470e 100644 --- a/flexmeasures/data/models/charts/belief_charts.py +++ b/flexmeasures/data/models/charts/belief_charts.py @@ -43,6 +43,7 @@ def bar_chart( } chart_specs = { "description": "A simple bar chart showing sensor data.", + # the sensor type is already shown as the y-axis title (avoid redundant info) "title": capitalize(sensor.name) if sensor.name != sensor.sensor_type else None, "layer": [ { @@ -196,6 +197,7 @@ def chart_for_multiple_sensors( [ f"{capitalize(sensor.name)}" for sensor in row_sensors + # the sensor type is already shown as the y-axis title (avoid redundant info) if sensor.name != sensor.sensor_type ] ), From 452e3c49fae68dd31727276581fdd6c37a000f1f Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 24 Mar 2023 10:22:17 +0100 Subject: [PATCH 45/73] Add return type annotations Signed-off-by: F.N. Claessen --- flexmeasures/data/schemas/generic_assets.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flexmeasures/data/schemas/generic_assets.py b/flexmeasures/data/schemas/generic_assets.py index b7aa0b86a..181318f68 100644 --- a/flexmeasures/data/schemas/generic_assets.py +++ b/flexmeasures/data/schemas/generic_assets.py @@ -17,13 +17,13 @@ class JSON(fields.Field): - def _deserialize(self, value, attr, data, **kwargs): + def _deserialize(self, value, attr, data, **kwargs) -> dict: try: return json.loads(value) except ValueError: raise ValidationError("Not a valid JSON string.") - def _serialize(self, value, attr, data, **kwargs): + def _serialize(self, value, attr, data, **kwargs) -> str: return json.dumps(value) From 6c76e936bbf8173951c1b42ef6bd8e0c95707ae4 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 24 Mar 2023 12:26:13 +0100 Subject: [PATCH 46/73] Add default sensors_to_show attribute to test asset Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/tests/test_assets_api.py | 3 +++ flexmeasures/conftest.py | 1 + 2 files changed, 4 insertions(+) diff --git a/flexmeasures/api/v3_0/tests/test_assets_api.py b/flexmeasures/api/v3_0/tests/test_assets_api.py index 6d2be3581..772b9e38d 100644 --- a/flexmeasures/api/v3_0/tests/test_assets_api.py +++ b/flexmeasures/api/v3_0/tests/test_assets_api.py @@ -179,6 +179,9 @@ def test_alter_an_asset_with_json_attributes( auth_token = prosumer1.get_auth_token() with AccountContext("Test Prosumer Account") as prosumer: prosumer_asset = prosumer.generic_assets[0] + assert prosumer_asset.attributes[ + "sensors_to_show" + ] # make sure we run this test on an asset with a non-empty sensors_to_show attribute asset_edit_response = client.patch( url_for("AssetAPI:patch", id=prosumer_asset.id), headers={"content-type": "application/json", "Authorization": auth_token}, diff --git a/flexmeasures/conftest.py b/flexmeasures/conftest.py index 285f2da4e..f6ce2a2d7 100644 --- a/flexmeasures/conftest.py +++ b/flexmeasures/conftest.py @@ -302,6 +302,7 @@ def create_generic_assets(db, setup_generic_asset_types, setup_accounts): name="Test grid connected battery storage", generic_asset_type=setup_generic_asset_types["battery"], owner=setup_accounts["Prosumer"], + attributes={"some-attribute": "some-value", "sensors_to_show": [1, 2]}, ) db.session.add(test_battery) test_wind_turbine = GenericAsset( From 51d1068159bcdbfd6380ab05ecc73a98baf62368 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 24 Mar 2023 12:26:43 +0100 Subject: [PATCH 47/73] Add test cases for bad sensors_to_show attribute Signed-off-by: F.N. Claessen --- .../api/v3_0/tests/test_assets_api.py | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_assets_api.py b/flexmeasures/api/v3_0/tests/test_assets_api.py index 772b9e38d..0a7aca430 100644 --- a/flexmeasures/api/v3_0/tests/test_assets_api.py +++ b/flexmeasures/api/v3_0/tests/test_assets_api.py @@ -147,15 +147,25 @@ def test_alter_an_asset(client, setup_api_test_data, setup_accounts): @pytest.mark.parametrize( - "bad_json_str", + "bad_json_str, error_msg", [ - None, - "{", - '{"hallo": world}', + (None, "may not be null"), + ("{", "Not a valid JSON"), + ('{"hallo": world}', "Not a valid JSON"), + ('{"sensors_to_show": [0, 1]}', "No sensor found"), # no sensor with ID 0 + ('{"sensors_to_show": [1, [0, 2]]}', "No sensor found"), # no sensor with ID 0 + ( + '{"sensors_to_show": [1, [2, [3, 4]]]}', + "should only contain", + ), # nesting level max 1 + ( + '{"sensors_to_show": [1, "2"]}', + "should only contain", + ), # non-integer sensor ID ], ) def test_alter_an_asset_with_bad_json_attributes( - client, setup_api_test_data, setup_accounts, bad_json_str + client, setup_api_test_data, setup_accounts, bad_json_str, error_msg ): """Check whether updating an asset's attributes with a badly structured JSON fails.""" with UserContext("test_prosumer_user@seita.nl") as prosumer1: @@ -169,6 +179,7 @@ def test_alter_an_asset_with_bad_json_attributes( ) print(f"Editing Response: {asset_edit_response.json}") assert asset_edit_response.status_code == 422 + assert error_msg in asset_edit_response.json["message"]["json"]["attributes"][0] def test_alter_an_asset_with_json_attributes( From 8c1d013992b5eb6f0724207332db018ab3467294 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 24 Mar 2023 12:27:24 +0100 Subject: [PATCH 48/73] Validate sensors_to_show attribute Signed-off-by: F.N. Claessen --- flexmeasures/data/schemas/generic_assets.py | 28 +++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/flexmeasures/data/schemas/generic_assets.py b/flexmeasures/data/schemas/generic_assets.py index 181318f68..89dec112d 100644 --- a/flexmeasures/data/schemas/generic_assets.py +++ b/flexmeasures/data/schemas/generic_assets.py @@ -14,6 +14,7 @@ ) from flexmeasures.auth.policy import user_has_admin_access from flexmeasures.cli import is_running as running_as_cli +from flexmeasures.utils.coding_utils import flatten_unique class JSON(fields.Field): @@ -77,6 +78,33 @@ def validate_account(self, account_id: int): "User is not allowed to create assets for this account." ) + @validates("attributes") + def validate_attributes(self, attributes: dict): + sensors_to_show = attributes.get("sensors_to_show", []) + + # Check type + if not isinstance(sensors_to_show, list): + raise ValidationError( + "sensors_to_show should be a list." + ) + for sensor_listing in sensors_to_show: + if not isinstance(sensor_listing, (int, list)): + raise ValidationError( + "sensors_to_show should only contain sensor IDs (integers) or lists thereof." + ) + if isinstance(sensor_listing, list): + for sensor_id in sensor_listing: + if not isinstance(sensor_id, int): + raise ValidationError( + "sensors_to_show should only contain sensor IDs (integers) or lists thereof." + ) + + # Check whether IDs represent accessible sensors + from flexmeasures.data.schemas import SensorIdField + sensor_ids = flatten_unique(sensors_to_show) + for sensor_id in sensor_ids: + SensorIdField().deserialize(sensor_id) + class GenericAssetTypeSchema(ma.SQLAlchemySchema): """ From 1ae0b431d91645fc6099fc239c6e8ab608a20291 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 24 Mar 2023 16:10:50 +0100 Subject: [PATCH 49/73] Simplify code lines Signed-off-by: F.N. Claessen --- flexmeasures/data/models/generic_assets.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/flexmeasures/data/models/generic_assets.py b/flexmeasures/data/models/generic_assets.py index 92c5b7ab6..facdf2c7b 100644 --- a/flexmeasures/data/models/generic_assets.py +++ b/flexmeasures/data/models/generic_assets.py @@ -299,8 +299,7 @@ def chart( :param dataset_name: optionally name the dataset used in the chart (the default name is sensor_) :returns: JSON string defining vega-lite chart specs """ - sensors_to_show = self.sensors_to_show - sensors = flatten_unique(sensors_to_show) + sensors = flatten_unique(self.sensors_to_show) for sensor in sensors: sensor.sensor_type = sensor.get_attribute("sensor_type", sensor.name) @@ -313,7 +312,7 @@ def chart( kwargs["event_ends_before"] = event_ends_before chart_specs = chart_type_to_chart_specs( chart_type, - sensors_to_show=sensors_to_show, + sensors_to_show=self.sensors_to_show, dataset_name=dataset_name, **kwargs, ) From 770fef6832de263f8d27183da7521b223a12019f Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 24 Mar 2023 16:13:23 +0100 Subject: [PATCH 50/73] black Signed-off-by: F.N. Claessen --- flexmeasures/data/schemas/generic_assets.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/flexmeasures/data/schemas/generic_assets.py b/flexmeasures/data/schemas/generic_assets.py index 89dec112d..469c29951 100644 --- a/flexmeasures/data/schemas/generic_assets.py +++ b/flexmeasures/data/schemas/generic_assets.py @@ -84,9 +84,7 @@ def validate_attributes(self, attributes: dict): # Check type if not isinstance(sensors_to_show, list): - raise ValidationError( - "sensors_to_show should be a list." - ) + raise ValidationError("sensors_to_show should be a list.") for sensor_listing in sensors_to_show: if not isinstance(sensor_listing, (int, list)): raise ValidationError( @@ -101,6 +99,7 @@ def validate_attributes(self, attributes: dict): # Check whether IDs represent accessible sensors from flexmeasures.data.schemas import SensorIdField + sensor_ids = flatten_unique(sensors_to_show) for sensor_id in sensor_ids: SensorIdField().deserialize(sensor_id) From 0611afded5447c2f4cd6ca34ee12121e1491da18 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 24 Mar 2023 16:21:00 +0100 Subject: [PATCH 51/73] Refactor: move util function Signed-off-by: F.N. Claessen --- flexmeasures/cli/data_add.py | 60 +---------------------- flexmeasures/data/services/__init__.py | 1 + flexmeasures/data/services/utils.py | 66 ++++++++++++++++++++++++++ 3 files changed, 69 insertions(+), 58 deletions(-) create mode 100644 flexmeasures/data/services/utils.py diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index ef1d04c89..1c974261c 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -2,7 +2,7 @@ from __future__ import annotations from datetime import datetime, timedelta -from typing import Dict, List, Optional, Tuple, Type +from typing import Dict, List, Optional, Tuple import json from marshmallow import validate @@ -12,7 +12,6 @@ from flask.cli import with_appcontext import click import getpass -from sqlalchemy import cast, literal, JSON, String from sqlalchemy.exc import IntegrityError from timely_beliefs.sensors.func_store.knowledge_horizons import x_days_ago_at_y_oclock import timely_beliefs as tb @@ -58,6 +57,7 @@ get_or_create_source, get_source_or_none, ) +from flexmeasures.data.services import get_or_create_model from flexmeasures.utils import flexmeasures_inflection from flexmeasures.utils.time_utils import server_now from flexmeasures.utils.unit_utils import convert_units, ur @@ -1271,59 +1271,3 @@ def parse_source(source): else: _source = get_or_create_source(source, source_type="CLI script") return _source - - -def get_or_create_model( - model_class: Type[GenericAsset | GenericAssetType | Sensor], **kwargs -) -> GenericAsset | GenericAssetType | Sensor: - """Get a model from the database or add it if it's missing. - - For example: - >>> weather_station_type = get_or_create_model( - >>> GenericAssetType, - >>> name="weather station", - >>> description="A weather station with various sensors.", - >>> ) - """ - - # unpack custom initialization parameters that map to multiple database columns - init_kwargs = kwargs.copy() - lookup_kwargs = kwargs.copy() - if "knowledge_horizon" in kwargs: - ( - lookup_kwargs["knowledge_horizon_fnc"], - lookup_kwargs["knowledge_horizon_par"], - ) = lookup_kwargs.pop("knowledge_horizon") - - # Find out which attributes are dictionaries mapped to JSON database columns, - # or callables mapped to string database columns (by their name) - filter_json_kwargs = {} - filter_by_kwargs = lookup_kwargs.copy() - for kw, arg in lookup_kwargs.items(): - model_attribute = getattr(model_class, kw) - if hasattr(model_attribute, "type") and isinstance(model_attribute.type, JSON): - filter_json_kwargs[kw] = filter_by_kwargs.pop(kw) - elif callable(arg) and isinstance(model_attribute.type, String): - # Callables are stored in the database by their name - # e.g. knowledge_horizon_fnc = x_days_ago_at_y_oclock - # is stored as "x_days_ago_at_y_oclock" - filter_by_kwargs[kw] = filter_by_kwargs[kw].__name__ - else: - # The kw is already present in filter_by_kwargs and doesn't need to be adapted - # i.e. it can be used as an argument to .filter_by() - pass - - # See if the model already exists as a db row - model_query = model_class.query.filter_by(**filter_by_kwargs) - for kw, arg in filter_json_kwargs.items(): - model_query = model_query.filter( - cast(getattr(model_class, kw), String) == cast(literal(arg, JSON()), String) - ) - model = model_query.one_or_none() - - # Create the model and add it to the database if it didn't already exist - if model is None: - model = model_class(**init_kwargs) - click.echo(f"Created {model}") - db.session.add(model) - return model diff --git a/flexmeasures/data/services/__init__.py b/flexmeasures/data/services/__init__.py index e69de29bb..9ec67c3f8 100644 --- a/flexmeasures/data/services/__init__.py +++ b/flexmeasures/data/services/__init__.py @@ -0,0 +1 @@ +from utils import get_or_create_model diff --git a/flexmeasures/data/services/utils.py b/flexmeasures/data/services/utils.py new file mode 100644 index 000000000..96dd9b9a4 --- /dev/null +++ b/flexmeasures/data/services/utils.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +from typing import Type + +import click +from sqlalchemy import JSON, String, cast, literal + +from flexmeasures import Sensor +from flexmeasures.data import db +from flexmeasures.data.models.generic_assets import GenericAsset, GenericAssetType + + +def get_or_create_model( + model_class: Type[GenericAsset | GenericAssetType | Sensor], **kwargs +) -> GenericAsset | GenericAssetType | Sensor: + """Get a model from the database or add it if it's missing. + + For example: + >>> weather_station_type = get_or_create_model( + >>> GenericAssetType, + >>> name="weather station", + >>> description="A weather station with various sensors.", + >>> ) + """ + + # unpack custom initialization parameters that map to multiple database columns + init_kwargs = kwargs.copy() + lookup_kwargs = kwargs.copy() + if "knowledge_horizon" in kwargs: + ( + lookup_kwargs["knowledge_horizon_fnc"], + lookup_kwargs["knowledge_horizon_par"], + ) = lookup_kwargs.pop("knowledge_horizon") + + # Find out which attributes are dictionaries mapped to JSON database columns, + # or callables mapped to string database columns (by their name) + filter_json_kwargs = {} + filter_by_kwargs = lookup_kwargs.copy() + for kw, arg in lookup_kwargs.items(): + model_attribute = getattr(model_class, kw) + if hasattr(model_attribute, "type") and isinstance(model_attribute.type, JSON): + filter_json_kwargs[kw] = filter_by_kwargs.pop(kw) + elif callable(arg) and isinstance(model_attribute.type, String): + # Callables are stored in the database by their name + # e.g. knowledge_horizon_fnc = x_days_ago_at_y_oclock + # is stored as "x_days_ago_at_y_oclock" + filter_by_kwargs[kw] = filter_by_kwargs[kw].__name__ + else: + # The kw is already present in filter_by_kwargs and doesn't need to be adapted + # i.e. it can be used as an argument to .filter_by() + pass + + # See if the model already exists as a db row + model_query = model_class.query.filter_by(**filter_by_kwargs) + for kw, arg in filter_json_kwargs.items(): + model_query = model_query.filter( + cast(getattr(model_class, kw), String) == cast(literal(arg, JSON()), String) + ) + model = model_query.one_or_none() + + # Create the model and add it to the database if it didn't already exist + if model is None: + model = model_class(**init_kwargs) + click.echo(f"Created {model}") + db.session.add(model) + return model From c16b01bcba8709154ddcfd4a3fd1c4d2c714099c Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 27 Mar 2023 11:06:24 +0200 Subject: [PATCH 52/73] flake8 Signed-off-by: F.N. Claessen --- flexmeasures/data/services/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/services/__init__.py b/flexmeasures/data/services/__init__.py index 9ec67c3f8..71b63b57b 100644 --- a/flexmeasures/data/services/__init__.py +++ b/flexmeasures/data/services/__init__.py @@ -1 +1 @@ -from utils import get_or_create_model +from utils import get_or_create_model # noqa F401 From 838d9f369689cdebe85b546d455c7b79adc08b1c Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 27 Mar 2023 11:14:14 +0200 Subject: [PATCH 53/73] Changelog entry plus upgrade instructions Signed-off-by: F.N. Claessen --- documentation/changelog.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index dacb07dc7..19aed13dd 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -8,9 +8,12 @@ v0.13.0 | April XX, 2023 .. warning:: The API endpoint (`[POST] /sensors/(id)/schedules/trigger `_) to make new schedules sunsets the deprecated (since v0.12) storage flexibility parameters (they move to the ``flex-model`` parameter group), as well as the parameters describing other sensors (they move to ``flex-context``). +.. warning:: Upgrading to this version requires running ``flexmeasures db upgrade`` (you can create a backup first with ``flexmeasures db-ops dump``). + New features ------------- * Keyboard control over replay [see `PR #562 `_] +* Overlay charts (e.g. power profiles) on the asset page using the `sensors_to_show` attribute, and distinguish plots by source (different trace), sensor (different color) and source type (different stroke dash) [see `PR #534 `_] * The ``FLEXMEASURES_MAX_PLANNING_HORIZON`` config setting can also be set as an integer number of planning steps rather than just as a fixed duration, which makes it possible to schedule further ahead in coarser time steps [see `PR #583 `_] * Different text styles for CLI output for errors, warnings or success messages. [see `PR #609 `_] From 1a4008a5aef73076dd87af9d87cca94a8176eb76 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 27 Mar 2023 11:20:07 +0200 Subject: [PATCH 54/73] Add documentation for nested sensors_to_show attribute Signed-off-by: F.N. Claessen --- documentation/views/asset-data.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/documentation/views/asset-data.rst b/documentation/views/asset-data.rst index 073e0786f..9631a4e26 100644 --- a/documentation/views/asset-data.rst +++ b/documentation/views/asset-data.rst @@ -17,6 +17,7 @@ This includes the possibility to specify which sensors the asset page should sho | | +.. note:: It is possible to overlay data for multiple sensors, by setting the `sensors_to_show` attribute to a nested list. For example, ``{"sensor_to_show": [3, [2, 4]]}`` would show the data for sensor 4 laid over the data for sensor 2. .. note:: While it is possible to show an arbitrary number of sensors this way, we recommend showing only the most crucial ones for faster loading, less page scrolling, and generally, a quick grasp of what the asset is up to. .. note:: Asset attributes can be edited through the CLI as well, with the CLI command ``flexmeasures edit attribute``. From 6f5c72722564e3d93f774d2fa4e7ae9cb8fce29b Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 27 Mar 2023 11:23:49 +0200 Subject: [PATCH 55/73] Resolve import error Signed-off-by: F.N. Claessen --- flexmeasures/data/services/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/services/__init__.py b/flexmeasures/data/services/__init__.py index 71b63b57b..f83100bd1 100644 --- a/flexmeasures/data/services/__init__.py +++ b/flexmeasures/data/services/__init__.py @@ -1 +1 @@ -from utils import get_or_create_model # noqa F401 +from .utils import get_or_create_model # noqa F401 From 5e00455d1fde8cde1e65a546f5416165775e78ed Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 27 Mar 2023 11:25:20 +0200 Subject: [PATCH 56/73] redundant space Signed-off-by: F.N. Claessen --- flexmeasures/data/services/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/services/__init__.py b/flexmeasures/data/services/__init__.py index f83100bd1..0eeb70932 100644 --- a/flexmeasures/data/services/__init__.py +++ b/flexmeasures/data/services/__init__.py @@ -1 +1 @@ -from .utils import get_or_create_model # noqa F401 +from .utils import get_or_create_model # noqa F401 From b37fea86b7e5c9d3671c72c058e0bf088aecee5e Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 27 Mar 2023 11:31:32 +0200 Subject: [PATCH 57/73] Resolve circular import Signed-off-by: F.N. Claessen --- flexmeasures/cli/data_add.py | 2 +- flexmeasures/data/services/__init__.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index 1c974261c..05242cca3 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -57,7 +57,7 @@ get_or_create_source, get_source_or_none, ) -from flexmeasures.data.services import get_or_create_model +from flexmeasures.data.services.utils import get_or_create_model from flexmeasures.utils import flexmeasures_inflection from flexmeasures.utils.time_utils import server_now from flexmeasures.utils.unit_utils import convert_units, ur diff --git a/flexmeasures/data/services/__init__.py b/flexmeasures/data/services/__init__.py index 0eeb70932..e69de29bb 100644 --- a/flexmeasures/data/services/__init__.py +++ b/flexmeasures/data/services/__init__.py @@ -1 +0,0 @@ -from .utils import get_or_create_model # noqa F401 From cbf8e17740bf8b54bcc7d1f15843446ad35edb4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Thu, 30 Mar 2023 15:59:03 +0200 Subject: [PATCH 58/73] add a missing output style MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- flexmeasures/cli/data_add.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index 05242cca3..c4753b83a 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -338,7 +338,7 @@ def add_source(name: str, model: str, version: str, source_type: str): source_type=source_type, ) db.session.commit() - print(f"Added source {source.__repr__()}") + click.secho(f"Added source {source.__repr__()}", **MsgStyle.SUCCESS) @fm_add_data.command("beliefs") From 3d42a6f0709b2afc2e44af0e734e6767810b0c22 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 5 Apr 2023 10:51:17 +0200 Subject: [PATCH 59/73] Fix GH Issue #604 Signed-off-by: F.N. Claessen --- .../versions/a528c3c81506_unique_generic_sensor_ids.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/migrations/versions/a528c3c81506_unique_generic_sensor_ids.py b/flexmeasures/data/migrations/versions/a528c3c81506_unique_generic_sensor_ids.py index cbcb784f8..b83560d83 100644 --- a/flexmeasures/data/migrations/versions/a528c3c81506_unique_generic_sensor_ids.py +++ b/flexmeasures/data/migrations/versions/a528c3c81506_unique_generic_sensor_ids.py @@ -203,7 +203,7 @@ def upgrade_data(): sequence_name = "%s_id_seq" % t_sensors.name # Set next id for table seq to just after max id of all old sensors combined connection.execute( - "SELECT setval('%s', %s, true);" + "SELECT setval('%s', %s, false);" # is_called = False % (sequence_name, max_asset_id + max_market_id + max_weather_sensor_id + 1) ) From f90ded66b4e5d605aff1c8f6e3deb881a9b1acda Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sun, 9 Apr 2023 14:59:09 +0200 Subject: [PATCH 60/73] Correct spacing of header markings Signed-off-by: F.N. Claessen --- documentation/index.rst | 2 +- documentation/tut/toy-example-from-scratch.rst | 12 ++++++------ flexmeasures/cli/data_show.py | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/documentation/index.rst b/documentation/index.rst index 9e9920e82..7f80d1089 100644 --- a/documentation/index.rst +++ b/documentation/index.rst @@ -94,7 +94,7 @@ Your journey, from dipping your toes in the water towards being a happy FlexMeas -Where to start reading ? +Where to start reading? -------------------------- You (the reader) might be a user connecting with a FlexMeasures server or working on hosting FlexMeasures. Maybe you are planning to develop a plugin or even core functionality. In :ref:`getting_started`, we have some helpful tips how to dive into this documentation! diff --git a/documentation/tut/toy-example-from-scratch.rst b/documentation/tut/toy-example-from-scratch.rst index dead6d1cd..4fb9709d5 100644 --- a/documentation/tut/toy-example-from-scratch.rst +++ b/documentation/tut/toy-example-from-scratch.rst @@ -129,9 +129,9 @@ If you want, you can inspect what you created: $ flexmeasures show account --id 1 - ============================= - Account Toy Account (ID:1): - ============================= + =========================== + Account Toy Account (ID: 1) + =========================== Account has no roles. @@ -151,9 +151,9 @@ If you want, you can inspect what you created: $ flexmeasures show asset --id 3 - =========================== - Asset toy-battery (ID:3): - =========================== + ========================= + Asset toy-battery (ID: 3) + ========================= Type Location Attributes ------- ----------------- --------------------- diff --git a/flexmeasures/cli/data_show.py b/flexmeasures/cli/data_show.py index b1ef6f891..9b29103b2 100644 --- a/flexmeasures/cli/data_show.py +++ b/flexmeasures/cli/data_show.py @@ -163,9 +163,9 @@ def show_generic_asset(asset): """ Show asset info and list sensors """ - click.echo(f"======{len(asset.name) * '='}=========") + click.echo(f"======{len(asset.name) * '='}========") click.echo(f"Asset {asset.name} (ID: {asset.id})") - click.echo(f"======{len(asset.name) * '='}=========\n") + click.echo(f"======{len(asset.name) * '='}========\n") asset_data = [ ( From 810c6ea1ff58f6ba8c7f116719a0bd34d603994c Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sun, 9 Apr 2023 15:00:35 +0200 Subject: [PATCH 61/73] Add sensors_to_show attribute to tutorial printout Signed-off-by: F.N. Claessen --- documentation/tut/toy-example-from-scratch.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/documentation/tut/toy-example-from-scratch.rst b/documentation/tut/toy-example-from-scratch.rst index 4fb9709d5..b080617e8 100644 --- a/documentation/tut/toy-example-from-scratch.rst +++ b/documentation/tut/toy-example-from-scratch.rst @@ -157,9 +157,10 @@ If you want, you can inspect what you created: Type Location Attributes ------- ----------------- --------------------- - battery (52.374, 4.88969) capacity_in_mw:0.5 - min_soc_in_mwh:0.05 - max_soc_in_mwh:0.45 + battery (52.374, 4.88969) capacity_in_mw: 0.5 + min_soc_in_mwh: 0.05 + max_soc_in_mwh: 0.45 + sensors_to_show: [3, [4, 2]] All sensors in asset: From fca5f6e3f133694da294abd2524009fbcb884283 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sun, 9 Apr 2023 15:01:17 +0200 Subject: [PATCH 62/73] Correct sensor name in tutorial Signed-off-by: F.N. Claessen --- documentation/tut/toy-example-from-scratch.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/documentation/tut/toy-example-from-scratch.rst b/documentation/tut/toy-example-from-scratch.rst index b080617e8..718ebfbdf 100644 --- a/documentation/tut/toy-example-from-scratch.rst +++ b/documentation/tut/toy-example-from-scratch.rst @@ -164,9 +164,9 @@ If you want, you can inspect what you created: All sensors in asset: - Id Name Unit Resolution Timezone Attributes - ---- -------- ------ ------------ ---------------- ------------ - 2 charging MW 15 minutes Europe/Amsterdam + Id Name Unit Resolution Timezone Attributes + ---- ----------- ------ ------------ ---------------- ------------ + 2 discharging MW 15 minutes Europe/Amsterdam Yes, that is quite a large battery :) @@ -274,7 +274,7 @@ Make a schedule Finally, we can create the schedule, which is the main benefit of FlexMeasures (smart real-time control). -We'll ask FlexMeasures for a schedule for our charging sensor (Id 2). We also need to specify what to optimise against. Here we pass the Id of our market price sensor (3). +We'll ask FlexMeasures for a schedule for our discharging sensor (Id 2). We also need to specify what to optimise against. Here we pass the Id of our market price sensor (3). To keep it short, we'll only ask for a 12-hour window starting at 7am. Finally, the scheduler should know what the state of charge of the battery is when the schedule starts (50%) and what its roundtrip efficiency is (90%). .. code-block:: console From eee7b97d532ab7bea7c9d2ce941e6a5f8c4d7b65 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sun, 9 Apr 2023 15:02:12 +0200 Subject: [PATCH 63/73] Correct column name in tutorial Signed-off-by: F.N. Claessen --- documentation/tut/toy-example-from-scratch.rst | 12 ++++++------ flexmeasures/cli/data_show.py | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/documentation/tut/toy-example-from-scratch.rst b/documentation/tut/toy-example-from-scratch.rst index 718ebfbdf..164f6dee9 100644 --- a/documentation/tut/toy-example-from-scratch.rst +++ b/documentation/tut/toy-example-from-scratch.rst @@ -16,7 +16,7 @@ Below are the ``flexmeasures`` CLI commands we'll run, and which we'll explain s .. code-block:: console - # setup an account with a user, a battery (Id 2) and a market (Id 3) + # setup an account with a user, battery (ID 2) and market (ID 3) $ flexmeasures add toy-account --kind battery # load prices to optimise the schedule against $ flexmeasures add beliefs --sensor-id 3 --source toy-user prices-tomorrow.csv --timezone Europe/Amsterdam @@ -143,7 +143,7 @@ If you want, you can inspect what you created: All assets: - Id Name Type Location + ID Name Type Location ---- ------------ -------- ----------------- 3 toy-battery battery (52.374, 4.88969) 2 toy-building building (52.374, 4.88969) @@ -164,7 +164,7 @@ If you want, you can inspect what you created: All sensors in asset: - Id Name Unit Resolution Timezone Attributes + ID Name Unit Resolution Timezone Attributes ---- ----------- ------ ------------ ---------------- ------------ 2 discharging MW 15 minutes Europe/Amsterdam @@ -234,7 +234,7 @@ Let's look at the price data we just loaded: .. code-block:: console $ flexmeasures show beliefs --sensor-id 3 --start ${TOMORROW}T00:00:00+01:00 --duration PT24H - Beliefs for Sensor 'day-ahead prices' (Id 3). + Beliefs for Sensor 'day-ahead prices' (ID 3). Data spans a day and starts at 2022-03-03 00:00:00+01:00. The time resolution (x-axis) is an hour. ┌────────────────────────────────────────────────────────────┐ @@ -274,7 +274,7 @@ Make a schedule Finally, we can create the schedule, which is the main benefit of FlexMeasures (smart real-time control). -We'll ask FlexMeasures for a schedule for our discharging sensor (Id 2). We also need to specify what to optimise against. Here we pass the Id of our market price sensor (3). +We'll ask FlexMeasures for a schedule for our discharging sensor (ID 2). We also need to specify what to optimise against. Here we pass the Id of our market price sensor (3). To keep it short, we'll only ask for a 12-hour window starting at 7am. Finally, the scheduler should know what the state of charge of the battery is when the schedule starts (50%) and what its roundtrip efficiency is (90%). .. code-block:: console @@ -289,7 +289,7 @@ Great. Let's see what we made: .. code-block:: console $ flexmeasures show beliefs --sensor-id 2 --start ${TOMORROW}T07:00:00+01:00 --duration PT12H - Beliefs for Sensor 'discharging' (Id 2). + Beliefs for Sensor 'discharging' (ID 2). Data spans 12 hours and starts at 2022-03-04 07:00:00+01:00. The time resolution (x-axis) is 15 minutes. ┌────────────────────────────────────────────────────────────┐ diff --git a/flexmeasures/cli/data_show.py b/flexmeasures/cli/data_show.py index 9b29103b2..0be61300c 100644 --- a/flexmeasures/cli/data_show.py +++ b/flexmeasures/cli/data_show.py @@ -343,9 +343,9 @@ def plot_beliefs( # Build title if len(sensors) == 1: - title = f"Beliefs for Sensor '{sensors[0].name}' (Id {sensors[0].id}).\n" + title = f"Beliefs for Sensor '{sensors[0].name}' (ID {sensors[0].id}).\n" else: - title = f"Beliefs for Sensor(s) [{', '.join([s.name for s in sensors])}], (Id(s): [{', '.join([str(s.id) for s in sensors])}]).\n" + title = f"Beliefs for Sensor(s) [{', '.join([s.name for s in sensors])}], (ID(s): [{', '.join([str(s.id) for s in sensors])}]).\n" title += f"Data spans {naturaldelta(duration)} and starts at {start}." if belief_time_before: title += f"\nOnly beliefs made before: {belief_time_before}." From 625f189a6331be649b8faa8cbdba28f4a4af1f7b Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sun, 9 Apr 2023 15:05:57 +0200 Subject: [PATCH 64/73] Shift sensor ids in tutorial Signed-off-by: F.N. Claessen --- documentation/index.rst | 8 +-- .../tut/toy-example-from-scratch.rst | 53 +++++++++---------- 2 files changed, 30 insertions(+), 31 deletions(-) diff --git a/documentation/index.rst b/documentation/index.rst index 7f80d1089..079454a77 100644 --- a/documentation/index.rst +++ b/documentation/index.rst @@ -39,12 +39,12 @@ A tiny, but complete example: Let's install FlexMeasures from scratch. Then, usi $ docker pull postgres; docker run --name pg-docker -e POSTGRES_PASSWORD=docker -e POSTGRES_DB=flexmeasures-db -d -p 5433:5432 postgres:latest $ export SQLALCHEMY_DATABASE_URI="postgresql://postgres:docker@127.0.0.1:5433/flexmeasures-db" && export SECRET_KEY=notsecret $ flexmeasures db upgrade # create tables - $ flexmeasures add toy-account --kind battery # setup account & a user, a battery (Id 2) and a market (Id 3) - $ flexmeasures add beliefs --sensor-id 3 --source toy-user prices-tomorrow.csv --timezone utc # load prices, also possible per API - $ flexmeasures add schedule for-storage --sensor-id 2 --consumption-price-sensor 3 \ + $ flexmeasures add toy-account --kind battery # setup account incl. a user, battery (ID 1) and market (ID 2) + $ flexmeasures add beliefs --sensor-id 2 --source toy-user prices-tomorrow.csv --timezone utc # load prices, also possible per API + $ flexmeasures add schedule for-storage --sensor-id 1 --consumption-price-sensor 2 \ --start ${TOMORROW}T07:00+01:00 --duration PT12H \ --soc-at-start 50% --roundtrip-efficiency 90% # this is also possible per API - $ flexmeasures show beliefs --sensor-id 2 --start ${TOMORROW}T07:00:00+01:00 --duration PT12H # also visible per UI, of course + $ flexmeasures show beliefs --sensor-id 1 --start ${TOMORROW}T07:00:00+01:00 --duration PT12H # also visible per UI, of course We discuss this in more depth at :ref:`tut_toy_schedule`. diff --git a/documentation/tut/toy-example-from-scratch.rst b/documentation/tut/toy-example-from-scratch.rst index 164f6dee9..bd162d437 100644 --- a/documentation/tut/toy-example-from-scratch.rst +++ b/documentation/tut/toy-example-from-scratch.rst @@ -16,12 +16,12 @@ Below are the ``flexmeasures`` CLI commands we'll run, and which we'll explain s .. code-block:: console - # setup an account with a user, battery (ID 2) and market (ID 3) + # setup an account with a user, battery (ID 1) and market (ID 2) $ flexmeasures add toy-account --kind battery # load prices to optimise the schedule against - $ flexmeasures add beliefs --sensor-id 3 --source toy-user prices-tomorrow.csv --timezone Europe/Amsterdam + $ flexmeasures add beliefs --sensor-id 2 --source toy-user prices-tomorrow.csv --timezone Europe/Amsterdam # make the schedule - $ flexmeasures add schedule for-storage --sensor-id 2 --consumption-price-sensor 3 \ + $ flexmeasures add schedule for-storage --sensor-id 1 --consumption-price-sensor 2 \ --start ${TOMORROW}T07:00+01:00 --duration PT12H \ --soc-at-start 50% --roundtrip-efficiency 90% @@ -117,9 +117,9 @@ FlexMeasures offers a command to create a toy account with a battery: $ flexmeasures add toy-account --kind battery Toy account Toy Account with user toy-user@flexmeasures.io created successfully. You might want to run `flexmeasures show account --id 1` - The sensor recording battery power is . - The sensor recording day-ahead prices is . - The sensor recording solar forecasts is . + The sensor recording battery power is . + The sensor recording day-ahead prices is . + The sensor recording solar forecasts is . And with that, we're done with the structural data for this tutorial! @@ -145,14 +145,13 @@ If you want, you can inspect what you created: ID Name Type Location ---- ------------ -------- ----------------- - 3 toy-battery battery (52.374, 4.88969) - 2 toy-building building (52.374, 4.88969) - 1 toy-solar solar (52.374, 4.88969) + 1 toy-battery battery (52.374, 4.88969) + 3 toy-solar solar (52.374, 4.88969) - $ flexmeasures show asset --id 3 + $ flexmeasures show asset --id 1 ========================= - Asset toy-battery (ID: 3) + Asset toy-battery (ID: 1) ========================= Type Location Attributes @@ -160,13 +159,13 @@ If you want, you can inspect what you created: battery (52.374, 4.88969) capacity_in_mw: 0.5 min_soc_in_mwh: 0.05 max_soc_in_mwh: 0.45 - sensors_to_show: [3, [4, 2]] + sensors_to_show: [2, [3, 1]] All sensors in asset: ID Name Unit Resolution Timezone Attributes ---- ----------- ------ ------------ ---------------- ------------ - 2 discharging MW 15 minutes Europe/Amsterdam + 1 discharging MW 15 minutes Europe/Amsterdam Yes, that is quite a large battery :) @@ -187,7 +186,7 @@ Visit `http://localhost:5000/assets `_ (username i Add some price data --------------------------------------- -Now to add price data. First, we'll create the csv file with prices (EUR/MWh, see the setup for sensor 3 above) for tomorrow. +Now to add price data. First, we'll create the csv file with prices (EUR/MWh, see the setup for sensor 2 above) for tomorrow. .. code-block:: console @@ -222,7 +221,7 @@ This is time series data, in FlexMeasures we call "beliefs". Beliefs can also be .. code-block:: console - $ flexmeasures add beliefs --sensor-id 3 --source toy-user prices-tomorrow.csv --timezone Europe/Amsterdam + $ flexmeasures add beliefs --sensor-id 2 --source toy-user prices-tomorrow.csv --timezone Europe/Amsterdam Successfully created beliefs In FlexMeasures, all beliefs have a data source. Here, we use the username of the user we created earlier. We could also pass a user ID, or the name of a new data source we want to use for CLI scripts. @@ -233,8 +232,8 @@ Let's look at the price data we just loaded: .. code-block:: console - $ flexmeasures show beliefs --sensor-id 3 --start ${TOMORROW}T00:00:00+01:00 --duration PT24H - Beliefs for Sensor 'day-ahead prices' (ID 3). + $ flexmeasures show beliefs --sensor-id 2 --start ${TOMORROW}T00:00:00+01:00 --duration PT24H + Beliefs for Sensor 'day-ahead prices' (ID 2). Data spans a day and starts at 2022-03-03 00:00:00+01:00. The time resolution (x-axis) is an hour. ┌────────────────────────────────────────────────────────────┐ @@ -261,7 +260,7 @@ Let's look at the price data we just loaded: -Again, we can also view these prices in the `FlexMeasures UI `_: +Again, we can also view these prices in the `FlexMeasures UI `_: .. image:: https://github.com/FlexMeasures/screenshots/raw/main/tut/toy-schedule/sensor-data-prices.png :align: center @@ -274,12 +273,12 @@ Make a schedule Finally, we can create the schedule, which is the main benefit of FlexMeasures (smart real-time control). -We'll ask FlexMeasures for a schedule for our discharging sensor (ID 2). We also need to specify what to optimise against. Here we pass the Id of our market price sensor (3). +We'll ask FlexMeasures for a schedule for our discharging sensor (ID 1). We also need to specify what to optimise against. Here we pass the Id of our market price sensor (3). To keep it short, we'll only ask for a 12-hour window starting at 7am. Finally, the scheduler should know what the state of charge of the battery is when the schedule starts (50%) and what its roundtrip efficiency is (90%). .. code-block:: console - $ flexmeasures add schedule for-storage --sensor-id 2 --consumption-price-sensor 3 \ + $ flexmeasures add schedule for-storage --sensor-id 1 --consumption-price-sensor 2 \ --start ${TOMORROW}T07:00+01:00 --duration PT12H \ --soc-at-start 50% --roundtrip-efficiency 90% New schedule is stored. @@ -288,8 +287,8 @@ Great. Let's see what we made: .. code-block:: console - $ flexmeasures show beliefs --sensor-id 2 --start ${TOMORROW}T07:00:00+01:00 --duration PT12H - Beliefs for Sensor 'discharging' (ID 2). + $ flexmeasures show beliefs --sensor-id 1 --start ${TOMORROW}T07:00:00+01:00 --duration PT12H + Beliefs for Sensor 'discharging' (ID 1). Data spans 12 hours and starts at 2022-03-04 07:00:00+01:00. The time resolution (x-axis) is 15 minutes. ┌────────────────────────────────────────────────────────────┐ @@ -317,7 +316,7 @@ Great. Let's see what we made: Here, negative values denote output from the grid, so that's when the battery gets charged. -We can also look at the charging schedule in the `FlexMeasures UI `_ (reachable via the asset page for the battery): +We can also look at the charging schedule in the `FlexMeasures UI `_ (reachable via the asset page for the battery): .. image:: https://github.com/FlexMeasures/screenshots/raw/main/tut/toy-schedule/sensor-data-charging.png :align: center @@ -333,7 +332,7 @@ Take into account solar production So far we haven't taken into account any other devices that consume or produce electricity. We'll now add solar production forecasts and reschedule, to see the effect of solar on the available headroom for the battery. -First, we'll create a new csv file with solar forecasts (MW, see the setup for sensor 4 above) for tomorrow. +First, we'll create a new csv file with solar forecasts (MW, see the setup for sensor 3 above) for tomorrow. .. code-block:: console @@ -375,7 +374,7 @@ Setting the data source type to "forecaster" helps FlexMeasures to visualize dis $ flexmeasures add source --name "toy-forecaster" --type forecaster Added source - $ flexmeasures add beliefs --sensor-id 4 --source 2 solar-tomorrow.csv --timezone Europe/Amsterdam + $ flexmeasures add beliefs --sensor-id 3 --source 2 solar-tomorrow.csv --timezone Europe/Amsterdam Successfully created beliefs The one-hour CSV data is automatically resampled to the 15-minute resolution of the sensor that is recording solar production. @@ -386,8 +385,8 @@ Now, we'll reschedule the battery while taking into account the solar production .. code-block:: console - $ flexmeasures add schedule for-storage --sensor-id 2 --consumption-price-sensor 3 \ - --inflexible-device-sensor 4 \ + $ flexmeasures add schedule for-storage --sensor-id 1 --consumption-price-sensor 2 \ + --inflexible-device-sensor 3 \ --start ${TOMORROW}T07:00+01:00 --duration PT12H \ --soc-at-start 50% --roundtrip-efficiency 90% New schedule is stored. From 1467fc6183101651037c3e7156134c841efc465e Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sun, 9 Apr 2023 15:11:49 +0200 Subject: [PATCH 65/73] fix capitalization Signed-off-by: F.N. Claessen --- documentation/tut/toy-example-from-scratch.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/tut/toy-example-from-scratch.rst b/documentation/tut/toy-example-from-scratch.rst index bd162d437..1f71c8cd1 100644 --- a/documentation/tut/toy-example-from-scratch.rst +++ b/documentation/tut/toy-example-from-scratch.rst @@ -10,7 +10,7 @@ Let's walk through an example from scratch! We'll ... - load hourly prices - optimize a 12h-schedule for a battery that is half full -What do you need? Your own computer, with one of two situations: Either you have `Docker `_ or your computer supports Python 3.8+, pip and PostgresDB. The former might be easier, see the installation step below. But you choose. +What do you need? Your own computer, with one of two situations: either you have `Docker `_ or your computer supports Python 3.8+, pip and PostgresDB. The former might be easier, see the installation step below. But you choose. Below are the ``flexmeasures`` CLI commands we'll run, and which we'll explain step by step. There are some other crucial steps for installation and setup, so this becomes a complete example from scratch, but this is the meat: From 1f54b81b76fef122bed34d4ce7d6e413d69631cf Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sun, 9 Apr 2023 15:20:36 +0200 Subject: [PATCH 66/73] Update uniplot output (looks like later versions have a more intuitive choice of y-axis ticks) Signed-off-by: F.N. Claessen --- .../tut/toy-example-from-scratch.rst | 72 +++++++++---------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/documentation/tut/toy-example-from-scratch.rst b/documentation/tut/toy-example-from-scratch.rst index 1f71c8cd1..d7c65216e 100644 --- a/documentation/tut/toy-example-from-scratch.rst +++ b/documentation/tut/toy-example-from-scratch.rst @@ -237,25 +237,25 @@ Let's look at the price data we just loaded: Data spans a day and starts at 2022-03-03 00:00:00+01:00. The time resolution (x-axis) is an hour. ┌────────────────────────────────────────────────────────────┐ - │ ▗▀▚▖ │ 18EUR/MWh - │ ▞ ▝▌ │ - │ ▐ ▚ │ - │ ▗▘ ▐ │ - │ ▌ ▌ ▖ │ - │ ▞ ▚ ▗▄▀▝▄ │ - │ ▗▘ ▐ ▗▞▀ ▚ │ 13EUR/MWh - │ ▗▄▘ ▌ ▐▘ ▚ │ - │ ▗▞▘ ▚ ▌ ▚ │ - │▞▘ ▝▄ ▗ ▐ ▝▖ │ - │ ▚▄▄▀▚▄▄ ▞▘▚ ▌ ▝▖ │ - │ ▀▀▛ ▚ ▐ ▚ │ - │ ▚ ▗▘ ▚│ 8EUR/MWh - │ ▌ ▗▘ ▝│ - │ ▝▖ ▞ │ - │ ▐▖ ▗▀ │ - │ ▝▚▄▄▄▄▘ │ + │ ▗▀▚▖ │ + │ ▗▘ ▝▖ │ + │ ▞ ▌ │ + │ ▟ ▐ │ 15EUR/MWh + │ ▗▘ ▝▖ ▗ │ + │ ▗▘ ▚ ▄▞▘▚▖ │ + │ ▞ ▐ ▄▀▘ ▝▄ │ + │ ▄▞ ▌ ▛ ▖ │ + │▀ ▚ ▐ ▝▖ │ + │ ▝▚ ▖ ▗▘ ▝▖ │ 10EUR/MWh + │ ▀▄▄▞▀▄▄ ▗▀▝▖ ▞ ▐ │ + │ ▀▀▜▘ ▝▚ ▗▘ ▚ │ + │ ▌ ▞ ▌│ + │ ▝▖ ▞ ▝│ + │ ▐ ▞ │ + │ ▚ ▗▞ │ 5EUR/MWh + │ ▀▚▄▄▄▄▘ │ └────────────────────────────────────────────────────────────┘ - 5 10 15 20 + 5 10 15 20 ██ day-ahead prices @@ -292,26 +292,26 @@ Great. Let's see what we made: Data spans 12 hours and starts at 2022-03-04 07:00:00+01:00. The time resolution (x-axis) is 15 minutes. ┌────────────────────────────────────────────────────────────┐ - │ ▐▌ ▐▀▀▌ ▛▀▀│ - │ ▐▌ ▞ ▚ ▌ │ 0.4MW - │ ▌▌ ▌ ▐ ▗▘ │ - │ ▌▚ ▌ ▐ ▐ │ - │ ▗▘▐ ▗▘ ▐ ▐ │ - │ ▐ ▐ ▐ ▌ ▞ │ 0.2MW - │ ▌ ▐ ▐ ▌ ▌ │ - │ ▗▘ ▌ ▐ ▌ ▌ │ - │▀▀▀▀▀▀▀───▀▀▀▀▌─────▌────▝▀▀▀▀▀▀▀▀▌─────▐▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▘───│ 0MW - │ ▌ ▗▘ ▐ ▞ │ - │ ▌ ▐ ▐ ▗▘ │ - │ ▚ ▌ ▐ ▐ │ -0.2MW - │ ▐ ▗▘ ▌ ▌ │ - │ ▐ ▐ ▌ ▌ │ - │ ▝▖ ▞ ▌ ▐ │ - │ ▌ ▌ ▚ ▐ │ -0.4MW - │ ▙▄▄▘ ▐▄▄▌ │ + │ ▐ ▐▀▀▌ ▛▀▀│ 0.5MW + │ ▞▌ ▌ ▌ ▌ │ + │ ▌▌ ▌ ▐ ▗▘ │ + │ ▌▌ ▌ ▐ ▐ │ + │ ▐ ▐ ▐ ▐ ▐ │ + │ ▐ ▐ ▐ ▝▖ ▞ │ + │ ▌ ▐ ▐ ▌ ▌ │ + │ ▐ ▝▖ ▌ ▌ ▌ │ + │▀▘───▀▀▀▀▖─────▌────▀▀▀▀▀▀▀▀▀▌─────▐▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▘───│ 0.0MW + │ ▌ ▐ ▚ ▌ │ + │ ▌ ▞ ▐ ▗▘ │ + │ ▌ ▌ ▐ ▞ │ + │ ▐ ▐ ▝▖ ▌ │ + │ ▐ ▐ ▌ ▗▘ │ + │ ▐ ▌ ▌ ▐ │ + │ ▝▖ ▌ ▌ ▞ │ + │ ▙▄▟ ▐▄▄▌ │ -0.5MW └────────────────────────────────────────────────────────────┘ 10 20 30 40 - ██ discharging + ██ discharging Here, negative values denote output from the grid, so that's when the battery gets charged. From 4dc0361c51018e60bcbe91470be599e43c59d9f7 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sun, 9 Apr 2023 15:34:16 +0200 Subject: [PATCH 67/73] Correct data source IDs Signed-off-by: F.N. Claessen --- documentation/tut/toy-example-from-scratch.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation/tut/toy-example-from-scratch.rst b/documentation/tut/toy-example-from-scratch.rst index d7c65216e..9391830fa 100644 --- a/documentation/tut/toy-example-from-scratch.rst +++ b/documentation/tut/toy-example-from-scratch.rst @@ -373,8 +373,8 @@ Setting the data source type to "forecaster" helps FlexMeasures to visualize dis .. code-block:: console $ flexmeasures add source --name "toy-forecaster" --type forecaster - Added source - $ flexmeasures add beliefs --sensor-id 3 --source 2 solar-tomorrow.csv --timezone Europe/Amsterdam + Added source + $ flexmeasures add beliefs --sensor-id 3 --source 4 solar-tomorrow.csv --timezone Europe/Amsterdam Successfully created beliefs The one-hour CSV data is automatically resampled to the 15-minute resolution of the sensor that is recording solar production. From b393cad4dcbbb5002b56ea01248891a3233d23e7 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sun, 9 Apr 2023 15:37:33 +0200 Subject: [PATCH 68/73] test Signed-off-by: F.N. Claessen --- flexmeasures/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/app.py b/flexmeasures/app.py index 7f0a450c7..6b4492aa0 100644 --- a/flexmeasures/app.py +++ b/flexmeasures/app.py @@ -134,7 +134,7 @@ def create( from flexmeasures.ui import register_at as register_ui_at - register_ui_at(app) + register_ui_at(apetstp) # Profile endpoints (if needed, e.g. during development) From d0f509b5e1b59403f63d820fab49c0cc1857b41e Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sun, 9 Apr 2023 15:53:50 +0200 Subject: [PATCH 69/73] Revert "test" This reverts commit b393cad4dcbbb5002b56ea01248891a3233d23e7. --- flexmeasures/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/app.py b/flexmeasures/app.py index 6b4492aa0..7f0a450c7 100644 --- a/flexmeasures/app.py +++ b/flexmeasures/app.py @@ -134,7 +134,7 @@ def create( from flexmeasures.ui import register_at as register_ui_at - register_ui_at(apetstp) + register_ui_at(app) # Profile endpoints (if needed, e.g. during development) From 57e61a7f60fa2ab6031137fec0323f2c8e95ccea Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sun, 9 Apr 2023 16:05:03 +0200 Subject: [PATCH 70/73] Resolve circular import coming to light through mypy Signed-off-by: F.N. Claessen --- flexmeasures/utils/coding_utils.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/flexmeasures/utils/coding_utils.py b/flexmeasures/utils/coding_utils.py index 5cf5fccf3..24522666c 100644 --- a/flexmeasures/utils/coding_utils.py +++ b/flexmeasures/utils/coding_utils.py @@ -3,9 +3,13 @@ import functools import time import inspect +import typing from typing import Union from flask import current_app +if typing.TYPE_CHECKING: + from flexmeasures.data.models.time_series import Sensor + def make_registering_decorator(foreign_decorator): """ @@ -123,9 +127,8 @@ def sort_dict(unsorted_dict: dict) -> dict: def flatten_unique( - nested_list_of_objects: list[int | list[int]] - | list["Sensor" | list["Sensor"]], # noqa F821 -) -> list[int] | list["Sensor"]: # noqa F821 + nested_list_of_objects: list[int | list[int]] | list[Sensor | list[Sensor]], +) -> list[int] | list[Sensor]: """Returns unique objects in a possibly nested (one level) list of objects. For example: From f98304d6edc69329b62cb89ebce5ef94278f10c3 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sun, 9 Apr 2023 20:00:58 +0200 Subject: [PATCH 71/73] Generalize util function Signed-off-by: F.N. Claessen --- flexmeasures/utils/coding_utils.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/flexmeasures/utils/coding_utils.py b/flexmeasures/utils/coding_utils.py index 24522666c..5c2f1b3c0 100644 --- a/flexmeasures/utils/coding_utils.py +++ b/flexmeasures/utils/coding_utils.py @@ -3,13 +3,9 @@ import functools import time import inspect -import typing from typing import Union from flask import current_app -if typing.TYPE_CHECKING: - from flexmeasures.data.models.time_series import Sensor - def make_registering_decorator(foreign_decorator): """ @@ -126,9 +122,7 @@ def sort_dict(unsorted_dict: dict) -> dict: return sorted_dict -def flatten_unique( - nested_list_of_objects: list[int | list[int]] | list[Sensor | list[Sensor]], -) -> list[int] | list[Sensor]: +def flatten_unique(nested_list_of_objects: list) -> list: """Returns unique objects in a possibly nested (one level) list of objects. For example: From dc58d9ab8efc486f4ed7a55d90c142928fe4d460 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 10 Apr 2023 23:42:47 +0200 Subject: [PATCH 72/73] Add documentation for new CLI command Signed-off-by: F.N. Claessen --- documentation/cli/change_log.rst | 5 +++++ documentation/cli/commands.rst | 1 + 2 files changed, 6 insertions(+) diff --git a/documentation/cli/change_log.rst b/documentation/cli/change_log.rst index 5b3edb70c..ce268cff6 100644 --- a/documentation/cli/change_log.rst +++ b/documentation/cli/change_log.rst @@ -4,6 +4,11 @@ FlexMeasures CLI Changelog ********************** +since v0.13.0 | April XX, 2023 +================================= + +* Add ``flexmeasures add source`` CLI command for adding a new data source. + since v0.12.0 | January 04, 2023 ================================= diff --git a/documentation/cli/commands.rst b/documentation/cli/commands.rst index 347e90659..f2bd3c40c 100644 --- a/documentation/cli/commands.rst +++ b/documentation/cli/commands.rst @@ -33,6 +33,7 @@ of which some are referred to in this documentation. ``flexmeasures add asset`` Create a new asset. ``flexmeasures add sensor`` Add a new sensor. ``flexmeasures add beliefs`` Load beliefs from file. +``flexmeasures add source`` Add a new data source. ``flexmeasures add forecasts`` Create forecasts. ``flexmeasures add schedule for-storage`` Create a charging schedule for a storage asset. ``flexmeasures add holidays`` Add holiday annotations to accounts and/or assets. From deedb7d5985cb0fe676b434b368ca34c23de9c82 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 10 Apr 2023 23:44:35 +0200 Subject: [PATCH 73/73] Add documentation for new CLI option Signed-off-by: F.N. Claessen --- documentation/cli/change_log.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/documentation/cli/change_log.rst b/documentation/cli/change_log.rst index ce268cff6..417eb1022 100644 --- a/documentation/cli/change_log.rst +++ b/documentation/cli/change_log.rst @@ -8,6 +8,7 @@ since v0.13.0 | April XX, 2023 ================================= * Add ``flexmeasures add source`` CLI command for adding a new data source. +* Add ``--inflexible-device-sensor`` option to ``flexmeasures add schedule``. since v0.12.0 | January 04, 2023 =================================