From e3803b0f1374b9e3ddd7599319f818eac591b749 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Sun, 18 Sep 2022 16:35:06 +0200 Subject: [PATCH 01/14] First implementation of loading scheduler function from non-FM code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- flexmeasures/data/services/scheduling.py | 111 ++++++++++++++++------- 1 file changed, 77 insertions(+), 34 deletions(-) diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py index 24915ffaf..a20be186c 100644 --- a/flexmeasures/data/services/scheduling.py +++ b/flexmeasures/data/services/scheduling.py @@ -1,5 +1,9 @@ from datetime import datetime, timedelta -from typing import List, Optional +from typing import List, Tuple, Optional, Callable +import os +import sys +import importlib.util +from importlib.abc import Loader from flask import current_app import click @@ -24,7 +28,7 @@ """ -DEFAULT_RESOLUTION = timedelta(minutes=15) +DEFAULT_RESOLUTION = timedelta(minutes=15) # make_schedule can also fallback to sensor resoution def create_scheduling_job( @@ -138,50 +142,45 @@ def make_schedule( soc_targets = pd.Series( np.nan, index=pd.date_range(start, end, freq=resolution, closed="right") ) + + data_source_name = "FlexMeasures" - if sensor.generic_asset.generic_asset_type.name == "battery": - consumption_schedule = schedule_battery( - sensor, - start, - end, - resolution, - soc_at_start, - soc_targets, - soc_min, - soc_max, - roundtrip_efficiency, - consumption_price_sensor=consumption_price_sensor, - production_price_sensor=production_price_sensor, - inflexible_device_sensors=inflexible_device_sensors, - belief_time=belief_time, - ) + # Choose which algorithm to use + if "custom-scheduler" in sensor.attributes: + scheduler_specs = sensor.attributes.get("custom-scheduler") + scheduler, data_source_name = load_custom_scheduler(scheduler_specs) + elif sensor.generic_asset.generic_asset_type.name == "battery": + scheduler = schedule_battery elif sensor.generic_asset.generic_asset_type.name in ( "one-way_evse", "two-way_evse", ): - consumption_schedule = schedule_charging_station( - sensor, - start, - end, - resolution, - soc_at_start, - soc_targets, - soc_min, - soc_max, - roundtrip_efficiency, - consumption_price_sensor=consumption_price_sensor, - production_price_sensor=production_price_sensor, - inflexible_device_sensors=inflexible_device_sensors, - belief_time=belief_time, - ) + scheduler = schedule_charging_station + else: raise ValueError( "Scheduling is not (yet) supported for asset type %s." % sensor.generic_asset.generic_asset_type ) + consumption_schedule = scheduler( + sensor, + start, + end, + resolution, + soc_at_start, + soc_targets, + soc_min, + soc_max, + roundtrip_efficiency, + consumption_price_sensor=consumption_price_sensor, + production_price_sensor=production_price_sensor, + inflexible_device_sensors=inflexible_device_sensors, + belief_time=belief_time, + ) + data_source = get_data_source( - data_source_name="Seita", + data_source_name=data_source_name, data_source_type="scheduling script", ) if rq_job: @@ -204,6 +203,50 @@ def make_schedule( return True +def load_custom_scheduler(scheduler_specs: dict) -> Tuple[Callable, str]: + """ + Read in custom scheduling spec. + Attempt to load the Callable, also derive a data source name. + + Example specs: + + { + "path": "/path/to/module.py", + "function": "name_of_function", + "source": "source name" + } + + """ + assert isinstance(scheduler_specs, dict), f"Scheduler specs is {type(scheduler_specs)}, should be a dict" + assert "path" in scheduler_specs, "scheduler specs have no 'path'." + assert "function" in scheduler_specs, "scheduler specs have no 'function'" + + source_name = scheduler_specs.get("source", f"Custom scheduler - {scheduler_specs['function']}") + scheduler_name = scheduler_specs["function"] + + # import module + module_path = scheduler_specs["path"] + module_name = module_path.split("/")[-1] + assert os.path.exists(module_path), f"Module {module_path} cannot be found." + spec = importlib.util.spec_from_file_location( + scheduler_name, module_path + ) + assert spec, f"Could not load specs for scheduleing module at {module_path}." + module = importlib.util.module_from_spec(spec) + sys.modules[scheduler_name] = module + assert isinstance(spec.loader, Loader) + spec.loader.exec_module(module) + assert module, f"Module {module_path} could not be loaded." + + # get scheduling function + assert ( + hasattr(module, scheduler_specs["function"]), + f"Module at {module_path} has no function {scheduler_specs['function']}" + ) + + return getattr(module, scheduler_specs["function"]), source_name + + def handle_scheduling_exception(job, exc_type, exc_value, traceback): """ Store exception as job meta data. From 2a7306fdc5acf7ce023ebf0264db65cf3381be23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Sun, 18 Sep 2022 18:46:57 +0200 Subject: [PATCH 02/14] test loading of custom scheduler with dummy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- flexmeasures/data/tests/dummy_scheduler.py | 4 ++++ .../data/tests/test_scheduling_jobs.py | 20 ++++++++++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 flexmeasures/data/tests/dummy_scheduler.py diff --git a/flexmeasures/data/tests/dummy_scheduler.py b/flexmeasures/data/tests/dummy_scheduler.py new file mode 100644 index 000000000..3c6413170 --- /dev/null +++ b/flexmeasures/data/tests/dummy_scheduler.py @@ -0,0 +1,4 @@ + +def compute_a_schedule(): + """Just a test scheduler.""" + pass \ No newline at end of file diff --git a/flexmeasures/data/tests/test_scheduling_jobs.py b/flexmeasures/data/tests/test_scheduling_jobs.py index 1e100cd40..950a4d933 100644 --- a/flexmeasures/data/tests/test_scheduling_jobs.py +++ b/flexmeasures/data/tests/test_scheduling_jobs.py @@ -1,11 +1,13 @@ # flake8: noqa: E402 +from typing import Callable from datetime import datetime, timedelta +import os import pytz from flexmeasures.data.models.data_sources import DataSource from flexmeasures.data.models.time_series import Sensor, TimedBelief from flexmeasures.data.tests.utils import work_on_rq, exception_reporter -from flexmeasures.data.services.scheduling import create_scheduling_job +from flexmeasures.data.services.scheduling import create_scheduling_job, load_custom_scheduler def test_scheduling_a_battery(db, app, add_battery_assets, setup_test_data): @@ -47,3 +49,19 @@ def test_scheduling_a_battery(db, app, add_battery_assets, setup_test_data): ) print([v.event_value for v in power_values]) assert len(power_values) == 96 + + +def test_loading_custom_schedule(): + """ + Simply check if loading a custom scheduler works. + """ + path_to_here = os.path.dirname(__file__) + scheduler_specs = { + "path": os.path.join(path_to_here, "dummy_scheduler.py"), + "function": "compute_a_schedule", + "source": "Test Source" + } + custom_scheduler, data_source = load_custom_scheduler(scheduler_specs) + assert data_source == "Test Source" + assert custom_scheduler.__name__ == "compute_a_schedule" + assert custom_scheduler.__doc__ == "Just a test scheduler." \ No newline at end of file From e3707f179c4bef1469b5df2eac7253f962d7236e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Wed, 21 Sep 2022 11:46:18 +0200 Subject: [PATCH 03/14] changelog entry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- documentation/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 9248c2e26..cb8bd4d85 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -9,6 +9,7 @@ New features ------------- * Hit the replay button to replay what happened, available on the sensor and asset pages [see `PR #463 `_] +* Ability to provide your own custom scheduling function [see `PR #505 `_] * Improved import of time series data from CSV file: 1) drop duplicate records with warning, and 2) allow configuring which column contains explicit recording times for each data point (use case: import forecasts) [see `PR #501 `_] Bugfixes From 1a3d0a1630a01c007d855b368ffe2a53d902b423 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Wed, 21 Sep 2022 11:50:00 +0200 Subject: [PATCH 04/14] complete the switch to default data source name for scheduling being 'FlexMeasures' MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- flexmeasures/api/v3_0/sensors.py | 4 ++-- flexmeasures/api/v3_0/tests/test_sensor_schedules.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index 0ff900f0a..1be8c966f 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -518,9 +518,9 @@ def get_schedule(self, sensor: Sensor, job_id: str, duration: timedelta, **kwarg return unknown_schedule("Scheduling job has an unknown status.") schedule_start = job.kwargs["start"] - schedule_data_source_name = "Seita" + schedule_data_source_name = "FlexMeasures" scheduler_source = DataSource.query.filter_by( - name="Seita", type="scheduling script" + name=schedule_data_source_name, type="scheduling script" ).one_or_none() if scheduler_source is None: return unknown_schedule( diff --git a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py index 9c9ac912a..dd74cfe86 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py @@ -68,7 +68,7 @@ def test_trigger_and_get_schedule( # check results are in the database resolution = timedelta(minutes=15) scheduler_source = DataSource.query.filter_by( - name="Seita", type="scheduling script" + name="FlexMeasures", type="scheduling script" ).one_or_none() assert ( scheduler_source is not None From 51586f394b6b8a19f75589619c7d6602f67a4741 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Wed, 21 Sep 2022 11:51:14 +0200 Subject: [PATCH 05/14] more complete dummy scheduler for testing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- flexmeasures/data/tests/dummy_scheduler.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/flexmeasures/data/tests/dummy_scheduler.py b/flexmeasures/data/tests/dummy_scheduler.py index 3c6413170..5b48be93c 100644 --- a/flexmeasures/data/tests/dummy_scheduler.py +++ b/flexmeasures/data/tests/dummy_scheduler.py @@ -1,4 +1,20 @@ +from datetime import datetime, timedelta -def compute_a_schedule(): - """Just a test scheduler.""" - pass \ No newline at end of file +import pandas as pd + +from flexmeasures.data.models.time_series import Sensor + + +def compute_a_schedule( + sensor: Sensor, + start: datetime, + end: datetime, + resolution: timedelta, + *args, + **kwargs +): + """Just a dummy scheduler.""" + return pd.Series( + sensor.get_attribute("capacity_in_mw"), + index=pd.date_range(start, end, freq=resolution, closed="right"), + ) From bec8cf1454d14b4a8b8b52f5a25f97d779d2c0a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Wed, 21 Sep 2022 11:54:05 +0200 Subject: [PATCH 06/14] allow a custom scheduler to be present in an importable package MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- flexmeasures/data/services/scheduling.py | 67 +++++++++------ .../data/tests/test_scheduling_jobs.py | 82 ++++++++++++++++--- .../tests/test_scheduling_jobs_fresh_db.py | 2 +- 3 files changed, 112 insertions(+), 39 deletions(-) diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py index a20be186c..876026d68 100644 --- a/flexmeasures/data/services/scheduling.py +++ b/flexmeasures/data/services/scheduling.py @@ -28,7 +28,9 @@ """ -DEFAULT_RESOLUTION = timedelta(minutes=15) # make_schedule can also fallback to sensor resoution +DEFAULT_RESOLUTION = timedelta( + minutes=15 +) # make_schedule can also fallback to sensor resoution def create_scheduling_job( @@ -142,13 +144,13 @@ def make_schedule( soc_targets = pd.Series( np.nan, index=pd.date_range(start, end, freq=resolution, closed="right") ) - + data_source_name = "FlexMeasures" # Choose which algorithm to use if "custom-scheduler" in sensor.attributes: scheduler_specs = sensor.attributes.get("custom-scheduler") - scheduler, data_source_name = load_custom_scheduler(scheduler_specs) + scheduler, data_source_name = load_custom_scheduler(scheduler_specs) elif sensor.generic_asset.generic_asset_type.name == "battery": scheduler = schedule_battery elif sensor.generic_asset.generic_asset_type.name in ( @@ -156,7 +158,7 @@ def make_schedule( "two-way_evse", ): scheduler = schedule_charging_station - + else: raise ValueError( "Scheduling is not (yet) supported for asset type %s." @@ -178,7 +180,7 @@ def make_schedule( inflexible_device_sensors=inflexible_device_sensors, belief_time=belief_time, ) - + data_source = get_data_source( data_source_name=data_source_name, data_source_type="scheduling script", @@ -207,42 +209,53 @@ def load_custom_scheduler(scheduler_specs: dict) -> Tuple[Callable, str]: """ Read in custom scheduling spec. Attempt to load the Callable, also derive a data source name. - + Example specs: { - "path": "/path/to/module.py", + "module": "/path/to/module.py", # or sthg importable, e.g. "package.module" "function": "name_of_function", "source": "source name" } - + """ - assert isinstance(scheduler_specs, dict), f"Scheduler specs is {type(scheduler_specs)}, should be a dict" - assert "path" in scheduler_specs, "scheduler specs have no 'path'." + assert isinstance( + scheduler_specs, dict + ), f"Scheduler specs is {type(scheduler_specs)}, should be a dict" + assert "module" in scheduler_specs, "scheduler specs have no 'module'." assert "function" in scheduler_specs, "scheduler specs have no 'function'" - source_name = scheduler_specs.get("source", f"Custom scheduler - {scheduler_specs['function']}") + source_name = scheduler_specs.get( + "source", f"Custom scheduler - {scheduler_specs['function']}" + ) scheduler_name = scheduler_specs["function"] # import module - module_path = scheduler_specs["path"] - module_name = module_path.split("/")[-1] - assert os.path.exists(module_path), f"Module {module_path} cannot be found." - spec = importlib.util.spec_from_file_location( - scheduler_name, module_path - ) - assert spec, f"Could not load specs for scheduleing module at {module_path}." - module = importlib.util.module_from_spec(spec) - sys.modules[scheduler_name] = module - assert isinstance(spec.loader, Loader) - spec.loader.exec_module(module) - assert module, f"Module {module_path} could not be loaded." + module_descr = scheduler_specs["module"] + if os.path.exists(module_descr): + spec = importlib.util.spec_from_file_location(scheduler_name, module_descr) + assert spec, f"Could not load specs for scheduling module at {module_descr}." + module = importlib.util.module_from_spec(spec) + sys.modules[scheduler_name] = module + assert isinstance(spec.loader, Loader) + spec.loader.exec_module(module) + else: # assume importable module + try: + module = importlib.import_module(module_descr) + except TypeError: + current_app.log.error(f"Cannot load {module_descr}.") + raise + except ModuleNotFoundError: + current_app.logger.error( + f"Attempted to import module {module_descr} (as it is not a valid file path), but it is not installed." + ) + raise + assert module, f"Module {module_descr} could not be loaded." # get scheduling function - assert ( - hasattr(module, scheduler_specs["function"]), - f"Module at {module_path} has no function {scheduler_specs['function']}" - ) + assert hasattr( + module, scheduler_specs["function"] + ), "Module at {module_descr} has no function {scheduler_specs['function']}" return getattr(module, scheduler_specs["function"]), source_name diff --git a/flexmeasures/data/tests/test_scheduling_jobs.py b/flexmeasures/data/tests/test_scheduling_jobs.py index 950a4d933..79adba856 100644 --- a/flexmeasures/data/tests/test_scheduling_jobs.py +++ b/flexmeasures/data/tests/test_scheduling_jobs.py @@ -1,13 +1,17 @@ # flake8: noqa: E402 -from typing import Callable from datetime import datetime, timedelta import os + import pytz +import pytest from flexmeasures.data.models.data_sources import DataSource from flexmeasures.data.models.time_series import Sensor, TimedBelief from flexmeasures.data.tests.utils import work_on_rq, exception_reporter -from flexmeasures.data.services.scheduling import create_scheduling_job, load_custom_scheduler +from flexmeasures.data.services.scheduling import ( + create_scheduling_job, + load_custom_scheduler, +) def test_scheduling_a_battery(db, app, add_battery_assets, setup_test_data): @@ -36,7 +40,7 @@ def test_scheduling_a_battery(db, app, add_battery_assets, setup_test_data): work_on_rq(app.queues["scheduling"], exc_handler=exception_reporter) scheduler_source = DataSource.query.filter_by( - name="Seita", type="scheduling script" + name="FlexMeasures", type="scheduling script" ).one_or_none() assert ( scheduler_source is not None @@ -51,17 +55,73 @@ def test_scheduling_a_battery(db, app, add_battery_assets, setup_test_data): assert len(power_values) == 96 -def test_loading_custom_schedule(): +scheduler_specs = { + "module": None, + "function": "compute_a_schedule", + "source": "Test Source", +} + + +def make_module_descr(is_path): + if is_path: + path_to_here = os.path.dirname(__file__) + return os.path.join(path_to_here, "dummy_scheduler.py") + else: + return "flexmeasures.data.tests.dummy_scheduler" + + +@pytest.mark.parametrize("is_path", [False, True]) +def test_loading_custom_scheduler(is_path: bool): """ Simply check if loading a custom scheduler works. """ - path_to_here = os.path.dirname(__file__) - scheduler_specs = { - "path": os.path.join(path_to_here, "dummy_scheduler.py"), - "function": "compute_a_schedule", - "source": "Test Source" - } + scheduler_specs["module"] = make_module_descr(is_path) custom_scheduler, data_source = load_custom_scheduler(scheduler_specs) assert data_source == "Test Source" assert custom_scheduler.__name__ == "compute_a_schedule" - assert custom_scheduler.__doc__ == "Just a test scheduler." \ No newline at end of file + assert custom_scheduler.__doc__ == "Just a dummy scheduler." + + +@pytest.mark.parametrize("is_path", [False, True]) +def test_assigning_custom_scheduler(db, app, add_battery_assets, is_path: bool): + """ + Test if the custom scheduler is picked up when we assign it to a Sensor, + and that its dummy values are saved. + """ + scheduler_specs["module"] = make_module_descr(is_path) + + battery = Sensor.query.filter(Sensor.name == "Test battery").one_or_none() + battery.attributes["custom-scheduler"] = scheduler_specs + + tz = pytz.timezone("Europe/Amsterdam") + start = tz.localize(datetime(2015, 1, 2)) + end = tz.localize(datetime(2015, 1, 3)) + resolution = timedelta(minutes=15) + + job = create_scheduling_job( + battery.id, start, end, belief_time=start, resolution=resolution + ) + print("Job: %s" % job.id) + + work_on_rq(app.queues["scheduling"], exc_handler=exception_reporter) + + scheduler_source = DataSource.query.filter_by( + name=scheduler_specs["source"], type="scheduling script" + ).one_or_none() + assert ( + scheduler_source is not None + ) # Make sure the scheduler data source is now there + + power_values = ( + TimedBelief.query.filter(TimedBelief.sensor_id == battery.id) + .filter(TimedBelief.source_id == scheduler_source.id) + .all() + ) + assert len(power_values) == 96 + # test for negative value as we schedule consumption + assert all( + [ + v.event_value == -1 * battery.get_attribute("capacity_in_mw") + for v in power_values + ] + ) diff --git a/flexmeasures/data/tests/test_scheduling_jobs_fresh_db.py b/flexmeasures/data/tests/test_scheduling_jobs_fresh_db.py index f1f4bd5ea..a304762f3 100644 --- a/flexmeasures/data/tests/test_scheduling_jobs_fresh_db.py +++ b/flexmeasures/data/tests/test_scheduling_jobs_fresh_db.py @@ -56,7 +56,7 @@ def test_scheduling_a_charging_station( work_on_rq(app.queues["scheduling"], exc_handler=exception_reporter) scheduler_source = DataSource.query.filter_by( - name="Seita", type="scheduling script" + name="FlexMeasures", type="scheduling script" ).one_or_none() assert ( scheduler_source is not None From 7ae7715e2fb7a3d38adf8564bace0594e3ad36e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Wed, 21 Sep 2022 11:54:50 +0200 Subject: [PATCH 07/14] add documentation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- documentation/plugin/customisation.rst | 64 ++++++++++++++++++++ documentation/plugin/introduction.rst | 4 +- documentation/tut/forecasting_scheduling.rst | 2 + 3 files changed, 67 insertions(+), 3 deletions(-) diff --git a/documentation/plugin/customisation.rst b/documentation/plugin/customisation.rst index 1841082d4..6d8414e5b 100644 --- a/documentation/plugin/customisation.rst +++ b/documentation/plugin/customisation.rst @@ -5,6 +5,70 @@ Plugin Customizations ======================= +Adding your own scheduling algorithm +------------------------------------- + +FlexMeasures comes with in-built scheduling algorithms for often-used use cases. However, you can use your own algorithm, as well. + +The idea is that you'd still use FlexMeasures' API to post flexibility states and trigger new schedules to be computed (see :ref:`posting_flex_states`), +but in the background your custom scheduling algorithm is being used. + +Let's walk through an example! + +First, we need to write a function which accepts arguments just like the in-built schedulers (their code is `here `_). +The following minimal example gives you an idea of the inputs and outputs: + +.. code-block:: python + + from datetime import datetime, timedelta + import pandas as pd + from flexmeasures.data.models.time_series import Sensor + + def compute_a_schedule( + sensor: Sensor, + start: datetime, + end: datetime, + resolution: timedelta, + *args, + **kwargs + ): + """Just a dummy scheduler, advising to do nothing""" + return pd.Series( + 0, index=pd.date_range(start, end, freq=resolution, closed="right") + ) + + +.. note:: It's possible to add arguments which describe the asset flexibility and the EMS context in more detail. For example, + for storage assets we support various state-of-charge parameters. For now, the existing schedulers are the best documentation. + + +Finally, make your scheduler be the one which FlexMeasures will use for certain sensors: + + +.. code-block:: python + + from flexmeasures.data.models.time_series import Sensor + + scheduler_specs = { + "module": "flexmeasures.data.tests.dummy_scheduler", # or a file path, see note below + "function": "compute_a_schedule", + "source": "My Opinion" + } + + my_sensor = Sensor.query.filter(Sensor.name == "My power sensor on a flexible asset").one_or_none() + my_sensor.attributes["custom-scheduler"] = scheduler_specs + + +From now on, all schedules (see :ref:`tut_forecasting_scheduling`) which are requested for this sensor should +get computed by your custom function! For later lookup, the data will be linked to a new data source with the name "My Opinion". + +.. note:: To describe the module, we used an importable module here (actually a custom scheduling function we use to test this). + You can also provide a full file path to the module, e.g. "/path/to/my_file.py". + + +.. todo:: We're planning to use a similar approach to allow for custom forecasting algorithms, as well. + + Adding your own style sheets ---------------------------- diff --git a/documentation/plugin/introduction.rst b/documentation/plugin/introduction.rst index 3d9c573a9..7f48b4050 100644 --- a/documentation/plugin/introduction.rst +++ b/documentation/plugin/introduction.rst @@ -8,8 +8,6 @@ This is eventually how energy flexibility services are built on top of FlexMeasu In an nutshell, a FlexMeasures plugin adds functionality via one or more `Flask Blueprints `_. -.. todo:: We'll use this to allow for custom forecasting and scheduling algorithms, as well. - How to make FlexMeasures load your plugin ------------------------------------------ @@ -34,4 +32,4 @@ To hit the ground running with that approach, we provide a `CookieCutter templat It also includes a few Blueprint examples and best practices. -Continue reading the :ref:`plugin_showcase`. \ No newline at end of file +Continue reading the :ref:`plugin_showcase` or possibilities to do :ref:`plugin_customization`. \ No newline at end of file diff --git a/documentation/tut/forecasting_scheduling.rst b/documentation/tut/forecasting_scheduling.rst index d7f8836dc..69bc3fc2b 100644 --- a/documentation/tut/forecasting_scheduling.rst +++ b/documentation/tut/forecasting_scheduling.rst @@ -12,6 +12,8 @@ Let's take a look at how FlexMeasures users can access information from these se If you want to learn more about the actual algorithms used in the background, head over to :ref:`algorithms`. +.. note:: FlexMeasures comes with in-built scheduling algorithms. You can use your own algorithm, as well, see :ref:`plugin-customization`. + Maintaining the queues ------------------------------------ From e9574a842cdc82d81a495871e9ba749171464b95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Wed, 21 Sep 2022 12:20:37 +0200 Subject: [PATCH 08/14] one more data source renaming in tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- flexmeasures/api/v1_3/tests/test_api_v1_3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/api/v1_3/tests/test_api_v1_3.py b/flexmeasures/api/v1_3/tests/test_api_v1_3.py index 54429774a..6be2515f8 100644 --- a/flexmeasures/api/v1_3/tests/test_api_v1_3.py +++ b/flexmeasures/api/v1_3/tests/test_api_v1_3.py @@ -90,7 +90,7 @@ def test_post_udi_event_and_get_device_message( # check results are in the database resolution = timedelta(minutes=15) scheduler_source = DataSource.query.filter_by( - name="Seita", type="scheduling script" + name="FlexMeasures", type="scheduling script" ).one_or_none() assert ( scheduler_source is not None From 19d5d5132db8e8f463a850fa5f74acd696f7fddb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Wed, 21 Sep 2022 12:28:36 +0200 Subject: [PATCH 09/14] save custom data source name on job, so it can be looked up in get_schedule endpoint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- flexmeasures/api/v3_0/sensors.py | 2 ++ flexmeasures/data/services/scheduling.py | 3 +++ flexmeasures/data/tests/test_scheduling_jobs.py | 8 +++++++- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index 1be8c966f..f7320624b 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -519,6 +519,8 @@ def get_schedule(self, sensor: Sensor, job_id: str, duration: timedelta, **kwarg schedule_start = job.kwargs["start"] schedule_data_source_name = "FlexMeasures" + if "data_source_name" in job.meta: + schedule_data_source_name = job.meta["data_source_name"] scheduler_source = DataSource.query.filter_by( name=schedule_data_source_name, type="scheduling script" ).one_or_none() diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py index 876026d68..2270c0886 100644 --- a/flexmeasures/data/services/scheduling.py +++ b/flexmeasures/data/services/scheduling.py @@ -151,6 +151,9 @@ def make_schedule( if "custom-scheduler" in sensor.attributes: scheduler_specs = sensor.attributes.get("custom-scheduler") scheduler, data_source_name = load_custom_scheduler(scheduler_specs) + if rq_job: + rq_job.meta["data_source_name"] = data_source_name + rq_job.save_meta() elif sensor.generic_asset.generic_asset_type.name == "battery": scheduler = schedule_battery elif sensor.generic_asset.generic_asset_type.name in ( diff --git a/flexmeasures/data/tests/test_scheduling_jobs.py b/flexmeasures/data/tests/test_scheduling_jobs.py index 79adba856..725bf11e3 100644 --- a/flexmeasures/data/tests/test_scheduling_jobs.py +++ b/flexmeasures/data/tests/test_scheduling_jobs.py @@ -4,6 +4,7 @@ import pytz import pytest +from rq.job import Job from flexmeasures.data.models.data_sources import DataSource from flexmeasures.data.models.time_series import Sensor, TimedBelief @@ -105,8 +106,13 @@ def test_assigning_custom_scheduler(db, app, add_battery_assets, is_path: bool): work_on_rq(app.queues["scheduling"], exc_handler=exception_reporter) + # make sure we saved the data source for later lookup + redis_connection = app.queues["scheduling"].connection + finished_job = Job.fetch(job.id, connection=redis_connection) + assert finished_job.meta["data_source_name"] == scheduler_specs["source"] + scheduler_source = DataSource.query.filter_by( - name=scheduler_specs["source"], type="scheduling script" + name=finished_job.meta["data_source_name"], type="scheduling script" ).one_or_none() assert ( scheduler_source is not None From 2ddd7cce6be0de6fa573acc8a283ff088f17aecd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Wed, 21 Sep 2022 13:29:03 +0200 Subject: [PATCH 10/14] one more data source renaming in tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- flexmeasures/api/v1_3/implementations.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flexmeasures/api/v1_3/implementations.py b/flexmeasures/api/v1_3/implementations.py index c62442e00..5e97c1545 100644 --- a/flexmeasures/api/v1_3/implementations.py +++ b/flexmeasures/api/v1_3/implementations.py @@ -144,9 +144,9 @@ def get_device_message_response(generic_asset_name_groups, duration): return unknown_schedule("Scheduling job has an unknown status.") schedule_start = job.kwargs["start"] - schedule_data_source_name = "Seita" + schedule_data_source_name = "FlexMeasures" scheduler_source = DataSource.query.filter_by( - name="Seita", type="scheduling script" + name=schedule_data_source_name, type="scheduling script" ).one_or_none() if scheduler_source is None: return unknown_schedule( From ed45916b38f98ab954a654862a334533d14092ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Wed, 21 Sep 2022 22:21:28 +0200 Subject: [PATCH 11/14] move back from renaming default script data source - we made issue #508 to work on this specifically MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- flexmeasures/api/v1_3/implementations.py | 2 +- flexmeasures/api/v1_3/tests/test_api_v1_3.py | 2 +- flexmeasures/api/v3_0/sensors.py | 2 +- flexmeasures/api/v3_0/tests/test_sensor_schedules.py | 2 +- flexmeasures/data/services/scheduling.py | 2 +- flexmeasures/data/tests/test_scheduling_jobs.py | 6 ++++-- flexmeasures/data/tests/test_scheduling_jobs_fresh_db.py | 6 ++++-- 7 files changed, 13 insertions(+), 9 deletions(-) diff --git a/flexmeasures/api/v1_3/implementations.py b/flexmeasures/api/v1_3/implementations.py index 5e97c1545..8b2a5b857 100644 --- a/flexmeasures/api/v1_3/implementations.py +++ b/flexmeasures/api/v1_3/implementations.py @@ -144,7 +144,7 @@ def get_device_message_response(generic_asset_name_groups, duration): return unknown_schedule("Scheduling job has an unknown status.") schedule_start = job.kwargs["start"] - schedule_data_source_name = "FlexMeasures" + schedule_data_source_name = "Seita" scheduler_source = DataSource.query.filter_by( name=schedule_data_source_name, type="scheduling script" ).one_or_none() diff --git a/flexmeasures/api/v1_3/tests/test_api_v1_3.py b/flexmeasures/api/v1_3/tests/test_api_v1_3.py index 6be2515f8..54429774a 100644 --- a/flexmeasures/api/v1_3/tests/test_api_v1_3.py +++ b/flexmeasures/api/v1_3/tests/test_api_v1_3.py @@ -90,7 +90,7 @@ def test_post_udi_event_and_get_device_message( # check results are in the database resolution = timedelta(minutes=15) scheduler_source = DataSource.query.filter_by( - name="FlexMeasures", type="scheduling script" + name="Seita", type="scheduling script" ).one_or_none() assert ( scheduler_source is not None diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index f7320624b..8ec9c0200 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -518,7 +518,7 @@ def get_schedule(self, sensor: Sensor, job_id: str, duration: timedelta, **kwarg return unknown_schedule("Scheduling job has an unknown status.") schedule_start = job.kwargs["start"] - schedule_data_source_name = "FlexMeasures" + schedule_data_source_name = "Seita" if "data_source_name" in job.meta: schedule_data_source_name = job.meta["data_source_name"] scheduler_source = DataSource.query.filter_by( diff --git a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py index dd74cfe86..9c9ac912a 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py @@ -68,7 +68,7 @@ def test_trigger_and_get_schedule( # check results are in the database resolution = timedelta(minutes=15) scheduler_source = DataSource.query.filter_by( - name="FlexMeasures", type="scheduling script" + name="Seita", type="scheduling script" ).one_or_none() assert ( scheduler_source is not None diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py index 2270c0886..b835cadee 100644 --- a/flexmeasures/data/services/scheduling.py +++ b/flexmeasures/data/services/scheduling.py @@ -145,7 +145,7 @@ def make_schedule( np.nan, index=pd.date_range(start, end, freq=resolution, closed="right") ) - data_source_name = "FlexMeasures" + data_source_name = "Seita" # Choose which algorithm to use if "custom-scheduler" in sensor.attributes: diff --git a/flexmeasures/data/tests/test_scheduling_jobs.py b/flexmeasures/data/tests/test_scheduling_jobs.py index 725bf11e3..e86727ff7 100644 --- a/flexmeasures/data/tests/test_scheduling_jobs.py +++ b/flexmeasures/data/tests/test_scheduling_jobs.py @@ -28,7 +28,9 @@ def test_scheduling_a_battery(db, app, add_battery_assets, setup_test_data): resolution = timedelta(minutes=15) assert ( - DataSource.query.filter_by(name="Seita", type="scheduling script").one_or_none() + DataSource.query.filter_by( + name="FlexMeasures", type="scheduling script" + ).one_or_none() is None ) # Make sure the scheduler data source isn't there @@ -41,7 +43,7 @@ def test_scheduling_a_battery(db, app, add_battery_assets, setup_test_data): work_on_rq(app.queues["scheduling"], exc_handler=exception_reporter) scheduler_source = DataSource.query.filter_by( - name="FlexMeasures", type="scheduling script" + name="Seita", type="scheduling script" ).one_or_none() assert ( scheduler_source is not None diff --git a/flexmeasures/data/tests/test_scheduling_jobs_fresh_db.py b/flexmeasures/data/tests/test_scheduling_jobs_fresh_db.py index a304762f3..0e2a52cc7 100644 --- a/flexmeasures/data/tests/test_scheduling_jobs_fresh_db.py +++ b/flexmeasures/data/tests/test_scheduling_jobs_fresh_db.py @@ -37,7 +37,9 @@ def test_scheduling_a_charging_station( soc_targets.loc[target_soc_datetime] = target_soc assert ( - DataSource.query.filter_by(name="Seita", type="scheduling script").one_or_none() + DataSource.query.filter_by( + name="FlexMeasures", type="scheduling script" + ).one_or_none() is None ) # Make sure the scheduler data source isn't there @@ -56,7 +58,7 @@ def test_scheduling_a_charging_station( work_on_rq(app.queues["scheduling"], exc_handler=exception_reporter) scheduler_source = DataSource.query.filter_by( - name="FlexMeasures", type="scheduling script" + name="Seita", type="scheduling script" ).one_or_none() assert ( scheduler_source is not None From 08bd6cc5995532989048ccb6e4908ec3ddd8b218 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Wed, 21 Sep 2022 22:25:57 +0200 Subject: [PATCH 12/14] forgot one more data source renaming MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- flexmeasures/data/services/scheduling.py | 4 +--- flexmeasures/data/tests/test_scheduling_jobs_fresh_db.py | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py index b835cadee..566879bbc 100644 --- a/flexmeasures/data/services/scheduling.py +++ b/flexmeasures/data/services/scheduling.py @@ -28,9 +28,7 @@ """ -DEFAULT_RESOLUTION = timedelta( - minutes=15 -) # make_schedule can also fallback to sensor resoution +DEFAULT_RESOLUTION = timedelta(minutes=15) def create_scheduling_job( diff --git a/flexmeasures/data/tests/test_scheduling_jobs_fresh_db.py b/flexmeasures/data/tests/test_scheduling_jobs_fresh_db.py index 0e2a52cc7..f1f4bd5ea 100644 --- a/flexmeasures/data/tests/test_scheduling_jobs_fresh_db.py +++ b/flexmeasures/data/tests/test_scheduling_jobs_fresh_db.py @@ -37,9 +37,7 @@ def test_scheduling_a_charging_station( soc_targets.loc[target_soc_datetime] = target_soc assert ( - DataSource.query.filter_by( - name="FlexMeasures", type="scheduling script" - ).one_or_none() + DataSource.query.filter_by(name="Seita", type="scheduling script").one_or_none() is None ) # Make sure the scheduler data source isn't there From 4e90a0e873f42d4a181b38d2cc3bf0c2e0d64a4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Sat, 15 Oct 2022 17:11:14 +0200 Subject: [PATCH 13/14] implement review comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- documentation/plugin/customisation.rst | 9 +++++---- documentation/plugin/introduction.rst | 2 +- flexmeasures/data/services/scheduling.py | 6 +++--- flexmeasures/data/tests/dummy_scheduler.py | 2 +- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/documentation/plugin/customisation.rst b/documentation/plugin/customisation.rst index 6d8414e5b..4f7b6fdb3 100644 --- a/documentation/plugin/customisation.rst +++ b/documentation/plugin/customisation.rst @@ -22,6 +22,7 @@ The following minimal example gives you an idea of the inputs and outputs: from datetime import datetime, timedelta import pandas as pd + from pandas.tseries.frequencies import to_offset from flexmeasures.data.models.time_series import Sensor def compute_a_schedule( @@ -34,15 +35,15 @@ The following minimal example gives you an idea of the inputs and outputs: ): """Just a dummy scheduler, advising to do nothing""" return pd.Series( - 0, index=pd.date_range(start, end, freq=resolution, closed="right") + 0, index=pd.date_range(start, end, freq=resolution, inclusive="left") ) -.. note:: It's possible to add arguments which describe the asset flexibility and the EMS context in more detail. For example, +.. note:: It's possible to add arguments that describe the asset flexibility and the EMS context in more detail. For example, for storage assets we support various state-of-charge parameters. For now, the existing schedulers are the best documentation. -Finally, make your scheduler be the one which FlexMeasures will use for certain sensors: +Finally, make your scheduler be the one that FlexMeasures will use for certain sensors: .. code-block:: python @@ -52,7 +53,7 @@ Finally, make your scheduler be the one which FlexMeasures will use for certain scheduler_specs = { "module": "flexmeasures.data.tests.dummy_scheduler", # or a file path, see note below "function": "compute_a_schedule", - "source": "My Opinion" + "source": "My Company" } my_sensor = Sensor.query.filter(Sensor.name == "My power sensor on a flexible asset").one_or_none() diff --git a/documentation/plugin/introduction.rst b/documentation/plugin/introduction.rst index 7f48b4050..8a0eff851 100644 --- a/documentation/plugin/introduction.rst +++ b/documentation/plugin/introduction.rst @@ -3,7 +3,7 @@ Writing Plugins ==================== -You can extend FlexMeasures with functionality like UI pages, API endpoints, or CLI functions. +You can extend FlexMeasures with functionality like UI pages, API endpoints, CLI functions and custom scheduling algorithms. This is eventually how energy flexibility services are built on top of FlexMeasures! In an nutshell, a FlexMeasures plugin adds functionality via one or more `Flask Blueprints `_. diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py index 566879bbc..0e9d2524b 100644 --- a/flexmeasures/data/services/scheduling.py +++ b/flexmeasures/data/services/scheduling.py @@ -243,15 +243,15 @@ def load_custom_scheduler(scheduler_specs: dict) -> Tuple[Callable, str]: else: # assume importable module try: module = importlib.import_module(module_descr) - except TypeError: - current_app.log.error(f"Cannot load {module_descr}.") + except TypeError as te: + current_app.log.error(f"Cannot load {module_descr}: {te}.") raise except ModuleNotFoundError: current_app.logger.error( f"Attempted to import module {module_descr} (as it is not a valid file path), but it is not installed." ) raise - assert module, f"Module {module_descr} could not be loaded." + assert module, f"Module {module_descr} could not be loaded." # get scheduling function assert hasattr( diff --git a/flexmeasures/data/tests/dummy_scheduler.py b/flexmeasures/data/tests/dummy_scheduler.py index 5b48be93c..4ff9242a9 100644 --- a/flexmeasures/data/tests/dummy_scheduler.py +++ b/flexmeasures/data/tests/dummy_scheduler.py @@ -16,5 +16,5 @@ def compute_a_schedule( """Just a dummy scheduler.""" return pd.Series( sensor.get_attribute("capacity_in_mw"), - index=pd.date_range(start, end, freq=resolution, closed="right"), + index=pd.date_range(start, end, freq=resolution, inclusive="left"), ) From 6c1301b61929861391889522591922a8de9d6237 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Sat, 15 Oct 2022 17:43:00 +0200 Subject: [PATCH 14/14] we cannot use inclusive yet, while we require/support Pandas 1.25 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- documentation/plugin/customisation.rst | 2 +- flexmeasures/data/tests/dummy_scheduler.py | 11 ++++++----- flexmeasures/data/tests/test_scheduling_jobs.py | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/documentation/plugin/customisation.rst b/documentation/plugin/customisation.rst index 4f7b6fdb3..bbe51fafe 100644 --- a/documentation/plugin/customisation.rst +++ b/documentation/plugin/customisation.rst @@ -35,7 +35,7 @@ The following minimal example gives you an idea of the inputs and outputs: ): """Just a dummy scheduler, advising to do nothing""" return pd.Series( - 0, index=pd.date_range(start, end, freq=resolution, inclusive="left") + 0, index=pd.date_range(start, end, freq=resolution, closed="left") ) diff --git a/flexmeasures/data/tests/dummy_scheduler.py b/flexmeasures/data/tests/dummy_scheduler.py index 4ff9242a9..cf4f0ec89 100644 --- a/flexmeasures/data/tests/dummy_scheduler.py +++ b/flexmeasures/data/tests/dummy_scheduler.py @@ -1,8 +1,7 @@ from datetime import datetime, timedelta -import pandas as pd - from flexmeasures.data.models.time_series import Sensor +from flexmeasures.data.models.planning.utils import initialize_series def compute_a_schedule( @@ -14,7 +13,9 @@ def compute_a_schedule( **kwargs ): """Just a dummy scheduler.""" - return pd.Series( - sensor.get_attribute("capacity_in_mw"), - index=pd.date_range(start, end, freq=resolution, inclusive="left"), + return initialize_series( # simply creates a Pandas Series repeating one value + data=sensor.get_attribute("capacity_in_mw"), + start=start, + end=end, + resolution=resolution, ) diff --git a/flexmeasures/data/tests/test_scheduling_jobs.py b/flexmeasures/data/tests/test_scheduling_jobs.py index e86727ff7..a0111a445 100644 --- a/flexmeasures/data/tests/test_scheduling_jobs.py +++ b/flexmeasures/data/tests/test_scheduling_jobs.py @@ -59,7 +59,7 @@ def test_scheduling_a_battery(db, app, add_battery_assets, setup_test_data): scheduler_specs = { - "module": None, + "module": None, # use make_module_descr, see below "function": "compute_a_schedule", "source": "Test Source", }