From 64c28efb409ee990440c566b30b2b0d8552de2a7 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Wed, 24 Apr 2024 23:06:44 +0200 Subject: [PATCH 1/6] Convert the units of the input data to a target unit. Signed-off-by: Victor Garcia Reolid --- flexmeasures/data/models/reporting/pandas_reporter.py | 11 +++++++++++ flexmeasures/data/schemas/io.py | 1 + 2 files changed, 12 insertions(+) diff --git a/flexmeasures/data/models/reporting/pandas_reporter.py b/flexmeasures/data/models/reporting/pandas_reporter.py index 3f4afb203..b8b37f8c2 100644 --- a/flexmeasures/data/models/reporting/pandas_reporter.py +++ b/flexmeasures/data/models/reporting/pandas_reporter.py @@ -5,6 +5,7 @@ from copy import deepcopy, copy from flask import current_app +from flexmeasures.utils.unit_utils import convert_units import timely_beliefs as tb import pandas as pd from flexmeasures.data.models.reporting import Reporter @@ -31,6 +32,12 @@ class PandasReporter(Reporter): data: dict[str, tb.BeliefsDataFrame | pd.DataFrame] = None + def _get_input_target_unit(self, name : str) -> str | None: + for required_input in self._config["required_input"]: + if name in required_input.get("name"): + return required_input.get("unit") + return None + def fetch_data( self, start: datetime, @@ -71,6 +78,10 @@ def fetch_data( for source in bdf.sources.unique(): self.data[f"source_{source.id}"] = source + unit = self._get_input_target_unit(name) + if unit is not None: + bdf *= convert_units(1, from_unit=sensor.unit, to_unit=unit, event_resolution=sensor.event_resolution) + # store BeliefsDataFrame as local variable self.data[name] = bdf diff --git a/flexmeasures/data/schemas/io.py b/flexmeasures/data/schemas/io.py index 4c202d137..e2ecab2dc 100644 --- a/flexmeasures/data/schemas/io.py +++ b/flexmeasures/data/schemas/io.py @@ -8,6 +8,7 @@ class RequiredInput(Schema): name = fields.Str(required=True) + unit = fields.Str(required=False) class Input(Schema): From 01519171aa3411a9a0d2536913e9cf119bd53412 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Mon, 29 Apr 2024 19:52:33 +0200 Subject: [PATCH 2/6] add test Signed-off-by: Victor Garcia Reolid --- .../data/models/reporting/tests/conftest.py | 27 ++++++++- .../models/reporting/tests/test_aggregator.py | 10 ++-- .../reporting/tests/test_pandas_reporter.py | 57 ++++++++++++++++++- 3 files changed, 83 insertions(+), 11 deletions(-) diff --git a/flexmeasures/data/models/reporting/tests/conftest.py b/flexmeasures/data/models/reporting/tests/conftest.py index a9f3516f7..1a79407c2 100644 --- a/flexmeasures/data/models/reporting/tests/conftest.py +++ b/flexmeasures/data/models/reporting/tests/conftest.py @@ -123,7 +123,9 @@ def setup_dummy_data(db, app, generic_report): db.session.add(dummy_asset) - sensor1 = Sensor("sensor 1", generic_asset=dummy_asset, event_resolution="1h") + sensor1 = Sensor( + "sensor 1", generic_asset=dummy_asset, event_resolution="1h", unit="kW" + ) db.session.add(sensor1) sensor2 = Sensor("sensor 2", generic_asset=dummy_asset, event_resolution="1h") db.session.add(sensor2) @@ -134,6 +136,14 @@ def setup_dummy_data(db, app, generic_report): timezone="Europe/Amsterdam", ) db.session.add(sensor3) + sensor4 = Sensor( + "sensor 4", + generic_asset=dummy_asset, + event_resolution="PT15M", + timezone="Europe/Amsterdam", + unit="kW", + ) + db.session.add(sensor4) report_sensor = Sensor( "report sensor", generic_asset=generic_report, event_resolution="1h" @@ -161,7 +171,6 @@ def setup_dummy_data(db, app, generic_report): for sensor in [sensor1, sensor2]: for si, source in enumerate([source1, source2]): for t in range(10): - print(si) beliefs.append( TimedBelief( event_start=datetime(2023, 4, 10, tzinfo=utc) @@ -246,7 +255,19 @@ def setup_dummy_data(db, app, generic_report): ) ) + # add data for sensor 4 + for t in range(24 * 3): + beliefs.append( + TimedBelief( + event_start=datetime(2023, 1, 1, tzinfo=utc) + timedelta(hours=t), + belief_horizon=timedelta(hours=24), + event_value=1, + sensor=sensor4, + source=source1, + ) + ) + db.session.add_all(beliefs) db.session.commit() - yield sensor1, sensor2, sensor3, report_sensor, daily_report_sensor + yield sensor1, sensor2, sensor3, sensor4, report_sensor, daily_report_sensor diff --git a/flexmeasures/data/models/reporting/tests/test_aggregator.py b/flexmeasures/data/models/reporting/tests/test_aggregator.py index 6d1a85554..cee177bcb 100644 --- a/flexmeasures/data/models/reporting/tests/test_aggregator.py +++ b/flexmeasures/data/models/reporting/tests/test_aggregator.py @@ -36,7 +36,7 @@ def test_aggregator(setup_dummy_data, aggregation_method, expected_value, db): 7) prod: -1 = (1) * (-1) 8) median: even number of elements, mean of the most central elements, 0 = ((1) + (-1))/2 """ - s1, s2, s3, report_sensor, daily_report_sensor = setup_dummy_data + s1, s2, s3, s4, report_sensor, daily_report_sensor = setup_dummy_data agg_reporter = AggregatorReporter(method=aggregation_method) @@ -64,7 +64,7 @@ def test_aggregator(setup_dummy_data, aggregation_method, expected_value, db): def test_aggregator_reporter_weights( setup_dummy_data, weight_1, weight_2, expected_result, db ): - s1, s2, s3, report_sensor, daily_report_sensor = setup_dummy_data + s1, s2, s3, s4, report_sensor, daily_report_sensor = setup_dummy_data reporter_config = dict(method="sum", weights={"s1": weight_1, "sensor_2": weight_2}) @@ -91,7 +91,7 @@ def test_aggregator_reporter_weights( def test_dst_transition(setup_dummy_data, db): - s1, s2, s3, report_sensor, daily_report_sensor = setup_dummy_data + s1, s2, s3, s4, report_sensor, daily_report_sensor = setup_dummy_data agg_reporter = AggregatorReporter() @@ -121,7 +121,7 @@ def test_dst_transition(setup_dummy_data, db): def test_resampling(setup_dummy_data, db): - s1, s2, s3, report_sensor, daily_report_sensor = setup_dummy_data + s1, s2, s3, s4, report_sensor, daily_report_sensor = setup_dummy_data agg_reporter = AggregatorReporter() @@ -165,7 +165,7 @@ def test_source_transition(setup_dummy_data, db): array is prioritized. """ - s1, s2, s3, report_sensor, daily_report_sensor = setup_dummy_data + s1, s2, s3, s4, report_sensor, daily_report_sensor = setup_dummy_data agg_reporter = AggregatorReporter() diff --git a/flexmeasures/data/models/reporting/tests/test_pandas_reporter.py b/flexmeasures/data/models/reporting/tests/test_pandas_reporter.py index 64c8ba7fd..9b9fd8aca 100644 --- a/flexmeasures/data/models/reporting/tests/test_pandas_reporter.py +++ b/flexmeasures/data/models/reporting/tests/test_pandas_reporter.py @@ -6,7 +6,7 @@ def test_reporter(app, setup_dummy_data): - s1, s2, s3, report_sensor, daily_report_sensor = setup_dummy_data + s1, s2, s3, s4, report_sensor, daily_report_sensor = setup_dummy_data reporter_config = dict( required_input=[{"name": "sensor_1"}, {"name": "sensor_2"}], @@ -76,7 +76,7 @@ def test_reporter(app, setup_dummy_data): def test_reporter_repeated(setup_dummy_data): """check that calling compute doesn't change the result""" - s1, s2, s3, report_sensor, daily_report_sensor = setup_dummy_data + s1, s2, s3, s4, report_sensor, daily_report_sensor = setup_dummy_data reporter_config = dict( required_input=[{"name": "sensor_1"}, {"name": "sensor_2"}], @@ -129,7 +129,7 @@ def test_reporter_repeated(setup_dummy_data): def test_reporter_empty(setup_dummy_data): """check that calling compute with missing data returns an empty report""" - s1, s2, s3, report_sensor, daily_report_sensor = setup_dummy_data + s1, s2, s3, s3, report_sensor, daily_report_sensor = setup_dummy_data config = dict( required_input=[{"name": "sensor_1"}], @@ -159,3 +159,54 @@ def test_reporter_empty(setup_dummy_data): ) assert report[0]["data"].empty + + +def test_pandas_reporter_unit_conversion(app, setup_dummy_data): + """ + Check that the unit conversion feature can handle the following cases: + - kW -> kW + - kW -> MW + - kW -> MWh + """ + s1, s2, s3, s4, report_sensor, daily_report_sensor = setup_dummy_data + + reporter_config = dict( + required_input=[ + {"name": "sensor_4_kw"}, + {"name": "sensor_4_mw", "unit": "MW"}, + {"name": "sensor_4_mwh", "unit": "MWh"}, + ], + required_output=[ + {"name": "sensor_4_kw"}, + {"name": "sensor_4_mw"}, + {"name": "sensor_4_mwh"}, + ], + transformations=[], + ) + + reporter = PandasReporter(config=reporter_config) + + start = datetime(2023, 1, 1, tzinfo=utc) + end = datetime(2023, 1, 2, tzinfo=utc) + input = [ + dict(name="sensor_4_kw", sensor=s4), + dict(name="sensor_4_mw", sensor=s4), + dict(name="sensor_4_mwh", sensor=s4), + ] + output = [ + dict(name="sensor_4_kw", sensor=s4), + dict(name="sensor_4_mw", sensor=s4), + dict(name="sensor_4_mwh", sensor=s4), + ] + + report = reporter.compute(start=start, end=end, input=input, output=output) + result_kw = report[0]["data"] + result_mw = report[1]["data"] + result_mwh = report[2]["data"] + + assert ( + result_mw.event_value.values == result_kw.event_value.values / 1000 + ).all() # MW = kW / 1000 + assert ( + result_mwh.event_value.values == result_mw.event_value.values * 0.25 + ).all() # MWh = MW * 0.25 (resolution = 15 min) From e8c80f6762a39f506d7294134323e9a16f90799d Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Mon, 29 Apr 2024 19:56:46 +0200 Subject: [PATCH 3/6] add changelog entry Signed-off-by: Victor Garcia Reolid --- documentation/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index c66de6d20..977e87f2d 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -11,6 +11,7 @@ New features * Add `asset//status` page to view asset statuses [see `PR #41 `_ and `PR #1035 `_] * Support `start_date` and `end_date` query parameters for the asset page [see `PR #1030 `_] +* Add unit conversion to the input data of the `PandasReporter` [see `PR #1044 `_] Bugfixes ----------- From 6bd4b9d05bda9b7fc1ed1ed91ae518e06d1e694a Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Tue, 30 Apr 2024 16:48:30 +0200 Subject: [PATCH 4/6] fix refactor Signed-off-by: Victor Garcia Reolid --- .../data/models/reporting/tests/test_pandas_reporter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/models/reporting/tests/test_pandas_reporter.py b/flexmeasures/data/models/reporting/tests/test_pandas_reporter.py index 9b9fd8aca..8c7c71374 100644 --- a/flexmeasures/data/models/reporting/tests/test_pandas_reporter.py +++ b/flexmeasures/data/models/reporting/tests/test_pandas_reporter.py @@ -129,7 +129,7 @@ def test_reporter_repeated(setup_dummy_data): def test_reporter_empty(setup_dummy_data): """check that calling compute with missing data returns an empty report""" - s1, s2, s3, s3, report_sensor, daily_report_sensor = setup_dummy_data + s1, s2, s3, s4, report_sensor, daily_report_sensor = setup_dummy_data config = dict( required_input=[{"name": "sensor_1"}], From 9f1582a112e5bb9bc15335d6580ac9195c3867ea Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Thu, 23 May 2024 00:37:28 +0200 Subject: [PATCH 5/6] add output unit conversion Signed-off-by: Victor Garcia Reolid --- .../data/models/reporting/pandas_reporter.py | 24 ++++++++++++++--- .../data/models/reporting/tests/conftest.py | 19 +++++++++----- .../reporting/tests/test_pandas_reporter.py | 26 ++++++++++++++----- flexmeasures/data/schemas/io.py | 1 + .../data/schemas/reporting/pandas_reporter.py | 4 +-- 5 files changed, 57 insertions(+), 17 deletions(-) diff --git a/flexmeasures/data/models/reporting/pandas_reporter.py b/flexmeasures/data/models/reporting/pandas_reporter.py index b8b37f8c2..78c568e36 100644 --- a/flexmeasures/data/models/reporting/pandas_reporter.py +++ b/flexmeasures/data/models/reporting/pandas_reporter.py @@ -32,12 +32,18 @@ class PandasReporter(Reporter): data: dict[str, tb.BeliefsDataFrame | pd.DataFrame] = None - def _get_input_target_unit(self, name : str) -> str | None: + def _get_input_target_unit(self, name: str) -> str | None: for required_input in self._config["required_input"]: if name in required_input.get("name"): return required_input.get("unit") return None + def _get_output_target_unit(self, name: str) -> str | None: + for required_output in self._config["required_output"]: + if name in required_output.get("name"): + return required_output.get("unit") + return None + def fetch_data( self, start: datetime, @@ -80,7 +86,12 @@ def fetch_data( unit = self._get_input_target_unit(name) if unit is not None: - bdf *= convert_units(1, from_unit=sensor.unit, to_unit=unit, event_resolution=sensor.event_resolution) + bdf *= convert_units( + 1, + from_unit=sensor.unit, + to_unit=unit, + event_resolution=sensor.event_resolution, + ) # store BeliefsDataFrame as local variable self.data[name] = bdf @@ -110,7 +121,6 @@ def _compute_report(self, **kwargs) -> list[dict[str, Any]]: if belief_time is None: belief_time = server_now() - # apply pandas transformations to the dataframes in `self.data` self._apply_transformations() @@ -137,6 +147,14 @@ def _compute_report(self, **kwargs) -> list[dict[str, Any]]: output_data = self._clean_belief_series( output_data, belief_time, belief_horizon ) + output_unit = self._get_output_target_unit(name) + if output_unit is not None: + output_data *= convert_units( + 1, + from_unit=output_unit, + to_unit=output_data.sensor.unit, + event_resolution=output_data.sensor.event_resolution, + ) result["data"] = output_data diff --git a/flexmeasures/data/models/reporting/tests/conftest.py b/flexmeasures/data/models/reporting/tests/conftest.py index 1a79407c2..abfeca507 100644 --- a/flexmeasures/data/models/reporting/tests/conftest.py +++ b/flexmeasures/data/models/reporting/tests/conftest.py @@ -124,35 +124,42 @@ def setup_dummy_data(db, app, generic_report): db.session.add(dummy_asset) sensor1 = Sensor( - "sensor 1", generic_asset=dummy_asset, event_resolution="1h", unit="kW" + "sensor 1", + generic_asset=dummy_asset, + event_resolution=timedelta(hours=1), + unit="kW", ) db.session.add(sensor1) - sensor2 = Sensor("sensor 2", generic_asset=dummy_asset, event_resolution="1h") + sensor2 = Sensor( + "sensor 2", generic_asset=dummy_asset, event_resolution=timedelta(hours=1) + ) db.session.add(sensor2) sensor3 = Sensor( "sensor 3", generic_asset=dummy_asset, - event_resolution="1h", + event_resolution=timedelta(hours=1), timezone="Europe/Amsterdam", ) db.session.add(sensor3) sensor4 = Sensor( "sensor 4", generic_asset=dummy_asset, - event_resolution="PT15M", + event_resolution=timedelta(minutes=15), timezone="Europe/Amsterdam", unit="kW", ) db.session.add(sensor4) report_sensor = Sensor( - "report sensor", generic_asset=generic_report, event_resolution="1h" + "report sensor", + generic_asset=generic_report, + event_resolution=timedelta(hours=1), ) db.session.add(report_sensor) daily_report_sensor = Sensor( "daily report sensor", generic_asset=generic_report, - event_resolution="1D", + event_resolution=timedelta(days=1), timezone="Europe/Amsterdam", ) diff --git a/flexmeasures/data/models/reporting/tests/test_pandas_reporter.py b/flexmeasures/data/models/reporting/tests/test_pandas_reporter.py index 8c7c71374..d9898c64c 100644 --- a/flexmeasures/data/models/reporting/tests/test_pandas_reporter.py +++ b/flexmeasures/data/models/reporting/tests/test_pandas_reporter.py @@ -167,11 +167,13 @@ def test_pandas_reporter_unit_conversion(app, setup_dummy_data): - kW -> kW - kW -> MW - kW -> MWh + - kW -> W -> kW """ s1, s2, s3, s4, report_sensor, daily_report_sensor = setup_dummy_data reporter_config = dict( required_input=[ + {"name": "sensor_4"}, {"name": "sensor_4_kw"}, {"name": "sensor_4_mw", "unit": "MW"}, {"name": "sensor_4_mwh", "unit": "MWh"}, @@ -180,8 +182,12 @@ def test_pandas_reporter_unit_conversion(app, setup_dummy_data): {"name": "sensor_4_kw"}, {"name": "sensor_4_mw"}, {"name": "sensor_4_mwh"}, + # Assume that the internal operations that produce sensor_4_output_w have "W" + {"name": "sensor_4_output_w", "unit": "W"}, + ], + transformations=[ + {"df_input": "sensor_4", "method": "copy", "df_output": "sensor_4_output_w"} ], - transformations=[], ) reporter = PandasReporter(config=reporter_config) @@ -189,6 +195,7 @@ def test_pandas_reporter_unit_conversion(app, setup_dummy_data): start = datetime(2023, 1, 1, tzinfo=utc) end = datetime(2023, 1, 2, tzinfo=utc) input = [ + dict(name="sensor_4", sensor=s4), dict(name="sensor_4_kw", sensor=s4), dict(name="sensor_4_mw", sensor=s4), dict(name="sensor_4_mwh", sensor=s4), @@ -197,16 +204,23 @@ def test_pandas_reporter_unit_conversion(app, setup_dummy_data): dict(name="sensor_4_kw", sensor=s4), dict(name="sensor_4_mw", sensor=s4), dict(name="sensor_4_mwh", sensor=s4), + dict(name="sensor_4_output_w", sensor=s4), ] report = reporter.compute(start=start, end=end, input=input, output=output) result_kw = report[0]["data"] result_mw = report[1]["data"] result_mwh = report[2]["data"] + result_output_w = report[3]["data"] + # MW = kW / 1000 + assert (result_mw.event_value.values == result_kw.event_value.values / 1000).all() + + # MWh = MW * 0.25 (resolution = 15 min) + assert (result_mwh.event_value.values == result_mw.event_value.values * 0.25).all() + + # Input is in kW; the operations transform the data to produce values in W and it transforms the values to the output sensor unit (kW). + # In summary, Input = 1 kW -(copy the values)-> 1 W -> 0.001 kW assert ( - result_mw.event_value.values == result_kw.event_value.values / 1000 - ).all() # MW = kW / 1000 - assert ( - result_mwh.event_value.values == result_mw.event_value.values * 0.25 - ).all() # MWh = MW * 0.25 (resolution = 15 min) + result_output_w.event_value.values == result_kw.event_value.values * 0.001 + ).all() diff --git a/flexmeasures/data/schemas/io.py b/flexmeasures/data/schemas/io.py index e2ecab2dc..5d1043fa5 100644 --- a/flexmeasures/data/schemas/io.py +++ b/flexmeasures/data/schemas/io.py @@ -69,3 +69,4 @@ class Output(Schema): class RequiredOutput(Schema): name = fields.Str(required=True) column = fields.Str(required=False) + unit = fields.Str(required=False) diff --git a/flexmeasures/data/schemas/reporting/pandas_reporter.py b/flexmeasures/data/schemas/reporting/pandas_reporter.py index 6dcac010f..93321bc70 100644 --- a/flexmeasures/data/schemas/reporting/pandas_reporter.py +++ b/flexmeasures/data/schemas/reporting/pandas_reporter.py @@ -56,10 +56,10 @@ class PandasReporterConfigSchema(ReporterConfigSchema): { "required_input" : [ - {"name" : "df1} + {"name" : "df1", "unit" : "MWh"} ], "required_output" : [ - {"name" : "df2"} + {"name" : "df2", "unit" : "kWh"} ], "transformations" : [ { From d7fe4ab12f56ea07330ee47c2fd7eb8c67454c10 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Thu, 23 May 2024 16:45:45 +0200 Subject: [PATCH 6/6] Mention output in changelog Signed-off-by: Victor Garcia Reolid --- documentation/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 88d9f3b98..ba34e1094 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -10,7 +10,7 @@ v0.22.0 | June XX, 2024 New features ------------- -* Add unit conversion to the input data of the `PandasReporter` [see `PR #1044 `_] +* Add unit conversion to the input and output data of the `PandasReporter` [see `PR #1044 `_] main * Add option `droplevels` to the `PandasReporter` to drop all the levels except the `event_start` and `event_value` [see `PR #1043 `_] * `PandasReporter` accepts the parameter `use_latest_version_only` to filter input data [see `PR #1045 `_]