From 6bb6933019181a076cd217a796d9ad1be325d4bc Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Thu, 1 Jun 2023 12:00:01 +0200 Subject: [PATCH 01/23] fix: get fresh sensor instance to avoid sqlalchemy.exc.InvalidRequestError Signed-off-by: Victor Garcia Reolid --- flexmeasures/data/models/time_series.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/flexmeasures/data/models/time_series.py b/flexmeasures/data/models/time_series.py index 18b9e115b..09cdc8f75 100644 --- a/flexmeasures/data/models/time_series.py +++ b/flexmeasures/data/models/time_series.py @@ -584,6 +584,9 @@ def __init__( source: tb.DBBeliefSource, **kwargs, ): + # get a Sensor instance attached to the database session (input sensor is detached) + # check out Issue #683 for more details + sensor = Sensor.query.get(sensor.id) tb.TimedBeliefDBMixin.__init__(self, sensor, source, **kwargs) tb_utils.remove_class_init_kwargs(tb.TimedBeliefDBMixin, kwargs) db.Model.__init__(self, **kwargs) From 45d48872138d104870be023de3c033755be1e52c Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Thu, 1 Jun 2023 12:29:32 +0200 Subject: [PATCH 02/23] fx: handle the case of not finding the sensor in the datbase Signed-off-by: Victor Garcia Reolid --- flexmeasures/data/models/time_series.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/flexmeasures/data/models/time_series.py b/flexmeasures/data/models/time_series.py index 09cdc8f75..e4ebf3326 100644 --- a/flexmeasures/data/models/time_series.py +++ b/flexmeasures/data/models/time_series.py @@ -586,7 +586,10 @@ def __init__( ): # get a Sensor instance attached to the database session (input sensor is detached) # check out Issue #683 for more details - sensor = Sensor.query.get(sensor.id) + _sensor = Sensor.query.get(sensor.id) + if _sensor: + sensor = _sensor + tb.TimedBeliefDBMixin.__init__(self, sensor, source, **kwargs) tb_utils.remove_class_init_kwargs(tb.TimedBeliefDBMixin, kwargs) db.Model.__init__(self, **kwargs) From fbbd0c28f339c7a928a84b0296af4e52de7dae0c Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Tue, 6 Jun 2023 14:58:09 +0200 Subject: [PATCH 03/23] feat: add AggregatorReporter Signed-off-by: Victor Garcia Reolid --- .../data/models/reporting/aggregatator.py | 83 +++++++++++++++++++ .../models/reporting/tests/test_aggregator.py | 37 +++++++++ .../data/schemas/reporting/aggregation.py | 65 +++++++++++++++ 3 files changed, 185 insertions(+) create mode 100644 flexmeasures/data/models/reporting/aggregatator.py create mode 100644 flexmeasures/data/models/reporting/tests/test_aggregator.py create mode 100644 flexmeasures/data/schemas/reporting/aggregation.py diff --git a/flexmeasures/data/models/reporting/aggregatator.py b/flexmeasures/data/models/reporting/aggregatator.py new file mode 100644 index 000000000..54fae8a01 --- /dev/null +++ b/flexmeasures/data/models/reporting/aggregatator.py @@ -0,0 +1,83 @@ +from __future__ import annotations + +from typing import Any +from datetime import datetime, timedelta + +import timely_beliefs as tb +import pandas as pd + +from flexmeasures.data.models.reporting import Reporter +from flexmeasures.data.schemas.reporting.aggregation import ( + AggregatorSchema, + AggregationMethod, +) +from flexmeasures.data.models.time_series import TimedBelief +from flexmeasures.utils.time_utils import server_now + + +class AggregatorReporter(Reporter): + """This reporter applies an aggregation function to multiple sensors""" + + __version__ = "1" + __author__ = None + schema = AggregatorSchema() + transformations: list[dict[str, Any]] = None + final_df_output: str = None + + def deserialize_config(self): + # call Reporter deserialize_config + super().deserialize_config() + + # extract AggregatorReporter specific fields + self.method = self.reporter_config.get("method") + self.weights = self.reporter_config.get("weights", dict()) + + def _compute( + self, + start: datetime, + end: datetime, + input_resolution: timedelta | None = None, + belief_time: datetime | None = None, + ) -> tb.BeliefsDataFrame: + """ + This method merges all the BeliefDataFrames into a single one, dropping + all indexes but event_start, and applies an aggregation function over the + columns. + """ + + dataframes = [] + + for belief_search_config in self.beliefs_search_configs: + # if alias is not in belief_search_config, using the Sensor id instead + column_name = belief_search_config.get( + "alias", f"sensor_{belief_search_config['sensor'].id}" + ) + data = self.data[column_name].droplevel([1, 2, 3]) + + # apply weight + if column_name in self.weights: + data *= self.weights[column_name] + + dataframes.append(data) + + output_df = pd.concat(dataframes, axis=1) + + # apply aggregation method + if self.method == AggregationMethod.SUM: + output_df = output_df.sum(axis=1) + elif self.method == AggregationMethod.MEAN: + output_df = output_df.mean(axis=1) + + # convert BeliefSeries to BeliefDataFrame + timed_beliefs = [ + TimedBelief( + sensor=output_df.sensor, + source=self.data_source, + belief_time=server_now(), + event_start=event_start, + event_value=event_value, + ) + for event_start, event_value in output_df.items() + ] + + return tb.BeliefsDataFrame(timed_beliefs) diff --git a/flexmeasures/data/models/reporting/tests/test_aggregator.py b/flexmeasures/data/models/reporting/tests/test_aggregator.py new file mode 100644 index 000000000..6627c8745 --- /dev/null +++ b/flexmeasures/data/models/reporting/tests/test_aggregator.py @@ -0,0 +1,37 @@ +import pytest + +from flexmeasures.data.models.reporting.aggregatator import AggregatorReporter + +from datetime import datetime +from pytz import utc + + +@pytest.mark.parametrize( + "aggregation_method, expected_value", [("SUM", 2), ("MEAN", 1)] +) +def test_aggregator(setup_dummy_data, aggregation_method, expected_value): + """ """ + s1, s2, reporter_sensor = setup_dummy_data + + reporter_config_raw = dict( + beliefs_search_configs=[ + dict(sensor=s1.id, source=1), + dict(sensor=s2.id, source=2), + ], # TODO: make source compulsory + method=aggregation_method, + ) + + agg_reporter = AggregatorReporter( + reporter_sensor, reporter_config_raw=reporter_config_raw + ) + + result = agg_reporter.compute( + start=datetime(2023, 5, 10, tzinfo=utc), + end=datetime(2023, 5, 11, tzinfo=utc), + ) + + # check that we got a result for 24 hours + assert len(result) == 24 + + # check that the value is equal to expected_value + assert (result == expected_value).all().event_value diff --git a/flexmeasures/data/schemas/reporting/aggregation.py b/flexmeasures/data/schemas/reporting/aggregation.py new file mode 100644 index 000000000..33eeed0fe --- /dev/null +++ b/flexmeasures/data/schemas/reporting/aggregation.py @@ -0,0 +1,65 @@ +from marshmallow import fields, ValidationError, validates_schema + +from flexmeasures.data.schemas.reporting import ReporterConfigSchema + +from enum import Enum + + +class AggregationMethod(Enum): + SUM = "SUM" + MEAN = "MEAN" + + +class AggregatorSchema(ReporterConfigSchema): + """Schema for the reporter_config of the AggregatorReporter + + Example: + .. code-block:: json + { + "beliefs_search_configs": [ + { + "sensor": 1, + "source" : 1, + "alias" : "PV" + }, + { + "sensor": 1, + "source" : 2, + "alias" : "CONSUMPTION" + } + ], + "method" : "SUM", + "weights" : { + "PV" : 1.0, + "CONSUMPTION" : -1.0 + } + } + """ + + method = fields.Enum(AggregationMethod, required=True) + weights = fields.Dict(fields.Str(), fields.Float()) + + @validates_schema + def validate_source(self, data, **kwargs): + + for beliefs_search_config in data["beliefs_search_configs"]: + if "source" not in beliefs_search_config: + raise ValidationError("`source` is a required field.") + + @validates_schema + def validate_weights(self, data, **kwargs): + if "weights" not in data: + return + + # get aliases + aliases = [] + for beliefs_search_config in data["beliefs_search_configs"]: + if "alias" in beliefs_search_config: + aliases.append(beliefs_search_config.get("alias")) + + # check the the aliases in weights are defined + for alias in data.get("weights").keys(): + if alias not in aliases: + raise ValidationError( + f"alias `{alias}` not defined in `beliefs_search_config`" + ) From 28a8b55500f522b19c86c08d1ce86a1eaaa74016 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Tue, 6 Jun 2023 15:24:47 +0200 Subject: [PATCH 04/23] feat: add fixture Signed-off-by: Victor Garcia Reolid --- .../data/models/reporting/tests/conftest.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/flexmeasures/data/models/reporting/tests/conftest.py b/flexmeasures/data/models/reporting/tests/conftest.py index 347196842..621e3515e 100644 --- a/flexmeasures/data/models/reporting/tests/conftest.py +++ b/flexmeasures/data/models/reporting/tests/conftest.py @@ -63,6 +63,20 @@ def setup_dummy_data(db, app): ) ) + # add simple data consisting of 24 hourly events with value 1.0 + # to be used to test the AggregatorReporter + for sensor, source in zip([sensor1, sensor2], [source1, source2]): + for t in range(24): + beliefs.append( + TimedBelief( + event_start=datetime(2023, 5, 10, tzinfo=utc) + timedelta(hours=t), + belief_horizon=timedelta(hours=24), + event_value=1, + sensor=sensor, + source=source, + ) + ) + db.session.add_all(beliefs) db.session.commit() From 0bc9450472a0deb23c940506411b7c51c4ca32c4 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Fri, 9 Jun 2023 11:12:28 +0200 Subject: [PATCH 05/23] fix: typo in file name Signed-off-by: Victor Garcia Reolid --- .../data/models/reporting/{aggregatator.py => aggregator.py} | 0 flexmeasures/data/models/reporting/tests/test_aggregator.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename flexmeasures/data/models/reporting/{aggregatator.py => aggregator.py} (100%) diff --git a/flexmeasures/data/models/reporting/aggregatator.py b/flexmeasures/data/models/reporting/aggregator.py similarity index 100% rename from flexmeasures/data/models/reporting/aggregatator.py rename to flexmeasures/data/models/reporting/aggregator.py diff --git a/flexmeasures/data/models/reporting/tests/test_aggregator.py b/flexmeasures/data/models/reporting/tests/test_aggregator.py index 6627c8745..47f989bc6 100644 --- a/flexmeasures/data/models/reporting/tests/test_aggregator.py +++ b/flexmeasures/data/models/reporting/tests/test_aggregator.py @@ -1,6 +1,6 @@ import pytest -from flexmeasures.data.models.reporting.aggregatator import AggregatorReporter +from flexmeasures.data.models.reporting.aggregator import AggregatorReporter from datetime import datetime from pytz import utc From 6a2b37895a0a95c025a621a2cf988b4c7075500f Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Fri, 9 Jun 2023 11:13:26 +0200 Subject: [PATCH 06/23] Set author in PandasReporter and AggregatorReporter Signed-off-by: Victor Garcia Reolid --- flexmeasures/data/models/reporting/aggregator.py | 2 +- flexmeasures/data/models/reporting/pandas_reporter.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/flexmeasures/data/models/reporting/aggregator.py b/flexmeasures/data/models/reporting/aggregator.py index 54fae8a01..28457602d 100644 --- a/flexmeasures/data/models/reporting/aggregator.py +++ b/flexmeasures/data/models/reporting/aggregator.py @@ -19,7 +19,7 @@ class AggregatorReporter(Reporter): """This reporter applies an aggregation function to multiple sensors""" __version__ = "1" - __author__ = None + __author__ = "Seita" schema = AggregatorSchema() transformations: list[dict[str, Any]] = None final_df_output: str = None diff --git a/flexmeasures/data/models/reporting/pandas_reporter.py b/flexmeasures/data/models/reporting/pandas_reporter.py index 3b98bb2f6..4169cfd69 100644 --- a/flexmeasures/data/models/reporting/pandas_reporter.py +++ b/flexmeasures/data/models/reporting/pandas_reporter.py @@ -18,7 +18,7 @@ class PandasReporter(Reporter): """This reporter applies a series of pandas methods on""" __version__ = "1" - __author__ = None + __author__ = "Seita" schema = PandasReporterConfigSchema() transformations: list[dict[str, Any]] = None final_df_output: str = None From 3447b3bad15e31d2dfa2e33c9ec70f620c516a00 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Fri, 9 Jun 2023 11:15:05 +0200 Subject: [PATCH 07/23] style: remove unnecesay class property Signed-off-by: Victor Garcia Reolid --- flexmeasures/data/models/reporting/aggregator.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/flexmeasures/data/models/reporting/aggregator.py b/flexmeasures/data/models/reporting/aggregator.py index 28457602d..7319b4467 100644 --- a/flexmeasures/data/models/reporting/aggregator.py +++ b/flexmeasures/data/models/reporting/aggregator.py @@ -1,6 +1,5 @@ from __future__ import annotations -from typing import Any from datetime import datetime, timedelta import timely_beliefs as tb @@ -21,8 +20,6 @@ class AggregatorReporter(Reporter): __version__ = "1" __author__ = "Seita" schema = AggregatorSchema() - transformations: list[dict[str, Any]] = None - final_df_output: str = None def deserialize_config(self): # call Reporter deserialize_config From 7644cd64b36964eed375f27a3c453431c77e8371 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Fri, 9 Jun 2023 12:18:43 +0200 Subject: [PATCH 08/23] test: add description of test Signed-off-by: Victor Garcia Reolid --- .../models/reporting/tests/test_aggregator.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/flexmeasures/data/models/reporting/tests/test_aggregator.py b/flexmeasures/data/models/reporting/tests/test_aggregator.py index 47f989bc6..71637a9b7 100644 --- a/flexmeasures/data/models/reporting/tests/test_aggregator.py +++ b/flexmeasures/data/models/reporting/tests/test_aggregator.py @@ -7,17 +7,27 @@ @pytest.mark.parametrize( - "aggregation_method, expected_value", [("SUM", 2), ("MEAN", 1)] + "aggregation_method, expected_value", + [ + ("SUM", 2), # for every time period, 1 (s1) + 1 (s2) = 2 (reporter_sensor) + ( + "MEAN", + 1, + ), # for every time period, mean[1 (s1), 1 (s2)] = 1 (reporter_sensor) + ], ) def test_aggregator(setup_dummy_data, aggregation_method, expected_value): - """ """ + """ + This test computes the aggreagation of two sensors containing 24 entries + with value 1.0. + """ s1, s2, reporter_sensor = setup_dummy_data reporter_config_raw = dict( beliefs_search_configs=[ dict(sensor=s1.id, source=1), dict(sensor=s2.id, source=2), - ], # TODO: make source compulsory + ], method=aggregation_method, ) From 03a91d7c8b56b34db07fa6bb54617959987fd762 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Fri, 9 Jun 2023 12:21:08 +0200 Subject: [PATCH 09/23] fix: vectorized bdf creation Signed-off-by: Victor Garcia Reolid --- .../data/models/reporting/aggregator.py | 39 ++++++++----------- 1 file changed, 17 insertions(+), 22 deletions(-) diff --git a/flexmeasures/data/models/reporting/aggregator.py b/flexmeasures/data/models/reporting/aggregator.py index 7319b4467..ffa64a6bc 100644 --- a/flexmeasures/data/models/reporting/aggregator.py +++ b/flexmeasures/data/models/reporting/aggregator.py @@ -6,11 +6,8 @@ import pandas as pd from flexmeasures.data.models.reporting import Reporter -from flexmeasures.data.schemas.reporting.aggregation import ( - AggregatorSchema, - AggregationMethod, -) -from flexmeasures.data.models.time_series import TimedBelief +from flexmeasures.data.schemas.reporting.aggregation import AggregatorSchema + from flexmeasures.utils.time_utils import server_now @@ -44,6 +41,9 @@ def _compute( dataframes = [] + if belief_time is None: + belief_time = server_now() + for belief_search_config in self.beliefs_search_configs: # if alias is not in belief_search_config, using the Sensor id instead column_name = belief_search_config.get( @@ -60,21 +60,16 @@ def _compute( output_df = pd.concat(dataframes, axis=1) # apply aggregation method - if self.method == AggregationMethod.SUM: - output_df = output_df.sum(axis=1) - elif self.method == AggregationMethod.MEAN: - output_df = output_df.mean(axis=1) - - # convert BeliefSeries to BeliefDataFrame - timed_beliefs = [ - TimedBelief( - sensor=output_df.sensor, - source=self.data_source, - belief_time=server_now(), - event_start=event_start, - event_value=event_value, - ) - for event_start, event_value in output_df.items() - ] + output_df = getattr(output_df, self.method.value)(axis=1) + + # convert BeliefsSeries into a BeliefsDataFrame + output_df = output_df.to_frame("event_value") + output_df["belief_time"] = [belief_time] * len(output_df) + output_df["cumulative_probability"] = [0.5] * len(output_df) + output_df["source"] = [self.data_source] * len(output_df) + + output_df = output_df.set_index( + ["belief_time", "source", "cumulative_probability"], append=True + ) - return tb.BeliefsDataFrame(timed_beliefs) + return output_df From cfe2e49bc3e4fb9416c2168acbcb994f0b345f06 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Fri, 9 Jun 2023 12:21:59 +0200 Subject: [PATCH 10/23] style: improve Exception message Signed-off-by: Victor Garcia Reolid --- flexmeasures/data/schemas/reporting/aggregation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/schemas/reporting/aggregation.py b/flexmeasures/data/schemas/reporting/aggregation.py index 33eeed0fe..f5d2d5964 100644 --- a/flexmeasures/data/schemas/reporting/aggregation.py +++ b/flexmeasures/data/schemas/reporting/aggregation.py @@ -61,5 +61,5 @@ def validate_weights(self, data, **kwargs): for alias in data.get("weights").keys(): if alias not in aliases: raise ValidationError( - f"alias `{alias}` not defined in `beliefs_search_config`" + f"alias `{alias}` in `weights` is not defined in `beliefs_search_config`" ) From ad4684e051ccf97dda0e5f0e50e7a05e3202f5f9 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Fri, 9 Jun 2023 12:22:20 +0200 Subject: [PATCH 11/23] fix: lowercase enum value Signed-off-by: Victor Garcia Reolid --- flexmeasures/data/schemas/reporting/aggregation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flexmeasures/data/schemas/reporting/aggregation.py b/flexmeasures/data/schemas/reporting/aggregation.py index f5d2d5964..a8d2e96d9 100644 --- a/flexmeasures/data/schemas/reporting/aggregation.py +++ b/flexmeasures/data/schemas/reporting/aggregation.py @@ -6,8 +6,8 @@ class AggregationMethod(Enum): - SUM = "SUM" - MEAN = "MEAN" + SUM = "sum" + MEAN = "mean" class AggregatorSchema(ReporterConfigSchema): From a8cadb113841bf74e61b0bf7331a4ddb01b75543 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Fri, 9 Jun 2023 12:46:51 +0200 Subject: [PATCH 12/23] feat: make method and weights optional Signed-off-by: Victor Garcia Reolid --- flexmeasures/data/schemas/reporting/aggregation.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/flexmeasures/data/schemas/reporting/aggregation.py b/flexmeasures/data/schemas/reporting/aggregation.py index a8d2e96d9..5a86741a1 100644 --- a/flexmeasures/data/schemas/reporting/aggregation.py +++ b/flexmeasures/data/schemas/reporting/aggregation.py @@ -36,8 +36,10 @@ class AggregatorSchema(ReporterConfigSchema): } """ - method = fields.Enum(AggregationMethod, required=True) - weights = fields.Dict(fields.Str(), fields.Float()) + method = fields.Enum( + AggregationMethod, required=False, dump_default=AggregationMethod.MEAN + ) + weights = fields.Dict(fields.Str(), fields.Float(), required=False) @validates_schema def validate_source(self, data, **kwargs): From 20ac071b97249ffbb5fd90904b1c24f5d6e758c3 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Fri, 9 Jun 2023 12:54:22 +0200 Subject: [PATCH 13/23] feat: set SUM as the feault aggregation method Signed-off-by: Victor Garcia Reolid --- flexmeasures/data/schemas/reporting/aggregation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/schemas/reporting/aggregation.py b/flexmeasures/data/schemas/reporting/aggregation.py index 5a86741a1..cbe6bfe45 100644 --- a/flexmeasures/data/schemas/reporting/aggregation.py +++ b/flexmeasures/data/schemas/reporting/aggregation.py @@ -37,7 +37,7 @@ class AggregatorSchema(ReporterConfigSchema): """ method = fields.Enum( - AggregationMethod, required=False, dump_default=AggregationMethod.MEAN + AggregationMethod, required=False, dump_default=AggregationMethod.SUM ) weights = fields.Dict(fields.Str(), fields.Float(), required=False) From b12054d86d49af4a6eb79b29d9bf0e72686a26aa Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 9 Jun 2023 15:21:44 +0200 Subject: [PATCH 14/23] typo Signed-off-by: F.N. Claessen --- flexmeasures/data/models/reporting/tests/test_aggregator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/models/reporting/tests/test_aggregator.py b/flexmeasures/data/models/reporting/tests/test_aggregator.py index 71637a9b7..3de645666 100644 --- a/flexmeasures/data/models/reporting/tests/test_aggregator.py +++ b/flexmeasures/data/models/reporting/tests/test_aggregator.py @@ -18,7 +18,7 @@ ) def test_aggregator(setup_dummy_data, aggregation_method, expected_value): """ - This test computes the aggreagation of two sensors containing 24 entries + This test computes the aggregation of two sensors containing 24 entries with value 1.0. """ s1, s2, reporter_sensor = setup_dummy_data From 7598b3a99b00ca7d0b58c28f5f423510cb023812 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 9 Jun 2023 15:48:34 +0200 Subject: [PATCH 15/23] refactor: simplify column value assignments Signed-off-by: F.N. Claessen --- flexmeasures/data/models/reporting/aggregator.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flexmeasures/data/models/reporting/aggregator.py b/flexmeasures/data/models/reporting/aggregator.py index ffa64a6bc..3bec644f2 100644 --- a/flexmeasures/data/models/reporting/aggregator.py +++ b/flexmeasures/data/models/reporting/aggregator.py @@ -64,9 +64,9 @@ def _compute( # convert BeliefsSeries into a BeliefsDataFrame output_df = output_df.to_frame("event_value") - output_df["belief_time"] = [belief_time] * len(output_df) - output_df["cumulative_probability"] = [0.5] * len(output_df) - output_df["source"] = [self.data_source] * len(output_df) + output_df["belief_time"] = belief_time + output_df["cumulative_probability"] = 0.5 + output_df["source"] = self.data_source output_df = output_df.set_index( ["belief_time", "source", "cumulative_probability"], append=True From 13e00d79034ea2a12affc8ded0801193ea6e92a5 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 9 Jun 2023 16:55:11 +0200 Subject: [PATCH 16/23] typo Signed-off-by: F.N. Claessen --- flexmeasures/data/schemas/reporting/aggregation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/schemas/reporting/aggregation.py b/flexmeasures/data/schemas/reporting/aggregation.py index cbe6bfe45..bf0c39962 100644 --- a/flexmeasures/data/schemas/reporting/aggregation.py +++ b/flexmeasures/data/schemas/reporting/aggregation.py @@ -59,7 +59,7 @@ def validate_weights(self, data, **kwargs): if "alias" in beliefs_search_config: aliases.append(beliefs_search_config.get("alias")) - # check the the aliases in weights are defined + # check that the aliases in weights are defined for alias in data.get("weights").keys(): if alias not in aliases: raise ValidationError( From fad8b336e09aff7189b962e61eeadacc07cfa557 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Fri, 9 Jun 2023 17:53:46 +0200 Subject: [PATCH 17/23] fix: use aggregate function instead of getattr Signed-off-by: Victor Garcia Reolid --- flexmeasures/data/models/reporting/aggregator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/models/reporting/aggregator.py b/flexmeasures/data/models/reporting/aggregator.py index 3bec644f2..952edda5c 100644 --- a/flexmeasures/data/models/reporting/aggregator.py +++ b/flexmeasures/data/models/reporting/aggregator.py @@ -60,7 +60,7 @@ def _compute( output_df = pd.concat(dataframes, axis=1) # apply aggregation method - output_df = getattr(output_df, self.method.value)(axis=1) + output_df = output_df.aggregate(self.method, axis=1) # convert BeliefsSeries into a BeliefsDataFrame output_df = output_df.to_frame("event_value") From fadae53e61e9b9b96afe2e2cb1fa62f3f98b3bc5 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Fri, 9 Jun 2023 17:54:17 +0200 Subject: [PATCH 18/23] feat: allow users to pass any string Signed-off-by: Victor Garcia Reolid --- .../data/schemas/reporting/aggregation.py | 21 ++++++------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/flexmeasures/data/schemas/reporting/aggregation.py b/flexmeasures/data/schemas/reporting/aggregation.py index bf0c39962..a42c6d7b9 100644 --- a/flexmeasures/data/schemas/reporting/aggregation.py +++ b/flexmeasures/data/schemas/reporting/aggregation.py @@ -2,13 +2,6 @@ from flexmeasures.data.schemas.reporting import ReporterConfigSchema -from enum import Enum - - -class AggregationMethod(Enum): - SUM = "sum" - MEAN = "mean" - class AggregatorSchema(ReporterConfigSchema): """Schema for the reporter_config of the AggregatorReporter @@ -20,25 +13,23 @@ class AggregatorSchema(ReporterConfigSchema): { "sensor": 1, "source" : 1, - "alias" : "PV" + "alias" : "pv" }, { "sensor": 1, "source" : 2, - "alias" : "CONSUMPTION" + "alias" : "consumption" } ], - "method" : "SUM", + "method" : "sum", "weights" : { - "PV" : 1.0, - "CONSUMPTION" : -1.0 + "pv" : 1.0, + "consumption" : -1.0 } } """ - method = fields.Enum( - AggregationMethod, required=False, dump_default=AggregationMethod.SUM - ) + method = fields.Str(required=False, dump_default="sum") weights = fields.Dict(fields.Str(), fields.Float(), required=False) @validates_schema From 3983b2bdda152d8884ac856de327d8981f595094 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Fri, 9 Jun 2023 17:55:02 +0200 Subject: [PATCH 19/23] test: add more test cases Signed-off-by: Victor Garcia Reolid --- .../data/models/reporting/tests/conftest.py | 9 ++++--- .../models/reporting/tests/test_aggregator.py | 25 ++++++++++++++----- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/flexmeasures/data/models/reporting/tests/conftest.py b/flexmeasures/data/models/reporting/tests/conftest.py index 621e3515e..bab14470b 100644 --- a/flexmeasures/data/models/reporting/tests/conftest.py +++ b/flexmeasures/data/models/reporting/tests/conftest.py @@ -63,15 +63,18 @@ def setup_dummy_data(db, app): ) ) - # add simple data consisting of 24 hourly events with value 1.0 + # add simple data consisting of 24 hourly events with value 1.0 for + # sensor1 and -1.0 for sensor2 # to be used to test the AggregatorReporter - for sensor, source in zip([sensor1, sensor2], [source1, source2]): + for sensor, source, value in zip( + [sensor1, sensor2], [source1, source2], [1.0, -1.0] + ): for t in range(24): beliefs.append( TimedBelief( event_start=datetime(2023, 5, 10, tzinfo=utc) + timedelta(hours=t), belief_horizon=timedelta(hours=24), - event_value=1, + event_value=value, sensor=sensor, source=source, ) diff --git a/flexmeasures/data/models/reporting/tests/test_aggregator.py b/flexmeasures/data/models/reporting/tests/test_aggregator.py index 3de645666..eeaf842ff 100644 --- a/flexmeasures/data/models/reporting/tests/test_aggregator.py +++ b/flexmeasures/data/models/reporting/tests/test_aggregator.py @@ -9,17 +9,30 @@ @pytest.mark.parametrize( "aggregation_method, expected_value", [ - ("SUM", 2), # for every time period, 1 (s1) + 1 (s2) = 2 (reporter_sensor) - ( - "MEAN", - 1, - ), # for every time period, mean[1 (s1), 1 (s2)] = 1 (reporter_sensor) + ("sum", 0), + ("mean", 0), + ("var", 2), + ("std", 2**0.5), + ("max", 1), + ("min", -1), + ("prod", -1), + ("median", 0), ], ) def test_aggregator(setup_dummy_data, aggregation_method, expected_value): """ This test computes the aggregation of two sensors containing 24 entries - with value 1.0. + with value 1.0 and -1, respectively, for sensors 1 and 2. + + Test cases: + 1) sum: 0 = 1 + (-1) + 2) mean: 0 = ((1) + (-1))/2 + 3) var: 2 = (1)^2 + (-1)^2 + 4) std: sqrt(2) = sqrt((1)^2 + (-1)^2) + 5) max: 1 = max(1, -1) + 6) min: -1 = min(1, -1) + 7) prod: -1 = (1) * (-1) + 8) median: even number of elements, mean of the most central elements, 0 = ((1) + (-1))/2 """ s1, s2, reporter_sensor = setup_dummy_data From d3d55dcd8a033b8cc5c61c9063f6fa75d8cae22c Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Fri, 9 Jun 2023 17:59:43 +0200 Subject: [PATCH 20/23] feat: weights and method as class attribute Signed-off-by: Victor Garcia Reolid --- flexmeasures/data/models/reporting/aggregator.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flexmeasures/data/models/reporting/aggregator.py b/flexmeasures/data/models/reporting/aggregator.py index 952edda5c..e74596ae0 100644 --- a/flexmeasures/data/models/reporting/aggregator.py +++ b/flexmeasures/data/models/reporting/aggregator.py @@ -17,6 +17,8 @@ class AggregatorReporter(Reporter): __version__ = "1" __author__ = "Seita" schema = AggregatorSchema() + weights: dict + method: str def deserialize_config(self): # call Reporter deserialize_config From 7780ecfb2e76f740193c3e773f64e58ce62f804d Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sat, 10 Jun 2023 19:15:57 +0200 Subject: [PATCH 21/23] simplify float to int Signed-off-by: F.N. Claessen --- flexmeasures/data/models/reporting/tests/conftest.py | 7 +++---- .../data/models/reporting/tests/test_aggregator.py | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/flexmeasures/data/models/reporting/tests/conftest.py b/flexmeasures/data/models/reporting/tests/conftest.py index bab14470b..e3b3ad50f 100644 --- a/flexmeasures/data/models/reporting/tests/conftest.py +++ b/flexmeasures/data/models/reporting/tests/conftest.py @@ -63,11 +63,10 @@ def setup_dummy_data(db, app): ) ) - # add simple data consisting of 24 hourly events with value 1.0 for - # sensor1 and -1.0 for sensor2 - # to be used to test the AggregatorReporter + # add simple data for testing the AggregatorReporter: + # 24 hourly events with value 1 for sensor1 and value -1 for sensor2 for sensor, source, value in zip( - [sensor1, sensor2], [source1, source2], [1.0, -1.0] + [sensor1, sensor2], [source1, source2], [1, -1] ): for t in range(24): beliefs.append( diff --git a/flexmeasures/data/models/reporting/tests/test_aggregator.py b/flexmeasures/data/models/reporting/tests/test_aggregator.py index eeaf842ff..8cd287fa4 100644 --- a/flexmeasures/data/models/reporting/tests/test_aggregator.py +++ b/flexmeasures/data/models/reporting/tests/test_aggregator.py @@ -22,7 +22,7 @@ def test_aggregator(setup_dummy_data, aggregation_method, expected_value): """ This test computes the aggregation of two sensors containing 24 entries - with value 1.0 and -1, respectively, for sensors 1 and 2. + with value 1 and -1, respectively, for sensors 1 and 2. Test cases: 1) sum: 0 = 1 + (-1) From 03cec949747a38546ebaa5b6ad4e3af5c156a67d Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sat, 10 Jun 2023 19:25:04 +0200 Subject: [PATCH 22/23] changelog entry Signed-off-by: F.N. Claessen --- documentation/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index fbe7da151..1e3bf4c2a 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -10,7 +10,6 @@ New features ------------- * Add multiple maxima and minima constraints into `StorageScheduler` [see `PR #680 `_] -* Introduction of the classes `Reporter` and `PandasReporter` [see `PR #641 `_] * Add CLI command ``flexmeasures add report`` [see `PR #659 `_] * Add CLI command ``flexmeasures show reporters`` [see `PR #686 `_] * Add CLI command ``flexmeasures show schedulers`` [see `PR #708 `_] @@ -22,6 +21,7 @@ Bugfixes Infrastructure / Support ---------------------- +* Introduction of the classes `Reporter`, `PandasReporter` and `AggregatorReporter` to help customize your own reporter functions (experimental) [see `PR #641 `_ and `PR #712 `_] * The setting FLEXMEASURES_PLUGINS can be set as environment variable now (as a comma-separated list) [see `PR #660 `_] * Packaging was modernized to stop calling setup.py directly [see `PR #671 `_] * Remove API versions 1.0, 1.1, 1.2, 1.3 and 2.0, while allowing hosts to switch between ``HTTP status 410 (Gone)`` and ``HTTP status 404 (Not Found)`` responses [see `PR #667 `_] From 3375335772a95384edd4df44ea18d969faa907a0 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sat, 10 Jun 2023 19:28:30 +0200 Subject: [PATCH 23/23] black Signed-off-by: F.N. Claessen --- flexmeasures/data/models/reporting/tests/conftest.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/flexmeasures/data/models/reporting/tests/conftest.py b/flexmeasures/data/models/reporting/tests/conftest.py index e3b3ad50f..a75a1a465 100644 --- a/flexmeasures/data/models/reporting/tests/conftest.py +++ b/flexmeasures/data/models/reporting/tests/conftest.py @@ -65,9 +65,7 @@ def setup_dummy_data(db, app): # add simple data for testing the AggregatorReporter: # 24 hourly events with value 1 for sensor1 and value -1 for sensor2 - for sensor, source, value in zip( - [sensor1, sensor2], [source1, source2], [1, -1] - ): + for sensor, source, value in zip([sensor1, sensor2], [source1, source2], [1, -1]): for t in range(24): beliefs.append( TimedBelief(