From 713d0cd3c45a350f780c2fc55af0bd9db68093d3 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 12 Jan 2022 15:05:47 +0100 Subject: [PATCH 1/5] Sensor.search_beliefs loads most recent beliefs by default Signed-off-by: F.N. Claessen --- flexmeasures/data/models/time_series.py | 2 +- flexmeasures/data/tests/test_queries.py | 2 +- .../data/tests/test_time_series_services.py | 14 +++++++------- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/flexmeasures/data/models/time_series.py b/flexmeasures/data/models/time_series.py index 304ae613a..5e3972e23 100644 --- a/flexmeasures/data/models/time_series.py +++ b/flexmeasures/data/models/time_series.py @@ -186,7 +186,7 @@ def search_beliefs( source: Optional[ Union[DataSource, List[DataSource], int, List[int], str, List[str]] ] = None, - most_recent_beliefs_only: bool = False, + most_recent_beliefs_only: bool = True, most_recent_events_only: bool = False, most_recent_only: bool = None, # deprecated one_deterministic_belief_per_event: bool = False, diff --git a/flexmeasures/data/tests/test_queries.py b/flexmeasures/data/tests/test_queries.py index 1f4232753..651a5294f 100644 --- a/flexmeasures/data/tests/test_queries.py +++ b/flexmeasures/data/tests/test_queries.py @@ -236,7 +236,7 @@ def test_query_beliefs(setup_beliefs): TimedBelief.search(sensor, source=source), TimedBelief.search(sensor.id, source=source), TimedBelief.search(sensor.name, source=source), - sensor.search_beliefs(source=source), + sensor.search_beliefs(source=source, most_recent_beliefs_only=False), tb.BeliefsDataFrame(sensor.beliefs)[ tb.BeliefsDataFrame(sensor.beliefs).index.get_level_values("source") == source diff --git a/flexmeasures/data/tests/test_time_series_services.py b/flexmeasures/data/tests/test_time_series_services.py index 9d9589aec..d14c0cbe9 100644 --- a/flexmeasures/data/tests/test_time_series_services.py +++ b/flexmeasures/data/tests/test_time_series_services.py @@ -14,7 +14,7 @@ def test_drop_unchanged_beliefs(setup_beliefs): # Set a reference for the number of beliefs stored and their belief times sensor = Sensor.query.filter_by(name="epex_da").one_or_none() - bdf = sensor.search_beliefs() + bdf = sensor.search_beliefs(most_recent_beliefs_only=False) num_beliefs_before = len(bdf) belief_times_before = bdf.belief_times @@ -22,7 +22,7 @@ def test_drop_unchanged_beliefs(setup_beliefs): save_to_db(bdf) # Verify that no new beliefs were saved - bdf = sensor.search_beliefs() + bdf = sensor.search_beliefs(most_recent_beliefs_only=False) assert len(bdf) == num_beliefs_before # See what happens when storing all beliefs with their belief time updated @@ -32,7 +32,7 @@ def test_drop_unchanged_beliefs(setup_beliefs): save_to_db(bdf) # Verify that no new beliefs were saved - bdf = sensor.search_beliefs() + bdf = sensor.search_beliefs(most_recent_beliefs_only=False) assert len(bdf) == num_beliefs_before assert list(bdf.belief_times) == list(belief_times_before) @@ -42,7 +42,7 @@ def test_do_not_drop_beliefs_copied_by_another_source(setup_beliefs): # Set a reference for the number of beliefs stored sensor = Sensor.query.filter_by(name="epex_da").one_or_none() - bdf = sensor.search_beliefs() + bdf = sensor.search_beliefs(most_recent_beliefs_only=False) num_beliefs_before = len(bdf) # See what happens when storing all belief with their source updated @@ -53,7 +53,7 @@ def test_do_not_drop_beliefs_copied_by_another_source(setup_beliefs): save_to_db(bdf) # Verify that all the new beliefs were added - bdf = sensor.search_beliefs() + bdf = sensor.search_beliefs(most_recent_beliefs_only=False) num_beliefs_after = len(bdf) assert num_beliefs_after == 2 * num_beliefs_before @@ -68,7 +68,7 @@ def test_do_not_drop_changed_probabilistic_belief(setup_beliefs): # Set a reference for the number of beliefs stored sensor = Sensor.query.filter_by(name="epex_da").one_or_none() - bdf = sensor.search_beliefs(source="ENTSO-E") + bdf = sensor.search_beliefs(source="ENTSO-E", most_recent_beliefs_only=False) num_beliefs_before = len(bdf) # See what happens when storing a belief with more certainty one hour later @@ -91,6 +91,6 @@ def test_do_not_drop_changed_probabilistic_belief(setup_beliefs): save_to_db(new_belief) # Verify that the whole probabilistic belief was added - bdf = sensor.search_beliefs(source="ENTSO-E") + bdf = sensor.search_beliefs(source="ENTSO-E", most_recent_beliefs_only=False) num_beliefs_after = len(bdf) assert num_beliefs_after == num_beliefs_before + len(new_belief) From 6f6452cd529a94b2ee69a5a5e6114f99e05826a3 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 12 Jan 2022 15:06:52 +0100 Subject: [PATCH 2/5] TimedBelief.search loads most recent beliefs by default Signed-off-by: F.N. Claessen --- flexmeasures/data/models/time_series.py | 2 +- flexmeasures/data/services/time_series.py | 1 + flexmeasures/data/tests/test_queries.py | 14 +++++++++----- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/flexmeasures/data/models/time_series.py b/flexmeasures/data/models/time_series.py index 5e3972e23..b365eb959 100644 --- a/flexmeasures/data/models/time_series.py +++ b/flexmeasures/data/models/time_series.py @@ -378,7 +378,7 @@ def search( user_source_ids: Optional[Union[int, List[int]]] = None, source_types: Optional[List[str]] = None, exclude_source_types: Optional[List[str]] = None, - most_recent_beliefs_only: bool = False, + most_recent_beliefs_only: bool = True, most_recent_events_only: bool = False, most_recent_only: bool = None, # deprecated one_deterministic_belief_per_event: bool = False, diff --git a/flexmeasures/data/services/time_series.py b/flexmeasures/data/services/time_series.py index e2832ea8e..5b1538d72 100644 --- a/flexmeasures/data/services/time_series.py +++ b/flexmeasures/data/services/time_series.py @@ -329,6 +329,7 @@ def drop_unchanged_beliefs(bdf: tb.BeliefsDataFrame) -> tb.BeliefsDataFrame: event_ends_before=bdf.event_ends[-1], beliefs_before=bdf.lineage.belief_times[0], # unique belief time source=bdf.lineage.sources[0], # unique source + most_recent_beliefs_only=False, ) previous_most_recent_beliefs_in_db = belief_utils.select_most_recent_belief( previous_beliefs_in_db diff --git a/flexmeasures/data/tests/test_queries.py b/flexmeasures/data/tests/test_queries.py index 651a5294f..048ecccbe 100644 --- a/flexmeasures/data/tests/test_queries.py +++ b/flexmeasures/data/tests/test_queries.py @@ -233,9 +233,9 @@ def test_query_beliefs(setup_beliefs): sensor = Sensor.query.filter_by(name="epex_da").one_or_none() source = DataSource.query.filter_by(name="ENTSO-E").one_or_none() bdfs = [ - TimedBelief.search(sensor, source=source), - TimedBelief.search(sensor.id, source=source), - TimedBelief.search(sensor.name, source=source), + TimedBelief.search(sensor, source=source, most_recent_beliefs_only=False), + TimedBelief.search(sensor.id, source=source, most_recent_beliefs_only=False), + TimedBelief.search(sensor.name, source=source, most_recent_beliefs_only=False), sensor.search_beliefs(source=source, most_recent_beliefs_only=False), tb.BeliefsDataFrame(sensor.beliefs)[ tb.BeliefsDataFrame(sensor.beliefs).index.get_level_values("source") @@ -255,7 +255,9 @@ def test_persist_beliefs(setup_beliefs, setup_test_data): """ sensor = Sensor.query.filter_by(name="epex_da").one_or_none() source = DataSource.query.filter_by(name="ENTSO-E").one_or_none() - bdf: tb.BeliefsDataFrame = TimedBelief.search(sensor, source=source) + bdf: tb.BeliefsDataFrame = TimedBelief.search( + sensor, source=source, most_recent_beliefs_only=False + ) # Form new beliefs df = bdf.reset_index() @@ -266,5 +268,7 @@ def test_persist_beliefs(setup_beliefs, setup_test_data): ) TimedBelief.add(bdf) - bdf: tb.BeliefsDataFrame = TimedBelief.search(sensor, source=source) + bdf: tb.BeliefsDataFrame = TimedBelief.search( + sensor, source=source, most_recent_beliefs_only=False + ) assert len(bdf) == setup_beliefs * 2 From 57d8dcf8632a7b83273cf5af35e8cecd2b21aa3b Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 12 Jan 2022 15:09:19 +0100 Subject: [PATCH 3/5] Add docstring comments Signed-off-by: F.N. Claessen --- flexmeasures/data/models/time_series.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/flexmeasures/data/models/time_series.py b/flexmeasures/data/models/time_series.py index b365eb959..68a99ee34 100644 --- a/flexmeasures/data/models/time_series.py +++ b/flexmeasures/data/models/time_series.py @@ -194,6 +194,8 @@ def search_beliefs( ) -> Union[tb.BeliefsDataFrame, str]: """Search all beliefs about events for this sensor. + If you don't set any filters, you get the most recent beliefs about all events. + :param event_starts_after: only return beliefs about events that start after this datetime (inclusive) :param event_ends_before: only return beliefs about events that end before this datetime (inclusive) :param beliefs_after: only return beliefs formed after this datetime (inclusive) @@ -387,6 +389,8 @@ def search( ) -> Union[tb.BeliefsDataFrame, Dict[str, tb.BeliefsDataFrame]]: """Search all beliefs about events for the given sensors. + If you don't set any filters, you get the most recent beliefs about all events. + :param sensors: search only these sensors, identified by their instance or id (both unique) or name (non-unique) :param event_starts_after: only return beliefs about events that start after this datetime (inclusive) :param event_ends_before: only return beliefs about events that end before this datetime (inclusive) From b71d89c78b5ee202029bd00a7e99a1a4ed3fa6ff Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 12 Jan 2022 17:38:14 +0100 Subject: [PATCH 4/5] Changelog entry Signed-off-by: F.N. Claessen --- documentation/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index be9094405..91eb570b0 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -27,6 +27,7 @@ Infrastructure / Support * Migrate attributes of assets, markets and weather sensors to our new sensor model [see `PR #254 `_ and `project 9 `_] * Migrate all time series data to our new sensor data model based on the `timely beliefs `_ lib [see `PR #286 `_ and `project 9 `_] * Support the new asset model (which describes the organisational structure, rather than sensors and data) in UI and API. Until the transition to our new data model is completed, the new API for assets is at `/api/dev/generic_assets`. [see `PR #251 `_ and `PR #290 `_] +* Internal search methods return most recent beliefs by default, also for charts, which can make them load a lot faster [see `PR #307 `_ and `PR #312 `_] v0.7.1 | November 08, 2021 From 48834736e8483a43f2a1d8a3a564bae5e4dba536 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 17 Jan 2022 09:38:06 +0100 Subject: [PATCH 5/5] Add todo for open timely-beliefs issue Signed-off-by: F.N. Claessen --- flexmeasures/data/services/time_series.py | 1 + 1 file changed, 1 insertion(+) diff --git a/flexmeasures/data/services/time_series.py b/flexmeasures/data/services/time_series.py index 5b1538d72..515fb8424 100644 --- a/flexmeasures/data/services/time_series.py +++ b/flexmeasures/data/services/time_series.py @@ -331,6 +331,7 @@ def drop_unchanged_beliefs(bdf: tb.BeliefsDataFrame) -> tb.BeliefsDataFrame: source=bdf.lineage.sources[0], # unique source most_recent_beliefs_only=False, ) + # todo: delete next line and set most_recent_beliefs_only=True when this is resolved: https://github.com/SeitaBV/timely-beliefs/issues/97 previous_most_recent_beliefs_in_db = belief_utils.select_most_recent_belief( previous_beliefs_in_db )