diff --git a/documentation/changelog.rst b/documentation/changelog.rst index be9094405..91eb570b0 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -27,6 +27,7 @@ Infrastructure / Support * Migrate attributes of assets, markets and weather sensors to our new sensor model [see `PR #254 `_ and `project 9 `_] * Migrate all time series data to our new sensor data model based on the `timely beliefs `_ lib [see `PR #286 `_ and `project 9 `_] * Support the new asset model (which describes the organisational structure, rather than sensors and data) in UI and API. Until the transition to our new data model is completed, the new API for assets is at `/api/dev/generic_assets`. [see `PR #251 `_ and `PR #290 `_] +* Internal search methods return most recent beliefs by default, also for charts, which can make them load a lot faster [see `PR #307 `_ and `PR #312 `_] v0.7.1 | November 08, 2021 diff --git a/flexmeasures/data/models/time_series.py b/flexmeasures/data/models/time_series.py index 304ae613a..68a99ee34 100644 --- a/flexmeasures/data/models/time_series.py +++ b/flexmeasures/data/models/time_series.py @@ -186,7 +186,7 @@ def search_beliefs( source: Optional[ Union[DataSource, List[DataSource], int, List[int], str, List[str]] ] = None, - most_recent_beliefs_only: bool = False, + most_recent_beliefs_only: bool = True, most_recent_events_only: bool = False, most_recent_only: bool = None, # deprecated one_deterministic_belief_per_event: bool = False, @@ -194,6 +194,8 @@ def search_beliefs( ) -> Union[tb.BeliefsDataFrame, str]: """Search all beliefs about events for this sensor. + If you don't set any filters, you get the most recent beliefs about all events. + :param event_starts_after: only return beliefs about events that start after this datetime (inclusive) :param event_ends_before: only return beliefs about events that end before this datetime (inclusive) :param beliefs_after: only return beliefs formed after this datetime (inclusive) @@ -378,7 +380,7 @@ def search( user_source_ids: Optional[Union[int, List[int]]] = None, source_types: Optional[List[str]] = None, exclude_source_types: Optional[List[str]] = None, - most_recent_beliefs_only: bool = False, + most_recent_beliefs_only: bool = True, most_recent_events_only: bool = False, most_recent_only: bool = None, # deprecated one_deterministic_belief_per_event: bool = False, @@ -387,6 +389,8 @@ def search( ) -> Union[tb.BeliefsDataFrame, Dict[str, tb.BeliefsDataFrame]]: """Search all beliefs about events for the given sensors. + If you don't set any filters, you get the most recent beliefs about all events. + :param sensors: search only these sensors, identified by their instance or id (both unique) or name (non-unique) :param event_starts_after: only return beliefs about events that start after this datetime (inclusive) :param event_ends_before: only return beliefs about events that end before this datetime (inclusive) diff --git a/flexmeasures/data/services/time_series.py b/flexmeasures/data/services/time_series.py index e2832ea8e..515fb8424 100644 --- a/flexmeasures/data/services/time_series.py +++ b/flexmeasures/data/services/time_series.py @@ -329,7 +329,9 @@ def drop_unchanged_beliefs(bdf: tb.BeliefsDataFrame) -> tb.BeliefsDataFrame: event_ends_before=bdf.event_ends[-1], beliefs_before=bdf.lineage.belief_times[0], # unique belief time source=bdf.lineage.sources[0], # unique source + most_recent_beliefs_only=False, ) + # todo: delete next line and set most_recent_beliefs_only=True when this is resolved: https://github.com/SeitaBV/timely-beliefs/issues/97 previous_most_recent_beliefs_in_db = belief_utils.select_most_recent_belief( previous_beliefs_in_db ) diff --git a/flexmeasures/data/tests/test_queries.py b/flexmeasures/data/tests/test_queries.py index 1f4232753..048ecccbe 100644 --- a/flexmeasures/data/tests/test_queries.py +++ b/flexmeasures/data/tests/test_queries.py @@ -233,10 +233,10 @@ def test_query_beliefs(setup_beliefs): sensor = Sensor.query.filter_by(name="epex_da").one_or_none() source = DataSource.query.filter_by(name="ENTSO-E").one_or_none() bdfs = [ - TimedBelief.search(sensor, source=source), - TimedBelief.search(sensor.id, source=source), - TimedBelief.search(sensor.name, source=source), - sensor.search_beliefs(source=source), + TimedBelief.search(sensor, source=source, most_recent_beliefs_only=False), + TimedBelief.search(sensor.id, source=source, most_recent_beliefs_only=False), + TimedBelief.search(sensor.name, source=source, most_recent_beliefs_only=False), + sensor.search_beliefs(source=source, most_recent_beliefs_only=False), tb.BeliefsDataFrame(sensor.beliefs)[ tb.BeliefsDataFrame(sensor.beliefs).index.get_level_values("source") == source @@ -255,7 +255,9 @@ def test_persist_beliefs(setup_beliefs, setup_test_data): """ sensor = Sensor.query.filter_by(name="epex_da").one_or_none() source = DataSource.query.filter_by(name="ENTSO-E").one_or_none() - bdf: tb.BeliefsDataFrame = TimedBelief.search(sensor, source=source) + bdf: tb.BeliefsDataFrame = TimedBelief.search( + sensor, source=source, most_recent_beliefs_only=False + ) # Form new beliefs df = bdf.reset_index() @@ -266,5 +268,7 @@ def test_persist_beliefs(setup_beliefs, setup_test_data): ) TimedBelief.add(bdf) - bdf: tb.BeliefsDataFrame = TimedBelief.search(sensor, source=source) + bdf: tb.BeliefsDataFrame = TimedBelief.search( + sensor, source=source, most_recent_beliefs_only=False + ) assert len(bdf) == setup_beliefs * 2 diff --git a/flexmeasures/data/tests/test_time_series_services.py b/flexmeasures/data/tests/test_time_series_services.py index 9d9589aec..d14c0cbe9 100644 --- a/flexmeasures/data/tests/test_time_series_services.py +++ b/flexmeasures/data/tests/test_time_series_services.py @@ -14,7 +14,7 @@ def test_drop_unchanged_beliefs(setup_beliefs): # Set a reference for the number of beliefs stored and their belief times sensor = Sensor.query.filter_by(name="epex_da").one_or_none() - bdf = sensor.search_beliefs() + bdf = sensor.search_beliefs(most_recent_beliefs_only=False) num_beliefs_before = len(bdf) belief_times_before = bdf.belief_times @@ -22,7 +22,7 @@ def test_drop_unchanged_beliefs(setup_beliefs): save_to_db(bdf) # Verify that no new beliefs were saved - bdf = sensor.search_beliefs() + bdf = sensor.search_beliefs(most_recent_beliefs_only=False) assert len(bdf) == num_beliefs_before # See what happens when storing all beliefs with their belief time updated @@ -32,7 +32,7 @@ def test_drop_unchanged_beliefs(setup_beliefs): save_to_db(bdf) # Verify that no new beliefs were saved - bdf = sensor.search_beliefs() + bdf = sensor.search_beliefs(most_recent_beliefs_only=False) assert len(bdf) == num_beliefs_before assert list(bdf.belief_times) == list(belief_times_before) @@ -42,7 +42,7 @@ def test_do_not_drop_beliefs_copied_by_another_source(setup_beliefs): # Set a reference for the number of beliefs stored sensor = Sensor.query.filter_by(name="epex_da").one_or_none() - bdf = sensor.search_beliefs() + bdf = sensor.search_beliefs(most_recent_beliefs_only=False) num_beliefs_before = len(bdf) # See what happens when storing all belief with their source updated @@ -53,7 +53,7 @@ def test_do_not_drop_beliefs_copied_by_another_source(setup_beliefs): save_to_db(bdf) # Verify that all the new beliefs were added - bdf = sensor.search_beliefs() + bdf = sensor.search_beliefs(most_recent_beliefs_only=False) num_beliefs_after = len(bdf) assert num_beliefs_after == 2 * num_beliefs_before @@ -68,7 +68,7 @@ def test_do_not_drop_changed_probabilistic_belief(setup_beliefs): # Set a reference for the number of beliefs stored sensor = Sensor.query.filter_by(name="epex_da").one_or_none() - bdf = sensor.search_beliefs(source="ENTSO-E") + bdf = sensor.search_beliefs(source="ENTSO-E", most_recent_beliefs_only=False) num_beliefs_before = len(bdf) # See what happens when storing a belief with more certainty one hour later @@ -91,6 +91,6 @@ def test_do_not_drop_changed_probabilistic_belief(setup_beliefs): save_to_db(new_belief) # Verify that the whole probabilistic belief was added - bdf = sensor.search_beliefs(source="ENTSO-E") + bdf = sensor.search_beliefs(source="ENTSO-E", most_recent_beliefs_only=False) num_beliefs_after = len(bdf) assert num_beliefs_after == num_beliefs_before + len(new_belief)