Issue 311 sensible defaults for searching for beliefs

documentation/changelog.rst

@@ -27,6 +27,7 @@ Infrastructure / Support
 * Migrate attributes of assets, markets and weather sensors to our new sensor model [see `PR #254 <http://www.github.com/FlexMeasures/flexmeasures/pull/254>`_ and `project 9 <http://www.github.com/FlexMeasures/flexmeasures/projects/9>`_]
 * Migrate all time series data to our new sensor data model based on the `timely beliefs <https://github.com/SeitaBV/timely-beliefs>`_ lib [see `PR #286 <http://www.github.com/FlexMeasures/flexmeasures/pull/286>`_ and `project 9 <http://www.github.com/FlexMeasures/flexmeasures/projects/9>`_]
 * Support the new asset model (which describes the organisational structure, rather than sensors and data) in UI and API. Until the transition to our new data model is completed, the new API for assets is at `/api/dev/generic_assets`. [see `PR #251 <http://www.github.com/FlexMeasures/flexmeasures/pull/251>`_ and `PR #290 <http://www.github.com/FlexMeasures/flexmeasures/pulls/290>`_]
+* Internal search methods return most recent beliefs by default, also for charts, which can make them load a lot faster [see `PR #307 <http://www.github.com/FlexMeasures/flexmeasures/pull/307>`_ and `PR #312 <http://www.github.com/FlexMeasures/flexmeasures/pull/312>`_]
 
 
 v0.7.1 | November 08, 2021

flexmeasures/data/models/time_series.py

@@ -186,14 +186,16 @@ def search_beliefs(
         source: Optional[
             Union[DataSource, List[DataSource], int, List[int], str, List[str]]
         ] = None,
-        most_recent_beliefs_only: bool = False,
+        most_recent_beliefs_only: bool = True,
         most_recent_events_only: bool = False,
         most_recent_only: bool = None,  # deprecated
         one_deterministic_belief_per_event: bool = False,
         as_json: bool = False,
     ) -> Union[tb.BeliefsDataFrame, str]:
         """Search all beliefs about events for this sensor.
 
+        If you don't set any filters, you get the most recent beliefs about all events.
+
         :param event_starts_after: only return beliefs about events that start after this datetime (inclusive)
         :param event_ends_before: only return beliefs about events that end before this datetime (inclusive)
         :param beliefs_after: only return beliefs formed after this datetime (inclusive)
@@ -378,7 +380,7 @@ def search(
         user_source_ids: Optional[Union[int, List[int]]] = None,
         source_types: Optional[List[str]] = None,
         exclude_source_types: Optional[List[str]] = None,
-        most_recent_beliefs_only: bool = False,
+        most_recent_beliefs_only: bool = True,
         most_recent_events_only: bool = False,
         most_recent_only: bool = None,  # deprecated
         one_deterministic_belief_per_event: bool = False,
@@ -387,6 +389,8 @@ def search(
     ) -> Union[tb.BeliefsDataFrame, Dict[str, tb.BeliefsDataFrame]]:
         """Search all beliefs about events for the given sensors.
 
+        If you don't set any filters, you get the most recent beliefs about all events.
+
         :param sensors: search only these sensors, identified by their instance or id (both unique) or name (non-unique)
         :param event_starts_after: only return beliefs about events that start after this datetime (inclusive)
         :param event_ends_before: only return beliefs about events that end before this datetime (inclusive)

flexmeasures/data/services/time_series.py

@@ -329,7 +329,9 @@ def drop_unchanged_beliefs(bdf: tb.BeliefsDataFrame) -> tb.BeliefsDataFrame:
         event_ends_before=bdf.event_ends[-1],
         beliefs_before=bdf.lineage.belief_times[0],  # unique belief time
         source=bdf.lineage.sources[0],  # unique source
+        most_recent_beliefs_only=False,
     )
+    # todo: delete next line and set most_recent_beliefs_only=True when this is resolved: https://github.com/SeitaBV/timely-beliefs/issues/97
     previous_most_recent_beliefs_in_db = belief_utils.select_most_recent_belief(
         previous_beliefs_in_db
     )

flexmeasures/data/tests/test_queries.py

@@ -233,10 +233,10 @@ def test_query_beliefs(setup_beliefs):
     sensor = Sensor.query.filter_by(name="epex_da").one_or_none()
     source = DataSource.query.filter_by(name="ENTSO-E").one_or_none()
     bdfs = [
-        TimedBelief.search(sensor, source=source),
-        TimedBelief.search(sensor.id, source=source),
-        TimedBelief.search(sensor.name, source=source),
-        sensor.search_beliefs(source=source),
+        TimedBelief.search(sensor, source=source, most_recent_beliefs_only=False),
+        TimedBelief.search(sensor.id, source=source, most_recent_beliefs_only=False),
+        TimedBelief.search(sensor.name, source=source, most_recent_beliefs_only=False),
+        sensor.search_beliefs(source=source, most_recent_beliefs_only=False),
         tb.BeliefsDataFrame(sensor.beliefs)[
             tb.BeliefsDataFrame(sensor.beliefs).index.get_level_values("source")
             == source
@@ -255,7 +255,9 @@ def test_persist_beliefs(setup_beliefs, setup_test_data):
     """
     sensor = Sensor.query.filter_by(name="epex_da").one_or_none()
     source = DataSource.query.filter_by(name="ENTSO-E").one_or_none()
-    bdf: tb.BeliefsDataFrame = TimedBelief.search(sensor, source=source)
+    bdf: tb.BeliefsDataFrame = TimedBelief.search(
+        sensor, source=source, most_recent_beliefs_only=False
+    )
 
     # Form new beliefs
     df = bdf.reset_index()
@@ -266,5 +268,7 @@ def test_persist_beliefs(setup_beliefs, setup_test_data):
     )
 
     TimedBelief.add(bdf)
-    bdf: tb.BeliefsDataFrame = TimedBelief.search(sensor, source=source)
+    bdf: tb.BeliefsDataFrame = TimedBelief.search(
+        sensor, source=source, most_recent_beliefs_only=False
+    )
     assert len(bdf) == setup_beliefs * 2

flexmeasures/data/tests/test_time_series_services.py

@@ -14,15 +14,15 @@ def test_drop_unchanged_beliefs(setup_beliefs):
 
     # Set a reference for the number of beliefs stored and their belief times
     sensor = Sensor.query.filter_by(name="epex_da").one_or_none()
-    bdf = sensor.search_beliefs()
+    bdf = sensor.search_beliefs(most_recent_beliefs_only=False)
     num_beliefs_before = len(bdf)
     belief_times_before = bdf.belief_times
 
     # See what happens when storing all existing beliefs verbatim
     save_to_db(bdf)
 
     # Verify that no new beliefs were saved
-    bdf = sensor.search_beliefs()
+    bdf = sensor.search_beliefs(most_recent_beliefs_only=False)
     assert len(bdf) == num_beliefs_before
 
     # See what happens when storing all beliefs with their belief time updated
@@ -32,7 +32,7 @@ def test_drop_unchanged_beliefs(setup_beliefs):
     save_to_db(bdf)
 
     # Verify that no new beliefs were saved
-    bdf = sensor.search_beliefs()
+    bdf = sensor.search_beliefs(most_recent_beliefs_only=False)
     assert len(bdf) == num_beliefs_before
     assert list(bdf.belief_times) == list(belief_times_before)
 
@@ -42,7 +42,7 @@ def test_do_not_drop_beliefs_copied_by_another_source(setup_beliefs):
 
     # Set a reference for the number of beliefs stored
     sensor = Sensor.query.filter_by(name="epex_da").one_or_none()
-    bdf = sensor.search_beliefs()
+    bdf = sensor.search_beliefs(most_recent_beliefs_only=False)
     num_beliefs_before = len(bdf)
 
     # See what happens when storing all belief with their source updated
@@ -53,7 +53,7 @@ def test_do_not_drop_beliefs_copied_by_another_source(setup_beliefs):
     save_to_db(bdf)
 
     # Verify that all the new beliefs were added
-    bdf = sensor.search_beliefs()
+    bdf = sensor.search_beliefs(most_recent_beliefs_only=False)
     num_beliefs_after = len(bdf)
     assert num_beliefs_after == 2 * num_beliefs_before
 
@@ -68,7 +68,7 @@ def test_do_not_drop_changed_probabilistic_belief(setup_beliefs):
 
     # Set a reference for the number of beliefs stored
     sensor = Sensor.query.filter_by(name="epex_da").one_or_none()
-    bdf = sensor.search_beliefs(source="ENTSO-E")
+    bdf = sensor.search_beliefs(source="ENTSO-E", most_recent_beliefs_only=False)
     num_beliefs_before = len(bdf)
 
     # See what happens when storing a belief with more certainty one hour later
@@ -91,6 +91,6 @@ def test_do_not_drop_changed_probabilistic_belief(setup_beliefs):
     save_to_db(new_belief)
 
     # Verify that the whole probabilistic belief was added
-    bdf = sensor.search_beliefs(source="ENTSO-E")
+    bdf = sensor.search_beliefs(source="ENTSO-E", most_recent_beliefs_only=False)
     num_beliefs_after = len(bdf)
     assert num_beliefs_after == num_beliefs_before + len(new_belief)