feat: Support Pandas 2 (#673)

* Unlink internal logic from API endpoints Signed-off-by: F.N. Claessen <felix@seita.nl> * Remove endpoint implementations Signed-off-by: F.N. Claessen <felix@seita.nl> * Turning off the sunset switch now starts returning 404 responses Signed-off-by: F.N. Claessen <felix@seita.nl> * changelog entries Signed-off-by: F.N. Claessen <felix@seita.nl> * typo Signed-off-by: F.N. Claessen <felix@seita.nl> * Fix tests: missing inclusive Signed-off-by: F.N. Claessen <felix@seita.nl> * Update docs for Pandas 2 syntax Signed-off-by: F.N. Claessen <felix@seita.nl> * Fix test due to failure to propagate metadata on empty DataFrames Signed-off-by: F.N. Claessen <felix@seita.nl> * upgrade timetomodel Signed-off-by: F.N. Claessen <felix@seita.nl> * upgrade timely-beliefs Signed-off-by: F.N. Claessen <felix@seita.nl> * fix: workaround for misbehaving pd.Timedelta Signed-off-by: F.N. Claessen <felix@seita.nl> * fix: modernize use of .loc in test Signed-off-by: F.N. Claessen <felix@seita.nl> * docs: changelog entry Signed-off-by: F.N. Claessen <felix@seita.nl> * fix: use belief_time if passed, and avoid using iteritems Signed-off-by: F.N. Claessen <felix@seita.nl> * fix: flake8 Signed-off-by: F.N. Claessen <felix@seita.nl> --------- Signed-off-by: F.N. Claessen <felix@seita.nl>
FlexMeasures · Jun 13, 2023 · d4178d6 · d4178d6
1 parent 4185880
commit d4178d6
Show file tree

Hide file tree

Showing 8 changed files with 24 additions and 25 deletions.
diff --git a/documentation/changelog.rst b/documentation/changelog.rst
@@ -25,6 +25,7 @@ Infrastructure / Support
 * The setting FLEXMEASURES_PLUGINS can be set as environment variable now (as a comma-separated list) [see `PR #660 <https://www.github.com/FlexMeasures/flexmeasures/pull/660>`_]
 * Packaging was modernized to stop calling setup.py directly [see `PR #671 <https://www.github.com/FlexMeasures/flexmeasures/pull/671>`_]
 * Remove API versions 1.0, 1.1, 1.2, 1.3 and 2.0, while making sure that sunset endpoints keep returning ``HTTP status 410 (Gone)`` responses [see `PR #667 <https://www.github.com/FlexMeasures/flexmeasures/pull/667>`_ and `PR #717 <https://www.github.com/FlexMeasures/flexmeasures/pull/717>`_]
+* Support Pandas 2 [see `PR #673 <https://www.github.com/FlexMeasures/flexmeasures/pull/673>`_]
 * Add code documentation from package structure and docstrings to official docs [see `PR #698 <https://www.github.com/FlexMeasures/flexmeasures/pull/698>`_]
 
 .. warning:: The setting `FLEXMEASURES_PLUGIN_PATHS` has been deprecated since v0.7. It has now been sunset. Please replace it with :ref:`plugin-config`.

diff --git a/documentation/plugin/customisation.rst b/documentation/plugin/customisation.rst
@@ -42,7 +42,7 @@ The following minimal example gives you an idea of some meta information you can
             """
             return pd.Series(
                 self.sensor.get_attribute("capacity_in_mw"),
-                index=pd.date_range(self.start, self.end, freq=self.resolution, closed="left"),
+                index=pd.date_range(self.start, self.end, freq=self.resolution, inclusive="left"),
             )
     
         def deserialize_config(self):

diff --git a/flexmeasures/data/models/planning/linear_optimization.py b/flexmeasures/data/models/planning/linear_optimization.py
@@ -74,7 +74,8 @@ def device_scheduler(  # noqa C901
 
     # Check if commitments have the same time window and resolution as the constraints
     start = device_constraints[0].index.to_pydatetime()[0]
-    resolution = pd.to_timedelta(device_constraints[0].index.freq)
+    # Workaround for https://github.com/pandas-dev/pandas/issues/53643. Was: resolution = pd.to_timedelta(device_constraints[0].index.freq)
+    resolution = pd.to_timedelta(device_constraints[0].index.freq).to_pytimedelta()
     end = device_constraints[0].index.to_pydatetime()[-1] + resolution
     if len(commitment_quantities) != 0:
         start_c = commitment_quantities[0].index.to_pydatetime()[0]

diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py
@@ -566,15 +566,15 @@ def compute_schedule(flex_model):
 
     # test for soc_minima
     # check that the local minimum constraint is respected
-    assert soc_schedule2.loc[datetime(2015, 1, 2, 7)] >= 3.5
+    assert soc_schedule2.loc["2015-01-02T08:00:00+01:00"] >= 3.5
 
     # test for soc_maxima
     # check that the local maximum constraint is respected
-    assert soc_schedule2.loc[datetime(2015, 1, 2, 14)] <= 1.0
+    assert soc_schedule2.loc["2015-01-02T15:00:00+01:00"] <= 1.0
 
     # test for soc_targets
     # check that the SOC target (at 19 pm, local time) is met
-    assert soc_schedule2.loc[datetime(2015, 1, 2, 18)] == 2.0
+    assert soc_schedule2.loc["2015-01-02T19:00:00+01:00"] == 2.0
 
 
 @pytest.mark.parametrize(

diff --git a/flexmeasures/data/models/reporting/pandas_reporter.py b/flexmeasures/data/models/reporting/pandas_reporter.py
@@ -10,7 +10,6 @@
 from flexmeasures.data.schemas.reporting.pandas_reporter import (
     PandasReporterConfigSchema,
 )
-from flexmeasures.data.models.time_series import TimedBelief
 from flexmeasures.utils.time_utils import server_now
 
 
@@ -43,6 +42,9 @@ def _compute(
         defined in `final_df_output` field of the report_config.
         """
 
+        if belief_time is None:
+            belief_time = server_now()
+
         # apply pandas transformations to the dataframes in `self.data`
         self._apply_transformations()
 
@@ -51,9 +53,9 @@ def _compute(
         if isinstance(final_output, tb.BeliefsDataFrame):
 
             # filing the missing indexes with default values:
-            # belief_time=server_now(), cummulative_probability=0.5, source=data_source
+            # belief_time=belief_time, cummulative_probability=0.5, source=data_source
             if "belief_time" not in final_output.index.names:
-                final_output["belief_time"] = [server_now()] * len(final_output)
+                final_output["belief_time"] = [belief_time] * len(final_output)
                 final_output = final_output.set_index("belief_time", append=True)
 
             if "cumulative_probability" not in final_output.index.names:
@@ -71,18 +73,13 @@ def _compute(
             )
 
         elif isinstance(final_output, tb.BeliefsSeries):
-
-            timed_beliefs = [
-                TimedBelief(
-                    sensor=final_output.sensor,
-                    source=self.data_source,
-                    belief_time=server_now(),
-                    event_start=event_start,
-                    event_value=event_value,
-                )
-                for event_start, event_value in final_output.iteritems()
-            ]
-            final_output = tb.BeliefsDataFrame(timed_beliefs)
+            final_output = final_output.to_frame("event_value")
+            final_output["belief_time"] = belief_time
+            final_output["cumulative_probability"] = 0.5
+            final_output["source"] = self.data_source
+            final_output = final_output.set_index(
+                ["belief_time", "source", "cumulative_probability"], append=True
+            )
 
         return final_output
 

diff --git a/flexmeasures/data/models/time_series.py b/flexmeasures/data/models/time_series.py
@@ -711,7 +711,7 @@ def search(
                 # todo: compute median of collective belief instead of median of first belief (update expected test results accordingly)
                 # todo: move to timely-beliefs: select mean/median belief
                 if (
-                    bdf.lineage.number_of_sources == 1
+                    bdf.lineage.number_of_sources <= 1
                     and bdf.lineage.probabilistic_depth == 1
                 ):
                     # Fast track, no need to loop over beliefs

diff --git a/requirements/app.in b/requirements/app.in
@@ -29,8 +29,8 @@ redis >4.5, <5
 tldextract
 pyomo>=5.6
 tabulate
-timetomodel>=0.7.1
-timely-beliefs[forecast]>=1.18
+timetomodel>=0.7.3
+timely-beliefs[forecast]>=1.20.1
 python-dotenv
 # a backport, not needed in Python3.8
 importlib_metadata

diff --git a/requirements/app.txt b/requirements/app.txt
@@ -314,9 +314,9 @@ tabulate==0.9.0
     # via -r requirements/app.in
 threadpoolctl==3.1.0
     # via scikit-learn
-timely-beliefs[forecast]==1.19.0
+timely-beliefs[forecast]==1.20.1
     # via -r requirements/app.in
-timetomodel==0.7.2
+timetomodel==0.7.3
     # via -r requirements/app.in
 tldextract==3.4.0
     # via -r requirements/app.in