From 8a5d07275f2d6ffd0cb8c05d05461437a4912754 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Fri, 16 Jun 2023 15:23:46 +0200 Subject: [PATCH 1/6] fix: consider decimal precision when validating equality constraints Signed-off-by: Victor Garcia Reolid --- flexmeasures/data/models/planning/storage.py | 132 +++++++++++++++---- 1 file changed, 107 insertions(+), 25 deletions(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 15ae79723..060ac8f08 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -3,7 +3,7 @@ import re import copy from datetime import datetime, timedelta -from typing import List, Dict +from typing import List, Dict, Optional import pandas as pd import numpy as np @@ -187,7 +187,11 @@ def compute(self, skip_validation: bool = False) -> pd.Series | None: if len(constraint_violations) > 0: # TODO: include hints from constraint_violations into the error message - raise ValueError("The input data yields an infeasible problem.") + message = create_constraint_violations_message(constraint_violations) + raise ValueError( + "The input data yields an infeasible problem. Constraint validation has found the following issues:\n" + + message + ) # Set up EMS constraints ems_constraints = initialize_df( @@ -383,6 +387,23 @@ def ensure_soc_min_max(self): ) +def create_constraint_violations_message(constraint_violations: list) -> str: + """Create a human-readable message with the constraint_violations. + + :param constraint_violations: list with the constraint violations + :return: human-readable message + """ + message = "" + + for c in constraint_violations: + message += f"t={c['dt']} | {c['violation']}\n" + + if len(message) > 1: + message = message[:-1] + + return message + + def build_device_soc_values( soc_values: List[Dict[str, datetime | float]] | pd.Series, soc_at_start: float, @@ -578,25 +599,33 @@ def validate_storage_constraints( # 1) min >= soc_min soc_min = (soc_min - soc_at_start) * timedelta(hours=1) / resolution _constraints["soc_min(t)"] = soc_min - constraint_violations += validate_constraint(_constraints, "soc_min(t) <= min(t)") + constraint_violations += validate_constraint( + _constraints, "soc_min(t)", "<=", "min(t)" + ) # 2) max <= soc_max soc_max = (soc_max - soc_at_start) * timedelta(hours=1) / resolution _constraints["soc_max(t)"] = soc_max - constraint_violations += validate_constraint(_constraints, "max(t) <= soc_max(t)") + constraint_violations += validate_constraint( + _constraints, "max(t)", "<=", "soc_max(t)" + ) ######################################## # B. Validation in the same time frame # ######################################## # 1) min <= max - constraint_violations += validate_constraint(_constraints, "min(t) <= max(t)") + constraint_violations += validate_constraint(_constraints, "min(t)", "<=", "max(t)") # 2) min <= equals - constraint_violations += validate_constraint(_constraints, "min(t) <= equals(t)") + constraint_violations += validate_constraint( + _constraints, "min(t)", "<=", "equals(t)" + ) # 3) equals <= max - constraint_violations += validate_constraint(_constraints, "equals(t) <= max(t)") + constraint_violations += validate_constraint( + _constraints, "equals(t)", "<=", "max(t)" + ) ########################################## # C. Validation in different time frames # @@ -609,32 +638,38 @@ def validate_storage_constraints( # 1) equals(t) - equals(t-1) <= derivative_max(t) constraint_violations += validate_constraint( - _constraints, "equals(t) - equals(t-1) <= derivative_max(t) * factor_w_wh(t)" + _constraints, + "equals(t) - equals(t-1)", + "<=", + "derivative_max(t) * factor_w_wh(t)", ) # 2) derivative_min(t) <= equals(t) - equals(t-1) constraint_violations += validate_constraint( - _constraints, "derivative_min(t) * factor_w_wh(t) <= equals(t) - equals(t-1)" + _constraints, + "derivative_min(t) * factor_w_wh(t)", + "<=", + "equals(t) - equals(t-1)", ) # 3) min(t) - max(t-1) <= derivative_max(t) constraint_violations += validate_constraint( - _constraints, "min(t) - max(t-1) <= derivative_max(t) * factor_w_wh(t)" + _constraints, "min(t) - max(t-1)", "<=", "derivative_max(t) * factor_w_wh(t)" ) # 4) max(t) - min(t-1) >= derivative_min(t) constraint_violations += validate_constraint( - _constraints, "derivative_min(t) * factor_w_wh(t) <= max(t) - min(t-1)" + _constraints, "derivative_min(t) * factor_w_wh(t)", "<=", "max(t) - min(t-1)" ) # 5) equals(t) - max(t-1) <= derivative_max(t) constraint_violations += validate_constraint( - _constraints, "equals(t) - max(t-1) <= derivative_max(t) * factor_w_wh(t)" + _constraints, "equals(t) - max(t-1)", "<=", "derivative_max(t) * factor_w_wh(t)" ) # 6) derivative_min(t) <= equals(t) - min(t-1) constraint_violations += validate_constraint( - _constraints, "derivative_min(t) * factor_w_wh(t) <= equals(t) - min(t-1)" + _constraints, "derivative_min(t) * factor_w_wh(t)", "<=", "equals(t) - min(t-1)" ) return constraint_violations @@ -658,8 +693,33 @@ def get_pattern_match_word(word: str) -> str: return regex + re.escape(word) + regex +def sanitize_expression(expression: str, columns: list) -> tuple(str, list): + """Wrap column in commas to accept arbitrary column names (e.g. with spaces). + + :param expression: expression to sanitize + :param columns: list with the name of the columns of the input data for the expression. + :return: sanitized expression and columns (variables) used in the expression + """ + + _expression = copy.copy(expression) + columns_involved = [] + + for column in columns: + + if re.search(get_pattern_match_word(column), _expression): + columns_involved.append(column) + + _expression = re.sub(get_pattern_match_word(column), f"`{column}`", _expression) + + return _expression, columns_involved + + def validate_constraint( - constraints_df: pd.DataFrame, constraint_expression: str + constraints_df: pd.DataFrame, + lhs_expression: str, + inequality: str, + rhs_expression: str, + round_to_decimals: Optional[int] = 6, ) -> list[dict]: """Validate the feasibility of a given set of constraints. @@ -670,21 +730,43 @@ def validate_constraint( :return: List of constraint violations, specifying their time, constraint and violation. """ - columns_involved = [] + constraint_expression = f"{lhs_expression} {inequality} {rhs_expression}" - eval_expression = copy.copy(constraint_expression) + constraints_df_columns = list(constraints_df.columns) - for column in constraints_df.columns: - if re.search(get_pattern_match_word(column), eval_expression): - columns_involved.append(column) + lhs_expression, columns_lhs = sanitize_expression( + lhs_expression, constraints_df_columns + ) + rhs_expression, columns_rhs = sanitize_expression( + rhs_expression, constraints_df_columns + ) - eval_expression = re.sub( - get_pattern_match_word(column), f"`{column}`", eval_expression - ) + columns_involved = columns_lhs + columns_rhs + + lhs = constraints_df.fillna(0).eval(lhs_expression).round(round_to_decimals) + rhs = constraints_df.fillna(0).eval(rhs_expression).round(round_to_decimals) + + condition = None + + inequality = inequality.strip() + + if inequality == "<=": + condition = lhs <= rhs + elif inequality == "<": + condition = lhs < rhs + elif inequality == ">=": + condition = lhs >= rhs + elif inequality == ">": + condition = lhs > rhs + elif inequality == "==": + condition = lhs == rhs + elif inequality == "!=": + condition = lhs != rhs + else: + raise ValueError(f"Inequality `{inequality} not supported.") time_condition_fails = constraints_df.index[ - ~constraints_df.fillna(0).eval(eval_expression) - & ~constraints_df[columns_involved].isna().any(axis=1) + ~condition & ~constraints_df[columns_involved].isna().any(axis=1) ] constraint_violations = [] @@ -695,7 +777,7 @@ def validate_constraint( for column in constraints_df.columns: value_replaced = re.sub( get_pattern_match_word(column), - f"{column} [{constraints_df.loc[dt, column]}]", + f"{column} [{constraints_df.loc[dt, column]}] ", value_replaced, ) From 82ab89def0427368cdc43185fa0aaf1312fb5db6 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Fri, 16 Jun 2023 15:26:32 +0200 Subject: [PATCH 2/6] test: add test cases to check for the equality validation considering decimal resolution Signed-off-by: Victor Garcia Reolid --- .../data/models/planning/tests/test_solver.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py index 48f00f771..0f7fba07e 100644 --- a/flexmeasures/data/models/planning/tests/test_solver.py +++ b/flexmeasures/data/models/planning/tests/test_solver.py @@ -640,6 +640,26 @@ def test_add_storage_constraints( @pytest.mark.parametrize( "value_min1, value_equals1, value_max1, value_min2, value_equals2, value_max2, expected_constraint_type_violations", [ + (1, np.nan, 9, 1, np.nan, 9, []), # base case + (1, np.nan, 10, 1, np.nan, 10, []), # exact equality + ( + 1, + np.nan, + 10 + 0.5e-6, + 1, + np.nan, + 10, + [], + ), # equality considering the precision (6 decimal figures) + ( + 1, + np.nan, + 10 + 1e-5, + 1, + np.nan, + 10, + ["max(t) <= soc_max(t)"], + ), # difference of 0.5e-5 > 1e-6 (1, np.nan, 9, 2, np.nan, 20, ["max(t) <= soc_max(t)"]), (-1, np.nan, 9, 1, np.nan, 9, ["soc_min(t) <= min(t)"]), (1, 10, 9, 1, np.nan, 9, ["equals(t) <= max(t)"]), From b132594356e1491455d1669ecf0536ac41ade290 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Fri, 16 Jun 2023 15:34:01 +0200 Subject: [PATCH 3/6] fix: add self.round_to_decimals to create_constraint_violations_message Signed-off-by: Victor Garcia Reolid --- flexmeasures/data/models/planning/storage.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 060ac8f08..0e6df7c92 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -187,7 +187,9 @@ def compute(self, skip_validation: bool = False) -> pd.Series | None: if len(constraint_violations) > 0: # TODO: include hints from constraint_violations into the error message - message = create_constraint_violations_message(constraint_violations) + message = create_constraint_violations_message( + constraint_violations, self.round_to_decimals + ) raise ValueError( "The input data yields an infeasible problem. Constraint validation has found the following issues:\n" + message From f9e401c0a4dc5be207d54329c66018e907dafb0d Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 16 Jun 2023 20:45:01 +0200 Subject: [PATCH 4/6] fix: fix and modernize type annotations Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/storage.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 0e6df7c92..2ab59f954 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -3,7 +3,6 @@ import re import copy from datetime import datetime, timedelta -from typing import List, Dict, Optional import pandas as pd import numpy as np @@ -407,7 +406,7 @@ def create_constraint_violations_message(constraint_violations: list) -> str: def build_device_soc_values( - soc_values: List[Dict[str, datetime | float]] | pd.Series, + soc_values: list[dict[str, datetime | float]] | pd.Series, soc_at_start: float, start_of_schedule: datetime, end_of_schedule: datetime, @@ -476,9 +475,9 @@ def add_storage_constraints( end: datetime, resolution: timedelta, soc_at_start: float, - soc_targets: List[Dict[str, datetime | float]] | pd.Series | None, - soc_maxima: List[Dict[str, datetime | float]] | pd.Series | None, - soc_minima: List[Dict[str, datetime | float]] | pd.Series | None, + soc_targets: list[dict[str, datetime | float]] | pd.Series | None, + soc_maxima: list[dict[str, datetime | float]] | pd.Series | None, + soc_minima: list[dict[str, datetime | float]] | pd.Series | None, soc_max: float, soc_min: float, ) -> pd.DataFrame: @@ -695,7 +694,7 @@ def get_pattern_match_word(word: str) -> str: return regex + re.escape(word) + regex -def sanitize_expression(expression: str, columns: list) -> tuple(str, list): +def sanitize_expression(expression: str, columns: list) -> tuple[str, list]: """Wrap column in commas to accept arbitrary column names (e.g. with spaces). :param expression: expression to sanitize @@ -721,7 +720,7 @@ def validate_constraint( lhs_expression: str, inequality: str, rhs_expression: str, - round_to_decimals: Optional[int] = 6, + round_to_decimals: int | None = 6, ) -> list[dict]: """Validate the feasibility of a given set of constraints. @@ -814,7 +813,7 @@ def prepend_serie(serie: pd.Series, value) -> pd.Series: #################### @deprecated(build_device_soc_values, "0.14") def build_device_soc_targets( - targets: List[Dict[str, datetime | float]] | pd.Series, + targets: list[dict[str, datetime | float]] | pd.Series, soc_at_start: float, start_of_schedule: datetime, end_of_schedule: datetime, From 161ca9de1209df7cb31820e902c64be6ecee953c Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 16 Jun 2023 20:51:47 +0200 Subject: [PATCH 5/6] fix(docs): update docstring with split and new parameters Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/storage.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 2ab59f954..a46f5a88b 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -724,11 +724,16 @@ def validate_constraint( ) -> list[dict]: """Validate the feasibility of a given set of constraints. - :param constraints_df: DataFrame with the constraints - :param constraint_expression: inequality expression following pd.eval format. - No need to use the syntax `column` to reference - column, just use the column name. - :return: List of constraint violations, specifying their time, constraint and violation. + :param constraints_df: DataFrame with the constraints + :param lhs_expression: left-hand side of the inequality expression following pd.eval format. + No need to use the syntax `column` to reference + column, just use the column name. + :param inequality: inequality operator, one of ('<=', '<', '>=', '>', '==', '!='). + :param rhs_expression: right-hand side of the inequality expression following pd.eval format. + No need to use the syntax `column` to reference + column, just use the column name. + :param round_to_decimals: Number of decimals to round off to before validating constraints. + :return: List of constraint violations, specifying their time, constraint and violation. """ constraint_expression = f"{lhs_expression} {inequality} {rhs_expression}" From a7c28478bc8405d711d881c9c431354cc12aea6b Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 16 Jun 2023 20:55:07 +0200 Subject: [PATCH 6/6] docs: changelog entry Signed-off-by: F.N. Claessen --- documentation/changelog.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 069eb9f8e..98260ae8a 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -16,6 +16,15 @@ Infrastructure / Support ---------------------- +v0.14.1 | June XX, 2023 +============================ + +Bugfixes +----------- + +* Relax constraint validation of `StorageScheduler` to accommodate violations caused by floating point precision [see `PR #731 `_] + + v0.14.0 | June 15, 2023 ============================