Merge pull request #295 from NREL/logic_validator

Add validation for common apply logic error.

.github/workflows/ci.yml

@@ -6,7 +6,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.7', '3.8', '3.9', '3.10']
+        python-version: ['3.8', '3.9', '3.10']
     name: Tests - Python ${{ matrix.python-version }}
     steps:
       - uses: actions/checkout@v2

buildstockbatch/base.py

@@ -23,6 +23,7 @@
 import zipfile
 import csv
 from collections import defaultdict, Counter
+import pprint
 
 from buildstockbatch.__version__ import __schema_version__
 from buildstockbatch import (
@@ -255,6 +256,7 @@ def validate_project(project_file):
         assert(BuildStockBatchBase.validate_xor_nor_schema_keys(project_file))
         assert(BuildStockBatchBase.validate_reference_scenario(project_file))
         assert(BuildStockBatchBase.validate_options_lookup(project_file))
+        assert(BuildStockBatchBase.validate_logic(project_file))
         assert(BuildStockBatchBase.validate_measure_references(project_file))
         assert(BuildStockBatchBase.validate_postprocessing_spec(project_file))
         assert(BuildStockBatchBase.validate_resstock_version(project_file))
@@ -431,7 +433,7 @@ def get_all_option_str(source_str, inp):
                             in enumerate(inp)], [])
             elif type(inp) == dict:
                 if len(inp) > 1:
-                    raise ValueError(f"{source_str} the logic is malformed.")
+                    raise ValidationError(f"{source_str} the logic is malformed. Dict can't have more than one entry")
                 source_str += f", in {list(inp.keys())[0]}"
                 return sum([get_all_option_str(source_str, i) for i in inp.values()], [])
 
@@ -500,7 +502,89 @@ def get_all_option_str(source_str, inp):
             return True
         else:
             logger.error(error_message)
-            raise ValueError(error_message)
+            raise ValidationError(error_message)
+
+    @staticmethod
+    def validate_logic(project_file):
+        """
+        Validates that the apply logic has basic consistency.
+        Currently checks the following rules:
+        1. A 'and' block or a 'not' block doesn't contain two identical options entry. For example, the following is an
+        invalid block because no building can have both of those characteristics.
+        not:
+           - HVAC Heating Efficiency|ASHP, SEER 10, 6.2 HSPF
+           - HVAC Heating Efficiency|ASHP, SEER 13, 7.7 HSPF
+        """
+        cfg = get_project_configuration(project_file)
+
+        printer = pprint.PrettyPrinter()
+
+        def get_option(element):
+            return element.split('|')[0] if isinstance(element, str) else None
+
+        def get_logic_problems(logic, parent=None):
+            if isinstance(logic, list):
+                all_options = [opt for el in logic if (opt := get_option(el))]
+                problems = []
+                if parent in ['not', 'and', None, '&&']:
+                    for opt, count in Counter(all_options).items():
+                        if count > 1:
+                            parent_name = parent or 'and'
+                            problem_text = f"Option '{opt}' occurs {count} times in a '{parent_name}' block. "\
+                                f"It should occur at max one times. This is the block:\n{printer.pformat(logic)}"
+                            if parent is None:
+                                problem_text += "\nRemember a list without a parent is considered an 'and' block."
+                            problems.append(problem_text)
+                for el in logic:
+                    problems += get_logic_problems(el)
+                return problems
+            elif isinstance(logic, dict):
+                assert len(logic) == 1
+                for key, val in logic.items():
+                    if key not in ['or', 'and', 'not']:
+                        raise ValidationError(f"Invalid key {key}. Only 'or', 'and' and 'not' is allowed.")
+                    return get_logic_problems(val, parent=key)
+            elif isinstance(logic, str):
+                if '&&' not in logic:
+                    return []
+                entries = logic.split('&&')
+                return get_logic_problems(entries, parent="&&")
+            else:
+                raise ValidationError(f"Invalid logic element {logic} with type {type(logic)}")
+
+        all_problems = []
+        if 'upgrades' in cfg:
+            for upgrade_count, upgrade in enumerate(cfg['upgrades']):
+                upgrade_name = upgrade.get('upgrade_name', '')
+                source_str_upgrade = f"upgrade '{upgrade_name}' (Upgrade Number:{upgrade_count})"
+                for option_count, option in enumerate(upgrade['options']):
+                    option_name = option.get('option', '')
+                    source_str_option = source_str_upgrade + f", option '{option_name}' (Option Number:{option_count})"
+                    if 'apply_logic' in option:
+                        if problems := get_logic_problems(option['apply_logic']):
+                            all_problems.append((source_str_option, problems, option['apply_logic']))
+
+                if 'package_apply_logic' in upgrade:
+                    source_str_package = source_str_upgrade + ", in package_apply_logic"
+                    if problems := get_logic_problems(upgrade['package_apply_logic']):
+                        all_problems.append((source_str_package, problems, upgrade['package_apply_logic']))
+
+        #  TODO: refactor this into Sampler.validate_args
+        if 'downselect' in cfg or "downselect" in cfg.get('sampler', {}).get('type'):
+            source_str = "in downselect logic"
+            logic = cfg['downselect']['logic'] if 'downselect' in cfg else cfg['sampler']['args']['logic']
+            if problems := get_logic_problems(logic):
+                all_problems.append((source_str, problems, logic))
+
+        if all_problems:
+            error_str = ''
+            for location, problems, logic in all_problems:
+                error_str += f"There are following problems in {location} with this logic\n{printer.pformat(logic)}\n"
+                problem_str = "\n".join(problems)
+                error_str += f"The problems are:\n{problem_str}\n"
+            raise ValidationError(error_str)
+        else:
+            return True
 
     @staticmethod
     def validate_measure_references(project_file):
@@ -556,7 +640,7 @@ def get_errors(source_str, measure_str):
         else:
             error_message = 'Measure name(s)/directory(ies) is(are) invalid. \n' + error_message
             logger.error(error_message)
-            raise ValueError(error_message)
+            raise ValidationError(error_message)
 
     @staticmethod
     def validate_reference_scenario(project_file):

buildstockbatch/test/test_inputs/complete-schema.yml

@@ -72,7 +72,7 @@ upgrades:
             - Lighting|100% LED, Low Efficacy
           - not: Lighting|100% CFL
           - not: Geometry Garage|3 Car
-          - and:
+          - or:
             - Lighting|60% CFL Hardwired, 34% CFL Plugin
             - Lighting|60% LED Hardwired, 34% CFL Plugin
           - Geometry Stories|1

buildstockbatch/test/test_inputs/enforce-validate-options-all-good.yml

@@ -0,0 +1,26 @@
+buildstock_directory: test_openstudio_buildstock
+project_directory: project_singlefamilydetached
+baseline:
+  n_datapoints: 30
+  n_buildings_represented: 81221016
+  sampling_algorithm: quota
+upgrades:
+  - upgrade_name: good upgrade
+    options:
+      - option: Vintage|<1940
+        apply_logic:
+          - or:
+            - Insulation Slab|Good Option
+            - Insulation Slab|None
+          - not: Insulation Wall|Good Option
+          - and:
+              - Vintage|1960s||Vintage|1960s
+              - Insulation Slab|None
+      - option: Insulation Finished Basement|Good Option
+        apply_logic:
+          - Insulation Unfinished Basement|Extra Argument
+    package_apply_logic: Vintage|1960s||Vintage|1940s
+downselect:
+  logic: Vintage|2000s
+  resample: False
+schema_version: 0.3

buildstockbatch/test/test_inputs/enforce-validate-options-good.yml

@@ -12,14 +12,16 @@ upgrades:
           - or:
             - Insulation Slab|Good Option
             - Insulation Slab|None
-          - not: Insulation Wall|Good Option
+          - not: 
+            - Insulation Wall|Good Option
+            - Insulation Wall|Good Option  # Two Insulation Wall under 'not'. Should be caught by logic validator
           - and:
               - Vintage|1960s||Vintage|1960s
-              - Vintage|1980s
+              - Vintage|1980s  # Two Vintages under and. Should be caught by logic validator
       - option: Insulation Finished Basement|Good Option
         apply_logic:
           - Insulation Unfinished Basement|Extra Argument
-    package_apply_logic: Vintage|1960s||Vintage|1940s
+    package_apply_logic: Vintage|1960s&&Vintage|1940s  # Two Vintages under 'and'. Should be caught by logic validator
 downselect:
   logic: Vintage|2000s
   resample: False

buildstockbatch/test/test_validation.py

@@ -146,7 +146,7 @@ def test_bad_measures(project_file):
             assert "Found unexpected argument key include_enduse_subcategory" in er
 
         else:
-            raise Exception("measures_and_arguments was supposed to raise ValueError for"
+            raise Exception("measures_and_arguments was supposed to raise ValidationError for"
                             " enforce-validate-measures-bad.yml")
 
 
@@ -186,7 +186,7 @@ def test_good_options_validation(project_file):
 def test_bad_options_validation(project_file):
     try:
         BuildStockBatchBase.validate_options_lookup(project_file)
-    except ValueError as er:
+    except ValidationError as er:
         er = str(er)
         assert "Insulation Slab(Good) Option" in er
         assert "Insulation Unfinished&Basement" in er
@@ -218,7 +218,7 @@ def test_good_measures_validation(project_file):
 def test_bad_measures_validation(project_file):
     try:
         BuildStockBatchBase.validate_measure_references(project_file)
-    except ValueError as er:
+    except ValidationError as er:
         er = str(er)
         assert "Measure directory" in er
         assert "not found" in er
@@ -240,4 +240,26 @@ def test_bad_postprocessing_spec_validation(project_file):
         er = str(er)
         assert "bad_partition_column" in er
     else:
-        raise Exception("validate_options was supposed to raise ValueError for enforce-validate-options-bad-2.yml")
+        raise Exception("validate_options was supposed to raise ValidationError for enforce-validate-options-bad-2.yml")
+
+
+@pytest.mark.parametrize("project_file", [
+    os.path.join(example_yml_dir, 'enforce-validate-options-good.yml')
+])
+def test_logic_validation_fail(project_file):
+    try:
+        BuildStockBatchBase.validate_logic(project_file)
+    except ValidationError as er:
+        er = str(er)
+        assert "'Insulation Wall' occurs 2 times in a 'not' block" in er
+        assert "'Vintage' occurs 2 times in a 'and' block" in er
+        assert "'Vintage' occurs 2 times in a '&&' block" in er
+    else:
+        raise Exception("validate_options was supposed to raise ValidationError for enforce-validate-options-good.yml")
+
+
+@pytest.mark.parametrize("project_file", [
+    os.path.join(example_yml_dir, 'enforce-validate-options-all-good.yml')
+])
+def test_logic_validation_pass(project_file):
+    BuildStockBatchBase.validate_logic(project_file)

docs/changelog/changelog_dev.rst

@@ -60,3 +60,11 @@ Development Changelog
         :tickets:
 
         For ResStock the OpenStudio version has changed to v3.4.0.
+
+    .. change::
+        :tags: general, feature
+        :pullreq: 295
+        :tickets:
+
+        Add basic logic validation that checks for incorrect use of 'and' and 'not' block.
+        BSB requires at least python 3.8.

setup.py

@@ -49,7 +49,7 @@ def run_tests(self):
     long_description_content_type='text/markdown',
     url=metadata['__url__'],
     packages=setuptools.find_packages(),
-    python_requires='>=3.7',
+    python_requires='>=3.8',
     package_data={
         'buildstockbatch': ['*.sh', 'schemas/*.yaml'],
         '': ['LICENSE']