ResStock-HPXML

buildstockbatch/aws/aws.py

@@ -1800,6 +1800,8 @@ def run_batch(self):
                 project_path = pathlib.Path(self.project_dir)
                 buildstock_path = pathlib.Path(self.buildstock_dir)
                 tar_f.add(buildstock_path / 'measures', 'measures')
+                if os.path.exists(buildstock_path / 'resources/hpxml-measures'):
+                    tar_f.add(buildstock_path / 'resources/hpxml-measures', 'resources/hpxml-measures')
                 tar_f.add(buildstock_path / 'resources', 'lib/resources')
                 tar_f.add(project_path / 'housing_characteristics', 'lib/housing_characteristics')
 

buildstockbatch/base.py

@@ -38,8 +38,9 @@
 
 class BuildStockBatchBase(object):
 
-    DEFAULT_OS_VERSION = '2.9.1'
-    DEFAULT_OS_SHA = '3472e8b799'
+    # http://openstudio-builds.s3-website-us-east-1.amazonaws.com
+    DEFAULT_OS_VERSION = '3.1.0'
+    DEFAULT_OS_SHA = 'e165090621'
     CONTAINER_RUNTIME = None
     LOGO = '''
      _ __         _     __,              _ __
@@ -172,17 +173,32 @@ def cleanup_sim_dir(sim_dir, dest_fs, simout_ts_dir, upgrade_id, building_id):
 
         # Convert the timeseries data to parquet
         # and copy it to the results directory
-        timeseries_filepath = os.path.join(sim_dir, 'run', 'enduse_timeseries.csv')
-        schedules_filepath = os.path.join(sim_dir, 'generated_files', 'schedules.csv')
+        results_timeseries_filepath = os.path.join(sim_dir, 'run', 'results_timeseries.csv')
+        timeseries_filepath = results_timeseries_filepath
+        skiprows = [1]
+        # FIXME: Allowing both names here for compatibility. Should consolidate on one timeseries filename.
+        if not os.path.isfile(results_timeseries_filepath):
+            enduse_timeseries_filepath = os.path.join(sim_dir, 'run', 'enduse_timeseries.csv')
+            timeseries_filepath = enduse_timeseries_filepath
+            skiprows = False
+        schedules_filepath = ''
+        if os.path.isdir(os.path.join(sim_dir, 'generated_files')):
+            for file in os.listdir(os.path.join(sim_dir, 'generated_files')):
+                if file.endswith('schedules.csv'):
+                    schedules_filepath = os.path.join(sim_dir, 'generated_files', file)
         if os.path.isfile(timeseries_filepath):
             # Find the time columns present in the enduse_timeseries file
             possible_time_cols = ['time', 'Time', 'TimeDST', 'TimeUTC']
             cols = pd.read_csv(timeseries_filepath, index_col=False, nrows=0).columns.tolist()
             actual_time_cols = [c for c in cols if c in possible_time_cols]
             if not actual_time_cols:
-                logger.error(f'Did not find any time column ({possible_time_cols}) in enduse_timeseries.csv.')
-                raise RuntimeError(f'Did not find any time column ({possible_time_cols}) in enduse_timeseries.csv.')
-            tsdf = pd.read_csv(timeseries_filepath, parse_dates=actual_time_cols)
+                logger.error(f'Did not find any time column ({possible_time_cols}) in {timeseries_filepath}.')
+                raise RuntimeError(f'Did not find any time column ({possible_time_cols}) in {timeseries_filepath}.')
+            if skiprows:
+                tsdf = pd.read_csv(timeseries_filepath, parse_dates=actual_time_cols, skiprows=skiprows)
+                tsdf['TimeDST'] = tsdf['Time']  # FIXME: Actually write TimeDST to results_timeseries.csv?
+            else:
+                tsdf = pd.read_csv(timeseries_filepath, parse_dates=actual_time_cols)
             if os.path.isfile(schedules_filepath):
                 schedules = pd.read_csv(schedules_filepath)
                 schedules.rename(columns=lambda x: f'schedules_{x}', inplace=True)
@@ -540,6 +556,12 @@ def process_results(self, skip_combine=False, force_upload=False):
         self.get_dask_client()  # noqa: F841
 
         do_timeseries = 'timeseries_csv_export' in self.cfg['workflow_generator']['args'].keys()
+        if not do_timeseries:
+            if 'simulation_output_report' in self.cfg['workflow_generator']['args'].keys():
+                if 'timeseries_frequency' in self.cfg['workflow_generator']['args']['simulation_output_report'].keys():
+                    do_timeseries = \
+                        (self.cfg['workflow_generator']['args']['simulation_output_report']['timeseries_frequency'] !=
+                            'none')
 
         fs = LocalFileSystem()
         if not skip_combine:

buildstockbatch/eagle.py

@@ -229,6 +229,11 @@ def run_job_batch(self, job_array_number):
             pathlib.Path(self.buildstock_dir) / 'measures',
             self.local_buildstock_dir / 'measures'
         )
+        if os.path.exists(pathlib.Path(self.buildstock_dir) / 'resources/hpxml-measures'):
+            self.clear_and_copy_dir(
+                pathlib.Path(self.buildstock_dir) / 'resources/hpxml-measures',
+                self.local_buildstock_dir / 'resources/hpxml-measures'
+            )
         self.clear_and_copy_dir(
             self.weather_dir,
             self.local_weather_dir
@@ -336,6 +341,12 @@ def run_building(cls, output_dir, cfg, n_datapoints, i, upgrade_idx=None):
                 container_symlink = os.path.join('/var/simdata/openstudio', os.path.basename(src))
                 runscript.append('ln -s {} {}'.format(*map(shlex.quote, (container_mount, container_symlink))))
 
+            if os.path.exists(os.path.join(cls.local_buildstock_dir, 'resources/hpxml-measures')):
+                runscript.append('ln -s /resources /var/simdata/openstudio/resources')
+                src = os.path.join(cls.local_buildstock_dir, 'resources/hpxml-measures')
+                container_mount = '/resources/hpxml-measures'
+                args.extend(['-B', '{}:{}:ro'.format(src, container_mount)])
+
             # Build the openstudio command that will be issued within the singularity container
             # If custom gems are to be used in the singularity container add extra bundle arguments to the cli command
             cli_cmd = 'openstudio run -w in.osw'

buildstockbatch/localdocker.py

@@ -103,6 +103,9 @@ def run_building(cls, project_dir, buildstock_dir, weather_dir, docker_image, re
             (os.path.join(project_dir, 'housing_characteristics'), 'lib/housing_characteristics', 'ro'),
             (weather_dir, 'weather', 'ro')
         ]
+        if os.path.exists(os.path.join(buildstock_dir, 'resources', 'hpxml-measures')):
+            bind_mounts += [(os.path.join(buildstock_dir, 'resources', 'hpxml-measures'),
+                            'resources/hpxml-measures', 'ro')]
         docker_volume_mounts = dict([(key, {'bind': f'/var/simdata/openstudio/{bind}', 'mode': mode}) for key, bind, mode in bind_mounts])  # noqa E501
         for bind in bind_mounts:
             dir_to_make = os.path.join(sim_dir, *bind[1].split('/'))

buildstockbatch/postprocessing.py

@@ -41,14 +41,18 @@ def read_data_point_out_json(fs, reporting_measures, filename):
         with fs.open(filename, 'r') as f:
             d = json.load(f)
     except (FileNotFoundError, json.JSONDecodeError):
-        return None
-    else:
-        if 'SimulationOutputReport' not in d:
-            d['SimulationOutputReport'] = {'applicable': False}
-        for reporting_measure in reporting_measures:
-            if reporting_measure not in d:
-                d[reporting_measure] = {'applicable': False}
-        return d
+        try:
+            with fs.open(filename.replace('data_point_out', 'results'), 'r') as f:
+                d = json.load(f)
+        except (FileNotFoundError, json.JSONDecodeError):
+            return None
+
+    if 'SimulationOutputReport' not in d:
+        d['SimulationOutputReport'] = {'applicable': False}
+    for reporting_measure in reporting_measures + ['UpgradeCosts']:
+        if reporting_measure not in d:
+            d[reporting_measure] = {'applicable': False}
+    return d
 
 
 def to_camelcase(x):
@@ -82,7 +86,7 @@ def flatten_datapoint_json(reporting_measures, d):
         new_d[f'{col2}.{k}'] = v
 
     # additional reporting measures
-    for col in reporting_measures:
+    for col in reporting_measures + ['UpgradeCosts']:
         for k, v in d.get(col, {}).items():
             new_d[f'{col}.{k}'] = v
 
@@ -107,12 +111,30 @@ def read_out_osw(fs, filename):
         ]
         for key in keys_to_copy:
             out_d[key] = d.get(key, None)
-        for step in d.get('steps', []):
-            if step['measure_dir_name'] == 'BuildExistingModel':
-                out_d['building_id'] = step['arguments']['building_id']
         return out_d
 
 
+def read_job_files(fs, started, finished):
+    jobs = {'completed_status': 'Fail'}
+    try:
+        with fs.open(started, 'r') as f:
+            started_at = re.search(r'Started Workflow (.*\s.*?)\s', f.readline()).group(1)
+            started_at = started_at.replace('-', '').replace(':', '').replace(' ', 'T') + 'Z'
+            jobs['started_at'] = started_at
+    except:
+        return None
+    try:
+        with fs.open(finished, 'r') as f:
+            completed_at = re.search(r'Finished Workflow (.*\s.*?)\s', f.readline()).group(1)
+            completed_at = completed_at.replace('-', '').replace(':', '').replace(' ', 'T') + 'Z'
+            jobs['completed_at'] = completed_at
+    except:
+        return None
+    else:
+        jobs['completed_status'] = 'Success'
+    return jobs
+
+
 def read_simulation_outputs(fs, reporting_measures, sim_dir, upgrade_id, building_id):
     """Read the simulation outputs and return as a dict
 
@@ -139,6 +161,9 @@ def read_simulation_outputs(fs, reporting_measures, sim_dir, upgrade_id, buildin
     out_osw = read_out_osw(fs, f'{sim_dir}/out.osw')
     if out_osw:
         dpout.update(out_osw)
+    else: # for when run_options: fast=true
+        jobs = read_job_files(fs, f'{sim_dir}/run/started.job', f'{sim_dir}/run/finished.job')
+        dpout.update(jobs)
     dpout['upgrade'] = upgrade_id
     dpout['building_id'] = building_id
     return dpout
@@ -194,6 +219,10 @@ def clean_up_results_df(df, cfg, keep_upgrade_id=False):
                                         col.startswith(to_camelcase(reporting_measure))])
         sorted_cols += reporting_measure_cols
 
+    upgrade_costs_cols = sorted([col for col in results_df.columns if
+                                col.startswith(to_camelcase('UpgradeCosts'))])
+    sorted_cols += upgrade_costs_cols
+
     results_df = results_df.reindex(columns=sorted_cols, copy=False)
 
     # for col in results_df.columns:

buildstockbatch/test/test_postprocessing.py

@@ -33,7 +33,7 @@ def test_report_additional_results_csv_columns(basic_residential_project_file):
         tarf.extractall(sim_out_dir)
 
     dpouts2 = []
-    for filename in sim_out_dir.rglob('data_point_out.json'):
+    for filename in sim_out_dir.rglob('results.json'):
         with filename.open('rt', encoding='utf-8') as f:
             dpout = json.load(f)
         dpout['ReportingMeasure1'] = {'column_1': 1, 'column_2': 2}

buildstockbatch/workflow_generator/__init__.py

@@ -2,3 +2,4 @@
 
 from .residential import ResidentialDefaultWorkflowGenerator  # noqa F041
 from .commercial import CommercialDefaultWorkflowGenerator  # noqa F041
+from .residential_hpxml import ResidentialHpxmlWorkflowGenerator  # noqa F041

buildstockbatch/workflow_generator/residential_hpxml.py

@@ -0,0 +1,161 @@
+# -*- coding: utf-8 -*-
+
+"""
+buildstockbatch.workflow_generator.residential_hpxml
+~~~~~~~~~~~~~~~
+This object contains the residential classes for generating OSW files from individual samples
+
+:author: Joe Robertson
+:copyright: (c) 2021 by The Alliance for Sustainable Energy
+:license: BSD-3
+"""
+
+import datetime as dt
+import json
+import logging
+import re
+import yamale
+
+from .base import WorkflowGeneratorBase
+
+logger = logging.getLogger(__name__)
+
+
+class ResidentialHpxmlWorkflowGenerator(WorkflowGeneratorBase):
+
+    @classmethod
+    def validate(cls, cfg):
+        """Validate arguments
+
+        :param cfg: project configuration
+        :type cfg: dict
+        """
+        schema_yml = """
+        build_existing_model: map(required=False)
+        simulation_output_report: map(required=False)
+        ---
+        measure-spec:
+            measure_dir_name: str(required=True)
+            arguments: map(required=False)
+        """
+        workflow_generator_args = cfg['workflow_generator']['args']
+        schema_yml = re.sub(r'^ {8}', '', schema_yml, flags=re.MULTILINE)
+        schema = yamale.make_schema(content=schema_yml, parser='ruamel')
+        data = yamale.make_data(content=json.dumps(workflow_generator_args), parser='ruamel')
+        yamale.validate(schema, data, strict=True)
+        return True
+
+    def create_osw(self, sim_id, building_id, upgrade_idx):
+        """
+        Generate and return the osw as a python dict
+
+        :param sim_id: simulation id, looks like 'bldg0000001up01'
+        :param building_id: integer building id to use from the sampled buildstock.csv
+        :param upgrade_idx: integer index of the upgrade scenario to apply, None if baseline
+        """
+        # Default argument values
+        workflow_args = {
+            'build_existing_model': {},
+            'simulation_output_report': {},
+        }
+        workflow_args.update(self.cfg['workflow_generator'].get('args', {}))
+
+        logger.debug('Generating OSW, sim_id={}'.format(sim_id))
+
+        sim_ctl_run_prd_args = {
+            'simulation_control_run_period_begin_month': 1,
+            'simulation_control_run_period_begin_day_of_month': 1,
+            'simulation_control_run_period_end_month': 12,
+            'simulation_control_run_period_end_day_of_month': 31,
+            'simulation_control_run_period_calendar_year': 2007,
+        }
+
+        bld_exist_model_args = {
+            'building_id': building_id,
+            'workflow_json': 'measure-info.json',
+            'sample_weight': self.n_datapoints / self.cfg['baseline']['n_buildings_represented'],
+        }
+        bld_exist_model_args.update(sim_ctl_run_prd_args)
+        bld_exist_model_args.update(workflow_args['build_existing_model'])
+
+        sim_out_rep_args = {
+            'timeseries_frequency': True,
+            'include_timeseries_fuel_consumptions': False,
+            'include_timeseries_end_use_consumptions': False,
+            'include_timeseries_hot_water_uses': False,
+            'include_timeseries_total_loads': False,
+            'include_timeseries_component_loads': False,
+            'include_timeseries_unmet_loads': False,
+            'include_timeseries_zone_temperatures': False,
+            'include_timeseries_airflows': False,
+            'include_timeseries_weather': False,
+        }
+        sim_out_rep_args.update(workflow_args['simulation_output_report'])
+
+        osw = {
+            'id': sim_id,
+            'steps': [
+                {
+                    'measure_dir_name': 'BuildExistingModel',
+                    'arguments': bld_exist_model_args
+                }
+            ],
+            'created_at': dt.datetime.now().isoformat(),
+            'measure_paths': [
+                'measures',
+                'resources/hpxml-measures'
+            ],
+            'run_options': {
+                'fast': True,
+                'skip_expand_objects': True,
+                'skip_energyplus_preprocess': True
+            }
+        }
+
+        osw['steps'].extend([
+            {
+                'measure_dir_name': 'SimulationOutputReport',
+                'arguments': workflow_args['simulation_output_report']
+            },
+            {
+                'measure_dir_name': 'UpgradeCosts',
+                'arguments': {}
+            },
+            {
+                'measure_dir_name': 'ServerDirectoryCleanup',
+                'arguments': {}
+            }
+        ])
+
+        if upgrade_idx is not None:
+            measure_d = self.cfg['upgrades'][upgrade_idx]
+            apply_upgrade_measure = {
+                'measure_dir_name': 'ApplyUpgrade',
+                'arguments': {
+                    'run_measure': 1
+                }
+            }
+            if 'upgrade_name' in measure_d:
+                apply_upgrade_measure['arguments']['upgrade_name'] = measure_d['upgrade_name']
+            for opt_num, option in enumerate(measure_d['options'], 1):
+                apply_upgrade_measure['arguments']['option_{}'.format(opt_num)] = option['option']
+                if 'lifetime' in option:
+                    apply_upgrade_measure['arguments']['option_{}_lifetime'.format(opt_num)] = option['lifetime']
+                if 'apply_logic' in option:
+                    apply_upgrade_measure['arguments']['option_{}_apply_logic'.format(opt_num)] = \
+                        self.make_apply_logic_arg(option['apply_logic'])
+                for cost_num, cost in enumerate(option.get('costs', []), 1):
+                    for arg in ('value', 'multiplier'):
+                        if arg not in cost:
+                            continue
+                        apply_upgrade_measure['arguments']['option_{}_cost_{}_{}'.format(opt_num, cost_num, arg)] = \
+                            cost[arg]
+            if 'package_apply_logic' in measure_d:
+                apply_upgrade_measure['arguments']['package_apply_logic'] = \
+                    self.make_apply_logic_arg(measure_d['package_apply_logic'])
+
+            build_existing_model_idx = \
+                [x['measure_dir_name'] == 'BuildExistingModel' for x in osw['steps']].index(True)
+            osw['steps'].insert(build_existing_model_idx + 1, apply_upgrade_measure)
+
+        return osw