Skip to content

Commit

Permalink
Merge pull request #208 from NREL/restructure-v3
Browse files Browse the repository at this point in the history
ResStock-HPXML
  • Loading branch information
nmerket committed Mar 2, 2022
2 parents 0724658 + 5b6296a commit f60c0e1
Show file tree
Hide file tree
Showing 13 changed files with 478 additions and 15 deletions.
2 changes: 2 additions & 0 deletions buildstockbatch/aws/aws.py
Original file line number Diff line number Diff line change
Expand Up @@ -1800,6 +1800,8 @@ def run_batch(self):
project_path = pathlib.Path(self.project_dir)
buildstock_path = pathlib.Path(self.buildstock_dir)
tar_f.add(buildstock_path / 'measures', 'measures')
if os.path.exists(buildstock_path / 'resources/hpxml-measures'):
tar_f.add(buildstock_path / 'resources/hpxml-measures', 'resources/hpxml-measures')
tar_f.add(buildstock_path / 'resources', 'lib/resources')
tar_f.add(project_path / 'housing_characteristics', 'lib/housing_characteristics')

Expand Down
34 changes: 28 additions & 6 deletions buildstockbatch/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@

class BuildStockBatchBase(object):

# http://openstudio-builds.s3-website-us-east-1.amazonaws.com
DEFAULT_OS_VERSION = '3.3.0'
DEFAULT_OS_SHA = 'ad235ff36e'
CONTAINER_RUNTIME = None
Expand Down Expand Up @@ -178,17 +179,31 @@ def cleanup_sim_dir(sim_dir, dest_fs, simout_ts_dir, upgrade_id, building_id):

# Convert the timeseries data to parquet
# and copy it to the results directory
timeseries_filepath = os.path.join(sim_dir, 'run', 'enduse_timeseries.csv')
schedules_filepath = os.path.join(sim_dir, 'generated_files', 'schedules.csv')
results_timeseries_filepath = os.path.join(sim_dir, 'run', 'results_timeseries.csv')
timeseries_filepath = results_timeseries_filepath
skiprows = [1]
# FIXME: Allowing both names here for compatibility. Should consolidate on one timeseries filename.
if not os.path.isfile(results_timeseries_filepath):
enduse_timeseries_filepath = os.path.join(sim_dir, 'run', 'enduse_timeseries.csv')
timeseries_filepath = enduse_timeseries_filepath
skiprows = False
schedules_filepath = ''
if os.path.isdir(os.path.join(sim_dir, 'generated_files')):
for file in os.listdir(os.path.join(sim_dir, 'generated_files')):
if file.endswith('schedules.csv'):
schedules_filepath = os.path.join(sim_dir, 'generated_files', file)
if os.path.isfile(timeseries_filepath):
# Find the time columns present in the enduse_timeseries file
possible_time_cols = ['time', 'Time', 'TimeDST', 'TimeUTC']
cols = pd.read_csv(timeseries_filepath, index_col=False, nrows=0).columns.tolist()
actual_time_cols = [c for c in cols if c in possible_time_cols]
if not actual_time_cols:
logger.error(f'Did not find any time column ({possible_time_cols}) in enduse_timeseries.csv.')
raise RuntimeError(f'Did not find any time column ({possible_time_cols}) in enduse_timeseries.csv.')
tsdf = pd.read_csv(timeseries_filepath, parse_dates=actual_time_cols)
logger.error(f'Did not find any time column ({possible_time_cols}) in {timeseries_filepath}.')
raise RuntimeError(f'Did not find any time column ({possible_time_cols}) in {timeseries_filepath}.')
if skiprows:
tsdf = pd.read_csv(timeseries_filepath, parse_dates=actual_time_cols, skiprows=skiprows)
else:
tsdf = pd.read_csv(timeseries_filepath, parse_dates=actual_time_cols)
if os.path.isfile(schedules_filepath):
schedules = pd.read_csv(schedules_filepath)
schedules.rename(columns=lambda x: f'schedules_{x}', inplace=True)
Expand Down Expand Up @@ -542,7 +557,14 @@ def get_dask_client(self):
def process_results(self, skip_combine=False, force_upload=False):
self.get_dask_client() # noqa: F841

do_timeseries = 'timeseries_csv_export' in self.cfg['workflow_generator']['args'].keys()
if self.cfg['workflow_generator']['type'] == 'residential_hpxml':
if 'simulation_output_report' in self.cfg['workflow_generator']['args'].keys():
if 'timeseries_frequency' in self.cfg['workflow_generator']['args']['simulation_output_report'].keys():
do_timeseries = \
(self.cfg['workflow_generator']['args']['simulation_output_report']['timeseries_frequency'] !=
'none')
else:
do_timeseries = 'timeseries_csv_export' in self.cfg['workflow_generator']['args'].keys()

fs = LocalFileSystem()
if not skip_combine:
Expand Down
11 changes: 11 additions & 0 deletions buildstockbatch/eagle.py
Original file line number Diff line number Diff line change
Expand Up @@ -230,6 +230,11 @@ def run_job_batch(self, job_array_number):
pathlib.Path(self.buildstock_dir) / 'measures',
self.local_buildstock_dir / 'measures'
)
if os.path.exists(pathlib.Path(self.buildstock_dir) / 'resources/hpxml-measures'):
self.clear_and_copy_dir(
pathlib.Path(self.buildstock_dir) / 'resources/hpxml-measures',
self.local_buildstock_dir / 'resources/hpxml-measures'
)
self.clear_and_copy_dir(
self.weather_dir,
self.local_weather_dir
Expand Down Expand Up @@ -351,6 +356,12 @@ def run_building(cls, output_dir, cfg, n_datapoints, i, upgrade_idx=None):
container_symlink = os.path.join('/var/simdata/openstudio', os.path.basename(src))
runscript.append('ln -s {} {}'.format(*map(shlex.quote, (container_mount, container_symlink))))

if os.path.exists(os.path.join(cls.local_buildstock_dir, 'resources/hpxml-measures')):
runscript.append('ln -s /resources /var/simdata/openstudio/resources')
src = os.path.join(cls.local_buildstock_dir, 'resources/hpxml-measures')
container_mount = '/resources/hpxml-measures'
args.extend(['-B', '{}:{}:ro'.format(src, container_mount)])

# Build the openstudio command that will be issued within the singularity container
# If custom gems are to be used in the singularity container add extra bundle arguments to the cli command
cli_cmd = 'openstudio run -w in.osw'
Expand Down
3 changes: 3 additions & 0 deletions buildstockbatch/localdocker.py
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,9 @@ def run_building(cls, project_dir, buildstock_dir, weather_dir, docker_image, re
(os.path.join(project_dir, 'housing_characteristics'), 'lib/housing_characteristics', 'ro'),
(weather_dir, 'weather', 'ro')
]
if os.path.exists(os.path.join(buildstock_dir, 'resources', 'hpxml-measures')):
bind_mounts += [(os.path.join(buildstock_dir, 'resources', 'hpxml-measures'),
'resources/hpxml-measures', 'ro')]
docker_volume_mounts = dict([(key, {'bind': f'/var/simdata/openstudio/{bind}', 'mode': mode}) for key, bind, mode in bind_mounts]) # noqa E501
for bind in bind_mounts:
dir_to_make = os.path.join(sim_dir, *bind[1].split('/'))
Expand Down
26 changes: 21 additions & 5 deletions buildstockbatch/postprocessing.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,12 @@ def read_data_point_out_json(fs, reporting_measures, filename):
except (FileNotFoundError, json.JSONDecodeError):
return None
else:
if 'SimulationOutputReport' not in d:
d['SimulationOutputReport'] = {'applicable': False}
sim_out_report = 'SimulationOutputReport'
if 'ReportSimulationOutput' in d:
sim_out_report = 'ReportSimulationOutput'

if sim_out_report not in d:
d[sim_out_report] = {'applicable': False}
for reporting_measure in reporting_measures:
if reporting_measure not in d:
d[reporting_measure] = {'applicable': False}
Expand Down Expand Up @@ -78,11 +82,16 @@ def flatten_datapoint_json(reporting_measures, d):
# TODO @nmerket @rajeee is there a way to not apply this to Commercial jobs? It doesn't hurt, but it is weird for us
units = int(new_d.get(f'{col1}.units_represented', 1))
new_d[f'{col1}.units_represented'] = units
col2 = 'SimulationOutputReport'
sim_out_report = 'SimulationOutputReport'
if 'ReportSimulationOutput' in d:
sim_out_report = 'ReportSimulationOutput'
col2 = sim_out_report
for k, v in d.get(col2, {}).items():
new_d[f'{col2}.{k}'] = v

# additional reporting measures
if sim_out_report == 'ReportSimulationOutput':
reporting_measures += ['UpgradeCosts']
for col in reporting_measures:
for k, v in d.get(col, {}).items():
new_d[f'{col}.{k}'] = v
Expand Down Expand Up @@ -187,8 +196,15 @@ def clean_up_results_df(df, cfg, keep_upgrade_id=False):
first_few_cols.insert(2, 'job_id')

build_existing_model_cols = sorted([col for col in results_df.columns if col.startswith('build_existing_model')])
simulation_output_cols = sorted([col for col in results_df.columns if col.startswith('simulation_output_report')])
sorted_cols = first_few_cols + build_existing_model_cols + simulation_output_cols
sim_output_report_cols = sorted([col for col in results_df.columns if col.startswith('simulation_output_report')])
report_sim_output_cols = sorted([col for col in results_df.columns if col.startswith('report_simulation_output')])
upgrade_costs_cols = sorted([col for col in results_df.columns if col.startswith('upgrade_costs')])
sorted_cols = \
first_few_cols + \
build_existing_model_cols + \
sim_output_report_cols + \
report_sim_output_cols + \
upgrade_costs_cols

remaining_cols = sorted(set(results_df.columns.values).difference(sorted_cols))
sorted_cols += remaining_cols
Expand Down
1 change: 1 addition & 0 deletions buildstockbatch/workflow_generator/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,4 @@

from .residential import ResidentialDefaultWorkflowGenerator # noqa F041
from .commercial import CommercialDefaultWorkflowGenerator # noqa F041
from .residential_hpxml import ResidentialHpxmlWorkflowGenerator # noqa F041
223 changes: 223 additions & 0 deletions buildstockbatch/workflow_generator/residential_hpxml.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,223 @@
# -*- coding: utf-8 -*-

"""
buildstockbatch.workflow_generator.residential_hpxml
~~~~~~~~~~~~~~~
This object contains the residential classes for generating OSW files from individual samples
:author: Joe Robertson
:copyright: (c) 2021 by The Alliance for Sustainable Energy
:license: BSD-3
"""

import datetime as dt
import json
import logging
import re
import yamale

from .base import WorkflowGeneratorBase

logger = logging.getLogger(__name__)


class ResidentialHpxmlWorkflowGenerator(WorkflowGeneratorBase):

@classmethod
def validate(cls, cfg):
"""Validate arguments
:param cfg: project configuration
:type cfg: dict
"""
schema_yml = """
measures_to_ignore: list(str(), required=False)
build_existing_model: map(required=False)
emissions: list(include('scenario-spec'), required=False)
reporting_measures: list(include('measure-spec'), required=False)
simulation_output_report: map(required=False)
server_directory_cleanup: map(required=False)
---
scenario-spec:
scenario_name: str(required=True)
type: str(required=True)
elec_folder: str(required=True)
gas_value: num(required=False)
propane_value: num(required=False)
oil_value: num(required=False)
wood_value: num(required=False)
measure-spec:
measure_dir_name: str(required=True)
arguments: map(required=False)
"""
workflow_generator_args = cfg['workflow_generator']['args']
schema_yml = re.sub(r'^ {8}', '', schema_yml, flags=re.MULTILINE)
schema = yamale.make_schema(content=schema_yml, parser='ruamel')
data = yamale.make_data(content=json.dumps(workflow_generator_args), parser='ruamel')
yamale.validate(schema, data, strict=True)
return True

def reporting_measures(self):
"""Return a list of reporting measures to include in outputs"""
workflow_args = self.cfg['workflow_generator'].get('args', {})
return [x['measure_dir_name'] for x in workflow_args.get('reporting_measures', [])]

def create_osw(self, sim_id, building_id, upgrade_idx):
"""
Generate and return the osw as a python dict
:param sim_id: simulation id, looks like 'bldg0000001up01'
:param building_id: integer building id to use from the sampled buildstock.csv
:param upgrade_idx: integer index of the upgrade scenario to apply, None if baseline
"""
# Default argument values
workflow_args = {
'build_existing_model': {},
'measures': [],
'simulation_output_report': {},
'server_directory_cleanup': {}
}
workflow_args.update(self.cfg['workflow_generator'].get('args', {}))

logger.debug('Generating OSW, sim_id={}'.format(sim_id))

sim_ctl_args = {}

bld_exist_model_args = {
'building_id': building_id,
'sample_weight': self.n_datapoints / self.cfg['baseline']['n_buildings_represented']
}
if 'measures_to_ignore' in workflow_args:
bld_exist_model_args['measures_to_ignore'] = '|'.join(workflow_args['measures_to_ignore'])
bld_exist_model_args.update(sim_ctl_args)
bld_exist_model_args.update(workflow_args['build_existing_model'])

if 'emissions' in workflow_args:
emissions = workflow_args['emissions']
emissions_map = [['emissions_scenario_names', 'scenario_name'],
['emissions_types', 'type'],
['emissions_electricity_folders', 'elec_folder'],
['emissions_natural_gas_values', 'gas_value'],
['emissions_propane_values', 'propane_value'],
['emissions_fuel_oil_values', 'oil_value'],
['emissions_wood_values', 'wood_value']]
for arg, item in emissions_map:
bld_exist_model_args[arg] = ','.join([str(s.get(item)) for s in emissions])

sim_out_rep_args = {
'timeseries_frequency': 'none',
'include_timeseries_fuel_consumptions': False,
'include_timeseries_end_use_consumptions': True,
'include_timeseries_emissions': False,
'include_timeseries_hot_water_uses': False,
'include_timeseries_total_loads': True,
'include_timeseries_component_loads': False,
'include_timeseries_zone_temperatures': False,
'include_timeseries_airflows': False,
'include_timeseries_weather': False,
'add_timeseries_dst_column': True,
'add_timeseries_utc_column': True
}
sim_out_rep_args.update(workflow_args['simulation_output_report'])

osw = {
'id': sim_id,
'steps': [
{
'measure_dir_name': 'BuildExistingModel',
'arguments': bld_exist_model_args
}
],
'created_at': dt.datetime.now().isoformat(),
'measure_paths': [
'measures',
'resources/hpxml-measures'
],
'run_options': {
'skip_zip_results': True
}
}

osw['steps'].extend(workflow_args['measures'])

server_dir_cleanup_args = {
'retain_in_osm': False,
'retain_in_idf': True,
'retain_pre_process_idf': False,
'retain_eplusout_audit': False,
'retain_eplusout_bnd': False,
'retain_eplusout_eio': False,
'retain_eplusout_end': False,
'retain_eplusout_err': False,
'retain_eplusout_eso': False,
'retain_eplusout_mdd': False,
'retain_eplusout_mtd': False,
'retain_eplusout_rdd': False,
'retain_eplusout_shd': False,
'retain_eplusout_sql': False,
'retain_eplustbl_htm': False,
'retain_sqlite_err': False,
'retain_stdout_energyplus': False,
'retain_stdout_expandobject': False,
'retain_schedules_csv': True
}
server_dir_cleanup_args.update(workflow_args['server_directory_cleanup'])

osw['steps'].extend([
{
'measure_dir_name': 'ReportSimulationOutput',
'arguments': sim_out_rep_args
},
{
'measure_dir_name': 'ReportHPXMLOutput',
'arguments': {}
},
{
'measure_dir_name': 'UpgradeCosts',
'arguments': {}
},
{
'measure_dir_name': 'ServerDirectoryCleanup',
'arguments': server_dir_cleanup_args
}
])

if upgrade_idx is not None:
measure_d = self.cfg['upgrades'][upgrade_idx]
apply_upgrade_measure = {
'measure_dir_name': 'ApplyUpgrade',
'arguments': {
'run_measure': 1
}
}
if 'upgrade_name' in measure_d:
apply_upgrade_measure['arguments']['upgrade_name'] = measure_d['upgrade_name']
for opt_num, option in enumerate(measure_d['options'], 1):
apply_upgrade_measure['arguments']['option_{}'.format(opt_num)] = option['option']
if 'lifetime' in option:
apply_upgrade_measure['arguments']['option_{}_lifetime'.format(opt_num)] = option['lifetime']
if 'apply_logic' in option:
apply_upgrade_measure['arguments']['option_{}_apply_logic'.format(opt_num)] = \
self.make_apply_logic_arg(option['apply_logic'])
for cost_num, cost in enumerate(option.get('costs', []), 1):
for arg in ('value', 'multiplier'):
if arg not in cost:
continue
apply_upgrade_measure['arguments']['option_{}_cost_{}_{}'.format(opt_num, cost_num, arg)] = \
cost[arg]
if 'package_apply_logic' in measure_d:
apply_upgrade_measure['arguments']['package_apply_logic'] = \
self.make_apply_logic_arg(measure_d['package_apply_logic'])

build_existing_model_idx = \
[x['measure_dir_name'] == 'BuildExistingModel' for x in osw['steps']].index(True)
osw['steps'].insert(build_existing_model_idx + 1, apply_upgrade_measure)

if 'reporting_measures' in workflow_args:
for reporting_measure in workflow_args['reporting_measures']:
if 'arguments' not in reporting_measure:
reporting_measure['arguments'] = {}
reporting_measure['measure_type'] = 'ReportingMeasure'
osw['steps'].insert(-1, reporting_measure) # right before ServerDirectoryCleanup

return osw

0 comments on commit f60c0e1

Please sign in to comment.