Skip to content

Commit

Permalink
Merge pull request #291 from NREL/eagle_tmp_dir
Browse files Browse the repository at this point in the history 
Mounting a temp dir into the singularity container
  • Loading branch information
@nmerket
nmerket committed Oct 3, 2022
2 parents ed0334b + 817a59a commit 538d06e
Show file tree
Hide file tree
Showing 4 changed files with 128 additions and 97 deletions.
149 changes: 79 additions & 70 deletions buildstockbatch/eagle.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import shutil
import subprocess
import sys
import tempfile
import time
import yaml
import csv
Expand All @@ -54,7 +55,7 @@ class EagleBatch(BuildStockBatchBase):
hpc_name = 'eagle'
min_sims_per_job = 36 * 2

local_scratch = pathlib.Path('/tmp/scratch')
local_scratch = pathlib.Path(os.environ.get('LOCAL_SCRATCH', '/tmp/scratch'))
local_project_dir = local_scratch / 'project'
local_buildstock_dir = local_scratch / 'buildstock'
local_weather_dir = local_scratch / 'weather'
Expand Down Expand Up @@ -344,76 +345,84 @@ def run_building(cls, output_dir, cfg, n_datapoints, i, upgrade_idx=None):
cls.local_weather_dir,
]

# Build the command to instantiate and configure the singularity container the simulation is run inside
local_resources_dir = cls.local_buildstock_dir / 'resources'
args = [
'singularity', 'exec',
'--contain',
'-e',
'--pwd', '/var/simdata/openstudio',
'-B', f'{sim_dir}:/var/simdata/openstudio',
'-B', f'{local_resources_dir}:/lib/resources',
'-B', f'{cls.local_housing_characteristics_dir}:/lib/housing_characteristics'
]
runscript = [
'ln -s /lib /var/simdata/openstudio/lib'
]
for src in dirs_to_mount:
container_mount = '/' + os.path.basename(src)
args.extend(['-B', '{}:{}:ro'.format(src, container_mount)])
container_symlink = os.path.join('/var/simdata/openstudio', os.path.basename(src))
runscript.append('ln -s {} {}'.format(*map(shlex.quote, (container_mount, container_symlink))))

if os.path.exists(os.path.join(cls.local_buildstock_dir, 'resources/hpxml-measures')):
runscript.append('ln -s /resources /var/simdata/openstudio/resources')
src = os.path.join(cls.local_buildstock_dir, 'resources/hpxml-measures')
container_mount = '/resources/hpxml-measures'
args.extend(['-B', '{}:{}:ro'.format(src, container_mount)])

# Build the openstudio command that will be issued within the singularity container
# If custom gems are to be used in the singularity container add extra bundle arguments to the cli command
cli_cmd = 'openstudio run -w in.osw'
if cfg.get('baseline', dict()).get('custom_gems', False):
cli_cmd = 'openstudio --bundle /var/oscli/Gemfile --bundle_path /var/oscli/gems run -w in.osw --debug'
if get_bool_env_var('MEASURESONLY'):
cli_cmd += ' --measures_only'
runscript.append(cli_cmd)
args.extend([
str(cls.local_singularity_img),
'bash', '-x'
])
env_vars = dict(os.environ)
env_vars['SINGULARITYENV_BUILDSTOCKBATCH_VERSION'] = bsb_version
logger.debug('\n'.join(map(str, args)))
with open(os.path.join(sim_dir, 'singularity_output.log'), 'w') as f_out:
try:
subprocess.run(
args,
check=True,
input='\n'.join(runscript).encode('utf-8'),
stdout=f_out,
stderr=subprocess.STDOUT,
cwd=cls.local_output_dir,
env=env_vars,
)
except subprocess.CalledProcessError:
pass
finally:
# Clean up the symbolic links we created in the container
for mount_dir in dirs_to_mount + [os.path.join(sim_dir, 'lib')]:
try:
os.unlink(os.path.join(sim_dir, os.path.basename(mount_dir)))
except FileNotFoundError:
pass

# Clean up simulation directory
cls.cleanup_sim_dir(
sim_dir,
fs,
f'{output_dir}/results/simulation_output/timeseries',
upgrade_id,
i
# Create a temporary directory for the simulation to use
with tempfile.TemporaryDirectory(dir=cls.local_scratch, prefix=f"{sim_id}_") as tmpdir:

# Build the command to instantiate and configure the singularity container the simulation is run inside
local_resources_dir = cls.local_buildstock_dir / 'resources'
args = [
'singularity', 'exec',
'--contain',
'-e',
'--pwd', '/var/simdata/openstudio',
'-B', f'{sim_dir}:/var/simdata/openstudio',
'-B', f'{local_resources_dir}:/lib/resources',
'-B', f'{cls.local_housing_characteristics_dir}:/lib/housing_characteristics',
'-B', f'{tmpdir}:/tmp'
]
runscript = [
'ln -s /lib /var/simdata/openstudio/lib'
]
for src in dirs_to_mount:
container_mount = '/' + os.path.basename(src)
args.extend(['-B', '{}:{}:ro'.format(src, container_mount)])
container_symlink = os.path.join('/var/simdata/openstudio', os.path.basename(src))
runscript.append('ln -s {} {}'.format(*map(shlex.quote, (container_mount, container_symlink))))

if os.path.exists(os.path.join(cls.local_buildstock_dir, 'resources/hpxml-measures')):
runscript.append('ln -s /resources /var/simdata/openstudio/resources')
src = os.path.join(cls.local_buildstock_dir, 'resources/hpxml-measures')
container_mount = '/resources/hpxml-measures'
args.extend(['-B', '{}:{}:ro'.format(src, container_mount)])

# Build the openstudio command that will be issued within the
# singularity container If custom gems are to be used in the
# singularity container add extra bundle arguments to the cli
# command
cli_cmd = 'openstudio run -w in.osw'
if cfg.get('baseline', dict()).get('custom_gems', False):
cli_cmd = (
'openstudio --bundle /var/oscli/Gemfile --bundle_path /var/oscli/gems run -w in.osw --debug'
)
if get_bool_env_var('MEASURESONLY'):
cli_cmd += ' --measures_only'
runscript.append(cli_cmd)
args.extend([
str(cls.local_singularity_img),
'bash', '-x'
])
env_vars = dict(os.environ)
env_vars['SINGULARITYENV_BUILDSTOCKBATCH_VERSION'] = bsb_version
logger.debug('\n'.join(map(str, args)))
with open(os.path.join(sim_dir, 'singularity_output.log'), 'w') as f_out:
try:
subprocess.run(
args,
check=True,
input='\n'.join(runscript).encode('utf-8'),
stdout=f_out,
stderr=subprocess.STDOUT,
cwd=cls.local_output_dir,
env=env_vars,
)
except subprocess.CalledProcessError:
pass
finally:
# Clean up the symbolic links we created in the container
for mount_dir in dirs_to_mount + [os.path.join(sim_dir, 'lib')]:
try:
os.unlink(os.path.join(sim_dir, os.path.basename(mount_dir)))
except FileNotFoundError:
pass

# Clean up simulation directory
cls.cleanup_sim_dir(
sim_dir,
fs,
f'{output_dir}/results/simulation_output/timeseries',
upgrade_id,
i
)

reporting_measures = cls.get_reporting_measures(cfg)
dpout = postprocessing.read_simulation_outputs(fs, reporting_measures, sim_dir, upgrade_id, i)
Expand Down
28 changes: 20 additions & 8 deletions buildstockbatch/test/test_eagle.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,8 @@ def test_hpc_run_building(mock_subprocess, monkeypatch, basic_residential_projec
with patch.object(EagleBatch, 'weather_dir', None), \
patch.object(EagleBatch, 'singularity_image', '/path/to/singularity.simg'), \
patch.object(EagleBatch, 'create_osw', return_value=osw_dict), \
patch.object(EagleBatch, 'make_sim_dir', return_value=('bldg0000001up00', sim_path)):
patch.object(EagleBatch, 'make_sim_dir', return_value=('bldg0000001up00', sim_path)), \
patch.object(EagleBatch, 'local_scratch', tmp_path):

# Normal run
run_bldg_args = [
Expand All @@ -50,16 +51,25 @@ def test_hpc_run_building(mock_subprocess, monkeypatch, basic_residential_projec
'-e',
'--pwd',
'/var/simdata/openstudio',
'-B', f'{sim_path}:/var/simdata/openstudio',
'-B', '/tmp/scratch/buildstock/resources:/lib/resources',
'-B', '/tmp/scratch/housing_characteristics:/lib/housing_characteristics',
'-B', '/tmp/scratch/buildstock/measures:/measures:ro',
'-B', '/tmp/scratch/weather:/weather:ro',
]
end_expected_singularity_args = [
'/tmp/scratch/openstudio.simg',
'bash', '-x'
]
mock_subprocess.run.assert_called_once()
assert mock_subprocess.run.call_args[0][0] == expected_singularity_args
args = mock_subprocess.run.call_args[0][0]
for a, b in [args[i:i+2] for i in range(6, len(args) - 3, 2)]:
assert a == '-B'
assert b.split(':')[1] in (
'/var/simdata/openstudio',
'/lib/resources',
'/lib/housing_characteristics',
'/measures',
'/weather',
'/tmp',
)
assert mock_subprocess.run.call_args[0][0][0:6] == expected_singularity_args
assert mock_subprocess.run.call_args[0][0][-3:] == end_expected_singularity_args
called_kw = mock_subprocess.run.call_args[1]
assert called_kw.get('check') is True
assert 'input' in called_kw
Expand All @@ -75,7 +85,8 @@ def test_hpc_run_building(mock_subprocess, monkeypatch, basic_residential_projec
monkeypatch.setenv('MEASURESONLY', '1')
EagleBatch.run_building(*run_bldg_args)
mock_subprocess.run.assert_called_once()
assert mock_subprocess.run.call_args[0][0] == expected_singularity_args
assert mock_subprocess.run.call_args[0][0][0:6] == expected_singularity_args
assert mock_subprocess.run.call_args[0][0][-3:] == end_expected_singularity_args
called_kw = mock_subprocess.run.call_args[1]
assert called_kw.get('check') is True
assert 'input' in called_kw
Expand Down Expand Up @@ -223,6 +234,7 @@ def sequential_parallel(**kwargs):
mocker.patch.object(EagleBatch, 'local_housing_characteristics_dir',
results_dir / 'local_housing_characteristics_dir')
mocker.patch.object(EagleBatch, 'results_dir', results_dir)
mocker.patch.object(EagleBatch, 'local_scratch', results_dir.parent)

def make_sim_dir_mock(building_id, upgrade_idx, base_dir, overwrite_existing=False):
real_upgrade_idx = 0 if upgrade_idx is None else upgrade_idx + 1
Expand Down
40 changes: 21 additions & 19 deletions buildstockbatch/workflow_generator/residential_hpxml.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,25 +138,27 @@ def create_osw(self, sim_id, building_id, upgrade_idx):

if 'utility_bills' in workflow_args:
utility_bills = workflow_args['utility_bills']
utility_bills_map = [['utility_bill_scenario_names', 'scenario_name'],
['utility_bill_electricity_fixed_charges', 'elec_fixed_charge'],
['utility_bill_electricity_marginal_rates', 'elec_marginal_rate'],
['utility_bill_natural_gas_fixed_charges', 'gas_fixed_charge'],
['utility_bill_natural_gas_marginal_rates', 'gas_marginal_rate'],
['utility_bill_propane_fixed_charges', 'propane_fixed_charge'],
['utility_bill_propane_marginal_rates', 'propane_marginal_rate'],
['utility_bill_fuel_oil_fixed_charges', 'oil_fixed_charge'],
['utility_bill_fuel_oil_marginal_rates', 'oil_marginal_rate'],
['utility_bill_wood_fixed_charges', 'wood_fixed_charge'],
['utility_bill_wood_marginal_rates', 'wood_marginal_rate'],
['utility_bill_pv_compensation_types', 'pv_compensation_type'],
['utility_bill_pv_net_metering_annual_excess_sellback_rate_types',
'pv_net_metering_annual_excess_sellback_rate_type'],
['utility_bill_pv_net_metering_annual_excess_sellback_rates',
'pv_net_metering_annual_excess_sellback_rate'],
['utility_bill_pv_feed_in_tariff_rates', 'pv_feed_in_tariff_rate'],
['utility_bill_pv_monthly_grid_connection_fee_units', 'pv_monthly_grid_connection_fee_units'],
['utility_bill_pv_monthly_grid_connection_fees', 'pv_monthly_grid_connection_fee']]
utility_bills_map = [
['utility_bill_scenario_names', 'scenario_name'],
['utility_bill_electricity_fixed_charges', 'elec_fixed_charge'],
['utility_bill_electricity_marginal_rates', 'elec_marginal_rate'],
['utility_bill_natural_gas_fixed_charges', 'gas_fixed_charge'],
['utility_bill_natural_gas_marginal_rates', 'gas_marginal_rate'],
['utility_bill_propane_fixed_charges', 'propane_fixed_charge'],
['utility_bill_propane_marginal_rates', 'propane_marginal_rate'],
['utility_bill_fuel_oil_fixed_charges', 'oil_fixed_charge'],
['utility_bill_fuel_oil_marginal_rates', 'oil_marginal_rate'],
['utility_bill_wood_fixed_charges', 'wood_fixed_charge'],
['utility_bill_wood_marginal_rates', 'wood_marginal_rate'],
['utility_bill_pv_compensation_types', 'pv_compensation_type'],
['utility_bill_pv_net_metering_annual_excess_sellback_rate_types',
'pv_net_metering_annual_excess_sellback_rate_type'],
['utility_bill_pv_net_metering_annual_excess_sellback_rates',
'pv_net_metering_annual_excess_sellback_rate'],
['utility_bill_pv_feed_in_tariff_rates', 'pv_feed_in_tariff_rate'],
['utility_bill_pv_monthly_grid_connection_fee_units', 'pv_monthly_grid_connection_fee_units'],
['utility_bill_pv_monthly_grid_connection_fees', 'pv_monthly_grid_connection_fee']
]
for arg, item in utility_bills_map:
bld_exist_model_args[arg] = ','.join([str(s.get(item, '')) for s in utility_bills])

Expand Down
8 changes: 8 additions & 0 deletions docs/changelog/changelog_dev.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,3 +93,11 @@ Development Changelog
:tickets: 305

Now reruns jobs where the job.out-x is missing entirely.

.. change::
:tags: bugfix, eagle
:pullreq: 291

Mounts a temp dir into the container to avoid using the RAM disk.
Especially helpful for large schedules. Fixes `NREL/OpenStudio-HPXML#1070 <https://github.com/NREL/OpenStudio-HPXML/issues/1070>`_.

0 comments on commit 538d06e

Please sign in to comment.