Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Mounting a temp dir into the singularity container #291

Merged
merged 6 commits into from
Oct 3, 2022
Merged

Mounting a temp dir into the singularity container #291

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
aee28c8
mounting a temporary directory into the singularity container
nmerket May 18, 2022
90e5572
fixing env var issue for testing
nmerket May 18, 2022
41a4c48
fixing tests
nmerket May 18, 2022
0b0f3d6
Merge branch 'develop' into eagle_tmp_dir
nmerket Oct 3, 2022
84328ff
style fixes
nmerket Oct 3, 2022
817a59a
adding changelog entry
nmerket Oct 3, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
149 changes: 79 additions & 70 deletions buildstockbatch/eagle.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import shutil
import subprocess
import sys
import tempfile
import time
import yaml
import csv
Expand All @@ -54,7 +55,7 @@ class EagleBatch(BuildStockBatchBase):
hpc_name = 'eagle'
min_sims_per_job = 36 * 2

local_scratch = pathlib.Path('/tmp/scratch')
local_scratch = pathlib.Path(os.environ.get('LOCAL_SCRATCH', '/tmp/scratch'))
local_project_dir = local_scratch / 'project'
local_buildstock_dir = local_scratch / 'buildstock'
local_weather_dir = local_scratch / 'weather'
Expand Down Expand Up @@ -344,76 +345,84 @@ def run_building(cls, output_dir, cfg, n_datapoints, i, upgrade_idx=None):
cls.local_weather_dir,
]

# Build the command to instantiate and configure the singularity container the simulation is run inside
local_resources_dir = cls.local_buildstock_dir / 'resources'
args = [
'singularity', 'exec',
'--contain',
'-e',
'--pwd', '/var/simdata/openstudio',
'-B', f'{sim_dir}:/var/simdata/openstudio',
'-B', f'{local_resources_dir}:/lib/resources',
'-B', f'{cls.local_housing_characteristics_dir}:/lib/housing_characteristics'
]
runscript = [
'ln -s /lib /var/simdata/openstudio/lib'
]
for src in dirs_to_mount:
container_mount = '/' + os.path.basename(src)
args.extend(['-B', '{}:{}:ro'.format(src, container_mount)])
container_symlink = os.path.join('/var/simdata/openstudio', os.path.basename(src))
runscript.append('ln -s {} {}'.format(*map(shlex.quote, (container_mount, container_symlink))))

if os.path.exists(os.path.join(cls.local_buildstock_dir, 'resources/hpxml-measures')):
runscript.append('ln -s /resources /var/simdata/openstudio/resources')
src = os.path.join(cls.local_buildstock_dir, 'resources/hpxml-measures')
container_mount = '/resources/hpxml-measures'
args.extend(['-B', '{}:{}:ro'.format(src, container_mount)])

# Build the openstudio command that will be issued within the singularity container
# If custom gems are to be used in the singularity container add extra bundle arguments to the cli command
cli_cmd = 'openstudio run -w in.osw'
if cfg.get('baseline', dict()).get('custom_gems', False):
cli_cmd = 'openstudio --bundle /var/oscli/Gemfile --bundle_path /var/oscli/gems run -w in.osw --debug'
if get_bool_env_var('MEASURESONLY'):
cli_cmd += ' --measures_only'
runscript.append(cli_cmd)
args.extend([
str(cls.local_singularity_img),
'bash', '-x'
])
env_vars = dict(os.environ)
env_vars['SINGULARITYENV_BUILDSTOCKBATCH_VERSION'] = bsb_version
logger.debug('\n'.join(map(str, args)))
with open(os.path.join(sim_dir, 'singularity_output.log'), 'w') as f_out:
try:
subprocess.run(
args,
check=True,
input='\n'.join(runscript).encode('utf-8'),
stdout=f_out,
stderr=subprocess.STDOUT,
cwd=cls.local_output_dir,
env=env_vars,
)
except subprocess.CalledProcessError:
pass
finally:
# Clean up the symbolic links we created in the container
for mount_dir in dirs_to_mount + [os.path.join(sim_dir, 'lib')]:
try:
os.unlink(os.path.join(sim_dir, os.path.basename(mount_dir)))
except FileNotFoundError:
pass

# Clean up simulation directory
cls.cleanup_sim_dir(
sim_dir,
fs,
f'{output_dir}/results/simulation_output/timeseries',
upgrade_id,
i
# Create a temporary directory for the simulation to use
with tempfile.TemporaryDirectory(dir=cls.local_scratch, prefix=f"{sim_id}_") as tmpdir:

# Build the command to instantiate and configure the singularity container the simulation is run inside
local_resources_dir = cls.local_buildstock_dir / 'resources'
args = [
'singularity', 'exec',
'--contain',
'-e',
'--pwd', '/var/simdata/openstudio',
'-B', f'{sim_dir}:/var/simdata/openstudio',
'-B', f'{local_resources_dir}:/lib/resources',
'-B', f'{cls.local_housing_characteristics_dir}:/lib/housing_characteristics',
'-B', f'{tmpdir}:/tmp'
]
runscript = [
'ln -s /lib /var/simdata/openstudio/lib'
]
for src in dirs_to_mount:
container_mount = '/' + os.path.basename(src)
args.extend(['-B', '{}:{}:ro'.format(src, container_mount)])
container_symlink = os.path.join('/var/simdata/openstudio', os.path.basename(src))
runscript.append('ln -s {} {}'.format(*map(shlex.quote, (container_mount, container_symlink))))

if os.path.exists(os.path.join(cls.local_buildstock_dir, 'resources/hpxml-measures')):
runscript.append('ln -s /resources /var/simdata/openstudio/resources')
src = os.path.join(cls.local_buildstock_dir, 'resources/hpxml-measures')
container_mount = '/resources/hpxml-measures'
args.extend(['-B', '{}:{}:ro'.format(src, container_mount)])

# Build the openstudio command that will be issued within the
# singularity container If custom gems are to be used in the
# singularity container add extra bundle arguments to the cli
# command
cli_cmd = 'openstudio run -w in.osw'
if cfg.get('baseline', dict()).get('custom_gems', False):
cli_cmd = (
'openstudio --bundle /var/oscli/Gemfile --bundle_path /var/oscli/gems run -w in.osw --debug'
)
if get_bool_env_var('MEASURESONLY'):
cli_cmd += ' --measures_only'
runscript.append(cli_cmd)
args.extend([
str(cls.local_singularity_img),
'bash', '-x'
])
env_vars = dict(os.environ)
env_vars['SINGULARITYENV_BUILDSTOCKBATCH_VERSION'] = bsb_version
logger.debug('\n'.join(map(str, args)))
with open(os.path.join(sim_dir, 'singularity_output.log'), 'w') as f_out:
try:
subprocess.run(
args,
check=True,
input='\n'.join(runscript).encode('utf-8'),
stdout=f_out,
stderr=subprocess.STDOUT,
cwd=cls.local_output_dir,
env=env_vars,
)
except subprocess.CalledProcessError:
pass
finally:
# Clean up the symbolic links we created in the container
for mount_dir in dirs_to_mount + [os.path.join(sim_dir, 'lib')]:
try:
os.unlink(os.path.join(sim_dir, os.path.basename(mount_dir)))
except FileNotFoundError:
pass

# Clean up simulation directory
cls.cleanup_sim_dir(
sim_dir,
fs,
f'{output_dir}/results/simulation_output/timeseries',
upgrade_id,
i
)

reporting_measures = cls.get_reporting_measures(cfg)
dpout = postprocessing.read_simulation_outputs(fs, reporting_measures, sim_dir, upgrade_id, i)
Expand Down
28 changes: 20 additions & 8 deletions buildstockbatch/test/test_eagle.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,8 @@ def test_hpc_run_building(mock_subprocess, monkeypatch, basic_residential_projec
with patch.object(EagleBatch, 'weather_dir', None), \
patch.object(EagleBatch, 'singularity_image', '/path/to/singularity.simg'), \
patch.object(EagleBatch, 'create_osw', return_value=osw_dict), \
patch.object(EagleBatch, 'make_sim_dir', return_value=('bldg0000001up00', sim_path)):
patch.object(EagleBatch, 'make_sim_dir', return_value=('bldg0000001up00', sim_path)), \
patch.object(EagleBatch, 'local_scratch', tmp_path):

# Normal run
run_bldg_args = [
Expand All @@ -50,16 +51,25 @@ def test_hpc_run_building(mock_subprocess, monkeypatch, basic_residential_projec
'-e',
'--pwd',
'/var/simdata/openstudio',
'-B', f'{sim_path}:/var/simdata/openstudio',
'-B', '/tmp/scratch/buildstock/resources:/lib/resources',
'-B', '/tmp/scratch/housing_characteristics:/lib/housing_characteristics',
'-B', '/tmp/scratch/buildstock/measures:/measures:ro',
'-B', '/tmp/scratch/weather:/weather:ro',
]
end_expected_singularity_args = [
'/tmp/scratch/openstudio.simg',
'bash', '-x'
]
mock_subprocess.run.assert_called_once()
assert mock_subprocess.run.call_args[0][0] == expected_singularity_args
args = mock_subprocess.run.call_args[0][0]
for a, b in [args[i:i+2] for i in range(6, len(args) - 3, 2)]:
assert a == '-B'
assert b.split(':')[1] in (
'/var/simdata/openstudio',
'/lib/resources',
'/lib/housing_characteristics',
'/measures',
'/weather',
'/tmp',
)
assert mock_subprocess.run.call_args[0][0][0:6] == expected_singularity_args
assert mock_subprocess.run.call_args[0][0][-3:] == end_expected_singularity_args
called_kw = mock_subprocess.run.call_args[1]
assert called_kw.get('check') is True
assert 'input' in called_kw
Expand All @@ -75,7 +85,8 @@ def test_hpc_run_building(mock_subprocess, monkeypatch, basic_residential_projec
monkeypatch.setenv('MEASURESONLY', '1')
EagleBatch.run_building(*run_bldg_args)
mock_subprocess.run.assert_called_once()
assert mock_subprocess.run.call_args[0][0] == expected_singularity_args
assert mock_subprocess.run.call_args[0][0][0:6] == expected_singularity_args
assert mock_subprocess.run.call_args[0][0][-3:] == end_expected_singularity_args
called_kw = mock_subprocess.run.call_args[1]
assert called_kw.get('check') is True
assert 'input' in called_kw
Expand Down Expand Up @@ -223,6 +234,7 @@ def sequential_parallel(**kwargs):
mocker.patch.object(EagleBatch, 'local_housing_characteristics_dir',
results_dir / 'local_housing_characteristics_dir')
mocker.patch.object(EagleBatch, 'results_dir', results_dir)
mocker.patch.object(EagleBatch, 'local_scratch', results_dir.parent)

def make_sim_dir_mock(building_id, upgrade_idx, base_dir, overwrite_existing=False):
real_upgrade_idx = 0 if upgrade_idx is None else upgrade_idx + 1
Expand Down
40 changes: 21 additions & 19 deletions buildstockbatch/workflow_generator/residential_hpxml.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,25 +138,27 @@ def create_osw(self, sim_id, building_id, upgrade_idx):

if 'utility_bills' in workflow_args:
utility_bills = workflow_args['utility_bills']
utility_bills_map = [['utility_bill_scenario_names', 'scenario_name'],
['utility_bill_electricity_fixed_charges', 'elec_fixed_charge'],
['utility_bill_electricity_marginal_rates', 'elec_marginal_rate'],
['utility_bill_natural_gas_fixed_charges', 'gas_fixed_charge'],
['utility_bill_natural_gas_marginal_rates', 'gas_marginal_rate'],
['utility_bill_propane_fixed_charges', 'propane_fixed_charge'],
['utility_bill_propane_marginal_rates', 'propane_marginal_rate'],
['utility_bill_fuel_oil_fixed_charges', 'oil_fixed_charge'],
['utility_bill_fuel_oil_marginal_rates', 'oil_marginal_rate'],
['utility_bill_wood_fixed_charges', 'wood_fixed_charge'],
['utility_bill_wood_marginal_rates', 'wood_marginal_rate'],
['utility_bill_pv_compensation_types', 'pv_compensation_type'],
['utility_bill_pv_net_metering_annual_excess_sellback_rate_types',
'pv_net_metering_annual_excess_sellback_rate_type'],
['utility_bill_pv_net_metering_annual_excess_sellback_rates',
'pv_net_metering_annual_excess_sellback_rate'],
['utility_bill_pv_feed_in_tariff_rates', 'pv_feed_in_tariff_rate'],
['utility_bill_pv_monthly_grid_connection_fee_units', 'pv_monthly_grid_connection_fee_units'],
['utility_bill_pv_monthly_grid_connection_fees', 'pv_monthly_grid_connection_fee']]
utility_bills_map = [
['utility_bill_scenario_names', 'scenario_name'],
['utility_bill_electricity_fixed_charges', 'elec_fixed_charge'],
['utility_bill_electricity_marginal_rates', 'elec_marginal_rate'],
['utility_bill_natural_gas_fixed_charges', 'gas_fixed_charge'],
['utility_bill_natural_gas_marginal_rates', 'gas_marginal_rate'],
['utility_bill_propane_fixed_charges', 'propane_fixed_charge'],
['utility_bill_propane_marginal_rates', 'propane_marginal_rate'],
['utility_bill_fuel_oil_fixed_charges', 'oil_fixed_charge'],
['utility_bill_fuel_oil_marginal_rates', 'oil_marginal_rate'],
['utility_bill_wood_fixed_charges', 'wood_fixed_charge'],
['utility_bill_wood_marginal_rates', 'wood_marginal_rate'],
['utility_bill_pv_compensation_types', 'pv_compensation_type'],
['utility_bill_pv_net_metering_annual_excess_sellback_rate_types',
'pv_net_metering_annual_excess_sellback_rate_type'],
['utility_bill_pv_net_metering_annual_excess_sellback_rates',
'pv_net_metering_annual_excess_sellback_rate'],
['utility_bill_pv_feed_in_tariff_rates', 'pv_feed_in_tariff_rate'],
['utility_bill_pv_monthly_grid_connection_fee_units', 'pv_monthly_grid_connection_fee_units'],
['utility_bill_pv_monthly_grid_connection_fees', 'pv_monthly_grid_connection_fee']
]
for arg, item in utility_bills_map:
bld_exist_model_args[arg] = ','.join([str(s.get(item, '')) for s in utility_bills])

Expand Down
8 changes: 8 additions & 0 deletions docs/changelog/changelog_dev.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,3 +93,11 @@ Development Changelog
:tickets: 305

Now reruns jobs where the job.out-x is missing entirely.

.. change::
:tags: bugfix, eagle
:pullreq: 291

Mounts a temp dir into the container to avoid using the RAM disk.
Especially helpful for large schedules. Fixes `NREL/OpenStudio-HPXML#1070 <https://github.com/NREL/OpenStudio-HPXML/issues/1070>`_.