Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Move more shared code into DockerBatchBase #422

Merged
merged 3 commits into from Dec 8, 2023
Merged

Move more shared code into DockerBatchBase #422

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
40aa0ab
Move shared code into DockerBatchBase
nweires Dec 4, 2023
bc66dd5
Add changelog entry
nweires Dec 4, 2023
d826e51
Replace bucket and prefix params with a single path
nweires Dec 5, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
11 changes: 5 additions & 6 deletions buildstockbatch/cloud/docker_base.py
View file
Open in desktop
Expand Up @@ -234,7 +234,7 @@ def get_epws_to_download(cls, sim_dir, jobs_d):
return epws_to_download

@classmethod
def run_simulations(cls, cfg, job_id, jobs_d, sim_dir, fs, bucket, prefix):
def run_simulations(cls, cfg, job_id, jobs_d, sim_dir, fs, output_path):
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I like this too. I kind of wish I had done this everywhere and then parsed bucket and prefix out as needed.

"""
Run one batch of simulations.

Expand All @@ -245,15 +245,14 @@ def run_simulations(cls, cfg, job_id, jobs_d, sim_dir, fs, bucket, prefix):
:param jobs_d: Contents of a single job JSON file; contains the list of buildings to simulate in this job.
:param sim_dir: Path to the (local) directory where job files are stored.
:param fs: Filesystem to use when writing outputs to storage bucket
:param bucket: Name of the storage bucket to upload results to.
:param prefix: File prefix to use when writing to storage bucket.
:param output_path: File path (typically `bucket/prefix`) to write outputs to.
"""
local_fs = LocalFileSystem()
reporting_measures = cls.get_reporting_measures(cfg)
dpouts = []
simulation_output_tar_filename = sim_dir.parent / "simulation_outputs.tar.gz"
asset_dirs = os.listdir(sim_dir)
ts_output_dir = (f"{bucket}/{prefix}/results/simulation_output/timeseries",)
ts_output_dir = (f"{output_path}/results/simulation_output/timeseries",)

with tarfile.open(str(simulation_output_tar_filename), "w:gz") as simout_tar:
for building_id, upgrade_idx in jobs_d["batch"]:
Expand Down Expand Up @@ -316,12 +315,12 @@ def run_simulations(cls, cfg, job_id, jobs_d, sim_dir, fs, bucket, prefix):
# Upload simulation outputs tarfile to s3
fs.put(
str(simulation_output_tar_filename),
f"{bucket}/{prefix}/results/simulation_output/simulations_job{job_id}.tar.gz",
f"{output_path}/results/simulation_output/simulations_job{job_id}.tar.gz",
)

# Upload aggregated dpouts as a json file
with fs.open(
f"{bucket}/{prefix}/results/simulation_output/results_job{job_id}.json.gz",
f"{output_path}/results/simulation_output/results_job{job_id}.json.gz",
"wb",
) as f1:
with gzip.open(f1, "wt", encoding="utf-8") as f2:
Expand Down
2 changes: 1 addition & 1 deletion buildstockbatch/test/test_docker_base.py
View file
Open in desktop
Expand Up @@ -120,7 +120,7 @@ def test_run_simulations(basic_residential_project_file):
bucket = temp_path / "bucket"
os.makedirs(bucket / "test_prefix" / "results" / "simulation_output")

DockerBatchBase.run_simulations(cfg, 0, jobs_d, sim_dir, fs, bucket, "test_prefix")
DockerBatchBase.run_simulations(cfg, 0, jobs_d, sim_dir, fs, f"{bucket}/test_prefix")

output_dir = bucket / "test_prefix" / "results" / "simulation_output"
assert sorted(os.listdir(output_dir)) == ["results_job0.json.gz", "simulations_job0.tar.gz"]
Expand Down