Kestrel

.github/ISSUE_TEMPLATE/bug_report.md

@@ -29,8 +29,8 @@ log file here
 ```
 
 **Platform (please complete the following information):**
-- Simulation platform: [e.g. Eagle, AWS, local docker; please label with this as well]
-- BuildStockBatch version, branch, or sha: 
+- Simulation platform: [e.g. Kestrel, Eagle, AWS, local docker; please label with this as well]
+- BuildStockBatch version, branch, or sha:
 - resstock or comstock repo version, branch, or sha:
 - Local Desktop OS: [e.g. Windows, Mac, Linux, especially important if running locally]
 

.github/PULL_REQUEST_TEMPLATE/pull_request_template.md

@@ -14,5 +14,5 @@ Not all may apply
 - [ ] All other unit and integration tests passing
 - [ ] Update validation for project config yaml file changes
 - [ ] Update existing documentation
-- [ ] Run a small batch run on Eagle to make sure it all works if you made changes that will affect Eagle
+- [ ] Run a small batch run on Kestrel/Eagle to make sure it all works if you made changes that will affect Kestrel/Eagle
 - [ ] Add to the changelog_dev.rst file and propose migration text in the pull request

.gitignore

@@ -5,6 +5,7 @@ __pycache__
 docs/_build
 dask-worker-space
 venv/*
+.venv
 *.DS_Store
 *.zip
 .pytest_cache
@@ -15,4 +16,4 @@ coverage/
 .coverage
 build/
 .env
-.history
+.history

buildstockbatch/base.py

@@ -3,7 +3,7 @@
 """
 buildstockbatch.base
 ~~~~~~~~~~~~~~~
-This is the base class mixed into the deployment specific classes (i.e. eagle, local)
+This is the base class mixed into the deployment specific classes (i.e. kestrel, local)
 
 :author: Noel Merket
 :copyright: (c) 2018 by The Alliance for Sustainable Energy

buildstockbatch/eagle.sh

@@ -12,4 +12,4 @@ df -h
 module load conda singularity-container
 source activate "$MY_CONDA_ENV"
 
-time python -u -m buildstockbatch.eagle "$PROJECTFILE"
+time python -u -m buildstockbatch.hpc eagle "$PROJECTFILE"

buildstockbatch/eagle_postprocessing.sh

@@ -29,4 +29,4 @@ pdsh -w $SLURM_JOB_NODELIST_PACK_GROUP_1 "df -i; df -h"
 $MY_CONDA_ENV/bin/dask scheduler --scheduler-file $SCHEDULER_FILE &> $OUT_DIR/dask_scheduler.out &
 pdsh -w $SLURM_JOB_NODELIST_PACK_GROUP_1 "$MY_CONDA_ENV/bin/dask worker --scheduler-file $SCHEDULER_FILE --local-directory /tmp/scratch/dask --nworkers ${NPROCS} --nthreads 1 --memory-limit ${MEMORY}MB" &> $OUT_DIR/dask_workers.out &
 
-time python -u -m buildstockbatch.eagle "$PROJECTFILE"
+time python -u -m buildstockbatch.hpc eagle "$PROJECTFILE"

buildstockbatch/kestrel.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+#SBATCH --nodes=1
+#SBATCH --ntasks=1
+#SBATCH --tmp=1000000
+
+echo "Job ID: $SLURM_JOB_ID"
+echo "Hostname: $HOSTNAME"
+echo "QOS: $SLURM_JOB_QOS"
+
+df -i
+df -h
+
+module load python apptainer
+source "$MY_PYTHON_ENV/bin/activate"
+
+time python -u -m buildstockbatch.hpc kestrel "$PROJECTFILE"

buildstockbatch/kestrel_postprocessing.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+#SBATCH --tmp=1000000
+
+echo "begin kestrel_postprocessing.sh"
+
+echo "Job ID: $SLURM_JOB_ID"
+echo "Hostname: $HOSTNAME"
+
+df -i
+df -h
+
+module load python apptainer
+source "$MY_PYTHON_ENV/bin/activate"
+
+export POSTPROCESS=1
+
+echo "UPLOADONLY: ${UPLOADONLY}"
+echo "MEMORY: ${MEMORY}"
+echo "NPROCS: ${NPROCS}"
+
+SCHEDULER_FILE=$OUT_DIR/dask_scheduler.json
+
+echo "head node"
+echo $SLURM_JOB_NODELIST_PACK_GROUP_0
+echo "workers"
+echo $SLURM_JOB_NODELIST_PACK_GROUP_1
+
+pdsh -w $SLURM_JOB_NODELIST_PACK_GROUP_1 "free -h"
+pdsh -w $SLURM_JOB_NODELIST_PACK_GROUP_1 "df -i; df -h"
+
+$MY_PYTHON_ENV/bin/dask scheduler --scheduler-file $SCHEDULER_FILE &> $OUT_DIR/dask_scheduler.out &
+pdsh -w $SLURM_JOB_NODELIST_PACK_GROUP_1 "$MY_PYTHON_ENV/bin/dask worker --scheduler-file $SCHEDULER_FILE --local-directory /tmp/scratch/dask --nworkers ${NPROCS} --nthreads 1 --memory-limit ${MEMORY}MB" &> $OUT_DIR/dask_workers.out &
+
+time python -u -m buildstockbatch.hpc kestrel "$PROJECTFILE"

buildstockbatch/sampler/base.py

@@ -40,7 +40,7 @@ def __init__(self, parent):
         """
         Create the buildstock.csv file required for batch simulations using this class.
 
-        Multiple sampling methods are available to support local & eagle analyses, as well as to support multiple\
+        Multiple sampling methods are available to support local & hpc analyses, as well as to support multiple\
         sampling strategies. Currently there are separate implementations for commercial & residential stock types\
         due to unique requirements created by the commercial tsv set.
 

buildstockbatch/schemas/v0.3.yaml

@@ -6,6 +6,7 @@ weather_files_url: str(required=False)
 sampler: include('sampler-spec', required=True)
 workflow_generator: include('workflow-generator-spec', required=True)
 eagle: include('hpc-spec', required=False)
+kestrel: include('hpc-spec', required=False)
 aws: include('aws-spec', required=False)
 output_directory: regex('^(.*\/)?[a-z][a-z0-9_]*\/?$', required=True)
 sys_image_dir: str(required=False)
@@ -48,7 +49,7 @@ hpc-spec:
 hpc-postprocessing-spec:
   time: int(required=True)
   n_workers: int(min=1, max=32, required=False)
-  node_memory_mb: enum(85248, 180224, 751616, required=False)
+  node_memory_mb: int(min=85248, max=751616, required=False)
   n_procs: int(min=1, max=36, required=False)
   parquet_memory_mb: int(min=100, max=4096, required=False)
 

buildstockbatch/test/test_eagle.py

@@ -8,14 +8,14 @@
 from unittest.mock import patch
 import gzip
 
-from buildstockbatch.eagle import user_cli, EagleBatch
+from buildstockbatch.hpc import user_cli, EagleBatch
 from buildstockbatch.base import BuildStockBatchBase
 from buildstockbatch.utils import get_project_configuration, read_csv
 
 here = os.path.dirname(os.path.abspath(__file__))
 
 
-@patch("buildstockbatch.eagle.subprocess")
+@patch("buildstockbatch.hpc.subprocess")
 def test_hpc_run_building(mock_subprocess, monkeypatch, basic_residential_project_file):
     tar_filename = (
         pathlib.Path(__file__).resolve().parent / "test_results" / "simulation_output" / "simulations_job0.tar.gz"
@@ -93,9 +93,9 @@ def test_hpc_run_building(mock_subprocess, monkeypatch, basic_residential_projec
 
 
 @patch("buildstockbatch.base.BuildStockBatchBase.validate_options_lookup")
-@patch("buildstockbatch.eagle.EagleBatch.validate_output_directory_eagle")
-@patch("buildstockbatch.eagle.EagleBatch.validate_singularity_image_eagle")
-@patch("buildstockbatch.eagle.subprocess")
+@patch("buildstockbatch.hpc.EagleBatch.validate_output_directory_eagle")
+@patch("buildstockbatch.hpc.EagleBatch.validate_singularity_image_eagle")
+@patch("buildstockbatch.hpc.subprocess")
 def test_user_cli(
     mock_subprocess,
     mock_validate_singularity_image_eagle,
@@ -167,7 +167,7 @@ def test_user_cli(
     assert "0" == mock_subprocess.run.call_args[1]["env"]["MEASURESONLY"]
 
 
-@patch("buildstockbatch.eagle.subprocess")
+@patch("buildstockbatch.hpc.subprocess")
 def test_qos_high_job_submit(mock_subprocess, basic_residential_project_file, monkeypatch):
     mock_subprocess.run.return_value.stdout = "Submitted batch job 1\n"
     mock_subprocess.PIPE = None

buildstockbatch/test/test_validation.py

@@ -17,7 +17,7 @@
 import tempfile
 import json
 import pathlib
-from buildstockbatch.eagle import EagleBatch
+from buildstockbatch.hpc import EagleBatch
 from buildstockbatch.local import LocalBatch
 from buildstockbatch.base import BuildStockBatchBase, ValidationError
 from buildstockbatch.test.shared_testing_stuff import (
@@ -379,7 +379,7 @@ def test_validate_singularity_image_eagle(mocker, basic_residential_project_file
         with open(temp_yml, "w") as f:
             yaml.dump(cfg, f, Dumper=yaml.SafeDumper)
         with pytest.raises(ValidationError, match=r"image does not exist"):
-            EagleBatch.validate_singularity_image_eagle(str(temp_yml))
+            EagleBatch.validate_singularity_image_hpc(str(temp_yml))
 
 
 def test_validate_sampler_good_buildstock(basic_residential_project_file):

create_kestrel_env.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+DEV=0
+while getopts de: option
+do
+case "${option}"
+in
+d) DEV=1;;
+e) PYTHON_ENVS_DIR=${OPTARG};;
+esac
+done
+
+if [ -z "$PYTHON_ENVS_DIR" ]
+then
+    PYTHON_ENVS_DIR=/kfs2/shared-projects/buildstock/envs
+fi
+
+MY_PYTHON_ENV_NAME=${@:$OPTIND:1}
+if [ -z "$MY_PYTHON_ENV_NAME" ]
+then
+    echo "Environment name not provided"
+    exit 1
+fi
+
+MY_PYTHON_ENV="$PYTHON_ENVS_DIR/$MY_PYTHON_ENV_NAME"
+echo "Creating $MY_PYTHON_ENV"
+module load python
+python -m venv --clear --upgrade-deps --prompt "$MY_PYTHON_ENV_NAME" "$MY_PYTHON_ENV"
+source "$MY_PYTHON_ENV/bin/activate"
+which pip
+if [ $DEV -eq 1 ]
+then
+    pip install --no-cache-dir -e ".[dev]"
+else
+    pip install --no-cache-dir .
+fi

setup.py

@@ -65,7 +65,8 @@
     entry_points={
         "console_scripts": [
             "buildstock_local=buildstockbatch.local:main",
-            "buildstock_eagle=buildstockbatch.eagle:user_cli",
+            "buildstock_eagle=buildstockbatch.hpc:eagle_cli",
+            "buildstock_kestrel=buildstockbatch.hpc:kestrel_cli",
             "buildstock_aws=buildstockbatch.aws.aws:main",
         ]
     },