Merge pull request #316 from jaredthomas68/greenheart-mpi

Greenheart MPI

greenheart/tools/optimization/fileIO.py

@@ -0,0 +1,110 @@
+"""
+This file is based on the WISDEM file of the same name: https://github.com/WISDEM/WISDEM
+"""
+
+import os
+import pickle
+
+import numpy as np
+import pandas as pd
+import scipy.io as sio
+
+
+def save_data(fname, prob, npz_file=True, mat_file=True, xls_file=True):
+    # Remove file extension
+    froot = os.path.splitext(fname)[0]
+
+    # Get all OpenMDAO inputs and outputs into a dictionary
+    var_dict = prob.model.list_inputs(prom_name=True, units=True, desc=True, out_stream=None)
+    out_dict = prob.model.list_outputs(prom_name=True, units=True, desc=True, out_stream=None)
+    var_dict.extend(out_dict)
+
+    # Pickle the full archive so that we can load it back in if we need
+    with open(froot + ".pkl", "wb") as f:
+        pickle.dump(var_dict, f)
+
+    # Reduce to variables we can save for matlab or python
+    if npz_file or mat_file:
+        array_dict = {}
+        for k in range(len(var_dict)):
+            unit_str = var_dict[k][1]["units"]
+            if unit_str is None or unit_str == "Unavailable":
+                unit_str = ""
+            elif len(unit_str) > 0:
+                unit_str = "_" + unit_str
+
+            iname = var_dict[k][1]["prom_name"] + unit_str
+            value = var_dict[k][1]["val"]
+
+            if iname in array_dict:
+                continue
+
+            if type(value) in [type(np.array([])), type(0.0), type(0), np.float64, np.int64]:
+                array_dict[iname] = value
+            elif type(value) == type(True):
+                array_dict[iname] = np.bool_(value)
+            elif type(value) == type(""):
+                array_dict[iname] = np.str_(value)
+            elif type(value) == type([]):
+                temp_val = np.empty(len(value), dtype=object)
+                temp_val[:] = value[:]
+                array_dict[iname] = temp_val
+            # else:
+            #    print(var_dict[k])
+
+    # Save to numpy compatible
+    if npz_file:
+        kwargs = {key: array_dict[key] for key in array_dict.keys()}
+        np.savez_compressed(froot + ".npz", **kwargs)
+
+    # Save to matlab compatible
+    if mat_file:
+        sio.savemat(froot + ".mat", array_dict, long_field_names=True)
+
+    if xls_file:
+        data = {}
+        data["variables"] = []
+        data["units"] = []
+        data["values"] = []
+        data["description"] = []
+        for k in range(len(var_dict)):
+            unit_str = var_dict[k][1]["units"]
+            if unit_str is None:
+                unit_str = ""
+
+            iname = var_dict[k][1]["prom_name"]
+            if iname in data["variables"]:
+                continue
+
+            data["variables"].append(iname)
+            data["units"].append(unit_str)
+            data["values"].append(var_dict[k][1]["val"])
+            data["description"].append(var_dict[k][1]["desc"])
+        df = pd.DataFrame(data)
+        df.to_excel(froot + ".xlsx", index=False)
+        df.to_csv(froot + ".csv", index=False)
+
+
+def load_data(fname, prob):
+    # Remove file extension
+    froot = os.path.splitext(fname)[0]
+
+    # Load in the pickled data
+    with open(froot + ".pkl", "rb") as f:
+        var_dict = pickle.load(f)
+
+    # Store into Problem object
+    for k in range(len(var_dict)):
+        iname = var_dict[k][0]
+        iname2 = var_dict[k][1]["prom_name"]
+        value = var_dict[k][1]["val"]
+        try:
+            prob.set_val(iname, value)
+        except:
+            pass
+        try:
+            prob.set_val(iname2, value)
+        except:
+            pass
+
+    return prob

greenheart/tools/optimization/gc_run_greenheart.py

@@ -1,16 +1,21 @@
+"""
+This file is based on the WISDEM file 'runWISDEM.py`: https://github.com/WISDEM/WISDEM
+"""
+
 import os
 import sys
 import logging
 import warnings
 
 import numpy as np
-# from mpi4py import MPI
 import openmdao.api as om
 
 
 from greenheart.simulation.greenheart_simulation import GreenHeartSimulationConfig, setup_greenheart_simulation
 from greenheart.tools.optimization.gc_PoseOptimization import PoseOptimization
 from greenheart.tools.optimization.openmdao import GreenHeartComponent
+from greenheart.tools.optimization.mpi_tools import MPI, map_comm_heirarchical
+from greenheart.tools.optimization import fileIO
 
 def run_greenheart(config:GreenHeartSimulationConfig, overridden_values=None, run_only=False):
     """This functions sets up and runs greenheart. It can be used for analysis runs, optimizations, design of experiments, or step size studies
@@ -25,38 +30,43 @@ def run_greenheart(config:GreenHeartSimulationConfig, overridden_values=None, ru
         config: see Args
     """
     # Initialize openmdao problem. If running with multiple processors in MPI, use parallel finite differencing equal to the number of cores used.
-    # Otherwise, initialize the WindPark system normally. Get the rank number for parallelization. We only print output files using the root processor.
+    # Otherwise, initialize the GreenHEART system normally. Get the rank number for parallelization. We only print output files using the root processor.
     myopt = PoseOptimization(config)
 
-    # if MPI:
-    #     n_DV = myopt.get_number_design_variables()
-
-    #     # Extract the number of cores available
-    #     max_cores = MPI.COMM_WORLD.Get_size()
-
-    #     if max_cores > n_DV:
-    #         raise ValueError(
-    #             "ERROR: please reduce the number of cores, currently set to "
-    #             + str(max_cores)
-    #             + ", to the number of finite differences "
-    #             + str(n_DV)
-    #             + ", which is equal to the number of design variables DV for forward differencing"
-    #             + " and DV times 2 for central differencing,"
-    #             + " or the parallelization logic will not work"
-    #         )
-
-    #     # Define the color map for the parallelization, determining the maximum number of parallel finite difference (FD) evaluations based on the number of design variables (DV).
-    #     n_FD = min([max_cores, n_DV])
-
-    #     # Define the color map for the cores
-    #     n_FD = max([n_FD, 1])
-    #     comm_map_down, comm_map_up, color_map = map_comm_heirarchical(n_FD, 1)
-    #     rank = MPI.COMM_WORLD.Get_rank()
-    #     color_i = color_map[rank]
-    #     comm_i = MPI.COMM_WORLD.Split(color_i, 1)
-    # else:
-    color_i = 0
-    rank = 0
+    if MPI:
+        n_DV = myopt.get_number_design_variables()
+
+        # Extract the number of cores available
+        max_cores = MPI.COMM_WORLD.Get_size()
+
+        if max_cores > n_DV and not config.greenheart_config["opt_options"]["driver"]["design_of_experiments"]["flag"]:
+            raise ValueError(
+                "ERROR: please reduce the number of cores, currently set to "
+                + str(max_cores)
+                + ", to the number of finite differences "
+                + str(n_DV)
+                + ", which is equal to the number of design variables DV for forward differencing"
+                + " and DV times 2 for central differencing,"
+                + " or the parallelization logic will not work"
+            )
+
+        if config.greenheart_config["opt_options"]["driver"]["design_of_experiments"]["flag"]:
+            n_FD = max_cores
+
+        else:
+            # Define the color map for the parallelization, determining the maximum number of parallel finite difference (FD) evaluations based on the number of design variables (DV).
+            n_FD = min([max_cores, n_DV])
+
+            # Define the color map for the cores
+            n_FD = max([n_FD, 1])
+
+        comm_map_down, comm_map_up, color_map = map_comm_heirarchical(n_FD, 1)
+        rank = MPI.COMM_WORLD.Get_rank()
+        color_i = color_map[rank]
+        comm_i = MPI.COMM_WORLD.Split(color_i, 1)
+    else:
+        color_i = 0
+        rank = 0
 
     folder_output = config.output_dir
 
@@ -99,13 +109,13 @@ def run_greenheart(config:GreenHeartSimulationConfig, overridden_values=None, ru
         logger.info("Started")
 
     if color_i == 0:  # the top layer of cores enters
-        # if MPI:
-        #     # Parallel settings for OpenMDAO
-        #     prob = om.Problem(model=om.Group(num_par_fd=n_FD), comm=comm_i, reports=False)
+        if MPI:
+            # Parallel settings for OpenMDAO
+            prob = om.Problem(model=om.Group(num_par_fd=n_FD), comm=comm_i, reports=False)
 
-        # else:
-        # Sequential finite differencing
-        prob = om.Problem(model=om.Group(), reports=False)
+        else:
+            # Sequential finite differencing
+            prob = om.Problem(model=om.Group(), reports=False)
 
         prob.model.add_subsystem(
                 'greenheart', GreenHeartComponent(config=config, design_variables=design_variables),  
@@ -118,6 +128,8 @@ def run_greenheart(config:GreenHeartSimulationConfig, overridden_values=None, ru
             prob = myopt.set_objective(prob)
             prob = myopt.set_design_variables(prob, config, hi)
             prob = myopt.set_constraints(prob, hi)
+
+        if config.greenheart_config["opt_options"]["recorder"]["flag"]:
             prob = myopt.set_recorders(prob)
 
         # Setup openmdao problem
@@ -134,6 +146,7 @@ def run_greenheart(config:GreenHeartSimulationConfig, overridden_values=None, ru
         # Place the last design variables from a previous run into the problem.
         # This needs to occur after the above setup() and yaml2openmdao() calls
         # so these values are correctly placed in the problem.
+
         if not run_only:
             prob = myopt.set_restart(prob)
 
@@ -148,7 +161,7 @@ def run_greenheart(config:GreenHeartSimulationConfig, overridden_values=None, ru
                     checks = prob.check_partials(compact_print=True)
 
             sys.stdout.flush()
-
+            
             if config.greenheart_config["opt_options"]["driver"]["step_size_study"]["flag"]:
                 prob.run_model()
                 study_options = config.greenheart_config["opt_options"]["driver"]["step_size_study"]
@@ -177,16 +190,18 @@ def run_greenheart(config:GreenHeartSimulationConfig, overridden_values=None, ru
                 prob.run_driver()
         else:
             prob.run_model()
+        if config.greenheart_config["opt_options"]["recorder"]["flag"]:
+            prob.record("final_state")
 
-        # if (not MPI) or (MPI and rank == 0):
-        #     # Save data coming from openmdao to an output yaml file
-        #     froot_out = os.path.join(folder_output, config.greenheart_config["opt_options"]["general"]["fname_output"])
+        if (not MPI) or (MPI and rank == 0):
+            # Save data coming from openmdao to an output yaml file
+            froot_out = os.path.join(folder_output, config.greenheart_config["opt_options"]["general"]["fname_output"])
 
-        #     # Save data to numpy and matlab arrays
-        #     fileIO.save_data(froot_out, prob)
+            # Save data to numpy and matlab arrays
+            fileIO.save_data(froot_out, prob)
 
     if rank == 0:
         return prob, config
     else:
-        return [], [], []
+        return [], []