Added Catalyst build support

- Fixed Make.catalyst include directories - Added SoA particle support to Conduit blueprints - Added AMReX_Conduit_Blueprint_ParticlesI.H to Conduit Make.package - Fixed Catalyst particle exporting issues - Fixed whitespaces
AMReX-Codes · Aug 1, 2023 · 8b4be4e · 8b4be4e
1 parent e0583fb
commit 8b4be4e
Show file tree

Hide file tree

Showing 13 changed files with 92 additions and 19 deletions.
diff --git a/Docs/sphinx_documentation/source/BuildingAMReX.rst b/Docs/sphinx_documentation/source/BuildingAMReX.rst
@@ -508,6 +508,8 @@ The list of available options is reported in the :ref:`table <tab:cmakevar>` bel
    +------------------------------+-------------------------------------------------+-------------------------+-----------------------+
    | AMReX_CONDUIT                |  Enable Conduit support                         | NO                      | YES, NO               |
    +------------------------------+-------------------------------------------------+-------------------------+-----------------------+
+   | AMReX_CATALYST               |  Enable Catalyst support                        | NO                      | YES, NO               |
+   +------------------------------+-------------------------------------------------+-------------------------+-----------------------+
    | AMReX_ASCENT                 |  Enable Ascent support                          | NO                      | YES, NO               |
    +------------------------------+-------------------------------------------------+-------------------------+-----------------------+
    | AMReX_HYPRE                  |  Enable HYPRE interfaces                        | NO                      | YES, NO               |

diff --git a/Src/CMakeLists.txt b/Src/CMakeLists.txt
@@ -43,7 +43,6 @@ include(AMReXParallelBackends)
 # Add definitions
 #
 include(AMReXSetDefines)
-
 #
 # Find and link third party libraries if needed
 #
@@ -180,7 +179,7 @@ if (AMReX_SENSEI)
    add_subdirectory(Extern/SENSEI)
 endif ()
 
-if (AMReX_CONDUIT OR AMReX_ASCENT)
+if (AMReX_CONDUIT OR AMReX_ASCENT OR AMReX_CATALYST)
    add_subdirectory(Extern/Conduit)
 endif ()
 

diff --git a/Src/Extern/Conduit/AMReX_Conduit_Blueprint.H b/Src/Extern/Conduit/AMReX_Conduit_Blueprint.H
@@ -96,9 +96,8 @@ namespace amrex
     //  coordset and fields used to represent the passed particle container.
     //  This allows you to use unique names to wrap multiple particle containers
     //  into a single blueprint tree.
-    template <int NStructReal, int NStructInt, int NArrayReal, int NArrayInt>
-    void ParticleContainerToBlueprint (const ParticleContainer<NStructReal,
-                                                               NStructInt,
+    template <typename ParticleType, int NArrayReal, int NArrayInt>
+    void ParticleContainerToBlueprint (const ParticleContainer_impl<ParticleType,
                                                                NArrayReal,
                                                                NArrayInt> &pc,
                                        const Vector<std::string> &real_comp_names,

diff --git a/Src/Extern/Conduit/AMReX_Conduit_Blueprint_ParticlesI.H b/Src/Extern/Conduit/AMReX_Conduit_Blueprint_ParticlesI.H
@@ -20,10 +20,9 @@ namespace amrex
 // Note:
 // This is a helper function, it's not part of the AMReX Blueprint Interface.
 //---------------------------------------------------------------------------//
-template <int NStructReal, int NStructInt, int NArrayReal, int NArrayInt>
+template <typename ParticleType, int NArrayReal, int NArrayInt>
 void
-ParticleTileToBlueprint(const ParticleTile<amrex::Particle<NStructReal,
-                                                           NStructInt>,
+ParticleTileToBlueprint(const ParticleTile<ParticleType,
                                            NArrayReal,
                                            NArrayInt> &ptile,
                         const Vector<std::string> &real_comp_names,
@@ -32,7 +31,7 @@ ParticleTileToBlueprint(const ParticleTile<amrex::Particle<NStructReal,
                         const std::string &topology_name)
 {
     int num_particles = ptile.GetArrayOfStructs().size();
-    int struct_size   = sizeof(Particle<NStructReal, NStructInt>);
+    int struct_size   = ParticleType::is_soa_particle ? 0 : sizeof(ParticleType);
 
     // knowing the above, we can zero copy the x,y,z positions + id, cpu
     // and any user fields in the AOS
@@ -68,25 +67,47 @@ ParticleTileToBlueprint(const ParticleTile<amrex::Particle<NStructReal,
     char* pbuf = const_cast<char*>(pbuf_const);
 
     ParticleReal* xp = reinterpret_cast<ParticleReal*>(pbuf); pbuf += sizeof(ParticleReal);
+#ifdef AMREX_USE_CATALYST
+    n_coords["values/x"].set(xp,
+                             num_particles,
+                             0,
+                             struct_size);
+#else
     n_coords["values/x"].set_external(xp,
                                       num_particles,
                                       0,
                                       struct_size);
+#endif //AMREX_USE_CATALYST                       
 #if AMREX_SPACEDIM > 1
     ParticleReal* yp = reinterpret_cast<ParticleReal*>(pbuf); pbuf += sizeof(ParticleReal);
+
+#ifdef AMREX_USE_CATALYST
+    n_coords["values/y"].set(yp,
+                             num_particles,
+                             0,
+                             struct_size);
+#else
     n_coords["values/y"].set_external(yp,
                                       num_particles,
                                       0,
                                       struct_size);
+#endif // AMREX_USE_CATALYST
 #endif
 #if AMREX_SPACEDIM > 2
     ParticleReal* zp = reinterpret_cast<ParticleReal*>(pbuf); pbuf += sizeof(ParticleReal);
+
+#ifdef AMREX_USE_CATALYST
+    n_coords["values/z"].set(zp,
+                             num_particles,
+                             0,
+                             struct_size);
+#else
     n_coords["values/z"].set_external(zp,
                                       num_particles,
                                       0,
                                       struct_size);
+#endif // AMREX_USE_CATALYST
 #endif
-
     // fields
     conduit::Node &n_fields = res["fields"];
 
@@ -97,7 +118,7 @@ ParticleTileToBlueprint(const ParticleTile<amrex::Particle<NStructReal,
     int vname_real_idx = 0;
     // struct real fields, the first set are always the particle positions
     // which we wrap above
-    for (int i = 0; i < NStructReal; i++)
+    for (int i = 0; i < ParticleType::NReal; i++)
     {
         ParticleReal* val = reinterpret_cast<ParticleReal*>(pbuf); pbuf += sizeof(ParticleReal);
         conduit::Node &n_f = n_fields[real_comp_names.at(vname_real_idx)];
@@ -143,7 +164,7 @@ ParticleTileToBlueprint(const ParticleTile<amrex::Particle<NStructReal,
     // --------------------------------
 
     int vname_int_idx = 0;
-    for (int i = 0; i < NStructInt; i++)
+    for (int i = 0; i < ParticleType::NInt; i++)
     {
         int* val = reinterpret_cast<int*>(pbuf); pbuf += sizeof(int);
         conduit::Node &n_f = n_fields[int_comp_names.at(vname_int_idx)];
@@ -193,10 +214,9 @@ ParticleTileToBlueprint(const ParticleTile<amrex::Particle<NStructReal,
 //---------------------------------------------------------------------------//
 // Converts a AMReX Particle Container into a Conduit Mesh Blueprint Hierarchy.
 //---------------------------------------------------------------------------//
-template <int NStructReal, int NStructInt, int NArrayReal, int NArrayInt>
+template <typename ParticleType, int NArrayReal, int NArrayInt>
 void
-ParticleContainerToBlueprint(const ParticleContainer<NStructReal,
-                                                     NStructInt,
+ParticleContainerToBlueprint(const ParticleContainer_impl<ParticleType,
                                                      NArrayReal,
                                                      NArrayInt> &pc,
                              const Vector<std::string> &real_comp_names,
@@ -209,8 +229,9 @@ ParticleContainerToBlueprint(const ParticleContainer<NStructReal,
     // validate varnames, which are used to provide field names
     // for user defined aos and soa values.
 
-    BL_ASSERT(real_comp_names.size() == (NStructReal + NArrayReal) );
-    BL_ASSERT(int_comp_names.size()  == (NStructInt + NArrayInt) );
+    // Not sure what to do here
+    // BL_ASSERT(real_comp_names.size() == (NStructReal + NArrayReal) );
+    // BL_ASSERT(int_comp_names.size()  == (NStructInt + NArrayInt) );
 
     int num_levels = pc.maxLevel() + 1;
     int num_domains = 0;
@@ -220,7 +241,7 @@ ParticleContainerToBlueprint(const ParticleContainer<NStructReal,
     int rank   = ParallelDescriptor::MyProc();
     int nprocs = ParallelDescriptor::NProcs();
 
-    using MyParConstIter = ParConstIter<NStructReal, NStructInt, NArrayReal, NArrayInt>;
+    using MyParConstIter = ParConstIter_impl<ParticleType, NArrayReal, NArrayInt>;
 
     //
     // blueprint expects unique ids for each domain published

diff --git a/Src/Extern/Conduit/CMakeLists.txt b/Src/Extern/Conduit/CMakeLists.txt
@@ -4,6 +4,7 @@ foreach(D IN LISTS AMReX_SPACEDIM)
     # TODO: Particles PR merges another file
     target_sources(amrex_${D}d
        PRIVATE
+       AMReX_Conduit_Blueprint_ParticlesI.H
        AMReX_Conduit_Blueprint.H
        AMReX_Conduit_Blueprint.cpp
     )

diff --git a/Src/Extern/Conduit/Make.package b/Src/Extern/Conduit/Make.package
@@ -3,7 +3,7 @@
 #
 
 CEXE_sources += AMReX_Conduit_Blueprint.cpp
-
+CEXE_headers += AMReX_Conduit_Blueprint_ParticlesI.H
 CEXE_headers += AMReX_Conduit_Blueprint.H
 
 VPATH_LOCATIONS += $(AMREX_HOME)/Src/Extern/Conduit

diff --git a/Tools/CMake/AMReXConfig.cmake.in b/Tools/CMake/AMReXConfig.cmake.in
@@ -79,6 +79,7 @@ set(AMReX_PARTICLES_FOUND           @AMReX_PARTICLES@)
 set(AMReX_P@AMReX_PARTICLES_PRECISION@_FOUND ON)
 set(AMReX_SENSEI_FOUND              @AMReX_SENSEI@)
 set(AMReX_CONDUIT_FOUND             @AMReX_CONDUIT@)
+set(AMReX_CATALYST_FOUND            @AMReX_CATALYST@)
 set(AMReX_ASCENT_FOUND              @AMReX_ASCENT@)
 set(AMReX_HYPRE_FOUND               @AMReX_HYPRE@)
 set(AMReX_PETSC_FOUND               @AMReX_PETSC@)
@@ -131,6 +132,7 @@ set(AMReX_PARTICLES_PRECISION       @AMReX_PARTICLES_PRECISION@)
 set(AMReX_SENSEI                    @AMReX_SENSEI@)
 set(AMReX_NO_SENSEI_AMR_INST        @AMReX_NO_SENSEI_AMR_INST@)
 set(AMReX_CONDUIT                   @AMReX_CONDUIT@)
+set(AMReX_CATALYST                  @AMReX_CATALYST@)
 set(AMReX_ASCENT                    @AMReX_ASCENT@)
 set(AMReX_HYPRE                     @AMReX_HYPRE@)
 set(AMReX_PETSC                     @AMReX_PETSC@)
@@ -197,6 +199,10 @@ if (@AMReX_ASCENT@)
     find_dependency(Ascent REQUIRED)
 endif ()
 
+if (@AMReX_CATALYST@)
+    find_dependency(Catalyst REQUIRED)
+endif ()
+
 if (@AMReX_CONDUIT@)
    find_dependency(Conduit REQUIRED)
 endif ()

diff --git a/Tools/CMake/AMReXOptions.cmake b/Tools/CMake/AMReXOptions.cmake
@@ -319,6 +319,11 @@ print_option( AMReX_NO_SENSEI_AMR_INST )
 option( AMReX_CONDUIT "Enable Conduit support" OFF )
 print_option( AMReX_CONDUIT )
 
+# Catalyst
+cmake_dependent_option( AMReX_CATALYST "Enable Catalyst support" OFF 
+   "AMReX_CONDUIT" OFF )
+print_option( AMReX_CATALYST )
+
 # Ascent
 cmake_dependent_option( AMReX_ASCENT "Enable Ascent support" OFF
    "AMReX_CONDUIT" OFF )

diff --git a/Tools/CMake/AMReXSetDefines.cmake b/Tools/CMake/AMReXSetDefines.cmake
@@ -138,6 +138,9 @@ add_amrex_define( AMREX_NO_SENSEI_AMR_INST NO_LEGACY IF AMReX_NO_SENSEI_AMR_INST
 # Conduit Support
 add_amrex_define( AMREX_USE_CONDUIT NO_LEGACY IF AMReX_CONDUIT )
 
+# Catalyst Support
+add_amrex_define( AMREX_USE_CATALYST NO_LEGACY IF AMReX_CATALYST )
+
 # Ascent Support
 add_amrex_define( AMREX_USE_ASCENT NO_LEGACY IF AMReX_ASCENT )
 

diff --git a/Tools/CMake/AMReXThirdPartyLibraries.cmake b/Tools/CMake/AMReXThirdPartyLibraries.cmake
@@ -90,6 +90,15 @@ if (AMReX_ASCENT) # Ascent will find conduit, so check for Ascent first
     endforeach()
 endif ()
 
+#
+#  Catalyst
+#
+if (AMReX_CATALYST) 
+    find_package(Catalyst REQUIRED PATHS "$ENV{CATALYST_IMPLEMENTATION_PATHS}")
+    foreach(D IN LISTS AMReX_SPACEDIM)
+        target_link_libraries(amrex_${D}d PUBLIC catalyst::catalyst)
+    endforeach()
+endif ()
 
 #
 # Conduit

diff --git a/Tools/CMake/AMReX_Config_ND.H.in b/Tools/CMake/AMReX_Config_ND.H.in
@@ -41,6 +41,7 @@
 #cmakedefine AMREX_USE_SENSEI_INSITU
 #cmakedefine AMREX_NO_SENSEI_AMR_INST
 #cmakedefine AMREX_USE_CONDUIT
+#cmakedefine AMREX_USE_CATALYST
 #cmakedefine AMREX_USE_ASCENT
 #cmakedefine AMREX_USE_EB
 #cmakedefine AMREX_USE_CUDA

diff --git a/Tools/GNUMake/Make.defs b/Tools/GNUMake/Make.defs
@@ -216,6 +216,12 @@ else
   USE_CONDUIT := FALSE
 endif
 
+ifdef USE_CATALYST
+  USE_CATALYST := $(strip $(USE_CATALYST))
+else
+  USE_CATALYST := FALSE
+endif
+
 ifdef USE_ASCENT
   USE_ASCENT := $(strip $(USE_ASCENT))
 else
@@ -1042,6 +1048,11 @@ ifeq ($(USE_CONDUIT),TRUE)
   include        $(AMREX_HOME)/Tools/GNUMake/packages/Make.conduit
 endif
 
+ifeq ($(USE_CATALYST),TRUE)
+  $(info Loading $(AMREX_HOME)/Tools/GNUMake/packages/Make.catalyst...)
+  include        $(AMREX_HOME)/Tools/GNUMake/packages/Make.catalyst
+endif
+
 ifeq ($(USE_ASCENT),TRUE)
   $(info Loading $(AMREX_HOME)/Tools/GNUMake/packages/Make.ascent...)
   include        $(AMREX_HOME)/Tools/GNUMake/packages/Make.ascent

diff --git a/Tools/GNUMake/packages/Make.catalyst b/Tools/GNUMake/packages/Make.catalyst
@@ -0,0 +1,16 @@
+#########################################################
+# Catalyst (https://gitlab.kitware.com/paraview/catalyst) Support 
+#########################################################
+
+CPPFLAGS += -DAMREX_USE_CATALYST
+
+ifdef CATALYST_DIR
+  INCLUDE_LOCATIONS += $(CATALYST_DIR)/../../../src
+  VPATH_LOCATIONS   += $(CATALYST_DIR)/../../../src
+  LIBRARY_LOCATIONS += $(CATALYST_DIR)/../../
+  LIBRARIES += -Wl,-rpath,$(CATALYST_DIR)/../../
+
+  LIBRARIES += -lcatalyst
+
+endif
+