diff --git a/.gitlab/corona-build-and-test-extra.yml b/.gitlab/corona-build-and-test-extra.yml index a94300f85b..29624d700f 100644 --- a/.gitlab/corona-build-and-test-extra.yml +++ b/.gitlab/corona-build-and-test-extra.yml @@ -23,6 +23,6 @@ rocm_5_1_1_clang_13_0_0_desul_atomics: variables: - SPEC: " +rocm ~openmp +desul amdgpu_target=gfx906 %clang@13.0.0 ^hip@5.1.1 ^blt@develop" + SPEC: " ~shared +rocm ~openmp +tests +desul amdgpu_target=gfx906 %clang@13.0.0 ^hip@5.1.1 ^blt@develop" extends: .build_and_test_on_corona diff --git a/.gitlab/custom-jobs-and-variables.yml b/.gitlab/custom-jobs-and-variables.yml index 53f36c56cd..a81ace1156 100644 --- a/.gitlab/custom-jobs-and-variables.yml +++ b/.gitlab/custom-jobs-and-variables.yml @@ -18,7 +18,7 @@ variables: # Arguments for job level allocation RUBY_BUILD_AND_TEST_JOB_ALLOC: "--time=45 --nodes=1" # Project specific variants for ruby - PROJECT_RUBY_VARIANTS: "+openmp " + PROJECT_RUBY_VARIANTS: "~shared +openmp +tests" # Project specific deps for ruby PROJECT_RUBY_DEPS: "" @@ -28,7 +28,7 @@ variables: # Arguments for job level allocation CORONA_BUILD_AND_TEST_JOB_ALLOC: "--time-limit=45m --nodes=1" # Project specific variants for corona - PROJECT_CORONA_VARIANTS: "~openmp " + PROJECT_CORONA_VARIANTS: "~shared ~openmp +tests" # Project specific deps for corona PROJECT_CORONA_DEPS: "^blt@develop " @@ -37,7 +37,7 @@ variables: # Arguments for job level allocation LASSEN_BUILD_AND_TEST_JOB_ALLOC: "1 -W 60" # Project specific variants for lassen - PROJECT_LASSEN_VARIANTS: "+openmp " + PROJECT_LASSEN_VARIANTS: "~shared +openmp +tests" # Project specific deps for lassen PROJECT_LASSEN_DEPS: "" diff --git a/.gitlab/lassen-build-and-test-extra.yml b/.gitlab/lassen-build-and-test-extra.yml index 0442a602bd..da0a050581 100644 --- a/.gitlab/lassen-build-and-test-extra.yml +++ b/.gitlab/lassen-build-and-test-extra.yml @@ -102,7 +102,7 @@ xl_16_1_1_12_gcc_8_3_1_cuda_11_1_0: clang_14_0_5: variables: - SPEC: " +openmp %clang@14.0.5" + SPEC: " ~shared +openmp +tests %clang@14.0.5" extends: .build_and_test_on_lassen ########## @@ -111,18 +111,18 @@ clang_14_0_5: clang_12_0_1_cuda_11_5_0: variables: - SPEC: " +openmp +cuda cuda_arch=70 %clang@12.0.1 ^cuda@11.5.0" + SPEC: " ~shared +openmp +tests +cuda cuda_arch=70 %clang@12.0.1 ^cuda@11.5.0" extends: .build_and_test_on_lassen gcc_8_3_1_cuda_11_1_0: variables: - SPEC: " +openmp +cuda %gcc@8.3.1 cuda_arch=70 ^cuda@11.1.0" + SPEC: " ~shared +openmp +tests +cuda %gcc@8.3.1 cuda_arch=70 ^cuda@11.1.0" extends: .build_and_test_on_lassen gcc_8_3_1_cuda_11_5_0_ats_disabled: extends: .build_and_test_on_lassen variables: - SPEC: " +openmp +cuda %gcc@8.3.1 cuda_arch=70 ^cuda@11.5.0" + SPEC: " ~shared +openmp +tests +cuda %gcc@8.3.1 cuda_arch=70 ^cuda@11.5.0" LASSEN_BUILD_AND_TEST_JOB_ALLOC: "1 --atsdisable -W 60" ########## @@ -131,16 +131,16 @@ gcc_8_3_1_cuda_11_5_0_ats_disabled: clang_13_0_1_libcpp: variables: - SPEC: " +openmp %clang@13.0.1+libcpp" + SPEC: " ~shared +openmp +tests %clang@13.0.1+libcpp" extends: .build_and_test_on_lassen clang_14_0_5_asan: variables: - SPEC: " +openmp %clang@14.0.5 cxxflags=-fsanitize=address" + SPEC: " ~shared +openmp +tests %clang@14.0.5 cxxflags=-fsanitize=address" ASAN_OPTIONS: "detect_leaks=1" extends: .build_and_test_on_lassen gcc_8_3_1_cuda_10_1_168_desul_atomics: variables: - SPEC: " +openmp +cuda +desul %gcc@8.3.1 cuda_arch=70 ^cuda@10.1.168" + SPEC: " ~shared +openmp +tests +cuda +desul %gcc@8.3.1 cuda_arch=70 ^cuda@10.1.168" extends: .build_and_test_on_lassen diff --git a/.gitlab/ruby-build-and-test-extra.yml b/.gitlab/ruby-build-and-test-extra.yml index 9bebc62530..33faa56442 100644 --- a/.gitlab/ruby-build-and-test-extra.yml +++ b/.gitlab/ruby-build-and-test-extra.yml @@ -35,24 +35,24 @@ pgi_20_1_gcc_local_8_3_1: clang_9_0_0_openmp_off: variables: - SPEC: " ~openmp %clang@9.0.0" + SPEC: " ~shared ~openmp +tests %clang@9.0.0" extends: .build_and_test_on_ruby gcc_8_1_0_openmp_default: variables: - SPEC: " %gcc@8.1.0" + SPEC: " ~shared +tests %gcc@8.1.0" RUBY_BUILD_AND_TEST_JOB_ALLOC: "--time=60 --nodes=1" extends: .build_and_test_on_ruby icpc_19_1_0: variables: - SPEC: " +openmp %intel@19.1.0" + SPEC: " ~shared +openmp +tests %intel@19.1.0" RUBY_BUILD_AND_TEST_JOB_ALLOC: "--time=40 --nodes=1" extends: .build_and_test_on_ruby # OTHERS clang_10_0_1_gcc_8_3_1_desul_atomics: variables: - SPEC: " +openmp +desul %clang@10.0.1 cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1" + SPEC: " ~shared +openmp +tests +desul %clang@10.0.1 cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1" extends: .build_and_test_on_ruby diff --git a/.uberenv_config.json b/.uberenv_config.json index 2fc700f855..dc692a9045 100644 --- a/.uberenv_config.json +++ b/.uberenv_config.json @@ -1,7 +1,7 @@ { "package_name" : "raja", "package_version" : "develop", -"package_final_phase" : "hostconfig", +"package_final_phase" : "initconfig", "package_source_dir" : "../..", "spack_url": "https://github.com/spack/spack.git", "spack_branch": "v0.18.1", diff --git a/CMakeLists.txt b/CMakeLists.txt index b8ebfb77e1..ff6600cd13 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,7 +16,7 @@ include(CMakeDependentOption) # Set version number set(RAJA_VERSION_MAJOR 2022) set(RAJA_VERSION_MINOR 10) -set(RAJA_VERSION_PATCHLEVEL 2) +set(RAJA_VERSION_PATCHLEVEL 3) if (RAJA_LOADED AND (NOT RAJA_LOADED STREQUAL "${RAJA_VERSION_MAJOR}.${RAJA_VERSION_MINOR}.${RAJA_VERSION_PATCHLEVEL}")) message(FATAL_ERROR "You are mixing RAJA versions. Loaded is ${RAJA_LOADED}, expected ${RAJA_VERSION_MAJOR}.${RAJA_VERSION_MINOR}.${RAJA_VERSION_PATCHLEVEL}") diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 0f6b40cdf4..f34f191608 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -20,6 +20,37 @@ Notable changes include: * Bug fixes/improvements: +Version 2022.10.3 -- Release date 2022-12-01 +============================================ + +This release fixes a few issues that were found after the v2022.10.2 release. + +Notable changes include: + + * Update camp submodule to v2022.10.1 + * Update BLT submodule to commit 8c229991 (includes fixes for crayftn + hip) + + * Properly export 'roctx' target when CMake variable RAJA_ENABLE_ROCTX is on. + * Fix CMake logic for exporting desul targets when desul atomics are enabled. + * Fix the way we use CMake to find the rocPRIM module to follow CMake + best practices. + * Add missing template parameter pack argument in RAJA::statement::For + execution policy construct used in RAJA::kernel implementation for OpenMP + target back-end. + * Change to use compile-time GPU thread block size in RAJA::forall + implementation. This improves performance of GPU kernels, especially + those using the RAJA HIP back-end. + * Added RAJA plugin support, including CHAI support, for RAJA::launch. + * Replaced 'DEVICE' macro with alias to 'device_mem_pool_t' to prevent name + conflicts with other libraries. + * Updated User Guide documentation about CMake variable used to pass + compiler flags for OpenMP target back-end. This changed with CMake + minimum required version bump in v2022.10.0. + * Adjust ordering of BLT and camp target inclusion in RAJA CMake usage to + fix an issue with projects using external camp vs. RAJA submodule. + + + Version 2022.10.2 -- Release date 2022-11-08 ============================================ diff --git a/blt b/blt index 97ea54d892..8c229991e6 160000 --- a/blt +++ b/blt @@ -1 +1 @@ -Subproject commit 97ea54d892b4b1d56736830575c3db62e3d7674d +Subproject commit 8c229991e65e7a9603c621b47cb3ba158bb7468c diff --git a/cmake/SetupPackages.cmake b/cmake/SetupPackages.cmake index 45dadbee4b..dc107e1fe3 100644 --- a/cmake/SetupPackages.cmake +++ b/cmake/SetupPackages.cmake @@ -82,11 +82,11 @@ if (RAJA_ENABLE_HIP) endif() if (RAJA_ENABLE_EXTERNAL_ROCPRIM) - include(cmake/thirdparty/FindRocPRIM.cmake) - if (ROCPRIM_FOUND) + find_package(rocPRIM) + if (rocPRIM_FOUND) blt_import_library( NAME rocPRIM - INCLUDES ${ROCPRIM_INCLUDE_DIRS} + INCLUDES ${rocPRIM_INCLUDE_DIRS} TREAT_INCLUDES_AS_SYSTEM ON EXPORTABLE ON) else() @@ -101,13 +101,16 @@ if (RAJA_ENABLE_HIP AND RAJA_ENABLE_ROCTX) include(FindRoctracer) blt_import_library(NAME roctx INCLUDES ${ROCTX_INCLUDE_DIRS} - LIBRARIES ${ROCTX_LIBRARIES}) + LIBRARIES ${ROCTX_LIBRARIES} + EXPORTABLE ON + TREAT_INCLUDES_AS_SYSTEM ON) endif () set(TPL_DEPS) blt_list_append(TO TPL_DEPS ELEMENTS nvtoolsext IF RAJA_ENABLE_NV_TOOLS_EXT) blt_list_append(TO TPL_DEPS ELEMENTS cub IF RAJA_ENABLE_EXTERNAL_CUB) blt_list_append(TO TPL_DEPS ELEMENTS rocPRIM IF RAJA_ENABLE_EXTERNAL_ROCPRIM) +blt_list_append(TO TPL_DEPS ELEMENTS roctx IF RAJA_ENABLE_ROCTX) set(RAJA_NEEDS_BLT_TPLS False) if (RAJA_ENABLE_CUDA OR RAJA_ENABLE_HIP OR RAJA_ENABLE_OPENMP OR RAJA_ENABLE_MPI) diff --git a/cmake/thirdparty/FindRocPRIM.cmake b/cmake/thirdparty/FindRocPRIM.cmake deleted file mode 100644 index 4279f48d1b..0000000000 --- a/cmake/thirdparty/FindRocPRIM.cmake +++ /dev/null @@ -1,29 +0,0 @@ -############################################################################### -# Copyright (c) 2016-22, Lawrence Livermore National Security, LLC -# and other RAJA project contributors. See the RAJA/LICENSE file for details. -# -# SPDX-License-Identifier: (BSD-3-Clause) -############################################################################### - -include (FindPackageHandleStandardArgs) - -find_path(ROCPRIM_INCLUDE_DIRS - NAMES rocprim/rocprim.hpp - HINTS - ${ROCPRIM_DIR}/ - ${ROCPRIM_DIR}/include - ${ROCPRIM_DIR}/rocprim/include - ${HIP_ROOT_DIR}/../rocprim - ${HIP_ROOT_DIR}/../rocprim/include - ${HIP_ROOT_DIR}/../include) - -find_package_handle_standard_args( - ROCPRIM - DEFAULT_MSG - ROCPRIM_INCLUDE_DIRS) - -if (ROCPRIM_INCLUDE_DIRS) - set(ROCPRIM_FOUND True) -else () - set(ROCPRIM_FOUND False) -endif() diff --git a/cmake/thirdparty/FindrocPRIM.cmake b/cmake/thirdparty/FindrocPRIM.cmake new file mode 100644 index 0000000000..702ef52013 --- /dev/null +++ b/cmake/thirdparty/FindrocPRIM.cmake @@ -0,0 +1,54 @@ +############################################################################### +# Copyright (c) 2016-22, Lawrence Livermore National Security, LLC +# and other RAJA project contributors. See the RAJA/LICENSE file for details. +# +# SPDX-License-Identifier: (BSD-3-Clause) +############################################################################### + +#[=======================================================================[.rst: + +FindrocPRIM +------- + +Finds the rocPRIM package. + +Result Variables +^^^^^^^^^^^^^^^^ + +This will define the following variables: + +``rocPRIM_FOUND`` +True if the system has the rocPRIM library. +``rocPRIM_INCLUDE_DIRS`` +Include directories needed to use rocPRIM. + +Cache Variables +^^^^^^^^^^^^^^^ + +The following cache variables may also be set: + +``rocPRIM_INCLUDE_DIR`` +The directory containing ``rocprim.hpp``. + +#]=======================================================================] + +include (FindPackageHandleStandardArgs) + +find_path(rocPRIM_INCLUDE_DIR + NAMES rocprim/rocprim.hpp + HINTS + ${ROCPRIM_DIR}/ + ${HIP_ROOT_DIR}/../ + PATH_SUFFIXES + include + rocprim + rocprim/include) + +find_package_handle_standard_args( + rocPRIM + DEFAULT_MSG + rocPRIM_INCLUDE_DIR) + +if (rocPRIM_FOUND) + set(rocPRIM_INCLUDE_DIRS ${rocPRIM_INCLUDE_DIR}) +endif() diff --git a/docs/conf.py b/docs/conf.py index 20c5ef60e0..51e0336b1a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -88,7 +88,7 @@ # The short X.Y version. version = u'2022.10' # The full version, including alpha/beta/rc tags. -release = u'2022.10.2' +release = u'2022.10.3' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/sphinx/user_guide/getting_started.rst b/docs/sphinx/user_guide/getting_started.rst index e4483d25c3..1c525d2dfa 100644 --- a/docs/sphinx/user_guide/getting_started.rst +++ b/docs/sphinx/user_guide/getting_started.rst @@ -323,16 +323,19 @@ OpenMP ^^^^^^^ To use OpenMP target offload GPU execution, additional options may need to be -passed to the compiler. The variable ``OpenMP_CXX_FLAGS`` is used for this. -Option syntax follows the CMake *list* pattern. For example, to specify OpenMP -target options for NVIDIA GPUs using a clang-based compiler, one may do -something like:: +passed to the compiler. BLT variables are used for this. Option syntax follows +the CMake *list* pattern. For example, to specify OpenMP target options for +NVIDIA GPUs using a clang-based compiler, one may do something like:: cmake \ ... \ - -DOpenMP_CXX_FLAGS="-fopenmp;-fopenmp-targets=nvptx64-nvidia-cuda" \ + -DBLT_OPENMP_COMPILE_FLAGS="-fopenmp;-fopenmp-targets=nvptx64-nvidia-cuda" \ + -DBLT_OPENMP_LINK_FLAGS="-fopenmp;-fopenmp-targets=nvptx64-nvidia-cuda" \ ... +Compiler flags are passed to other compilers similarly, using flags specific to +the compiler. Typically, the compile and link flags are the same as shown here. + ---------------------------------------- RAJA Example Build Configuration Files ---------------------------------------- diff --git a/host-configs/lc-builds/toss4/gcc_X.cmake b/host-configs/lc-builds/toss4/gcc_X.cmake new file mode 100755 index 0000000000..017fabca22 --- /dev/null +++ b/host-configs/lc-builds/toss4/gcc_X.cmake @@ -0,0 +1,16 @@ +############################################################################### +# Copyright (c) 2016-22, Lawrence Livermore National Security, LLC +# and RAJA project contributors. See the RAJA/LICENSE file for details. +# +# SPDX-License-Identifier: (BSD-3-Clause) +############################################################################### + +set(RAJA_COMPILER "RAJA_COMPILER_GNU" CACHE STRING "") + +set(CMAKE_CXX_FLAGS_RELEASE "-Ofast -march=native" CACHE STRING "") +set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-Ofast -march=native -g" CACHE STRING "") +set(CMAKE_CXX_FLAGS_DEBUG "-O0 -g" CACHE STRING "") + +set(RAJA_DATA_ALIGN 64 CACHE STRING "") + +set(RAJA_HOST_CONFIG_LOADED On CACHE BOOL "") diff --git a/host-configs/lc-builds/toss4/icpc_X.cmake b/host-configs/lc-builds/toss4/icpc_X.cmake new file mode 100755 index 0000000000..9810b093f6 --- /dev/null +++ b/host-configs/lc-builds/toss4/icpc_X.cmake @@ -0,0 +1,16 @@ +############################################################################### +# Copyright (c) 2016-22, Lawrence Livermore National Security, LLC +# and RAJA project contributors. See the RAJA/LICENSE file for details. +# +# SPDX-License-Identifier: (BSD-3-Clause) +############################################################################### + +set(RAJA_COMPILER "RAJA_COMPILER_ICC" CACHE STRING "") + +set(CMAKE_CXX_FLAGS_RELEASE "-O3 -march=native -ansi-alias -diag-disable cpu-dispatch" CACHE STRING "") +set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O3 -g -march=native -ansi-alias -diag-disable cpu-dispatch" CACHE STRING "") +set(CMAKE_CXX_FLAGS_DEBUG "-O0 -g" CACHE STRING "") + +set(RAJA_DATA_ALIGN 64 CACHE STRING "") + +set(RAJA_HOST_CONFIG_LOADED On CACHE BOOL "") diff --git a/host-configs/lc-builds/toss4/icpx_X.cmake b/host-configs/lc-builds/toss4/icpx_X.cmake new file mode 100755 index 0000000000..3b1e6cd194 --- /dev/null +++ b/host-configs/lc-builds/toss4/icpx_X.cmake @@ -0,0 +1,16 @@ +############################################################################### +# Copyright (c) 2016-22, Lawrence Livermore National Security, LLC +# and RAJA project contributors. See the RAJA/LICENSE file for details. +# +# SPDX-License-Identifier: (BSD-3-Clause) +############################################################################### + +set(RAJA_COMPILER "RAJA_COMPILER_ICC" CACHE STRING "") + +set(CMAKE_CXX_FLAGS_RELEASE "-O3 -march=native" CACHE STRING "") +set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O3 -g -march=native" CACHE STRING "") +set(CMAKE_CXX_FLAGS_DEBUG "-O0 -g" CACHE STRING "") + +set(RAJA_DATA_ALIGN 64 CACHE STRING "") + +set(RAJA_HOST_CONFIG_LOADED On CACHE BOOL "") diff --git a/include/RAJA/pattern/launch/launch_core.hpp b/include/RAJA/pattern/launch/launch_core.hpp index 12d6f16f6f..28dca62ff0 100644 --- a/include/RAJA/pattern/launch/launch_core.hpp +++ b/include/RAJA/pattern/launch/launch_core.hpp @@ -227,8 +227,20 @@ void launch(LaunchParams const ¶ms, const char *kernel_name, BODY const &bod { //Take the first policy as we assume the second policy is not user defined. //We rely on the user to pair launch and loop policies correctly. + util::PluginContext context{util::make_context()}; + util::callPreCapturePlugins(context); + + using RAJA::util::trigger_updates_before; + auto p_body = trigger_updates_before(body); + + util::callPostCapturePlugins(context); + + util::callPreLaunchPlugins(context); + using launch_t = LaunchExecute; - launch_t::exec(params, kernel_name, body); + launch_t::exec(params, kernel_name, p_body); + + util::callPostLaunchPlugins(context); } @@ -242,22 +254,23 @@ void launch(ExecPlace place, LaunchParams const ¶ms, BODY const &body) template void launch(ExecPlace place, const LaunchParams ¶ms, const char *kernel_name, BODY const &body) { + + //Forward to single policy launch API - simplifies testing of plugins switch (place) { case ExecPlace::HOST: { - using launch_t = LaunchExecute; - launch_t::exec(params, kernel_name, body); + launch>(params, kernel_name, body); break; } #ifdef RAJA_DEVICE_ACTIVE case ExecPlace::DEVICE: { - using launch_t = LaunchExecute; - launch_t::exec(params, kernel_name, body); + launch>(params, kernel_name, body); break; } #endif default: RAJA_ABORT_OR_THROW("Unknown launch place or device is not enabled"); } + } // Helper function to retrieve a resource based on the run-time policy - if a device is active @@ -291,28 +304,55 @@ launch(RAJA::resources::Resource res, LaunchParams const ¶ms, const char *ke { ExecPlace place; - if(res.get_platform() == camp::resources::v1::Platform::host) { + if(res.get_platform() == RAJA::Platform::host) { place = RAJA::ExecPlace::HOST; }else{ place = RAJA::ExecPlace::DEVICE; } + // + //Configure plugins + // +#ifdef RAJA_DEVICE_ACTIVE + util::PluginContext context{place == ExecPlace::HOST ? + util::make_context() + : util::make_context()}; +#else + util::PluginContext context{util::make_context()}; +#endif + + util::callPreCapturePlugins(context); + + using RAJA::util::trigger_updates_before; + auto p_body = trigger_updates_before(body); + + util::callPostCapturePlugins(context); + + util::callPreLaunchPlugins(context); + switch (place) { case ExecPlace::HOST: { using launch_t = LaunchExecute; - return launch_t::exec(res, params, kernel_name, body); break; + resources::EventProxy e_proxy = launch_t::exec(res, params, kernel_name, p_body); + util::callPostLaunchPlugins(context); + return e_proxy; } #ifdef RAJA_DEVICE_ACTIVE case ExecPlace::DEVICE: { using launch_t = LaunchExecute; - return launch_t::exec(res, params, kernel_name, body); break; + resources::EventProxy e_proxy = launch_t::exec(res, params, kernel_name, p_body); + util::callPostLaunchPlugins(context); + return e_proxy; } #endif default: { RAJA_ABORT_OR_THROW("Unknown launch place or device is not enabled"); } } - //Should not get here; + + RAJA_ABORT_OR_THROW("Unknown launch place"); + + //^^ RAJA will abort before getting here return resources::EventProxy(res); } diff --git a/include/RAJA/pattern/params/reducer.hpp b/include/RAJA/pattern/params/reducer.hpp index 5c4858a14a..d094a729c1 100644 --- a/include/RAJA/pattern/params/reducer.hpp +++ b/include/RAJA/pattern/params/reducer.hpp @@ -5,10 +5,8 @@ #include "RAJA/util/SoAPtr.hpp" #if defined(RAJA_CUDA_ACTIVE) -#define DEVICE cuda #include "RAJA/policy/cuda/MemUtils_CUDA.hpp" #elif defined(RAJA_HIP_ACTIVE) -#define DEVICE hip #include "RAJA/policy/hip/MemUtils_HIP.hpp" #endif @@ -72,6 +70,12 @@ namespace expt namespace detail { +#if defined(RAJA_CUDA_ACTIVE) + using device_mem_pool_t = RAJA::cuda::device_mempool_type; +#elif defined(RAJA_HIP_ACTIVE) + using device_mem_pool_t = RAJA::hip::device_mempool_type; +#endif + // // // Basic Reducer @@ -91,11 +95,11 @@ namespace detail #if defined(RAJA_CUDA_ACTIVE) || defined(RAJA_HIP_ACTIVE) // Device related attributes. value_type * devicetarget = nullptr; - RAJA::detail::SoAPtr device_mem; + RAJA::detail::SoAPtr device_mem; unsigned int * device_count = nullptr; #endif - using ARG_TUP_T = camp::tuple; + using ARG_TUP_T = camp::tuple; RAJA_HOST_DEVICE ARG_TUP_T get_lambda_arg_tup() { return camp::make_tuple(&val); } using ARG_LIST_T = typename ARG_TUP_T::TList; diff --git a/include/RAJA/policy/cuda/forall.hpp b/include/RAJA/policy/cuda/forall.hpp index bc0936c353..ce278c4ee2 100644 --- a/include/RAJA/policy/cuda/forall.hpp +++ b/include/RAJA/policy/cuda/forall.hpp @@ -81,15 +81,17 @@ cuda_dim_t getGridDim(cuda_dim_member_t len, cuda_dim_t blockDim) * ****************************************************************************** */ +template __device__ __forceinline__ unsigned int getGlobalIdx_1D_1D() { unsigned int blockId = blockIdx.x; - unsigned int threadId = blockId * blockDim.x + threadIdx.x; + unsigned int threadId = blockId * BlockSize + threadIdx.x; return threadId; } +template __device__ __forceinline__ unsigned int getGlobalNumThreads_1D_1D() { - unsigned int numThreads = blockDim.x * gridDim.x; + unsigned int numThreads = BlockSize * gridDim.x; return numThreads; } @@ -144,7 +146,7 @@ __launch_bounds__(BlockSize, BlocksPerSM) __global__ using RAJA::internal::thread_privatize; auto privatizer = thread_privatize(loop_body); auto& body = privatizer.get_priv(); - auto ii = static_cast(getGlobalIdx_1D_1D()); + auto ii = static_cast(getGlobalIdx_1D_1D()); if (ii < length) { body(idx[ii]); } @@ -167,7 +169,7 @@ __launch_bounds__(BlockSize, 1) __global__ using RAJA::internal::thread_privatize; auto privatizer = thread_privatize(loop_body); auto& body = privatizer.get_priv(); - auto ii = static_cast(getGlobalIdx_1D_1D()); + auto ii = static_cast(getGlobalIdx_1D_1D()); if ( ii < length ) { RAJA::expt::invoke_body( f_params, body, idx[ii] ); diff --git a/include/RAJA/policy/hip/forall.hpp b/include/RAJA/policy/hip/forall.hpp index ab3109b354..1aa447633e 100644 --- a/include/RAJA/policy/hip/forall.hpp +++ b/include/RAJA/policy/hip/forall.hpp @@ -77,15 +77,17 @@ hip_dim_t getGridDim(hip_dim_member_t len, hip_dim_t blockDim) * ****************************************************************************** */ +template __device__ __forceinline__ unsigned int getGlobalIdx_1D_1D() { unsigned int blockId = blockIdx.x; - unsigned int threadId = blockId * blockDim.x + threadIdx.x; + unsigned int threadId = blockId * BlockSize + threadIdx.x; return threadId; } +template __device__ __forceinline__ unsigned int getGlobalNumThreads_1D_1D() { - unsigned int numThreads = blockDim.x * gridDim.x; + unsigned int numThreads = BlockSize * gridDim.x; return numThreads; } @@ -139,7 +141,7 @@ __launch_bounds__(BlockSize, 1) __global__ using RAJA::internal::thread_privatize; auto privatizer = thread_privatize(loop_body); auto& body = privatizer.get_priv(); - auto ii = static_cast(getGlobalIdx_1D_1D()); + auto ii = static_cast(getGlobalIdx_1D_1D()); if (ii < length) { body(idx[ii]); } @@ -161,7 +163,7 @@ __launch_bounds__(BlockSize, 1) __global__ using RAJA::internal::thread_privatize; auto privatizer = thread_privatize(loop_body); auto& body = privatizer.get_priv(); - auto ii = static_cast(getGlobalIdx_1D_1D()); + auto ii = static_cast(getGlobalIdx_1D_1D()); if ( ii < length ) { RAJA::expt::invoke_body( f_params, body, idx[ii] ); diff --git a/include/RAJA/policy/openmp_target/kernel/For.hpp b/include/RAJA/policy/openmp_target/kernel/For.hpp index c5d60d01a9..08f64e227a 100644 --- a/include/RAJA/policy/openmp_target/kernel/For.hpp +++ b/include/RAJA/policy/openmp_target/kernel/For.hpp @@ -57,7 +57,7 @@ struct StatementExecutor{}, TypedRangeSegment(0, len), for_wrapper); + forall_impl(r, omp_target_parallel_for_exec{}, TypedRangeSegment(0, len), for_wrapper, RAJA::expt::get_empty_forall_param_pack()); } }; diff --git a/include/RAJA/policy/tbb/forall.hpp b/include/RAJA/policy/tbb/forall.hpp index de811a4251..3f8bd966db 100644 --- a/include/RAJA/policy/tbb/forall.hpp +++ b/include/RAJA/policy/tbb/forall.hpp @@ -99,9 +99,9 @@ forall_impl(resources::Host host_res, [=](const brange& r, ForallParam fp) { using RAJA::internal::thread_privatize; auto privatizer = thread_privatize(loop_body); - auto body = privatizer.get_priv(); + auto& body = privatizer.get_priv(); for (auto i = r.begin(); i != r.end(); ++i) - expt::invoke_body(fp, loop_body, b[i]); + expt::invoke_body(fp, body, b[i]); return fp; }, @@ -138,7 +138,7 @@ forall_impl(resources::Host host_res, ::tbb::parallel_for(brange(0, dist, p.grain_size), [=](const brange& r) { using RAJA::internal::thread_privatize; auto privatizer = thread_privatize(loop_body); - auto body = privatizer.get_priv(); + auto& body = privatizer.get_priv(); for (auto i = r.begin(); i != r.end(); ++i) body(b[i]); }); @@ -196,9 +196,9 @@ forall_impl(resources::Host host_res, [=](const brange& r, ForallParam fp) { using RAJA::internal::thread_privatize; auto privatizer = thread_privatize(loop_body); - auto body = privatizer.get_priv(); + auto& body = privatizer.get_priv(); for (auto i = r.begin(); i != r.end(); ++i) - expt::invoke_body(fp, loop_body, b[i]); + expt::invoke_body(fp, body, b[i]); return fp; }, @@ -240,7 +240,7 @@ forall_impl(resources::Host host_res, [=](const brange& r) { using RAJA::internal::thread_privatize; auto privatizer = thread_privatize(loop_body); - auto body = privatizer.get_priv(); + auto& body = privatizer.get_priv(); for (auto i = r.begin(); i != r.end(); ++i) body(b[i]); }, diff --git a/scripts/gitlab/build_and_test.sh b/scripts/gitlab/build_and_test.sh index 4e96cfcf45..2c7041fefb 100755 --- a/scripts/gitlab/build_and_test.sh +++ b/scripts/gitlab/build_and_test.sh @@ -127,8 +127,7 @@ fi build_dir="${build_root}/build_${hostconfig//.cmake/}" install_dir="${build_root}/install_${hostconfig//.cmake/}" -# TODO: This is from Umpire, could it work with RAJA ? -#cmake_exe=`grep 'CMake executable' ${hostconfig_path} | cut -d ':' -f 2 | xargs` +cmake_exe=`grep 'CMake executable' ${hostconfig_path} | cut -d ':' -f 2 | xargs` # Build if [[ "${option}" != "--deps-only" && "${option}" != "--test-only" ]] @@ -161,17 +160,14 @@ then then module unload rocm fi - - module load cmake/3.23.1 || module load cmake/3.20.2 - - cmake \ + $cmake_exe \ -C ${hostconfig_path} \ -DCMAKE_INSTALL_PREFIX=${install_dir} \ ${project_dir} - if ! cmake --build . -j ${core_counts[$truehostname]} + if ! $cmake_exe --build . -j ${core_counts[$truehostname]} then echo "ERROR: compilation failed, building with verbose output..." - cmake --build . --verbose -j 1 + $cmake_exe --build . --verbose -j 1 else make install fi @@ -225,9 +221,9 @@ then echo "ERROR: failure(s) while running CTest" && exit 1 fi - if grep -q -i "ENABLE_HIP.*ON" ${hostconfig_path} || grep -q -i "RAJA_ENABLE_DESUL_ATOMICS.*ON" ${hostconfig_path} + if grep -q -i "ENABLE_HIP.*ON" ${hostconfig_path} then - echo "WARNING: not testing install with HIP or desul" + echo "WARNING: not testing install with HIP" else if [[ ! -d ${install_dir} ]] then @@ -236,8 +232,8 @@ then cd ${install_dir}/examples/RAJA/using-with-cmake mkdir build && cd build - if ! cmake -C ../host-config.cmake ..; then - echo "ERROR: running cmake for using-with-cmake test" && exit 1 + if ! $cmake_exe -C ../host-config.cmake ..; then + echo "ERROR: running $cmake_exe for using-with-cmake test" && exit 1 fi if ! make; then diff --git a/scripts/lc-builds/blueos_clang_omptarget.sh b/scripts/lc-builds/blueos_clang_omptarget.sh index e0aa7104a7..67a477288b 100755 --- a/scripts/lc-builds/blueos_clang_omptarget.sh +++ b/scripts/lc-builds/blueos_clang_omptarget.sh @@ -40,7 +40,8 @@ cmake \ -DENABLE_OPENMP=On \ -DENABLE_CUDA=Off \ -DRAJA_ENABLE_TARGET_OPENMP=On \ - -DOpenMP_CXX_FLAGS="-fopenmp;-fopenmp-targets=nvptx64-nvidia-cuda" \ + -DBLT_OPENMP_COMPILE_FLAGS="-fopenmp;-fopenmp-targets=nvptx64-nvidia-cuda" \ + -DBLT_OPENMP_LINK_FLAGS="-fopenmp;-fopenmp-targets=nvptx64-nvidia-cuda" \ -DCMAKE_INSTALL_PREFIX=../install_${BUILD_SUFFIX} \ "$@" \ .. diff --git a/scripts/lc-builds/blueos_xl_omptarget.sh b/scripts/lc-builds/blueos_xl_omptarget.sh index 2b3fadcbb0..6eb7e162b2 100755 --- a/scripts/lc-builds/blueos_xl_omptarget.sh +++ b/scripts/lc-builds/blueos_xl_omptarget.sh @@ -37,7 +37,8 @@ cmake \ -C ../host-configs/lc-builds/blueos/xl_X.cmake \ -DENABLE_OPENMP=On \ -DRAJA_ENABLE_TARGET_OPENMP=On \ - -DOpenMP_CXX_FLAGS="-qoffload;-qsmp=omp;-qalias=noansi" \ + -DBLT_OPENMP_COMPILE_FLAGS="-qoffload;-qsmp=omp;-qalias=noansi" \ + -DBLT_OPENMP_LINK_FLAGS="-qoffload;-qsmp=omp;-qalias=noansi" \ -DCMAKE_INSTALL_PREFIX=../install_${BUILD_SUFFIX} \ "$@" \ .. diff --git a/scripts/lc-builds/toss3_icpc.sh b/scripts/lc-builds/toss3_icpc.sh index 5f2474bb44..7e0487dacd 100755 --- a/scripts/lc-builds/toss3_icpc.sh +++ b/scripts/lc-builds/toss3_icpc.sh @@ -50,7 +50,6 @@ module load cmake/3.20.2 ## cmake \ - -DBLT_CXX_STD=c++14 \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_CXX_COMPILER=/usr/tce/packages/intel/intel-${COMP_VER}/bin/icpc \ -DCMAKE_C_COMPILER=/usr/tce/packages/intel/intel-${COMP_VER}/bin/icc \ diff --git a/scripts/lc-builds/toss4_amdclang.sh b/scripts/lc-builds/toss4_amdclang.sh index 615060490e..4aaa36b207 100755 --- a/scripts/lc-builds/toss4_amdclang.sh +++ b/scripts/lc-builds/toss4_amdclang.sh @@ -43,6 +43,9 @@ echo "Creating build directory build_${BUILD_SUFFIX} and generating configuratio echo "Configuration extra arguments:" echo " $@" echo +echo "To use fp64 HW atomics you must configure with these options when using gfx90a and hip >= 5.2" +echo " -DCMAKE_CXX_FLAGS=\"-munsafe-fp-atomics\"" +echo rm -rf build_${BUILD_SUFFIX} >/dev/null mkdir build_${BUILD_SUFFIX} && cd build_${BUILD_SUFFIX} diff --git a/scripts/lc-builds/toss4_amdclang_asan.sh b/scripts/lc-builds/toss4_amdclang_asan.sh index a886d1c0b6..02925093ff 100755 --- a/scripts/lc-builds/toss4_amdclang_asan.sh +++ b/scripts/lc-builds/toss4_amdclang_asan.sh @@ -43,6 +43,9 @@ echo "Creating build directory ${BUILD_SUFFIX} and generating configuration in i echo "Configuration extra arguments:" echo " $@" echo +echo "To use fp64 HW atomics you must configure with these options when using gfx90a and hip >= 5.2" +echo " -DCMAKE_CXX_FLAGS=\"-munsafe-fp-atomics\"" +echo rm -rf build_${BUILD_SUFFIX} >/dev/null mkdir build_${BUILD_SUFFIX} && cd build_${BUILD_SUFFIX} diff --git a/scripts/lc-builds/toss4_cce_hip.sh b/scripts/lc-builds/toss4_cce_hip.sh index 187e797a5f..fc325e0948 100755 --- a/scripts/lc-builds/toss4_cce_hip.sh +++ b/scripts/lc-builds/toss4_cce_hip.sh @@ -34,6 +34,9 @@ echo "Creating build directory build_${BUILD_SUFFIX} and generating configuratio echo "Configuration extra arguments:" echo " $@" echo +echo "To use fp64 HW atomics you must configure with these options when using gfx90a and hip >= 5.2" +echo " -DCMAKE_CXX_FLAGS=\"-munsafe-fp-atomics\"" +echo rm -rf build_${BUILD_SUFFIX} >/dev/null mkdir build_${BUILD_SUFFIX} && cd build_${BUILD_SUFFIX} @@ -41,12 +44,10 @@ mkdir build_${BUILD_SUFFIX} && cd build_${BUILD_SUFFIX} module load cmake/3.24.2 -module load cce/${COMP_VER} - cmake \ -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_C_COMPILER=$(which cc) \ - -DCMAKE_CXX_COMPILER=$(which CC) \ + -DCMAKE_C_COMPILER="/usr/tce/packages/cce-tce/cce-${COMP_VER}/bin/craycc" \ + -DCMAKE_CXX_COMPILER="/usr/tce/packages/cce-tce/cce-${COMP_VER}/bin/crayCC" \ -DHIP_PATH=/opt/rocm-${HIP_VER}/hip \ -DCMAKE_HIP_ARCHITECTURES=${HIP_ARCH} \ -DGPU_TARGETS=${HIP_ARCH} \ @@ -69,7 +70,7 @@ echo " Please note that you have to have a consistent build environment" echo " when you make RAJA as cmake may reconfigure; load the appropriate" echo " cce module (${COMP_VER}) when building." echo -echo " module load cce/${COMP_VER}" +echo " module load cce-tce/${COMP_VER}" echo " srun -n1 make" echo echo "***********************************************************************" diff --git a/scripts/lc-builds/toss4_gcc.sh b/scripts/lc-builds/toss4_gcc.sh new file mode 100755 index 0000000000..532d2ff130 --- /dev/null +++ b/scripts/lc-builds/toss4_gcc.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +############################################################################### +# Copyright (c) 2016-22, Lawrence Livermore National Security, LLC +# and RAJA project contributors. See the RAJA/LICENSE file for details. +# +# SPDX-License-Identifier: (BSD-3-Clause) +############################################################################### + +if [ "$1" == "" ]; then + echo + echo "You must pass a compiler version number to script. For example," + echo " toss4_gcc.sh 10.3.1" + exit +fi + +COMP_VER=$1 +shift 1 + +BUILD_SUFFIX=lc_toss4-gcc-${COMP_VER} + +echo +echo "Creating build directory build_${BUILD_SUFFIX} and generating configuration in it" +echo "Configuration extra arguments:" +echo " $@" +echo + +rm -rf build_${BUILD_SUFFIX} 2>/dev/null +mkdir build_${BUILD_SUFFIX} && cd build_${BUILD_SUFFIX} + +module load cmake/3.21.1 + +cmake \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_CXX_COMPILER=/usr/tce/packages/gcc/gcc-${COMP_VER}/bin/g++ \ + -DBLT_CXX_STD=c++14 \ + -C ../host-configs/lc-builds/toss4/gcc_X.cmake \ + -DENABLE_OPENMP=On \ + -DCMAKE_INSTALL_PREFIX=../install_${BUILD_SUFFIX} \ + "$@" \ + .. diff --git a/scripts/lc-builds/toss4_hipcc.sh b/scripts/lc-builds/toss4_hipcc.sh new file mode 100755 index 0000000000..0e3d8bdd6d --- /dev/null +++ b/scripts/lc-builds/toss4_hipcc.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash + +############################################################################### +# Copyright (c) 2016-22, Lawrence Livermore National Security, LLC +# and RAJA project contributors. See the RAJA/LICENSE file for details. +# +# SPDX-License-Identifier: (BSD-3-Clause) +############################################################################### + +if [[ $# -lt 2 ]]; then + echo + echo "You must pass 2 or more arguments to the script (in this order): " + echo " 1) compiler version number" + echo " 2) HIP compute architecture" + echo " 3...) optional arguments to cmake" + echo + echo "For example: " + echo " toss4_hipcc.sh 4.1.0 gfx906" + exit +fi + +COMP_VER=$1 +COMP_ARCH=$2 +shift 2 + +HOSTCONFIG="hip_3_X" + +if [[ ${COMP_VER} == 4.* ]] +then +##HIP_CLANG_FLAGS="-mllvm -amdgpu-fixed-function-abi=1" + HOSTCONFIG="hip_4_link_X" +elif [[ ${COMP_VER} == 3.* ]] +then + HOSTCONFIG="hip_3_X" +else + echo "Unknown hip version, using ${HOSTCONFIG} host-config" +fi + +BUILD_SUFFIX=lc_toss4-hipcc-${COMP_VER}-${COMP_ARCH} + +echo +echo "Creating build directory build_${BUILD_SUFFIX} and generating configuration in it" +echo "Configuration extra arguments:" +echo " $@" +echo +echo "To use fp64 HW atomics you must configure with these options when using gfx90a and hip >= 5.2" +echo " -DCMAKE_CXX_FLAGS=\"-munsafe-fp-atomics\"" +echo + +rm -rf build_${BUILD_SUFFIX} >/dev/null +mkdir build_${BUILD_SUFFIX} && cd build_${BUILD_SUFFIX} + + +module load cmake/3.23.1 + +# unload rocm to avoid configuration problems where the loaded rocm and COMP_VER +# are inconsistent causing the rocprim from the module to be used unexpectedly +module unload rocm + + +cmake \ + -DCMAKE_BUILD_TYPE=Release \ + -DROCM_ROOT_DIR="/opt/rocm-${COMP_VER}" \ + -DHIP_ROOT_DIR="/opt/rocm-${COMP_VER}/hip" \ + -DHIP_PATH=/opt/rocm-${COMP_VER}/bin \ + -DCMAKE_C_COMPILER=/opt/rocm-${COMP_VER}/bin/hipcc \ + -DCMAKE_CXX_COMPILER=/opt/rocm-${COMP_VER}/bin/hipcc \ + -DCMAKE_HIP_ARCHITECTURES="${COMP_ARCH}" \ + -DGPU_TARGETS="${COMP_ARCH}" \ + -DAMDGPU_TARGETS="${COMP_ARCH}" \ + -DBLT_CXX_STD=c++14 \ + -C "../host-configs/lc-builds/toss4/${HOSTCONFIG}.cmake" \ + -DENABLE_HIP=ON \ + -DENABLE_OPENMP=ON \ + -DENABLE_CUDA=OFF \ + -DCMAKE_INSTALL_PREFIX=../install_${BUILD_SUFFIX} \ + "$@" \ + .. + +echo +echo "***********************************************************************" +echo +echo "cd into directory build_${BUILD_SUFFIX} and run make to build RAJA" +echo +echo " Please note that you have to have a consistent build environment" +echo " when you make RAJA as cmake may reconfigure; unload the rocm module" +echo " or load the appropriate rocm module (${COMP_VER}) when building." +echo +echo " module unload rocm" +echo " srun -n1 make" +echo +echo "***********************************************************************" diff --git a/scripts/lc-builds/toss4_icpc.sh b/scripts/lc-builds/toss4_icpc.sh new file mode 100755 index 0000000000..4c46e80f9c --- /dev/null +++ b/scripts/lc-builds/toss4_icpc.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash + +############################################################################### +# Copyright (c) 2016-22, Lawrence Livermore National Security, LLC +# and RAJA project contributors. See the RAJA/LICENSE file for details. +# +# SPDX-License-Identifier: (BSD-3-Clause) +############################################################################### + +if [ "$1" == "" ]; then + echo + echo "You must pass a compiler version number to script. For example," + echo " toss4_icpc.sh 2021.6.0" + exit +fi + +COMP_VER=$1 +shift 1 + +USE_TBB=On + +BUILD_SUFFIX=lc_toss4-icpc-${COMP_VER} + +echo +echo "Creating build directory build_${BUILD_SUFFIX} and generating configuration in it" +echo "Configuration extra arguments:" +echo " $@" +echo + +rm -rf build_${BUILD_SUFFIX} 2>/dev/null +mkdir build_${BUILD_SUFFIX} && cd build_${BUILD_SUFFIX} + +module load cmake/3.21.1 + +## +# CMake option -DRAJA_ENABLE_FORCEINLINE_RECURSIVE=Off used to speed up compile +# times at a potential cost of slower 'forall' execution. +## + +cmake \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_CXX_COMPILER=/usr/tce/packages/intel-classic-tce/intel-classic-${COMP_VER}/bin/icpc \ + -DCMAKE_C_COMPILER=/usr/tce/packages/intel-classic-tce/intel-classic-${COMP_VER}/bin/icc \ + -DBLT_CXX_STD=c++14 \ + -C ../host-configs/lc-builds/toss4/icpc_X.cmake \ + -DRAJA_ENABLE_FORCEINLINE_RECURSIVE=Off \ + -DENABLE_OPENMP=On \ + -DRAJA_ENABLE_TBB=${USE_TBB} \ + -DCMAKE_INSTALL_PREFIX=../install_${BUILD_SUFFIX} \ + "$@" \ + .. + +echo +echo "***********************************************************************" +echo +echo "cd into directory build_${BUILD_SUFFIX} and run make to build RAJA" +echo +echo " Please note that you may need to add some intel openmp libraries to your" +echo " LD_LIBRARY_PATH to run with openmp." +echo +echo " LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:/usr/tce/packages/intel-classic-tce/intel-classic-2021.6.0/compiler/lib/intel64_lin" +echo +echo "***********************************************************************" diff --git a/scripts/lc-builds/toss4_icpx.sh b/scripts/lc-builds/toss4_icpx.sh new file mode 100755 index 0000000000..7c2a3c4561 --- /dev/null +++ b/scripts/lc-builds/toss4_icpx.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +############################################################################### +# Copyright (c) 2016-22, Lawrence Livermore National Security, LLC +# and RAJA project contributors. See the RAJA/LICENSE file for details. +# +# SPDX-License-Identifier: (BSD-3-Clause) +############################################################################### + +if [ "$1" == "" ]; then + echo + echo "You must pass a compiler version number to script. For example," + echo " toss4_icpx.sh 2022.1.0" + exit +fi + +COMP_VER=$1 +shift 1 + +USE_TBB=On + +BUILD_SUFFIX=lc_toss4-icpx-${COMP_VER} + +echo +echo "Creating build directory build_${BUILD_SUFFIX} and generating configuration in it" +echo "Configuration extra arguments:" +echo " $@" +echo + +rm -rf build_${BUILD_SUFFIX} 2>/dev/null +mkdir build_${BUILD_SUFFIX} && cd build_${BUILD_SUFFIX} + +module load cmake/3.21.1 + +# +# Note: we are using the intel-tce install path as the vanilla intel install +# path is not in /usr/tce/packages +# + +cmake \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_CXX_COMPILER=/usr/tce/packages/intel-tce/intel-${COMP_VER}/bin/icpx \ + -DCMAKE_C_COMPILER=/usr/tce/packages/intel-tce/intel-${COMP_VER}/bin/icx \ + -DBLT_CXX_STD=c++14 \ + -C ../host-configs/lc-builds/toss4/icpx_X.cmake \ + -DENABLE_OPENMP=On \ + -DRAJA_ENABLE_TBB=${USE_TBB} \ + -DCMAKE_INSTALL_PREFIX=../install_${BUILD_SUFFIX} \ + "$@" \ + .. diff --git a/scripts/radiuss-spack-configs b/scripts/radiuss-spack-configs index 1ce0f4421c..8d955b4a49 160000 --- a/scripts/radiuss-spack-configs +++ b/scripts/radiuss-spack-configs @@ -1 +1 @@ -Subproject commit 1ce0f4421cfe6be4200ae9aa8abd113e09ee4c2d +Subproject commit 8d955b4a49406a12dc44f1a9baab7ab7fa9c68df diff --git a/scripts/spack_packages/blt/package.py b/scripts/spack_packages/blt/package.py new file mode 100644 index 0000000000..87fb122dff --- /dev/null +++ b/scripts/spack_packages/blt/package.py @@ -0,0 +1,40 @@ +# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from spack.package import * + + +class Blt(Package): + """BLT is a streamlined CMake-based foundation for Building, Linking and + Testing large-scale high performance computing (HPC) applications.""" + + homepage = "https://github.com/LLNL/blt" + url = "https://github.com/LLNL/blt/archive/v0.4.0.tar.gz" + git = "https://github.com/LLNL/blt.git" + tags = ["radiuss"] + + maintainers = ["white238", "davidbeckingsale"] + + version("develop", branch="develop") + version("main", branch="main") + # Note: 0.4.0+ contains a breaking change to BLT created targets + # if you export targets this could cause problems in downstream + # projects if not handled properly. More info here: + # https://llnl-blt.readthedocs.io/en/develop/tutorial/exporting_targets.html + version("0.5.2", sha256="95b924cfbb2bddd9b1a92e96603b2fd485a19721d59ddf8ff50baefc1714d7ea") + version("0.5.1", sha256="ff7e87eefc48704a0721b66174612b945955adaa0a56aa69dd0473074fa4badf") + version("0.5.0", sha256="5f680ef922d0e0a7ff1b1a5fc8aa107cd4f543ad888cbc9b12639bea72a6ab1f") + version("0.4.1", sha256="16cc3e067ddcf48b99358107e5035a17549f52dcc701a35cd18a9d9f536826c1") + version("0.4.0", sha256="f3bc45d28b9b2eb6df43b75d4f6f89a1557d73d012da7b75bac1be0574767193") + version("0.3.6", sha256="6276317c29e7ff8524fbea47d9288ddb40ac06e9f9da5e878bf9011e2c99bf71") + version("0.3.5", sha256="68a1c224bb9203461ae6f5ab0ff3c50b4a58dcce6c2d2799489a1811f425fb84") + version("0.3.0", sha256="bb917a67cb7335d6721c997ba9c5dca70506006d7bba5e0e50033dd0836481a5") + version("0.2.5", sha256="3a000f60194e47b3e5623cc528cbcaf88f7fea4d9620b3c7446ff6658dc582a5") + version("0.2.0", sha256="c0cadf1269c2feb189e398a356e3c49170bc832df95e5564e32bdbb1eb0fa1b3") + + depends_on("cmake", type="run") + + def install(self, spec, prefix): + install_tree(".", prefix) diff --git a/scripts/spack_packages/camp/package.py b/scripts/spack_packages/camp/package.py index 3bff14ef61..4e25af8022 100644 --- a/scripts/spack_packages/camp/package.py +++ b/scripts/spack_packages/camp/package.py @@ -8,6 +8,24 @@ from spack.package import * +def hip_repair_options(options, spec): + # there is only one dir like this, but the version component is unknown + options.append( + "-DHIP_CLANG_INCLUDE_PATH=" + + glob.glob("{}/lib/clang/*/include".format(spec["llvm-amdgpu"].prefix))[0] + ) + + +def hip_repair_cache(options, spec): + # there is only one dir like this, but the version component is unknown + options.append( + cmake_cache_path( + "HIP_CLANG_INCLUDE_PATH", + glob.glob("{}/lib/clang/*/include".format(spec["llvm-amdgpu"].prefix))[0], + ) + ) + + class Camp(CMakePackage, CudaPackage, ROCmPackage): """ Compiler agnostic metaprogramming library providing concepts, @@ -15,69 +33,66 @@ class Camp(CMakePackage, CudaPackage, ROCmPackage): """ homepage = "https://github.com/LLNL/camp" - git = "https://github.com/LLNL/camp.git" - url = "https://github.com/LLNL/camp/archive/v0.1.0.tar.gz" + git = "https://github.com/LLNL/camp.git" + url = "https://github.com/LLNL/camp/archive/v0.1.0.tar.gz" - maintainers = ['trws'] + maintainers = ["trws"] - version('main', branch='main', submodules='True') - version('2022.03.0', sha256='e9090d5ee191ea3a8e36b47a8fe78f3ac95d51804f1d986d931e85b8f8dad721') - version('0.3.0', sha256='129431a049ca5825443038ad5a37a86ba6d09b2618d5fe65d35f83136575afdb') - version('0.2.3', sha256='58a0f3bd5eadb588d7dc83f3d050aff8c8db639fc89e8d6553f9ce34fc2421a7') - version('0.2.2', sha256='194d38b57e50e3494482a7f94940b27f37a2bee8291f2574d64db342b981d819') - version('0.1.0', sha256='fd4f0f2a60b82a12a1d9f943f8893dc6fe770db493f8fae5ef6f7d0c439bebcc') + version("main", branch="main", submodules="True") + version("2022.10.0", sha256="3561c3ef00bbcb61fe3183c53d49b110e54910f47e7fc689ad9ccce57e55d6b8") + version("2022.03.2", sha256="bc4aaeacfe8f2912e28f7a36fc731ab9e481bee15f2c6daf0cb208eed3f201eb") + version("2022.03.0", sha256="e9090d5ee191ea3a8e36b47a8fe78f3ac95d51804f1d986d931e85b8f8dad721") + version("0.3.0", sha256="129431a049ca5825443038ad5a37a86ba6d09b2618d5fe65d35f83136575afdb") + version("0.2.3", sha256="58a0f3bd5eadb588d7dc83f3d050aff8c8db639fc89e8d6553f9ce34fc2421a7") + version("0.2.2", sha256="194d38b57e50e3494482a7f94940b27f37a2bee8291f2574d64db342b981d819") + version("0.1.0", sha256="fd4f0f2a60b82a12a1d9f943f8893dc6fe770db493f8fae5ef6f7d0c439bebcc") # TODO: figure out gtest dependency and then set this default True. - variant('tests', default=False, description='Build tests') - variant('openmp', default=False, description='Build with OpenMP support') + variant("tests", default=False, description="Build tests") + variant("openmp", default=False, description="Build with OpenMP support") - depends_on('cub', when='+cuda') + depends_on("cub", when="+cuda") - depends_on('blt') + depends_on("blt") def cmake_args(self): spec = self.spec options = [] - options.append("-DBLT_SOURCE_DIR={0}".format(spec['blt'].prefix)) + options.append("-DBLT_SOURCE_DIR={0}".format(spec["blt"].prefix)) - if '+cuda' in spec: + if "+cuda" in spec: options.extend([ - '-DENABLE_CUDA=ON', - '-DCUDA_TOOLKIT_ROOT_DIR=%s' % (spec['cuda'].prefix)]) - - if not spec.satisfies('cuda_arch=none'): - cuda_arch = spec.variants['cuda_arch'].value - options.append('-DCMAKE_CUDA_ARCHITECTURES={0}'.format(cuda_arch[0])) - options.append('-DCUDA_ARCH=sm_{0}'.format(cuda_arch[0])) - flag = '-arch sm_{0}'.format(cuda_arch[0]) - options.append('-DCMAKE_CUDA_FLAGS:STRING={0}'.format(flag)) + "-DENABLE_CUDA=ON", + "-DCUDA_TOOLKIT_ROOT_DIR=%s" % (spec["cuda"].prefix) + ]) + + if not spec.satisfies("cuda_arch=none"): + cuda_arch = spec.variants["cuda_arch"].value + options.append("-DCMAKE_CUDA_ARCHITECTURES={0}".format(cuda_arch[0])) + options.append("-DCUDA_ARCH=sm_{0}".format(cuda_arch[0])) + flag = "-arch sm_{0}".format(cuda_arch[0]) + options.append("-DCMAKE_CUDA_FLAGS:STRING={0}".format(flag)) else: - options.append('-DENABLE_CUDA=OFF') + options.append("-DENABLE_CUDA=OFF") - if '+rocm' in spec: + if "+rocm" in spec: options.extend([ - '-DENABLE_HIP=ON', - '-DHIP_ROOT_DIR={0}'.format(spec['hip'].prefix) + "-DENABLE_HIP=ON", + "-DHIP_ROOT_DIR={0}".format(spec["hip"].prefix) ]) - archs = self.spec.variants['amdgpu_target'].value - if archs != 'none': + + hip_repair_options(options, spec) + + archs = self.spec.variants["amdgpu_target"].value + if archs != "none": arch_str = ",".join(archs) - options.append( - '-DHIP_HIPCC_FLAGS=--amdgpu-target={0}'.format(arch_str) - ) - # there is only one dir like this, but the version component is unknown - options.append( - "-DHIP_CLANG_INCLUDE_PATH=" + glob.glob( - "{}/lib/clang/*/include".format(spec['llvm-amdgpu'].prefix) - )[0] - ) + options.append("-DHIP_HIPCC_FLAGS=--amdgpu-target={0}".format(arch_str)) else: - options.append('-DENABLE_HIP=OFF') - - options.append(self.define_from_variant('ENABLE_TESTS', 'tests')) - options.append(self.define_from_variant('ENABLE_OPENMP', 'openmp')) + options.append("-DENABLE_HIP=OFF") + options.append(self.define_from_variant("ENABLE_OPENMP", "openmp")) + options.append(self.define_from_variant("ENABLE_TESTS", "tests")) return options diff --git a/scripts/spack_packages/raja/package.py b/scripts/spack_packages/raja/package.py index 63e0888c1f..875e63107d 100644 --- a/scripts/spack_packages/raja/package.py +++ b/scripts/spack_packages/raja/package.py @@ -1,382 +1,309 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) - -from spack import * - -import glob -import os import socket +import glob -from os import environ as env -from os.path import join as pjoin - -import re - -def cmake_cache_entry(name, value, comment=""): - """Generate a string for a cmake cache variable""" - - return 'set(%s "%s" CACHE PATH "%s")\n\n' % (name,value,comment) - - -def cmake_cache_string(name, string, comment=""): - """Generate a string for a cmake cache variable""" - - return 'set(%s "%s" CACHE STRING "%s")\n\n' % (name,string,comment) - - -def cmake_cache_option(name, boolean_value, comment=""): - """Generate a string for a cmake configuration option""" - - value = "ON" if boolean_value else "OFF" - return 'set(%s %s CACHE BOOL "%s")\n\n' % (name,value,comment) - - -def get_spec_path(spec, package_name, path_replacements = {}, use_bin = False) : - """Extracts the prefix path for the given spack package - path_replacements is a dictionary with string replacements for the path. - """ - - if not use_bin: - path = spec[package_name].prefix - else: - path = spec[package_name].prefix.bin - - path = os.path.realpath(path) - - for key in path_replacements: - path = path.replace(key, path_replacements[key]) - - return path - - -class Raja(CMakePackage, CudaPackage, ROCmPackage): - """RAJA Performance Portability Abstractions for C++ HPC Applications.""" - - homepage = "https://github.com/LLNL/RAJA" - git = "https://github.com/LLNL/RAJA.git" - tags = ['radiuss', 'e4s'] - - maintainers = ['davidbeckingsale'] - - version('develop', branch='develop', submodules='True') - version('main', branch='main', submodules='True') - version('0.14.1', tag='v0.14.1', submodules="True") - version('0.14.0', tag='v0.14.0', submodules="True") - version('0.13.0', tag='v0.13.0', submodules="True") - version('0.12.1', tag='v0.12.1', submodules="True") - version('0.12.0', tag='v0.12.0', submodules="True") - version('0.11.0', tag='v0.11.0', submodules="True") - version('0.10.1', tag='v0.10.1', submodules="True") - version('0.10.0', tag='v0.10.0', submodules="True") - version('0.9.0', tag='v0.9.0', submodules="True") - version('0.8.0', tag='v0.8.0', submodules="True") - version('0.7.0', tag='v0.7.0', submodules="True") - version('0.6.0', tag='v0.6.0', submodules="True") - version('0.5.3', tag='v0.5.3', submodules="True") - version('0.5.2', tag='v0.5.2', submodules="True") - version('0.5.1', tag='v0.5.1', submodules="True") - version('0.5.0', tag='v0.5.0', submodules="True") - version('0.4.1', tag='v0.4.1', submodules="True") - version('0.4.0', tag='v0.4.0', submodules="True") - - variant('openmp', default=True, description='Build OpenMP backend') - variant('shared', default=False, description='Build Shared Libs') - variant('libcpp', default=False, description='Uses libc++ instead of libstdc++') - variant('tests', default='basic', values=('none', 'basic', 'benchmarks'), - multi=False, description='Tests to run') - variant('desul', default=False, description='Build Desul Atomics backend') - - depends_on('cmake@3.9:', type='build') - - depends_on('blt@0.4.1', type='build', when='@main') - depends_on('blt@0.4.1:', type='build') - - depends_on('camp') - depends_on('camp@main') # TODO: remove this ASAP - depends_on('camp+rocm', when='+rocm') - depends_on('camp+openmp', when='+openmp') - for val in ROCmPackage.amdgpu_targets: - depends_on('camp amdgpu_target=%s' % val, when='amdgpu_target=%s' % val) - - depends_on('camp+cuda', when='+cuda') - for sm_ in CudaPackage.cuda_arch_values: - depends_on('camp cuda_arch={0}'.format(sm_), - when='cuda_arch={0}'.format(sm_)) - - conflicts('+openmp', when='+rocm') - depends_on('rocprim', when='+rocm') +from spack.package import * +from spack.pkg.builtin.camp import hip_repair_cache + + +class Raja(CachedCMakePackage, CudaPackage, ROCmPackage): + """RAJA Parallel Framework.""" + + homepage = "https://software.llnl.gov/RAJA/" + git = "https://github.com/LLNL/RAJA.git" + tags = ["radiuss", "e4s"] + + maintainers = ["davidbeckingsale"] + + version("develop", branch="develop", submodules=False) + version("main", branch="main", submodules=False) + version("2022.10.1", tag="v2022.10.1", submodules=False) + version("2022.10.0", tag="v2022.10.0", submodules=False) + version("2022.03.1", tag="v2022.03.1", submodules=False) + version("2022.03.0", tag="v2022.03.0", submodules=False) + version("0.14.0", tag="v0.14.0", submodules="True") + version("0.13.0", tag="v0.13.0", submodules="True") + version("0.12.1", tag="v0.12.1", submodules="True") + version("0.12.0", tag="v0.12.0", submodules="True") + version("0.11.0", tag="v0.11.0", submodules="True") + version("0.10.1", tag="v0.10.1", submodules="True") + version("0.10.0", tag="v0.10.0", submodules="True") + version("0.9.0", tag="v0.9.0", submodules="True") + version("0.8.0", tag="v0.8.0", submodules="True") + version("0.7.0", tag="v0.7.0", submodules="True") + version("0.6.0", tag="v0.6.0", submodules="True") + version("0.5.3", tag="v0.5.3", submodules="True") + version("0.5.2", tag="v0.5.2", submodules="True") + version("0.5.1", tag="v0.5.1", submodules="True") + version("0.5.0", tag="v0.5.0", submodules="True") + version("0.4.1", tag="v0.4.1", submodules="True") + version("0.4.0", tag="v0.4.0", submodules="True") + + # export targets when building pre-2.4.0 release with BLT 0.4.0+ + patch( + "https://github.com/LLNL/RAJA/commit/eca1124ee4af380d6613adc6012c307d1fd4176b.patch?full_index=1", + sha256="12bb78c00b6683ad3e7fd4e3f87f9776bae074b722431b79696bc862816735ef", + when="@:0.13.0 ^blt@0.4:", + ) + + variant("openmp", default=True, description="Build OpenMP backend") + variant("shared", default=True, description="Build Shared Libs") + variant("examples", default=True, description="Build examples.") + variant("exercises", default=True, description="Build exercises.") + # TODO: figure out gtest dependency and then set this default True + # and remove the +tests conflict below. + variant("tests", default=False, description="Build tests") + variant("libcpp", default=False, description="Uses libc++ instead of libstdc++") + variant("desul", default=False, description="Build Desul Atomics backend") + + depends_on("blt") + depends_on("blt@0.5.2:", type="build", when="@2022.10.0:") + depends_on("blt@0.5.0:", type="build", when="@0.14.1:") + depends_on("blt@0.4.1", type="build", when="@0.14.0") + depends_on("blt@0.4.0:", type="build", when="@0.13.0") + depends_on("blt@0.3.6:", type="build", when="@:0.12.0") + + depends_on("camp@0.2.2:0.2.3", when="@0.14.0") + depends_on("camp@0.1.0", when="@0.10.0:0.13.0") + depends_on("camp@2022.10.0:", when="@2022.10.0:") + depends_on("camp@2022.03.2:", when="@2022.03.0:") + depends_on("camp@main", when="@main") + depends_on("camp@main", when="@develop") + depends_on("camp+openmp", when="+openmp") + + depends_on("cmake@3.20:", when="@2022.10.0:", type="build") + depends_on("cmake@3.23:", when="@2022.10.0: +rocm", type="build") + depends_on("cmake@3.14:", when="@2022.03.0:", type="build") + depends_on("cmake@:3.20", when="@2022.03.0:2022.03 +rocm", type="build") + + depends_on("llvm-openmp", when="+openmp %apple-clang") + + depends_on("rocprim", when="+rocm") + with when("+rocm @0.12.0:"): + depends_on("camp+rocm") + for arch in ROCmPackage.amdgpu_targets: + depends_on( + "camp+rocm amdgpu_target={0}".format(arch), when="amdgpu_target={0}".format(arch) + ) + conflicts("+openmp") - phases = ['hostconfig', 'cmake', 'build', 'install'] + with when("+cuda @0.12.0:"): + depends_on("camp+cuda") + for sm_ in CudaPackage.cuda_arch_values: + depends_on("camp +cuda cuda_arch={0}".format(sm_), when="cuda_arch={0}".format(sm_)) def _get_sys_type(self, spec): - sys_type = str(spec.architecture) - # if on llnl systems, we can use the SYS_TYPE + sys_type = spec.architecture if "SYS_TYPE" in env: sys_type = env["SYS_TYPE"] return sys_type - def _get_host_config_path(self, spec): - var='' - if '+cuda' in spec: - var= '-'.join([var,'cuda']) - if '+libcpp' in spec: - var='-'.join([var,'libcpp']) - - host_config_path = "hc-%s-%s-%s%s-%s.cmake" % (socket.gethostname().rstrip('1234567890'), - self._get_sys_type(spec), - spec.compiler, - var, - spec.dag_hash()) - dest_dir = self.stage.source_path - host_config_path = os.path.abspath(pjoin(dest_dir, host_config_path)) - return host_config_path - - def hostconfig(self, spec, prefix, py_site_pkgs_dir=None): - """ - This method creates a 'host-config' file that specifies - all of the options used to configure and build Umpire. - - For more details about 'host-config' files see: - http://software.llnl.gov/conduit/building.html - - Note: - The `py_site_pkgs_dir` arg exists to allow a package that - subclasses this package provide a specific site packages - dir when calling this function. `py_site_pkgs_dir` should - be an absolute path or `None`. - - This is necessary because the spack `site_packages_dir` - var will not exist in the base class. For more details - on this issue see: https://github.com/spack/spack/issues/6261 - """ - - ####################### - # Compiler Info - ####################### - c_compiler = env["SPACK_CC"] - cpp_compiler = env["SPACK_CXX"] - - # Even though we don't have fortran code in our project we sometimes - # use the Fortran compiler to determine which libstdc++ to use - f_compiler = "" - if "SPACK_FC" in env.keys(): - # even if this is set, it may not exist - # do one more sanity check - if os.path.isfile(env["SPACK_FC"]): - f_compiler = env["SPACK_FC"] - - ####################################################################### - # By directly fetching the names of the actual compilers we appear - # to doing something evil here, but this is necessary to create a - # 'host config' file that works outside of the spack install env. - ####################################################################### - - sys_type = self._get_sys_type(spec) - - ############################################## - # Find and record what CMake is used - ############################################## - - cmake_exe = spec['cmake'].command.path - cmake_exe = os.path.realpath(cmake_exe) - - host_config_path = self._get_host_config_path(spec) - cfg = open(host_config_path, "w") - cfg.write("###################\n".format("#" * 60)) - cfg.write("# Generated host-config - Edit at own risk!\n") - cfg.write("###################\n".format("#" * 60)) - cfg.write("# Copyright 2016-22, Lawrence Livermore National Security, LLC\n") - cfg.write("# and RAJA project contributors. See the RAJA/LICENSE file\n") - cfg.write("# for details.\n") - cfg.write("#\n") - cfg.write("# SPDX-License-Identifier: (BSD-3-Clause) \n") - cfg.write("###################\n\n".format("#" * 60)) - - cfg.write("#------------------\n".format("-" * 60)) - cfg.write("# SYS_TYPE: {0}\n".format(sys_type)) - cfg.write("# Compiler Spec: {0}\n".format(spec.compiler)) - cfg.write("# CMake executable path: %s\n" % cmake_exe) - cfg.write("#------------------\n\n".format("-" * 60)) - - cfg.write(cmake_cache_string("CMAKE_BUILD_TYPE", spec.variants['build_type'].value)) - - ####################### - # Compiler Settings - ####################### - - cfg.write("#------------------\n".format("-" * 60)) - cfg.write("# Compilers\n") - cfg.write("#------------------\n\n".format("-" * 60)) - cfg.write(cmake_cache_entry("CMAKE_C_COMPILER", c_compiler)) - cfg.write(cmake_cache_entry("CMAKE_CXX_COMPILER", cpp_compiler)) + @property + # TODO: name cache file conditionally to cuda and libcpp variants + def cache_name(self): + hostname = socket.gethostname() + if "SYS_TYPE" in env: + hostname = hostname.rstrip("1234567890") + return "{0}-{1}-{2}@{3}-{4}.cmake".format( + hostname, + self._get_sys_type(self.spec), + self.spec.compiler.name, + self.spec.compiler.version, + self.spec.dag_hash(8) + ) + + def initconfig_compiler_entries(self): + spec = self.spec + # Default entries are already defined in CachedCMakePackage, inherit them: + entries = super(Raja, self).initconfig_compiler_entries() + + # Switch to hip as a CPP compiler. + # adrienbernede-22-11: + # This was only done in upstream Spack raja package. + # I could not find the equivalent logic in Spack source, so keeping it. + if "+rocm" in spec: + entries.insert(0, cmake_cache_path("CMAKE_CXX_COMPILER", spec["hip"].hipcc)) + + # Override CachedCMakePackage CMAKE_C_FLAGS and CMAKE_CXX_FLAGS add + # +libcpp specific flags + flags = spec.compiler_flags # use global spack compiler flags - cflags = ' '.join(spec.compiler_flags['cflags']) + cppflags = " ".join(flags["cppflags"]) + if cppflags: + # avoid always ending up with " " with no flags defined + cppflags += " " + + cflags = cppflags + " ".join(flags["cflags"]) if "+libcpp" in spec: - cflags += ' '.join([cflags,"-DGTEST_HAS_CXXABI_H_=0"]) + cflags += " ".join([cflags,"-DGTEST_HAS_CXXABI_H_=0"]) if cflags: - cfg.write(cmake_cache_entry("CMAKE_C_FLAGS", cflags)) + entries.append(cmake_cache_string("CMAKE_C_FLAGS", cflags)) - cxxflags = ' '.join(spec.compiler_flags['cxxflags']) + cxxflags = cppflags + " ".join(flags["cxxflags"]) if "+libcpp" in spec: - cxxflags += ' '.join([cxxflags,"-stdlib=libc++ -DGTEST_HAS_CXXABI_H_=0"]) + cxxflags += " ".join([cxxflags,"-stdlib=libc++ -DGTEST_HAS_CXXABI_H_=0"]) if cxxflags: - cfg.write(cmake_cache_entry("CMAKE_CXX_FLAGS", cxxflags)) - - # TODO (bernede1@llnl.gov): Is this useful for RAJA? - if ("gfortran" in f_compiler) and ("clang" in cpp_compiler): - libdir = pjoin(os.path.dirname( - os.path.dirname(f_compiler)), "lib") - flags = "" - for _libpath in [libdir, libdir + "64"]: - if os.path.exists(_libpath): - flags += " -Wl,-rpath,{0}".format(_libpath) - description = ("Adds a missing libstdc++ rpath") - #if flags: - # cfg.write(cmake_cache_string("BLT_EXE_LINKER_FLAGS", flags, - # description)) - - gcc_toolchain_regex = re.compile("--gcc-toolchain=(.*)") - gcc_name_regex = re.compile(".*gcc-name.*") - - using_toolchain = list(filter(gcc_toolchain_regex.match, spec.compiler_flags['cxxflags'])) - if(using_toolchain): - gcc_toolchain_path = gcc_toolchain_regex.match(using_toolchain[0]) - using_gcc_name = list(filter(gcc_name_regex.match, spec.compiler_flags['cxxflags'])) - compilers_using_toolchain = ["pgi", "xl", "icpc"] - if any(compiler in cpp_compiler for compiler in compilers_using_toolchain): - if using_toolchain or using_gcc_name: - cfg.write(cmake_cache_entry("BLT_CMAKE_IMPLICIT_LINK_DIRECTORIES_EXCLUDE", - "/usr/tce/packages/gcc/gcc-4.9.3/lib64;/usr/tce/packages/gcc/gcc-4.9.3/gnu/lib64/gcc/powerpc64le-unknown-linux-gnu/4.9.3;/usr/tce/packages/gcc/gcc-4.9.3/gnu/lib64;/usr/tce/packages/gcc/gcc-4.9.3/lib64/gcc/x86_64-unknown-linux-gnu/4.9.3")) - - compilers_using_cxx14 = ["intel-17", "intel-18", "xl"] - if any(compiler in cpp_compiler for compiler in compilers_using_cxx14): - cfg.write(cmake_cache_entry("BLT_CXX_STD", "c++14")) + entries.append(cmake_cache_string("CMAKE_CXX_FLAGS", cxxflags)) + + return entries + + def initconfig_hardware_entries(self): + spec = self.spec + entries = super(Raja, self).initconfig_hardware_entries() + + entries.append(cmake_cache_option("ENABLE_OPENMP", "+openmp" in spec)) if "+cuda" in spec: - cfg.write("#------------------{0}\n".format("-" * 60)) - cfg.write("# Cuda\n") - cfg.write("#------------------{0}\n\n".format("-" * 60)) - - cfg.write(cmake_cache_option("ENABLE_CUDA", True)) - - cudatoolkitdir = spec['cuda'].prefix - cfg.write(cmake_cache_entry("CUDA_TOOLKIT_ROOT_DIR", - cudatoolkitdir)) - cudacompiler = "${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc" - cfg.write(cmake_cache_entry("CMAKE_CUDA_COMPILER", - cudacompiler)) - - if ("xl" in cpp_compiler): - cfg.write(cmake_cache_entry("CMAKE_CUDA_FLAGS", "-Xcompiler -O3 -Xcompiler -qxlcompatmacros -Xcompiler -qalias=noansi " + - "-Xcompiler -qsmp=omp -Xcompiler -qhot -Xcompiler -qnoeh -Xcompiler -qsuppress=1500-029 " + - "-Xcompiler -qsuppress=1500-036 -Xcompiler -qsuppress=1500-030")) - cuda_release_flags = "-O3" - cuda_reldebinf_flags = "-O3 -g" - cuda_debug_flags = "-O0 -g" - - cfg.write(cmake_cache_string("BLT_CXX_STD", "c++14")) - elif ("gcc" in cpp_compiler): - cuda_release_flags = "-O3 -Xcompiler -Ofast -Xcompiler -finline-functions -Xcompiler -finline-limit=20000" - cuda_reldebinf_flags = "-O3 -g -Xcompiler -Ofast -Xcompiler -finline-functions -Xcompiler -finline-limit=20000" - cuda_debug_flags = "-O0 -g -Xcompiler -O0 -Xcompiler -finline-functions -Xcompiler -finline-limit=20000" - else: - cuda_release_flags = "-O3 -Xcompiler -Ofast -Xcompiler -finline-functions" - cuda_reldebinf_flags = "-O3 -g -Xcompiler -Ofast -Xcompiler -finline-functions" - cuda_debug_flags = "-O0 -g -Xcompiler -O0 -Xcompiler -finline-functions" - - cfg.write(cmake_cache_string("CMAKE_CUDA_FLAGS_RELEASE", cuda_release_flags)) - cfg.write(cmake_cache_string("CMAKE_CUDA_FLAGS_RELWITHDEBINFO", cuda_reldebinf_flags)) - cfg.write(cmake_cache_string("CMAKE_CUDA_FLAGS_DEBUG", cuda_debug_flags)) - - if not spec.satisfies('cuda_arch=none'): - cuda_arch = spec.variants['cuda_arch'].value - cfg.write(cmake_cache_string("CMAKE_CUDA_ARCHITECTURES", '{0}'.format(cuda_arch[0]))) + entries.append(cmake_cache_option("ENABLE_CUDA", True)) + if not spec.satisfies("cuda_arch=none"): + cuda_arch = spec.variants["cuda_arch"].value + entries.append(cmake_cache_string("CUDA_ARCH", "sm_{0}".format(cuda_arch[0]))) + entries.append( + cmake_cache_string("CMAKE_CUDA_ARCHITECTURES", "{0}".format(cuda_arch[0])) + ) else: - cfg.write(cmake_cache_option("ENABLE_CUDA", False)) + entries.append(cmake_cache_option("ENABLE_CUDA", False)) if "+rocm" in spec: - cfg.write("#------------------{0}\n".format("-" * 60)) - cfg.write("# HIP\n") - cfg.write("#------------------{0}\n\n".format("-" * 60)) - - cfg.write(cmake_cache_option("ENABLE_HIP", True)) - - hip_root = spec['hip'].prefix - rocm_root = hip_root + "/.." - hip_arch = spec.variants['amdgpu_target'].value - cfg.write(cmake_cache_entry("HIP_ROOT_DIR", - hip_root)) - # there is only one dir like this, but the version component is unknown - cfg.write( - cmake_cache_path( - "HIP_CLANG_INCLUDE_PATH", - glob.glob( - "{}/lib/clang/*/include".format(spec['llvm-amdgpu'].prefix) - )[0] + entries.append(cmake_cache_option("ENABLE_HIP", True)) + entries.append(cmake_cache_path("HIP_ROOT_DIR", "{0}".format(spec["hip"].prefix))) + hip_repair_cache(entries, spec) + archs = self.spec.variants["amdgpu_target"].value + if archs != "none": + arch_str = ",".join(archs) + entries.append( + cmake_cache_string("HIP_HIPCC_FLAGS", "--amdgpu-target={0}".format(arch_str)) + ) + entries.append( + cmake_cache_string("CMAKE_HIP_ARCHITECTURES", arch_str) ) - ) - cfg.write(cmake_cache_entry("ROCM_ROOT_DIR", - rocm_root)) - cfg.write(cmake_cache_entry("HIP_PATH", - rocm_root + '/llvm/bin')) - cfg.write(cmake_cache_entry("CMAKE_HIP_ARCHITECTURES", hip_arch[0])) - - if ('%gcc' in spec) or (using_toolchain): - if ('%gcc' in spec): - gcc_bin = os.path.dirname(self.compiler.cxx) - gcc_prefix = join_path(gcc_bin, '..') - else: - gcc_prefix = gcc_toolchain_path.group(1) - cfg.write(cmake_cache_entry("HIP_CLANG_FLAGS", - "--gcc-toolchain={0}".format(gcc_prefix))) - cfg.write(cmake_cache_entry("CMAKE_EXE_LINKER_FLAGS", - " -Wl,-rpath {}/lib64".format(gcc_prefix))) - else: - cfg.write(cmake_cache_option("ENABLE_HIP", False)) + entries.append(cmake_cache_option("ENABLE_HIP", False)) + + return entries - cfg.write("#------------------{0}\n".format("-" * 60)) - cfg.write("# Other\n") - cfg.write("#------------------{0}\n\n".format("-" * 60)) + def initconfig_package_entries(self): + spec = self.spec + entries = [] + + option_prefix = "RAJA_" if spec.satisfies("@0.14.0:") else "" + + # TPL locations + entries.append("#------------------{0}".format("-" * 60)) + entries.append("# TPLs") + entries.append("#------------------{0}\n".format("-" * 60)) + + entries.append(cmake_cache_path("BLT_SOURCE_DIR", spec["blt"].prefix)) + if "camp" in self.spec: + entries.append(cmake_cache_path("camp_DIR", spec["camp"].prefix)) - cfg.write(cmake_cache_string("RAJA_RANGE_ALIGN", "4")) - cfg.write(cmake_cache_string("RAJA_RANGE_MIN_LENGTH", "32")) - cfg.write(cmake_cache_string("RAJA_DATA_ALIGN", "64")) + # Build options + entries.append("#------------------{0}".format("-" * 60)) + entries.append("# Build Options") + entries.append("#------------------{0}\n".format("-" * 60)) - cfg.write(cmake_cache_option("RAJA_HOST_CONFIG_LOADED", True)) + entries.append(cmake_cache_string( + "CMAKE_BUILD_TYPE", spec.variants["build_type"].value)) + entries.append(cmake_cache_option("BUILD_SHARED_LIBS", "+shared" in spec)) - # shared vs static libs - cfg.write(cmake_cache_option("BUILD_SHARED_LIBS","+shared" in spec)) - cfg.write(cmake_cache_option("ENABLE_OPENMP","+openmp" in spec)) - cfg.write(cmake_cache_option("RAJA_ENABLE_DESUL_ATOMICS","+desul" in spec)) + entries.append(cmake_cache_option("RAJA_ENABLE_DESUL_ATOMICS", "+desul" in spec)) if "+desul" in spec: - cfg.write(cmake_cache_string("BLT_CXX_STD","c++14")) + entries.append(cmake_cache_string("BLT_CXX_STD","c++14")) if "+cuda" in spec: - cfg.write(cmake_cache_string("CMAKE_CUDA_STANDARD", "14")) - - cfg.write(cmake_cache_option("ENABLE_BENCHMARKS", 'tests=benchmarks' in spec)) - cfg.write(cmake_cache_option("ENABLE_TESTS", not 'tests=none' in spec or self.run_tests)) - cfg.write(cmake_cache_string("camp_DIR", spec['camp'].prefix)) + entries.append(cmake_cache_string("CMAKE_CUDA_STANDARD", "14")) + + entries.append( + cmake_cache_option("{}ENABLE_EXAMPLES".format(option_prefix), "+examples" in spec) + ) + if spec.satisfies("@0.14.0:"): + entries.append( + cmake_cache_option( + "{}ENABLE_EXERCISES".format(option_prefix), "+exercises" in spec + ) + ) + else: + entries.append(cmake_cache_option("ENABLE_EXERCISES", "+exercises" in spec)) + + ### #TODO: Treat the workaround when building tests with spack wrapper + ### # For now, removing it to test CI, which builds tests outside of wrapper. + ### # Work around spack adding -march=ppc64le to SPACK_TARGET_ARGS which + ### # is used by the spack compiler wrapper. This can go away when BLT + ### # removes -Werror from GTest flags + ### if self.spec.satisfies("%clang target=ppc64le:") or ( not self.run_tests and not "+tests" in spec): + if not self.run_tests and not "+tests" in spec: + entries.append(cmake_cache_option("ENABLE_TESTS", False)) + else: + entries.append(cmake_cache_option("ENABLE_TESTS", True)) - ####################### - # Close and save - ####################### - cfg.write("\n") - cfg.close() + entries.append(cmake_cache_option("RAJA_HOST_CONFIG_LOADED", True)) - print("OUT: host-config file {0}".format(host_config_path)) + return entries def cmake_args(self): - spec = self.spec - host_config_path = self._get_host_config_path(spec) - options = [] - options.extend(['-C', host_config_path]) - return options + + @property + def build_relpath(self): + """Relative path to the cmake build subdirectory.""" + return join_path("..", self.build_dirname) + + @run_after("install") + def setup_build_tests(self): + """Copy the build test files after the package is installed to a + relative install test subdirectory for use during `spack test run`.""" + # Now copy the relative files + self.cache_extra_test_sources(self.build_relpath) + + # Ensure the path exists since relying on a relative path at the + # same level as the normal stage source path. + mkdirp(self.install_test_root) + + @property + def _extra_tests_path(self): + # TODO: The tests should be converted to re-build and run examples + # TODO: using the installed libraries. + return join_path(self.install_test_root, self.build_relpath, "bin") + + def _test_examples(self): + """Perform very basic checks on a subset of copied examples.""" + checks = [ + ( + "ex5_line-of-sight_solution", + [r"RAJA sequential", r"RAJA OpenMP", r"result -- PASS"], + ), + ( + "ex6_stencil-offset-layout_solution", + [r"RAJA Views \(permuted\)", r"result -- PASS"], + ), + ( + "ex8_tiled-matrix-transpose_solution", + [r"parallel top inner loop", r"collapsed inner loops", r"result -- PASS"], + ), + ("kernel-dynamic-tile", [r"Running index", r"(24,24)"]), + ("plugin-example", [r"Launching host kernel for the 10 time"]), + ("tut_batched-matrix-multiply", [r"result -- PASS"]), + ("wave-eqn", [r"Max Error = 2", r"Evolved solution to time"]), + ] + for exe, expected in checks: + reason = "test: checking output of {0} for {1}".format(exe, expected) + self.run_test( + exe, + [], + expected, + installed=False, + purpose=reason, + skip_missing=True, + work_dir=self._extra_tests_path, + ) + + def test(self): + """Perform smoke tests.""" + self._test_examples() diff --git a/share/raja/cmake/RAJA-config.cmake.in b/share/raja/cmake/RAJA-config.cmake.in index 3baa8b68a2..a9271781cd 100644 --- a/share/raja/cmake/RAJA-config.cmake.in +++ b/share/raja/cmake/RAJA-config.cmake.in @@ -32,6 +32,9 @@ # @PACKAGE_INIT@ +# cache the prefix dir (could be overriden by find_dependency) +set(RAJA_PACKAGE_PREFIX_DIR ${PACKAGE_PREFIX_DIR}) + include(CMakeFindDependencyMacro) if (@RAJA_NEEDS_BLT_TPLS@) @@ -40,6 +43,14 @@ if (@RAJA_NEEDS_BLT_TPLS@) endif () endif() +#Needs to come before camp +#Camp performs a check on the BLT target +set(BLT_TGTS "${CMAKE_CURRENT_LIST_DIR}/bltTargets.cmake") +if(EXISTS "${BLT_TGTS}") +include("${BLT_TGTS}") +endif() +unset(BLT_TGTS) + if (NOT TARGET camp) set(RAJA_CAMP_DIR "@camp_DIR@") if(NOT camp_DIR) @@ -49,14 +60,24 @@ if (NOT TARGET camp) find_dependency(camp CONFIG NO_DEFAULT_PATH PATHS ${camp_DIR} ${camp_DIR}/lib/cmake/camp - @PACKAGE_CMAKE_INSTALL_PREFIX@ - @PACKAGE_CMAKE_INSTALL_PREFIX@/lib/cmake/camp) + ${RAJA_PACKAGE_PREFIX_DIR} + ${RAJA_PACKAGE_PREFIX_DIR}/lib/cmake/camp) endif () -set(BLT_TGTS "${CMAKE_CURRENT_LIST_DIR}/bltTargets.cmake") -if(EXISTS "${BLT_TGTS}") -include("${BLT_TGTS}") +if (@RAJA_ENABLE_DESUL_ATOMICS@) + if (NOT TARGET desul_atomics) + set(RAJA_DESUL_DIR "@desul_DIR@") + if(NOT desul_DIR) + set(desul_DIR ${RAJA_DESUL_DIR}) + endif() + + find_dependency(desul_atomics CONFIG NO_DEFAULT_PATH PATHS + ${desul_DIR} + ${desul_DIR}/lib/cmake/desul + ${RAJA_PACKAGE_PREFIX_DIR} + ${RAJA_PACKAGE_PREFIX_DIR}/lib/cmake/desul) + endif () endif() -unset(BLT_TGTS) + include("${CMAKE_CURRENT_LIST_DIR}/RAJATargets.cmake") check_required_components("@PROJECT_NAME@") diff --git a/test/functional/dynamic_forall/resource-segment/tests/test-dynamic-forall-resource-RangeSegment.hpp b/test/functional/dynamic_forall/resource-segment/tests/test-dynamic-forall-resource-RangeSegment.hpp index 0b09079093..fccdf6880b 100644 --- a/test/functional/dynamic_forall/resource-segment/tests/test-dynamic-forall-resource-RangeSegment.hpp +++ b/test/functional/dynamic_forall/resource-segment/tests/test-dynamic-forall-resource-RangeSegment.hpp @@ -66,7 +66,9 @@ TYPED_TEST_P(DynamicForallResourceRangeSegmentTest, RangeSegmentForallResource) using POLICY_LIST = typename camp::at>::type; +#if defined(RAJA_DEVICE_ACTIVE) constexpr int N = camp::size::value; +#endif //If N == 2 host, no openmp is available //If N == 3 host, openmp is available diff --git a/test/include/RAJA_test-plugin-launchpol.hpp b/test/include/RAJA_test-plugin-launchpol.hpp new file mode 100644 index 0000000000..b3677144dc --- /dev/null +++ b/test/include/RAJA_test-plugin-launchpol.hpp @@ -0,0 +1,35 @@ +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// +// Copyright (c) 2016-22, Lawrence Livermore National Security, LLC +// and RAJA project contributors. See the RAJA/LICENSE file for details. +// +// SPDX-License-Identifier: (BSD-3-Clause) +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// + +// +// Kernel execution policy lists used throughout plugin tests +// + +#ifndef __RAJA_test_plugin_launchpol_HPP__ +#define __RAJA_test_plugin_launchpol_HPP__ + +#include "RAJA/RAJA.hpp" + +#include "camp/list.hpp" + +// Sequential execution policy types +using SequentialPluginLaunchExecPols = camp::list>; + +#if defined(RAJA_ENABLE_OPENMP) +using OpenMPPluginLaunchExecPols = camp::list>; +#endif + +#if defined(RAJA_ENABLE_CUDA) +using CudaPluginLaunchExecPols = camp::list>>; +#endif + +#if defined(RAJA_ENABLE_HIP) +using HipPluginLaunchExecPols = camp::list>>; + +#endif + +#endif // __RAJA_test_plugin_kernelpol_HPP__ diff --git a/test/include/RAJA_test-plugin-resource-launchpol.hpp b/test/include/RAJA_test-plugin-resource-launchpol.hpp new file mode 100644 index 0000000000..c2c2dccb9a --- /dev/null +++ b/test/include/RAJA_test-plugin-resource-launchpol.hpp @@ -0,0 +1,35 @@ +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// +// Copyright (c) 2016-22, Lawrence Livermore National Security, LLC +// and RAJA project contributors. See the RAJA/LICENSE file for details. +// +// SPDX-License-Identifier: (BSD-3-Clause) +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// + +// +// Kernel execution policy lists used throughout plugin tests +// + +#ifndef __RAJA_test_plugin_resource_launchpol_HPP__ +#define __RAJA_test_plugin_resource_launchpol_HPP__ + +#include "RAJA/RAJA.hpp" + +#include "camp/list.hpp" + +// Sequential execution policy types +using SequentialPluginResourceLaunchExecPols = camp::list>; + +#if defined(RAJA_ENABLE_OPENMP) +using OpenMPPluginResourceLaunchExecPols = camp::list>; +#endif + +#if defined(RAJA_ENABLE_CUDA) +using CudaPluginResourceLaunchExecPols = camp::list>>; +#endif + +#if defined(RAJA_ENABLE_HIP) +using HipPluginResourceLaunchExecPols = camp::list>>; + +#endif + +#endif // __RAJA_test_plugin_kernelpol_HPP__ diff --git a/test/install/using-with-cmake/CMakeLists.txt b/test/install/using-with-cmake/CMakeLists.txt index 32f8baa1da..27b397bbaf 100644 --- a/test/install/using-with-cmake/CMakeLists.txt +++ b/test/install/using-with-cmake/CMakeLists.txt @@ -5,7 +5,11 @@ # SPDX-License-Identifier: (BSD-3-Clause) ############################################################################### -cmake_minimum_required(VERSION 3.14) +if (ENABLE_HIP) + cmake_minimum_required(VERSION 3.23) +else() + cmake_minimum_required(VERSION 3.20) +endif() project(using_with_cmake) @@ -19,4 +23,4 @@ cmake_minimum_required(VERSION 3.14) add_executable(using-with-cmake using-with-cmake.cpp) target_link_libraries(using-with-cmake RAJA) - \ No newline at end of file + diff --git a/test/integration/plugin/CMakeLists.txt b/test/integration/plugin/CMakeLists.txt index da41e2a0bd..360d63164c 100644 --- a/test/integration/plugin/CMakeLists.txt +++ b/test/integration/plugin/CMakeLists.txt @@ -32,6 +32,28 @@ foreach( BACKEND ${PLUGIN_BACKENDS} ) PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/tests) endforeach() +foreach( BACKEND ${PLUGIN_BACKENDS} ) + configure_file( test-plugin-launch.cpp.in + test-plugin-launch-${BACKEND}.cpp ) + raja_add_test( NAME test-plugin-launch-${BACKEND} + SOURCES ${CMAKE_CURRENT_BINARY_DIR}/test-plugin-launch-${BACKEND}.cpp + plugin_to_test.cpp ) + + target_include_directories(test-plugin-launch-${BACKEND}.exe + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/tests) +endforeach() + +foreach( BACKEND ${PLUGIN_BACKENDS} ) + configure_file( test-plugin-resource-launch.cpp.in + test-plugin-resource-launch-${BACKEND}.cpp ) + raja_add_test( NAME test-plugin-resource-launch-${BACKEND} + SOURCES ${CMAKE_CURRENT_BINARY_DIR}/test-plugin-resource-launch-${BACKEND}.cpp + plugin_to_test.cpp ) + + target_include_directories(test-plugin-resource-launch-${BACKEND}.exe + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/tests) +endforeach() + set(DISPATCHERS Direct IndirectFunction IndirectVirtual) foreach( BACKEND ${PLUGIN_BACKENDS} ) diff --git a/test/integration/plugin/test-plugin-launch.cpp.in b/test/integration/plugin/test-plugin-launch.cpp.in new file mode 100644 index 0000000000..7df20f0ba4 --- /dev/null +++ b/test/integration/plugin/test-plugin-launch.cpp.in @@ -0,0 +1,38 @@ +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// +// Copyright (c) 2016-22, Lawrence Livermore National Security, LLC +// and RAJA project contributors. See the RAJA/LICENSE file for details. +// +// SPDX-License-Identifier: (BSD-3-Clause) +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// + +// +// test/include headers +// +#include "RAJA_test-base.hpp" +#include "RAJA_test-camp.hpp" +#include "RAJA_test-platform.hpp" + +#include "RAJA_test-plugin-launchpol.hpp" + +// +// Header for tests in ./tests directory +// +// Note: CMake adds ./tests as an include dir for these tests. +// +#include "test-plugin-launch.hpp" + + +// +// Cartesian product of types used in parameterized tests +// +using @BACKEND@PluginLaunchTypes = + Test< camp::cartesian_product<@BACKEND@PluginLaunchExecPols, + @BACKEND@ResourceList, + @BACKEND@PlatformList > >::Types; + +// +// Instantiate parameterized test +// +INSTANTIATE_TYPED_TEST_SUITE_P(@BACKEND@, + PluginLaunchTest, + @BACKEND@PluginLaunchTypes); diff --git a/test/integration/plugin/test-plugin-resource-launch.cpp.in b/test/integration/plugin/test-plugin-resource-launch.cpp.in new file mode 100644 index 0000000000..573982ba38 --- /dev/null +++ b/test/integration/plugin/test-plugin-resource-launch.cpp.in @@ -0,0 +1,38 @@ +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// +// Copyright (c) 2016-22, Lawrence Livermore National Security, LLC +// and RAJA project contributors. See the RAJA/LICENSE file for details. +// +// SPDX-License-Identifier: (BSD-3-Clause) +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// + +// +// test/include headers +// +#include "RAJA_test-base.hpp" +#include "RAJA_test-camp.hpp" +#include "RAJA_test-platform.hpp" + +#include "RAJA_test-plugin-resource-launchpol.hpp" + +// +// Header for tests in ./tests directory +// +// Note: CMake adds ./tests as an include dir for these tests. +// +#include "test-plugin-resource-launch.hpp" + + +// +// Cartesian product of types used in parameterized tests +// +using @BACKEND@PluginResourceLaunchTypes = + Test< camp::cartesian_product<@BACKEND@PluginResourceLaunchExecPols, + @BACKEND@ResourceList, + @BACKEND@PlatformList > >::Types; + +// +// Instantiate parameterized test +// +INSTANTIATE_TYPED_TEST_SUITE_P(@BACKEND@, + PluginResourceLaunchTest, + @BACKEND@PluginResourceLaunchTypes); diff --git a/test/integration/plugin/tests/test-plugin-launch.hpp b/test/integration/plugin/tests/test-plugin-launch.hpp new file mode 100644 index 0000000000..6aa1744b6c --- /dev/null +++ b/test/integration/plugin/tests/test-plugin-launch.hpp @@ -0,0 +1,89 @@ +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// +// Copyright (c) 2016-22, Lawrence Livermore National Security, LLC +// and RAJA project contributors. See the RAJA/LICENSE file for details. +// +// SPDX-License-Identifier: (BSD-3-Clause) +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// + +/// +/// Header file containing basic integration tests for plugins with launch. +/// + +#ifndef __TEST_PLUGIN_LAUNCH_HPP__ +#define __TEST_PLUGIN_LAUNCH_HPP__ + +#include "test-plugin.hpp" + + +// Check that the plugin is called with the right Platform. +// Check that the plugin is called the correct number of times, +// once before and after each launch capture for the capture counter, +// once before and after each launch invocation for the launch counter. + +// test with basic launch +template +void PluginLaunchTestImpl() +{ + SetupPluginVars spv(WORKING_RES::get_default()); + + CounterData* data = plugin_test_resource->allocate(10); + + for (int i = 0; i < 10; i++) { + + //Keep PluginTestCallable within a scope to ensure + //destruction, consistent with other test + { + PluginTestCallable p_callable{data}; + + RAJA::launch + (RAJA::LaunchParams(RAJA::Teams(1), RAJA::Threads(1)), + [=] RAJA_HOST_DEVICE(RAJA::LaunchContext RAJA_UNUSED_ARG(ctx)) + { + p_callable(i); + }); + } + + CounterData loop_data; + plugin_test_resource->memcpy(&loop_data, &data[i], sizeof(CounterData)); + ASSERT_EQ(loop_data.capture_platform_active, PLATFORM); + ASSERT_EQ(loop_data.capture_counter_pre, i+1); + ASSERT_EQ(loop_data.capture_counter_post, i); + ASSERT_EQ(loop_data.launch_platform_active, PLATFORM); + ASSERT_EQ(loop_data.launch_counter_pre, i+1); + ASSERT_EQ(loop_data.launch_counter_post, i); + } + + CounterData plugin_data; + plugin_test_resource->memcpy(&plugin_data, plugin_test_data, sizeof(CounterData)); + ASSERT_EQ(plugin_data.capture_platform_active, RAJA::Platform::undefined); + ASSERT_EQ(plugin_data.capture_counter_pre, 10); + ASSERT_EQ(plugin_data.capture_counter_post, 10); + ASSERT_EQ(plugin_data.launch_platform_active, RAJA::Platform::undefined); + ASSERT_EQ(plugin_data.launch_counter_pre, 10); + ASSERT_EQ(plugin_data.launch_counter_post, 10); + + plugin_test_resource->deallocate(data); +} + + +TYPED_TEST_SUITE_P(PluginLaunchTest); +template +class PluginLaunchTest : public ::testing::Test +{ +}; + +TYPED_TEST_P(PluginLaunchTest, PluginLaunch) +{ + using LaunchPolicy = typename camp::at>::type; + using ResType = typename camp::at>::type; + using PlatformHolder = typename camp::at>::type; + + PluginLaunchTestImpl( ); +} + +REGISTER_TYPED_TEST_SUITE_P(PluginLaunchTest, + PluginLaunch); + +#endif //__TEST_PLUGIN_LAUNCH_HPP__ diff --git a/test/integration/plugin/tests/test-plugin-resource-launch.hpp b/test/integration/plugin/tests/test-plugin-resource-launch.hpp new file mode 100644 index 0000000000..76e10a7118 --- /dev/null +++ b/test/integration/plugin/tests/test-plugin-resource-launch.hpp @@ -0,0 +1,91 @@ +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// +// Copyright (c) 2016-22, Lawrence Livermore National Security, LLC +// and RAJA project contributors. See the RAJA/LICENSE file for details. +// +// SPDX-License-Identifier: (BSD-3-Clause) +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// + +/// +/// Header file containing basic integration tests for plugins with launch. +/// + +#ifndef __TEST_PLUGIN_RESOURCE_LAUNCH_HPP__ +#define __TEST_PLUGIN_RESOURCE_LAUNCH_HPP__ + +#include "test-plugin.hpp" + + +// Check that the plugin is called with the right Platform. +// Check that the plugin is called the correct number of times, +// once before and after each launch capture for the capture counter, +// once before and after each launch invocation for the launch counter. + +// test with basic launch +template +void PluginResourceLaunchTestImpl() +{ + WORKING_RES res; + + SetupPluginVars spv(res); + + CounterData* data = plugin_test_resource->allocate(10); + + for (int i = 0; i < 10; i++) { + + //Keep PluginTestCallable within a scope to ensure + //destruction, consistent with other test + { + PluginTestCallable p_callable{data}; + + RAJA::launch + (res, RAJA::LaunchParams(RAJA::Teams(1), RAJA::Threads(1)), + [=] RAJA_HOST_DEVICE(RAJA::LaunchContext ctx) + { + p_callable(i); + }); + } + + CounterData loop_data; + plugin_test_resource->memcpy(&loop_data, &data[i], sizeof(CounterData)); + ASSERT_EQ(loop_data.capture_platform_active, PLATFORM); + ASSERT_EQ(loop_data.capture_counter_pre, i+1); + ASSERT_EQ(loop_data.capture_counter_post, i); + ASSERT_EQ(loop_data.launch_platform_active, PLATFORM); + ASSERT_EQ(loop_data.launch_counter_pre, i+1); + ASSERT_EQ(loop_data.launch_counter_post, i); + } + + CounterData plugin_data; + plugin_test_resource->memcpy(&plugin_data, plugin_test_data, sizeof(CounterData)); + ASSERT_EQ(plugin_data.capture_platform_active, RAJA::Platform::undefined); + ASSERT_EQ(plugin_data.capture_counter_pre, 10); + ASSERT_EQ(plugin_data.capture_counter_post, 10); + ASSERT_EQ(plugin_data.launch_platform_active, RAJA::Platform::undefined); + ASSERT_EQ(plugin_data.launch_counter_pre, 10); + ASSERT_EQ(plugin_data.launch_counter_post, 10); + + plugin_test_resource->deallocate(data); +} + + +TYPED_TEST_SUITE_P(PluginResourceLaunchTest); +template +class PluginResourceLaunchTest : public ::testing::Test +{ +}; + +TYPED_TEST_P(PluginResourceLaunchTest, PluginResourceLaunch) +{ + using LaunchPolicy = typename camp::at>::type; + using ResType = typename camp::at>::type; + using PlatformHolder = typename camp::at>::type; + + PluginResourceLaunchTestImpl( ); +} + +REGISTER_TYPED_TEST_SUITE_P(PluginResourceLaunchTest, + PluginResourceLaunch); + +#endif //__TEST_PLUGIN_LAUNCH_HPP__ diff --git a/test/unit/workgroup/CMakeLists.txt b/test/unit/workgroup/CMakeLists.txt index 0de12ba336..3cd18b7ef5 100644 --- a/test/unit/workgroup/CMakeLists.txt +++ b/test/unit/workgroup/CMakeLists.txt @@ -74,6 +74,10 @@ if(RAJA_TEST_EXHAUSTIVE OR NOT RAJA_COMPILER MATCHES "RAJA_COMPILER_Intel") endif() set(Dispatcher_SUBTESTS Single) +if(RAJA_ENABLE_TARGET_OPENMP) + # WorkGroup dispatcher for OpenMPTarget not implemented yet + list(REMOVE_ITEM BACKENDS OpenMPTarget) +endif() buildunitworkgrouptest(Dispatcher "${Dispatcher_SUBTESTS}" "${DISPATCHERS}" "${BACKENDS}") set(WorkStorage_SUBTESTS Constructor Iterator InsertCall Multiple) diff --git a/tpl/camp b/tpl/camp index 9a6b8216a9..a1c74aade4 160000 --- a/tpl/camp +++ b/tpl/camp @@ -1 +1 @@ -Subproject commit 9a6b8216a9c5f6d8f05a77fc1402fa7e91043d5c +Subproject commit a1c74aade443c6332e953e1ca5ad5e9e5b5baf21