Merge pull request #1393 from LLNL/v2022.10.3-RC

V2022.10.3 rc

.gitlab/corona-build-and-test-extra.yml

@@ -23,6 +23,6 @@
 
 rocm_5_1_1_clang_13_0_0_desul_atomics:
   variables:
-    SPEC: " +rocm ~openmp +desul amdgpu_target=gfx906 %clang@13.0.0 ^hip@5.1.1 ^blt@develop"
+    SPEC: " ~shared +rocm ~openmp +tests +desul amdgpu_target=gfx906 %clang@13.0.0 ^hip@5.1.1 ^blt@develop"
   extends: .build_and_test_on_corona
 

.gitlab/custom-jobs-and-variables.yml

@@ -18,7 +18,7 @@ variables:
 # Arguments for job level allocation
   RUBY_BUILD_AND_TEST_JOB_ALLOC: "--time=45 --nodes=1"
 # Project specific variants for ruby
-  PROJECT_RUBY_VARIANTS: "+openmp "
+  PROJECT_RUBY_VARIANTS: "~shared +openmp +tests"
 # Project specific deps for ruby
   PROJECT_RUBY_DEPS: ""
 
@@ -28,7 +28,7 @@ variables:
 # Arguments for job level allocation
   CORONA_BUILD_AND_TEST_JOB_ALLOC: "--time-limit=45m --nodes=1"
 # Project specific variants for corona
-  PROJECT_CORONA_VARIANTS: "~openmp "
+  PROJECT_CORONA_VARIANTS: "~shared ~openmp +tests"
 # Project specific deps for corona
   PROJECT_CORONA_DEPS: "^blt@develop "
 
@@ -37,7 +37,7 @@ variables:
 # Arguments for job level allocation
   LASSEN_BUILD_AND_TEST_JOB_ALLOC: "1 -W 60"
 # Project specific variants for lassen
-  PROJECT_LASSEN_VARIANTS: "+openmp "
+  PROJECT_LASSEN_VARIANTS: "~shared +openmp +tests"
 # Project specific deps for lassen
   PROJECT_LASSEN_DEPS: ""
 

.gitlab/lassen-build-and-test-extra.yml

@@ -102,7 +102,7 @@ xl_16_1_1_12_gcc_8_3_1_cuda_11_1_0:
 
 clang_14_0_5:
   variables:
-    SPEC: " +openmp %clang@14.0.5"
+    SPEC: " ~shared +openmp +tests %clang@14.0.5"
   extends: .build_and_test_on_lassen
 
 ##########
@@ -111,18 +111,18 @@ clang_14_0_5:
 
 clang_12_0_1_cuda_11_5_0:
   variables:
-    SPEC: " +openmp +cuda cuda_arch=70 %clang@12.0.1 ^cuda@11.5.0"
+    SPEC: " ~shared +openmp +tests +cuda cuda_arch=70 %clang@12.0.1 ^cuda@11.5.0"
   extends: .build_and_test_on_lassen
 
 gcc_8_3_1_cuda_11_1_0:
   variables:
-    SPEC: " +openmp +cuda %gcc@8.3.1 cuda_arch=70 ^cuda@11.1.0"
+    SPEC: " ~shared +openmp +tests +cuda %gcc@8.3.1 cuda_arch=70 ^cuda@11.1.0"
   extends: .build_and_test_on_lassen
 
 gcc_8_3_1_cuda_11_5_0_ats_disabled:
   extends: .build_and_test_on_lassen
   variables:
-    SPEC: " +openmp +cuda %gcc@8.3.1 cuda_arch=70 ^cuda@11.5.0"
+    SPEC: " ~shared +openmp +tests +cuda %gcc@8.3.1 cuda_arch=70 ^cuda@11.5.0"
     LASSEN_BUILD_AND_TEST_JOB_ALLOC: "1 --atsdisable -W 60"
 
 ##########
@@ -131,16 +131,16 @@ gcc_8_3_1_cuda_11_5_0_ats_disabled:
 
 clang_13_0_1_libcpp:
   variables:
-    SPEC: " +openmp %clang@13.0.1+libcpp"
+    SPEC: " ~shared +openmp +tests %clang@13.0.1+libcpp"
   extends: .build_and_test_on_lassen
 
 clang_14_0_5_asan:
   variables:
-    SPEC: " +openmp %clang@14.0.5 cxxflags=-fsanitize=address"
+    SPEC: " ~shared +openmp +tests %clang@14.0.5 cxxflags=-fsanitize=address"
     ASAN_OPTIONS: "detect_leaks=1"
   extends: .build_and_test_on_lassen
 
 gcc_8_3_1_cuda_10_1_168_desul_atomics:
   variables:
-    SPEC: " +openmp +cuda +desul %gcc@8.3.1 cuda_arch=70 ^cuda@10.1.168"
+    SPEC: " ~shared +openmp +tests +cuda +desul %gcc@8.3.1 cuda_arch=70 ^cuda@10.1.168"
   extends: .build_and_test_on_lassen

.gitlab/ruby-build-and-test-extra.yml

@@ -35,24 +35,24 @@ pgi_20_1_gcc_local_8_3_1:
 
 clang_9_0_0_openmp_off:
   variables:
-    SPEC: " ~openmp %clang@9.0.0"
+    SPEC: " ~shared ~openmp +tests %clang@9.0.0"
   extends: .build_and_test_on_ruby
 
 gcc_8_1_0_openmp_default:
   variables:
-    SPEC: " %gcc@8.1.0"
+    SPEC: " ~shared +tests %gcc@8.1.0"
     RUBY_BUILD_AND_TEST_JOB_ALLOC: "--time=60 --nodes=1"
   extends: .build_and_test_on_ruby
 
 icpc_19_1_0:
   variables:
-    SPEC: " +openmp %intel@19.1.0"
+    SPEC: " ~shared +openmp +tests %intel@19.1.0"
     RUBY_BUILD_AND_TEST_JOB_ALLOC: "--time=40 --nodes=1"
   extends: .build_and_test_on_ruby
 
 # OTHERS
 clang_10_0_1_gcc_8_3_1_desul_atomics:
   variables:
-    SPEC: " +openmp +desul %clang@10.0.1 cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1"
+    SPEC: " ~shared +openmp +tests +desul %clang@10.0.1 cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1"
   extends: .build_and_test_on_ruby
 

.uberenv_config.json

@@ -1,7 +1,7 @@
 {
 "package_name" : "raja",
 "package_version" : "develop",
-"package_final_phase" : "hostconfig",
+"package_final_phase" : "initconfig",
 "package_source_dir" : "../..",
 "spack_url": "https://github.com/spack/spack.git",
 "spack_branch": "v0.18.1",

CMakeLists.txt

@@ -16,7 +16,7 @@ include(CMakeDependentOption)
 # Set version number
 set(RAJA_VERSION_MAJOR 2022)
 set(RAJA_VERSION_MINOR 10)
-set(RAJA_VERSION_PATCHLEVEL 2)
+set(RAJA_VERSION_PATCHLEVEL 3)
 
 if (RAJA_LOADED AND (NOT RAJA_LOADED STREQUAL "${RAJA_VERSION_MAJOR}.${RAJA_VERSION_MINOR}.${RAJA_VERSION_PATCHLEVEL}"))
   message(FATAL_ERROR "You are mixing RAJA versions. Loaded is ${RAJA_LOADED}, expected ${RAJA_VERSION_MAJOR}.${RAJA_VERSION_MINOR}.${RAJA_VERSION_PATCHLEVEL}")

RELEASE_NOTES.md

@@ -20,6 +20,37 @@ Notable changes include:
   * Bug fixes/improvements:
 
 
+Version 2022.10.3 -- Release date 2022-12-01
+============================================
+
+This release fixes a few issues that were found after the v2022.10.2 release.
+
+Notable changes include:
+
+  * Update camp submodule to v2022.10.1
+  * Update BLT submodule to commit 8c229991 (includes fixes for crayftn + hip)
+
+  * Properly export 'roctx' target when CMake variable RAJA_ENABLE_ROCTX is on. 
+  * Fix CMake logic for exporting desul targets when desul atomics are enabled.
+  * Fix the way we use CMake to find the rocPRIM module to follow CMake
+    best practices.
+  * Add missing template parameter pack argument in RAJA::statement::For
+    execution policy construct used in RAJA::kernel implementation for OpenMP 
+    target back-end.
+  * Change to use compile-time GPU thread block size in RAJA::forall 
+    implementation. This improves performance of GPU kernels, especially 
+    those using the RAJA HIP back-end.
+  * Added RAJA plugin support, including CHAI support, for RAJA::launch.
+  * Replaced 'DEVICE' macro with alias to 'device_mem_pool_t' to prevent name 
+    conflicts with other libraries.
+  * Updated User Guide documentation about CMake variable used to pass 
+    compiler flags for OpenMP target back-end. This changed with CMake
+    minimum required version bump in v2022.10.0.
+  * Adjust ordering of BLT and camp target inclusion in RAJA CMake usage to 
+    fix an issue with projects using external camp vs. RAJA submodule.
+
+
+
 Version 2022.10.2 -- Release date 2022-11-08
 ============================================
 

cmake/SetupPackages.cmake

@@ -82,11 +82,11 @@ if (RAJA_ENABLE_HIP)
   endif()
 
   if (RAJA_ENABLE_EXTERNAL_ROCPRIM)
-    include(cmake/thirdparty/FindRocPRIM.cmake)
-    if (ROCPRIM_FOUND)
+    find_package(rocPRIM)
+    if (rocPRIM_FOUND)
       blt_import_library(
         NAME rocPRIM
-        INCLUDES ${ROCPRIM_INCLUDE_DIRS}
+        INCLUDES ${rocPRIM_INCLUDE_DIRS}
         TREAT_INCLUDES_AS_SYSTEM ON
         EXPORTABLE ON)
     else()
@@ -101,13 +101,16 @@ if (RAJA_ENABLE_HIP AND RAJA_ENABLE_ROCTX)
   include(FindRoctracer)
   blt_import_library(NAME roctx
                      INCLUDES ${ROCTX_INCLUDE_DIRS}
-                     LIBRARIES ${ROCTX_LIBRARIES})
+                     LIBRARIES ${ROCTX_LIBRARIES}
+                     EXPORTABLE ON
+                     TREAT_INCLUDES_AS_SYSTEM ON)
 endif ()
 
 set(TPL_DEPS)
 blt_list_append(TO TPL_DEPS ELEMENTS nvtoolsext IF RAJA_ENABLE_NV_TOOLS_EXT)
 blt_list_append(TO TPL_DEPS ELEMENTS cub IF RAJA_ENABLE_EXTERNAL_CUB)
 blt_list_append(TO TPL_DEPS ELEMENTS rocPRIM IF RAJA_ENABLE_EXTERNAL_ROCPRIM)
+blt_list_append(TO TPL_DEPS ELEMENTS roctx IF RAJA_ENABLE_ROCTX)
 
 set(RAJA_NEEDS_BLT_TPLS False)
 if (RAJA_ENABLE_CUDA OR RAJA_ENABLE_HIP OR RAJA_ENABLE_OPENMP OR RAJA_ENABLE_MPI)

cmake/thirdparty/FindrocPRIM.cmake

@@ -0,0 +1,54 @@
+###############################################################################
+# Copyright (c) 2016-22, Lawrence Livermore National Security, LLC
+# and other RAJA project contributors. See the RAJA/LICENSE file for details.
+#
+# SPDX-License-Identifier: (BSD-3-Clause)
+###############################################################################
+
+#[=======================================================================[.rst:
+
+FindrocPRIM
+-------
+
+Finds the rocPRIM package.
+
+Result Variables
+^^^^^^^^^^^^^^^^
+
+This will define the following variables:
+
+``rocPRIM_FOUND``
+True if the system has the rocPRIM library.
+``rocPRIM_INCLUDE_DIRS``
+Include directories needed to use rocPRIM.
+
+Cache Variables
+^^^^^^^^^^^^^^^
+
+The following cache variables may also be set:
+
+``rocPRIM_INCLUDE_DIR``
+The directory containing ``rocprim.hpp``.
+
+#]=======================================================================]
+
+include (FindPackageHandleStandardArgs)
+
+find_path(rocPRIM_INCLUDE_DIR
+  NAMES rocprim/rocprim.hpp
+  HINTS
+    ${ROCPRIM_DIR}/
+    ${HIP_ROOT_DIR}/../
+  PATH_SUFFIXES
+    include
+    rocprim
+    rocprim/include)
+
+find_package_handle_standard_args(
+  rocPRIM
+  DEFAULT_MSG
+  rocPRIM_INCLUDE_DIR)
+
+if (rocPRIM_FOUND)
+  set(rocPRIM_INCLUDE_DIRS ${rocPRIM_INCLUDE_DIR})
+endif()

docs/conf.py

@@ -88,7 +88,7 @@
 # The short X.Y version.
 version = u'2022.10'
 # The full version, including alpha/beta/rc tags.
-release = u'2022.10.2'
+release = u'2022.10.3'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/sphinx/user_guide/getting_started.rst

@@ -323,16 +323,19 @@ OpenMP
 ^^^^^^^
 
 To use OpenMP target offload GPU execution, additional options may need to be
-passed to the compiler. The variable ``OpenMP_CXX_FLAGS`` is used for this.
-Option syntax follows the CMake *list* pattern. For example, to specify OpenMP 
-target options for NVIDIA GPUs using a clang-based compiler, one may do
-something like::
+passed to the compiler. BLT variables are used for this. Option syntax follows 
+the CMake *list* pattern. For example, to specify OpenMP target options for 
+NVIDIA GPUs using a clang-based compiler, one may do something like::
 
    cmake \
      ... \
-     -DOpenMP_CXX_FLAGS="-fopenmp;-fopenmp-targets=nvptx64-nvidia-cuda" \
+     -DBLT_OPENMP_COMPILE_FLAGS="-fopenmp;-fopenmp-targets=nvptx64-nvidia-cuda" \
+     -DBLT_OPENMP_LINK_FLAGS="-fopenmp;-fopenmp-targets=nvptx64-nvidia-cuda" \
      ...
 
+Compiler flags are passed to other compilers similarly, using flags specific to
+the compiler. Typically, the compile and link flags are the same as shown here.
+
 ----------------------------------------
 RAJA Example Build Configuration Files
 ----------------------------------------

host-configs/lc-builds/toss4/gcc_X.cmake

@@ -0,0 +1,16 @@
+###############################################################################
+# Copyright (c) 2016-22, Lawrence Livermore National Security, LLC
+# and RAJA project contributors. See the RAJA/LICENSE file for details.
+#
+# SPDX-License-Identifier: (BSD-3-Clause)
+###############################################################################
+
+set(RAJA_COMPILER "RAJA_COMPILER_GNU" CACHE STRING "")
+
+set(CMAKE_CXX_FLAGS_RELEASE "-Ofast -march=native" CACHE STRING "")
+set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-Ofast -march=native -g" CACHE STRING "")
+set(CMAKE_CXX_FLAGS_DEBUG "-O0 -g" CACHE STRING "")
+
+set(RAJA_DATA_ALIGN 64 CACHE STRING "")
+
+set(RAJA_HOST_CONFIG_LOADED On CACHE BOOL "")

host-configs/lc-builds/toss4/icpc_X.cmake

@@ -0,0 +1,16 @@
+###############################################################################
+# Copyright (c) 2016-22, Lawrence Livermore National Security, LLC
+# and RAJA project contributors. See the RAJA/LICENSE file for details.
+#
+# SPDX-License-Identifier: (BSD-3-Clause)
+###############################################################################
+
+set(RAJA_COMPILER "RAJA_COMPILER_ICC" CACHE STRING "")
+
+set(CMAKE_CXX_FLAGS_RELEASE "-O3 -march=native -ansi-alias -diag-disable cpu-dispatch" CACHE STRING "")
+set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O3 -g -march=native -ansi-alias -diag-disable cpu-dispatch" CACHE STRING "")
+set(CMAKE_CXX_FLAGS_DEBUG "-O0 -g" CACHE STRING "")
+
+set(RAJA_DATA_ALIGN 64 CACHE STRING "")
+
+set(RAJA_HOST_CONFIG_LOADED On CACHE BOOL "")

host-configs/lc-builds/toss4/icpx_X.cmake

@@ -0,0 +1,16 @@
+###############################################################################
+# Copyright (c) 2016-22, Lawrence Livermore National Security, LLC
+# and RAJA project contributors. See the RAJA/LICENSE file for details.
+#
+# SPDX-License-Identifier: (BSD-3-Clause)
+###############################################################################
+
+set(RAJA_COMPILER "RAJA_COMPILER_ICC" CACHE STRING "")
+
+set(CMAKE_CXX_FLAGS_RELEASE "-O3 -march=native" CACHE STRING "")
+set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O3 -g -march=native" CACHE STRING "")
+set(CMAKE_CXX_FLAGS_DEBUG "-O0 -g" CACHE STRING "")
+
+set(RAJA_DATA_ALIGN 64 CACHE STRING "")
+
+set(RAJA_HOST_CONFIG_LOADED On CACHE BOOL "")