Add HIP support

CMakeLists.txt

@@ -13,7 +13,7 @@
 
 # Mandatory call to project
 
-cmake_minimum_required (VERSION 3.10)
+cmake_minimum_required (VERSION 3.21)
 
 project(opm-simulators C CXX)
 
@@ -33,6 +33,19 @@ option(BUILD_FLOW_ALU_GRID "Build flow blackoil with alu grid" OFF)
 option(USE_DAMARIS_LIB "Use the Damaris library for asynchronous I/O?" OFF)
 option(USE_BDA_BRIDGE "Enable the BDA bridge (GPU/AMGCL solvers)" ON)
 option(USE_TRACY_PROFILER "Enable tracy profiling" OFF)
+option(CONVERT_CUDA_TO_HIP "Convert CUDA code to HIP (to run on AMD cards)" OFF)
+
+if (CONVERT_CUDA_TO_HIP)
+  enable_language(HIP)
+  message("CUDA code will be hipified")
+  set(HAVE_CUDA 1) # we still need this defined so that the preprocessor does not remove the code
+  set(CUDA_FOUND ON)
+  set(USE_HIP 1)
+  find_package(hip REQUIRED)
+  find_package(hipsparse REQUIRED)
+  find_package(hipblas REQUIRED)
+  link_libraries(roc::hipblas roc::hipsparse)
+endif()
 
 # The following was copied from CMakeLists.txt in opm-common.
 # TODO: factor out the common parts in opm-common and opm-simulator as a cmake module
@@ -166,6 +179,7 @@ if(NOT CMAKE_DISABLE_FIND_PACKAGE_CUDA AND
     # Hence we call it unconditionally
     # The WellContributions kernel uses __shfl_down_sync, which was introduced in CUDA 9.0
     find_package(CUDA)
+    set(CUDA_FOUND ON)
   endif()
   if(CUDA_FOUND AND CUDA_VERSION VERSION_LESS "9.0")
     set(CUDA_FOUND OFF)
@@ -174,9 +188,13 @@ if(NOT CMAKE_DISABLE_FIND_PACKAGE_CUDA AND
   endif()
 endif()
 
+find_package(CUDAToolkit)
 if(CUDA_FOUND)
   set(HAVE_CUDA 1)
-  include_directories(${CUDA_INCLUDE_DIRS})
+  if(NOT USE_HIP) # no need to include CUDA files if we use rocm stack
+    include_directories(${CUDA_INCLUDE_DIRS})
+    include_directories(${CUDAToolkit_INCLUDE_DIRS})
+  endif()
 endif()
 
 find_package(OpenCL)
@@ -311,7 +329,7 @@ macro (files_hook)
     set(HDF5_FOUND OFF)
     unset(HAVE_HDF5)
   endif()
-  if(HAVE_ROCSPARSE AND HAVE_CUDA)
+  if(HAVE_ROCSPARSE AND HAVE_CUDA AND USE_BDA_BRIDGE) # unsure if this is the correct way to change this
     message(WARNING "WARNING! Using CUDA and ROCm at the same time is not allowed. Please choose only one of them by setting CMAKE_DISABLE_FIND_PACKAGE_<rocsparse|CUDA>=<ON|OFF>. Disabling CUDA...\n")
     set(CUDA_FOUND OFF)
     unset(HAVE_CUDA)
@@ -719,13 +737,19 @@ add_custom_target(extra_test ${CMAKE_CTEST_COMMAND} -C ExtraTests)
 # must link libraries after target 'opmsimulators' has been defined
 
 if(CUDA_FOUND)
-  target_link_libraries( opmsimulators PUBLIC ${CUDA_cusparse_LIBRARY} )
-  target_link_libraries( opmsimulators PUBLIC ${CUDA_cublas_LIBRARY} )
+  if (NOT USE_HIP)
+    target_link_libraries( opmsimulators PUBLIC ${CUDA_cusparse_LIBRARY} )
+    target_link_libraries( opmsimulators PUBLIC ${CUDA_cublas_LIBRARY} )
+  endif()
   if(USE_BDA_BRIDGE)
     set_tests_properties(cusparseSolver PROPERTIES LABELS gpu_cuda)
   endif()
 
   # CUISTL
+  set(gpu_label "gpu_cuda")
+  if(USE_HIP)
+    set(gpu_label "gpu_hip")
+  endif()
   set_tests_properties(cusparse_safe_call
                        cublas_safe_call
                        cuda_safe_call
@@ -740,7 +764,7 @@ if(CUDA_FOUND)
                        cuseqilu0
                        cuowneroverlapcopy
                        solver_adapter
-                       PROPERTIES LABELS gpu_cuda)
+                       PROPERTIES LABELS ${gpu_label})
 endif()
 
 if(USE_BDA_BRIDGE)

CMakeLists_files.cmake

@@ -20,6 +20,39 @@
 #                       you should only add to this list if the *user* of
 #                       the library needs it.
 
+# This macro adds a cuda/hip source file to the correct source file list
+# it takes in the list to add it to, the path to the cuistl directory, and then
+# the rest of the file path after cuistl. The reason for splitting this into to
+# paths is to simplify replacing the cuistl part with hipistl.
+# Cuda files are added as they are, whereas hip files should be added after
+# hipification, we a dependency that will trigger when the cuda source code is
+# changed.
+macro (ADD_CUDA_OR_HIP_FILE LIST DIR FILE)
+  set (cuda_file_path "${PROJECT_SOURCE_DIR}/${DIR}/cuistl/${FILE}")
+
+  if(CUDA_FOUND AND NOT CONVERT_CUDA_TO_HIP)
+    list (APPEND ${LIST} "${DIR}/cuistl/${FILE}")
+  else()
+    # we must hipify the code
+    # and include the correct path which is in the build/binary dir
+    string(REPLACE ".cu" ".hip" HIP_SOURCE_FILE ${FILE})
+    set (hip_file_path "${PROJECT_BINARY_DIR}/${DIR}/hipistl/${HIP_SOURCE_FILE}")
+    file(RELATIVE_PATH relpath ${PROJECT_SOURCE_DIR} ${hip_file_path})
+    execute_process(COMMAND bash "${PROJECT_SOURCE_DIR}/bin/hipify_file.sh" ${cuda_file_path} ${hip_file_path})
+
+    # add a custom command that will hipify again if the cuda code it depends on changes
+    add_custom_command(
+        OUTPUT ${hip_file_path}
+        COMMAND bash "${PROJECT_SOURCE_DIR}/bin/hipify_file.sh" ${cuda_file_path} ${hip_file_path}
+        DEPENDS ${cuda_file_path}
+        COMMENT "Rehipifying because of change in ${cuda_file_path}"
+    )
+
+    # set_source_files_properties(${relpath} PROPERTIES LANGUAGE HIP)
+    list(APPEND ${LIST} ${relpath})
+  endif()
+endmacro()
+
 # originally generated with the command:
 # find opm -name '*.c*' -printf '\t%p\n' | sort
 list (APPEND MAIN_SOURCE_FILES
@@ -161,51 +194,51 @@ if (Damaris_FOUND AND MPI_FOUND AND USE_DAMARIS_LIB)
     opm/simulators/utils/initDamarisXmlFile.cpp
   )
 endif()
-if(CUDA_FOUND)
-  # CUISTL SOURCE
-  list (APPEND MAIN_SOURCE_FILES opm/simulators/linalg/cuistl/detail/CuBlasHandle.cpp)
-  list (APPEND MAIN_SOURCE_FILES opm/simulators/linalg/cuistl/detail/cusparse_matrix_operations.cu)
-  list (APPEND MAIN_SOURCE_FILES opm/simulators/linalg/cuistl/detail/CuSparseHandle.cpp)
-  list (APPEND MAIN_SOURCE_FILES opm/simulators/linalg/cuistl/CuVector.cpp)
-  list (APPEND MAIN_SOURCE_FILES opm/simulators/linalg/cuistl/detail/vector_operations.cu)
-  list (APPEND MAIN_SOURCE_FILES opm/simulators/linalg/cuistl/CuSparseMatrix.cpp)
-  list (APPEND MAIN_SOURCE_FILES opm/simulators/linalg/cuistl/CuDILU.cpp)
-  list (APPEND MAIN_SOURCE_FILES opm/simulators/linalg/cuistl/CuJac.cpp)
-  list (APPEND MAIN_SOURCE_FILES opm/simulators/linalg/cuistl/CuSeqILU0.cpp)
-  list (APPEND MAIN_SOURCE_FILES opm/simulators/linalg/cuistl/set_device.cpp)
 
-  # CUISTL HEADERS
-  list (APPEND PUBLIC_HEADER_FILES opm/simulators/linalg/cuistl/detail/cuda_safe_call.hpp)
-  list (APPEND PUBLIC_HEADER_FILES opm/simulators/linalg/cuistl/detail/cusparse_matrix_operations.hpp)
-  list (APPEND PUBLIC_HEADER_FILES opm/simulators/linalg/cuistl/detail/cusparse_safe_call.hpp)
-  list (APPEND PUBLIC_HEADER_FILES opm/simulators/linalg/cuistl/detail/cublas_safe_call.hpp)
-  list (APPEND PUBLIC_HEADER_FILES opm/simulators/linalg/cuistl/detail/cuda_check_last_error.hpp)
-  list (APPEND PUBLIC_HEADER_FILES opm/simulators/linalg/cuistl/detail/CuBlasHandle.hpp)
-  list (APPEND PUBLIC_HEADER_FILES opm/simulators/linalg/cuistl/detail/CuSparseHandle.hpp)
-  list (APPEND PUBLIC_HEADER_FILES opm/simulators/linalg/cuistl/CuDILU.hpp)
-  list (APPEND PUBLIC_HEADER_FILES opm/simulators/linalg/cuistl/CuJac.hpp)
-  list (APPEND PUBLIC_HEADER_FILES opm/simulators/linalg/cuistl/CuVector.hpp)
-  list (APPEND PUBLIC_HEADER_FILES opm/simulators/linalg/cuistl/CuSparseMatrix.hpp)
-  list (APPEND PUBLIC_HEADER_FILES opm/simulators/linalg/cuistl/detail/CuMatrixDescription.hpp)
-  list (APPEND PUBLIC_HEADER_FILES opm/simulators/linalg/cuistl/detail/CuSparseResource.hpp)
-  list (APPEND PUBLIC_HEADER_FILES opm/simulators/linalg/cuistl/detail/CuSparseResource_impl.hpp)
-  list (APPEND PUBLIC_HEADER_FILES opm/simulators/linalg/cuistl/detail/safe_conversion.hpp)
-  list (APPEND PUBLIC_HEADER_FILES opm/simulators/linalg/cuistl/detail/cublas_wrapper.hpp)
-  list (APPEND PUBLIC_HEADER_FILES opm/simulators/linalg/cuistl/detail/cusparse_wrapper.hpp)
-  list (APPEND PUBLIC_HEADER_FILES opm/simulators/linalg/cuistl/detail/cusparse_constants.hpp)
-  list (APPEND PUBLIC_HEADER_FILES opm/simulators/linalg/cuistl/detail/vector_operations.hpp)
-  list (APPEND PUBLIC_HEADER_FILES opm/simulators/linalg/cuistl/detail/has_function.hpp)
-  list (APPEND PUBLIC_HEADER_FILES opm/simulators/linalg/cuistl/detail/preconditioner_should_call_post_pre.hpp)
-  list (APPEND PUBLIC_HEADER_FILES opm/simulators/linalg/cuistl/PreconditionerAdapter.hpp)
-  list (APPEND PUBLIC_HEADER_FILES opm/simulators/linalg/cuistl/CuSeqILU0.hpp)
-  list (APPEND PUBLIC_HEADER_FILES opm/simulators/linalg/cuistl/detail/fix_zero_diagonal.hpp)
-  list (APPEND PUBLIC_HEADER_FILES opm/simulators/linalg/cuistl/PreconditionerConvertFieldTypeAdapter.hpp)
-  list (APPEND PUBLIC_HEADER_FILES opm/simulators/linalg/cuistl/CuOwnerOverlapCopy.hpp)
-  list (APPEND PUBLIC_HEADER_FILES opm/simulators/linalg/cuistl/SolverAdapter.hpp)
-  list (APPEND PUBLIC_HEADER_FILES opm/simulators/linalg/cuistl/CuBlockPreconditioner.hpp)
-  list (APPEND PUBLIC_HEADER_FILES opm/simulators/linalg/cuistl/PreconditionerHolder.hpp)
-  list (APPEND PUBLIC_HEADER_FILES opm/simulators/linalg/cuistl/set_device.hpp)
+# add these files if we should compile the hip code
+if (HAVE_CUDA)
+  ADD_CUDA_OR_HIP_FILE(MAIN_SOURCE_FILES opm/simulators/linalg detail/CuBlasHandle.cpp)
+  ADD_CUDA_OR_HIP_FILE(MAIN_SOURCE_FILES opm/simulators/linalg detail/cusparse_matrix_operations.cu)
+  ADD_CUDA_OR_HIP_FILE(MAIN_SOURCE_FILES opm/simulators/linalg detail/CuSparseHandle.cpp)
+  ADD_CUDA_OR_HIP_FILE(MAIN_SOURCE_FILES opm/simulators/linalg CuVector.cpp)
+  ADD_CUDA_OR_HIP_FILE(MAIN_SOURCE_FILES opm/simulators/linalg detail/vector_operations.cu)
+  ADD_CUDA_OR_HIP_FILE(MAIN_SOURCE_FILES opm/simulators/linalg CuSparseMatrix.cpp)
+  ADD_CUDA_OR_HIP_FILE(MAIN_SOURCE_FILES opm/simulators/linalg CuDILU.cpp)
+  ADD_CUDA_OR_HIP_FILE(MAIN_SOURCE_FILES opm/simulators/linalg CuJac.cpp)
+  ADD_CUDA_OR_HIP_FILE(MAIN_SOURCE_FILES opm/simulators/linalg CuSeqILU0.cpp)
+  ADD_CUDA_OR_HIP_FILE(MAIN_SOURCE_FILES opm/simulators/linalg set_device.cpp)
 
+  # HEADERS
+  ADD_CUDA_OR_HIP_FILE(PUBLIC_HEADER_FILES opm/simulators/linalg detail/cuda_safe_call.hpp)
+  ADD_CUDA_OR_HIP_FILE(PUBLIC_HEADER_FILES opm/simulators/linalg detail/cusparse_matrix_operations.hpp)
+  ADD_CUDA_OR_HIP_FILE(PUBLIC_HEADER_FILES opm/simulators/linalg detail/cusparse_safe_call.hpp)
+  ADD_CUDA_OR_HIP_FILE(PUBLIC_HEADER_FILES opm/simulators/linalg detail/cublas_safe_call.hpp)
+  ADD_CUDA_OR_HIP_FILE(PUBLIC_HEADER_FILES opm/simulators/linalg detail/cuda_check_last_error.hpp)
+  ADD_CUDA_OR_HIP_FILE(PUBLIC_HEADER_FILES opm/simulators/linalg detail/CuBlasHandle.hpp)
+  ADD_CUDA_OR_HIP_FILE(PUBLIC_HEADER_FILES opm/simulators/linalg detail/CuSparseHandle.hpp)
+  ADD_CUDA_OR_HIP_FILE(PUBLIC_HEADER_FILES opm/simulators/linalg CuDILU.hpp)
+  ADD_CUDA_OR_HIP_FILE(PUBLIC_HEADER_FILES opm/simulators/linalg CuJac.hpp)
+  ADD_CUDA_OR_HIP_FILE(PUBLIC_HEADER_FILES opm/simulators/linalg CuVector.hpp)
+  ADD_CUDA_OR_HIP_FILE(PUBLIC_HEADER_FILES opm/simulators/linalg CuSparseMatrix.hpp)
+  ADD_CUDA_OR_HIP_FILE(PUBLIC_HEADER_FILES opm/simulators/linalg detail/CuMatrixDescription.hpp)
+  ADD_CUDA_OR_HIP_FILE(PUBLIC_HEADER_FILES opm/simulators/linalg detail/CuSparseResource.hpp)
+  ADD_CUDA_OR_HIP_FILE(PUBLIC_HEADER_FILES opm/simulators/linalg detail/CuSparseResource_impl.hpp)
+  ADD_CUDA_OR_HIP_FILE(PUBLIC_HEADER_FILES opm/simulators/linalg detail/safe_conversion.hpp)
+  ADD_CUDA_OR_HIP_FILE(PUBLIC_HEADER_FILES opm/simulators/linalg detail/cublas_wrapper.hpp)
+  ADD_CUDA_OR_HIP_FILE(PUBLIC_HEADER_FILES opm/simulators/linalg detail/cusparse_wrapper.hpp)
+  ADD_CUDA_OR_HIP_FILE(PUBLIC_HEADER_FILES opm/simulators/linalg detail/cusparse_constants.hpp)
+  ADD_CUDA_OR_HIP_FILE(PUBLIC_HEADER_FILES opm/simulators/linalg detail/vector_operations.hpp)
+  ADD_CUDA_OR_HIP_FILE(PUBLIC_HEADER_FILES opm/simulators/linalg detail/has_function.hpp)
+  ADD_CUDA_OR_HIP_FILE(PUBLIC_HEADER_FILES opm/simulators/linalg detail/preconditioner_should_call_post_pre.hpp)
+  ADD_CUDA_OR_HIP_FILE(PUBLIC_HEADER_FILES opm/simulators/linalg PreconditionerAdapter.hpp)
+  ADD_CUDA_OR_HIP_FILE(PUBLIC_HEADER_FILES opm/simulators/linalg CuSeqILU0.hpp)
+  ADD_CUDA_OR_HIP_FILE(PUBLIC_HEADER_FILES opm/simulators/linalg detail/fix_zero_diagonal.hpp)
+  ADD_CUDA_OR_HIP_FILE(PUBLIC_HEADER_FILES opm/simulators/linalg PreconditionerConvertFieldTypeAdapter.hpp)
+  ADD_CUDA_OR_HIP_FILE(PUBLIC_HEADER_FILES opm/simulators/linalg CuOwnerOverlapCopy.hpp)
+  ADD_CUDA_OR_HIP_FILE(PUBLIC_HEADER_FILES opm/simulators/linalg SolverAdapter.hpp)
+  ADD_CUDA_OR_HIP_FILE(PUBLIC_HEADER_FILES opm/simulators/linalg CuBlockPreconditioner.hpp)
+  ADD_CUDA_OR_HIP_FILE(PUBLIC_HEADER_FILES opm/simulators/linalg PreconditionerHolder.hpp)
+  ADD_CUDA_OR_HIP_FILE(PUBLIC_HEADER_FILES opm/simulators/linalg set_device.hpp)
 endif()
 
 if(USE_BDA_BRIDGE)
@@ -312,24 +345,25 @@ if(CUDA_FOUND)
   if(USE_BDA_BRIDGE)
     list(APPEND TEST_SOURCE_FILES tests/test_cusparseSolver.cpp)
   endif()
-  list(APPEND TEST_SOURCE_FILES tests/cuistl/test_converttofloatadapter.cpp)
-  list(APPEND TEST_SOURCE_FILES tests/cuistl/test_cublas_handle.cpp)
-  list(APPEND TEST_SOURCE_FILES tests/cuistl/test_cublas_safe_call.cpp)
-  list(APPEND TEST_SOURCE_FILES tests/cuistl/test_cusparse_safe_call.cpp)
-  list(APPEND TEST_SOURCE_FILES tests/cuistl/test_cuda_safe_call.cpp)
-  list(APPEND TEST_SOURCE_FILES tests/cuistl/test_cuda_check_last_error.cpp)
-  list(APPEND TEST_SOURCE_FILES tests/cuistl/test_cujac.cpp)
-  list(APPEND TEST_SOURCE_FILES tests/cuistl/test_cuowneroverlapcopy.cpp)
-  list(APPEND TEST_SOURCE_FILES tests/cuistl/test_cuseqilu0.cpp)
-  list(APPEND TEST_SOURCE_FILES tests/cuistl/test_cusparse_handle.cpp)
-  list(APPEND TEST_SOURCE_FILES tests/cuistl/test_cuSparse_matrix_operations.cpp)
-  list(APPEND TEST_SOURCE_FILES tests/cuistl/test_cusparsematrix.cpp)
-  list(APPEND TEST_SOURCE_FILES tests/cuistl/test_cuvector.cpp)
-  list(APPEND TEST_SOURCE_FILES tests/cuistl/test_cuVector_operations.cpp)
-  list(APPEND TEST_SOURCE_FILES tests/cuistl/test_safe_conversion.cpp)
-  list(APPEND TEST_SOURCE_FILES tests/cuistl/test_solver_adapter.cpp)
-
+endif()
 
+if (HAVE_CUDA)
+  ADD_CUDA_OR_HIP_FILE(TEST_SOURCE_FILES tests test_converttofloatadapter.cpp)
+  ADD_CUDA_OR_HIP_FILE(TEST_SOURCE_FILES tests test_cublas_handle.cpp)
+  ADD_CUDA_OR_HIP_FILE(TEST_SOURCE_FILES tests test_cublas_safe_call.cpp)
+  ADD_CUDA_OR_HIP_FILE(TEST_SOURCE_FILES tests test_cusparse_safe_call.cpp)
+  ADD_CUDA_OR_HIP_FILE(TEST_SOURCE_FILES tests test_cuda_safe_call.cpp)
+  ADD_CUDA_OR_HIP_FILE(TEST_SOURCE_FILES tests test_cuda_check_last_error.cpp)
+  ADD_CUDA_OR_HIP_FILE(TEST_SOURCE_FILES tests test_cujac.cpp)
+  ADD_CUDA_OR_HIP_FILE(TEST_SOURCE_FILES tests test_cuowneroverlapcopy.cpp)
+  ADD_CUDA_OR_HIP_FILE(TEST_SOURCE_FILES tests test_cuseqilu0.cpp)
+  ADD_CUDA_OR_HIP_FILE(TEST_SOURCE_FILES tests test_cusparse_handle.cpp)
+  ADD_CUDA_OR_HIP_FILE(TEST_SOURCE_FILES tests test_cuSparse_matrix_operations.cpp)
+  ADD_CUDA_OR_HIP_FILE(TEST_SOURCE_FILES tests test_cusparsematrix.cpp)
+  ADD_CUDA_OR_HIP_FILE(TEST_SOURCE_FILES tests test_cuvector.cpp)
+  ADD_CUDA_OR_HIP_FILE(TEST_SOURCE_FILES tests test_cuVector_operations.cpp)
+  ADD_CUDA_OR_HIP_FILE(TEST_SOURCE_FILES tests test_safe_conversion.cpp)
+  ADD_CUDA_OR_HIP_FILE(TEST_SOURCE_FILES tests test_solver_adapter.cpp)
 endif()
 
 if(USE_BDA_BRIDGE)
@@ -490,30 +524,6 @@ list (APPEND PUBLIC_HEADER_FILES
   opm/simulators/aquifers/BlackoilAquiferModel.hpp
   opm/simulators/aquifers/BlackoilAquiferModel_impl.hpp
   opm/simulators/aquifers/SupportsFaceTag.hpp
-  opm/simulators/linalg/bda/amgclSolverBackend.hpp
-  opm/simulators/linalg/bda/BdaBridge.hpp
-  opm/simulators/linalg/bda/BdaResult.hpp
-  opm/simulators/linalg/bda/BdaSolver.hpp
-  opm/simulators/linalg/bda/opencl/BILU0.hpp
-  opm/simulators/linalg/bda/BlockedMatrix.hpp
-  opm/simulators/linalg/bda/opencl/CPR.hpp
-  opm/simulators/linalg/bda/cuda/cuda_header.hpp
-  opm/simulators/linalg/bda/cuda/cusparseSolverBackend.hpp
-  opm/simulators/linalg/bda/opencl/ChowPatelIlu.hpp
-  opm/simulators/linalg/bda/opencl/BISAI.hpp
-  opm/simulators/linalg/bda/Reorder.hpp
-  opm/simulators/linalg/bda/opencl/opencl.hpp
-  opm/simulators/linalg/bda/opencl/openclKernels.hpp
-  opm/simulators/linalg/bda/opencl/OpenclMatrix.hpp
-  opm/simulators/linalg/bda/opencl/Preconditioner.hpp
-  opm/simulators/linalg/bda/opencl/openclSolverBackend.hpp
-  opm/simulators/linalg/bda/opencl/openclWellContributions.hpp
-  opm/simulators/linalg/bda/Matrix.hpp
-  opm/simulators/linalg/bda/MultisegmentWellContribution.hpp
-  opm/simulators/linalg/bda/rocalutionSolverBackend.hpp
-  opm/simulators/linalg/bda/rocsparseSolverBackend.hpp
-  opm/simulators/linalg/bda/rocsparseWellContributions.hpp
-  opm/simulators/linalg/bda/WellContributions.hpp
   opm/simulators/linalg/amgcpr.hh
   opm/simulators/linalg/DILU.hpp
   opm/simulators/linalg/twolevelmethodcpr.hh
@@ -524,7 +534,6 @@ list (APPEND PUBLIC_HEADER_FILES
   opm/simulators/linalg/FlowLinearSolverParameters.hpp
   opm/simulators/linalg/GraphColoring.hpp
   opm/simulators/linalg/ISTLSolver.hpp
-  opm/simulators/linalg/ISTLSolverBda.hpp
   opm/simulators/linalg/MatrixMarketSpecializations.hpp
   opm/simulators/linalg/OwningBlockPreconditioner.hpp
   opm/simulators/linalg/OwningTwoLevelPreconditioner.hpp
@@ -635,6 +644,35 @@ list (APPEND PUBLIC_HEADER_FILES
   opm/simulators/wells/WellTest.hpp
   opm/simulators/wells/WGState.hpp
   )
+if (USE_BDA_BRIDGE)
+  list (APPEND PUBLIC_HEADER_FILES
+    opm/simulators/linalg/bda/amgclSolverBackend.hpp
+    opm/simulators/linalg/bda/BdaBridge.hpp
+    opm/simulators/linalg/bda/BdaResult.hpp
+    opm/simulators/linalg/bda/BdaSolver.hpp
+    opm/simulators/linalg/bda/opencl/BILU0.hpp
+    opm/simulators/linalg/bda/BlockedMatrix.hpp
+    opm/simulators/linalg/bda/opencl/CPR.hpp
+    opm/simulators/linalg/bda/cuda/cuda_header.hpp
+    opm/simulators/linalg/bda/cuda/cusparseSolverBackend.hpp
+    opm/simulators/linalg/bda/opencl/ChowPatelIlu.hpp
+    opm/simulators/linalg/bda/opencl/BISAI.hpp
+    opm/simulators/linalg/bda/Reorder.hpp
+    opm/simulators/linalg/bda/opencl/opencl.hpp
+    opm/simulators/linalg/bda/opencl/openclKernels.hpp
+    opm/simulators/linalg/bda/opencl/OpenclMatrix.hpp
+    opm/simulators/linalg/bda/opencl/Preconditioner.hpp
+    opm/simulators/linalg/bda/opencl/openclSolverBackend.hpp
+    opm/simulators/linalg/bda/opencl/openclWellContributions.hpp
+    opm/simulators/linalg/bda/Matrix.hpp
+    opm/simulators/linalg/bda/MultisegmentWellContribution.hpp
+    opm/simulators/linalg/bda/rocalutionSolverBackend.hpp
+    opm/simulators/linalg/bda/rocsparseSolverBackend.hpp
+    opm/simulators/linalg/bda/rocsparseWellContributions.hpp
+    opm/simulators/linalg/bda/WellContributions.hpp
+    opm/simulators/linalg/ISTLSolverBda.hpp
+  )
+endif()
 
 if (Damaris_FOUND AND MPI_FOUND AND USE_DAMARIS_LIB)
   list (APPEND PUBLIC_HEADER_FILES

bin/hipify_file.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# the script is intended to be run like this: bash hipify_file.sh ${PROJECT_BUILD_DIR} ${PROJECT_BINARY_DIR}
+# it should be run automatically on the correct files through cmake
+input_file=$1
+output_file=$2
+
+# make sure the output folder exists
+mkdir -p $(dirname $output_file)
+
+# hipify out-of-place
+hipify-perl $input_file > $output_file
+
+# expand includes so we only need include_directories (path to hip)
+sed -i 's/^#include <hipblas\.h>/#include <hipblas\/hipblas.h>/g' $output_file
+sed -i 's/^#include <hipsparse\.h>/#include <hipsparse\/hipsparse.h>/g' $output_file
+# make sure includes refer to hipistl/ files (the ones that are also hipified)
+sed -i 's/cuistl\//hipistl\//g' $output_file
+
+echo "$output_file hipified"

opm-simulators-prereqs.cmake

@@ -22,6 +22,7 @@ set (opm-simulators_CONFIG_VAR
   HAVE_SUITESPARSE_UMFPACK
   HAVE_DAMARIS
   HAVE_HDF5
+  USE_HIP
   USE_TRACY
   )
 

opm/simulators/linalg/FlexibleSolver_impl.hpp

@@ -38,8 +38,12 @@
 #include <dune/istl/paamg/pinfo.hh>
 
 #if HAVE_CUDA
+#if USE_HIP
+#include <opm/simulators/linalg/hipistl/SolverAdapter.hpp>
+#else
 #include <opm/simulators/linalg/cuistl/SolverAdapter.hpp>
 #endif
+#endif
 
 namespace Dune
 {