Skip to content

Commit

Permalink
Merge branch 'master' into sharedmem_3d_y_blitz_storage_order_mpi_by_…
Browse files Browse the repository at this point in the history
…two_threads
  • Loading branch information
pdziekan committed Nov 18, 2021
2 parents 3964132 + 990a884 commit 4bf8ef7
Show file tree
Hide file tree
Showing 38 changed files with 332 additions and 86 deletions.
215 changes: 215 additions & 0 deletions .github/workflows/test_libmpdataxx.yml
@@ -0,0 +1,215 @@
name: Test libmpdata++

on:
push:
branches: [ master ]
pull_request:
branches: [ master ]

jobs:
# run unit tests
unit:
runs-on: ubuntu-20.04
strategy:
matrix:
build_type: ["Debug", "Release"]
mpi: ["none", "mvapich2"]
include:
- mpi: "none"
tag: "ubuntu_20_04_cuda_11_4"
cxx: "g++"
ctest_options: ""
- mpi: "mvapich2"
tag: "ubuntu_20_04_cuda_11_4_mvapich2_v1"
cxx: "mpic++"
ctest_options: "-LE SlowWithMpi" # exclude tests with the label SlowWithMpi
steps:
- uses: actions/checkout@v2

- name: Install libmpdata++
uses: igfuw/libmpdataxx_install@v0.1
with:
build_type: ${{matrix.build_type}}
threads: ${{matrix.threads}}
path: ${{ github.workspace }}/libmpdata++
install_prefix: ${{ github.workspace }}/installed
tag: ${{ matrix.tag }}
cxx: ${{ matrix.cxx }}

- run: mkdir ${{ github.workspace }}/tests/unit/build

- name: configure,make and run unit tests #all in one call, because singularity always creates a sandbox, what takes ca. 1 min. TODO: fix this (newer singularity from apt?)
working-directory: ${{ github.workspace }}/tests/unit
run: VERBOSE=1 OMP_NUM_THREADS=4 singularity exec $SI bash -c "cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -DCMAKE_CXX_COMPILER=${{matrix.cxx}} && cmake --build build --config ${{matrix.build_type}} -j4 && cd build && ctest ${{matrix.ctest_options}} || cat Testing/Temporary/LastTest.log /"

# Alternative command, but it does not run tests. Need to specify --test-command, but to what?
# - name: configure,make and run unit tests #all in one call, because singularity always creates a sandbox, what takes ca. 1 min. TODO: fix this (newer singularity from apt?)
# working-directory: ${{ github.workspace }}/tests/unit/build
# run: VERBOSE=1 OMP_NUM_THREADS=4 singularity exec $SI ctest --build-and-test .. . --build-generator "Unix Makefiles" -LE SlowWithMpi --build-options "-DCMAKE_CXX_COMPILER=${{matrix.cxx}}" "-DCMAKE_BUILD_TYPE=${{matrix.build_type}}" || cat Testing/Temporary/LastTest.log /

sandbox:
runs-on: ubuntu-20.04
strategy:
matrix:
build_type: ["Release"] # Debug would be too slow
mpi: ["none", "mvapich2"]
include:
- mpi: "none"
tag: "ubuntu_20_04_cuda_11_4"
cxx: "g++"
ctest_options: "-LE NotInCI"
- mpi: "mvapich2"
tag: "ubuntu_20_04_cuda_11_4_mvapich2_v1"
cxx: "mpic++"
ctest_options: "-LE 'NotInCI|SlowWithMpi'" # exclude tests with either label

steps:
- uses: actions/checkout@v2

- name: Install libmpdata++
uses: igfuw/libmpdataxx_install@v0.1
with:
build_type: ${{matrix.build_type}}
threads: ${{matrix.threads}}
path: ${{ github.workspace }}/libmpdata++
install_prefix: ${{ github.workspace }}/installed
tag: ${{ matrix.tag }}
cxx: ${{ matrix.cxx }}

- run: mkdir ${{ github.workspace }}/tests/sandbox/build

- name: configure,make and run selected sandbox tests
working-directory: ${{ github.workspace }}/tests/sandbox
run: VERBOSE=1 OMP_NUM_THREADS=4 singularity exec $SI bash -c "cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -DCMAKE_CXX_COMPILER=${{matrix.cxx}} && cmake --build build --config ${{matrix.build_type}} -j4 && cd build && ctest ${{matrix.ctest_options}} || cat Testing/Temporary/LastTest.log /"

sandbox_slow_tests:
runs-on: ubuntu-20.04
strategy:
matrix:
build_type: ["Release"] # Debug would be too slow
mpi: ["mvapich2"]
tests: ["pbl_smg_short", "pbl_iles_short", "shear_layer", "convergence_adv_diffusion"]
include:
- mpi: "mvapich2"
tag: "ubuntu_20_04_cuda_11_4_mvapich2_v1"
cxx: "mpic++"

steps:
- uses: actions/checkout@v2

- name: Install libmpdata++
uses: igfuw/libmpdataxx_install@v0.1
with:
build_type: ${{matrix.build_type}}
threads: ${{matrix.threads}}
path: ${{ github.workspace }}/libmpdata++
install_prefix: ${{ github.workspace }}/installed
tag: ${{ matrix.tag }}
cxx: ${{ matrix.cxx }}

- run: mkdir ${{ github.workspace }}/tests/sandbox/build

- name: configure,make and run selected sandbox tests
working-directory: ${{ github.workspace }}/tests/sandbox
run: VERBOSE=1 OMP_NUM_THREADS=4 singularity exec $SI bash -c "cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -DCMAKE_CXX_COMPILER=${{matrix.cxx}} && cmake --build build --config ${{matrix.build_type}} -j4 && cd build && ctest -R ${{matrix.tests}} || cat Testing/Temporary/LastTest.log /"

# run tests from the 2015 GMD paper
paper:
runs-on: ubuntu-20.04
strategy:
matrix:
build_type: ["Release"]
mpi: ["none", "mvapich2"]
include:
- mpi: "none"
tag: "ubuntu_20_04_cuda_11_4"
cxx: "g++"
- mpi: "mvapich2"
tag: "ubuntu_20_04_cuda_11_4_mvapich2_v1"
cxx: "mpic++"
ctest_options: "-LE SlowWithMpi" # exclude tests with the label SlowWithMpi

steps:
- uses: actions/checkout@v2

- name: Install libmpdata++
uses: igfuw/libmpdataxx_install@v0.1
with:
build_type: ${{matrix.build_type}}
threads: ${{matrix.threads}}
path: ${{ github.workspace }}/libmpdata++
install_prefix: ${{ github.workspace }}/installed
tag: ${{ matrix.tag }}
cxx: ${{ matrix.cxx }}

- run: mkdir ${{ github.workspace }}/tests/paper_2015_GMD/build

- name: configure,make and run selected sandbox tests
working-directory: ${{ github.workspace }}/tests/paper_2015_GMD
run: VERBOSE=1 OMP_NUM_THREADS=4 singularity exec $SI bash -c "cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -DCMAKE_CXX_COMPILER=${{matrix.cxx}} && cmake --build build --config ${{matrix.build_type}} -j4 && cd build && ctest ${{matrix.ctest_options}} || cat Testing/Temporary/LastTest.log /"

nair_jablonowski_2008:
runs-on: ubuntu-20.04
strategy:
matrix:
build_type: ["Release"]
mpi: ["none"]
include:
- mpi: "none"
tag: "ubuntu_20_04_cuda_11_4"
cxx: "g++"

steps:
- uses: actions/checkout@v2

- name: Install libmpdata++
uses: igfuw/libmpdataxx_install@v0.1
with:
build_type: ${{matrix.build_type}}
threads: ${{matrix.threads}}
path: ${{ github.workspace }}/libmpdata++
install_prefix: ${{ github.workspace }}/installed
tag: ${{ matrix.tag }}
cxx: ${{ matrix.cxx }}

- run: mkdir ${{ github.workspace }}/tests/nair_jablonowski_2008/build

- name: configure,make and run selected sandbox tests
working-directory: ${{ github.workspace }}/tests/nair_jablonowski_2008
run: VERBOSE=1 OMP_NUM_THREADS=4 singularity exec $SI bash -c "cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -DCMAKE_CXX_COMPILER=${{matrix.cxx}} && cmake --build build --config ${{matrix.build_type}} -j4 && cd build && ctest || cat Testing/Temporary/LastTest.log /"

elliptic_drop:
runs-on: ubuntu-20.04
strategy:
matrix:
build_type: ["Release"]
mpi: ["none"]
include:
- mpi: "none"
tag: "ubuntu_20_04_cuda_11_4"
cxx: "g++"

steps:
- uses: actions/checkout@v2

- name: Install libmpdata++
uses: igfuw/libmpdataxx_install@v0.1
with:
build_type: ${{matrix.build_type}}
threads: ${{matrix.threads}}
path: ${{ github.workspace }}/libmpdata++
install_prefix: ${{ github.workspace }}/installed
tag: ${{ matrix.tag }}
cxx: ${{ matrix.cxx }}

- name: checkout elliptic_drop
uses: actions/checkout@v2
with:
repository: igfuw/shallow-water-elliptic-drop
path: shallow-water-elliptic-drop

- run: mkdir ${{ github.workspace }}/shallow-water-elliptic-drop/numerical/build

- name: configure,make and run tests
working-directory: ${{ github.workspace }}/shallow-water-elliptic-drop/numerical
run: VERBOSE=1 OMP_NUM_THREADS=4 singularity exec -B ${{ github.workspace }}/installed/ $SI bash -c "cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -DCMAKE_CXX_COMPILER=${{matrix.cxx}} -Dlibmpdata++_DIR=${{ github.workspace }}/installed/share/libmpdata++ && cmake --build build --config ${{matrix.build_type}} -j4 && cd build && ctest || cat Testing/Temporary/LastTest.log /"
49 changes: 32 additions & 17 deletions libmpdata++-config.cmake
Expand Up @@ -7,6 +7,8 @@ else()
cmake_minimum_required(VERSION 2.8.8)
endif()

cmake_minimum_required(VERSION 3.14) # for setting output variable of try_compile

# the policies we care about:
# - CMP0025 - make CMake distinguis between Apple and LLVM clang
# - CMP0042 - make CMake use RPATHs on OSX
Expand Down Expand Up @@ -216,7 +218,7 @@ if(HDF5_FOUND)
set(msg "Checking if MPI-HDF5 is usable from C++...")
set(pfx "HDF5/MPI/C++ check")
message(STATUS ${msg})
execute_process(COMMAND "mktemp" "-d" RESULT_VARIABLE status OUTPUT_VARIABLE tmpdir)
execute_process(COMMAND "mktemp" "-d" RESULT_VARIABLE status OUTPUT_VARIABLE tmpdir OUTPUT_STRIP_TRAILING_WHITESPACE)
if (NOT status EQUAL 0)
message(FATAL_ERROR "${pfx}: mkdtemp failed")
endif()
Expand All @@ -229,27 +231,40 @@ if(HDF5_FOUND)
int main()
{
boost::mpi::environment e;
hid_t plist_id = H5Pcreate(H5P_FILE_ACCESS);
H5Pset_fapl_mpio(plist_id, MPI_COMM_WORLD, MPI_INFO_NULL);
H5::H5File(\"test.h5\", H5F_ACC_TRUNC, H5P_DEFAULT, plist_id);
H5Pclose(plist_id);
hid_t plist_id = H5Pcreate(H5P_FILE_ACCESS);
H5Pset_fapl_mpio(plist_id, MPI_COMM_WORLD, MPI_INFO_NULL);
H5::H5File(\"test.h5\", H5F_ACC_TRUNC, H5P_DEFAULT, plist_id);
H5Pclose(plist_id);
}
")

execute_process(
COMMAND "${CMAKE_CXX_COMPILER}" "test.cpp" "-I${Boost_INCLUDE_DIRS}" "-I${HDF5_INCLUDE_DIRS}" ${HDF5_LIBRARIES} ${Boost_LIBRARIES}# the order of HDF/Boost matters here!
# Boost_LIBRARY_DIRS has to be in LD_RUN_PATH, tried to specify it through rpath/rpath-link but failed; at runtime it correctly finds libboost-mpi (direct dependency), but fails on boost-serialization (which is a dependency of boost-mpi)
#COMMAND "${CMAKE_CXX_COMPILER}" "test.cpp" "-I${Boost_INCLUDE_DIRS}" "-Wl,-rpath,${Boost_LIBRARY_DIRS},-rpath-link,${Boost_LIBRARY_DIRS}" ${HDF5_LIBRARIES} ${Boost_LIBRARIES}# the order of HDF/Boost matters here!
WORKING_DIRECTORY ${tmpdir}
RESULT_VARIABLE status
ERROR_VARIABLE error
)
if (NOT status EQUAL 0)
message(FATAL_ERROR "${pfx}: compilation failed\n ${error}")
# message("${CMAKE_CXX_COMPILER}" "test.cpp" "-I${Boost_INCLUDE_DIRS}" "-I${HDF5_INCLUDE_DIRS}" ${HDF5_LIBRARIES} ${Boost_LIBRARIES})

# execute_process(
# COMMAND "${CMAKE_CXX_COMPILER}" "test.cpp" "-I${Boost_INCLUDE_DIRS}" "-I${HDF5_INCLUDE_DIRS}" ${HDF5_LIBRARIES} ${Boost_LIBRARIES}# the order of HDF/Boost matters here!
# # Boost_LIBRARY_DIRS has to be in LD_RUN_PATH, tried to specify it through rpath/rpath-link but failed; at runtime it correctly finds libboost-mpi (direct dependency), but fails on boost-serialization (which is a dependency of boost-mpi)
# #COMMAND "${CMAKE_CXX_COMPILER}" "test.cpp" "-I${Boost_INCLUDE_DIRS}" "-Wl,-rpath,${Boost_LIBRARY_DIRS},-rpath-link,${Boost_LIBRARY_DIRS}" ${HDF5_LIBRARIES} ${Boost_LIBRARIES}# the order of HDF/Boost matters here!
# WORKING_DIRECTORY ${tmpdir}
# RESULT_VARIABLE status
# ERROR_VARIABLE error
# )

try_compile(status ${CMAKE_BINARY_DIR} "${tmpdir}/test.cpp"
OUTPUT_VARIABLE error
CMAKE_FLAGS INCLUDE_DIRECTORIES ${Boost_INCLUDE_DIRS},{HDF5_INCLUDE_DIRS}
LINK_LIBRARIES ${Boost_LIBRARIES}
LINK_LIBRARIES ${HDF5_LIBRARIES}
COPY_FILE ${tmpdir}/test_hdf5_mpi
COPY_FILE_ERROR copy_error
)

if (status EQUAL 0)
message(FATAL_ERROR "${pfx}: compilation failed\n status: ${status}\n copy file error: ${copy_error}\n output: ${error}")
endif()
message(STATUS "${msg} - compilation OK")

execute_process(
COMMAND "mpiexec" "-np" "1" "./a.out"
COMMAND "mpiexec" "-np" "1" "./test_hdf5_mpi"
WORKING_DIRECTORY ${tmpdir}
RESULT_VARIABLE status
ERROR_VARIABLE error
Expand All @@ -266,7 +281,7 @@ if(HDF5_FOUND)
# detecting if it runs under mpirun (missing libhwloc-plugins issue:
# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=790540
# )
execute_process(COMMAND ${libmpdataxx_MPIRUN} "-np" "2" "./a.out"
execute_process(COMMAND ${libmpdataxx_MPIRUN} "-np" "2" "./test_hdf5_mpi"
WORKING_DIRECTORY ${tmpdir}
RESULT_VARIABLE status
ERROR_VARIABLE error
Expand Down
2 changes: 1 addition & 1 deletion tests/paper_2015_GMD/0_basic_example/CMakeLists.txt
@@ -1 +1 @@
libmpdataxx_add_test_gi(basic_example)
libmpdataxx_add_test_gi(basic_example true)
2 changes: 1 addition & 1 deletion tests/paper_2015_GMD/1_advscheme_opts/CMakeLists.txt
@@ -1 +1 @@
libmpdataxx_add_test_gi(advscheme_opts)
libmpdataxx_add_test_gi(advscheme_opts true)
3 changes: 2 additions & 1 deletion tests/paper_2015_GMD/2_convergence_1d/CMakeLists.txt
Expand Up @@ -15,8 +15,9 @@ if(NOT USE_MPI)
NAME convergence_1d_plot
COMMAND bash -c "
for i in ${CMAKE_CURRENT_SOURCE_DIR}/refdata/*.txt.gz; do
python ${CMAKE_CURRENT_SOURCE_DIR}/plot.py ${CMAKE_CURRENT_BINARY_DIR}/`basename \${i/.gz/}`;
python3 ${CMAKE_CURRENT_SOURCE_DIR}/plot.py ${CMAKE_CURRENT_BINARY_DIR}/`basename \${i/.gz/}`;
done;
"
)
endif()
set_property(TEST convergence_1d PROPERTY LABELS SlowWithMpi) # the test is super slow with MPI, make it easy to disable it
10 changes: 5 additions & 5 deletions tests/paper_2015_GMD/2_convergence_1d/plot.py
Expand Up @@ -11,10 +11,11 @@
import matplotlib
matplotlib.use('Svg')

from numpy import loadtxt, zeros, linspace
from numpy import loadtxt, zeros, mgrid
from sys import argv
from math import pi, log, cos, sin, sqrt
from matplotlib.mlab import griddata
from scipy.interpolate import griddata
from scipy.interpolate import interp2d
from matplotlib.patches import Circle, Path, PathPatch

import matplotlib.pyplot as plt
Expand Down Expand Up @@ -42,9 +43,8 @@
ngrid = 800 * 2
levels = range(-32,-3) #[-28,-27,-26,-25,-24,-23,-22,-21,-20,-19,-18,-17,-16,-15,-14,-13,-12,-11,-10,-9,-8,-7]

xi = linspace(mn, mx, ngrid)
yi = linspace(mn, mx, ngrid)
zi = griddata(x,y,err,xi,yi,interp='linear')
xi, yi = mgrid[mn:mx:complex(ngrid), mn:mx:complex(ngrid)]
zi = griddata((x,y),err,(xi,yi),method='linear')

fig = plt.gcf()
fig.gca().set_xlim(mn,mx)
Expand Down
2 changes: 1 addition & 1 deletion tests/paper_2015_GMD/3_rotating_cone_2d/CMakeLists.txt
@@ -1,4 +1,4 @@
libmpdataxx_add_test_gi(rotating_cone_2d)
libmpdataxx_add_test_gi(rotating_cone_2d false) # disable comparison of data sent to Gnuplot, because the data depends on compiler's optimization (result of blitz::rint in particular?)
if(NOT USE_MPI)
add_test(
NAME rotating_cone_stats_diff
Expand Down
2 changes: 1 addition & 1 deletion tests/paper_2015_GMD/6_coupled_harmosc/CMakeLists.txt
@@ -1,4 +1,4 @@
libmpdataxx_add_test_gi(coupled_harmosc)
libmpdataxx_add_test_gi(coupled_harmosc true)
if(NOT USE_MPI)
add_test(
NAME harmosc_stats_diff
Expand Down
10 changes: 5 additions & 5 deletions tests/paper_2015_GMD/7_shallow_water/CMakeLists.txt
Expand Up @@ -13,12 +13,12 @@ if(NOT USE_MPI)
")
add_test(shallow_water_1d_plot bash -c "
for dir in 1d_fct_abs 1d_fct_iga; do
python ${CMAKE_CURRENT_SOURCE_DIR}/plot_1d.py $dir;
python3 ${CMAKE_CURRENT_SOURCE_DIR}/plot_1d.py $dir;
done
")
add_test(shallow_water_1d_stats bash -c "
for dir in 1d_fct_abs 1d_fct_iga; do
python ${CMAKE_CURRENT_SOURCE_DIR}/errors_1d.py $dir &&
python3 ${CMAKE_CURRENT_SOURCE_DIR}/errors_1d.py $dir &&
zdiff $dir'_stats.txt' ${CMAKE_CURRENT_SOURCE_DIR}/refdata/$dir'_stats.txt.gz' || exit 1;
done
")
Expand All @@ -43,15 +43,15 @@ add_test(shallow_water_2d_diff bash -c "
if(NOT USE_MPI)
add_test(shallow_water_2d_plot bash -c "
for dir in 2d_fct_abs 2d_fct_iga; do
python ${CMAKE_CURRENT_SOURCE_DIR}/plot_2d.py $dir;
python3 ${CMAKE_CURRENT_SOURCE_DIR}/plot_2d.py $dir;
done
")
add_test(shallow_water_3d_plot bash -c "
python ${CMAKE_CURRENT_SOURCE_DIR}/plot_2d_3d.py;
python3 ${CMAKE_CURRENT_SOURCE_DIR}/plot_2d_3d.py;
")
add_test(shallow_water_2d_stats bash -c "
for dir in 2d_fct_abs 2d_fct_iga; do
python ${CMAKE_CURRENT_SOURCE_DIR}/errors_2d.py $dir &&
python3 ${CMAKE_CURRENT_SOURCE_DIR}/errors_2d.py $dir &&
zdiff $dir'_stats.txt' ${CMAKE_CURRENT_SOURCE_DIR}/refdata/$dir'_stats.txt.gz' || exit 1;
done
")
Expand Down
4 changes: 3 additions & 1 deletion tests/paper_2015_GMD/8_boussinesq_2d/CMakeLists.txt
@@ -1,7 +1,9 @@
libmpdataxx_add_test_gi(boussinesq_2d)
libmpdataxx_add_test_gi(boussinesq_2d false) # don't compare results point-by-point with reference, because compiler optimization (-Ofast) affects them a little
if(NOT USE_MPI)
add_test(
NAME boussinesq_stats_diff
COMMAND zdiff ${CMAKE_CURRENT_SOURCE_DIR}/refdata/stats.txt.gz stats.txt
)
endif()

set_property(TEST init_boussinesq_2d calc_boussinesq_2d clean_boussinesq_2d PROPERTY LABELS SlowWithMpi)

0 comments on commit 4bf8ef7

Please sign in to comment.