Skip to content

Tesseract GPU compilation

Peter Boyle edited this page Oct 25, 2019 · 2 revisions
####################################
# Grid directory and support directory
####################################
export prefix=$HOME/prefix
export grid=$HOME/GridCompile
mkdir $prefix
mkdir $grid

##################
#LIME
##################
cd $prefix
wget http://usqcd-software.github.io/downloads/c-lime/lime-1.3.2.tar.gz
tar xvzf lime-1.3.2.tar.gz
cd lime-1.3.2
./configure --prefix $prefix
make all install

##################
#GMP - tesseract is badly configured
##################
cd $prefix
wget https://ftp.gnu.org/gnu/gmp/gmp-6.1.2.tar.bz2
tar xvjf gmp-6.1.2.tar.bz2
cd gmp-6.1.2
./configure --prefix $prefix
make all install

##################
#MPFR - summit is badly configured
##################
cd $prefix
wget https://www.mpfr.org/mpfr-current/mpfr-4.0.2.tar.gz
tar xvzf mpfr-4.0.2.tar.gz
cd mpfr-4.0.2
./configure --prefix $prefix --with-gmp=$prefix
make all install

###############
# FFTW
###############
cd $prefix
wget http://www.fftw.org/fftw-3.3.8.tar.gz
tar xvzf fftw-3.3.8.tar.gz
cd fftw-3.3.8
./configure --prefix $prefix  --enable-avx2 --enable-float --enable-


##################
#OpenSSL
##################
cd $prefix
wget  https://www.openssl.org/source/openssl-1.1.0l.tar.gz
tar xvzf openssl-1.1.0l.tar.gz
cd openssl-1.1.0l
./config --prefix=$prefix
make all install

##################
#Obtain Grid and support codes
##################
cd $grid
git clone https://paboyle@github.com/paboyle/Grid
cd Grid
./bootstrap.sh

#########################
# build for GPU
#########################
mkdir build-gpu
cd build-gpu

module load gcc
module load cuda/cuda-9.2

export MPICXX=mpicxx
export CXX=nvcc
export CXXFLAGS="-I/tessfs1/home/dp008/dp008/paboyle/Modules/openmpi/install/include/ -I/home/dp008/dp008/paboyle/Modules/hdf5/install/include/  -arch=sm_70 -gencode=arch=compute_70,code=compute_70  -Xcompiler -fopenmp "
export LIBS="-lmpi "
export LDFLAGS="-L/tessfs1/home/dp008/dp008/paboyle/Modules/openmpi/install/lib/ -L/tessfs1/home/dp008/dp008/paboyle/Modules/hdf5/install/lib/  "
export LD_LIBRARY_PATH=/home/dp008/dp008/paboyle/Modules/psm2-10.8/opa-psm2-IFS_RELEASE_10_8_0_0_204/build_release/:$LD_LIBRARY_PATH

source /home/dp008/dp008/paboyle/Modules/openmpi/module.sh
source /home/dp008/dp008/paboyle/Modules/hdf5/module.sh
source /home/dp008/dp008/paboyle/Modules/lime/lime-1.3.2/module.sh
source /home/dp008/dp008/paboyle/Modules/openssl/module.sh

../configure \
--enable-precision=single \
--enable-simd=GPU \
--enable-comms=mpi3-auto  \
--enable-gen-simd-width=32 \
--enable-openmp
make -j 24

#
# Test GPU
#
qsub -IVl select=1:ncpus=24,place=scatter,walltime=0:20:0 -A dp008 -q QGPU

module load gcc
module load cuda/cuda-9.2
source /home/dp008/dp008/paboyle/Modules/openmpi/module.sh
source /home/dp008/dp008/paboyle/Modules/hdf5/module.sh
source /home/dp008/dp008/paboyle/Modules/lime/lime-1.3.2/module.sh
source /home/dp008/dp008/paboyle/Modules/openssl/module.sh
export LD_LIBRARY_PATH=/home/dp008/dp008/paboyle/Modules/psm2-10.8/opa-psm2-IFS_RELEASE_10_8_0_0_204/build_release/:$LD_LIBRARY_PATH
export PSM2_CUDA=1
cd $PBS_O_WORKDIR/
mpirun -n 4 -map-by node ./Benchmark_dwf --mpi 2.2.1.1 --grid 48.48.48.48