configure

#!/usr/bin/env bash
#configure script adapted from Aquarius software (https://github.cxxom/devinamatthews/aquarius)


function usage
{
  echo -e 'Usage: configure [options]'
  echo
  echo -e '\t--install-dir=dir Specify where to install header and lib files (default is /usr/local,'
  echo -e '\t                  so headers will be installed to /usr/local/include and libs to /usr/local/lib)'
  echo
  echo -e '\t--build-dir=dir   Specify where to build object files, library, and executable (default is .)'
  echo
  echo -e '\t--with-lapack     Tells CTF build to enable LAPACK functionality regardless of whether LAPACK libs have been given.'
  echo
  echo -e '\t--with-scalapack  Tells CTF build to enable ScaLAPACK functionality regardless of whether ScaLAPACK libs have been given.'
  echo
  echo -e '\t--build-scalapack Tells CTF to download and build ScaLAPACK library.'
  echo
  echo -e '\t--with-hptt       Tells CTF build to enable HPTT functionality.'
  echo
  echo -e '\t--build-hptt      Tells CTF to download and build HPTT library.'
  echo
  echo -e '\t--with-cuda       Tells CTF to setup and use NVCC, NVCCFLAGS, and CUBLAS libs'
  echo
  echo -e '\t--no-dynamic      Turns off configuration and build of dynamic (shared) libraries (these are needed for Python codes and some C++ codes)'
  echo
  echo -e '\t--no-static       Turns off configuration and build of static libraries (these are needed for C++ codes)'
  echo
  echo -e '\t--verbose         Does not suppress tests of compile/link commands'
  echo
  echo -e '\tLIB_PATH=-Ldir    Specify paths to static libraries, e.g. -L/usr/local/lib to be used for test executables'
  echo
  echo -e '\tLD_LIB_PATH=-Ldir Specify paths to dynamic libraries to be used for test executables as part of LD_LIBRARY_PATH'
  echo
  echo -e '\tLIBS=-llibname    Specify list of static libraries to link to, by default "-lblas -llapack -lscalapack" or a subset'
  echo -e '\t                  (for each -l<name> ensure lib<name>.a is found in LIBRARY_PATH or LIB_PATH)'
  echo
  echo -e '\tLD_LIBS=-llibname Specify list of dynamic libraries to link to, by default "-lblas -llapack -lscalapack" or a subset'
  echo -e '\t                  (for each -l<name> ensure lib<name>.a is found in LD_LIBRARY_PATH or LD_LIB_PATH)'
  echo
  echo -e '\tCXX=compiler      Specify the C++ compiler (e.g. mpicxx)'
  echo
  echo -e '\tCXXFLAGS=flags    Specify the compiler flags (e.g. "-g -O0" for a lightweight debug build),'
  echo -e '\t                  can also be used to specify macros like -DOMP_OFF (turn off OpenMP) -DPROFILE -DPMPI (turn on performance profiling)'
  echo -e '\t                  -DVERBOSE=1 -DDEBUG, see docs and generated config.mk for details)'
  echo
  echo -e '\tLINKFLAGS=flags   Specify flags for creating the static library'
  echo
  echo -e '\tLDFLAGS=flags     Specify flags for creating the dynamic library'
  echo
  echo -e '\tINCLUDES=-Idir    Specify directories and header files to include (e.g. -I/path/to/mpi/header/file/)'
  echo
  echo -e 'Additionally, the variables AR, NVCC, NVCCFLAGS, and WARNFLAGS'
  echo -e 'can be set on the command line, e.g. ./configure CXX=g++ CXXFLAGS="-fopenmp -O2 -g".'
  echo
}

PARAMS=$0
for PARAM in "$@"
do
  PARAMS="${PARAMS} '${PARAM}'"
done
#Usage: 'testlink $1 $2 $3' where 
# $1 - library link line
# $2 - symbol (function)
# $3 - 0 if no printouts, 1 if verbose
function testlink
{
  rm -f a.out
  status=1
  cat > .test.cxx <<EOF
#if __cplusplus
  extern "C" 
#endif
  void $2();
  int main(){ $2(); return 0; }
EOF
  if [ $3 -eq 1 ]; then
    ( set -x; $CXX $DEFS $WARNFLAGS $CXXFLAGS $INCLUDES .test.cxx $1 $LINKFLAGS  2>&1 )
  else
    $CXX $DEFS $WARNFLAGS $CXXFLAGS $INCLUDES .test.cxx $1 $LINKFLAGS  > /dev/null 2>&1
  fi
  if [ -x a.out ]; then
    status=0
  fi
  rm -f .test.cxx a.out
  return $status
}

#Usage: 'testldlink $1 $2 $3' for checking symbol with dynamic library where 
# $1 - shared library link path
# $2 - shared library libs
# $3 - symbol (function)
# $4 - 0 if no printouts, 1 if verbose
function testldlink
{
  rm -f a.out
  status=1
  cat > .test.cxx <<EOF
#if __cplusplus
  extern "C" 
#endif
  void $3();
  void func(){ $3(); }
  int main(){ return 0; }
EOF
  if [ $IS_APPLE_CLANG -eq 0 ]; then
    TMP_LD_LIB_PATH=${1//"-L"/"-Wl,-rpath="}
  fi
  
  TMP_LD_LIB_PATH="$1 $TMP_LD_LIB_PATH"
  TMP_LD_LIBS="$2"
  if [ $4 -eq 1 ]; then
    ( set -x; $CXX $DEFS $WARNFLAGS $CXXFLAGS $INCLUDES .test.cxx $TMP_LD_LIB_PATH $TMP_LD_LIBS $LDFLAGS  2>&1 )
  else
    $CXX $DEFS $WARNFLAGS $CXXFLAGS $INCLUDES .test.cxx $TMP_LD_LIB_PATH $TMP_LD_LIBS $LDFLAGS  > /dev/null 2>&1
  fi
  if [ -x a.out ]; then
    status=0
    TMP_RAW_LD_LIB_PATH=${TMP_LD_LIB_PATH//" -L"/":"}
    TMP_RAW_LD_LIB_PATH=${RAW_LD_LIB_PATH//"-L"/""}
    TMP_RAW_LD_LIB_PATH=${RAW_LD_LIB_PATH//" "/""}
    if [ $4 -eq 1 ]; then
      ( set -x; LD_LIBRARY_PATH="$TMP_RAW_LD_LIB_PATH:$LD_LIBRARY_PATH" ./a.out 2>&1 )
    else
      LD_LIBRARY_PATH="$TMP_RAW_LD_LIB_PATH:$LD_LIBRARY_PATH" ./a.out > /dev/null 2>&1
    fi
    # Check if symbol is not yet defined, to ensure we really linked to dynamic and not static
    # Disabled because it seems this is not always a reliable check, things often work despite it failing
    #if [ $IS_APPLE_CLANG -eq 0 ]; then
    #  if [ $? -eq 0 ]; then
    #    UDEFSYMB=$(nm ./a.out | grep "$3" | grep -c "U")
    #    if [ $UDEFSYMB -eq 0 ]; then
    #      status=1
    #      if [ $4 -eq 1 ]; then
    #          echo "Undefined symbol not found, so we did not really link to dynamic library (but rather static)"
    #          ( set -x; nm ./a.out; )
    #      fi
    #    fi
    #  else
    #    status=0
    #  fi
    #fi
  fi
  rm -f .test.cxx a.out
  return $status
}

#Usage: 'testcompiler $1' where 
# $1 - 0 if no printouts, 1 if verbose
function testcompiler
{
  status=1
  cat > .test.cxx <<EOF
  int main(){ return 0; }
EOF
  if [ $1 -eq 1 ]; then
    ( set -x; $CXX $DEFS $WARNFLAGS $INCLUDES .test.cxx  2>&1 )
  else
    $CXX $DEFS $WARNFLAGS $INCLUDES .test.cxx > /dev/null 2>&1
  fi
  if [ -x a.out ]; then
    status=0
  fi
  rm -f .test.cxx a.out
  return $status
}

#Usage: 'testcpp11 $1' where 
# $1 - 0 if no printouts, 1 if verbose
function testcpp11
{
  status=1
  cat > .test.cxx <<EOF
  #include <type_traits>

  template <typename a> 
  class b {
    public:
    a s;
    b(a s_){ s=s_; }
  };
  template <typename a> 
  using tb = b<a>;

  int f(int k){
    tb<int> r = b<int>(k);
    return r.s;
  }

  template <typename a, bool b>
  inline typename std::enable_if<b, a>::type testmin(a x, a y){
    return x>y ? y : x;
  }
  template <typename a, bool b>
  inline typename std::enable_if<!b, a>::type testmin(a x, a y){
    return x;
  }
  int main(){ return testmin<int,1>(f(7),3)-testmin<int,0>(3,4); }
EOF
  if [ $1 -eq 1 ]; then
    ( set -x; $CXX $DEFS $WARNFLAGS $INCLUDES .test.cxx $LINKFLAGS  2>&1 )
  else
    $CXX $DEFS $WARNFLAGS $INCLUDES .test.cxx $LINKFLAGS  > /dev/null 2>&1
  fi
  if [ -x a.out ]; then
    status=0
  fi
  rm -f .test.cxx a.out
  return $status
}

#Usage: 'testopenmp'
function testopenmp
{
  status=1
  cat > .test.cxx <<EOF
  #include "omp.h"
  int main(){ 
    int i;
    int j = 0;
    omp_set_num_threads(1);
    #pragma omp for
    for (i=0; i<10; i++){
      j+=i;
    } 
    return j; 
  }
EOF
  $CXX $CXXFLAGS $DEFS $WARNFLAGS $INCLUDES .test.cxx  $LINKFLAGS  > /dev/null  2>&1
  
  if [ -x a.out ]; then
    status=0
  fi
  rm -f .test.cxx a.out
  return $status

}


#Usage: 'testmpi $1 $2' where 
# $1 is MPI_Datatype to test
# $2 - 0 if no printouts, 1 if verbose
function testmpi
{
  status=1
  cat > .test.cxx <<EOF
  #include "mpi.h"
  int main(){ 
    MPI_Init(NULL,NULL);
    MPI_Datatype a = $1;
    MPI_Finalize();
    return 0; 
  }
EOF
  if [ $2 -eq 1 ]; then
    ( set -x; $CXX $CXXFLAGS $DEFS $WARNFLAGS $INCLUDES .test.cxx  $LINKFLAGS 2>&1 )
  else
    $CXX $CXXFLAGS $DEFS $WARNFLAGS $INCLUDES .test.cxx  $LINKFLAGS > /dev/null 2>&1
  fi
  if [ -x a.out ]; then
    status=0
  fi
  rm -f .test.cxx a.out
  return $status

}


#Usage: 'testhptt $1' where 
# $1 - 0 if no printouts, 1 if verbose
function testhptt
{
  status=1
  cat > .test.cxx <<EOF
  #include <hptt.h>
  int main(){ 
    int i = sizeof(hptt::DoubleComplex);
    auto plan = hptt::create_plan( NULL, 0,
          hptt::DoubleComplex(1.0), NULL, 0, NULL,
          hptt::DoubleComplex(0.0), NULL, NULL,
          hptt::ESTIMATE, 1 );
    return 1; 
  }
EOF
  if [ $1 -eq 1 ]; then
    (set -x; $CXX $CXXFLAGS $DEFS $WARNFLAGS $INCLUDES .test.cxx  $LIB_PATH $LIBS $LINKFLAGS  2>&1 )
  else
    $CXX $CXXFLAGS $DEFS $WARNFLAGS $INCLUDES .test.cxx  $LIB_PATH $LIBS $LINKFLAGS  > /dev/null  2>&1
  fi
  
  if [ -x a.out ]; then
    status=0
  fi
  rm -f .test.cxx a.out
  return $status
}


#Usage: 'testcompiler $1' where 
# $1 - 0 if no printouts, 1 if verbose
function testnvcc
{
  status=1
  cat > .test.cu <<EOF
  #include <cuda_runtime.h>
  int main(){ 
    int ndev=0;
    cudaGetDeviceCount(&ndev);
    return 0; 
  }
EOF
  if [ $1 -eq 1 ]; then
    ( set -x; $NVCC $NVCCFLAGS $DEFS $INCLUDES .test.cu -c .test.o  2>&1 )
    ( set -x; $CXX $DEFS $WARNFLAGS $INCLUDES .test.o $LINKFLAGS 2>&1 )
  else
    $NVCC $NVCCFLAGS $DEFS $INCLUDES .test.cu -c .test.o > /dev/null 2>&1
    $CXX $DEFS $WARNFLAGS $INCLUDES .test.o $LINKFLAGS > /dev/null 2>&1
  fi
  if [ -x a.out ]; then
    status=0
  fi
  rm -f .test.cu .test.o a.out
  return $status
}

function testcublas
{
  status=1
  cat > .test.cu <<EOF
  #include <cuda_runtime.h>
  #include <cublas_v2.h>
  cublasHandle_t cuhandle;
  int main(){
    cublasStatus_t status = cublasCreate(&cuhandle);
    return 0; 
  }
EOF
  if [ $1 -eq 1 ]; then
    ( set -x; $NVCC $NVCCFLAGS $DEFS $INCLUDES .test.cu -c .test.o  2>&1 )
    ( set -x; $CXX $DEFS $WARNFLAGS $INCLUDES .test.o $LINKFLAGS 2>&1 )
  else
    $NVCC $NVCCFLAGS $DEFS $INCLUDES .test.cu -c .test.o > /dev/null 2>&1
    $CXX $DEFS $WARNFLAGS $INCLUDES .test.o $LINKFLAGS > /dev/null 2>&1
  fi
  if [ -x a.out ]; then
    status=0
  fi
  rm -f .test.cu a.out
  return $status
}

#Usage: 'testcompiler $1' where 
# $1 - 0 if no printouts, 1 if verbose
function testlambda
{
  status=1
  cat > .test.cxx <<EOF
  #include <functional>
  void a(std::function<void(int)>){}
  void a(std::function<void(int,int)>){}
  void f(int){};
  int main(){ 
    a(&f);
    return 0; 
  }
EOF
  if [ $1 -eq 1 ]; then
    ( set -x; $CXX $DEFS $WARNFLAGS $INCLUDES .test.cxx $LINKFLAGS  2>&1 )
  else
    $CXX $DEFS $WARNFLAGS $INCLUDES .test.cxx $LINKFLAGS  > /dev/null 2>&1
  fi
  if [ -x a.out ]; then
    status=0
  fi
  rm -f .test.cxx a.out
  return $status
}

function realcompiler
{
    echo -n 'Checking compiler type/version... '
    if $CXX --version > /dev/null 2>&1; then
        version=`$CXX --version 2>&1 | tr '\n' ' '`
    elif $CXX -V > /dev/null 2>&1; then
        version=`$CXX -V 2>&1 | tr '\n' ' '`
    else
        $CXX --version
        $CXX -V 
        echo 'Could not determine underlying C/C++ compilers. Specify MPI compiler via CXX=...'
        exit 1
    fi

    case $version in
        *Intel*)
            echo 'Using Intel compilers.'
            compiler=intel
            ;;
        *Portland*)
            echo 'Portland Group compilers are not supported. Specify MPI compiler via CXX=...'
            exit 1
            compiler=pgi
            ;;
        *Free\ Software*)
            echo 'Using GNU compilers.'
            compiler=gnu
            ;;
        *Cray*)
            echo 'Cray compilers are not supported.'
            exit 1
            compiler=cray
            ;;
        *clang*)
            #CLANG_VERSION=$(clang++ --version | grep -o "[0-9]\.[0-9]" | sort | head -n 1)
            #echo 'Using Clang/LLVM compiler, version '$CLANG_VERSION'.'
            echo 'Using Clang/LLVM compiler.'
            compiler=clang
            ;;
        *)
            echo 'Could not determine underlying C/C++ compilers. Specify MPI compiler via CXX=...'
            exit 1
            ;;
    esac
}

function defaultflags
{
    #
    # Set default compiler flags
    #
    case $compiler in
        intel)
            if [ "x$AR"        = "x" ]; then AR='xiar'; fi
            if [ "x$CXXFLAGS"  = "x" ]; then CXXFLAGS='-qopenmp -O3 -ipo'; fi
            if [ "x$NVCCFLAGS" = "x" ]; then NVCCLAGS=''; fi
            if [ "x$DEFS"      = "x" ]; then DEFS='-D_POSIX_C_SOURCE=200112L -D__STDC_LIMIT_MACROS'; fi
            if [ "x$WARNFLAGS"      = "x" ]; then WARNFLAGS='-Wall'; fi
            PASS_TO_LINKER='-Wl,'
            ;;
        gnu | clang)
            if [ "x$AR"        = "x" ]; then AR='ar'; fi
            if [ "x$CXXFLAGS"  = "x" ]; then CXXFLAGS='-O3'; fi
            if [ "x$NVCCFLAGS" = "x" ]; then NVCCLAGS=''; fi
            if [ "x$DEFS"      = "x" ]; then DEFS='-D_POSIX_C_SOURCE=200112L -D__STDC_LIMIT_MACROS'; fi
            if [ "x$WARNFLAGS"      = "x" ]; then WARNFLAGS='-Wall'; fi
            PASS_TO_LINKER='-Wl,'
            ;;
        *)
            echo 'No default flags known for given compilers. Make sure that you have set the'
            echo 'appropriate compiler flags manually.'
            if [ "x$AR"        = "x" ]; then AR='ar'; fi
            if [ "x$CXXFLAGS"  = "x" ]; then CXXFLAGS='-O3'; fi
            if [ "x$NVCCFLAGS" = "x" ]; then NVCCLAGS=''; fi
            if [ "x$DEFS"      = "x" ]; then DEFS='-D_POSIX_C_SOURCE=200112L -D__STDC_LIMIT_MACROS'; fi
            if [ "x$WARNFLAGS"      = "x" ]; then WARNFLAGS='-Wall'; fi
            PASS_TO_LINKER='-Wl,'
            ;;
    esac

    case $compiler in
        intel)
            if [ "x$SEQCXX"    = "x" ]; then SEQCXX='icpc'; fi
            ;;
        gnu )
            if [ "x$SEQCXX"    = "x" ]; then SEQCXX='g++'; fi
            ;;
        clang)
            if [ "x$SEQCXX"    = "x" ]; then SEQCXX='clang++'; fi
            ;;
        *)
            if [ "x$SEQCXX"    = "x" ]; then SEQCXX=$CXX; fi
            ;;
    esac
}

function check_if_apple
{
  status=1
  cat > .test.cxx <<EOF
#if defined(__APPLE__)
#error
#endif
  int main(){  return 0; }
EOF
  $CXX $DEFS $WARNFLAGS $CXXFLAGS $INCLUDES .test.cxx $LINKFLAGS  > /dev/null 2>&1
  if [ -x a.out ]; then
    status=0
  fi
  rm -f .test.cxx a.out
  return $status
}

#
# Parse command-line arguments
#
depstype=normal
WITH_CUDA=0
WITH_LAPACK=0
WITH_SCALAPACK=0
WITH_STATIC=1
WITH_DYNAMIC=1
BUILD_SCALAPACK=0
WITH_HPTT=0
BUILD_HPTT=0
VERBOSE=0
while [ "x$1" != "x" ]; do
  case $1 in
    --build-dir=*)
      eval BUILDDIR="$(printf "%q" "${1#--build-dir=}")"
      ;;
    --install-dir=*)
      eval INSTALLDIR="$(printf "%q" "${1#--install-dir=}")"
      ;;
    --with-lapack)
      WITH_LAPACK=1
      ;;
    --with-scalapack)
      WITH_SCALAPACK=1
      ;;
    --build-scalapack)
      BUILD_SCALAPACK=1
      ;;
    --with-hptt)
      WITH_HPTT=1
      ;;
    --build-hptt)
      BUILD_HPTT=1
      ;;
    --with-cuda)
      WITH_CUDA=1
      ;;
    --no-dynamic)
      WITH_DYNAMIC=0
      ;;
    --no-static)
      WITH_STATIC=0
      ;;
    --verbose)
      VERBOSE=1
      ;;
    --help)
      usage
      exit 0
      ;;
    LIB_PATH=*|\
    LD_LIB_PATH=*|\
    LIBS=*|\
    LD_LIBS=*|\
    CXX=*|\
    NVCC=*|\
    AR=*|\
    CXXFLAGS=*|\
    NVCCFLAGS=*|\
    LINKFLAGS=*|\
    LDFLAGS=*|\
    WARNFLAGS=*|\
    INCLUDES=*)
      eval "${1%%=*}=\"${1#*=}\""
      ;;
    *)
      echo "    WARNING: Unknown option \"$1\"."
      usage
      exit 1
      ;;
  esac
  shift
done

CTFDIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
if [ "x$BUILDDIR" = "x" ]; then
	if [ "x$CTFDIR" = `pwd` ]; then
	  BUILDDIR='.'
  else
	  BUILDDIR=`pwd`
  fi
fi

if [ "x$INSTALLDIR" = "x" ]; then
	INSTALLDIR='/usr/local'
fi

if [ "x$LIB_PATH" = "x" ]; then
	LIB_PATH=""
elif [ "x$LD_LIB_PATH" = "x" ]; then
  echo "LD_LIB_PATH was unspecified but LIB_PATH was, setting LD_LIB_PATH=$LIB_PATH."
	LD_LIB_PATH=$LIB_PATH
fi

if [ "x$LIBS" = "x" ]; then
	LIBS=""
elif [ "x$LD_LIBS" = "x" ]; then
  echo "LD_LIBS was unspecified but LIBS was, setting LD_LIBS=$LIBS."
	LD_LIBS=$LIBS
fi


if [ "x$CXX" = "x" ]; then
  CXX=mpicxx
fi

realcompiler
defaultflags

if [ $WITH_STATIC = 0 ]; then
  if [ $WITH_DYNAMIC = 0 ]; then
    echo
    echo -n 'Specifying --no-dynamic and --no-static does not make sense, what do you want to build?'
    echo
    exit 1
  fi
fi

echo -n 'Checking whether __APPLE__ is defined... '
IS_APPLE_CLANG=0
BDYNAMIC="-Wl,-Bdynamic"
BSTATIC="-Wl,-Bstatic"
check_if_apple;
status=$?
if [ $status = 1 ]; then
  echo -n 'yes, tacking on -D_DARWIN_C_SOURCE to DEFS and -std=c++0x to CXXFLAGS.'
  echo
  DEFS="$DEFS -D_DARWIN_C_SOURCE"
  CXXFLAGS="$CXXFLAGS -std=c++0x"
  if [ "$compiler" = "clang" ]; then
    echo -n 'Detected Apple + Clang, disabling use of rpath, Bdynamic, and Bstatic'
    echo
    IS_APPLE_CLANG=1
    BDYNAMIC=""
    BSTATIC=""
  fi
else
  echo -n 'no.'
  echo
fi

#
# Check that compiler works without error
#
echo -n 'Checking compiler (CXX)... '
if testcompiler 0; then
  echo 'successful.'
else
  echo 'FAILED! error below:'
  echo
  testcompiler 1
  echo
  exit 1
fi

#
# Check that CXXFLAGS works without error
#
echo -n 'Checking flags (CXXFLAGS)... '
ODEFS="$DEFS"
DEFS="$DEFS $CXXFLAGS"
if testcompiler 0; then
  echo 'successful.'
else
  echo 'FAILED! error below:'
  echo
  testcompiler 1
  echo
  exit 1
fi
DEFS="$ODEFS"


#
# Check that compiler works without error
#
echo -n 'Checking availability of C++11... '

if testcpp11 0; then
  echo 'successful.'
else
  ODEFS="$DEFS"
  DEFS="$DEFS $CXXFLAGS"
  if testcpp11 0; then
    echo 'successful.'
  else
    if [ "$NERSC_HOST" = "edison" ]; then
      echo 'FAILED! error below, try using module load gcc/4.8.2:'
      echo
      testcpp11 1
      echo
      exit 1
    else
      CXXFLAGS="$CXXFLAGS -std=c++0x"
      DEFS="$ODEFS $CXXFLAGS"
      if testcpp11 0; then
        echo 'successful (tacking on -std=c++0x to CXXFLAGS).'
      else
        echo 'FAILED! error below:'
        echo
        testcpp11 1
        echo
        exit 1
      fi
    fi
  DEFS="$ODEFS"
  fi
fi

#
# Check if MPI is provided
#
echo -n 'Checking for MPI... '
if testmpi MPI_CXX_DOUBLE_COMPLEX $VERBOSE; then
  echo 'MPI works.'
else
  if testmpi MPI::DOUBLE_COMPLEX $VERBOSE; then
    echo 'detected limited MPI type-naming convention, e.g. MPI_C_DOUBLE_COMPLEX, adapting to use MPI::... instead of MPI_CXX_... for some types.'
    DEFS="$DEFS -DUSE_MPI_CPP"
  else
    echo '    ERROR: unable to compile MPI test program, provide appropriate compiler, e.g. CXX=mpicxx or specify INCLUDES=-I/path/to/mpi.h/ as well as appropriate static/dynamic paths/libraries, see ./configure --help.'
    echo
    testmpi MPI_CXX_DOUBLE_COMPLEX 1;
    echo
    exit 1
  fi
fi
 
#
# Check if OpenMP is provided
#
echo -n 'Checking for OpenMP... '
if testopenmp; then
  echo 'OpenMP works.'
else
  OCXXFLAGS=$CXXFLAGS
  if [ "$compiler" = "clang" ]; then
    echo
    echo '    WARNING: not attempting usage of -fopenmp for Clang, reconfigure with appropriate flags to use OpenMP with Clang... '
    echo -n '  ... '
  else
    CXXFLAGS="$CXXFLAGS -fopenmp"
  fi
  if testopenmp; then
    echo 'using OpenMP via -fopenmp flag.'
  else
    echo 'unable to compile OpenMP test program, will build without OpenMP, to enable OpenMP please provide e.g. CXXFLAGS=-fopenmp.'
    CXXFLAGS="$OCXXFLAGS -DOMP_OFF"
  fi
fi
  

USE_BATCH_GEMM=-1
USE_MKL=0
#
# Check for BLAS/LAPACK/SCALAPACK libraries and determine BLAS/LAPACK naming convention (underscores)
#
if [ $WITH_STATIC = 1 ]; then
  echo -n 'Checking for static BLAS library... '
  if testlink "$LIB_PATH $LIBS" dgemm_ $VERBOSE; then
    UNDERSCORE=1
    echo 'static BLAS library found, with underscores.'
  else
    if testlink "$LIB_PATH $LIBS" dgemm $VERBOSE; then
      UNDERSCORE= 0
      echo 'static BLAS library found, without underscores.'
    else
      if [ "$compiler" = "intel" ] &&  testlink  "$LIB_PATH -mkl $LIBS" dgemm_ 0 ; then
        UNDERSCORE=1
        echo 'detected that -mkl works, speculatively using -mkl.'
        LIBS="-mkl $LIBS"
        USE_MKL=1
      elif testlink "$LIB_PATH $LIBS $BSTATIC -lblas $BDYNAMIC" dgemm_ $VERBOSE; then
        UNDERSCORE=1
        echo 'detected that -lblas works, speculatively using -lblas (with underscores).'
        LIBS="$LIBS -lblas"
      elif testlink "$LIB_PATH $LIBS $BSTATIC -lblas $BDYNAMIC" dgemm $VERBOSE; then
        UNDERSCORE=0
        echo 'detected that -lblas works, speculatively using -lblas (without underscores).'
        LIBS="$LIBS -lblas"
      else
        UNDERSCORE=1
        echo
        echo '    WARNING: static BLAS libirary did not work, executables will not build (errors for a few configurations below),'
        testlink "$LIB_PATH $LIBS" dgemm_ 1
        testlink "$LIB_PATH $LIBS" dgemm 1
        testlink "$LIB_PATH $LIBS $BSTATIC -lblas $BDYNAMIC" dgemm_ 1
        testlink "$LIB_PATH $LIBS $BSTATIC -lblas $BDYNAMIC" dgemm 1
        echo '    please specify correct LIB_PATH and LIBS (run ./configure --help to see all options),'
        echo '    CTF library can still build (via make), but executables will not!'
        echo
      fi
    fi
  fi
  echo -n 'Checking for availability of static batched gemm... '
  if [ $UNDERSCORE = 1 ]; then
    if testlink "$LIB_PATH $LIBS" dgemm_batch_ $VERBOSE; then
      echo 'available, will build with -DUSE_BATCH_GEMM.'
      USE_BATCH_GEMM=1
    else
      echo 'unavailable, will build without -DUSE_BATCH_GEMM.'
      USE_BATCH_GEMM=0
    fi
  else
    if testlink "$LIB_PATH $LIBS" dgemm_batch $VERBOSE; then
      echo 'available, will build with -DUSE_BATCH_GEMM.'
      USE_BATCH_GEMM=1
    else
      echo 'unavailable, will build without -DUSE_BATCH_GEMM.'
      USE_BATCH_GEMM=0
    fi
  fi
fi

if [ $WITH_DYNAMIC = 1 ]; then
  echo -n 'Checking for dynamic BLAS library... '
  if testldlink "$LD_LIB_PATH" "$LD_LIBS" dgemm_ $VERBOSE; then
    UNDERSCORE=1
    echo 'dynamic BLAS library found, with underscores.'
  else
    if testldlink "$LD_LIB_PATH" "$LD_LIBS" dgemm $VERBOSE; then
      UNDERSCORE= 0
      echo 'dynamic BLAS library found, without underscores.'
    else
      if [ "$compiler" = "intel" ] &&  testldlink  "$LD_LIB_PATH" "$LD_LIBS -mkl" dgemm_ $VERBOSE ; then
        UNDERSCORE=1
        echo 'detected that -mkl works, speculatively using -mkl.'
        LD_LIBS="$LD_LIBS -mkl"
        USE_MKL=1
      elif testldlink "$LD_LIB_PATH $LD_LIBS $BDYNAMIC -lblas" "$LD_LIBS" dgemm_ $VERBOSE; then
        UNDERSCORE=1
        echo "detected that $BDYNAMIC -lblas works, speculatively using (-Wl,-Bdynamic) -lblas (with underscores)."
        LD_LIBS="$LD_LIBS $BDYNAMIC -lblas"
      elif testldlink "$LD_LIB_PATH $LD_LIBS $BDYNAMIC -lblas" "$LD_LIBS" dgemm $VERBOSE; then
        UNDERSCORE=0
        echo "detected that $BDYNAMIC -lblas works, speculatively using $BDYNAMIC -lblas (without underscores)."
        LD_LIBS="$LD_LIBS $BDYNAMIC -lblas"
      else
        UNDERSCORE=1
        echo
        echo '    WARNING: dynamic BLAS libirary did not work, python executables will not run (errors for a few configurations below),'
        testldlink "$LD_LIB_PATH" "$LD_LIBS" dgemm_ 1
        testldlink "$LD_LIB_PATH" "$LD_LIBS" dgemm 1
        testldlink "$LD_LIB_PATH $LD_LIBS $BDYNAMIC -lblas" "$LD_LIBS" dgemm_ 1
        testldlink "$LD_LIB_PATH $LD_LIBS $BDYNAMIC -lblas" "$LD_LIBS" dgemm 1
        echo '    please specify correct LD_LIB_PATH and LD_LIBS (run ./configure --help to see all options),'
        echo '    CTF library can still build (via make), but executables will not!'
        echo
        exit 1
      fi
    fi
  fi
  echo -n 'Checking for availability of dynamic batched gemm... '
  if [ $USE_BATCH_GEMM = -1 ]; then
    if [ $UNDERSCORE = 1 ]; then
      if testldlink "$LD_LIB_PATH" "$LD_LIBS" dgemm_batch_ $VERBOSE; then
        echo 'available, will build with -DUSE_BATCH_GEMM.'
        USE_BATCH_GEMM=1
      else
        echo 'unavailable, will build without -DUSE_BATCH_GEMM.'
        USE_BATCH_GEMM=0
      fi
    else
      if testldlink "$LD_LIB_PATH" "$LD_LIBS" dgemm_batch $VERBOSE; then
        echo 'available, will build with -DUSE_BATCH_GEMM.'
        USE_BATCH_GEMM=1
      else
        echo 'unavailable, will build without -DUSE_BATCH_GEMM.'
        USE_BATCH_GEMM=0
      fi
    fi
  fi
fi


DEFS="$DEFS -DFTN_UNDERSCORE=$UNDERSCORE"
if [ $USE_BATCH_GEMM = 1 ]; then
  DEFS="$DEFS -DUSE_BATCH_GEMM"
fi


if [ $UNDERSCORE = 1 ] ; then
  DGEMM=dgemm_
  DGEQRF=dgeqrf_
  PDGEMM=pdgemm_
else
  DGEMM=dgemm
  DGEQRF=dgeqrf
  PDGEMM=pdgemm
fi

USING_LAPACK=0
if [ $WITH_STATIC = 1 ]; then
  echo -n 'Checking for static LAPACK library... '
  if testlink "$LIB_PATH $LIBS" $DGEQRF $VERBOSE; then
    echo 'static LAPACK found.'
    DEFS="$DEFS -DUSE_LAPACK"
    USING_LAPACK=1
  else
    if testlink "$LIB_PATH $BSTATIC -llapack $BDYNAMIC $LIBS" $DGEQRF $VERBOSE; then
      echo 'static LAPACK library found (-llapack).'
      LIBS="$BSTATIC -llapack $BDYNAMIC $LIBS"
      DEFS="$DEFS -DUSE_LAPACK"
      USING_LAPACK=1
    else
      OLINKFLAGS=$LINKFLAGS
      LINKFLAGS="$LINKFLAGS -lgfortran"
      if testlink "$LIB_PATH $BSTATIC -llapack $BDYNAMIC $LIBS" $DGEQRF $VERBOSE; then
        echo 'static LAPACK library found (-llapack -lgfortran).'
        LIBS="$BSTATIC -llapack $BDYNAMIC $LIBS"
        DEFS="$DEFS -DUSE_LAPACK"
        USING_LAPACK=1
      else
        LINKFLAGS="$OLINKFLAGS"
        if [ $WITH_LAPACK = 1 ]; then
          echo '  LAPACK not found, will build with LAPACK-dependent functionality anyway, since --with-lapackwas specified.'
          echo '  WARNING: you may not be able to build CTF tests and benchmarks in this configuration (but you can build the library and use in conjunction with a lapack library)'
          echo '  WARNING: executables will not build until you provide a BLAS library (error below),'
          testlink "$LIB_PATH $LIBS" $DGEQRF 1
          echo '    please specify correct LIB_PATH and LIBS (run ./configure --help to see all options),'
          echo '    CTF library can still build (via make), but executables will not!'
          DEFS="$DEFS -DUSE_LAPACK"
          USING_LAPACK=1
        else
          echo '  LAPACK not found, some functionality and tests will be unavailable,'
          echo '  to fix reconfigure and add --with-lapack and the appropriate library path'
          echo '  (LIB_PATH/LIBS for static, LD_LIB_PATH/LD_LIBS for dynamic)'
        fi 
      fi
    fi
  fi
fi

if [ $WITH_DYNAMIC = 1 ]; then
  echo -n 'Checking for dynamic LAPACK library... '
  if testldlink "$LD_LIB_PATH" "$LD_LIBS" $DGEQRF $VERBOSE; then
    echo 'dynamic LAPACK found.'
    if [ $USING_LAPACK = 0 ]; then 
      DEFS="$DEFS -DUSE_LAPACK"
    fi
  else
    if testldlink "$LD_LIB_PATH $BDYNAMIC -llapack" "$LD_LIBS" $DGEQRF $VERBOSE; then
      echo 'dynamic LAPACK library found ($BDYNAMIC -llapack).'
      LD_LIBS="$LD_LIBS $BDYNAMIC -llapack"
      if [ $USING_LAPACK = 0 ]; then 
        DEFS="$DEFS -DUSE_LAPACK"
      fi
    else
      if [ $WITH_LAPACK = 1 ]; then
        echo '  LAPACK not found, will build with LAPACK-dependent functionality anyway, since --with-lapackwas specified.'
        echo '  WARNING: you may not be able to build CTF tests and benchmarks in this configuration (but you can build the library and use in conjunction with a lapack library)'
        echo '  WARNING: executables will not build until you provide a BLAS library (error below),'
        testldlink "$LD_LIB_PATH" "$LD_LIBS" $DGEQRF 1
        echo '    please specify correct LD_LIB_PATH and LD_LIBS (run ./configure --help to see all options),'
        echo '    CTF library can still build (via make), but executables will not!'
        if [ $USING_LAPACK = 0 ]; then 
          DEFS="$DEFS -DUSE_LAPACK"
        fi
      else
        echo '  LAPACK not found, some functionality and tests will be unavailable,'
        echo '  to fix reconfigure and add --with-lapack and the appropriate library path'
        echo '  (LIB_PATH/LIBS for static, LD_LIB_PATH/LD_LIBS for dynamic)'
      fi 
    fi
  fi
fi

echo -n 'Checking for sparse MKL routines... '
if [ $WITH_STATIC = 1 ]; then
  if testlink "$LIB_PATH $LIBS" mkl_dcsrmultcsr $VERBOSE; then
    echo 'sparse MKL found.'
    USE_MKL=1
  elif testlink "$LIB_PATH $LIBS" mkl_dcsrmultcsr_ $VERBOSE; then
    echo 'sparse MKL found.'
    USE_MKL=1
  else
    echo 'MKL symbol not detected, assuming sparse MKL routines are unavailable (affects sparse BLAS performance).'
    USE_MKL=0
  fi
else
  if testldlink "$LD_LIB_PATH" "$LD_LIBS" mkl_dcsrmultcsr $VERBOSE; then
    echo 'sparse MKL found.'
    USE_MKL=1
  elif testldlink "$LD_LIB_PATH" "$LD_LIBS" mkl_dcsrmultcsr_ $VERBOSE; then
    echo 'sparse MKL found.'
    USE_MKL=1
  else
    echo 'MKL symbol not detected, assuming sparse MKL routines are unavailable (affects sparse BLAS performance).'
    USE_MKL=0
  fi
fi

if [ $USE_MKL = 1 ]; then
  DEFS="$DEFS -DUSE_MKL"
fi

if [ $BUILD_SCALAPACK = 1 ]; then
  echo -n 'Building ScaLAPACK using cmake (no options passed along)... '
  cd $BUILDDIR
  if [ -d scalapack ]; then
    read -p "found existing scalapack subdirectory,"$'\n'""$'\n'" overwrite?  (Y/N)? " -n 1 -r
    echo    # (optional) move to a new line
    if [[ $REPLY =~ ^[Yy]$ ]]
    then
        rm -rf scalapack
    fi
  fi
  if [ ! -d scalapack ]; then
    wget http://www.netlib.org/scalapack/scalapack-2.1.0.tgz -O scalapack.tgz;
    mkdir scalapack;
    tar -xzf scalapack.tgz -C scalapack --strip-components 1;
  fi
  cd scalapack/;
  mkdir -p build;
  cd build;
  if [ $WITH_STATIC = 1 ]; then
    cmake .. -DBUILD_STATIC_LIBS=ON -DBUILD_SHARED_LIBS=OFF
    make
    LIB_PATH="$LIB_PATH -L$BUILDDIR/scalapack/build/lib"
  fi
  if [ $WITH_DYNAMIC = 1 ]; then
    cmake .. -DBUILD_STATIC_LIBS=OFF -DBUILD_SHARED_LIBS=ON
    make
    LD_LIB_PATH="$LD_LIB_PATH -L$BUILDDIR/scalapack/build/lib"
  fi
  #STR=""
  #if [ $WITH_STATIC = 1 ]; then
  #  STR="$STR -DBUILD_STATIC_LIBS=ON"
  #else
  #  STR="$STR -DBUILD_STATIC_LIBS=OFF"
  #fi
  #if [ $WITH_DYNAMIC = 1 ]; then
  #  STR="$STR -DBUILD_SHARED_LIBS=ON"
  #else
  #  STR="$STR -DBUILD_SHARED_LIBS=OFF"
  #fi
  #cmake .. $STR
  cd $BUILDDIR
  echo "build completed in scalapack subdirectory."
fi

USING_SCALA=0
if [ $WITH_STATIC = 1 ]; then
  echo -n 'Checking for static ScaLAPACK... '
  if testlink "$LIB_PATH $LIBS" $PDGEMM $VERBOSE; then
    echo 'static SCALAPACK found.'
    DEFS="$DEFS -DUSE_SCALAPACK"
    USING_SCALA=1
  else
    if testlink "$LIB_PATH $BSTATIC -lscalapack $BDYNAMIC $LIBS" $PDGEMM $VERBOSE; then
      echo 'SCALAPACK library found (-lscalapack).'
      LIBS="$BSTATIC -lscalapack $BDYNAMIC $LIBS"
      DEFS="$DEFS -DUSE_SCALAPACK"
      USING_SCALA=1
    else
      if [ $WITH_SCALAPACK = 1 ]; then
        echo '  static ScaLAPACK not found, will build with ScaLAPACK-dependent functionality anyway, since --with-scalapack was specified.'
        echo '  WARNING: you may not be able to build ctf tests and benchmarks in this configuration (but you can build the library and use in conjunction with a scalapack library)'
        echo '  WARNING: executables will not build until you provide a scalapack library (error below),'
        testlink "$LIB_PATH $BSTATIC -lscalapack $BDYNAMIC $LIBS" $PDGEMM 1
        echo '    please specify correct LIB_PATH and LIBS or configure with'
        echo '    --build-scalapack to attempt to automatically download and build scalapack'
        echo '    (run ./configure --help to see all options),'
        echo '    CTF library can still build (via make), but executables will not!'
        DEFS="$DEFS -DUSE_SCALAPACK"
        USING_SCALA=1
      else
        echo '  static ScaLAPACK not found, some functionality and tests will be unavailable,'
        echo '  to fix reconfigure and add --with-scalapack and the appropriate library path'
        echo '  (LIB_PATH/LIBS for static, LD_LIB_PATH/LD_LIBS for dynamic) or configure with'
        echo '  --build-scalapack to attempt to automatically download and build scalapack'
      fi
    fi
  fi
fi

if [ $WITH_DYNAMIC = 1 ]; then
  echo -n 'Checking for dynamic ScaLAPACK... '
  if testldlink "$LD_LIB_PATH" "$LD_LIBS" $PDGEMM $VERBOSE; then
    echo 'dynamic SCALAPACK found.'
    if [ $USING_SCALA = 0 ]; then 
      DEFS="$DEFS -DUSE_SCALAPACK"
    fi
  else
    if testldlink "$LD_LIB_PATH $BDYNAMIC -lscalapack" "$LD_LIBS" $PDGEMM $VERBOSE; then
      echo 'SCALAPACK library found ($BDYNAMIC -lscalapack).'
      LD_LIBS="$BDYNAMIC -lscalapack $LD_LIBS"
      if [ $USING_SCALA = 0 ]; then 
        DEFS="$DEFS -DUSE_SCALAPACK"
      fi
    else
      if [ $WITH_SCALAPACK = 1 ]; then
        echo '  dynamic ScaLAPACK not found, will build with ScaLAPACK-dependent functionality anyway, since --with-scalapack was specified.'
        echo '  WARNING: you may not be able to build ctf tests and benchmarks in this configuration (but you can build the library and use in conjunction with a scalapack library)'
        echo '  WARNING: executables will not build until you provide a scalapack library (error below),'
        testldlink "$LD_LIB_PATH" "$LD_LIBS" $PDGEMM 1
        echo '    please specify correct LD_LIB_PATH and LD_LIBS (run ./configure --help to see all options),'
        echo '    CTF library can still build (via make), but executables will not!'
        if [ $USING_SCALA = 0 ]; then 
          DEFS="$DEFS -DUSE_SCALAPACK"
        fi
      else
        echo '  dynamic ScaLAPACK not found, some functionality and tests will be unavailable,'
        echo '  to fix reconfigure and add --with-scalapack and the appropriate library path'
        echo '  (LD_LIB_PATH/LD_LIBS for dynamic, LD_LD_LIB_PATH/LD_LD_LIBS for dynamic)'
      fi
    fi
  fi
fi

USE_HPTT=0
if [ $BUILD_HPTT = 1 ]; then
  echo -n "Building HPTT using CXX=$CXX... "
  cd $BUILDDIR
  if [ -d hptt ]; then
    read -p "found existing HPTT subdirectory,"$'\n'""$'\n'" overwrite?  (Y/N)? " -n 1 -r
    echo    # (optional) move to a new line
    if [[ $REPLY =~ ^[Yy]$ ]]
    then
      rm -rf hptt;
    fi
  fi
  if [ ! -d hptt ]; then
    wget https://github.com/solomonik/hptt/archive/master.zip -O hptt-master.zip;
    unzip -o hptt-master.zip;
    mv hptt-master hptt;
  fi
  cd hptt;
  CXX=$SEQCXX make
  LIB_PATH="$LIB_PATH -L$BUILDDIR/hptt/lib"
  LD_LIB_PATH="$LD_LIB_PATH -L$BUILDDIR/hptt/lib"
  INCLUDES="$INCLUDES -I$BUILDDIR/hptt/include"
  cd $BUILDDIR
  echo "build completed in $hptt subdirectory."
fi

#
# Check if HPTT is provided
# 
echo -n 'Checking for HPTT (optimized transposition library)... '
if testhptt $VERBOSE; then
  echo 'HPTT works.'
  USE_HPTT=1
else
  OLIBS=$LIBS
  OLD_LIBS=$LD_LIBS
  LIBS="$LIBS $BSTATIC -lhptt $BDYNAMIC"
  LD_LIBS="$LD_LIBS $BDYNAMIC -lhptt"
  if testhptt $VERBOSE; then
    echo 'HPTT works by tacking on -lhptt to LIBS and LD_LIBS.'
    USE_HPTT=1
  else
    if [ $BUILD_HPTT = 1 ] || [ $WITH_HPTT = 1 ]; then
      echo 'ERROR: HPTT does not work, aborting.'
      testhptt 1;
      echo
      exit 0
    else
      echo 'HPTT does not work, will use built-in transpose kernel.'
      LIBS=$OLIBS
      LD_LIBS=$OLD_LIBS
      USE_HPTT=0
    fi
  fi
fi

if [ $USE_HPTT = 1 ]; then
  DEFS="$DEFS -DUSE_HPTT"
fi

USE_CUDA=0
echo -n 'Checking whether to use CUDA... '
if [ "x$NVCC" = "x" ]; then
  if [ $WITH_CUDA = 1 ]; then
    echo -n 'automatic nvcc setup requested... '
    NVCC='nvcc -ccbin g++ -x cu'
    NVCCFLAGS='-O2 -std=c++11 -m64'
    LINKFLAGS="$LINKFLAGS -L/usr/local/cuda/lib64 -lcuda -lcudart"
    if testnvcc $VERBOSE; then
      echo 'nvcc defaults work successfully.'
      DEFS="$DEFS -DOFFLOAD -DUSE_CUDA"
      USE_CUDA=1
    else  
      echo 'FAILED! error below:'
      echo
      testnvcc 1;
      echo
      exit 1
    fi
  else
    echo 'CUDA will not be used.'
    NVCC='$(CXX) -x c++'
    NVCCFLAGS='$(CXXFLAGS)'
  fi
else
  echo 'testing provided NVCC and NVCCFLAGS... '
  DEFS="$DEFS -DOFFLOAD -DUSE_CUDA"
  USE_CUDA=1
  if testnvcc $VERBOSE; then
    echo 'successful.'
  else
    echo 'FAILED! error below:'
    echo
    testnvcc 1;
    echo
    exit 1
  fi
fi

if [ $USE_CUDA = 1 ]; then
  echo -n 'Checking whether CUBLAS libraries are provided... '
  if testcublas $VERBOSE; then
    echo 'CUBLAS works with provided LINKFLAGS.'
  else
    LINKFLAGS="$LINKFLAGS -lcublas"
    if testcublas $VERBOSE; then
      echo 'CUBLAS works with default -lcublas.'
    else
      echo 'FAILED! error with default flags below:'
      echo
      testnvcc 1
      echo
      exit 1
    fi
  fi 
fi

mkdir -p $BUILDDIR
mkdir -p $BUILDDIR/lib
mkdir -p $BUILDDIR/lib_shared
mkdir -p $BUILDDIR/lib_python
mkdir -p $BUILDDIR/obj
mkdir -p $BUILDDIR/obj_ext
mkdir -p $BUILDDIR/obj_shared
mkdir -p $BUILDDIR/bin

if [ -f $BUILDDIR/config.mk ] ; then
    if [ $BUILDDIR/config.mk -nt $BUILDDIR/how-did-i-configure ] ; then
        read -p "Execution will overwrite existing config.mk file with local modifications. "$'\n'"Are you sure you want to proceed? "$'\n'""$'\n'"(Y/N) :  " -n 1 -r
        echo    # (optional) move to a new line
        if [[ ! $REPLY =~ ^[Yy]$ ]]
        then
            exit 1
        fi
    fi
fi



RAW_LD_LIB_PATH=${LD_LIB_PATH//" -L"/":"}
RAW_LD_LIB_PATH=${RAW_LD_LIB_PATH//"-L"/""}
RAW_LD_LIB_PATH=${RAW_LD_LIB_PATH//" "/""}

if [ $IS_APPLE_CLANG -eq 0 ]; then
  if [ "x$LD_LIB_PATH" != "x" ]; then
    RLD_LIB_PATH=${LD_LIB_PATH//'-L'/'-Wl,-rpath='}
    LD_LIB_PATH="$LD_LIB_PATH $RLD_LIB_PATH"
  fi
fi

cat > $BUILDDIR/config.mk <<EOF
### INSTALLATION TARGET DIRECTORY (for make install)
INSTALL_DIR = $INSTALLDIR

### LINK TIME LIBRARIES AND FLAGS
#libraries and flags for link time (irrelevant if only building CTF lib and not examples/tests)
LIB_PATH     = $LIB_PATH
LIB_FILES    = $LIBS
LINKFLAGS    = $LINKFLAGS
LD_LIB_PATH  = $RAW_LD_LIB_PATH
SO_LIB_PATH  = $LD_LIB_PATH
SO_LIB_FILES = $LD_LIBS
LDFLAGS      = $LDFLAGS


### COMPILE TIME INCLUDES AND FLAGS
#C++ compiler 
CXX         = $CXX
#includes for compile time
INCLUDES    = $INCLUDES
#optimization flags, some intel compiler versions may run into errors when using -fast or -ipo
CXXFLAGS    = $CXXFLAGS $WARNFLAGS 
#command to make library out of object files
AR          = $AR

#macros to be defined throughout the code, use the below in combination with appropriate external libraries
#Include in DEFS -DUSE_LAPACK to build with LAPACK functionality, 
#                -DUSE_SCALAPACK to build with SCALAPACK functionality
#                -DUSE_BATCH_GEMM to build {without, with} batched BLAS routines
#                -DUSE_MKL to build with MKL sparse matrix kernels
#                -DUSE_HPTT to build with optimized tensor transposition routines from HPTT library
DEFS        = $DEFS 
$SPMKL_COMMENT

### Optional: PROFILING AND TUNING
#uncomment below to enable performance profiling
#DEFS       += -DPROFILE -DPMPI
#also uncomment below to enable performance profiling without epochs (started when first CTF World created and stopped when destroyed)
#DEFS       += -DAUTO_PROFILE
#uncomment below to enable automatic performance tuning (loses reproducibility of results)
#recommended usage is to run model_trainer with -DTUNE at scale for suitable duration to obtain suitable architecutral parameters, 
#then recompile with parameters without -DTUNE
#Note: -DTUNE requires lapack (include -mkl or -llapack in LIBS) and the inclusion of above performance profiling flags
#DEFS       += -DTUNE

### Optional: DEBUGGING AND VERBOSITY
#uncomment below to enable CTF execution output (1 for basic contraction information on start-up and contractions)
#DEFS       += -DVERBOSE=1
#uncomment to set debug level to dump information about mapping and internal CTF actions and activate asserts
#DEFS       += -DDEBUG=1

### FULL COMPILE COMMAND AND LIBRARIES
#used to compile all plain C++ files
FCXX        = \$(CXX) \$(CXXFLAGS) \$(DEFS) \$(INCLUDES)
#link-line for all executables
LIBS        = \$(LIB_PATH) \$(LIB_FILES) \$(LINKFLAGS)
#compiler for CUDA files (used to compile CUDA code only when -DOFFLOAD and -DUSE_CUDA are in DEFS, otherwise should be same as FCXX with -x c++)
OFFLOAD_CXX = $NVCC $NVCCFLAGS \$(DEFS) \$(INCLUDES)
EOF



if [ ! -f $BUILDDIR/include/ctf.hpp ] ; then
mkdir -p $BUILDDIR/include
echo "#include \"$CTFDIR/include/ctf.hpp\"" > $BUILDDIR/include/ctf.hpp
fi

if [ ! -e $BUILDDIR/Makefile ] ; then
cat > $BUILDDIR/Makefile <<EOF
CTFDIR=$CTFDIR
all:
%:
	export CTF_BUILD_DIR=$BUILDDIR; cd \$(CTFDIR); \$(MAKE) \$@;
EOF
fi
echo 'A config.mk file has been created (to adjust all settings edit the config.mk file manually or rerun ./configure to create a new one).'

if [ ! -e $BUILDDIR/requirements.txt ] ; then
cat > $BUILDDIR/requirements.txt <<EOF
cython>=0.22
numpy
EOF
fi


if [ -f $BUILDDIR/setup.py ] ; then
    if [ $BUILDDIR/setup.py -nt $BUILDDIR/how-did-i-configure ] ; then
        read -p "Execution will overwrite existing setup.py file with local modifications. Are you sure you ant to proceed "$'\n'""$'\n'"(Y/N)? " -n 1 -r
        echo    # (optional) move to a new line
        if [[ ! $REPLY =~ ^[Yy]$ ]]
        then
            exit 1
        fi
    fi
fi

CORRFLAGS=""
if [ $IS_APPLE_CLANG -eq 1 ]; then
  CORRFLAGS="-stdlib=libc++ -mmacosx-version-min=10.7"
  echo 'Tacking on -stdlib=libc++ -mmacosx-version-min=10.7 to setup.py (relevant only for Python build), since apple clang was detected.'
fi


cat > $BUILDDIR/setup.py <<EOF
from distutils.core import setup
from distutils.extension import Extension
import numpy

import os
os.environ["CXX"] = "$CXX"
os.environ["CC"] = "$CXX"

from Cython.Distutils import build_ext
from Cython.Build import cythonize

mod_profile = ("ctf.profile", ["ctf/profile.pyx"])
mod_linalg = ("ctf.linalg", ["ctf/linalg.pyx"])
mod_multilinear = ("ctf.multilinear", ["ctf/multilinear.pyx"])
mod_helper = ("ctf.helper", ["ctf/helper.pyx"])
mod_chelper = ("ctf.chelper", ["ctf/chelper.pyx"])
mod_tensor = ("ctf.tensor", ["ctf/tensor.pyx"])
mod_tensor_aux = ("ctf.tensor_aux", ["ctf/tensor_aux.pyx"])
mod_world = ("ctf.world", ["ctf/world.pyx"])
mod_partition = ("ctf.partition", ["ctf/partition.pyx"])
mod_rand = ("ctf.random", ["ctf/random.pyx"])
mod_term = ("ctf.term", ["ctf/term.pyx"])

mods = [mod_profile, mod_linalg, mod_multilinear, mod_helper, mod_chelper, mod_tensor, mod_tensor_aux, mod_world, mod_partition, mod_rand, mod_term]

ext_mods = []
for mod in mods:
    ext_mods.append(Extension(
            mod[0],
            mod[1],
            language="c++",
            include_dirs=[numpy.get_include(),"$BUILDDIR/include","."]+"$INCLUDES".replace("-I","").split(),
            libraries="-lctf $LD_LIBS".replace('-l','').replace('$BDYNAMIC','').split(),
            extra_compile_args="$CXXFLAGS $CORRFLAGS".split(),
            extra_link_args="-L$BUILDDIR/lib_shared $CXXFLAGS $LD_LIB_PATH $LINKFLAGS $LDFLAGS".split()
        ))

setup(name="CTF",packages=["ctf"],version="1.5.5",cmdclass = {'build_ext': build_ext},ext_modules = cythonize(ext_mods))

EOF
echo 'A setup.py file has been created (to adjust all settings edit the setup.py file manually or rerun ./configure to create a new one).'


echo $PARAMS >> $BUILDDIR/how-did-i-configure
NUMOFLINES=$(wc -l < "$BUILDDIR/how-did-i-configure")
if [ $NUMOFLINES -ge 100 ]; then
    tail -n +2 "$BUILDDIR/how-did-i-configure" > $BUILDDIR/how-did-i-configure.tmp
    mv $BUILDDIR/how-did-i-configure.tmp $BUILDDIR/how-did-i-configure
fi 


echo 'Configure finished successfully (see how-did-i-configure to determine how script was executed).'