Skip to content

[Manual] Devito on CX3 Imperial College HPC

Jump to bottom
George Bisbas edited this page Apr 3, 2024 · 6 revisions

Useful: HPC pages:

Job sizing guidance

# Do `ssh` to your login node (Usually logs to AMD EPYC 7742 64-Core Processor)
ssh gb4018@login.hpc.imperial.ac.uk

# Load Anaconda personal
module load anaconda3/personal

# Alternative path with Python3.11
module load tools/prod
module load Python/3.11.2-GCCcore-12.2.0-bare

# Load OpenMPI
module load tools/dev
module load OpenMPI/4.1.4-GCC-12.2.0

Load intel compilers (gcc, icc), tools..etc

While these nodes will run software built using the Intel MPI and Compiler, we strongly recommend rebuilding your software using GCC and OpenMPI as we observe greater performance with these.
# Load git module
module load git

# Following instructions from https://github.com/opesci/devito
git clone https://github.com/devitocodes/devito.git
cd devito
conda env create -f environment-dev.yml
source activate devito
pip install -e .

# If everything went fine, you should be able to run a typical operator. i.e.:
export DEVITO_LOGGING=DEBUG
export DEVITO_LANGUAGE=openmp
python examples/seismic/acoustic/acoustic_example.py

For MPI:

# Instal MPI requirements
pip install --force-reinstall --upgrade --no-cache-dir -r requirements-mpi.txt

For interactive jobs:

qsub -I -l select=1:ncpus=2:mem=8gb -l walltime=02:00:00

WIP .PBS for some intel machine

#!/bin/bash

#PBS -lselect=1:ncpus=24:mem=120gb:mpiprocs=2:ompthreads=12
#PBS -lwalltime=00:30:00

lscpu
whoami

cd /rdsgpfs/general/user/$(whoami)/home/devitocodes/devito

export DEVITO_HOME=/rdsgpfs/general/user/$(whoami)/home/devitocodes/devito

module load tools/prod
module load Python/3.11.2-GCCcore-12.2.0-bare
source environments/python311-env/bin/activate

module load intel-compilers/2022.2.1
module load mpi

export DEVITO_ARCH=intel
export DEVITO_LANGUAGE=openmp
export DEVITO_LOGGING=DEBUG
export DEVITO_MPI=1

# export TMPDIR=/rds/general/user/$(whoami)/home/devitocodes/cache

mpiexec python3 $DEVITO_HOME/examples/seismic/acoustic/acoustic_example.py d 100 100 100 -so 8 --tn 200 --autotune aggressive

For Rome:

#!/bin/bash

#PBS -lselect=1:ncpus=128:mem=100gb:mpiprocs=8:ompthreads=16
#PBS -lwalltime=02:00:00

lscpu
whoami

cd /rdsgpfs/general/user/$(whoami)/home/devitocodes/devito

export DEVITO_HOME=/rdsgpfs/general/user/$(whoami)/home/devitocodes/devito

module load tools/prod
module load Python/3.11.2-GCCcore-12.2.0-bare
source environments/python311-env/bin/activate

module load intel-compilers/2022.2.1
module load mpi

export DEVITO_ARCH=gcc
export DEVITO_LANGUAGE=openmp
export DEVITO_LOGGING=DEBUG
export DEVITO_MPI=1

# export TMPDIR=/rds/general/user/$(whoami)/home/devitocodes/cache

mpiexec python3 $DEVITO_HOME/examples/seismic/acoustic/acoustic_example.py d 100 100 100 -so 8 --tn 200 --autotune aggressive
Clone this wiki locally