diff --git a/configs/perlmutter.config b/configs/perlmutter.config new file mode 100644 index 0000000000..831b2759d9 --- /dev/null +++ b/configs/perlmutter.config @@ -0,0 +1,36 @@ +# NERSC Perlmutter (HPE Cray Compiler Wrappers) + +### Caution!! Load the module below before compiling +### module load gcc-native/12.3 + +CUDA_PATH /opt/nvidia/hpc_sdk/Linux_x86_64/23.9/cuda/12.2/ +FFTW2_PATH +FFTW3_PATH /opt/cray/pe/fftw/3.3.10.6/x86_milan/ +MPI_PATH /opt/cray/pe/craype/2.7.30/ # This is actually the path to CC (just for perlmutter). +HDF5_PATH /opt/cray/pe/hdf5/default/gnu/12.3 +#HDF5_PATH /opt/cray/pe/hdf5-parallel/1.12.2.9/gnu/12.3/ +GRACKLE_PATH +GSL_PATH +LIBYT_PATH +CUFFTDX_PATH + +# compilers +CXX CC +CXX_MPI CC # MPI compiler (The MPI_PATH/bin/ will be combined automatically) + +# flags +CXXFLAG -target-accel=nvidia80 # Also available in the environment variable +CXXFLAG -g +CXXFLAG -O2 + +OPENMPFLAG -fopenmp + +#LIBFLAG + +NVCCFLAG_COM -O3 -I/opt/cray/pe/mpich/8.1.28/ofi/gnu/12.3/include +#NVCCFLAG_COM -use_fast_math +NVCCFLAG_FLU -Xptxas -dlcm=ca -prec-div=false -ftz=true +NVCCFLAG_POT -Xptxas -dlcm=ca + +# gpu +GPU_COMPUTE_CAPABILITY 800 # NVIDIA A100 diff --git a/example/queue/submit_perlmutter_cpu.job b/example/queue/submit_perlmutter_cpu.job new file mode 100644 index 0000000000..fecb5dd84f --- /dev/null +++ b/example/queue/submit_perlmutter_cpu.job @@ -0,0 +1,21 @@ +#!/bin/bash +#SBATCH --account=YOUR_ACCOUNT +#SBATCH --job-name=GAMER_CPU +#SBATCH --qos=debug +#SBATCH --time=0:30:00 +#SBATCH --nodes=1 +#SBATCH --mail-type=BEGIN,END,FAIL +#SBATCH --mail-user=YOUR_EMAIL + +########## Do NOT modify this block ########## +#SBATCH --constraint=cpu +#SBATCH --ntasks-per-node=1 +#SBATCH --cpus-per-task=256 +export OMP_PROC_BIND=spread +export OMP_PLACES=threads +############################################## + +module load gcc-native/12.3 +module list # Record the loaded modules + +srun ./gamer diff --git a/example/queue/submit_perlmutter_gpu.job b/example/queue/submit_perlmutter_gpu.job new file mode 100644 index 0000000000..6de4b0fd61 --- /dev/null +++ b/example/queue/submit_perlmutter_gpu.job @@ -0,0 +1,22 @@ +#!/bin/bash +#SBATCH --account=YOUR_ACCOUNT_g +#SBATCH --job-name=GAMER_GPU +#SBATCH --qos=debug +#SBATCH --time=0:30:00 +#SBATCH --nodes=1 +#SBATCH --mail-type=BEGIN,END,FAIL +#SBATCH --mail-user=YOUR_EMAIL + +########## Do NOT modify this block ########## +#SBATCH --constraint=gpu +#SBATCH --ntasks-per-node=4 +#SBATCH --cpus-per-task=32 +#SBATCH --gpus-per-task=1 +export OMP_PROC_BIND=spread +export OMP_PLACES=threads +############################################## + +module load gcc-native/12.3 +module list # Record the loaded modules + +srun ./gamer