first slurm script+env and Makefile for Irene

MUMPS/machines/irene-amd/Makefile.inc

+#
+#  This file is part of MUMPS 5.2.1, released
+#  on Fri Jun 14 14:46:05 UTC 2019
+#
+#
+#
+# You have to choose one among the following two lines depending on
+# the type of analysis you want to perform. If you want to perform only
+# sequential analysis choose the first (remember to add -Dscotch in the ORDERINGSF
+# variable below); for both parallel and sequential analysis choose the second 
+# line (remember to add -Dptscotch in the ORDERINGSF variable below)
+
+LPORDDIR = $(topdir)/PORD/lib/
+IPORD    = -I$(topdir)/PORD/include/
+LPORD    = -L$(LPORDDIR) -lpord
+
+PARMETISDIR = $(CCCSCRATCHDIR)/AO2020/MUMPS/PARMETIS
+IMETIS      = -I$(METISDIR)/include -I$(PARMETISDIR)/include
+LMETIS      = -L$(PARMETISDIR)/lib -lparmetis -lmetis
+
+
+
+# You have to choose one among the following two lines depending on
+# the type of analysis you want to perform. If you want to perform only
+# sequential analysis choose the first (remember to add -Dmetis in the ORDERINGSF
+# variable below); for both parallel and sequential analysis choose the second 
+# line (remember to add -Dparmetis in the ORDERINGSF variable below)
+
+
+# The following variables will be used in the compilation process.
+# Please note that -Dptscotch and -Dparmetis imply -Dscotch and -Dmetis respectively.
+# If you want to use Metis 4.X or an older version, you should use -Dmetis4 instead of -Dmetis
+# or in addition with -Dparmetis (if you are using parmetis 3.X or older).
+ORDERINGSF  = -Dpord -Dparmetis
+ORDERINGSC  = $(ORDERINGSF)
+
+LORDERINGS = $(LMETIS) $(LPORD)
+IORDERINGSC = $(IMETIS) $(IPORD) 
+IORDERINGSF = $(IORDERINGSC)
+
+#End orderings
+########################################################################
+################################################################################
+
+PLAT    =
+LIBEXT  = .a
+OUTC    = -o 
+OUTF    = -o 
+RM = /bin/rm -f
+CC = mpicc
+FC = mpif90
+FL = mpif90
+AR = ar vr 
+#RANLIB = ranlib
+RANLIB  = echo
+# Make this variable point to the path where the Intel MKL library is
+# installed. It is set to the default install directory for Intel MKL.
+# MKLROOT=/opt/intel/mkl/lib/intel64
+#LAPACK = -L$(MKLROOT) -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core
+#SCALAP = -L$(MKLROOT) -lmkl_scalapack_lp64 -lmkl_blacs_intelmpi_lp64
+
+
+INCPAR =
+LIBPAR = -lscalapack -lblis-mt -lflame -fopenmp -lpthread -lamdlibm
+
+INCSEQ = -I$(topdir)/libseq
+LIBSEQ  = $(LAPACK) -L$(topdir)/libseq -lmpiseq
+
+#LIBBLAS = -L$(MKLROOT) -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core 
+LIBOTHERS = -lpthread
+
+#Preprocessor defs for calling Fortran from C (-DAdd_ or -DAdd__ or -DUPPER)
+CDEFS   = -DAdd_
+
+#Begin Optimized options
+OPTF    = -O3  -DBLR_MT -fopenmp 
+OPTL    = -O3 -fopenmp
+OPTC    = -O3 -fopenmp
+#End Optimized options
+ 
+INCS = $(INCPAR)
+LIBS = $(LIBPAR)
+LIBSEQNEEDED =

MUMPS/machines/irene-amd/batch_charge.slurm

+#!/bin/bash               
+#MSUB -r mumpsben # Name of the job
+#MSUB -N 200                # Number of Nodes
+#MSUB -n 6400                # Number of MPI processes
+#MSUB -c 4                # Number of threads
+#MSUB -T 4000              # Time requested in seconds
+#MSUB -m scratch
+#MSUB -q rome           # Partition
+#MSUB -A tgcc0090       # Project id 
+
+source env_bench
+
+set -x
+
+ulimit -s 400000
+
+ldir=.
+mumpstest=../mumps_bench
+
+
+export OMP_NUM_THREADS=4
+export BLIS_NUM_THREADS=4
+export OMP_PROC_BIND=TRUE
+export OMP_PLACES=cores
+
+#export OMP_STACKSIZE=800M
+export OMP_WAIT_POLICY=active
+export OMP_NESTED=false
+
+
+input=input.txt
+output=./output_${SLURM_JOBID}.log
+
+# block:cyclic pinning mask for 4 OMP on 4 phys cores+resp. hyperthread on irene
+export mask_list='0xF0000000000000000000000000000000F,0xF0000000000000000000000000000000F0000000000000000,0xF0000000000000000000000000000000F00000000,0xF0000000000000000000000000000000F000000000000000000000000,0xF0000000000000000000000000000000F0000,0xF0000000000000000000000000000000F00000000000000000000,0xF0000000000000000000000000000000F000000000000,0xF0000000000000000000000000000000F0000000000000000000000000000,0xF0000000000000000000000000000000F00,0xF0000000000000000000000000000000F000000000000000000,0xF0000000000000000000000000000000F0000000000,0xF0000000000000000000000000000000F00000000000000000000000000,0xF0000000000000000000000000000000F000000,0xF0000000000000000000000000000000F0000000000000000000000,0xF0000000000000000000000000000000F00000000000000,0xF0000000000000000000000000000000F000000000000000000000000000000,0xF0000000000000000000000000000000F0,0xF0000000000000000000000000000000F00000000000000000,0xF0000000000000000000000000000000F000000000,0xF0000000000000000000000000000000F0000000000000000000000000,0xF0000000000000000000000000000000F00000,0xF0000000000000000000000000000000F000000000000000000000,0xF0000000000000000000000000000000F0000000000000,0xF0000000000000000000000000000000F00000000000000000000000000000,0xF0000000000000000000000000000000F000,0xF0000000000000000000000000000000F0000000000000000000,0xF0000000000000000000000000000000F00000000000,0xF0000000000000000000000000000000F000000000000000000000000000,0xF0000000000000000000000000000000F0000000,0xF0000000000000000000000000000000F00000000000000000000000,0xF0000000000000000000000000000000F000000000000000,0xF0000000000000000000000000000000F0000000000000000000000000000000'
+
+
+time ccc_mprun -E "--cpu-bind=mask_cpu:${mask_list}" ${mumpstest} < ${input}   >& ${output}

MUMPS/machines/irene-amd/env_bench

+module purge
+module load gnu/8.3.0 mpi/openmpi/4.0.2
+export LD_LIBRARY_PATH=$CCCSCRATCHDIR/AO2020/MUMPS/PARMETIS/lib:$LD_LIBRARY_PATH
+
+export LD_LIBRARY_PATH=$CCCSCRATCHDIR/AO2020/AOCL/amd/aocl/2.1/amd-blis/lib:$LD_LIBRARY_PATH
+export LD_LIBRARY_PATH=$CCCSCRATCHDIR/AO2020/AOCL/amd/aocl/2.1/amd-libflame/lib:$LD_LIBRARY_PATH
+export LD_LIBRARY_PATH=$CCCSCRATCHDIR/AO2020/AOCL/amd/aocl/2.1/amd-libm/lib:$LD_LIBRARY_PATH
+export LD_LIBRARY_PATH=$CCCSCRATCHDIR/AO2020/AOCL/amd/aocl/2.1/amd-scalapack/lib:$LD_LIBRARY_PATH
+
+GIT_ROOT=`git rev-parse --show-toplevel`
+MUMPS_DIR=$GIT_ROOT/MUMPS
+EXE=$MUMPS_DIR/mumps_bench