added large test case for JZ CPU and GPU

DYNAMICO/machines/jean-zay-cpu/batch_large.slurm

+#!/bin/bash
+#SBATCH --job-name=dynamico      # nom du job
+#SBATCH --ntasks=1024              # Nombre total de processus MPI
+#SBATCH --cpus-per-task=5
+#SBATCH --hint=nomultithread       # 1 processus MPI par coeur physique (pas d'hyperthreading)
+#SBATCH --time=00:20:00            # Temps d?excution maximum demande (HH:MM:SS)
+#SBATCH --output=dynamico.out  # Nom du fichier de sortie
+#SBATCH --error=dynamico.err   # Nom du fichier d'erreur (ici commun avec la sortie)
+#SBATCH -A qbg@cpu 
+ 
+# on se place dans le rpertoire de soumission
+cd ${SLURM_SUBMIT_DIR}
+ 
+ulimit -a
+ulimit -s unlimited
+source env_bench 
+
+export OMP_NUM_THREADS=${SLURM_CPUS_PER_TASK}
+export OMP_NUM_THREADS=5
+export OMP_STACKSIZE=100M
+export OMP_PLACES=cores
+
+mkdir -p large_output
+cp run.def large_output
+cp const.def large_output
+cd large_output
+
+srun $EXE_NO_IO

DYNAMICO/machines/jean-zay-cpu/run_large.def

+INCLUDEDEF = const.def
+
+mpi_threading_mode=funneled
+#---------------- Mesh ----------------
+
+# Number of subdivisions on a main triangle : integer (default=40)
+nbp = 640
+
+# Number of vertical layers : integer (default=19)
+llm = 79
+
+# Vertical grid : [std|ncar|ncarl30] (default=std)
+disvert = std
+
+# Mesh optimisation : number of iterations : integer (default=0)
+optim_it = 1000
+
+# Sub splitting of main rhombus : integer (default=1)
+nsplit_i = 32
+nsplit_j = 32
+
+omp_level_size=5
+
+#itau_check_conserv=1
+itau_adv=2
+#---------------- Numerics ----------------
+
+#scheme
+
+scheme = RK2.5
+
+# Time step in s : real (default=480)
+dt = 30
+
+# Dissipation time for grad(div) : real (default=5000)
+tau_graddiv = 600
+
+# Exponent of grad(div) disspation : integer (default=1)
+nitergdiv = 2
+
+# Dissipation time for curl(curl) : real (default=5000)
+tau_gradrot = 600
+
+# Exponent of curl(curl) disspation : integer (default=1)
+nitergrot = 2
+
+# Dissipation time for div(grad) : real (default=5000)
+tau_divgrad = 600
+
+# Exponent of div(grad) disspation : integer (default=1)
+niterdivgrad = 2
+
+
+#---------------- Time ----------------
+
+# Run length in s : real (default=??)
+run_length = 864000
+
+
+enable_io=no
+
+# Interval in s between two outputs : integer (default=??)
+write_period = 864000
+
+# using xios (default=yes)
+xios_output=no
+
+#---------------- Physical parameters ----------------
+
+# Number of tracers : integer (default=1)
+nqtot = 2
+
+# Initial state : 
+#   [jablonowsky06|academic|dcmip[1-4]|heldsz|dcmip2_schaer_noshear] (default=jablonowsky06)
+etat0 = dcmip4
+
+# DCMIP 4 Test case : [1|2] (default=1)
+dcmip4_testcase = 1
+
+

DYNAMICO/machines/jean-zay-gpu/batch_large.slurm

+#!/bin/bash
+#SBATCH --job-name=dynamico      # nom du job
+#SBATCH --ntasks=80              # Nombre total de processus MPI
+#SBATCH --ntasks-per-node=4          # nombre de tâche MPI par noeud
+#SBATCH --gres=gpu:4                 # nombre de GPU à réserver par nœud
+#SBATCH --cpus-per-task=10           # nombre de coeurs à réserver par tâche#SBATCH --hint=nomultithread       # 1 processus MPI par coeur physique (pas d'hyperthreading)
+#SBATCH --distribution=block:block   # on épingle les tâches sur des coeurs contigus
+#SBATCH --time=01:00:00            # Temps d?excution maximum demande (HH:MM:SS)
+#SBATCH --output=dynamico.out  # Nom du fichier de sortie
+#SBATCH --error=dynamico.err   # Nom du fichier d'erreur (ici commun avec la sortie)
+#SBATCH -A qbg@gpu 
+ 
+# on se place dans le rpertoire de soumission
+ 
+ulimit -a
+ulimit -s unlimited
+source env_bench 
+
+mkdir -p medium_output
+cp run.def medium_output
+cp const.def medium_output
+cd medium_output
+
+srun $EXE_NO_IO 

DYNAMICO/machines/jean-zay-gpu/batch_medium.slurm

+#!/bin/bash
+#SBATCH --job-name=dynamico      # nom du job
+#SBATCH --ntasks=4              # Nombre total de processus MPI
+#SBATCH --ntasks-per-node=4          # nombre de tâche MPI par noeud
+#SBATCH --gres=gpu:4                 # nombre de GPU à réserver par nœud
+#SBATCH --cpus-per-task=10           # nombre de coeurs à réserver par tâche#SBATCH --hint=nomultithread       # 1 processus MPI par coeur physique (pas d'hyperthreading)
+#SBATCH --distribution=block:block   # on épingle les tâches sur des coeurs contigus
+#SBATCH --time=00:20:00            # Temps d?excution maximum demande (HH:MM:SS)
+#SBATCH --output=dynamico.out  # Nom du fichier de sortie
+#SBATCH --error=dynamico.err   # Nom du fichier d'erreur (ici commun avec la sortie)
+#SBATCH -A qbg@gpu 
+ 
+# on se place dans le rpertoire de soumission
+ 
+ulimit -a
+ulimit -s unlimited
+source env_bench 
+
+mkdir -p medium_output
+cp run.def medium_output
+cp const.def medium_output
+cd medium_output
+
+srun $EXE_NO_IO 
+

DYNAMICO/machines/jean-zay-gpu/env_bench

+# Load the correct modules:
+#
+module purge
+
+module load git
+module load gcc/9.1.0
+module load pgi/20.1
+module load cuda/10.1.2
+module load openmpi/4.0.2-cuda 
+module load hdf5/1.10.5-mpi-cuda
+module load netcdf/4.7.2-mpi-cuda 
+module load netcdf-fortran/4.5.2-mpi-cuda 
+
+export LD_PRELOAD=/gpfslocalsup/spack_soft/gcc/9.1.0/gcc-4.8.5-bzyowvwzeyqlfsnfyh3bgztur3uhca55/lib64/libstdc++.so
+export PSM2_CUDA_MEMCACHE_SIZE=256
+
+COMPILE_ARCH=JEANZAY_PGI_ACC
+PARALLEL_FLAG=mpi
+
+GIT_ROOT=`git rev-parse --show-toplevel`
+DYNAMICO_DIR=$GIT_ROOT/DYNAMICO/
+
+EXE=$DYNAMICO_DIR/DYNAMICO/bin/icosa_gcm.exe
+EXE_NO_IO=$DYNAMICO_DIR/DYNAMICO/bin_noio/icosa_gcm.exe
+EXE_DYN_XIOS=$DYNAMICO_DIR/DYNAMICO/bin_xios/icosa_gcm.exe
+EXE_XIOS=$DYNAMICO_DIR/XIOS/bin/xios_server.exe

DYNAMICO/machines/jean-zay-gpu/run_large.def

+INCLUDEDEF = const.def
+
+mpi_threading_mode=funneled
+#---------------- Mesh ----------------
+
+# Number of subdivisions on a main triangle : integer (default=40)
+nbp = 640
+
+# Number of vertical layers : integer (default=19)
+llm = 79
+
+# Vertical grid : [std|ncar|ncarl30] (default=std)
+disvert = std
+
+# Mesh optimisation : number of iterations : integer (default=0)
+optim_it = 1000
+
+# Sub splitting of main rhombus : integer (default=1)
+nsplit_i = 8
+nsplit_j = 8
+
+omp_level_size=1
+
+#itau_check_conserv=1
+itau_adv=2
+#---------------- Numerics ----------------
+
+#scheme
+
+scheme = RK2.5
+
+# Time step in s : real (default=480)
+dt = 30
+
+# Dissipation time for grad(div) : real (default=5000)
+tau_graddiv = 600
+
+# Exponent of grad(div) disspation : integer (default=1)
+nitergdiv = 2
+
+# Dissipation time for curl(curl) : real (default=5000)
+tau_gradrot = 600
+
+# Exponent of curl(curl) disspation : integer (default=1)
+nitergrot = 2
+
+# Dissipation time for div(grad) : real (default=5000)
+tau_divgrad = 600
+
+# Exponent of div(grad) disspation : integer (default=1)
+niterdivgrad = 2
+
+
+#---------------- Time ----------------
+
+# Run length in s : real (default=??)
+run_length = 864000
+
+
+enable_io=no
+
+# Interval in s between two outputs : integer (default=??)
+write_period = 864000
+
+# using xios (default=yes)
+xios_output=no
+
+#---------------- Physical parameters ----------------
+
+# Number of tracers : integer (default=1)
+nqtot = 2
+
+# Initial state : 
+#   [jablonowsky06|academic|dcmip[1-4]|heldsz|dcmip2_schaer_noshear] (default=jablonowsky06)
+etat0 = dcmip4
+
+# DCMIP 4 Test case : [1|2] (default=1)
+dcmip4_testcase = 1
+
+

DYNAMICO/machines/jean-zay-gpu/run_medium.def

+	INCLUDEDEF = const.def
+
+mpi_threading_mode=funneled
+#---------------- Mesh ----------------
+
+# Number of subdivisions on a main triangle : integer (default=40)
+nbp = 40
+
+# Number of vertical layers : integer (default=19)
+llm = 79
+
+# Vertical grid : [std|ncar|ncarl30] (default=std)
+disvert = std
+
+# Mesh optimisation : number of iterations : integer (default=0)
+optim_it = 1000
+
+# Sub splitting of main rhombus : integer (default=1)
+nsplit_i = 1
+nsplit_j = 1
+
+omp_level_size=1
+
+#itau_check_conserv=1
+itau_adv=2
+#---------------- Numerics ----------------
+
+#scheme
+
+scheme = RK2.5
+
+
+# Time step in s : real (default=480)
+dt = 720
+
+# Dissipation time for grad(div) : real (default=5000)
+tau_graddiv = 9000
+
+# Exponent of grad(div) disspation : integer (default=1)
+nitergdiv = 2
+
+# Dissipation time for curl(curl) : real (default=5000)
+tau_gradrot = 9000
+
+# Exponent of curl(curl) disspation : integer (default=1)
+nitergrot = 2
+
+# Dissipation time for div(grad) : real (default=5000)
+tau_divgrad = 9000
+
+# Exponent of div(grad) disspation : integer (default=1)
+niterdivgrad = 2
+
+
+#---------------- Time ----------------
+
+# Run length in s : real (default=??)
+run_length = 8640000
+
+
+enable_io=no
+
+# Interval in s between two outputs : integer (default=??)
+write_period = 8640
+
+# using xios (default=yes)
+xios_output=no
+
+#---------------- Physical parameters ----------------
+
+# Number of tracers : integer (default=1)
+nqtot = 2
+
+# Initial state : 
+#   [jablonowsky06|academic|dcmip[1-4]|heldsz|dcmip2_schaer_noshear] (default=jablonowsky06)
+etat0 = dcmip4
+
+# DCMIP 4 Test case : [1|2] (default=1)
+dcmip4_testcase = 1
+
+

DYNAMICO/testcase_large/.gitignore

+batch_large.slurm
+dynamico.out
+env_bench
+large_output
+results-DYNAMICO-large
+run.def
+checkconsicosa.txt

DYNAMICO/testcase_large/const.def

+
+#---------------- Planet ----------------
+
+# Planetary radius : real (default=6371220)
+radius = 6.37122e6
+
+# Gravity : real (default=9.80616)
+g = 9.80616
+
+# kappa=Rd/cpp : real (default=0.2857143)
+kappa = 0.2857143
+
+# Heat capacity at constant pressure : real (default=1004.70885)
+cpp = 1004.5
+
+# Reference pressure : real (default=101325)
+preff = 1e5
+
+# Planetary rotation rate : real (default=7.292E-5)
+omega = 7.292e-5
+
+
+#---------------- Constants ----------------
+
+# Reference temperature T0 for DCMIP test cases : real (default=300)
+ncar_T0 = 300
+
+# Reference pressure p0 for DCMIP test cases, used by disvert if disver=ncar : real (default=1e5)
+ncar_p0 = 1e5
+
+# Exponent for B(eta), used by disvert if disvert=ncar : integer (default=1)
+ncar_disvert_c = 1
+

DYNAMICO/testcase_large/prepare.sh

+#!/bin/bash
+
+if [ -z "$1" ]
+  then
+    echo "Please provide the targeted machine from:"
+    ls ../machines/
+    echo ""
+    echo "Example: ./prepare.sh occigen-bdw"
+    exit 1
+fi
+machine_dir="../machines/$1"
+
+cp $machine_dir/env_bench .
+cp $machine_dir/batch_large.slurm .
+cp $machine_dir/run_large.def run.def

DYNAMICO/testcase_large/run.sh

+#!/bin/bash
+
+sbatch batch_large.slurm

DYNAMICO/testcase_large/validate.sh

+#!/bin/bash
+set -e
+
+mass=`tail -n10 dynamico.out | grep "GLOB "| awk '{print $2}' | awk -F 'E' '{print $2}'`
+upper_bound=-11
+
+test=$(awk 'BEGIN{printf ('$mass'<'$upper_bound')?0:1}')
+echo $test
+
+if [ $test -eq "0" ] 
+	then 
+		echo "Validation passed"
+	else 
+		echo "Validation failed"
+		exit 1
+fi
+mkdir -p results-DYNAMICO-large
+cp dynamico.out results-DYNAMICO-large
+

DYNAMICO/testcase_medium_xios/validate.sh

+#!/bin/bash
+set -e
+
+mass=`tail -n10 dynamico.out | grep "GLOB "| awk '{print $2}' | awk -F 'E' '{print $2}'`
+upper_bound=-11
+
+test=$(awk 'BEGIN{printf ('$mass'<'$upper_bound')?0:1}')
+echo $test
+
+if [ $test -eq "0" ] 
+	then 
+		echo "Validation passed"
+	else 
+		echo "Validation failed"
+		exit 1
+fi
+mkdir -p results-DYNAMICO-medium
+cp dynamico.out results-DYNAMICO-medium
+