Last commit

GYSELAX/testcase_gpu/subgys

+#!/bin/bash
+
+# -----------------------------------------------------------------
+# script used for submit gysela5D job
+# -----------------------------------------------------------------
+arg=$#
+
+if [ $# -lt 1 ]; then
+    echo " Syntaxe:  subgys DATA_filename [debug|timer]         "
+    echo " with optional debug                            "
+    echo "      to run with executable in debug mode      "
+    echo " with optional timer                            "
+    echo "      to run with executable in timer mode      "
+    exit
+fi
+
+# case name
+# ---------
+CASENAME=`basename $1 | tr '[:lower:]' '[:upper:]'`
+OPTIONALMODE=`echo $2 | tr '[:lower:]' '[:upper:]'`
+MACHNAME=`echo $ARCH | tr '[:lower:]' '[:upper:]'`
+
+FIRSTCAR=${MACHNAME:0:1}
+PART=""
+if [ "$ARCH" == "marconi_knl" ]; then
+    PART="_KNL"
+fi
+if [ "$ARCH" == "marconi_skl" ]; then
+    PART="_SKL"
+fi
+CASE=D${FIRSTCAR}${PART}_${CASENAME}
+if [ "${OPTIONALMODE}" = "DEBUG" ]; then
+    echo " ==> run with executable in DEBUG mode "
+    CASE=${CASE}_DBG
+fi
+if [ "${OPTIONALMODE}" = "TIMER" ]; then
+    echo " ==> run with executable in TIMER mode "
+    CASE=${CASE}_TI
+fi
+
+# data name
+# ---------
+DATANAME=$1
+
+# job parameters
+# --------------
+# number of RUNS
+NB_RESTART=`grep NB_RESTART $DATANAME | head -n 1 | tr -s '=' ' ' |  awk '{print $2}'`
+# time in seconds format
+TIMEH=$(grep TIME $DATANAME | head -n 1 | sed -n 's/.*[^0-9]\([0-9]*[0-9]:[0-9][0-9]:[0-9][0-9]\).*/\1/p')
+TIME=$(echo $TIMEH |awk -F: '{print 3600*$1+60*$2+$3}')
+# job Name
+JOBNAME=`grep JOBNAME $DATANAME  | head -n 1 | tr -s '=' ' ' |  awk '{print $2}'`
+STATUSOK=0
+KILLCMD=""
+
+# MPI/OpenMP resources
+# --------------------
+NSPECIES=`grep NSPECIES $DATANAME | head -n 1 | tr -s '=' ' ' |  awk '{print $2}'`
+NPROC_R=`grep NPROC_R   $DATANAME | head -n 1 | tr -s '=' ' ' |  awk '{print $2}'`
+NPROC_TH=`grep NPROC_TH $DATANAME | head -n 1 | tr -s '=' ' ' |  awk '{print $2}'`
+NPROC_MU=`grep NPROC_MU $DATANAME | head -n 1 | tr -s '=' ' ' |  awk '{print $2}'`
+NTHREAD=`grep NTHREAD   $DATANAME | head -n 1 | tr -s '=' ' ' | tr -s ',' ' ' | awk '{print $2}'`
+echo " NTHREAD " $NTHREAD
+
+((NPES=NPROC_R*NPROC_TH*NPROC_MU*NSPECIES))
+((MPIPROCS=8/$NTHREAD))
+((NNODES=(NPES*NTHREAD)/8))
+if [ $NNODES -eq 0 ]; then
+    ((NNODES=1))
+fi
+
+# Validate DATA file
+# ------------------
+if [ -f ./validate_data ]; then
+    if ! /usr/bin/env python ./validate_data $DATANAME $OPTIONALMODE
+    then
+        echo
+        echo "  You can bypass this test by removing the file wk/validate_data"
+        echo "  If your data file is correct but fails to pass this test"
+        echo "  please send it to the gysela mailing list"
+        exit 1
+    fi
+else
+    echo "  wk/validate_data not found, skipping validation"
+fi
+
+# Environment
+# -----------
+DIR=$(basename $(dirname $PWD))
+
+# Debug mode
+# ----------
+if [ "${OPTIONALMODE}" = "DEBUG" ]; then
+    exec_file=gysela_dbg.exe
+    go_file_cmd="go_"$DATANAME"_dbg.cmd"
+else if [ "${OPTIONALMODE}" = "TIMER" ]; then
+    exec_file=gysela_ti.exe
+    go_file_cmd="go_"$DATANAME"_ti.cmd"
+    else
+        exec_file=gysela.exe
+	go_file_cmd="go_"$DATANAME".cmd"
+    fi
+fi
+if [ -f ${go_file_cmd} ]; then
+    rm -f ${go_file_cmd}
+fi
+
+# According to ARCH variable
+# --------------------------
+case ${ARCH} in
+
+# -----------------------------------------------------------------
+# occigen2 Broadwell (CINES)
+# -----------------------------------------------------------------
+occigen2)
+
+# Directories
+HOMEDIR=${GYSELA_PATH}
+WKDIR=${PWD}
+RESDIR=${PWD}/${CASE}
+
+# We launch 56 threads per node even with 28 cores.
+((THPNODE=56))
+((MPIPROCS=$THPNODE/$NTHREAD))
+((CPUTASK=$THPNODE/(2*$MPIPROCS)))
+((NCPUS=$NPES*$NTHREAD))
+((NNODES=($THPNODE-1+$NPES*$NTHREAD)/$THPNODE))
+if [ $NNODES -eq 0 ]; then
+    ((NNODES=1))
+fi
+
+if [ $NTHREAD -ne 14 -a $NTHREAD -ne 28 ]; then
+    echo "Please set NTHREAD to 14 or 28 in your input file"
+    exit
+fi
+#
+CMD_MPIRUN="srun -K1 -m block:block --resv-ports -n $NPES ${RESDIR}/${exec_file} 1>> ${RESDIR}/gysela_log.out 2>> ${RESDIR}/gysela_res.err"
+CMD_SUB="sbatch --wait ${go_file_cmd}"
+CMD_SUBRST=${CMD_SUB}
+
+TSLEEP=$(awk "BEGIN{printf(\"%.0f\",$TIME*.9);}")
+
+cat <<EOF > ${go_file_cmd}
+#!/bin/bash
+#SBATCH -J $JOBNAME             # jobname
+#SBATCH --constraint=BDW28      # processor Broadwell     
+#SBATCH --nodes=${NNODES}
+#BATCH --ntasks-per-node=${MPIPROCS}
+#BATCH --threads-per-core=1
+#BATCH --cpus-per-task=${CPUTASK}
+#SBATCH -t ${TIMEH}            # execute time (hh:mm:ss - max 08:00:00)
+#SBATCH --mem=50GB
+#SBATCH --exclusive
+#SBATCH -o $JOBNAME%j.out      # strout filename (%j is jobid)
+#SBATCH -e $JOBNAME%j.out      # stderr filename (%j is jobid)
+
+#export KMP_AFFINITY="verbose,scatter"
+export KMP_AFFINITY="verbose"
+export OMP_NUM_THREADS=${NTHREAD}
+ulimit -d unlimited
+ulimit -Sa
+ulimit -Ha
+ulimit -l unlimited
+
+unset FORT_BUFFERED
+
+set +v
+
+EOF
+
+;;
+
+
+*)
+esac
+
+# Data file
+# ---------
+grep "&PARALLEL" -A 500 $1 > data.${CASE}
+
+sed -i "s/  Nspecies .*/  Nspecies     = $NSPECIES/"  data.${CASE}
+sed -i "s/  Nbproc_r .*/  Nbproc_r     = $NPROC_R/"  data.${CASE}
+sed -i "s/  Nbproc_theta .*/  Nbproc_theta = $NPROC_TH/" data.${CASE}
+sed -i "s/  Nbproc_mu .*/  Nbproc_mu    = $NPROC_MU/" data.${CASE}
+sed -i "s/  Nbthread .*/  Nbthread     = $NTHREAD/"  data.${CASE}
+sed -i "s/  Nmu .*/  Nmu            = $((NPROC_MU-1))/"  data.${CASE}
+
+# Names of the profile files
+# --------------------------
+ns0_sp0_filename=`grep ns0_sp0_filename $DATANAME | head -n 1 | tr -s '=' ' ' |  awk '{print $2}'`
+ns0_sp1_filename=`grep ns0_sp1_filename $DATANAME | head -n 1 | tr -s '=' ' ' |  awk '{print $2}'`
+Ts0_sp0_filename=`grep Ts0_sp0_filename $DATANAME | head -n 1 | tr -s '=' ' ' |  awk '{print $2}'`
+Te0_filename=`grep Te0_filename $DATANAME | head -n 1 | tr -s '=' ' ' |  awk '{print $2}'`
+Ts0_sp1_filename=`grep Ts0_sp1_filename $DATANAME | head -n 1 | tr -s '=' ' ' |  awk '{print $2}'`
+q_filename=`grep q_filename $DATANAME | head -n 1 | tr -s '=' ' ' |  awk '{print $2}'`
+read_q=`grep read_q $DATANAME | head -n 1 | tr -s '=' ' ' |  awk '{print $2}'`
+Vpars0_sp0_filename=`grep Vpars0_sp0_filename $DATANAME | head -n 1 | tr -s '=' ' ' |  awk '{print $2}'`
+Vpars0_sp1_filename=`grep Vpars0_sp1_filename $DATANAME | head -n 1 | tr -s '=' ' ' |  awk '{print $2}'`
+ripple_filename=`grep ripple_filename $DATANAME | head -n 1 | tr -s '=' ' ' |  awk '{print $2}'`
+TF_ripple=`grep TF_ripple $DATANAME | head -n 1 | tr -s '=' ' ' |  awk '{print $2}'`
+buffer_filename=`grep buffer_filename $DATANAME | head -n 1 | tr -s '=' ' ' |  awk '{print $2}'`
+read_buffer_profile=`grep read_buffer_profile $DATANAME | head -n 1 | tr -s '=' ' ' | tr '[:upper:]' '[:lower:]' |  awk '{print $2}'`
+pulse_filename_r=`grep pulse_filename_r $DATANAME | head -n 1 | tr -s '=' ' ' |  awk '{print $2}'`
+pulse_filename_th=`grep pulse_filename_th $DATANAME | head -n 1 | tr -s '=' ' ' |  awk '{print $2}'`
+pulse_filename_phi=`grep pulse_filename_phi $DATANAME | head -n 1 | tr -s '=' ' ' |  awk '{print $2}'`
+read_pulse_profile=`grep read_pulse_profile $DATANAME | head -n 1 | tr -s '=' ' ' | tr '[:upper:]' '[:lower:]' |  awk '{print $2}'`
+
+# Reading of the input file for
+# ns0, Ts0, Te0 and q if they exist 
+# ------------------------------
+for ((ns=0;ns<=${NSPECIES}-1;ns++)); do
+    if [ $ns -eq "0" ]; then
+	#--> for density reading of first species
+	ns0_filename=${ns0_sp0_filename}
+	ns0_GYS="ns0_sp0.dat"
+	read_ns0=`grep read_ns0 $DATANAME | head -n 1 | tr -s '=' ' ' | tr -s ',' ' ' | awk '{print $2}'`
+	#--> for temperature reading of first species
+	Ts0_filename=${Ts0_sp0_filename}
+	Ts0_GYS="Ts0_sp0.dat"
+	read_Ts0=`grep read_Ts0 $DATANAME | head -n 1 | tr -s '=' ' ' | tr -s ',' ' ' | awk '{print $2}'`
+	#--> for temperature reading of the electrons
+	Te0_filename=${Te0_filename}
+	Te0_GYS="Te0.dat"
+	read_Te0=`grep read_Te0 $DATANAME | head -n 1 | tr -s '=' ' ' | tr -s ',' ' ' | awk '{print $2}'`
+	#--> for Vpars0 of first species
+	Vpars0_filename=${Vpars0_sp0_filename}
+	Vpars0_GYS="Vpars0_sp0.dat"
+	read_Vpars0=`grep read_Vpars0 $DATANAME | head -n 1 | tr -s '=' ' ' | tr -s ',' ' ' | awk '{print $2}'`	
+    fi 
+    if [ $ns -eq "1" ]; then
+	#--> for density reading of second species
+	ns0_filename=${ns0_sp1_filename}
+	ns0_GYS="ns0_sp1.dat"
+	read_ns0=`grep read_ns0 $DATANAME | head -n 1 | tr -s '=' ' ' | tr -s ',' ' ' | awk '{print $3}'`
+	#--> for temperature reading of second species
+	Ts0_filename=${Ts0_sp1_filename}
+	Ts0_GYS="Ts0_sp1.dat"
+	read_Ts0=`grep read_Ts0 $DATANAME | head -n 1 | tr -s '=' ' ' | tr -s ',' ' ' | awk '{print $3}'`
+	#--> for Vpars0 of second species
+	Vpars0_filename=${Vpars0_sp1_filename}
+	Vpars0_GYS="Vpars0_sp1.dat"
+	read_Vpars0=`grep read_Vpars0 $DATANAME | head -n 1 | tr -s '=' ' ' | tr -s ',' ' ' | awk '{print $3}'`	
+    fi
+    #--> copy of the file containing the ion density profile
+    if [ "$read_ns0" == '.true.' ]; then
+	echo " => reading of the file " ${ns0_filename} "for species " ${ns}
+	if [ -e ${ns0_filename} ]; then
+	    cp ${ns0_filename} ${RESDIR}/${ns0_GYS}
+	else
+	    echo "The file " ${ns0_filename} " does not exist for species " ${ns}
+	    exit
+	fi
+    fi
+    #--> copy of the file containing the ion temperature profile
+    if [ "$read_Ts0" == '.true.' ]; then
+	echo " => reading of the file " ${Ts0_filename} "for species " ${ns}
+	if [ -e ${Ts0_filename} ]; then
+	    cp ${Ts0_filename} ${RESDIR}/${Ts0_GYS}
+	else
+	    echo "The file " ${Ts0_filename} " does not exist for species " ${ns}
+	    exit
+	fi
+    fi
+    #--> copy of the file containing the electron temperature profile
+    if [ "$read_Te0" == '.true.' ]; then
+	echo " => reading of the file " ${Te0_filename}
+	if [ -e ${Te0_filename} ]; then
+	    cp ${Te0_filename} ${RESDIR}/${Te0_GYS}
+	else
+	    echo "The file " ${Te0_filename} " does not exist "
+	    exit
+	fi
+    fi
+    #--> copy of the file containing the parallel velocity profile
+    if [ "$read_Vpars0" == '.true.' ]; then
+       echo " => reading of the file " ${Vpars0_filename}
+       if [ -e ${Vpars0_filename} ]; then
+          cp ${Vpars0_filename}  ${RESDIR}/${Vpars0_GYS}
+       else
+          echo "The file " ${Vpars0_filename} " does not exist"
+          exit
+       fi
+    fi
+done
+
+#--> copy of the file containing the electron temperature profile
+if [ "$read_Te0" == '.true.' ]; then
+    echo " => reading of the file " ${Te0_filename}
+    if [ -e ${Te0_filename} ]; then
+	cp ${Te0_filename} ${RESDIR}/Te0.dat
+    else
+	echo "The file " ${Te0_filename} " does not exist"
+	exit
+    fi
+fi
+
+#--> copy of the file containing the safety factor profile
+if [ "$read_q" == '.true.' ]; then
+    echo " => reading of the file " ${q_filename}
+    if [ -e ${q_filename} ]; then
+	cp ${q_filename}  ${RESDIR}/safety_factor.dat
+    else
+	echo "The file " ${q_filename} " does not exist"
+	exit
+    fi
+fi
+
+#--> copy of the file containing the ripple profile
+if [ "$TF_ripple" == '.true.' ]; then    
+    if [ -e ${ripple_filename} ]; then
+	echo " => reading of the file " ${ripple_filename}
+	cp ${ripple_filename}  ${RESDIR}/ripple.h5
+    else
+	echo -e "\e[31m The file '${ripple_filename}' does not exist\e[0m"
+	echo " You can use script 'ripple.py' to generate ripple HDF5 data file"
+	echo " Usage:  /usr/bin/env python ${WKDIR}/ripple.py $DATANAME ${ripple_filename}"
+	exit 1
+    fi
+fi
+
+#--> copy of the file containing the radial buffer profile
+if [ "$read_buffer_profile" == '.true.' ]; then
+    echo " => reading of the file " ${buffer_filename}
+    if [ -e ${buffer_filename} ]; then
+	cp ${buffer_filename}  ${RESDIR}/buffer_profile.dat
+    else
+	echo "The file " ${buffer_filename} " does not exist"
+	exit
+    fi
+fi
+
+#--> copy of the file containing the radial Pulse profile
+if [ ! -z "$read_pulse_profile" ]; then 
+    if [ "$read_pulse_profile" -eq 1 ] || [ "$read_pulse_profile" -eq 3 ] || [ "$read_pulse_profile" -eq 5 ] || [ "$read_pulse_profile" -eq 7 ]; then
+	echo " => reading of the file " ${pulse_filename_r}
+	if [ -e ${pulse_filename_r} ]; then
+	    cp ${pulse_filename_r}  ${RESDIR}/pulse_profile_r.dat
+	else
+	    echo "The file " ${pulse_filename_r} " does not exist"
+	    exit
+	fi
+    fi
+    if [ "$read_pulse_profile" -eq 2 ] || [ "$read_pulse_profile" -eq 3 ] || [ "$read_pulse_profile" -gt 6 ]; then
+	echo " => reading of the file " ${pulse_filename_th}
+	if [ -e ${pulse_filename_th} ]; then
+	    cp ${pulse_filename_th}  ${RESDIR}/pulse_profile_th.dat
+	else
+	    echo "The file " ${pulse_filename_th} " does not exist"
+	    exit
+	fi
+    fi
+    if [ "$read_pulse_profile" -gt 4 ]; then
+	echo " => reading of the file " ${pulse_filename_phi}
+	if [ -e ${pulse_filename_phi} ]; then
+	    cp ${pulse_filename_phi}  ${RESDIR}/pulse_profile_phi.dat
+	else
+	    echo "The file " ${pulse_filename_phi} " does not exist"
+	    exit
+	fi
+    fi
+fi
+
+STATEDIR=init_state
+SRCSDIR=sources
+CLDIR=conservation_laws
+RPROFDIR=rprof
+F2DDIR=f2D
+PHI2DDIR=Phi2D
+COLLISIONDIR=collisions
+PHI3DDIR=Phi3D
+FM3DDIR=moment3D
+FLUXES3DDIR=fluxes3D
+SPDIR=spectra
+F5DDIR=f5D
+POPEDIR=POPE
+MTM_TRACE=mtm_trace
+RST_FILES=rst_files
+
+mkdir -p ${RESDIR}/${MTM_TRACE}
+
+for DIR in ${STATEDIR} ${SRCSDIR} ${CLDIR} ${RPROFDIR} ${F2DDIR} \
+    ${PHI2DDIR} ${COLLISIONDIR} ${PHI3DDIR} ${FM3DDIR} ${FLUXES3DDIR} \
+    ${SPDIR} ${F5DDIR} ${POPEDIR} ${RST_FILES}
+do
+    for ((i=0;i<=${NSPECIES}-1;i++)); do
+	if [ ! -d ${RESDIR}/sp$i/${DIR} ]; then
+	    mkdir -p ${RESDIR}/sp$i/${DIR}
+	fi
+    done
+done 
+
+# Init restart files 
+# --------------------------------
+echo " "
+SEARCH_SP0=`grep SEARCH_SP0 $DATANAME | head -n 1 | tr -s '=' ' ' |  awk '{print $2}'`
+if [ "${SEARCH_SP0}" = '' -o "${SEARCH_SP0}" = '0' ];then
+    echo " No restart from another simulation"
+    SEARCH_SP0=0
+else
+    DIR_SP0=`grep DIR_SP0 $DATANAME | head -n 1 | tr -s '=' ' ' |  awk '{print $2}'`
+    DIR_SP0_RST=$DIR_SP0'/sp0/rst_files'
+    if [ "$DIR_SP0_RST" == "" ]; then
+	printf "%s %0.f \t\t %s \n" " SEARCH_SP0 = " $SEARCH_SP0 " so you must specify a directory with restarts files !!"
+        exit
+    fi
+    # test if DIR_SP0_RST exist
+    echo " Restart with previous files from repository:" $DIR_SP0
+    if [ ! -d $DIR_SP0_RST ]; then
+	echo " Directory $DIR_SP0_RST doesn't exist !! "
+	exit
+    fi
+fi
+
+# Number of expected restart files
+CKPTMODE=`grep "^\s*checkpointing_strategy.*" $DATANAME | head -n 1 | tr -s '=' ' ' | tr -d '"' | awk '{print $2}' `
+if [ "$CKPTMODE" == "SION" ]; then
+    echo " checkpointing:  SION"
+    Nb_rst_exp=1
+    FIRST_RST_SUFF="sionlib"
+    ALL_RST_SUFF="sionlib"
+    EXTENSION_SUFF="sionlib"
+else
+    CKPTMODE="HDF5"
+    echo " checkpointing: HDF5"
+    ((Nb_rst_exp=NPROC_R*NPROC_TH*NPROC_MU))
+    FIRST_RST_SUFF="fn.p00000.h5"
+    ALL_RST_SUFF="fn.p*.h5"
+    EXTENSION_SUFF="h5"
+fi
+
+# Restart with previous simulation
+if [ ${SEARCH_SP0} -ne '0' ]; then
+    # Restart with previous sp0 files
+    #if [ -f ./check_nbrestart ]; then
+    #    echo "/usr/bin/env python ./check_nbrestart ${DIR_SP0} ${CKPTMODE} ${NB_RESTART} acronym"
+    #    TAB="`/usr/bin/env python ./check_nbrestart ${DIR_SP0} ${CKPTMODE} ${NB_RESTART} acronym`"
+    #fi
+    rst_acronym=$(echo $TAB | cut -d' ' -f2)
+    echo "RST_acronym -${rst_acronym}-"
+    # Copy this restart file
+    for ((i=0;i<=${NSPECIES}-1;i++)); do
+	DIR_RST=$DIR_SP0/sp$i/rst_files
+	if [ -e $DIR_RST ]; then
+            # Number of restart files in DIR_RST
+	    Nb_rst_DIR_RST=`ls $DIR_RST/gys_rst.sp$i.${rst_acronym}.${FIRST_RST_SUFF} | wc -l`
+            # Copy of this restart file
+	    rst_file='gys_rst.sp'$i'.'${rst_acronym}'.'${ALL_RST_SUFF}
+	    if [ -f ERR ]; then rm ERR; fi
+	    echo "  => rsync restart files for sp$i "
+	    rsync -avur ${DIR_RST}/${rst_file} $RESDIR/sp$i/rst_files/. 2>> ERR
+	    status=$?;
+	    if [ $status -ne 0 ]; then
+		echo -e " \e[31m Problem with rsync command !!  so exit \e[0m"
+		cat ERR
+		rm ERR
+		exit
+	    fi
+            # Number of restart files in result directory
+	    Nb_rst_count=`ls ${RESDIR}/sp$i/rst_files/gys_rst*${rst_acronym}.${FIRST_RST_SUFF} | wc -l`
+            # Compare this two numbers
+	    if [ $Nb_rst_count -eq $Nb_rst_DIR_RST ]; then
+		echo " ... rsync OK"; echo ""
+	    else
+                # RSYNC not OK ??
+		echo " \e[31m All the restart files doesn't exist !! \e[0m"
+		exit
+	    fi
+
+	    # Copy metadata file
+	    metadata="${DIR_RST}/metadata.${rst_acronym}.${EXTENSION_SUFF}"
+	    if [ -f ${metadata} ]; then
+		echo "${metadata} exist"
+
+		if [ -f ERR ]; then rm ERR; fi
+		echo "  => rsync ${metadata} file for sp$i"
+		rsync -avur ${metadata} $RESDIR/sp$i/rst_files/. 2>> ERR
+		status=$?;
+		if [ $status -ne 0 ]; then
+		    echo -e " \e[31m Problem with rsync command for metadata !! \e[0m"
+		    rm ERR
+		    exit
+		fi
+	    else
+		echo -e " \e[31m ! NO metadata file exist \e[0m"
+                # Test if restart files are in old format
+                # Test made also in create_rst_metadata.py
+		if [ -f ${DIR_RST}/gys_rst.sp$i.${rst_acronym}.${FIRST_RST_SUFF} ]; then
+		    if [ ${CKPTMODE} == "HDF5" ]; then
+			time_diag=`h5ls -d ${DIR_RST}/gys_rst.sp$i.${rst_acronym}.${FIRST_RST_SUFF}/time_diag`
+			time_diag=$(echo $time_diag | cut -d ' ' -f 6 | tr '\n' ' ')
+			if [ $time_diag != "" ]; then
+			    echo  " Restart files are in Old configuration, so executing script:"
+			    cmd="/usr/bin/env python ../post-process/Adapt_rst_files/create_rst_metadata.py ${RESDIR} ${i} ${rst_acronym}"
+			    echo -e "  \e[34m" $cmd "\e[0m"
+			    eval $cmd
+			    status=$?
+			    if [ $status -ne 0 ]; then
+				echo -e " \e[31m Problem with create_rst_metadata.py !! \e[0m"
+				rm ERR
+				exit
+			    fi
+			else
+			    echo -e " \e[31m Problem reading ${DIR_RST}/gys_rst.sp$i.${rst_acronym}.${FIRST_RST_SUFF} !! \e[0m"
+			fi
+		    else
+			echo -e " \e[31m Restart files are in Sionlib format, no metadata could be generated \e[0m"
+		    fi 
+		fi
+	    fi
+	fi
+    done
+else
+    if [ -e ${RESDIR}"/sp0/rst_files/gys_rst.sp0.n0.${FIRST_RST_SUFF}" ] || [ -e ${RESDIR}"/sp0/rst_files/gys_rst.sp0.n1.${FIRST_RST_SUFF}" ]; then
+	if [ -f ${WKDIR}/check_nbrestart ]; then
+	    TAB="`/usr/bin/env python ${WKDIR}/check_nbrestart ${RESDIR} ${CKPTMODE} ${NB_RESTART} acronym`"
+	fi
+	nb_restart=$(echo $TAB | cut -d' ' -f1)
+	rst_acronym=$(echo $TAB | cut -d' ' -f2)
+    else
+	nb_restart=-1
+	rst_acronym="n0"
+	echo " Initialisation ......................."
+    fi
+fi
+
+if [ -f ERR ]; then rm ERR; fi
+
+# Number of restart files in result directory
+if [ ! -f ${RESDIR}/sp0/rst_files/gys_rst.sp0.${rst_acronym}.${FIRST_RST_SUFF} ]; then
+    Nb_rst_count=0
+else
+    Nb_rst_count=`ls \${RESDIR}/sp0/rst_files/gys_rst.sp0.${rst_acronym}.${ALL_RST_SUFF} | wc -l`
+fi
+if [ $Nb_rst_exp -eq $Nb_rst_count ]; then
+    echo " Restart with files already in the directory"
+    echo ""
+fi
+
+# Test if a restart run is required
+if [ ${Nb_rst_count} -ne '0' ] && [ -f ./check_nbrestart ]; then
+    if ! /usr/bin/env python ./check_nbrestart ${RESDIR} ${CKPTMODE} ${NB_RESTART}; then exit 1; fi
+fi
+
+# Create autostop script
+# ----------------------
+touch ${RESDIR}/automatic_stop.sh
+chmod +x ${RESDIR}/automatic_stop.sh
+printf "sleep $TSLEEP\n"      > ${RESDIR}/automatic_stop.sh
+printf "$KILLCMD\n"          >> ${RESDIR}/automatic_stop.sh
+printf "touch gysela.stop\n" >> ${RESDIR}/automatic_stop.sh
+
+# Create script submission
+# ------------------------
+cp ${WKDIR}/${DATANAME} ${RESDIR}/input.${CASE}
+mv ${WKDIR}/data.${CASE} ${RESDIR}/DATA
+if [ ! -f ${RESDIR}/${exec_file} ]; then
+    if [ "${OPTIONALMODE}" = "DEBUG" ]; then    
+	if [ ! -f ${WKDIR}/gysela_dbg.exe ]; then
+	    echo " !! gysela not compiled in mode debug !! "
+	    exit
+	fi
+    fi
+    if [ "${OPTIONALMODE}" = "TIMER" ]; then    
+	if [ ! -f ${WKDIR}/gysela_ti.exe ]; then
+	    echo " !! gysela not compiled in mode timer !! "
+	    exit
+	fi
+    fi
+    cp ${WKDIR}/${exec_file} ${RESDIR}/${exec_file}
+fi
+cd ${WKDIR}
+
+cat <<EOF >> ${go_file_cmd}
+
+echo "list of loaded modules:"
+echo " =============================="
+module list || true
+echo " =============================="
+echo
+
+function halt_gys () {
+    if [ "$ARCH" == "ouessant" ]; then 
+        kill -s 9 0
+    else
+        kill -9 \$PID
+        rm -f auto.out auto.err autostop.pid mpistop.pid
+    fi
+    exit \$1
+}
+
+if [ -z "$WAITJOBEND" ]
+then
+set -vx
+fi
+cd ${RESDIR}
+
+# Script submission
+# -----------------
+if [ -e "sp0/rst_files/gys_rst.sp0.n0.${FIRST_RST_SUFF}" ] || [ -e "sp0/rst_files/gys_rst.sp0.n1.${FIRST_RST_SUFF}" ]; then
+    if [ -f ${WKDIR}/check_nbrestart ]; then
+       TAB="`/usr/bin/env python ${WKDIR}/check_nbrestart ${RESDIR} ${CKPTMODE} ${NB_RESTART} acronym`"
+    fi
+    nb_restart=\$(echo \$TAB | cut -d' ' -f1)
+    rst_acronym=\$(echo \$TAB | cut -d' ' -f2)
+else
+    nb_restart=-1
+    echo " Initialisation ......................."
+fi
+
+# Create script to stop gysela automatically
+# whenever the wall clock time limit is too low
+# ---------------------------------------------
+if [ -e gysela.stop ]; then
+    rm -f gysela.stop
+fi
+
+# test if nb_restart <= NB_RESTART
+# -----------------------------
+if [ "\$nb_restart" -le "$NB_RESTART" ]; then
+    echo " You want "$NB_RESTART" restart(s), and "\$nb_restart" are made "
+    ${RESDIR}/automatic_stop.sh 1>${RESDIR}/auto.out 2>${RESDIR}/auto.err &
+    PID=\$!
+    echo \$PID > ${RESDIR}/autostop.pid
+   
+    # run gysela.exe :
+    # ----------------
+    # Reset gysela_status file
+    echo "-1 : not started" >  gysela_status.txt
+   
+    export OMP_NUM_THREADS=${NTHREAD}
+    ${CMD_MPIRUN}
+    status=\$(head -n 1 gysela_status.txt| awk '{print \$1}')
+   
+    mv mtm_*.out ${MTM_TRACE}
+    for ((ns=0;ns<=${NSPECIES}-1;ns++)); do
+        mv sp\${ns}/coord_system.h5       sp\${ns}/${STATEDIR} 
+        mv sp\${ns}/init_state*.h5        sp\${ns}/${STATEDIR}
+        mv sp\${ns}/data_*.h5             sp\${ns}/${STATEDIR}
+        mv sp\${ns}/sources_*.h5          sp\${ns}/${SRCSDIR}
+        mv sp\${ns}/conservation_laws*.h5 sp\${ns}/${CLDIR}
+        mv sp\${ns}/rprof_*.h5            sp\${ns}/${RPROFDIR}
+        mv sp\${ns}/Phi2D*.h5             sp\${ns}/${PHI2DDIR}
+        mv sp\${ns}/f2D*.h5               sp\${ns}/${F2DDIR}
+        mv sp\${ns}/collision*.h5         sp\${ns}/${COLLISIONDIR}
+        mv sp\${ns}/Phi_3D*.h5            sp\${ns}/${PHI3DDIR}
+        mv sp\${ns}/Phi_3D*.bin           sp\${ns}/${PHI3DDIR} 2>/dev/null
+        mv sp\${ns}/n*_GC_3D_*.h5         sp\${ns}/${FM3DDIR}
+        mv sp\${ns}/V_GC_3D_*.h5          sp\${ns}/${FM3DDIR}
+        mv sp\${ns}/Ppar_GC_3D_*.h5       sp\${ns}/${FM3DDIR}
+        mv sp\${ns}/Pperp*_GC_3D_*.h5     sp\${ns}/${FM3DDIR}
+        mv sp\${ns}/Gamma_*_3D*.h5        sp\${ns}/${FLUXES3DDIR}
+        mv sp\${ns}/Q*_3D*.h5             sp\${ns}/${FLUXES3DDIR}
+        mv sp\${ns}/spectra*.h5           sp\${ns}/${SPDIR}
+        mv sp\${ns}/f5D*.h5               sp\${ns}/${F5DDIR}
+        mv sp\${ns}/POPE*.h5              sp\${ns}/${POPEDIR}
+        mv sp\${ns}/gys_rst.sp\${ns}.*.${EXTENSION_SUFF} sp\${ns}/${RST_FILES} 2>/dev/null
+        mv sp\${ns}/meta*.${EXTENSION_SUFF} sp\${ns}/${RST_FILES} 2>/dev/null
+    done
+  
+    cd ${WKDIR}
+   
+    if [ "$NB_RESTART" -ne "0" ]; then
+        if [ "\$status" -eq ${STATUSOK} ]; then
+            # new submission
+            # --------------
+            if [ -f ./check_nbrestart ]; then
+                if ! /usr/bin/env python ./check_nbrestart ${RESDIR} ${CKPTMODE} ${NB_RESTART}
+                then
+                    halt_gys 1
+                fi
+            fi
+            ${CMD_SUBRST}
+        fi
+    fi
+fi
+halt_gys 0
+EOF
+
+# -----------------------------------------------------------------
+# Test presence of CMD_MPIRUN  
+# -----------------------------------------------------------------
+command_mpirun=`echo $CMD_MPIRUN | awk '{print $1}'`
+test_command_mpirun=`which $command_mpirun 2> /dev/null`
+if [ ${#test_command_mpirun} -eq 0 ]; then
+    echo " "
+    echo "-----------------------------------------------------------------------"
+    echo -e "\033[31m***ERROR: \033[0m $command_mpirun is not installed --> load GYSELA environment"
+    echo "-----------------------------------------------------------------------"
+    echo " "
+    exit
+fi
+
+# -----------------------------------------------------------------                                                                                                                          
+# Launch submission
+# -----------------------------------------------------------------
+chmod +x ${go_file_cmd}
+cd ${WKDIR}
+echo " ... ${CMD_SUB}"
+${CMD_SUB}
+if [ -n "$WAITJOBEND" ]; then
+    cat "${RESDIR}/gysela_log.out" "${RESDIR}/gysela_res.err"
+fi

GYSELAX/testcase_xlarge/subgys

+#!/bin/bash
+
+# -----------------------------------------------------------------
+# script used for submit gysela5D job
+# -----------------------------------------------------------------
+arg=$#
+
+if [ $# -lt 1 ]; then
+    echo " Syntaxe:  subgys DATA_filename [debug|timer]         "
+    echo " with optional debug                            "
+    echo "      to run with executable in debug mode      "
+    echo " with optional timer                            "
+    echo "      to run with executable in timer mode      "
+    exit
+fi
+
+# case name
+# ---------
+CASENAME=`basename $1 | tr '[:lower:]' '[:upper:]'`
+OPTIONALMODE=`echo $2 | tr '[:lower:]' '[:upper:]'`
+MACHNAME=`echo $ARCH | tr '[:lower:]' '[:upper:]'`
+
+FIRSTCAR=${MACHNAME:0:1}
+PART=""
+if [ "$ARCH" == "marconi_knl" ]; then
+    PART="_KNL"
+fi
+if [ "$ARCH" == "marconi_skl" ]; then
+    PART="_SKL"
+fi
+CASE=D${FIRSTCAR}${PART}_${CASENAME}
+if [ "${OPTIONALMODE}" = "DEBUG" ]; then
+    echo " ==> run with executable in DEBUG mode "
+    CASE=${CASE}_DBG
+fi
+if [ "${OPTIONALMODE}" = "TIMER" ]; then
+    echo " ==> run with executable in TIMER mode "
+    CASE=${CASE}_TI
+fi
+
+# data name
+# ---------
+DATANAME=$1
+
+# job parameters
+# --------------
+# number of RUNS
+NB_RESTART=`grep NB_RESTART $DATANAME | head -n 1 | tr -s '=' ' ' |  awk '{print $2}'`
+# time in seconds format
+TIMEH=$(grep TIME $DATANAME | head -n 1 | sed -n 's/.*[^0-9]\([0-9]*[0-9]:[0-9][0-9]:[0-9][0-9]\).*/\1/p')
+TIME=$(echo $TIMEH |awk -F: '{print 3600*$1+60*$2+$3}')
+# job Name
+JOBNAME=`grep JOBNAME $DATANAME  | head -n 1 | tr -s '=' ' ' |  awk '{print $2}'`
+STATUSOK=0
+KILLCMD=""
+
+# MPI/OpenMP resources
+# --------------------
+NSPECIES=`grep NSPECIES $DATANAME | head -n 1 | tr -s '=' ' ' |  awk '{print $2}'`
+NPROC_R=`grep NPROC_R   $DATANAME | head -n 1 | tr -s '=' ' ' |  awk '{print $2}'`
+NPROC_TH=`grep NPROC_TH $DATANAME | head -n 1 | tr -s '=' ' ' |  awk '{print $2}'`
+NPROC_MU=`grep NPROC_MU $DATANAME | head -n 1 | tr -s '=' ' ' |  awk '{print $2}'`
+NTHREAD=`grep NTHREAD   $DATANAME | head -n 1 | tr -s '=' ' ' | tr -s ',' ' ' | awk '{print $2}'`
+echo " NTHREAD " $NTHREAD
+
+((NPES=NPROC_R*NPROC_TH*NPROC_MU*NSPECIES))
+((MPIPROCS=8/$NTHREAD))
+((NNODES=(NPES*NTHREAD)/8))
+if [ $NNODES -eq 0 ]; then
+    ((NNODES=1))
+fi
+
+# Validate DATA file
+# ------------------
+if [ -f ./validate_data ]; then
+    if ! /usr/bin/env python ./validate_data $DATANAME $OPTIONALMODE
+    then
+        echo
+        echo "  You can bypass this test by removing the file wk/validate_data"
+        echo "  If your data file is correct but fails to pass this test"
+        echo "  please send it to the gysela mailing list"
+        exit 1
+    fi
+else
+    echo "  wk/validate_data not found, skipping validation"
+fi
+
+# Environment
+# -----------
+DIR=$(basename $(dirname $PWD))
+
+# Debug mode
+# ----------
+if [ "${OPTIONALMODE}" = "DEBUG" ]; then
+    exec_file=gysela_dbg.exe
+    go_file_cmd="go_"$DATANAME"_dbg.cmd"
+else if [ "${OPTIONALMODE}" = "TIMER" ]; then
+    exec_file=gysela_ti.exe
+    go_file_cmd="go_"$DATANAME"_ti.cmd"
+    else
+        exec_file=gysela.exe
+	go_file_cmd="go_"$DATANAME".cmd"
+    fi
+fi
+if [ -f ${go_file_cmd} ]; then
+    rm -f ${go_file_cmd}
+fi
+
+# According to ARCH variable
+# --------------------------
+case ${ARCH} in
+
+# -----------------------------------------------------------------
+# occigen2 Broadwell (CINES)
+# -----------------------------------------------------------------
+occigen2)
+
+# Directories
+HOMEDIR=${GYSELA_PATH}
+WKDIR=${PWD}
+RESDIR=${PWD}/${CASE}
+
+# We launch 56 threads per node even with 28 cores.
+((THPNODE=56))
+((MPIPROCS=$THPNODE/$NTHREAD))
+((CPUTASK=$THPNODE/(2*$MPIPROCS)))
+((NCPUS=$NPES*$NTHREAD))
+((NNODES=($THPNODE-1+$NPES*$NTHREAD)/$THPNODE))
+if [ $NNODES -eq 0 ]; then
+    ((NNODES=1))
+fi
+
+if [ $NTHREAD -ne 14 -a $NTHREAD -ne 28 ]; then
+    echo "Please set NTHREAD to 14 or 28 in your input file"
+    exit
+fi
+#
+CMD_MPIRUN="srun -K1 -m block:block --resv-ports -n $NPES ${RESDIR}/${exec_file} 1>> ${RESDIR}/gysela_log.out 2>> ${RESDIR}/gysela_res.err"
+CMD_SUB="sbatch --wait ${go_file_cmd}"
+CMD_SUBRST=${CMD_SUB}
+
+TSLEEP=$(awk "BEGIN{printf(\"%.0f\",$TIME*.9);}")
+
+cat <<EOF > ${go_file_cmd}
+#!/bin/bash
+#SBATCH -J $JOBNAME             # jobname
+#SBATCH --constraint=BDW28      # processor Broadwell     
+#SBATCH --nodes=${NNODES}
+#BATCH --ntasks-per-node=${MPIPROCS}
+#BATCH --threads-per-core=1
+#BATCH --cpus-per-task=${CPUTASK}
+#SBATCH -t ${TIMEH}            # execute time (hh:mm:ss - max 08:00:00)
+#SBATCH --mem=50GB
+#SBATCH --exclusive
+#SBATCH -o $JOBNAME%j.out      # strout filename (%j is jobid)
+#SBATCH -e $JOBNAME%j.out      # stderr filename (%j is jobid)
+
+#export KMP_AFFINITY="verbose,scatter"
+export KMP_AFFINITY="verbose"
+export OMP_NUM_THREADS=${NTHREAD}
+ulimit -d unlimited
+ulimit -Sa
+ulimit -Ha
+ulimit -l unlimited
+
+unset FORT_BUFFERED
+
+set +v
+
+EOF
+
+;;
+
+
+*)
+esac
+
+# Data file
+# ---------
+grep "&PARALLEL" -A 500 $1 > data.${CASE}
+
+sed -i "s/  Nspecies .*/  Nspecies     = $NSPECIES/"  data.${CASE}
+sed -i "s/  Nbproc_r .*/  Nbproc_r     = $NPROC_R/"  data.${CASE}
+sed -i "s/  Nbproc_theta .*/  Nbproc_theta = $NPROC_TH/" data.${CASE}
+sed -i "s/  Nbproc_mu .*/  Nbproc_mu    = $NPROC_MU/" data.${CASE}
+sed -i "s/  Nbthread .*/  Nbthread     = $NTHREAD/"  data.${CASE}
+sed -i "s/  Nmu .*/  Nmu            = $((NPROC_MU-1))/"  data.${CASE}
+
+# Names of the profile files
+# --------------------------
+ns0_sp0_filename=`grep ns0_sp0_filename $DATANAME | head -n 1 | tr -s '=' ' ' |  awk '{print $2}'`
+ns0_sp1_filename=`grep ns0_sp1_filename $DATANAME | head -n 1 | tr -s '=' ' ' |  awk '{print $2}'`
+Ts0_sp0_filename=`grep Ts0_sp0_filename $DATANAME | head -n 1 | tr -s '=' ' ' |  awk '{print $2}'`
+Te0_filename=`grep Te0_filename $DATANAME | head -n 1 | tr -s '=' ' ' |  awk '{print $2}'`
+Ts0_sp1_filename=`grep Ts0_sp1_filename $DATANAME | head -n 1 | tr -s '=' ' ' |  awk '{print $2}'`
+q_filename=`grep q_filename $DATANAME | head -n 1 | tr -s '=' ' ' |  awk '{print $2}'`
+read_q=`grep read_q $DATANAME | head -n 1 | tr -s '=' ' ' |  awk '{print $2}'`
+Vpars0_sp0_filename=`grep Vpars0_sp0_filename $DATANAME | head -n 1 | tr -s '=' ' ' |  awk '{print $2}'`
+Vpars0_sp1_filename=`grep Vpars0_sp1_filename $DATANAME | head -n 1 | tr -s '=' ' ' |  awk '{print $2}'`
+ripple_filename=`grep ripple_filename $DATANAME | head -n 1 | tr -s '=' ' ' |  awk '{print $2}'`
+TF_ripple=`grep TF_ripple $DATANAME | head -n 1 | tr -s '=' ' ' |  awk '{print $2}'`
+buffer_filename=`grep buffer_filename $DATANAME | head -n 1 | tr -s '=' ' ' |  awk '{print $2}'`
+read_buffer_profile=`grep read_buffer_profile $DATANAME | head -n 1 | tr -s '=' ' ' | tr '[:upper:]' '[:lower:]' |  awk '{print $2}'`
+pulse_filename_r=`grep pulse_filename_r $DATANAME | head -n 1 | tr -s '=' ' ' |  awk '{print $2}'`
+pulse_filename_th=`grep pulse_filename_th $DATANAME | head -n 1 | tr -s '=' ' ' |  awk '{print $2}'`
+pulse_filename_phi=`grep pulse_filename_phi $DATANAME | head -n 1 | tr -s '=' ' ' |  awk '{print $2}'`
+read_pulse_profile=`grep read_pulse_profile $DATANAME | head -n 1 | tr -s '=' ' ' | tr '[:upper:]' '[:lower:]' |  awk '{print $2}'`
+
+# Reading of the input file for
+# ns0, Ts0, Te0 and q if they exist 
+# ------------------------------
+for ((ns=0;ns<=${NSPECIES}-1;ns++)); do
+    if [ $ns -eq "0" ]; then
+	#--> for density reading of first species
+	ns0_filename=${ns0_sp0_filename}
+	ns0_GYS="ns0_sp0.dat"
+	read_ns0=`grep read_ns0 $DATANAME | head -n 1 | tr -s '=' ' ' | tr -s ',' ' ' | awk '{print $2}'`
+	#--> for temperature reading of first species
+	Ts0_filename=${Ts0_sp0_filename}
+	Ts0_GYS="Ts0_sp0.dat"
+	read_Ts0=`grep read_Ts0 $DATANAME | head -n 1 | tr -s '=' ' ' | tr -s ',' ' ' | awk '{print $2}'`
+	#--> for temperature reading of the electrons
+	Te0_filename=${Te0_filename}
+	Te0_GYS="Te0.dat"
+	read_Te0=`grep read_Te0 $DATANAME | head -n 1 | tr -s '=' ' ' | tr -s ',' ' ' | awk '{print $2}'`
+	#--> for Vpars0 of first species
+	Vpars0_filename=${Vpars0_sp0_filename}
+	Vpars0_GYS="Vpars0_sp0.dat"
+	read_Vpars0=`grep read_Vpars0 $DATANAME | head -n 1 | tr -s '=' ' ' | tr -s ',' ' ' | awk '{print $2}'`	
+    fi 
+    if [ $ns -eq "1" ]; then
+	#--> for density reading of second species
+	ns0_filename=${ns0_sp1_filename}
+	ns0_GYS="ns0_sp1.dat"
+	read_ns0=`grep read_ns0 $DATANAME | head -n 1 | tr -s '=' ' ' | tr -s ',' ' ' | awk '{print $3}'`
+	#--> for temperature reading of second species
+	Ts0_filename=${Ts0_sp1_filename}
+	Ts0_GYS="Ts0_sp1.dat"
+	read_Ts0=`grep read_Ts0 $DATANAME | head -n 1 | tr -s '=' ' ' | tr -s ',' ' ' | awk '{print $3}'`
+	#--> for Vpars0 of second species
+	Vpars0_filename=${Vpars0_sp1_filename}
+	Vpars0_GYS="Vpars0_sp1.dat"
+	read_Vpars0=`grep read_Vpars0 $DATANAME | head -n 1 | tr -s '=' ' ' | tr -s ',' ' ' | awk '{print $3}'`	
+    fi
+    #--> copy of the file containing the ion density profile
+    if [ "$read_ns0" == '.true.' ]; then
+	echo " => reading of the file " ${ns0_filename} "for species " ${ns}
+	if [ -e ${ns0_filename} ]; then
+	    cp ${ns0_filename} ${RESDIR}/${ns0_GYS}
+	else
+	    echo "The file " ${ns0_filename} " does not exist for species " ${ns}
+	    exit
+	fi
+    fi
+    #--> copy of the file containing the ion temperature profile
+    if [ "$read_Ts0" == '.true.' ]; then
+	echo " => reading of the file " ${Ts0_filename} "for species " ${ns}
+	if [ -e ${Ts0_filename} ]; then
+	    cp ${Ts0_filename} ${RESDIR}/${Ts0_GYS}
+	else
+	    echo "The file " ${Ts0_filename} " does not exist for species " ${ns}
+	    exit
+	fi
+    fi
+    #--> copy of the file containing the electron temperature profile
+    if [ "$read_Te0" == '.true.' ]; then
+	echo " => reading of the file " ${Te0_filename}
+	if [ -e ${Te0_filename} ]; then
+	    cp ${Te0_filename} ${RESDIR}/${Te0_GYS}
+	else
+	    echo "The file " ${Te0_filename} " does not exist "
+	    exit
+	fi
+    fi
+    #--> copy of the file containing the parallel velocity profile
+    if [ "$read_Vpars0" == '.true.' ]; then
+       echo " => reading of the file " ${Vpars0_filename}
+       if [ -e ${Vpars0_filename} ]; then
+          cp ${Vpars0_filename}  ${RESDIR}/${Vpars0_GYS}
+       else
+          echo "The file " ${Vpars0_filename} " does not exist"
+          exit
+       fi
+    fi
+done
+
+#--> copy of the file containing the electron temperature profile
+if [ "$read_Te0" == '.true.' ]; then
+    echo " => reading of the file " ${Te0_filename}
+    if [ -e ${Te0_filename} ]; then
+	cp ${Te0_filename} ${RESDIR}/Te0.dat
+    else
+	echo "The file " ${Te0_filename} " does not exist"
+	exit
+    fi
+fi
+
+#--> copy of the file containing the safety factor profile
+if [ "$read_q" == '.true.' ]; then
+    echo " => reading of the file " ${q_filename}
+    if [ -e ${q_filename} ]; then
+	cp ${q_filename}  ${RESDIR}/safety_factor.dat
+    else
+	echo "The file " ${q_filename} " does not exist"
+	exit
+    fi
+fi
+
+#--> copy of the file containing the ripple profile
+if [ "$TF_ripple" == '.true.' ]; then    
+    if [ -e ${ripple_filename} ]; then
+	echo " => reading of the file " ${ripple_filename}
+	cp ${ripple_filename}  ${RESDIR}/ripple.h5
+    else
+	echo -e "\e[31m The file '${ripple_filename}' does not exist\e[0m"
+	echo " You can use script 'ripple.py' to generate ripple HDF5 data file"
+	echo " Usage:  /usr/bin/env python ${WKDIR}/ripple.py $DATANAME ${ripple_filename}"
+	exit 1
+    fi
+fi
+
+#--> copy of the file containing the radial buffer profile
+if [ "$read_buffer_profile" == '.true.' ]; then
+    echo " => reading of the file " ${buffer_filename}
+    if [ -e ${buffer_filename} ]; then
+	cp ${buffer_filename}  ${RESDIR}/buffer_profile.dat
+    else
+	echo "The file " ${buffer_filename} " does not exist"
+	exit
+    fi
+fi
+
+#--> copy of the file containing the radial Pulse profile
+if [ ! -z "$read_pulse_profile" ]; then 
+    if [ "$read_pulse_profile" -eq 1 ] || [ "$read_pulse_profile" -eq 3 ] || [ "$read_pulse_profile" -eq 5 ] || [ "$read_pulse_profile" -eq 7 ]; then
+	echo " => reading of the file " ${pulse_filename_r}
+	if [ -e ${pulse_filename_r} ]; then
+	    cp ${pulse_filename_r}  ${RESDIR}/pulse_profile_r.dat
+	else
+	    echo "The file " ${pulse_filename_r} " does not exist"
+	    exit
+	fi
+    fi
+    if [ "$read_pulse_profile" -eq 2 ] || [ "$read_pulse_profile" -eq 3 ] || [ "$read_pulse_profile" -gt 6 ]; then
+	echo " => reading of the file " ${pulse_filename_th}
+	if [ -e ${pulse_filename_th} ]; then
+	    cp ${pulse_filename_th}  ${RESDIR}/pulse_profile_th.dat
+	else
+	    echo "The file " ${pulse_filename_th} " does not exist"
+	    exit
+	fi
+    fi
+    if [ "$read_pulse_profile" -gt 4 ]; then
+	echo " => reading of the file " ${pulse_filename_phi}
+	if [ -e ${pulse_filename_phi} ]; then
+	    cp ${pulse_filename_phi}  ${RESDIR}/pulse_profile_phi.dat
+	else
+	    echo "The file " ${pulse_filename_phi} " does not exist"
+	    exit
+	fi
+    fi
+fi
+
+STATEDIR=init_state
+SRCSDIR=sources
+CLDIR=conservation_laws
+RPROFDIR=rprof
+F2DDIR=f2D
+PHI2DDIR=Phi2D
+COLLISIONDIR=collisions
+PHI3DDIR=Phi3D
+FM3DDIR=moment3D
+FLUXES3DDIR=fluxes3D
+SPDIR=spectra
+F5DDIR=f5D
+POPEDIR=POPE
+MTM_TRACE=mtm_trace
+RST_FILES=rst_files
+
+mkdir -p ${RESDIR}/${MTM_TRACE}
+
+for DIR in ${STATEDIR} ${SRCSDIR} ${CLDIR} ${RPROFDIR} ${F2DDIR} \
+    ${PHI2DDIR} ${COLLISIONDIR} ${PHI3DDIR} ${FM3DDIR} ${FLUXES3DDIR} \
+    ${SPDIR} ${F5DDIR} ${POPEDIR} ${RST_FILES}
+do
+    for ((i=0;i<=${NSPECIES}-1;i++)); do
+	if [ ! -d ${RESDIR}/sp$i/${DIR} ]; then
+	    mkdir -p ${RESDIR}/sp$i/${DIR}
+	fi
+    done
+done 
+
+# Init restart files 
+# --------------------------------
+echo " "
+SEARCH_SP0=`grep SEARCH_SP0 $DATANAME | head -n 1 | tr -s '=' ' ' |  awk '{print $2}'`
+if [ "${SEARCH_SP0}" = '' -o "${SEARCH_SP0}" = '0' ];then
+    echo " No restart from another simulation"
+    SEARCH_SP0=0
+else
+    DIR_SP0=`grep DIR_SP0 $DATANAME | head -n 1 | tr -s '=' ' ' |  awk '{print $2}'`
+    DIR_SP0_RST=$DIR_SP0'/sp0/rst_files'
+    if [ "$DIR_SP0_RST" == "" ]; then
+	printf "%s %0.f \t\t %s \n" " SEARCH_SP0 = " $SEARCH_SP0 " so you must specify a directory with restarts files !!"
+        exit
+    fi
+    # test if DIR_SP0_RST exist
+    echo " Restart with previous files from repository:" $DIR_SP0
+    if [ ! -d $DIR_SP0_RST ]; then
+	echo " Directory $DIR_SP0_RST doesn't exist !! "
+	exit
+    fi
+fi
+
+# Number of expected restart files
+CKPTMODE=`grep "^\s*checkpointing_strategy.*" $DATANAME | head -n 1 | tr -s '=' ' ' | tr -d '"' | awk '{print $2}' `
+if [ "$CKPTMODE" == "SION" ]; then
+    echo " checkpointing:  SION"
+    Nb_rst_exp=1
+    FIRST_RST_SUFF="sionlib"
+    ALL_RST_SUFF="sionlib"
+    EXTENSION_SUFF="sionlib"
+else
+    CKPTMODE="HDF5"
+    echo " checkpointing: HDF5"
+    ((Nb_rst_exp=NPROC_R*NPROC_TH*NPROC_MU))
+    FIRST_RST_SUFF="fn.p00000.h5"
+    ALL_RST_SUFF="fn.p*.h5"
+    EXTENSION_SUFF="h5"
+fi
+
+# Restart with previous simulation
+if [ ${SEARCH_SP0} -ne '0' ]; then
+    # Restart with previous sp0 files
+    #if [ -f ./check_nbrestart ]; then
+    #    echo "/usr/bin/env python ./check_nbrestart ${DIR_SP0} ${CKPTMODE} ${NB_RESTART} acronym"
+    #    TAB="`/usr/bin/env python ./check_nbrestart ${DIR_SP0} ${CKPTMODE} ${NB_RESTART} acronym`"
+    #fi
+    rst_acronym=$(echo $TAB | cut -d' ' -f2)
+    echo "RST_acronym -${rst_acronym}-"
+    # Copy this restart file
+    for ((i=0;i<=${NSPECIES}-1;i++)); do
+	DIR_RST=$DIR_SP0/sp$i/rst_files
+	if [ -e $DIR_RST ]; then
+            # Number of restart files in DIR_RST
+	    Nb_rst_DIR_RST=`ls $DIR_RST/gys_rst.sp$i.${rst_acronym}.${FIRST_RST_SUFF} | wc -l`
+            # Copy of this restart file
+	    rst_file='gys_rst.sp'$i'.'${rst_acronym}'.'${ALL_RST_SUFF}
+	    if [ -f ERR ]; then rm ERR; fi
+	    echo "  => rsync restart files for sp$i "
+	    rsync -avur ${DIR_RST}/${rst_file} $RESDIR/sp$i/rst_files/. 2>> ERR
+	    status=$?;
+	    if [ $status -ne 0 ]; then
+		echo -e " \e[31m Problem with rsync command !!  so exit \e[0m"
+		cat ERR
+		rm ERR
+		exit
+	    fi
+            # Number of restart files in result directory
+	    Nb_rst_count=`ls ${RESDIR}/sp$i/rst_files/gys_rst*${rst_acronym}.${FIRST_RST_SUFF} | wc -l`
+            # Compare this two numbers
+	    if [ $Nb_rst_count -eq $Nb_rst_DIR_RST ]; then
+		echo " ... rsync OK"; echo ""
+	    else
+                # RSYNC not OK ??
+		echo " \e[31m All the restart files doesn't exist !! \e[0m"
+		exit
+	    fi
+
+	    # Copy metadata file
+	    metadata="${DIR_RST}/metadata.${rst_acronym}.${EXTENSION_SUFF}"
+	    if [ -f ${metadata} ]; then
+		echo "${metadata} exist"
+
+		if [ -f ERR ]; then rm ERR; fi
+		echo "  => rsync ${metadata} file for sp$i"
+		rsync -avur ${metadata} $RESDIR/sp$i/rst_files/. 2>> ERR
+		status=$?;
+		if [ $status -ne 0 ]; then
+		    echo -e " \e[31m Problem with rsync command for metadata !! \e[0m"
+		    rm ERR
+		    exit
+		fi
+	    else
+		echo -e " \e[31m ! NO metadata file exist \e[0m"
+                # Test if restart files are in old format
+                # Test made also in create_rst_metadata.py
+		if [ -f ${DIR_RST}/gys_rst.sp$i.${rst_acronym}.${FIRST_RST_SUFF} ]; then
+		    if [ ${CKPTMODE} == "HDF5" ]; then
+			time_diag=`h5ls -d ${DIR_RST}/gys_rst.sp$i.${rst_acronym}.${FIRST_RST_SUFF}/time_diag`
+			time_diag=$(echo $time_diag | cut -d ' ' -f 6 | tr '\n' ' ')
+			if [ $time_diag != "" ]; then
+			    echo  " Restart files are in Old configuration, so executing script:"
+			    cmd="/usr/bin/env python ../post-process/Adapt_rst_files/create_rst_metadata.py ${RESDIR} ${i} ${rst_acronym}"
+			    echo -e "  \e[34m" $cmd "\e[0m"
+			    eval $cmd
+			    status=$?
+			    if [ $status -ne 0 ]; then
+				echo -e " \e[31m Problem with create_rst_metadata.py !! \e[0m"
+				rm ERR
+				exit
+			    fi
+			else
+			    echo -e " \e[31m Problem reading ${DIR_RST}/gys_rst.sp$i.${rst_acronym}.${FIRST_RST_SUFF} !! \e[0m"
+			fi
+		    else
+			echo -e " \e[31m Restart files are in Sionlib format, no metadata could be generated \e[0m"
+		    fi 
+		fi
+	    fi
+	fi
+    done
+else
+    if [ -e ${RESDIR}"/sp0/rst_files/gys_rst.sp0.n0.${FIRST_RST_SUFF}" ] || [ -e ${RESDIR}"/sp0/rst_files/gys_rst.sp0.n1.${FIRST_RST_SUFF}" ]; then
+	if [ -f ${WKDIR}/check_nbrestart ]; then
+	    TAB="`/usr/bin/env python ${WKDIR}/check_nbrestart ${RESDIR} ${CKPTMODE} ${NB_RESTART} acronym`"
+	fi
+	nb_restart=$(echo $TAB | cut -d' ' -f1)
+	rst_acronym=$(echo $TAB | cut -d' ' -f2)
+    else
+	nb_restart=-1
+	rst_acronym="n0"
+	echo " Initialisation ......................."
+    fi
+fi
+
+if [ -f ERR ]; then rm ERR; fi
+
+# Number of restart files in result directory
+if [ ! -f ${RESDIR}/sp0/rst_files/gys_rst.sp0.${rst_acronym}.${FIRST_RST_SUFF} ]; then
+    Nb_rst_count=0
+else
+    Nb_rst_count=`ls \${RESDIR}/sp0/rst_files/gys_rst.sp0.${rst_acronym}.${ALL_RST_SUFF} | wc -l`
+fi
+if [ $Nb_rst_exp -eq $Nb_rst_count ]; then
+    echo " Restart with files already in the directory"
+    echo ""
+fi
+
+# Test if a restart run is required
+if [ ${Nb_rst_count} -ne '0' ] && [ -f ./check_nbrestart ]; then
+    if ! /usr/bin/env python ./check_nbrestart ${RESDIR} ${CKPTMODE} ${NB_RESTART}; then exit 1; fi
+fi
+
+# Create autostop script
+# ----------------------
+touch ${RESDIR}/automatic_stop.sh
+chmod +x ${RESDIR}/automatic_stop.sh
+printf "sleep $TSLEEP\n"      > ${RESDIR}/automatic_stop.sh
+printf "$KILLCMD\n"          >> ${RESDIR}/automatic_stop.sh
+printf "touch gysela.stop\n" >> ${RESDIR}/automatic_stop.sh
+
+# Create script submission
+# ------------------------
+cp ${WKDIR}/${DATANAME} ${RESDIR}/input.${CASE}
+mv ${WKDIR}/data.${CASE} ${RESDIR}/DATA
+if [ ! -f ${RESDIR}/${exec_file} ]; then
+    if [ "${OPTIONALMODE}" = "DEBUG" ]; then    
+	if [ ! -f ${WKDIR}/gysela_dbg.exe ]; then
+	    echo " !! gysela not compiled in mode debug !! "
+	    exit
+	fi
+    fi
+    if [ "${OPTIONALMODE}" = "TIMER" ]; then    
+	if [ ! -f ${WKDIR}/gysela_ti.exe ]; then
+	    echo " !! gysela not compiled in mode timer !! "
+	    exit
+	fi
+    fi
+    cp ${WKDIR}/${exec_file} ${RESDIR}/${exec_file}
+fi
+cd ${WKDIR}
+
+cat <<EOF >> ${go_file_cmd}
+
+echo "list of loaded modules:"
+echo " =============================="
+module list || true
+echo " =============================="
+echo
+
+function halt_gys () {
+    if [ "$ARCH" == "ouessant" ]; then 
+        kill -s 9 0
+    else
+        kill -9 \$PID
+        rm -f auto.out auto.err autostop.pid mpistop.pid
+    fi
+    exit \$1
+}
+
+if [ -z "$WAITJOBEND" ]
+then
+set -vx
+fi
+cd ${RESDIR}
+
+# Script submission
+# -----------------
+if [ -e "sp0/rst_files/gys_rst.sp0.n0.${FIRST_RST_SUFF}" ] || [ -e "sp0/rst_files/gys_rst.sp0.n1.${FIRST_RST_SUFF}" ]; then
+    if [ -f ${WKDIR}/check_nbrestart ]; then
+       TAB="`/usr/bin/env python ${WKDIR}/check_nbrestart ${RESDIR} ${CKPTMODE} ${NB_RESTART} acronym`"
+    fi
+    nb_restart=\$(echo \$TAB | cut -d' ' -f1)
+    rst_acronym=\$(echo \$TAB | cut -d' ' -f2)
+else
+    nb_restart=-1
+    echo " Initialisation ......................."
+fi
+
+# Create script to stop gysela automatically
+# whenever the wall clock time limit is too low
+# ---------------------------------------------
+if [ -e gysela.stop ]; then
+    rm -f gysela.stop
+fi
+
+# test if nb_restart <= NB_RESTART
+# -----------------------------
+if [ "\$nb_restart" -le "$NB_RESTART" ]; then
+    echo " You want "$NB_RESTART" restart(s), and "\$nb_restart" are made "
+    ${RESDIR}/automatic_stop.sh 1>${RESDIR}/auto.out 2>${RESDIR}/auto.err &
+    PID=\$!
+    echo \$PID > ${RESDIR}/autostop.pid
+   
+    # run gysela.exe :
+    # ----------------
+    # Reset gysela_status file
+    echo "-1 : not started" >  gysela_status.txt
+   
+    export OMP_NUM_THREADS=${NTHREAD}
+    ${CMD_MPIRUN}
+    status=\$(head -n 1 gysela_status.txt| awk '{print \$1}')
+   
+    mv mtm_*.out ${MTM_TRACE}
+    for ((ns=0;ns<=${NSPECIES}-1;ns++)); do
+        mv sp\${ns}/coord_system.h5       sp\${ns}/${STATEDIR} 
+        mv sp\${ns}/init_state*.h5        sp\${ns}/${STATEDIR}
+        mv sp\${ns}/data_*.h5             sp\${ns}/${STATEDIR}
+        mv sp\${ns}/sources_*.h5          sp\${ns}/${SRCSDIR}
+        mv sp\${ns}/conservation_laws*.h5 sp\${ns}/${CLDIR}
+        mv sp\${ns}/rprof_*.h5            sp\${ns}/${RPROFDIR}
+        mv sp\${ns}/Phi2D*.h5             sp\${ns}/${PHI2DDIR}
+        mv sp\${ns}/f2D*.h5               sp\${ns}/${F2DDIR}
+        mv sp\${ns}/collision*.h5         sp\${ns}/${COLLISIONDIR}
+        mv sp\${ns}/Phi_3D*.h5            sp\${ns}/${PHI3DDIR}
+        mv sp\${ns}/Phi_3D*.bin           sp\${ns}/${PHI3DDIR} 2>/dev/null
+        mv sp\${ns}/n*_GC_3D_*.h5         sp\${ns}/${FM3DDIR}
+        mv sp\${ns}/V_GC_3D_*.h5          sp\${ns}/${FM3DDIR}
+        mv sp\${ns}/Ppar_GC_3D_*.h5       sp\${ns}/${FM3DDIR}
+        mv sp\${ns}/Pperp*_GC_3D_*.h5     sp\${ns}/${FM3DDIR}
+        mv sp\${ns}/Gamma_*_3D*.h5        sp\${ns}/${FLUXES3DDIR}
+        mv sp\${ns}/Q*_3D*.h5             sp\${ns}/${FLUXES3DDIR}
+        mv sp\${ns}/spectra*.h5           sp\${ns}/${SPDIR}
+        mv sp\${ns}/f5D*.h5               sp\${ns}/${F5DDIR}
+        mv sp\${ns}/POPE*.h5              sp\${ns}/${POPEDIR}
+        mv sp\${ns}/gys_rst.sp\${ns}.*.${EXTENSION_SUFF} sp\${ns}/${RST_FILES} 2>/dev/null
+        mv sp\${ns}/meta*.${EXTENSION_SUFF} sp\${ns}/${RST_FILES} 2>/dev/null
+    done
+  
+    cd ${WKDIR}
+   
+    if [ "$NB_RESTART" -ne "0" ]; then
+        if [ "\$status" -eq ${STATUSOK} ]; then
+            # new submission
+            # --------------
+            if [ -f ./check_nbrestart ]; then
+                if ! /usr/bin/env python ./check_nbrestart ${RESDIR} ${CKPTMODE} ${NB_RESTART}
+                then
+                    halt_gys 1
+                fi
+            fi
+            ${CMD_SUBRST}
+        fi
+    fi
+fi
+halt_gys 0
+EOF
+
+# -----------------------------------------------------------------
+# Test presence of CMD_MPIRUN  
+# -----------------------------------------------------------------
+command_mpirun=`echo $CMD_MPIRUN | awk '{print $1}'`
+test_command_mpirun=`which $command_mpirun 2> /dev/null`
+if [ ${#test_command_mpirun} -eq 0 ]; then
+    echo " "
+    echo "-----------------------------------------------------------------------"
+    echo -e "\033[31m***ERROR: \033[0m $command_mpirun is not installed --> load GYSELA environment"
+    echo "-----------------------------------------------------------------------"
+    echo " "
+    exit
+fi
+
+# -----------------------------------------------------------------                                                                                                                          
+# Launch submission
+# -----------------------------------------------------------------
+chmod +x ${go_file_cmd}
+cd ${WKDIR}
+echo " ... ${CMD_SUB}"
+${CMD_SUB}
+if [ -n "$WAITJOBEND" ]; then
+    cat "${RESDIR}/gysela_log.out" "${RESDIR}/gysela_res.err"
+fi