Merge branch 'develop' of https://dci-gitlab.cines.fr/dci/abs into develop

GYSELAX/machines/occigen-bdw/scaltest_TKE_A139_n14_dlogTi0.0

+#******************************************
+# job parameters
+#******************************************
+NB_RESTART   = 0
+TIME         = 00:30:00
+MACHINE_FILE =
+JOBNAME      = N14_DLOGTI0.0
+
+#*******************************************
+#  Variables used to specify if the
+#   simulation is a restart from
+#   another one
+#*******************************************
+SEARCH_SP0   = 0           ! 0 if no restart, 1 otherwise
+DIR_SP0      =             ! if SEARCH_SP0=1, name of the directory
+                                !   containing the major species restart files
+
+#*******************************************
+#  Variables for parallelization
+#*******************************************
+NSPECIES = 2
+NPROC_R  = 4
+NPROC_TH = 2
+NPROC_MU = 16
+NTHREAD  = 14
+
+#*******************************************
+#  Variables for radial profile input files
+#*******************************************
+ns0_sp0_filename    = ns0_sp0_test.dat
+ns0_sp1_filename    = ns0_sp1_test.dat
+Ts0_sp0_filename    = Ts0_sp0_test.dat
+Ts0_sp1_filename    = Ts0_sp1_test.dat
+q_filename          = safety_factor_test.dat
+Vpars0_sp0_filename = Vpar0_sp0_test.dat
+Vpars0_sp1_filename = Vpar0_sp1_test.dat
+ripple_filename     = ripple_constant.h5
+
+#******************************************
+# gysela input datas
+#******************************************
+&PARALLEL
+  Nspecies       = $NSPECIES
+  Nbproc_r       = $NPROC_R
+  Nbproc_theta   = $NPROC_TH
+  Nbproc_mu      = $NPROC_MU
+  Nbthread       = $NTHREAD
+/
+
+&MESH
+  CylindricalGeometry = .false.
+  Nr                  = 255
+  Ntheta              = 256
+  Nphi                = 64
+  Nvpar               = 127
+  Nmu                 = $((NPROC_MU-1))
+  small_radius        = 139.
+  rhomin              = 0.
+  rhomax              = 1.
+  Ltheta              = 6.283185307179586476925286766559005768394
+  Lphi                = 6.283185307179586476925286766559005768394
+  aspect_ratio        = 2.78
+  nb_vth0             = 7.
+  Lmu                 = 12.
+  mumin               = 0.,
+/
+
+&EQUIL
+! Define the choice for electrons
+!  0 = "ADIABATIC"
+!  1 = "FULL-KINETIC"
+!  2 = "TRAPPED-KINETIC"
+  electron_type              = 2
+! We consider as kinetic only the trapped electrons which have
+!  a maximum parallel velocity so that
+!   |vpar/vperp| < sqrt(2*eps)*kineticTE_coeff
+  kineticTE_coeff            = 1.
+! The possible choices for the equilibrium distribution function are:
+!   1 = "MAXWELLIAN"[default]
+!   2 = "ENERGETIC_PARTICLES"
+  fequilibrium_type          = 1 , 1
+  canonical_profile          = 0
+  tau0                       = 1.
+  A0                         = 1.
+  Z0                         = 1.
+  As                         = 2. , 0.02
+  Zs                         = 1. , -1.
+  species_frac               = 1. , 1.
+  read_ns0                   = .false. , .false.
+  read_Ts0                   = .false. , .false.
+  read_q                     = .false.
+  read_Vpars0                = .false. , .false.
+  profile_choice             = 5 , 5
+  rpeak                      = 0.5
+  kappa_ns0                  = 2.22 , 2.22
+  kappa_Ts0                  = 0.0, 6.92
+  deltar_ns0                 = 0.3 , 0.3
+  deltar_Ts0                 = 0.3 , 0.3
+  Vpars0                     = 0. 
+  q_profile                  = 4
+  q_param1                   = 0.854
+  q_param2                   = 0
+  q_param3                   = 2.184
+  q_param4                   = 0.
+  magnetic_drift             = .true.
+  perturb_amplitude          = 0.000001 , 0.000001
+  perturb_choice             = 2
+  m                          = 20
+  n                          = 14
+  !--> Gyroaverage tunable input parameters
+  gyroaverage                = .true. , .true.
+! The default 'gyro_strategy' is "PADE"
+! All possible choices:
+! {"PADE", "SPLINE", "HERMITE"}
+  gyro_strategy              = "HERMITE"
+! The default 'gyro_opt_hermite' is "BLOCKING"
+! All possible choices:
+! {"BLOCKING", "LAYOUT"}
+! gyro_opt_hermite           = ""
+! 'gyro_Nbpoints' is usefull only if
+! gyro_strategy={"SPLINE", "HERMITE"}
+  gyro_Nbpoints              = 8
+  TF_ripple                  = .false.
+  N_ripple                   = 0,
+/
+
+&COLLISION_OPERATOR
+  collision_strategy = "NONE" ! ("DEFAULT", "NONE", "GENERAL") !nouveau
+  nustar          = 0.0
+  coll_nbstep     = 1,
+/
+
+&BUFFER_REGION
+  coef_diffr        = 0.0 , 0.0
+  coef_diffth       = 0.0 ,0.0
+  buffer_asymmetry  = .false.
+  right_buffer_only = .true.
+  buffer_location   = 0.05
+  buffer_stiffness  = 0.025,
+/
+
+&ALGORITHM
+  Vlasov_scheme = "BSL"
+  aligned_strategy = "SPLINE"
+  skiphole         = .true.
+  QN_strategy      = "TKE"
+  QN_TKE_version   = 0
+  FFTpotential_filter        = 1
+  FFTpotential_filter_deltam = 1000.
+  deltat        = 2.
+  nbiter        = 4
+  dt_diag       = 2.
+/
+
+&SOURCE
+  flux_driven     = .true.
+  heat_source     = .false.
+  heat_version    = 0
+  Sce0            = 0.00002   , 0.
+  TS              = 1.5       , 1.
+  rhoS            = 0.        , 0.
+  LS              = 0.1       , 0.
+/
+
+&TEST
+  !--> Tests for QN solver
+  solve_QN             = .true.
+  !--> Tests for RHS of Boltzmann equation
+  RHS_only             = .false.
+  !--> Tests for equilibrium definition
+  canonical_vpar0      = 3
+  !--> Tests with simplified physics
+  BstareqB             = .false.
+  single_m             = .false.
+  single_n             = .true.
+  remove_n0            = .true.
+  Rosenbluth_Hinton    = .false.
+  !--> Tests for computer sciences
+  modulo_restart_iter  = 400
+/
+
+&OUTPUT
+  integration_CS       = .true.
+  diag_level           = 8
+  Phi3D_saving         = .true.
+  Moments3D_saving     = .true.
+  fluxes3D_GC_saving   = .true.
+  f5D_saving           = .true.
+  Phi3D_dt_diag        = 4.
+  Moments3D_dt_diag    = 4.
+  fluxes3D_GC_dt_diag  = 4.
+  f5D_dt_diag          = 8.
+/

GYSELAX/machines/occigen-bdw/smalltest_TKE_A139_n14_dlogTi0.0

+#******************************************
+# job parameters
+#******************************************
+NB_RESTART   = 0
+TIME         = 00:10:00
+MACHINE_FILE =
+JOBNAME      = N14_DLOGTI0.0
+
+#*******************************************
+#  Variables used to specify if the
+#   simulation is a restart from
+#   another one
+#*******************************************
+SEARCH_SP0   = 0           ! 0 if no restart, 1 otherwise
+DIR_SP0      =             ! if SEARCH_SP0=1, name of the directory
+                                !   containing the major species restart files
+
+#*******************************************
+#  Variables for parallelization
+#*******************************************
+NSPECIES = 2
+NPROC_R  = 1
+NPROC_TH = 2
+NPROC_MU = 2
+NTHREAD  = 14
+
+#*******************************************
+#  Variables for radial profile input files
+#*******************************************
+ns0_sp0_filename    = ns0_sp0_test.dat
+ns0_sp1_filename    = ns0_sp1_test.dat
+Ts0_sp0_filename    = Ts0_sp0_test.dat
+Ts0_sp1_filename    = Ts0_sp1_test.dat
+q_filename          = safety_factor_test.dat
+Vpars0_sp0_filename = Vpar0_sp0_test.dat
+Vpars0_sp1_filename = Vpar0_sp1_test.dat
+ripple_filename     = ripple_constant.h5
+
+#******************************************
+# gysela input datas
+#******************************************
+&PARALLEL
+  Nspecies       = $NSPECIES
+  Nbproc_r       = $NPROC_R
+  Nbproc_theta   = $NPROC_TH
+  Nbproc_mu      = $NPROC_MU
+  Nbthread       = $NTHREAD
+/
+
+&MESH
+  CylindricalGeometry = .false.
+  Nr                  = 31
+  Ntheta              = 128
+  Nphi                = 32
+  Nvpar               = 127
+  Nmu                 = $((NPROC_MU-1))
+  small_radius        = 139.
+  rhomin              = 0.
+  rhomax              = 1.
+  Ltheta              = 6.283185307179586476925286766559005768394
+  Lphi                = 6.283185307179586476925286766559005768394
+  aspect_ratio        = 2.78
+  nb_vth0             = 7.
+  Lmu                 = 12.
+  mumin               = 0.,
+/
+
+&EQUIL
+! Define the choice for electrons
+!  0 = "ADIABATIC"
+!  1 = "FULL-KINETIC"
+!  2 = "TRAPPED-KINETIC"
+  electron_type              = 2
+! We consider as kinetic only the trapped electrons which have
+!  a maximum parallel velocity so that
+!   |vpar/vperp| < sqrt(2*eps)*kineticTE_coeff
+  kineticTE_coeff            = 1.
+! The possible choices for the equilibrium distribution function are:
+!   1 = "MAXWELLIAN"[default]
+!   2 = "ENERGETIC_PARTICLES"
+  fequilibrium_type          = 1 , 1
+  canonical_profile          = 0
+  tau0                       = 1.
+  A0                         = 1.
+  Z0                         = 1.
+  As                         = 2. , 0.02
+  Zs                         = 1. , -1.
+  species_frac               = 1. , 1.
+  read_ns0                   = .false. , .false.
+  read_Ts0                   = .false. , .false.
+  read_q                     = .false.
+  read_Vpars0                = .false. , .false.
+  profile_choice             = 5 , 5
+  rpeak                      = 0.5
+  kappa_ns0                  = 2.22 , 2.22
+  kappa_Ts0                  = 0.0, 6.92
+  deltar_ns0                 = 0.3 , 0.3
+  deltar_Ts0                 = 0.3 , 0.3
+  Vpars0                     = 0. 
+  q_profile                  = 4
+  q_param1                   = 0.854
+  q_param2                   = 0
+  q_param3                   = 2.184
+  q_param4                   = 0.
+  magnetic_drift             = .true.
+  perturb_amplitude          = 0.001 , 0.001
+  perturb_choice             = 2
+  m                          = 20
+  n                          = 14
+  !--> Gyroaverage tunable input parameters
+  gyroaverage                = .true. , .true.
+! The default 'gyro_strategy' is "PADE"
+! All possible choices:
+! {"PADE", "SPLINE", "HERMITE"}
+  gyro_strategy              = "HERMITE"
+! The default 'gyro_opt_hermite' is "BLOCKING"
+! All possible choices:
+! {"BLOCKING", "LAYOUT"}
+! gyro_opt_hermite           = ""
+! 'gyro_Nbpoints' is usefull only if
+! gyro_strategy={"SPLINE", "HERMITE"}
+  gyro_Nbpoints              = 8
+  TF_ripple                  = .false.
+  N_ripple                   = 0,
+/
+
+&COLLISION_OPERATOR
+  collision_strategy = "NONE" ! ("DEFAULT", "NONE", "GENERAL") !nouveau
+  nustar          = 0.0
+  coll_nbstep     = 1,
+/
+
+&BUFFER_REGION
+  coef_diffr        = 0.0 , 0.0
+  coef_diffth       = 0.0 ,0.0
+  buffer_asymmetry  = .false.
+  right_buffer_only = .true.
+  buffer_location   = 0.05
+  buffer_stiffness  = 0.025,
+/
+
+&ALGORITHM
+  Vlasov_scheme = "BSL"
+  aligned_strategy = "SPLINE"
+  skiphole         = .true.
+  QN_strategy      = "TKE"
+  QN_TKE_version   = 0
+  FFTpotential_filter        = 1
+  FFTpotential_filter_deltam = 1000.
+  deltat        = 10.
+  nbiter        = 4
+  dt_diag       = 10.
+/
+
+&SOURCE
+  flux_driven     = .true.
+  heat_source     = .false.
+  heat_version    = 0
+  Sce0            = 0.00002   , 0.
+  TS              = 1.5       , 1.
+  rhoS            = 0.        , 0.
+  LS              = 0.1       , 0.
+/
+
+&TEST
+  !--> Tests for QN solver
+  solve_QN             = .true.
+  !--> Tests for RHS of Boltzmann equation
+  RHS_only             = .false.
+  !--> Tests for equilibrium definition
+  canonical_vpar0      = 3
+  !--> Tests with simplified physics
+  BstareqB             = .false.
+  single_m             = .false.
+  single_n             = .true.
+  remove_n0            = .true.
+  Rosenbluth_Hinton    = .false.
+  !--> Tests for computer sciences
+  modulo_restart_iter  = 400
+/
+
+&OUTPUT
+  integration_CS       = .true.
+  diag_level           = 8
+  Phi3D_saving         = .true.
+  Moments3D_saving     = .true.
+  fluxes3D_GC_saving   = .true.
+  f5D_saving           = .false.
+  Phi3D_dt_diag        = 20.
+  Moments3D_dt_diag    = 20.
+  fluxes3D_GC_dt_diag  = 20.
+  f5D_dt_diag          = 3000.
+/

GYSELAX/testcase_big/prepare.sh

+####################################
+# REMOVE PREVIOUS LOGS
+####################################
+rm -rf $GYSELA_PATH/wk/*BIG*TKE*
+rm -rf $GYSELA_PATH/wk/big*
+
+####################################
+# COMPILE THE TEST CASE
+####################################
+if [ ${ARCH} = 'occigen2' ]; then
+    cp ../machines/occigen-bdw/bigtest_TKE_A139_n14_dlogTi0.0 $GYSELA_PATH/wk/
+elif [ ${ARCH} = 'jeanzay' ]; then
+    cp ../machines/jean-zay-cpu/bigtest_TKE_A139_n14_dlogTi0.0 $GYSELA_PATH/wk/
+fi
+
+

GYSELAX/testcase_big/run.sh

+####################################
+# RUN SMALLTEST
+####################################
+cd $GYSELA_PATH/wk/
+./subgys bigtest_TKE_A139_n14_dlogTi0.0

GYSELAX/testcase_large/prepare.sh

+####################################
+# REMOVE PREVIOUS LOGS
+####################################
+rm -rf $GYSELA_PATH/wk/*SCAL*TKE*
+rm -rf $GYSELA_PATH/wk/scal*
+
+####################################
+# COMPILE THE TEST CASE
+####################################
+if [ ${ARCH} = 'occigen2' ]; then
+    cp ../machines/occigen-bdw/scaltest_TKE_A139_n14_dlogTi0.0 $GYSELA_PATH/wk/
+elif [ ${ARCH} = 'jeanzay' ]; then
+    cp ../machines/jean-zay-cpu/scaltest_TKE_A139_n14_dlogTi0.0 $GYSELA_PATH/wk/
+fi
+
+

GYSELAX/testcase_large/run.sh

+####################################
+# RUN SMALLTEST
+####################################
+cd $GYSELA_PATH/wk/
+./subgys scalbigtest_TKE_A139_n14_dlogTi0.0

GYSELAX/testcase_small/prepare.sh

+####################################
+# REMOVE PREVIOUS LOGS
+####################################
+rm -rf $GYSELA_PATH/wk/*SMALL*TKE*
+
+####################################
+# COMPILE THE TEST CASE
+####################################
+if [ ${ARCH} = 'occigen2' ]; then
+    cp ../machines/occigen-bdw/smalltest_TKE_A139_n14_dlogTi0.0 $GYSELA_PATH/wk/
+elif [ ${ARCH} = 'jeanzay' ]; then
+    cp ../machines/jean-zay-cpu/smalltest_TKE_A139_n14_dlogTi0.0 $GYSELA_PATH/wk/
+fi
+   
+

GYSELAX/testcase_small/run.sh

+####################################
+# RUN SMALLTEST
+####################################
+cd $GYSELA_PATH/wk/
+./subgys smalltest_TKE_A139_n14_dlogTi0.0

Gromacs/README.md

@@ -15,7 +15,7 @@ Technical information:
 * Language : C/C++/CUDA
 * Parallelism : MPI + OpenMP + CUDA
 * GPU acceleration : yes  
-* Scalability : high
+* Scalability : med
 * Vectorization: high
 
 MPI-Bound Code
@@ -55,11 +55,24 @@ source machines/occigen-bdw/env
 Run and validate the simulation:
 --------------------------------
 
-For each test case, given in a separate folder (e.g. testcase_ion_channel_small and testcase_BSG_large), you can find three scripts:
-
-*  prepare.sh: prepare the simulation (move data to the right location, recompile some minor changes, ...)
-*  run.sh : run the application and print out the evaluated metric in a file called results (first numeric column representing ns/day corresponds to the evaluated metric to take into account)
-*  validate.sh: validation of the simulation for (if the evaluated metric is print out and the simulation is completed with 20 000 steps for testcase_ion_channel_small and 1 200 000 steps for testcase_BSG_large, the bench is considered validated)
+For each test case, given in a separate folder (e.g. testcase_ion_channel_small, testcase_ion_channel_large and testcase_BSG_large), you can find three scripts:
+
+1.  prepare.sh: prepare the simulation (move data to the right location, recompile some minor changes, ...)
+2.  run.sh : run the application and print out the evaluated metric in a file called results (first numeric column representing ns/day corresponds to the evaluated metric to take into account)
+3.  validate.sh: validation of the simulation. 
+
+*  The bench testcase_ion_channel_small is considered validated if:
+    * the evaluated metric is print out ;
+    * the simulation is completed with  20 000 steps ;
+    * the files small_ion_channel.log, small_ion_channel.edr, results and slurm*.out are present in the directory results-gromacs-ion_channel_small.
+*  The bench testcase_ion_channel_large is considered validated if:
+    *  the evaluated metric is print out ;
+    *  the simulation is completed with 1 000 000 steps ;
+    *  the files large_ion_channel.log, large_ion_channel.edr, results and slurm*.out are present in the directory results-gromacs-ion_channel_large.
+*  The bench testcase_BSG_large is considered validated if:
+    *  the evaluated metric is print out ;
+    *  the simulation is completed with 1 200 000 steps ;
+    *  the files benchBSG.cpt, benchBSG.trr, benchBSG.xtc, benchBSG.log, benchBSG.edr, results and slurm*.out are present in the directory results-gromacs-BSG_large.
 
 For running and validating the simulation, one should be able to do:
 ```

Gromacs/machines/occigen-bdw/batch_large_BSG.slurm

-d #!/bin/bash
+#!/bin/bash
 #SBATCH -J benchBSG
 #SBATCH -C BDW28
 #SBATCH -t 01:30:00

Gromacs/testcase_BSG_large/validate.sh

@@ -12,6 +12,12 @@ cd $run_gromacs_dir_name
 perf=`grep Performance: results | awk '{ print $2 }'`
 end_sim=`grep "Statistics over 1200001 steps using 12001 frames" benchBSG.log`
 
+mkdir results-gromacs-BSG_large
+cp benchBSG.* results-gromacs-BSG_large/.
+cp slurm* results-gromacs-BSG_large/.
+cp results results-gromacs-BSG_large/.
+cd results-gromacs-BSG_large
+
 if [ -z "$perf" ] || [ -z "$end_sim" ]
 then
     echo "bench is not validated"

Gromacs/testcase_ion_channel_large/validate.sh

@@ -12,6 +12,12 @@ cd $run_gromacs_dir_name
 perf=`grep Performance results | awk '{ print $2 }'`
 end_sim=`grep "Statistics over 1000001 steps using 100001 frames" large_ion_channel.log`
 
+mkdir results-gromacs-ion_channel_large
+cp large_ion_channel* results-gromacs-ion_channel_large/.
+cp slurm* results-gromacs-ion_channel_large/.
+cp results results-gromacs-ion_channel_large/.
+cd results-gromacs-ion_channel_large
+
 if [ -z "$perf" ] || [ -z "$end_sim" ]
 then
     echo "bench is not validated"

Gromacs/testcase_ion_channel_small/validate.sh

@@ -12,6 +12,12 @@ cd $run_gromacs_dir_name
 perf=`grep Performance results | awk '{ print $2 }'`
 end_sim=`grep "Statistics over 20001 steps using 2001 frames" small_ion_channel.log`
 
+mkdir results-gromacs-ion_channel_small
+cp small_ion_channel* results-gromacs-ion_channel_small/.
+cp slurm* results-gromacs-ion_channel_small/.
+cp results results-gromacs-ion_channel_small/.
+cd results-gromacs-ion_channel_small
+
 if [ -z "$perf" ] || [ -z "$end_sim" ]
 then
     echo "bench is not validated"

Smilei/machines/occigen-bdw/batch_xlarge.slurm

@@ -5,7 +5,7 @@
 #SBATCH --cpus-per-task=3
 #SBATCH -J laser 
 #SBATCH --constraint=BDW28
-#SBATCH --time=01:00:00
+#SBATCH --time=02:00:00
 #SBATCH --output=xlarge_test.out
 
 ## Set the working dir to the dir in which the submission was called.
@@ -19,12 +19,14 @@ export KMP_AFFINITY=granularity=fine,compact,1,0,verbose
 mkdir -p xlarge_output
 cd xlarge_output
 
-if [[ ! -d testcase_xlarge ]] ; 
+if [[ ! -d ../testcase_xlarge ]] ; 
 then 
 	echo "Make sure you have the checkpoint localy." 
 	echo "If not, type: ../../testcase_xlarge/prepare.sh" 
   exit 0
 fi
 
+ln -s ../testcase_xlarge testcase_xlarge 
+
 
 srun --mpi=pmi2 -K1 -m block:cyclic -c $SLURM_CPUS_PER_TASK --resv-ports -n $SLURM_NTASKS $EXE $SMILEI_DIR/testcase_xlarge/test.py

Smilei/testcase_xlarge/README.md

@@ -11,7 +11,8 @@ It requires about 250GB to run.
 Prepare the test case
 ---------------------
 
-This test case restarts from a checkpoint. Make sure to run the prepare.sh script in your working directory before running the application.
+This test case restarts from a checkpoint. You must define the location of the checkpoint xlarge\_checkpoint.tar file in the prepare.sh script. 
+Make sure to run the prepare.sh script in your working directory before running the application.
 
 Case profile
 ------------

Smilei/testcase_xlarge/prepare.sh

 #!/bin/bash
-echo "No preparation needed"
+echo "Untar the checkpoint"
 
 tar -xvf /store/CINES/dci/SHARED/abs/Smilei/xlarge_checkpoint.tar 

YALES2BIO/testcase_big/batch

-#!/bin/bash
-#SBATCH -J 3D_RBC_SHEAR_BIGp
-#SBATCH --constraint=BDW28
-#SBATCH --nodes=32
-#SBATCH --cpus-per-task=1
-#SBATCH --time=00:59:00
-#SBATCH --exclusive
-
-source $YALES2_HOME/../machines/occigen-bdw/env_bench
-module load vtune/2019.2
-
-export cores=896
-
-#mkdir dump
-
-# deactivate the collective optimizations
-export OMPI_MCA_ess=^pmi
-export OMPI_MCA_pubsub=^pmi
-export OMPI_MCA_pml=ob1
-unset OMPI_MCA_mtl
-unset OMPI_MCA_coll_fca_enable
-unset OMPI_MCA_coll_fca_np
-export OMP_NUM_THREADS=1
-export MKL_NUM_THREADS=1
-export OMPI_MCA_btl_openib_warn_no_device_params_found=0
-
-export I_MPI_OFA_TRANSLATION_CACHE=0
-export I_MPI_DAPL_TRANSLATION_CACHE=0
-
-srun --kill-on-bad-exit -m block --resv-ports --cpu_bind=map_cpu:0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27 -n $cores aps ./rbc_shear_30microns
-
-

YALES2BIO/testcase_small/batch

-#!/bin/bash
-#SBATCH -J HELFER_15
-#SBATCH --constraint=BDW28
-#SBATCH --nodes=1
-#SBATCH --cpus-per-task=1
-#SBATCH --time=00:30:00
-#SBATCH --exclusive
-
-source $YALES2_HOME/../machines/occigen-bdw/env_bench
-
-export cores=28
-
-#mkdir dump
-
-# deactivate the collective optimizations
-export OMPI_MCA_ess=^pmi
-export OMPI_MCA_pubsub=^pmi
-export OMPI_MCA_pml=ob1
-unset OMPI_MCA_mtl
-unset OMPI_MCA_coll_fca_enable
-unset OMPI_MCA_coll_fca_np
-export OMP_NUM_THREADS=1
-export MKL_NUM_THREADS=1
-export OMPI_MCA_btl_openib_warn_no_device_params_found=0
-
-export I_MPI_OFA_TRANSLATION_CACHE=0
-export I_MPI_DAPL_TRANSLATION_CACHE=0
-
-srun --kill-on-bad-exit -m block --resv-ports --cpu_bind=map_cpu:0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27 -n $cores ./tube_10micron_036
-
-