Merge branch 'develop' of https://dci-gitlab.cines.fr/dci/abs into develop

.gitignore

+# Output
+*.log
+*.err
+*.out
+
+# Compiled source 
+*.exe
+*.o
+*.so
+
+# Vi 
+*.swp
+*.swo
+

Abinit/README.md

+Description:
+============
+
+Presentation
+------------
+
+ABINIT is a package whose main program allows one to find the total energy, charge density and electronic structure of systems made of electrons and nuclei (molecules and periodic solids) within Density Functional Theory (DFT), using pseudopotentials (or PAW atomic data) and a planewave basis. ABINIT also optimize the geometry according to the DFT forces and stresses, or perform molecular dynamics simulations using these forces, or generate phonons, Born effective charges, and dielectric tensors, based on Density-Functional Perturbation Theory, and many more properties. Excited states and spectra can be computed within the Many-Body Perturbation Theory (the GW approximation and the Bethe-Salpeter equation). DFT+U and Dynamical mean-field theory are available for strongly correlated materials
+
+
+Technical information:
+----------------------
+
+* website : http://abinit.org
+* Scientific domain : Density Functional Theory 
+* Language : C/Fortran
+* Parallelism : MPI + OpenMP
+* GPU acceleration : Yes (CUDA) 
+* Scalability : average
+* Vectorization: poor
+
+
+Compilation and simulation:
+===========================
+
+Download:
+---------
+
+```
+./download.sh
+```
+
+Compile:
+--------
+
+```
+source machines/occigen-bdw/env
+./compile.sh
+```
+
+`machines/occigen-bdw/env` contains the shell environment variables for compiling
+
+Run and validate the simulation:
+--------------------------------
+
+For each test case, given in a separate folder (e.g. testcase_small), you can find three scripts:
+
+*  prepare.sh: prepare the simulation (move data to the right location, recompile some minor changes, ...)
+*  run.sh : run the application and print out the evaluated metric
+*  validate.sh: validation of the simulation on a scientific point of view
+
+For running and validating the simulation, one should be able to do:
+```
+cd testcase_XXX
+./prepare.sh
+./run.sh
+./validate.sh
+```
+And getting no error code returned.
+Those steps can also be used in a batch file for running the simulation using a job scheduler.
+
+
+
+

Abinit/compile.sh

+export ABINIT_ROOT="$SCRATCHDIR/Abinit"
+export ETSFIO_ROOT="$SCRATCHDIR/Etsfio"
+export XC_ROOT="$SCRATCHDIR/LibXC"
+./configure --prefix=$ABINIT_ROOT CC=mpicc CXX=mpiCC FC=mpif90 F77=mpif77 --enable-mpi=yes --enable-openmp=yes --with-linalg-flavor=mkl+scalapack --with-linalg-libs="-lscalapack -L$MKL_LIBS -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core" --with-math-flavor=gsl --with-dft-flavor=libxc --with-libxc-incs=-I$XC_ROOT/include --with-libxc-libs="-lxcf90 -lxc" --with-fft-flavor=fftw3-mkl --with-trio-flavor=netcdf  --with-etsf-io-incs=-I$ETSFIO_ROOT/include --with-etsf-io-libs="-letsf_io_low_level -letsf_io_utils -letsf_io" --with-netcdf-incs=-I. --with-netcdf-libs="-lnetcdff -lnetcdf"
+
+make -j 20
+make -j 20 install

Abinit/download.sh

+#!/bin/sh
+
+wget https://www.abinit.org/sites/default/files/packages/abinit-8.10.3.tar.gz
+
+tar xf abinit-8.10.3.tar.gz
+
+cd abinit-8.10.3

RAMSES/compile.sh

+#!/bin/bash
+
+source machines/occigen-bdw/env
+
+compile(){
+    cd $ramses_dir/ramses/bin
+    make clean
+    # FFLAGS
+    sed -i s/"-cpp -DNDIM=\$(NDIM) -DNPRE=\$(NPRE) -DSOLVER\$(SOLVER) -DNVAR=\$(NVAR) -DNCHEM=\$(NCHEM)"/"-xHost -O3 -ip -cpp -DNDIM=\$(NDIM) -DNPRE=\$(NPRE) -DSOLVER\$(SOLVER) -DNVAR=\$(NVAR) -DNCHEM=\$(NCHEM)"/g $ramses_dir/ramses/bin/Makefile
+    pwd
+    make >& make.log
+}
+
+compile

RAMSES/download.sh

-#/bin/bash
+#!/bin/bash
 
-git clone https://bitbucket.org/rteyssie/ramses
-cd ramses
-git checkout stable_19_10 # Checkout a stable version 
+source machines/occigen-bdw/env
+
+git_download(){
+    git clone https://bitbucket.org/rteyssie/ramses
+    cd ramses
+    git checkout stable_19_10 # Checkout a stable version
+}
+
+tar_download(){
+    cd $ramses_dir
+    tar zxvf /opt/software/tarballs/ramses.tar.gz
+}
+cd $ramses_dir
+tar_download

RAMSES/machines/occigen-bdw/batch_small.slurm

+#!/bin/bash
+#SBATCH --job-name=DEBUG
+#SBATCH --nodes=11
+#SBATCH --ntasks=256
+#SBATCH --threads-per-core=1
+#SBATCH --time=00:30:00
+#SBATCH --output=ramses3d-BDW28-%j.out
+#SBATCH --error=ramses3d-BDW28-%j.err
+#SBATCH --constraint=BDW28
+set -x
+
+source ./env
+#set the working directory here:
+SIMU=$bench_dir/DEBUG/
+#indicate namelist file here (in the SIMU directory)
+NML="cosmo.nml"
+#locate the executable file here
+RAMSES=$ramses_dir/ramses/bin/ramses3d
+DATE=$( date +%Y-%m-%d_%Hh%M )
+cd $SIMU
+srun --mpi=pmi2 -K1 --resv-ports -n $SLURM_NTASKS  $RAMSES $NML > $SIMU'/run_'$DATE.log

RAMSES/machines/occigen-bdw/env

+#/bin/bash
+ 
+module purge
+module load intel/18.1 openmpi/intel/2.0.4
+
+export ramses_dir=$HOME/benchmarks/abs/$compiler/$compiler_version/$libmpi/$mpi_release
+export bench_dir=$SCRATCHDIR/benchmarks/abs/$compiler/$compiler_version/$libmpi/$mpi_release
+mkdir -p $bench_dir
+mkdir -p $ramses_dir

RAMSES/testcase_small/prepare.sh

+#!/bin/bash
+
+source ../machines/occigen-bdw/env
+
+# Copy input for small case
+cp $STOREDIR/testcase_small.tar.gz $bench_dir/.
+cd $bench_dir/
+tar zxvf testcase_small.tar.gz

YALES2BIO/testcase_big/Makefile

+#Makefile
+
+# parameters
+include $(YALES2_HOME)/src/Makefile.in
+
+PROG2MAKE=rbc_shear_30microns
+
+# defaults
+defaults: $(PROG2MAKE)
+
+# Libraries
+LIBS = $(YALES2_LIB)/libyales2main.a
+LINKS = -L$(YALES2_LIB) -lyales2main $(ADD_LIBS)
+
+# Program rule
+$(PROG2MAKE): %: %.o $(LIBS)
+	$(F90) $(FLD) -I. -I$(YALES2_LIB) -o $@ $@.o $(LINKS)
+
+# Generic rules
+%.o: %.f90 $(LIBS)
+	$(F90) $(FFLAGS) -I. -I$(YALES2_LIB) -c $*.f90
+
+%.c: %.c $(LIBS)
+	$(CC) $(CFLAGS) -c $*.c
+
+# clean
+veryclean: clean
+	rm -f dump/* 3D_RBC* slurm*
+
+clean:
+	rm -f $(PROG2MAKE) *.o *.mod *.log .gdb_history

YALES2BIO/testcase_big/batch

+#!/bin/bash
+#SBATCH -J 3D_RBC_SHEAR_BIGp
+#SBATCH --constraint=BDW28
+#SBATCH --nodes=32
+#SBATCH --cpus-per-task=1
+#SBATCH --time=00:59:00
+#SBATCH --exclusive
+
+source $YALES2_HOME/../machines/occigen-bdw/env_bench
+module load vtune/2019.2
+
+export cores=896
+
+#mkdir dump
+
+# deactivate the collective optimizations
+export OMPI_MCA_ess=^pmi
+export OMPI_MCA_pubsub=^pmi
+export OMPI_MCA_pml=ob1
+unset OMPI_MCA_mtl
+unset OMPI_MCA_coll_fca_enable
+unset OMPI_MCA_coll_fca_np
+export OMP_NUM_THREADS=1
+export MKL_NUM_THREADS=1
+export OMPI_MCA_btl_openib_warn_no_device_params_found=0
+
+export I_MPI_OFA_TRANSLATION_CACHE=0
+export I_MPI_DAPL_TRANSLATION_CACHE=0
+
+srun --kill-on-bad-exit -m block --resv-ports --cpu_bind=map_cpu:0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27 -n $cores aps ./rbc_shear_30microns
+
+

YALES2BIO/testcase_big/prepare.sh

+####################################
+# TEST THE ENVIRONMENT
+####################################
+echo "################################"
+echo "CHECKING THE ENVIRONMENT:"
+if [ $YALES2_HOSTTYPE = $BENCH_ARCH ]; then
+   if [ "$(ls -A $YALES2_HOME/src/main 2>/dev/null)" ]; then
+      echo ">> Environment test: SUCCESSFUL"
+   fi
+else
+   echo ">> Environment test: FAILED"
+   echo ">> Please source an environment"
+   exit
+fi
+
+####################################
+# COPY THE CASE IN SCRATCHDIR
+####################################
+currentdate=`date +"%Y-%m-%d"`
+mkdir $SCRATCHDIR/BENCH
+mkdir $SCRATCHDIR/BENCH/${currentdate}
+mkdir $SCRATCHDIR/BENCH/${currentdate}/YALES2BIO
+mkdir $SCRATCHDIR/BENCH/${currentdate}/YALES2BIO/testsmall_case
+
+cp -r ./* $SCRATCHDIR/BENCH/${currentdate}/YALES2BIO/testsmall_case/
+cd $SCRATCHDIR/BENCH/${currentdate}/YALES2BIO/testsmall_case/
+echo "testsmall_case is in:"
+pwd
+
+####################################
+# COMPILE THE TEST CASE
+####################################
+rm makeSmallCase.out makeSmallCase.err 2>/dev/null
+make veryclean 1>/dev/null 2>/dev/null
+make 1>makeSmallCase.out 2>makeSmallCase.err
+
+####################################
+# TEST THE CASE COMPILATION
+####################################
+echo "CHECKING THE CASE COMPILATION:"
+if test -f "tube_10micron_036"; then
+   echo ">> Case compilation: SUCCESSFUL"
+else
+   echo ">> Case compilation: FAILED"
+fi
+echo "################################"
+   
+

YALES2BIO/testcase_big/rbc_shear_30microns.f90

+!-----------------------------------------------------------------------------------
+! rbc_shear_30microns program
+!-----------------------------------------------------------------------------------
+
+!=================================================================
+program main
+
+      use yales2_m
+
+      implicit none
+
+      ! ------------------------
+      character(len=LEN_MAX) :: inputfile
+      ! ------------------------
+
+      inputfile = "rbc_shear_30microns.in"
+
+      ! ------------------------
+      ! init
+      call init_yales2(inputfile,initmpi=.true.)
+
+      ! ------------------------
+      ! temporal loop
+      call temporal_loop()
+
+      ! ------------------------
+      ! restart if needed
+      do while (solver%reset_and_restart)
+         call destroy_yales2(destroympi=.false.)
+         call init_yales2(inputfile,initmpi=.false.)
+         call temporal_loop()
+      end do
+
+      ! ------------------------
+      if (myrank==master) then
+         call print_message("Test passed successfully.",1)
+      end if
+      ! ------------------------
+      ! destroy
+      call destroy_yales2(destroympi=.true.)
+
+end program main
+!=================================================================
+
+!=================================================================
+subroutine initialize_data()
+
+      use yales2_m
+      use IFPORT
+
+      implicit none
+
+      ! ------------------------
+      type(grid_t), pointer :: grid
+      ! ------------------------
+      type(pt_set_t), pointer :: pt_set
+      type(pt_data_t), pointer :: initial_x_ptr,initial_indic_ptr
+      integer :: nset,npt_set
+      logical :: found
+      ! ------------------------
+
+      grid => solver%first_grid
+
+      pt_set => grid%first_pt_set
+      npt_set = count_pt_set(grid%first_pt_set)
+
+      ! loop on pt set
+      do nset=1,npt_set
+
+         ! usefull pt data 
+         call find_pt_data(pt_set%first_pt_data,"INITIAL_X",initial_x_ptr,resfound=found)
+         if (.not.found) then
+             call register_pt_data("INITIAL_X",PT_DATATYPE_REAL_VECTOR,pt_set,dump=.true.,restart=.true.,&
+                                   prescribed_at_injection=.false.,new_ptr=initial_x_ptr,warn_if_exists=.false.)
+         else
+             initial_x_ptr%dump = .true.
+             initial_x_ptr%restart = .true.
+         end if
+
+         call find_pt_data(pt_set%first_pt_data,"INITIAL_INDIC",initial_indic_ptr,resfound=found)
+         if (.not.found) then
+             call register_pt_data("INITIAL_INDIC",PT_DATATYPE_REAL_SCALAR,pt_set,dump=.true.,restart=.true.,&
+                                   prescribed_at_injection=.false.,new_ptr=initial_indic_ptr,warn_if_exists=.false.)
+         else
+             initial_indic_ptr%dump = .true.
+             initial_indic_ptr%restart = .true.
+         end if
+         ! copy the older position
+         ! next
+         pt_set => pt_set%next
+      end do
+
+
+end subroutine initialize_data
+!=================================================================
+
+!=================================================================
+subroutine temporal_loop_preproc()
+
+      use yales2_m
+
+      implicit none
+
+      ! ------------------------
+      ! ------------------------
+
+
+end subroutine temporal_loop_preproc
+!=================================================================
+
+!=================================================================
+subroutine temporal_loop_postproc()
+
+      use yales2_m
+
+      implicit none
+
+      ! ------------------------
+      ! ------------------------
+
+end subroutine temporal_loop_postproc
+!=================================================================

YALES2BIO/testcase_big/rbc_shear_30microns.in

+#Main parameters =========================
+# run
+
+SURFACE_SET surf_set001 VARIABLE_VISCOSITY  = FALSE
+SURFACE_SET surf_set001 VISCOSITY_RATIO  = 5.0
+
+SHEAR_RATE = 1250
+FOURIER = 0.5
+
+NITER_MAX = 2500
+DUMP_TIME_PERIOD = 1.0E-3
+DUMP_SURF_TIME_PERIOD = 1.0E-4
+
+KINEMATIC_VISCOSITY = 7.81250E-8
+BOUNDARY y0 U = -1.875E-2 0.0 0.0
+BOUNDARY y1 U = 1.875E-2 0.0 0.0
+
+##################################
+SOLVER_NAME = 'rbc_shear_30microns'
+SOLVER_TYPE = RBC
+NDIM = 3
+CFL = 0.9
+MOMENTUM_SOURCE_NUMBER=0.14
+ITER_INFO_PERIOD = 10
+
+DEBUG_RBCS = FALSE
+
+# schemes
+TIME_INT = RK4
+SPACE_INT = 4TH
+IMPLICIT_DIFFUSION = FALSE
+
+REAL_IMPLICIT_DIFFUSION = TRUE
+IMPLICIT_CONVERGENCE = 5.0E-26
+ID_SOLVER = PCG
+
+# debug
+DEBUG_LEVEL_MASTER = 3
+DEBUG_LEVEL_SLAVE = 1
+DEBUG_WAIT = 0
+
+# Grid partitioning
+NELEMENTPERGROUP = 500
+
+# parameters
+TURBULENCE_MODEL = NONE
+POISSON_SOLVER = DPCG
+PRESSURE_CONVERGENCE =  3.E-16
+NRECYCLED_RHS = 3
+#Q_CRITERION = 2
+MAIN_GRID_REFINEMENT = 0
+DENSITY = 1000.0
+HALO_SIZE = 0.353E-06
+#EXACT_RESTART = TRUE
+#GEOMETRIC_TOLERANCE = 1.0E-7
+
+#############################
+#BEGIN SURFACE SET PARAMETERS
+#############################
+#NUMBER OF RBCS IN THE SURFACE SET
+SURFACE_SET surf_set001 N_RBC = 1024
+
+#RESTART FROM MESH
+#SURFACE_SET surf_set001 STL_MESH = 'meshes/globule_bille_5006fa.stl'
+#SURFACE_SET surf_set001 RESTART_TYPE = "STL_MESH"
+#SURFACE_SET surf_set001 SCALING_MESH = 1.0E-06
+#SURFACE_SET surf_set001 REFERENCE_MEMBRANE_STATE = INITIAL
+
+#FROM RESTART
+SURFACE_SET surf_set001 RESTART_TYPE = "HDF"
+SURFACE_SET surf_set001 NPART = 28
+SURFACE_SET surf_set001 RESTART_HDF_MESH = './restart/channel_30x10_dx0.345micr.sol000010.surf_set001_**.sol.h5'
+
+SURFACE_SET surf_set001 REFERENCE_MEMBRANE_STATE = FROM_RESTART
+
+
+#FROM EQSHAPE
+#SURFACE_SET surf_set001 RESTART_TYPE = "EQSHAPE"
+#SURFACE_SET surf_set001 NPART = 1
+#SURFACE_SET surf_set001 RESTART_HDF_MESH = 'restart_cell_NB1_RKPM2/cell.sol000050.surf_set001_1.sol.h5'
+#SURFACE_SET surf_set001 REFERENCE_MEMBRANE_STATE = FROM_RESTART
+
+#MECHANICAL PROPERTIES
+SURFACE_SET surf_set001 SURFACES_TYPE = MEMBRANE
+SURFACE_SET surf_set001 MEMBRANE_PHYSICS = HEC
+SURFACE_SET surf_set001 REF_VOLUME = 9.28E-17
+SURFACE_SET surf_set001 REF_SURFACE = 1.33E-10
+SURFACE_SET surf_set001 VOL_CORRECTION = 1
+SURFACE_SET surf_set001 ELASTIC_MODULUS = 0.0
+SURFACE_SET surf_set001 CURVATURE_MODULUS = 3.0E-19
+SURFACE_SET surf_set001 AREA_MODULUS = 0.0
+SURFACE_SET surf_set001 REF_AREA_DIFFERENCE = 0.0
+SURFACE_SET surf_set001 BILAYER_THICKNESS = 0.0
+SURFACE_SET surf_set001 ADE_PENALTY = 0.0
+SURFACE_SET surf_set001 GRID_SPAC = 0.353E-06
+SURFACE_SET surf_set001 HYPERELASTIC_LAW = SK
+SURFACE_SET surf_set001 SK_SHEAR_MODULUS = 2.5E-6
+SURFACE_SET surf_set001 SK_AREA_TO_SHEAR_RATIO = 10.0
+SURFACE_SET surf_set001 C0 = 0.0
+
+SURFACE_SET surf_set001 MEMBRANE_TO_INTERNAL_VISCOSITY_RATIO  = 0.0
+VARIABLE_VISCOSITY_CORRECTION  = ROUGH
+VISCOSITY_CONVERGENCE = 1.0E-11
+
+#CALCULATION PARAMETERS
+#SURFACE_SET surf_set001 DUMP_TENSIONS = TRUE
+SURFACE_SET surf_set001 UNSTRUCTURED_FLUID_GRID = TRUE
+SURFACE_SET surf_set001 RKPM_ORDER = 2
+SURFACE_SET surf_set001 NB_RINGS = 1
+
+#REPULSIVE FORCES
+SURFACE_SET surf_set001 REPULSIVE_FORCES = TRUE
+SURFACE_SET surf_set001 REPULSIVE_FORCES_TYPE = LENNARD-JONES
+SURFACE_SET surf_set001 ZERO_FORCE_DISTANCE = 0.353E-06
+SURFACE_SET surf_set001 REPULSIVE_ENERGY = 1.2E-23
+
+##############################
+##END SURFACE SET PARAMETERS
+##############################
+
+# Particles
+PARTICLE_SET ptset1 DRAG = TRACER
+PARTICLE_SET ptset1 RESTART_TYPE = HDF
+PARTICLE_SET ptset1 RESTART_NPART = 28
+PARTICLE_SET ptset1 RESTART_HDF_SOLUTION = "./restart/channel_30x10_dx0.345micr.sol000010.ptset1_**.sol.h5"
+
+# Restart =================================
+
+RESTART_TYPE = HDF
+RESTART_NPART = 28
+RESTART_HDF_MESH = './restart/channel_30x10_dx0.345micr.sol000000_**.mesh.h5'
+RESTART_HDF_SOLUTION = './restart/channel_30x10_dx0.345micr.sol000010_**.sol.h5'
+
+#RESTART_TYPE = CART
+CART_NX = 448
+CART_NY = 168
+CART_NZ = 224
+CART_X_MIN = -8.0E-5
+CART_X_MAX = 8.0E-5
+CART_Y_MIN = -3.00E-5
+CART_Y_MAX = 3.00E-5
+CART_Z_MIN = -4.0E-5
+CART_Z_MAX = 4.0E-5
+
+# boundaries ==============================
+BOUNDARY x0 DESCRIPTION = 'Channel inlet'
+BOUNDARY x0 TYPE = PERIODIC
+BOUNDARY x0 LABEL = 'X'
+
+BOUNDARY x1 DESCRIPTION = 'Channel outlet'
+BOUNDARY x1 TYPE = PERIODIC
+BOUNDARY x1 LABEL = 'X'
+
+BOUNDARY y0 DESCRIPTION = 'Lower wall'
+BOUNDARY y0 TYPE = WALL
+BOUNDARY y0 WALL_TYPE = TRANSLATING
+BOUNDARY y0 LABEL = 'Y'
+
+BOUNDARY y1 DESCRIPTION = 'Upper wall'
+BOUNDARY y1 TYPE = WALL
+BOUNDARY y1 WALL_TYPE = TRANSLATING
+BOUNDARY y1 LABEL = 'Y'
+
+BOUNDARY z0 DESCRIPTION = 'Bottom wall'
+BOUNDARY z0 TYPE = PERIODIC
+BOUNDARY z0 LABEL = 'Z'
+
+BOUNDARY z1 DESCRIPTION = 'Top wall'
+BOUNDARY z1 TYPE = PERIODIC
+BOUNDARY z1 LABEL = 'Z'
+
+# Dump ====================================
+DUMP_TYPE = HDF
+DUMP_NPART = 28
+DUMP_PREFIX = 'dump/channel_30x10_dx0.345micr.sol'
+DUMP_PREFIX1 = 'dump/surface.sol'