Initial commit

Makefile

+CC=icc
+MPICC=mpicc
+LIB_OMP=-qopenmp
+
+SRC_DIR=src
+
+all: hello_mpi hello_omp hello_hybrid
+
+hello_mpi: $(SRC_DIR)/hello_mpi.c 
+	$(MPICC) -o bin/$@ $^ 
+
+hello_omp: $(SRC_DIR)/hello_omp.c 
+	$(CC) -o bin/$@ $^ $(LIB_OMP) 
+
+hello_hybrid: $(SRC_DIR)/hello_hybrid.c 
+	$(MPICC) -o bin/$@ $^ $(LIB_OMP) 
+
+clean:
+	rm bin/*

src/hello_hybrid.c

+#include <omp.h>
+#include <stdio.h>
+#include <mpi.h>
+#include <stdlib.h>
+#include <sys/unistd.h>
+
+int main(int argc,char**argv)
+{
+  int mpiRank, mpiSize;
+  char hostname[128];
+  int nthreads, tid, cpuid;  
+	int i, j=0;
+
+  MPI_Init(&argc,&argv);
+  MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank);
+  MPI_Comm_size(MPI_COMM_WORLD, &mpiSize);
+
+  gethostname(hostname, sizeof hostname);
+
+	#pragma omp parallel
+	{
+ 	nthreads = omp_get_num_threads();
+	}
+
+  if ( mpiRank== 0) 
+		printf("Run executed using %d MPI processes, with %d threads per process \n", mpiSize, nthreads);
+
+	for(i = 0; i < mpiSize; i++) {
+    MPI_Barrier(MPI_COMM_WORLD);
+    if (i == mpiRank) {
+  		printf("%s: MPI n° %d -> cpuid %d \n",hostname, mpiRank,sched_getcpu());
+		#pragma omp parallel private(tid, nthreads, cpuid) shared(i)
+  	{
+    	tid=omp_get_thread_num();
+    	nthreads = omp_get_num_threads();
+    	cpuid = sched_getcpu();
+
+			while(j < tid){
+				#pragma omp flush(j)
+			}  
+   		printf("\t thread n° %d -> cpuid %d \n", tid, cpuid);
+			j++;
+			#pragma omp flush(j)
+		}
+    }
+	}
+  MPI_Finalize();
+}

src/hello_mpi.c

+#include <mpi.h>
+#include <stdlib.h>
+#include <sys/unistd.h>
+#include <stdio.h>
+#include <string.h>
+
+int main(int argc,char**argv)
+{
+  int mpiRank, mpiSize, i;
+  char hostname[128];
+  int nthreads, tid, cpuid;  
+
+  MPI_Init(&argc,&argv);
+  MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank);
+  MPI_Comm_size(MPI_COMM_WORLD, &mpiSize);
+
+  gethostname(hostname, sizeof hostname);
+
+  if ( mpiRank== 0) 
+		printf("Run executed using %d MPI processes \n", mpiSize);
+
+	for(i = 0; i < mpiSize; i++) {
+    MPI_Barrier(MPI_COMM_WORLD);
+    if (i == mpiRank) {
+  		printf("%s: MPI n° %d -> cpuid %d \n",hostname, mpiRank,sched_getcpu());
+    }
+	}
+
+  MPI_Finalize();
+}

src/hello_omp.c

+#include <omp.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/unistd.h>
+
+int main(int argc,char**argv)
+{
+  char hostname[128];
+  int nthreads, cpuid, tid;  
+	int i=0;
+
+  gethostname(hostname, sizeof hostname);
+
+#pragma omp parallel private(tid, nthreads, cpuid) shared(i)
+  {
+    tid=omp_get_thread_num();
+    nthreads = omp_get_num_threads();
+    cpuid = sched_getcpu();
+
+		if(tid == 0)
+			printf("Run executed using %d threads \n", nthreads);
+
+		while(i < tid){
+			#pragma omp flush(i)
+		}  
+   	printf("on host %s, thread n° %d/%d, for cpuid %d \n", hostname, tid, nthreads, cpuid);
+		i++;
+			#pragma omp flush(i)
+	}
+
+}
+

test_scripts/occigen/bdw28/bdw28_hybrid.sh

+#!/bin/bash
+#SBATCH -J bdw28_hybrid
+#SBATCH --nodes=6
+#SBATCH --ntasks=12
+#SBATCH --ntasks-per-node=2
+#SBATCH --cpus-per-task=14
+#SBATCH --time=0:40:00
+#SBATCH -C BDW28 
+#SBATCH --exclusive
+#SBATCH --mem=50GB
+#SBATCH --output bdw_hybrid.output.slurm 
+
+set -e
+
+export I_MPI_DOMAIN=auto
+export I_MPI_PIN_RESPECT_CPUSET=0
+export I_MPI_DEBUG=4
+
+#Make sure that OMP_NUM_THREADS = cpus-per-task * KMP_HW_SUBSET
+export KMP_HW_SUBSET=1T
+export OMP_NUM_THREADS=14
+export KMP_AFFINITY=verbose,compact,1,0,granularity=fine
+
+module load intel intelmpi
+
+ulimit -s unlimited
+
+rm -f *.out
+
+srun ../../../bin/hello_hybrid
+

test_scripts/occigen/bdw28/bdw28_mpi.sh

+#!/bin/bash
+#SBATCH -J bdw28_mpi
+#SBATCH --nodes=6
+#SBATCH --ntasks=168
+#SBATCH --ntasks-per-node=28
+#SBATCH --time=0:40:00
+#SBATCH -C BDW28 
+#SBATCH --exclusive
+#SBATCH --mem=50GB
+#SBATCH --output bdw_mpi.output.slurm 
+
+set -e
+
+export I_MPI_DOMAIN=auto
+export I_MPI_PIN_RESPECT_CPUSET=0
+export I_MPI_DEBUG=4
+
+
+module load intel intelmpi
+
+ulimit -s unlimited
+
+srun ../../../bin/hello_mpi
+
+

test_scripts/occigen/bdw28/bdw28_omp.sh

+#!/bin/bash
+#SBATCH -J bdw28_omp 
+#SBATCH --nodes=1
+#SBATCH --ntasks=1
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=28
+#SBATCH --time=0:40:00
+#SBATCH -C BDW28
+#SBATCH --exclusive
+#SBATCH --mem=50GB
+#SBATCH --output bdw_omp.output.slurm 
+
+set -e
+
+#Make sure that OMP_NUM_THREADS = cpus-per-task * KMP_HW_SUBSET
+export KMP_HW_SUBSET=1T
+export OMP_NUM_THREADS=28
+export KMP_AFFINITY=verbose,compact,1,0,granularity=fine
+
+module load intel 
+
+ulimit -s unlimited
+
+rm -f *.out
+
+srun ../../../bin/hello_omp
+
+

test_scripts/occigen/hsw24/hsw24_hybrid.sh

+#!/bin/bash
+#SBATCH -J hsw24_hybrid
+#SBATCH --nodes=7
+#SBATCH --ntasks=14
+#SBATCH --ntasks-per-node=2
+#SBATCH --cpus-per-task=12
+#SBATCH --time=0:40:00
+#SBATCH -C HSW24 
+#SBATCH --exclusive
+#SBATCH --mem=50GB
+#SBATCH --output hsw_hybrid.output.slurm 
+
+set -e
+
+export I_MPI_DOMAIN=auto
+export I_MPI_PIN_RESPECT_CPUSET=0
+export I_MPI_DEBUG=4
+
+#Make sure that OMP_NUM_THREADS = cpus-per-task * KMP_HW_SUBSET
+export KMP_HW_SUBSET=1T
+export OMP_NUM_THREADS=12
+export KMP_AFFINITY=verbose,compact,1,0,granularity=fine
+
+module load intel intelmpi
+
+ulimit -s unlimited
+
+rm -f *.out
+
+srun ../../../bin/hello_hybrid
+

test_scripts/occigen/hsw24/hsw24_mpi.sh

+#!/bin/bash
+#SBATCH -J hsw24_mpi
+#SBATCH --nodes=7
+#SBATCH --ntasks=168
+#SBATCH --ntasks-per-node=24
+#SBATCH --cpus-per-task=1
+#SBATCH --time=0:40:00
+#SBATCH -C HSW24 
+#SBATCH --exclusive
+#SBATCH --mem=50GB
+#SBATCH --output hsw_mpi.output.slurm 
+
+set -e
+
+export I_MPI_DOMAIN=auto
+export I_MPI_PIN_RESPECT_CPUSET=0
+export I_MPI_DEBUG=4
+
+
+module load intel intelmpi
+
+ulimit -s unlimited
+
+srun ../../../bin/hello_mpi
+
+

test_scripts/occigen/hsw24/hsw24_omp.sh

+#!/bin/bash
+#SBATCH -J hsw24_omp 
+#SBATCH --nodes=1
+#SBATCH --ntasks=1
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=24
+#SBATCH --time=0:40:00
+#SBATCH -C HSW24 
+#SBATCH --exclusive
+#SBATCH --mem=50GB
+#SBATCH --output hsw_omp.output.slurm 
+
+set -e
+
+#Make sure that OMP_NUM_THREADS = cpus-per-task * KMP_HW_SUBSET
+export KMP_HW_SUBSET=1T
+export OMP_NUM_THREADS=24
+export KMP_AFFINITY=verbose,compact,1,0,granularity=fine
+
+module load intel 
+
+ulimit -s unlimited
+
+rm -f *.out
+
+srun ../../../bin/hello_omp
+
+