#!/bin/bash 

if [ $# -ne 5 ]; then
    echo $0: usage: run.sh.gtc \(#nodes\) \(#process per node\) \(#omp threads per node\) \(walltime in minute\) \(gpu or cpu\)
    exit 1
fi

nodes=$1
ppn=$2
ompthreads=$3
walltime=$4
mode=$5
let nmpi=$nodes*$ppn

if [ $mode == "gpu" ]; then
    echo "GPU Normal Run:"
    name="GTC-GPU"
else
    echo "CPU Normal Run:"
    name="GTC-CPU"
fi
#--------------------------------------
cat >batch.job <<EOF
#!/bin/bash
#BSUB -P csc190gtc
#BSUB -W ${walltime} 
#BSUB -alloc_flags "gpudefault"
#BSUB -o %J.out
#BSUB -e %J.err
#BSUB -J ${name}
#BSUB -nnodes ${nodes}
#---------------------------------
export OMP_NUM_THREADS=${ompthreads}
ulimit -s 10240

mkdir -p restart_dir1
mkdir -p restart_dir2
mkdir -p restart_dir

mkdir -p phi_dir
mkdir -p trackp_dir

jsrun -n${nmpi} -g1 -a1 -lgpu-cpu -c${ompthreads} ./gtc 

EOF
#---------------------------------------
bsub batch.job
echo total MPI ranks: ${nmpi}
echo OpenMP threads per process: ${ompthreads}
echo Options: ${opt}