#key parameters:
OPENMP=n
DOUBLE_PRECISION=y
ADIOS=n
BIGDATA=n
PETSc=y
DEBUG=n
GPU=n
GPU_UM=n
COMPILER=gun
EQ_test=n     # unit test equilibrium wrapper
#ICONFIG=TOKAMAK
ICONFIG=FRC

############################################################################
#             Makefile to build the GTC code
#           ==================================
#
# You only need to type "gmake" to build the code on the platforms
# defined below. The makefile runs the "uname -s" command to detect
# the operating system automatically. Other options are:
#
#  % gmake OPENMP=y       Builds the code with OpenMP support
#  % gmake OPENMP=n       Builds the code WITHOUT OpenMP support
#                         The default is with OpenMP support	
#
#  % gmake DOUBLE_PRECISION=y  Builds with 8-byte floating point precision
#                              The default is single precision
#
#  % gmake PETSc=y        Builds the code using the PETSc parallel matrix
#                         solving library.  This is necessary to run the
#                         electromagnetic version of GTC.
#
#  % gmake DEBUG=y        Compiles the files with debugging flags.
#                         The default is no debug option.
#
#  If COMPILER is set to 'default' the Makefile will select the appropriate
#  compiler for the target.  If the user wished to override the target
#  machine default, COMPILER may be set to 'portland', 'intel', or 'gnu'.
#
# Special targets:
#
#  % gmake clean      Removes the executable and all object files (*.o)
#
#  % gmake cleanomp   Removes the executable and the object files
#                     containing OpenMP directives
#
#  % gmake doc        Rebuilds the documentation.
#
#############################################################################
# Default executable name
CMD := gtc
ALL:${CMD}
LIB :=
LD_LIB :=

##### inquire the hostname to judge the system name, to determine what target we're compiling on
ifneq (,${HOST})
  SYSTEMS := ${HOST}
else
  SYSTEMS := $(shell hostname)
endif

##### target machine determines the default compiler, and paths to libraries #####
# edison
ifneq (,$(findstring edison,$(SYSTEMS)))
   ifeq ($(COMPILER),default)
      COMPILER := intel
   endif
   CMP := ftn
#   PETSC_OPT = -D_PETSc35BEFORE ##only needed for kink/tearing
   PETSC_LIB = -I${PETSC_DIR}/include -I${PETSC_DIR}/include/finclude
   NETCDF_LIB = -I${NETCDF_DIR}/include
   NETCDF_LD_LIB = -L${NETCDF_DIR}/lib -lnetcdf
   LIBD = /global/u2/x/xiao/AdioFranklin/lib
   INCD = /global/u2/x/xiao/AdioFranklin/include
   ADIOS_LIB = -I$(INCD)
   ADIOS_LD_LIB = -L$(LIBD) -ladios -lmxml
endif
#sun
ifneq (,$(findstring sun, $(SYSTEMS)))
    COMPILER := gnu
    CMP := mpif90
    HDF5_LIB=/home/xswei/software/hdf5-parallel/include
    OPT := -ffree-line-length-0
    PETSC_LIB = -I${PETSC_DIR}/include -I${PETSC_DIR}/${PETSC_ARCH}/include -I${PETSC_DIR}/include/petsc/finclude
    LIB:= -I$(HDF5_LIB)
    LD_LIB:= -lhdf5_fortran -L/home/xswei/software/hdf5-parallel/lib -L${PETSC_DIR}/${PETSC_ARCH}/lib
    ifeq (${PETSc},y)
      LD_LIB += -L${PETSC_DIR}/${PETSC_ARCH}/lib -lpetsc
    endif
    ADIOS_LIB = -I$(INCD)
    ADIOS_LD_LIB = -L$(LIBD) -ladios -lmxml
endif
# summit
ifneq (,$(findstring summit,$(shell hostname -d)))
   ifeq ($(COMPILER),default)
      COMPILER := portland
   endif
   CMP := mpif90
   ifeq ($(COMPILER),gnu)
      OPT := -ffree-line-length-0 -Wuninitialized
   endif
   HDF5_LIB := -I${OLCF_HDF5_ROOT}/include
   PETSC_LIB = -I${PETSC_DIR}/include 
   LIB:= $(HDF5_LIB)
   LD_LIB:= -lhdf5_fortran
   ifeq (${PETSc},y)
     LD_LIB += -lpetsc
   endif
   ADIOS_LIB = -I$(INCD)
   ADIOS_LD_LIB = -L$(LIBD) -ladios -lmxml
endif
# cori
ifneq (,$(findstring cori,$(SYSTEMS)))
   ifeq ($(COMPILER),default)
      COMPILER := intel
      #COMPILER := gnu
   endif
   CMP := ftn
#   OPT := -ffree-line-length-0
#   PETSC_OPT = -D_PETSc35BEFORE ##only needed for kink/tearing
   PETSC_LIB = -I${PETSC_DIR}/include -I${PETSC_DIR}/include/petsc/finclude
   NETCDF_LIB = -I${NETCDF_DIR}/include
   NETCDF_LD_LIB = -L${NETCDF_DIR}/lib -lnetcdf
   LIBD = /global/u2/x/xiao/AdioFranklin/lib
   INCD = /global/u2/x/xiao/AdioFranklin/include
   ADIOS_LIB = -I$(INCD)
   ADIOS_LD_LIB = -L$(LIBD) -ladios -lmxml
endif

# hopper
ifneq (,$(findstring hopper,$(SYSTEMS)))
   ifeq ($(COMPILER),default)
      COMPILER := portland
   endif
   CMP := ftn
   PETSC_LIB = -I${PETSC_DIR}/include -I${PETSC_DIR}/include/finclude
   NETCDF_LIB = -I${NETCDF_DIR}/include
   NETCDF_LD_LIB = -L${NETCDF_DIR}/lib -lnetcdf
   LIBD = /global/u2/x/xiao/AdioFranklin/lib
   INCD = /global/u2/x/xiao/AdioFranklin/include
   ADIOS_LIB = -I$(INCD)
   ADIOS_LD_LIB = -L$(LIBD) -ladios -lmxml
endif

# titan
ifneq (,$(findstring titan,$(SYSTEMS)))
   ifeq ($(COMPILER),default)
      ifeq ($(GPU),y)
	COMPILER := portland
      else
	COMPILER := intel
      endif
   endif
   CMP :=ftn
#   PETSC_OPT = -D_PETSc35BEFORE##only needed for kink/tearing
   PETSC_LIB = -I${PETSC_DIR}/include/petsc -I${PETSC_DIR}/include/petsc/finclude
   NETCDF_LIB = -I${NETCDF_DIR}/include
   NETCDF_LD_LIB = -L${NETCDF_DIR}/lib
   LIBD = /ccs/home/hardes/gtc_adios/jaguarlib
   INCD = /ccs/home/hardes/gtc_adios/jaguarinclude
   ADIOS_LIB = -I$(INCD)
   ADIOS_LD_LIB = -L$(LIBD) -ladios -lmxml
endif

# Tianhe-1A
ifneq (,$(findstring ln,$(SYSTEMS)))
   ifeq ($(COMPILER),default)
      ifeq ($(GPU),y)
	COMPILER := portland
      else
	COMPILER := intel
      endif
   endif
   CMP := mpif90

   PETSC_solver = petsc.o
   PETSC_OPT = -D_PETSc31P8ANDBEFORE
	 #PETSC_OPT += -D_PETSc30ANDBEFORE
   ifneq (,${PETSC_DIR})
      include ${PETSC_DIR}/conf/variables
   endif

   PETSC_LIB := -I${PETSC_DIR}/include -I${PETSC_DIR}/include/finclude 
   PETSC_LD_LIB := ${PETSC_KSP_LIB}

   NETCDF_LIB := -I${NETCDF_DIR}/include
   NETCDF_LD_LIB := -L${NETCDF_DIR}/lib -lnetcdf -lnetcdff -lnetcdf_c++\
		 -Wl,--rpath -Wl,${NETCDF_DIR}/lib
endif

# PKU cluster
ifneq (,$(findstring mgt,$(SYSTEMS)))
   PETSC_LIB = -Wl,-rpath,${PETSC_LIB_DIR} -L${PETSC_LIB_DIR} -lpetsc -lX11\
							 -L/share/soft/lapack-3.3.1 -llapack_LINUX -lblas_LINUX
endif


#iop
ifneq (,$(findstring indac,$(SYSTEMS)))
   ifeq ($(COMPILER),default)
      COMPILER := intel
   endif
   CMP := mpif90

   #PETSC_solver = petsc.o
   PETSC_OPT = -D_PETSc31P8ANDBEFORE
         #PETSC_OPT += -D_PETSc30ANDBEFORE
   ifneq (,${PETSC_DIR})
      include ${PETSC_DIR}/conf/variables
   endif

   PETSC_LIB := -I${PETSC_DIR}/include -I${PETSC_DIR}/include/finclude
   PETSC_LD_LIB := ${PETSC_KSP_LIB}

   NETCDF_LIB := -I${NETCDF_DIR}/include
   NETCDF_LD_LIB := -L${NETCDF_DIR}/lib -lnetcdf -lnetcdff -lnetcdf_c++\
                 -Wl,--rpath -Wl,${NETCDF_DIR}/lib
endif

#### compiler determines debugging, openmp, and double precision flags #####
# intel fortran
ifeq ($(COMPILER),intel)
   OPTIMOPT := -O
   #OPTIMOPT := -O3
   #OPTIMOPT += -fp-model fast=2
   #OPTIMOPT += -align array64byte
   #OPTIMOPT += -no-vec
# debug flags changed by L. Shi to suppress long warning outputs.
   #DEBUGOPT := -g -free -pc64 -check all -warn nounused -warn all -debug all -debug-parameters all -fp-stack-check
   DEBUGOPT := -g -free -pc64 -check 'all,noarg_temp_created' -warn none -debug all -debug-parameters all -fp-stack-check
   OMPOPT := -qopenmp
   DPOPT := -fpconstant
endif

# portland group fortran
ifeq ($(COMPILER),portland)
   OPTIMOPT := -O3 -Mfree -Kieee
   DEBUGOPT := -g -C -gopt -Mbounds -Mchkfpstk -Mchkptr -Mchkstk -Mcoff -Mdwarf3 -Melf -Mpgicoff -Minform=inform
   OMPOPT := -mp
   DPOPT := -DDOUBLE_PRECISION
   LD_LIB += -L/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20180914/linux-rhel7-ppc64le/pgi-18.10/petsc-3.7.2-4hldrigfxg5qlofldjuafy2dka3bsqpi/lib -L/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20180914/linux-rhel7-ppc64le/pgi-18.10/hdf5-1.10.3-wvb2rm6hzrii7fdm4d3iai2gihtx3arp/lib
endif

# GNU gfortran
ifeq ($(COMPILER),gnu)
   OPTIMOPT := -O
   DEBUGOPT := -fcheck=all,no-array-temps
   OMPOPT := -fopenmp
   DPOPT := -ffree-line-length-0
endif


### assemble all the options
#OPT :=
#LIB :=

ifeq ($(ICONFIG),FRC)
   ICONFIG_OPT = -D_FRC
   OPT += $(ICONFIG_OPT)
   EQ_interface ?=LAMYRIDGE_interface.o
   ifeq ($(EQ_test),y)
       EQ_test_OPT = -D_LR
       OPT += $(EQ_test_OPT)
   endif
else ifeq ($(ICONFIG),TOKAMAK)
   ICONFIG_OPT = -D_TOKAMAK
   OPT += $(ICONFIG_OPT)
   EQ_interface ?=EFIT_interface.o
   ifeq ($(EQ_test),y)
       EQ_test_OPT = -D_EFIT
       OPT += $(EQ_test_OPT)
   endif
endif

ifeq ($(BIGDATA),y)
  DATAOUT3D ?= dataout3d.o
  BIGDATA_OPT = -D_BIGDATA
  OPT += $(BIGDATA_OPT)
  ifeq ($(ADIOS),y)
    OPT := -DADIOS=1
    LIB := $(ADIOS_LIB)
    LD_LIB := $(ADIOS_LD_LIB)
  else
    LIB += $(NETCDF_LIB)
    LD_LIB += $(NETCDF_LD_LIB)
  endif
endif
ifeq ($(PETSc),y)
   PETSC_solver ?= petsc.o
   PETSC_OPT += -D_PETSc
   OPT += $(PETSC_OPT)
   LIB += $(PETSC_LIB)
   LD_LIB += $(PETSC_LD_LIB)
else
   PETSC_solver :=
endif

ifeq ($(OPENMP),y)
   OPT += $(OMPOPT)
endif
ifeq ($(DOUBLE_PRECISION),y)
   OPT += -DDOUBLE_PRECISION $(DPOPT)
endif
ifeq ($(DEBUG),y)
   OPT += $(DEBUGOPT)
else
   OPT += $(OPTIMOPT)
endif

ifeq ($(GPU),y)
  ifeq ($(GPU_UM),y)
    OPT := -DGPU_UM -Mcuda=cuda7.5 -Minfo=accel -acc -ta=nvidia:cc35,ptxinfo,maxregcount:64,managed $(OPT)
   else
    OPT := -Mcuda=cuda7.5 -Minfo=accel -acc -ta=nvidia:cc35,ptxinfo,maxregcount:64 $(OPT)
  endif
  #OPT := -Mcuda=cuda7.0 -Minfo=all -acc -ta=nvidia:cc35,ptxinfo,maxregcount:64,time,host $(OPT)
  NVCC := nvcc
  CUDA_OPT := -arch=sm_35 -O3\
	-I/opt/cray/mpt/7.4.0/gni/mpich-pgi/15.3/include
  CUDA_OBJ := shift_cuda.o
endif


##################################################################
# We add ".F90" to the list of suffixes to allow source files on which the
# co-processor will be run automatically.
.SUFFIXES: .o .F90 .F
.PHONY: clean

ifeq ($(ICONFIG),FRC)
# List of all the object files needed to build the FRC code
OBJ:=module.o initial.o function.o main.o\
     taehdf5.o equilibrium.o $(EQ_interface)\
     trace_fieldlines.o simulation_domain.o mapping_splines.o\
     grids_setup.o grids_eqdata.o grids_fields.o\
     cell_volume.o laplacian.o\
     $(PETSC_solver) fft_gl.o\
     shift.o $(CUDA_OBJ) tracking.o

OBJ +=loade.o pushe.o antenna.o local_pssolver.o fluid.o\
      loadi.o pushi.o charge.o data_output.o gkload.o gkpush.o\
      loadfki.o pushfki.o\
      setup.o restart.o #global_pssolver.o
ifeq ($(PETSc),y)
OBJ += global_pssolver.o
endif

else ifeq ($(ICONFIG),TOKAMAK)
# List of all object files needed to build the EFIT tokamak code
OBJ:= module.o $(PETSC_solver) function.o initial.o $(EQ_interface) equilibrium.o main_efit.o

endif

$(CMD): $(OBJ)
	$(CMP) $(OPT) -o $(CMD) $(OBJ) $(LD_LIB)

#========= Unit_test_equilibrium_interface
ifeq ($(EQ_test),y)
    ifeq ($(ICONFIG),FRC)

        OBJ_EQ:= module.o taehdf5.o LAMYRIDGE_interface.o

    else ifeq ($(ICONFIG),TOKAMAK)

        OBJ_EQ:= module.o EFIT_interface.o

    endif
endif
gtceq: $(OBJ_EQ)
	echo $(CMP) $(OPT)
	$(CMP) $(OPT) -o gtceq $(OBJ_EQ) $(LD_LIB)
#==============================file dependence================================
$(filter-out module.o,$(OBJ)): module.o  # all .o files except for module.o dependent on module.o

module.o : module.F90
	$(CMP) $(OPT) $(LIB) -c module.F90

ifeq ($(PETSc),y)
main.o : main.F90 module.o initial.o setup.o loade.o pushe.o antenna.o\
         local_pssolver.o fluid.o loadi.F90 pushi.o charge.o data_output.o \
         tracking.o restart.o pushfki.o loadfki.o global_pssolver.o
	$(CMP) $(OPT) $(LIB) -c main.F90
else
main.o : main.F90 module.o initial.o setup.o loade.o pushe.o antenna.o\
         local_pssolver.o fluid.o loadi.F90 pushi.o charge.o data_output.o \
         tracking.o restart.o pushfki.o loadfki.o
	$(CMP) $(OPT) $(LIB) -c main.F90
endif

main_efit.o : main_efit.F90 module.o initial.o equilibrium.o
	$(CMP) $(OPT) $(LIB) -c main_efit.F90

function.o: function.F90 module.o
	$(CMP) $(OPT) $(LIB) -c function.F90

equilibrium.o: equilibrium.F90 module.o function.o $(EQ_interface)
	$(CMP) $(OPT) $(LIB) -c equilibrium.F90

LAMYRIDGE_interface.o: LAMYRIDGE_interface.F90 module.o taehdf5.o 
	$(CMP) $(OPT) $(LIB) -c LAMYRIDGE_interface.F90

EFIT_interface.o: EFIT_interface.F90 module.o
	$(CMP) $(OPT) $(LIB) -c EFIT_interface.F90

trace_fieldlines.o: trace_fieldlines.F90 module.o function.o equilibrium.o 
	$(CMP) $(OPT) $(LIB) -c trace_fieldlines.F90

ifeq ($(PETSc),y)
initial.o: initial.F90 module.o petsc.o
	$(CMP) $(OPT) $(LIB) -c initial.F90
else
initial.o: initial.F90 module.o
	$(CMP) $(OPT) $(LIB) -c initial.F90
endif

ifeq ($(PETSc),y)
petsc.o: petsc.F90 module.o
	$(CMP) $(OPT) $(LIB) -c petsc.F90
endif

simulation_domain.o: simulation_domain.F90 module.o function.o equilibrium.o trace_fieldlines.o
	$(CMP) $(OPT) $(LIB) -c simulation_domain.F90

mapping_splines.o: mapping_splines.F90 module.o function.o equilibrium.o trace_fieldlines.o simulation_domain.o 
	$(CMP) $(OPT) $(LIB) -c mapping_splines.F90

grids_setup.o: grids_setup.F90 module.o function.o equilibrium.o mapping_splines.o trace_fieldlines.o simulation_domain.o
	$(CMP) $(OPT) $(LIB) -c grids_setup.F90

grids_eqdata.o: grids_eqdata.F90 module.o function.o equilibrium.o mapping_splines.o cell_volume.o grids_fields.o grids_setup.o
	$(CMP) $(OPT) $(LIB) -c grids_eqdata.F90

grids_fields.o: grids_fields.F90 module.o function.o grids_setup.o simulation_domain.o
	$(CMP) $(OPT) $(LIB) -c grids_fields.F90

ifeq ($(PETSc),y)
laplacian.o: laplacian.F90 module.o grids_setup.o simulation_domain.o grids_eqdata.o petsc.o
	$(CMP) $(OPT) $(LIB) -c laplacian.F90
else
laplacian.o: laplacian.F90 module.o grids_setup.o simulation_domain.o grids_eqdata.o
	$(CMP) $(OPT) $(LIB) -c laplacian.F90
endif

cell_volume.o: cell_volume.F90 module.o function.o equilibrium.o mapping_splines.o grids_setup.o simulation_domain.o
	$(CMP) $(OPT) $(LIB) -c cell_volume.F90

restart.o: restart.F90 module.o grids_fields.o data_output.o
	$(CMP) $(OPT) $(LIB) -c restart.F90


#=====================================================================

local_pssolver.o: local_pssolver.F90 module.o grids_setup.o simulation_domain.o grids_eqdata.o grids_fields.o
	$(CMP) $(OPT) $(LIB) -c local_pssolver.F90

antenna.o: antenna.F90 module.o grids_setup.o simulation_domain.o grids_eqdata.o grids_fields.o laplacian.o
	$(CMP) $(OPT) $(LIB) -c antenna.F90

fluid.o: fluid.F90 module.o grids_setup.o simulation_domain.o grids_eqdata.o grids_fields.o
	$(CMP) $(OPT) $(LIB) -c fluid.F90
loade.o: loade.F90 module.o function.o equilibrium.o mapping_splines.o grids_setup.o cell_volume.o simulation_domain.o grids_eqdata.o grids_fields.o
	$(CMP) $(OPT) $(LIB) -c loade.F90
pushe.o: pushe.F90 module.o function.o equilibrium.o mapping_splines.o grids_setup.o simulation_domain.o grids_eqdata.o grids_fields.o
	$(CMP) $(OPT) $(LIB) -c pushe.F90
loadi.o: loadi.F90 module.o function.o equilibrium.o mapping_splines.o grids_setup.o cell_volume.o simulation_domain.o grids_eqdata.o grids_fields.o restart.o
	$(CMP) $(OPT) $(LIB) -c loadi.F90 
pushi.o: pushi.F90 module.o function.o equilibrium.o mapping_splines.o grids_setup.o simulation_domain.o grids_eqdata.o grids_fields.o
	$(CMP) $(OPT) $(LIB) -c pushi.F90
gkpush.o: grids_setup.o module.o gkpush.F90 grids_fields.o
	$(CMP) $(OPT) $(LIB) -c gkpush.F90
gkload.o: grids_setup.o equilibrium.o module.o gkload.F90 grids_fields.o
	$(CMP) $(OPT) $(LIB) -c gkload.F90
pushifk.o: pushifk.F90 module.o function.o equilibrium.o mapping_splines.o grids_setup.o simulation_domain.o grids_eqdata.o grids_fields.o
	$(CMP) $(OPT) $(LIB) -c pushifk.F90
charge.o: charge.F90 module.o function.o equilibrium.o mapping_splines.o grids_setup.o simulation_domain.o grids_eqdata.o grids_fields.o
	$(CMP) $(OPT) $(LIB) -c charge.F90
data_output.o: data_output.F90 module.o function.o equilibrium.o mapping_splines.o grids_setup.o cell_volume.o simulation_domain.o grids_eqdata.o grids_fields.o fluid.o antenna.o
	$(CMP) $(OPT) $(LIB) -c data_output.F90

ifeq ($(PETSc),y)
global_pssolver.o: global_pssolver.F90 module.o function.o equilibrium.o mapping_splines.o grids_setup.o simulation_domain.o grids_eqdata.o grids_fields.o $(PETSC_solver) laplacian.o
	$(CMP) $(OPT) $(LIB) -c global_pssolver.F90
endif

tracking.o: tracking.F90 module.o 
	$(CMP) $(OPT) $(LIB) -c tracking.F90

.F90.o : module.o
	$(CMP) $(OPT) $(LIB) -c $<

shift_cuda.o : shift_cuda.cu
	$(NVCC) $(CUDA_OPT) -c shift_cuda.cu

setup.o: setup.F90 module.o function.o mapping_splines.o equilibrium.o laplacian.o cell_volume.o grids_setup.o simulation_domain.o grids_eqdata.o grids_fields.o fluid.o
	$(CMP) $(OPT) $(LIB) -c setup.F90
pushfki.o: pushfki.F90 module.o function.o equilibrium.o mapping_splines.o grids_setup.o simulation_domain.o grids_eqdata.o grids_fields.o
loadfki.o: loadfki.F90 module.o function.o equilibrium.o mapping_splines.o grids_setup.o cell_volume.o simulation_domain.o grids_eqdata.o grids_fields.o restart.o
# The following tag is meant to "clean" the directory by removing the
# executable along with all the object files created by the compilation 
# One only has to run:  gmake clean


clean::
	rm -f $(CMD) gtceq $(OBJ_EQ) $(OBJ) *.mod *.o *genmod.f90 *.optrpt