############################################################################
#             Makefile to build the GTC code
#           ==================================
#
# You only need to type "gmake" to build the code on the platforms
# defined below. The makefile runs the "uname -s" command to detect
# the operating system automatically. Other options are:
#
#  % gmake OPENMP=y       Builds the code with OpenMP support
#  % gmake OPENMP=n       Builds the code WITHOUT OpenMP support
#                         The default is with OpenMP support	
#
#  % gmake DOUBLE_PRECISION=y  Builds with 8-byte floating point precision
#                              The default is single precision
#
#  % gmake PETSc=y        Builds the code using the PETSc parallel matrix
#                         solving library.  This is necessary to run the
#                         electromagnetic version of GTC.
#
#  % gmake DEBUG=y        Compiles the files with debugging flags.
#                         The default is no debug option.
#
#  If COMPILER is set to 'default' the Makefile will select the appropriate
#  compiler for the target.  If the user wished to override the target
#  machine default, COMPILER may be set to 'portland', 'intel', or 'gnu'.
#
# Special targets:
#
#  % gmake clean      Removes the executable and all object files (*.o)
#
#  % gmake cleanomp   Removes the executable and the object files
#                     containing OpenMP directives
#
#  % gmake doc        Rebuilds the documentation.
#
#############################################################################
# Default executable name
CMD := gtc
ALL:${CMD}
INCLUDE :=
LIB :=
OPT :=

-include local.mk

#### compiler determines debugging, openmp, and double precision flags #####
# intel fortran
ifeq ($(COMPILER),intel)
   OPTIMOPT := -O
   #OPTIMOPT := -O3
   #OPTIMOPT += -fp-model fast=2
   #OPTIMOPT += -align array64byte
   #OPTIMOPT += -no-vec
# debug flags changed by L. Shi to suppress long warning outputs.
   #DEBUGOPT := -g -free -pc64 -check all -warn nounused -warn all -debug all -debug-parameters all -fp-stack-check
   DEBUGOPT := -g -free -pc64 -check 'all,noarg_temp_created' -warn none -debug all -debug-parameters all -fp-stack-check
   OMPOPT := -qopenmp
   DPOPT := -fpconstant
endif

# portland group fortran
ifeq ($(COMPILER),portland)
   OPTIMOPT := -O3 -Mfree -Kieee
   DEBUGOPT := -g -C -gopt -Mbounds -Mchkfpstk -Mchkptr -Mchkstk -Mcoff -Mdwarf3 -Melf -Mpgicoff -Minform=inform
   OMPOPT := -mp
#   DPOPT := -DDOUBLE_PRECISION
endif

# GNU gfortran
ifeq ($(COMPILER),gnu)
   OPT += -ffree-line-length-0 -Wuninitialized
   OPTIMOPT := -O
   DEBUGOPT := -g -fcheck=all,no-array-temps
   OMPOPT := -fopenmp
   DPOPT :=
endif


### assemble all the options

ifeq ($(ICONFIG),FRC)
   ICONFIG_OPT = -D_FRC
   OPT += $(ICONFIG_OPT)
   EQ_interface ?=LAMYRIDGE_interface.o
   ifeq ($(EQ_test),y)
       EQ_test_OPT = -D_LR
       OPT += $(EQ_test_OPT)
   endif
else ifeq ($(ICONFIG),TOKAMAK)
   ICONFIG_OPT = -D_TOKAMAK
   OPT += $(ICONFIG_OPT)
   EQ_interface ?=EFIT_interface.o
   ifeq ($(EQ_test),y)
       EQ_test_OPT = -D_EFIT
       OPT += $(EQ_test_OPT)
   endif
endif

# HDF5_INCLUDE, HDF5_LIB should be specified in all cases
INCLUDE += $(HDF5_INCLUDE)
LIB += $(HDF5_LIB)

# ADIOS_INCLUDE, ADIOS_LIB, NETCDF_INCLUDE, NETCDF_LIB should be specified if BIGDATA=y
ifeq ($(BIGDATA),y)
  DATAOUT3D ?= dataout3d.o
  BIGDATA_OPT = -D_BIGDATA
  OPT += $(BIGDATA_OPT)
  ifeq ($(ADIOS),y)
    OPT += -DADIOS=1
    INCLUDE += $(ADIOS_INCLUDE)
    LIB += $(ADIOS_LIB)
  else
    INCLUDE += $(NETCDF_INCLUDE)
    LIB += $(NETCDF_LIB)
  endif
endif

# PETSC_INLUCDE, PETSC_LIB should be specified if PETSc=y
ifeq ($(PETSc),y)
   PETSC_solver ?= petsc.o
   PETSC_OPT += -D_PETSc
   OPT += $(PETSC_OPT)
   INCLUDE += $(PETSC_INCLUDE)
   LIB += $(PETSC_LIB)
else
   PETSC_solver :=
endif

ifeq ($(OPENMP),y)
   OPT += $(OMPOPT)
endif
ifeq ($(DOUBLE_PRECISION),y)
   OPT += -DDOUBLE_PRECISION $(DPOPT)
endif
ifeq ($(DEBUG),y)
   OPT += $(DEBUGOPT)
else
   OPT += $(OPTIMOPT)
endif

ifeq ($(GPU),y)
  ifeq ($(GPU_UM),y)
    OPT += -DGPU_UM -Mcuda=cuda7.5 -Minfo=accel -acc -ta=nvidia:cc35,ptxinfo,maxregcount:64,managed $(OPT)
   else
    OPT += -Minfo=accel -acc -ta=nvidia:cc70,ptxinfo,maxregcount:160 $(OPT)
  endif
  #OPT := -Mcuda=cuda7.0 -Minfo=all -acc -ta=nvidia:cc35,ptxinfo,maxregcount:64,time,host $(OPT)
  NVCC := nvcc
  CUDA_OPT := -DCUDA_MPI -arch=sm_70 -O3\
	-I$(dir $(shell which mpif90))../include
  CUDA_OBJ := shift_cuda.o
  OPT += -DCUDA_MPI
  CUDA_OPT += -DCUDA_MPI
  ifeq ($(DOUBLE_PRECISION),y)
    CUDA_OPT += -DDOUBLE_PRECISION
  endif

  LIB += -lstdc++ -L$(dir $(shell which nvcc))../lib64 -lcudart -lcufft -lnvToolsExt
endif


##################################################################
# We add ".F90" to the list of suffixes to allow source files on which the
# co-processor will be run automatically.
.SUFFIXES: .o .F90 .F
.PHONY: clean

ifeq ($(ICONFIG),FRC)
# List of all the object files needed to build the FRC code
OBJ:=module.o initial.o function.o main.o\
     taehdf5.o equilibrium.o $(EQ_interface)\
     trace_fieldlines.o simulation_domain.o mapping_splines.o\
     grids_setup.o grids_eqdata.o grids_fields.o\
     cell_volume.o laplacian.o\
     $(PETSC_solver) fft_gl.o\
     shift.o $(CUDA_OBJ) tracking.o

OBJ +=loade.o pushe.o antenna.o local_pssolver.o fluid.o\
      loadi.o pushi.o charge.o data_output.o gkload.o gkpush.o\
      loadfki.o pushfki.o\
      setup.o restart.o #global_pssolver.o
ifeq ($(PETSc),y)
OBJ += global_pssolver.o
endif

else ifeq ($(ICONFIG),TOKAMAK)
# List of all object files needed to build the EFIT tokamak code
OBJ:= module.o $(PETSC_solver) function.o initial.o $(EQ_interface) equilibrium.o main_efit.o

endif

$(CMD): $(OBJ)
	$(CMP) $(OPT) -o $(CMD) $(OBJ) $(LIB)

#========= Unit_test_equilibrium_interface
ifeq ($(EQ_test),y)
    ifeq ($(ICONFIG),FRC)

        OBJ_EQ:= module.o taehdf5.o LAMYRIDGE_interface.o

    else ifeq ($(ICONFIG),TOKAMAK)

        OBJ_EQ:= module.o EFIT_interface.o

    endif
endif
gtceq: $(OBJ_EQ)
	echo $(CMP) $(OPT)
	$(CMP) $(OPT) -o gtceq $(OBJ_EQ) $(LIB)
#==============================file dependence================================
$(filter-out module.o,$(OBJ)): module.o  # all .o files except for module.o dependent on module.o

module.o : module.F90
	$(CMP) $(OPT) $(INCLUDE) -c module.F90

ifeq ($(PETSc),y)
main.o : main.F90 module.o initial.o setup.o loade.o pushe.o antenna.o\
         local_pssolver.o fluid.o loadi.F90 pushi.o charge.o data_output.o \
         tracking.o restart.o pushfki.o loadfki.o global_pssolver.o
	$(CMP) $(OPT) $(INCLUDE) -c main.F90
else
main.o : main.F90 module.o initial.o setup.o loade.o pushe.o antenna.o\
         local_pssolver.o fluid.o loadi.F90 pushi.o charge.o data_output.o \
         tracking.o restart.o pushfki.o loadfki.o
	$(CMP) $(OPT) $(INCLUDE) -c main.F90
endif

main_efit.o : main_efit.F90 module.o initial.o equilibrium.o
	$(CMP) $(OPT) $(INCLUDE) -c main_efit.F90

function.o: function.F90 module.o
	$(CMP) $(OPT) $(INCLUDE) -c function.F90

equilibrium.o: equilibrium.F90 module.o function.o $(EQ_interface)
	$(CMP) $(OPT) $(INCLUDE) -c equilibrium.F90

LAMYRIDGE_interface.o: LAMYRIDGE_interface.F90 module.o taehdf5.o 
	$(CMP) $(OPT) $(INCLUDE) -c LAMYRIDGE_interface.F90

EFIT_interface.o: EFIT_interface.F90 module.o
	$(CMP) $(OPT) $(INCLUDE) -c EFIT_interface.F90

trace_fieldlines.o: trace_fieldlines.F90 module.o function.o equilibrium.o 
	$(CMP) $(OPT) $(INCLUDE) -c trace_fieldlines.F90

ifeq ($(PETSc),y)
initial.o: initial.F90 module.o petsc.o
	$(CMP) $(OPT) $(INCLUDE) -c initial.F90
else
initial.o: initial.F90 module.o
	$(CMP) $(OPT) $(INCLUDE) -c initial.F90
endif

ifeq ($(PETSc),y)
petsc.o: petsc.F90 module.o
	$(CMP) $(OPT) $(INCLUDE) -c petsc.F90
endif

simulation_domain.o: simulation_domain.F90 module.o function.o equilibrium.o trace_fieldlines.o
	$(CMP) $(OPT) $(INCLUDE) -c simulation_domain.F90

mapping_splines.o: mapping_splines.F90 module.o function.o equilibrium.o trace_fieldlines.o simulation_domain.o 
	$(CMP) $(OPT) $(INCLUDE) -c mapping_splines.F90

grids_setup.o: grids_setup.F90 module.o function.o equilibrium.o mapping_splines.o trace_fieldlines.o simulation_domain.o
	$(CMP) $(OPT) $(INCLUDE) -c grids_setup.F90

grids_eqdata.o: grids_eqdata.F90 module.o function.o equilibrium.o mapping_splines.o cell_volume.o grids_fields.o grids_setup.o
	$(CMP) $(OPT) $(INCLUDE) -c grids_eqdata.F90

grids_fields.o: grids_fields.F90 module.o function.o grids_setup.o simulation_domain.o
	$(CMP) $(OPT) $(INCLUDE) -c grids_fields.F90

ifeq ($(PETSc),y)
laplacian.o: laplacian.F90 module.o grids_setup.o simulation_domain.o grids_eqdata.o petsc.o
	$(CMP) $(OPT) $(INCLUDE) -c laplacian.F90
else
laplacian.o: laplacian.F90 module.o grids_setup.o simulation_domain.o grids_eqdata.o
	$(CMP) $(OPT) $(INCLUDE) -c laplacian.F90
endif

cell_volume.o: cell_volume.F90 module.o function.o equilibrium.o mapping_splines.o grids_setup.o simulation_domain.o
	$(CMP) $(OPT) $(INCLUDE) -c cell_volume.F90

restart.o: restart.F90 module.o grids_fields.o data_output.o
	$(CMP) $(OPT) $(INCLUDE) -c restart.F90


#=====================================================================

local_pssolver.o: local_pssolver.F90 module.o grids_setup.o simulation_domain.o grids_eqdata.o grids_fields.o
	$(CMP) $(OPT) $(INCLUDE) -c local_pssolver.F90

antenna.o: antenna.F90 module.o grids_setup.o simulation_domain.o grids_eqdata.o grids_fields.o laplacian.o
	$(CMP) $(OPT) $(INCLUDE) -c antenna.F90

fluid.o: fluid.F90 module.o grids_setup.o simulation_domain.o grids_eqdata.o grids_fields.o
	$(CMP) $(OPT) $(INCLUDE) -c fluid.F90
loade.o: loade.F90 module.o function.o equilibrium.o mapping_splines.o grids_setup.o cell_volume.o simulation_domain.o grids_eqdata.o grids_fields.o
	$(CMP) $(OPT) $(INCLUDE) -c loade.F90
pushe.o: pushe.F90 module.o function.o equilibrium.o mapping_splines.o grids_setup.o simulation_domain.o grids_eqdata.o grids_fields.o
	$(CMP) $(OPT) $(INCLUDE) -c pushe.F90
loadi.o: loadi.F90 module.o function.o equilibrium.o mapping_splines.o grids_setup.o cell_volume.o simulation_domain.o grids_eqdata.o grids_fields.o restart.o
	$(CMP) $(OPT) $(INCLUDE) -c loadi.F90 
pushi.o: pushi.F90 module.o function.o equilibrium.o mapping_splines.o grids_setup.o simulation_domain.o grids_eqdata.o grids_fields.o
	$(CMP) $(OPT) $(INCLUDE) -c pushi.F90
gkpush.o: grids_setup.o module.o gkpush.F90 grids_fields.o grids_eqdata.o
	$(CMP) $(OPT) $(INCLUDE) -c gkpush.F90
gkload.o: grids_setup.o equilibrium.o module.o gkload.F90 grids_fields.o
	$(CMP) $(OPT) $(INCLUDE) -c gkload.F90
pushifk.o: pushifk.F90 module.o function.o equilibrium.o mapping_splines.o grids_setup.o simulation_domain.o grids_eqdata.o grids_fields.o
	$(CMP) $(OPT) $(INCLUDE) -c pushifk.F90
charge.o: charge.F90 module.o function.o equilibrium.o mapping_splines.o grids_setup.o simulation_domain.o grids_eqdata.o grids_fields.o
	$(CMP) $(OPT) $(INCLUDE) -c charge.F90
data_output.o: data_output.F90 module.o function.o equilibrium.o mapping_splines.o grids_setup.o cell_volume.o simulation_domain.o grids_eqdata.o grids_fields.o fluid.o antenna.o
	$(CMP) $(OPT) $(INCLUDE) -c data_output.F90

ifeq ($(PETSc),y)
global_pssolver.o: global_pssolver.F90 module.o function.o equilibrium.o mapping_splines.o grids_setup.o simulation_domain.o grids_eqdata.o grids_fields.o $(PETSC_solver) laplacian.o
	$(CMP) $(OPT) $(INCLUDE) -c global_pssolver.F90
endif

tracking.o: tracking.F90 module.o 
	$(CMP) $(OPT) $(INCLUDE) -c tracking.F90

.F90.o : module.o
	$(CMP) $(OPT) $(INCLUDE) -c $<

shift_cuda.o : shift_cuda.cu
	$(NVCC) $(CUDA_OPT) -c shift_cuda.cu

setup.o: setup.F90 module.o function.o mapping_splines.o equilibrium.o laplacian.o cell_volume.o grids_setup.o simulation_domain.o grids_eqdata.o grids_fields.o fluid.o
	$(CMP) $(OPT) $(INCLUDE) -c setup.F90
pushfki.o: pushfki.F90 module.o function.o equilibrium.o mapping_splines.o grids_setup.o simulation_domain.o grids_eqdata.o grids_fields.o
loadfki.o: loadfki.F90 module.o function.o equilibrium.o mapping_splines.o grids_setup.o cell_volume.o simulation_domain.o grids_eqdata.o grids_fields.o restart.o
# The following tag is meant to "clean" the directory by removing the
# executable along with all the object files created by the compilation 
# One only has to run:  gmake clean


clean::
	rm -f $(CMD) gtceq $(OBJ_EQ) $(OBJ) *.mod *.o *genmod.f90 *.optrpt