# This file is part of GTC version 4.4 
# GTC version 4.4 is released under the 3-Clause BSD license:

# Copyright (c) 2002,2010,2016, GTC Team (team leader: Zhihong Lin, zhihongl@uci.edu)
# All rights reserved.

# Redistribution and use in source and binary forms, with or without 
# modification, are permitted provided that the following conditions are met:

# 1. Redistributions of source code must retain the above copyright notice, 
#    this list of conditions and the following disclaimer.

# 2. Redistributions in binary form must reproduce the above copyright notice, 
#    this list of conditions and the following disclaimer in the documentation 
#    and/or other materials provided with the distribution.

# 3. Neither the name of the GTC Team nor the names of its contributors may be 
#    used to endorse or promote products derived from this software without 
#    specific prior written permission.
# ==============================================================================

#key parameters:
OPENMP=y
DOUBLE_PRECISION=n
ADIOS2=n
DEBUG=n
# set SDP_OUT to y to enable synthetic diagnostic platform outputs
SDP_OUT=n
COMPILER=default
ICONFIG=TOKAMAK
#ICONFIG=FRC
#ICONFIG=CYLINDER
#ICONFIG=TOROIDAL3D

# GPU and field solver flags
GPU=y
GPU_UM=n
PETSc=n
AMGX=y
HYPRE=n
HYPRE_USES_GPU=n
TIME_MPI=y
NVTX=n
#CUDA_MPI=n    #on summit, cuda mpi library is enforced loaded when GPU=y, this parameter is no longer used
ifeq ($(OLCF_AMGX_ROOT),)
  AMGX_ROOT = /ccs/proj/csc190/gtc/amgx/summit
else
  AMGX_ROOT = $(OLCF_AMGX_ROOT)
endif
ifeq ($(OLCF_HYPRE_ROOT),)
  ifeq ($(HYPRE_USES_GPU),y)
    HYPRE_ROOT = /ccs/proj/csc190/gtc/hypre_gpu/summit
  else
    HYPRE_ROOT = /ccs/proj/csc190/gtc/hypre/summit
  endif
else
  #ifeq ($(HYPRE_USES_GPU),y)
  #  $(error To use GPU-enabled Hypre, please unload the hypre module and use the custom hypre build)
  #endif
  HYPRE_ROOT = $(OLCF_HYPRE_ROOT)
endif
############################################################################
#             Makefile to build the GTC code
#           ==================================
#
# You only need to type "gmake" to build the code on the platforms
# defined below. The makefile runs the "uname -s" command to detect
# the operating system automatically. Other options are:
#
#  % gmake OPENMP=y       Builds the code with OpenMP support
#  % gmake OPENMP=n       Builds the code WITHOUT OpenMP support
#                         The default is with OpenMP support	
#
#  % gmake DOUBLE_PRECISION=y  Builds with 8-byte floating point precision
#                              The default is single precision
#
#  % gmake PETSc=y        Builds the code using the PETSc parallel matrix
#                         solving library.  This is necessary to run the
#                         electromagnetic version of GTC.
#
#  % gmake DEBUG=y        Compiles the files with debugging flags.
#                         The default is no debug option.
#
#  If COMPILER is set to 'default' the Makefile will select the appropriate
#  compiler for the target.  If the user wished to override the target
#  machine default, COMPILER may be set to 'portland', 'intel', or 'gnu'.
#
# Special targets:
#
#  % gmake clean      Removes the executable and all object files (*.o)
#
#  % gmake cleanomp   Removes the executable and the object files
#                     containing OpenMP directives
#
#  % gmake doc        Rebuilds the documentation.
#
#############################################################################
# Default executable name
CMD := gtc
ALL:${CMD}
LIB :=
LD_LIB :=

##### inquire the hostname to judge the system name, to determine what target we're compiling on
ifneq (,${HOST})
  SYSTEMS := ${HOST}
else
  ifneq (,$(findstring ppc64le,$(shell uname -m)))
    SYSTEMS := ppc64le
  else
    SYSTEMS := $(shell hostname)
  endif
endif

##### target machine determines the default compiler, and paths to libraries #####
# edison
ifneq (,$(findstring edison,$(SYSTEMS)))
   ifeq ($(COMPILER),default)
      COMPILER := intel
   endif
   CMP := ftn
#   PETSC_OPT = -D_PETSc35BEFORE ##only needed for kink/tearing
   PETSC_LIB = -I${PETSC_DIR}/include -I${PETSC_DIR}/include/finclude
   NETCDF_LIB = -I${NETCDF_DIR}/include
   NETCDF_LD_LIB = -L${NETCDF_DIR}/lib -lnetcdf
   LIBD = /global/u2/x/xiao/AdioFranklin/lib
   INCD = /global/u2/x/xiao/AdioFranklin/include
   ADIOS_LIB = -I$(INCD)
   ADIOS_LD_LIB = -L$(LIBD) -ladios -lmxml
endif

# cori
ifneq (,$(findstring cori,$(SYSTEMS)))
   ifeq ($(COMPILER),default)
      COMPILER := intel
   endif
   CMP := ftn
#   PETSC_OPT = -D_PETSc35BEFORE ##only needed for kink/tearing
   PETSC_LIB = -I${PETSC_DIR}/include -I${PETSC_DIR}/include/finclude
   NETCDF_LIB = -I${NETCDF_DIR}/include
   NETCDF_LD_LIB = -L${NETCDF_DIR}/lib -lnetcdf
   LIBD = /global/u2/x/xiao/AdioFranklin/lib
   INCD = /global/u2/x/xiao/AdioFranklin/include
   ADIOS2_DIR=/project/projectdirs/m808/kmehta/adios2-12.16.18/installation/intel/18.0.1
   ADIOS2_LIBS= -L${ADIOS2_DIR}/lib -ladios2_f -ladios2 -ladios2_sst -ladios2_evpath -ladios2_ffs -ladios2_dill -ladios2_enet  -ladios2_atl -lstdc++
endif

# titan
ifneq (,$(findstring titan,$(SYSTEMS)))
   ifeq ($(COMPILER),default)
      ifeq ($(GPU),y)
	COMPILER := portland
      else
	COMPILER := intel
      endif
   endif
   CMP :=ftn
#   PETSC_OPT = -D_PETSc35BEFORE##only needed for kink/tearing
   PETSC_LIB = -I${PETSC_DIR}/include/petsc -I${PETSC_DIR}/include/petsc/finclude
   NETCDF_LIB = -I${NETCDF_DIR}/include
   NETCDF_LD_LIB = -L${NETCDF_DIR}/lib
   LIBD = /ccs/home/hardes/gtc_adios/jaguarlib
   INCD = /ccs/home/hardes/gtc_adios/jaguarinclude
   CUDA_OPT := -I${MPICH_DIR}/include/
   ADIOS2_DIR=/lustre/atlas/world-shared/csc143/kmehta/adios2-installation/adios2-12.16.18/installation/pgi/18.4.0
   ADIOS2_LIBS= -L${ADIOS2_DIR}/lib -ladios2_f -ladios2 -ladios2_sst -ladios2_evpath -ladios2_ffs -ladios2_dill -ladios2_enet  -ladios2_atl -lbz2
endif

# summit
#ifneq (,$(findstring summit,$(SYSTEMS))) # can not work now
#ifneq (,$(findstring ppc64le,$(shell uname -m)))
ifneq (,$(findstring ppc64le,$(SYSTEMS)))
   CMP := mpif90
   CCMP := mpicc
   ifeq ($(COMPILER),default)
      ifeq ($(GPU),y)
         COMPILER := portland
      endif
   endif
   COMPILER := portland
   ifeq ($(PETSc),y)
     PETSC_LIB = -I${OLCF_PETSC_ROOT}/include
     PETSC_LD_LIB = -L${OLCF_PETSC_ROOT}/lib -R${OLCF_PETSC_ROOT}/lib -lpetsc
   endif
   NETCDF_LIB = -I${OLCF_NETCDF_FORTRAN_ROOT}/include
   NETCDF_LD_LIB = -L${OLCF_NETCDF_FORTRAN_ROOT}/lib -R${OLCF_NETCDF_FORTRAN_ROOT}/lib -lnetcdff -L${OLCF_CUDA_ROOT}/lib64/ -lnvToolsExt
   HDF5_LIB = -I${OLCF_HDF5_ROOT}/include
   HDF5_LD_LIB = -L${OLCF_HDF5_ROOT}/lib -R${OLCF_HDF5_ROOT}/lib -lhdf5_fortran 
   MPI_LIB := -I${OLCF_OMPI_ROOT}/include 
   CUDA_OPT := -I$(dir $(shell which mpif90))../include
   ifeq ($(AMGX),y)
     AMGX_LIB = -I${AMGX_ROOT}/../AMGX/base/include
     AMGX_LD_LIB = -L${AMGX_ROOT}/lib -R${AMGX_ROOT}/lib -lamgxsh -L$(CUDA_DIR)/targets/ppc64le-linux/lib -R$(CUDA_DIR)/targets/ppc64le-linux/lib -lcusparse
   endif
   ifeq ($(HYPRE),y)
     HYPRE_LIB = -I${HYPRE_ROOT}/hypre/src/hypre//include -I${HYPRE_ROOT}/include
     HYPRE_LD_LIB = -L${HYPRE_ROOT}/hypre/src/hypre/lib -R${HYPRE_ROOT}/hypre/src/hypre/lib -L${HYPRE_ROOT}/lib -R${HYPRE_ROOT}/lib -lHYPRE
     ifeq ($(HYPRE_USES_GPU),y)
       HYPRE_LD_LIB += -L${OLCF_CUDA_ROOT}/targets/ppc64le-linux/lib -R${OLCF_CUDA_ROOT}/targets/ppc64le-linux/lib -lnvToolsExt
     endif
   endif
   ADIOS2_DIR=/ccs/proj/phy130/wenhao/adios2-2.6.0/installation
   #ADIOS2_DIR=/gpfs/alpine/world-shared/csc143/kmehta/ADIOS2-2.5.0/installation/pgi/19.9/
   #ADIOS2_DIR=${OLCF_ADIOS2_ROOT}
   ADIOS2_LIBS=$(shell ${ADIOS2_DIR}/bin/adios2-config --fortran-libs)
endif

# power
ifneq (,$(findstring power,$(SYSTEMS)))
   COMPILER := portland
   CMP :=mpif90
#   PETSC_OPT = -D_PETSc35BEFORE##only needed for kink/tearing
   PETSC_LIB = -I${PETSC_DIR}/include
   PETSC_LD_LIB = -L${PETSC_DIR}/${PETSC_ARCH}/lib -lpetsc
   CUDA_OPT := -I$(dir $(shell which mpif90))../include
#   NETCDF_LIB = -I${NETCDF_DIR}/include
#   NETCDF_LD_LIB = -L${NETCDF_DIR}/lib
#   LIBD = /ccs/home/hardes/gtc_adios/jaguarlib
#   INCD = /ccs/home/hardes/gtc_adios/jaguarinclude
#   ADIOS_LIB = -I$(INCD)
#   ADIOS_LD_LIB = -L$(LIBD) -ladios -lmxml
endif

# Tianhe-1A
ifneq (,$(findstring ln,$(SYSTEMS)))
   ifeq ($(COMPILER),default)
      ifeq ($(GPU),y)
	COMPILER := portland
      else
	COMPILER := intel
      endif
   endif
   CMP := mpif90

   PETSC_solver = petsc.o
   PETSC_OPT = -D_PETSc31P8ANDBEFORE
	 #PETSC_OPT += -D_PETSc30ANDBEFORE
   ifneq (,${PETSC_DIR})
      include ${PETSC_DIR}/conf/variables
   endif

   PETSC_LIB := -I${PETSC_DIR}/include -I${PETSC_DIR}/include/finclude
   PETSC_LD_LIB := ${PETSC_KSP_LIB}

   NETCDF_LIB := -I${NETCDF_DIR}/include
   NETCDF_LD_LIB := -L${NETCDF_DIR}/lib -lnetcdf -lnetcdff -lnetcdf_c++\
		 -Wl,--rpath -Wl,${NETCDF_DIR}/lib
endif

# PKU cluster
ifneq (,$(findstring mgt,$(SYSTEMS)))
   PETSC_LIB = -Wl,-rpath,${PETSC_LIB_DIR} -L${PETSC_LIB_DIR} -lpetsc -lX11\
							 -L/share/soft/lapack-3.3.1 -llapack_LINUX -lblas_LINUX
endif

#sun
ifneq (,$(findstring sun,$(SYSTEMS)))
   ifeq ($(COMPILER),default)
      COMPILER := gnu
   endif
   CMP := mpif90

   HDF5_INCLUDE := -I/opt/hdf5_parallel/include
#   PETSC_INCLUDE := -I${PETSC_DIR}/include -I${PETSC_DIR}/${PETSC_ARCH}/include -I${PETSC_DIR}/include/petsc/finclude
   HDF5_LIB := -lhdf5_fortran -L/opt/hdf5_parallel/lib
   PETSC_LIB = -I${PETSC_DIR}/include -I${PETSC_DIR}/include/petsc/finclude -I${PETSC_DIR}/${PETSC_ARCH}/include
   PETSC_LD_LIB = -L${PETSC_DIR}/${PETSC_ARCH}/lib -lpetsc

#   ADIOS2_DIR=/home/kmehta/ADIOS2-2.6.0/installation/
#   ADIOS2_DIR=/home/wenhao/Downloads/adios2-2.6.0/installation
   ADIOS2_DIR=/opt/adios-2.6.0/installation
   ADIOS2_LIBS=$(shell ${ADIOS2_DIR}/bin/adios2-config --fortran-libs)

   NETCDF_DIR = /usr
   NETCDF_LIB = -I${NETCDF_DIR}/include/
   NETCDF_LD_LIB = -L${NETCDF_DIR}/lib/ -lnetcdff
endif

#iop
ifneq (,$(findstring indac,$(SYSTEMS)))
   ifeq ($(COMPILER),default)
      COMPILER := intel
   endif
   CMP := mpif90

   #PETSC_solver = petsc.o
   PETSC_OPT = -D_PETSc31P8ANDBEFORE
         #PETSC_OPT += -D_PETSc30ANDBEFORE
   ifneq (,${PETSC_DIR})
      include ${PETSC_DIR}/conf/variables
   endif

   PETSC_LIB := -I${PETSC_DIR}/include -I${PETSC_DIR}/include/finclude
   PETSC_LD_LIB := ${PETSC_KSP_LIB}

   NETCDF_LIB := -I${NETCDF_DIR}/include
   NETCDF_LD_LIB := -L${NETCDF_DIR}/lib -lnetcdf -lnetcdff -lnetcdf_c++\
                 -Wl,--rpath -Wl,${NETCDF_DIR}/lib
endif

#### compiler determines debugging, openmp, and double precision flags #####
# intel fortran
ifeq ($(COMPILER),intel)
   OPTIMOPT := -O
   #OPTIMOPT := -O3
   #OPTIMOPT += -fp-model fast=2
   #OPTIMOPT += -align array64byte
   #OPTIMOPT += -no-vec
# debug flags changed by L. Shi to suppress long warning outputs.
   #DEBUGOPT := -g -free -pc64 -check all -warn nounused -warn all -debug all -debug-parameters all -fp-stack-check
   DEBUGOPT := -g -free -pc64 -check 'all,noarg_temp_created' -warn none -debug all -debug-parameters all -fp-stack-check
   OMPOPT := -qopenmp
   DPOPT := -fpconstant
endif

# portland group fortran
ifeq ($(COMPILER),portland)
   OPTIMOPT := -fast -Mfree -Kieee
   #DEBUGOPT := -g -C -gopt -Mbounds -Mchkfpstk -Mchkptr -Mchkstk -Mcoff -Mdwarf3 -Melf -Mpgicoff -traceback -Minform=inform
   # -Mchkptr is currently not working well with GPU compiling, so temporarily opted out
   DEBUGOPT := -g -C -gopt  -Mchkfpstk  -Mchkstk -Mcoff -Mdwarf3 -Melf -Mpgicoff -Minform=inform -Minfo -Msummary="CodeSummary.json" -Mbounds #-Mchkptr
   OMPOPT := -mp
   DPOPT := -DDOUBLE_PRECISION
endif

### assemble all the options
OPT :=
LIB :=

# GNU gfortran
ifeq ($(COMPILER),gnu)
   OPT += -ffree-line-length-0 -Wuninitialized
   OPTIMOPT := -O
   DEBUGOPT := -g -fcheck=all,no-array-temps,no-bounds
#   DEBUGOPT := -g -fcheck=all,no-array-temps
   OMPOPT := -fopenmp
   DPOPT :=
endif

ifeq ($(ICONFIG),FRC)
  ICONFIG_OPT = -D_FRC
  OPT += $(ICONFIG_OPT)
else ifeq ($(ICONFIG),TOKAMAK)
  ICONFIG_OPT = -D_TOKAMAK
  OPT += $(ICONFIG_OPT)
else ifeq ($(ICONFIG),CYLINDER)
  ICONFIG_OPT = -D_CYLINDER
  OPT += $(ICONFIG_OPT)
else ifeq ($(ICONFIG),TOROIDAL3D)
  ICONFIG_OPT = -D_TOROIDAL3D
  OPT += $(ICONFIG_OPT)
endif

ifeq ($(SDP_OUT),y)
  OPT += -D_SDP
endif

# NetCDF library is always required
  LIB += $(NETCDF_LIB)
  LD_LIB += $(NETCDF_LD_LIB)

ifeq ($(ADIOS2), y)
  OPT += -DADIOS2 -I${ADIOS2_DIR}/include/adios2/fortran
#  OPTIMOPT += -DADIOS2 -I${ADIOS2_DIR}/include/adios2/fortran
  LD_LIB += ${ADIOS2_LIBS}
endif

ifeq ($(PETSc),y)
   PETSC_solver ?= petsc.o
   PETSC_OPT += -D_PETSc -D_USING_EXT_SOLVER
   OPT += $(PETSC_OPT)
   LIB += $(PETSC_LIB)
   LD_LIB += $(PETSC_LD_LIB)
   ifeq ($(AMGX),y)
     $(error Currently cannot enable multiple solvers in a single GTC build)
   endif
   ifeq ($(HYPRE),y)
     $(error Currently cannot enable multiple solvers in a single GTC build)
   endif
else
   PETSC_solver :=
endif

ifeq ($(AMGX),y)
   AMGX_solver ?= amgx.o
   AMGX_OPT += -D_AMGX -D_USING_EXT_SOLVER
   ifeq ($(DOUBLE_PRECISION),y)
     AMG_OPT += -DDOUBLE_PRECISION
   endif
   OPT += $(AMGX_OPT)
   LIB += $(AMGX_LIB)
   LD_LIB += $(AMGX_LD_LIB)
   ifeq ($(PETSc),y)
     $(error Currently cannot enable multiple solvers in a single GTC build)
   endif
   ifeq ($(HYPRE),y)
     $(error Currently cannot enable multiple solvers in a single GTC build)
   endif
else
   AMGX_solver :=
endif

ifeq ($(HYPRE),y)
   HYPRE_solver ?= hypre.o
   HYPRE_OPT += -D_HYPRE -D_USING_EXT_SOLVER
   ifeq ($(DOUBLE_PRECISION),y)
     HYPRE_OPT += -DDOUBLE_PRECISION
   endif
   ifeq ($(HYPRE_USES_GPU),y)
     HYPRE_OPT += -D_HYPRE_USES_GPU
   endif
   OPT += $(HYPRE_OPT)
   LIB += $(HYPRE_LIB)
   LD_LIB += $(HYPRE_LD_LIB)
   ifeq ($(PETSc),y)
     $(error Currently cannot enable multiple solvers in a single GTC build)
   endif
   ifeq ($(AMGX),y)
     $(error Currently cannot enable multiple solvers in a single GTC build)
   endif
else
   HYPRE_solver :=
endif

ifeq ($(OPENMP),y)
   OPT += $(OMPOPT)
endif

ifeq ($(DOUBLE_PRECISION),y)
   OPT += -DDOUBLE_PRECISION $(DPOPT)
endif

ifeq ($(DEBUG),y)
   OPT += $(DEBUGOPT)
else
   OPT += $(OPTIMOPT)
endif

ifeq ($(GPU),y)
  OPT:= -Minfo=accel -acc $(OPT)
  ifneq (,$(findstring titan,$(SYSTEMS)))
    ifeq ($(GPU_UM),y)
      OPT := -DGPU_UM -ta=nvidia:cc35,maxregcount:64,managed $(OPT)
    else
      OPT := -ta=nvidia:cc35,maxregcount:64 $(OPT)
    endif
    CUDA_OPT += -O3 -arch=sm_35
  endif
  #ifneq (,$(findstring summit,$(SYSTEMS)))
  #ifneq (,$(findstring ppc64le,$(shell uname -m)))
  ifneq (,$(findstring ppc64le,$(SYSTEMS)))
    ifeq ($(GPU_UM),y)
      OPT := CUDAROOT=$(OLCF_CUDA_ROOT) -DGPU_UM -Mcuda=9.0 -ta=nvidia:cc70,maxregcount:160,managed $(OPT)
      CUDA_OPT += -O3 -arch=sm_70
    else
      OPT := CUDAROOT=$(OLCF_CUDA_ROOT) -ta=nvidia:cc70,maxregcount:160 $(OPT)
      CUDA_OPT += -O3 -arch=sm_70
    endif
    OPT += -DCUDA_MPI
    ifeq ($(DOUBLE_PRECISION),y)
       CUDA_OPT += -DDOUBLE_PRECISION
    endif
    CUDA_OPT += -DCUDA_MPI
  endif

  NVCC := nvcc
  CUDA_OBJ := shift_cuda.o cufft.o
  LD_LIB += -lstdc++ -L$(dir $(shell which nvcc))../lib64 -lcudart -lcufft -lnvToolsExt
endif

ifeq ($(TIME_MPI),y)
  OPT := -DTIME_MPI $(OPT)
  CUDA_OPT += -DTIME_MPI
endif

ifeq ($(NVTX),y)
  CUDA_OPT += -DNVTX
  OPT := -DNVTX $(OPT)
endif
# On summit, cuda mpi is enforced used. On other machines with cuda mpi library, users should use following lines.
#ifeq ($(CUDA_MPI),y)
#  OPT += -DCUDA_MPI
#  CUDA_OPT += -DCUDA_MPI
#endif

# Equilibrium dimension related OBJs
ifeq ($(ICONFIG), TOROIDAL3D)
  EQ_OBJ := VMEC_interface.o
else
  EQ_OBJ := M3DC1_interface.o
endif

##################################################################
# We add ".F90" to the list of suffixes to allow source files on which the
# co-processor will be run automatically.
.SUFFIXES: .o .F90 .F
.PHONY: clean

# List of all the object files needed to build the code
OBJ:=simple_error_handler.o module.o function.o boundary_decay.o main.o setup.o restart.o diagnosis.o\
	snapshot.o poisson.o smooth.o field.o pushifk.o pushifk_boris.o bparasolver.o\
	tracking.o dataout3d.o pushfield.o eqdata.o eqplot.o\
	collision.o $(PETSC_solver) $(AMGX_solver) $(HYPRE_solver) fft_gl.o \
	load.o push.o shift.o charge.o radial_bin.o $(CUDA_OBJ) antenna.o utility.o ohm_law.o physical_meaning.o \
        $(EQ_OBJ)

ifeq ($(ADIOS2), y)
	OBJ += adios2_common.o adios2_history_diagnostics.o adios2_data1d_diagnostics.o
endif

HYBRID_OBJ:=hybridChargeParticle.o hybridPushParticle.o
KINETIC_OBJ:=gkChargeParticle.o gkPushParticle.o

OBJ += $(HYBRID_OBJ) $(KINETIC_OBJ)

ifeq ($(SDP_OUT),y)
  OBJ += SDP_interface.o
endif

# set dependencies on modules based on switches
DEPEND_MOD := 
ifeq ($(SDP_OUT),y)
  DEPEND_MOD += SDP_interface.o
endif

# set commonly depended modules
COMMON_MOD := module.o utility.o simple_error_handler.o
ifeq ($(ADIOS2), y)
	COMMON_MOD += adios2_common.o adios2_history_diagnostics.o adios2_data1d_diagnostics.o
endif

# Start building the executive
$(CMD): $(OBJ)
	$(CMP) $(OPT) -o $(CMD) $(OBJ) $(LD_LIB)

$(filter-out module.o,$(OBJ)): module.o

simple_error_handler.o : simple_error_handler.F90
	$(CMP) $(OPT) $(LIB) -c simple_error_handler.F90

module.o : module.F90
	$(CMP) $(OPT) $(LIB) -c module.F90

utility.o : utility.F90 module.o
	$(CMP) $(OPT) $(LIB) -c utility.F90

adios2_common.o : module.o simple_error_handler.o adios2_common.F90 
	$(CMP) $(OPT) $(LIB) -c adios2_common.F90

adios2_history_diagnostics.o : adios2_common.o module.o adios2_history_diagnostics.F90
	$(CMP) $(OPT) $(LIB) -c adios2_history_diagnostics.F90

adios2_data1d_diagnostics.o : adios2_common.o module.o adios2_data1d_diagnostics.F90
	$(CMP) $(OPT) $(LIB) -c adios2_data1d_diagnostics.F90

main.o : main.F90 $(COMMON_MOD) diagnosis.o antenna.o
	$(CMP) $(OPT) $(LIB) -c main.F90

function.o: function.F90 $(COMMON_MOD)
	$(CMP) $(OPT) $(LIB) -c function.F90

boundary_decay.o : boundary_decay.F90 $(COMMON_MOD) function.o
	$(CMP) $(OPT) $(LIB) -c boundary_decay.F90

antenna.o: antenna.F90 $(COMMON_MOD)
	$(CMP) $(OPT) $(LIB) -c antenna.F90

M3DC1_interface.o: M3DC1_interface.F90 $(COMMON_MOD) function.o boundary_decay.o
	$(CMP) $(OPT) $(LIB) -c M3DC1_interface.F90

VMEC_interface.o: VMEC_interface.F90 $(COMMON_MOD) function.o 
	$(CMP) $(OPT) $(LIB) -c VMEC_interface.F90

eqdata.o: eqdata.F90 $(EQ_OBJ) $(COMMON_MOD) function.o boundary_decay.o $(DEPEND_MOD)
	$(CMP) $(OPT) $(LIB) -c eqdata.F90

eqplot.o: eqplot.F90 $(COMMON_MOD) function.o $(DEPEND_MOD)
	$(CMP) $(OPT) $(LIB) -c eqplot.F90

snapshot.o: snapshot.F90 $(COMMON_MOD) function.o $(DEPEND_MOD) physical_meaning.o
	$(CMP) $(OPT) $(LIB) -c snapshot.F90

dataout3d.o: dataout3d.F90 $(COMMON_MOD) function.o
	$(CMP) $(OPT) $(LIB) -c dataout3d.F90

SDP_interface.o: SDP_interface.F90 $(COMMON_MOD) function.o 
	$(CMP) $(OPT) $(LIB) -c SDP_interface.F90

poisson.o: poisson.F90 $(COMMON_MOD) function.o antenna.o
	$(CMP) $(OPT) $(LIB) -c poisson.F90

pushfield.o: pushfield.F90 $(COMMON_MOD) function.o antenna.o
	$(CMP) $(OPT) $(LIB) -c pushfield.F90

ohm_law.o: ohm_law.F90 $(COMMON_MOD) function.o 
	$(CMP) $(OPT) $(LIB) -c ohm_law.F90

field.o: field.F90 $(COMMON_MOD) antenna.o
	$(CMP) $(OPT) $(LIB) -c field.F90

setup.o: setup.F90 $(COMMON_MOD) function.o boundary_decay.o antenna.o $(EQ_OBJ) diagnosis.o physical_meaning.o
	$(CMP) $(OPT) $(LIB) -c setup.F90

smooth.o: smooth.F90 $(COMMON_MOD) boundary_decay.o
	$(CMP) $(OPT) $(LIB) -c smooth.F90

load.o: load.F90 $(COMMON_MOD) function.o antenna.o
	$(CMP) $(OPT) $(LIB) -c load.F90

pushifk_boris.o: pushifk_boris.F90 $(COMMON_MOD) function.o
	$(CMP) $(OPT) $(LIB) -c pushifk_boris.F90 


tracking.o: tracking.F90 $(COMMON_MOD) function.o
	$(CMP) $(OPT) $(LIB) -c tracking.F90

#json_module.o: json_module.F90
#	$(CMP) $(OPT) $(LIB) -c json_module.F90

hybrid%.F90 : $(COMMON_MOD) boundary_decay.o function.o PushParticle.F90 ChargeParticle.F90
	$(CMP) $(OPT) $(LIB) -D_hybrid -E $*.F90 > $@

gk%.F90 : $(COMMON_MOD) boundary_decay.o function.o PushParticle.F90 ChargeParticle.F90
	$(CMP) $(OPT) $(LIB) -E $*.F90 > $@

shift_cuda.o : shift_cuda.cu shift_cuda.cuh
	$(NVCC) $(CUDA_OPT) -c shift_cuda.cu

cufft.o : cufft.cu 
	$(NVCC) $(CUDA_OPT) -c cufft.cu

amgx.o: amgx.cu
	$(NVCC) $(CUDA_OPT) -c amgx.cu $(AMGX_LIB) $(AMGX_OPT)

hypre.o: hypre.c
	$(CCMP) -c hypre.c $(HYPRE_LIB) $(HYPRE_OPT)

diagnosis.o: diagnosis.F90 $(COMMON_MOD) function.o antenna.o
	$(CMP) $(OPT) $(LIB) -c diagnosis.F90

shift.o: shift.F90 $(COMMON_MOD)
	$(CMP) $(OPT) $(LIB) -c shift.F90

restart.o: restart.F90 $(COMMON_MOD)
	$(CMP) $(OPT) $(LIB) -c restart.F90

physical_meaning.o : $(COMMON_MOD)
	$(CMP) $(OPT) $(LIB) -c physical_meaning.F90

.F90.o : $(COMMON_MOD)
	$(CMP) $(OPT) $(LIB) -c $<

# The following tag is meant to "clean" the directory by removing the
# executable along with all the object files created by the compilation
# One only has to run:  gmake clean

clean::
	rm -f $(CMD) $(OBJ) *.mod *.o *genmod.f90 *.optrpt *.json pgipar* *.dbg *.smod