-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathMakefile
executable file
·329 lines (278 loc) · 18 KB
/
Makefile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
#OPS_INSTALL_PATH := $(OPS_INSTALL_PATH)/c
OPS_INC = -I$(OPS_INSTALL_PATH)/include
OPS_LIB = -L$(OPS_INSTALL_PATH)/lib
CUDA_LIB = -L$(CUDA_INSTALL_PATH)/lib
CUDA_INC = -I$(CUDA_INSTALL_PATH)/include
OPENCL_INC = -I$(OPENCL_INSTALL_PATH)/include
OPENCL_LIB = -L$(OPENCL_INSTALL_PATH)/lib64 -lOpenCL
ifdef HDF5_INCLUDE_PATH
HDF5_INC := -I$(HDF5_INCLUDE_PATH)
ifdef CHECKPOINTING
HDF5_INC := $(HDF5_INC) -DCHECKPOINTING
endif
endif
ifdef HDF5_LIB_PATH
HDF5_LIB := -L$(HDF5_LIB_PATH) -lhdf5_hl -lhdf5 -lz
endif
ifeq ($(OPS_COMPILER),clang)
CPP = clang++
ifdef DEBUG
CPPFLAGS = -O0 -g -DUNIX -Wall -std=c++11 #-Wall
else
CPPFLAGS = -O3 -fPIC -DUNIX -Wall -std=c++11 #-Wall
endif
OMPFLAGS = ""#-fopenmp
MPICPP = mpic++
MPIFLAGS = $(CPPFLAGS)
else
ifeq ($(OPS_COMPILER),gnu)
CPP = g++
ifdef DEBUG
CPPFLAGS = -ggdb -DUNIX -Wall
else
CPPFLAGS = -O3 -std=c++11 -fPIC -DUNIX
endif
OMPFLAGS = -fopenmp
MPICPP = mpicxx
MPIFLAGS = $(CPPFLAGS)
else
ifeq ($(OPS_COMPILER),intel)
CPP = icpc
ifdef DEBUG
CCFLAGS = -O0 -g -no-prec-div -openmp -fp-model strict -fp-model source -prec-div -prec-sqrt -DMPICH_IGNORE_CXX_SEEK #-DOPS_DEBUG
else
#CCFLAGS = -O3 -ipo -no-prec-div -fp-model strict -fp-model source -prec-div -prec-sqrt -vec-report2 -xSSE4.2 -parallel #-DCOMM_PERF #-DDEBUG
CCFLAGS = -O3 -ipo -no-prec-div -restrict -fno-alias -fp-model strict -fp-model source -prec-div -prec-sqrt -DMPICH_IGNORE_CXX_SEEK#-vec-report
endif
CPPFLAGS = $(CCFLAGS)
OMPFLAGS = -openmp -openmp-report2
MPICPP = $(MPI_INSTALL_PATH)/bin/mpicxx
MPIFLAGS = $(CCFLAGS)
else
ifeq ($(OPS_COMPILER),mic)
CPP = icpc
ifdef DEBUG
CCFLAGS = -O0 -g -mmic -DMPICH_IGNORE_CXX_SEEK#-vec-report
else
#CCFLAGS = -O3 -ipo -no-prec-div -restrict -fno-alias -fp-model strict -fp-model source -prec-div -prec-sqrt -DMPICH_IGNORE_CXX_SEEK#-vec-report
#CCFLAGS = -O3 -ipo -no-prec-div -restrict -fno-alias -mmic -vec-report -finline -inline-forceinline -DSIMD_VEC=32 -DMPICH_IGNORE_CXX_SEEK#-vec-report
CCFLAGS = -O3 -ipo -no-prec-div -restrict -fno-alias -mmic -vec-report -DSIMD_VEC=32 -DMPICH_IGNORE_CXX_SEEK -finline -vec-report
endif
CPPFLAGS = $(CCFLAGS)
OMPFLAGS = -openmp -openmp-report2
MPICPP = $(MPI_INSTALL_PATH)/bin/mpicxx
MPIFLAGS = $(CCFLAGS)
OPENCL_LIB = -L$(OPENCL_INSTALL_PATH)/lib64 -lintelocl -lOpenCL #for offload opencl execution on the phi
else
ifeq ($(OPS_COMPILER),pgi)
CPP = pgc++
CC = pgcc
CUDA_ALIGN_FLAG := -D__x86_64 -D__align__\(n\)=__attribute__\(\(aligned\(n\)\)\) -D__location__\(a\)=__annotate__\(a\) -DCUDARTAPI=
ifdef DEBUG
CCFLAGS = -O2 -Minline -Kieee -Minform=inform -Minfo=all
else
CCFLAGS = -O3 -fast -Kieee -Minline -Minform=severe -Minfo=all -std=c++11
endif
CPPFLAGS = $(CCFLAGS)
OMPFLAGS = -mp
MPICPP = $(MPI_INSTALL_PATH)/bin/mpicxx
MPICC = $(MPI_INSTALL_PATH)/bin/mpicc
MPIFLAGS = $(CPPFLAGS)
else
ifeq ($(OPS_COMPILER),xl)
CPP = xlc++
ifdef DEBUG
CPPFLAGS = -O0 -g -qfullpath -Q -qlistopt -qattr=full -qlist -qreport -qxref=full -qsource -qsuppress=1506-224:1500-036 -qsrcmsg -qfloat=nomaf
else
CPPFLAGS = -O5 -q64 -qnoeh -qipa=partition=large -qfullpath -Q -qlistopt -qattr=full -qlist -qreport -qxref=full -qsource -qsuppress=1506-224:1500-036 -qsrcmsg -qfloat=nomaf
endif
OMPFLAGS = -fopenmp
MPICPP = $(MPI_INSTALL_PATH)/bin/mpicxx
MPIFLAGS = $(CPPFLAGS)
else
ifeq ($(OPS_COMPILER),cray)
CPP = CC
CUDA_ALIGN_FLAG := -D__x86_64 -D__align__\(n\)=__attribute__\(\(aligned\(n\)\)\) -D__location__\(a\)=__annotate__\(a\) -DCUDARTAPI=
ifdef DEBUG
CCFLAGS = -O2
else
CCFLAGS = -O3
endif
CPPFLAGS = $(CCFLAGS)
OMPFLAGS =
MPICPP = CC
MPICC = cc
MPIFLAGS = $(CPPFLAGS)
else
print:
@echo "unrecognised value for OPS_COMPILER"
endif
endif
endif
endif
endif
endif
endif
NVCC := $(CUDA_INSTALL_PATH)/bin/nvcc
# flags for nvcc
# set NV_ARCH to select the correct one
ifndef NV_ARCH
MESSAGE=select an NVIDA device to compile in CUDA, e.g. make NV_ARCH=KEPLER
NV_ARCH=Kepler
endif
ifeq ($(NV_ARCH),Fermi)
CODE_GEN_CUDA=-gencode arch=compute_20,code=sm_21
else
ifeq ($(NV_ARCH),Kepler)
CODE_GEN_CUDA=-gencode arch=compute_35,code=sm_35
endif
endif
ifdef DEBUG
NVCC_FLAGS = -O0 -g -G
else
NVCC_FLAGS = -O3 -std=c++11
endif
#
# master to make all versions
#
TARGETS = lbm2d_dev_seq lbm2d_seq lbm2d_dev_mpi lbm2d_mpi lbm2d_openmp lbm2d_mpi_openmp lbm2d_cuda lbm2d_mpi_cuda lbm2d_opencl lbm2d_mpi_opencl
ifeq ($(OPS_COMPILER),pgi)
#TARGETS += lbm2d_openacc lbm2d_mpi_openacc
else
#TARGETS += openacc_error
endif
all: clean $(TARGETS)
env :
@echo "MPLBDIR = $(MPLB_DIR)"
@echo "HDF5_INCLUDE_PATH = $(HDF5_INCLUDE_PATH)"
@echo "HDF5_LIB_PATH = $(HDF5_LIB_PATH)"
@echo "OPS_INSTALL_PATH = $(OPS_INSTALL_PATH)"
@echo "Compiler = $(OPS_COMPILER)"
mplb3d_seq :
$(CPP) -c $(CPPFLAGS) $(OPS_INC) $(MPLB_INC) $(HDF5_INC) $(OPS_LIB) $(HDF5_LIB) -DOPS_3D -DCPU -D$(LEVEL) -lops_seq -lops_hdf5_seq boundary.cpp
$(CPP) -c $(CPPFLAGS) $(OPS_INC) $(MPLB_INC) $(HDF5_INC) $(OPS_LIB) $(HDF5_LIB) -DOPS_3D -DCPU -D$(LEVEL) -lops_seq -lops_hdf5_seq configuration.cpp
$(CPP) -c $(CPPFLAGS) $(OPS_INC) $(MPLB_INC) $(HDF5_INC) $(OPS_LIB) $(HDF5_LIB) -DOPS_3D -DCPU -D$(LEVEL) -lops_seq -lops_hdf5_seq evolution3d.cpp
$(CPP) -c $(CPPFLAGS) $(OPS_INC) $(MPLB_INC) $(HDF5_INC) $(OPS_LIB) $(HDF5_LIB) -DOPS_3D -DCPU -D$(LEVEL) -lops_seq -lops_hdf5_seq flowfield.cpp
$(CPP) -c $(CPPFLAGS) $(OPS_INC) $(MPLB_INC) $(HDF5_INC) $(OPS_LIB) $(HDF5_LIB) -DOPS_3D -DCPU -D$(LEVEL) -lops_seq -lops_hdf5_seq hilemms_ops.cpp
$(CPP) -c $(CPPFLAGS) $(OPS_INC) $(MPLB_INC) $(HDF5_INC) $(OPS_LIB) $(HDF5_LIB) -DOPS_3D -DCPU -D$(LEVEL) -lops_seq -lops_hdf5_seq model.cpp
$(CPP) -c $(CPPFLAGS) $(OPS_INC) $(MPLB_INC) $(HDF5_INC) $(OPS_LIB) $(HDF5_LIB) -DOPS_3D -DCPU -D$(LEVEL) -lops_seq -lops_hdf5_seq scheme.cpp
$(CPP) -c $(CPPFLAGS) $(OPS_INC) $(MPLB_INC) $(HDF5_INC) $(OPS_LIB) $(HDF5_LIB) -DOPS_3D -DCPU -D$(LEVEL) -lops_seq -lops_hdf5_seq type.cpp
@ar crf libmplb3d_seq.a boundary.o configuration.o evolution3d.o flowfield.o hilemms_ops.o model.o scheme.o type.o
mplb3d_mpi:
$(MPICPP) -c $(MPIFLAGS) $(OPS_INC) $(MPLB_INC) $(HDF5_INC) $(OPS_LIB) $(HDF5_LIB) -DOPS_MPI -DOPS_3D -DCPU -D$(LEVEL) -lops_seq -lops_hdf5_seq boundary.cpp
$(MPICPP) -c $(MPIFLAGS) $(OPS_INC) $(MPLB_INC) $(HDF5_INC) $(OPS_LIB) $(HDF5_LIB) -DOPS_MPI -DOPS_3D -DCPU -D$(LEVEL) -lops_seq -lops_hdf5_seq configuration.cpp
$(MPICPP) -c $(MPIFLAGS) $(OPS_INC) $(MPLB_INC) $(HDF5_INC) $(OPS_LIB) $(HDF5_LIB) -DOPS_MPI -DOPS_3D -DCPU -D$(LEVEL) -lops_seq -lops_hdf5_seq evolution3d.cpp
$(MPICPP) -c $(MPIFLAGS) $(OPS_INC) $(MPLB_INC) $(HDF5_INC) $(OPS_LIB) $(HDF5_LIB) -DOPS_MPI -DOPS_3D -DCPU -D$(LEVEL) -lops_seq -lops_hdf5_seq flowfield.cpp
$(MPICPP) -c $(MPIFLAGS) $(OPS_INC) $(MPLB_INC) $(HDF5_INC) $(OPS_LIB) $(HDF5_LIB) -DOPS_MPI -DOPS_3D -DCPU -D$(LEVEL) -lops_seq -lops_hdf5_seq hilemms_ops.cpp
$(MPICPP) -c $(MPIFLAGS) $(OPS_INC) $(MPLB_INC) $(HDF5_INC) $(OPS_LIB) $(HDF5_LIB) -DOPS_MPI -DOPS_3D -DCPU -D$(LEVEL) -lops_seq -lops_hdf5_seq model.cpp
$(MPICPP) -c $(MPIFLAGS) $(OPS_INC) $(MPLB_INC) $(HDF5_INC) $(OPS_LIB) $(HDF5_LIB) -DOPS_MPI -DOPS_3D -DCPU -D$(LEVEL) -lops_seq -lops_hdf5_seq scheme.cpp
$(MPICPP) -c $(MPIFLAGS) $(OPS_INC) $(MPLB_INC) $(HDF5_INC) $(OPS_LIB) $(HDF5_LIB) -DOPS_MPI -DCPU -D$(LEVEL) -lops_seq -lops_hdf5_seq type.cpp
@ar crf libmplb3d_mpi.a boundary.o configuration.o evolution3d.o flowfield.o hilemms_ops.o model.o scheme.o type.o
mplb2d_seq :
$(CPP) -c $(CPPFLAGS) $(OPS_INC) $(MPLB_INC) $(HDF5_INC) $(OPS_LIB) $(HDF5_LIB) -DOPS_2D -DCPU -D$(LEVEL) -lops_seq -lops_hdf5_seq boundary.cpp
$(CPP) -c $(CPPFLAGS) $(OPS_INC) $(MPLB_INC) $(HDF5_INC) $(OPS_LIB) $(HDF5_LIB) -DOPS_2D -DCPU -D$(LEVEL) -lops_seq -lops_hdf5_seq configuration.cpp
$(CPP) -c $(CPPFLAGS) $(OPS_INC) $(MPLB_INC) $(HDF5_INC) $(OPS_LIB) $(HDF5_LIB) -DOPS_2D -DCPU -D$(LEVEL) -lops_seq -lops_hdf5_seq evolution.cpp
$(CPP) -c $(CPPFLAGS) $(OPS_INC) $(MPLB_INC) $(HDF5_INC) $(OPS_LIB) $(HDF5_LIB) -DOPS_2D -DCPU -D$(LEVEL) -lops_seq -lops_hdf5_seq flowfield.cpp
$(CPP) -c $(CPPFLAGS) $(OPS_INC) $(MPLB_INC) $(HDF5_INC) $(OPS_LIB) $(HDF5_LIB) -DOPS_2D -DCPU -D$(LEVEL) -lops_seq -lops_hdf5_seq hilemms_ops.cpp
$(CPP) -c $(CPPFLAGS) $(OPS_INC) $(MPLB_INC) $(HDF5_INC) $(OPS_LIB) $(HDF5_LIB) -DOPS_2D -DCPU -D$(LEVEL) -lops_seq -lops_hdf5_seq model.cpp
$(CPP) -c $(CPPFLAGS) $(OPS_INC) $(MPLB_INC) $(HDF5_INC) $(OPS_LIB) $(HDF5_LIB) -DOPS_2D -DCPU -D$(LEVEL) -lops_seq -lops_hdf5_seq scheme.cpp
$(CPP) -c $(CPPFLAGS) $(OPS_INC) $(MPLB_INC) $(HDF5_INC) $(OPS_LIB) $(HDF5_LIB) -DOPS_2D -DCPU -D$(LEVEL) -lops_seq -lops_hdf5_seq type.cpp
@ar crf libmplb2d_seq.a boundary.o configuration.o evolution.o flowfield.o hilemms_ops.o model.o scheme.o type.o
mplb2d_mpi:
$(MPICPP) -c $(MPIFLAGS) $(OPS_INC) $(MPLB_INC) $(HDF5_INC) $(OPS_LIB) $(HDF5_LIB) -DOPS_MPI -DOPS_2D -DCPU -D$(LEVEL) -lops_seq -lops_hdf5_seq boundary.cpp
$(MPICPP) -c $(MPIFLAGS) $(OPS_INC) $(MPLB_INC) $(HDF5_INC) $(OPS_LIB) $(HDF5_LIB) -DOPS_MPI -DOPS_2D -DCPU -D$(LEVEL) -lops_seq -lops_hdf5_seq configuration.cpp
$(MPICPP) -c $(MPIFLAGS) $(OPS_INC) $(MPLB_INC) $(HDF5_INC) $(OPS_LIB) $(HDF5_LIB) -DOPS_MPI -DOPS_2D -DCPU -D$(LEVEL) -lops_seq -lops_hdf5_seq evolution.cpp
$(MPICPP) -c $(MPIFLAGS) $(OPS_INC) $(MPLB_INC) $(HDF5_INC) $(OPS_LIB) $(HDF5_LIB) -DOPS_MPI -DOPS_2D -DCPU -D$(LEVEL) -lops_seq -lops_hdf5_seq flowfield.cpp
$(MPICPP) -c $(MPIFLAGS) $(OPS_INC) $(MPLB_INC) $(HDF5_INC) $(OPS_LIB) $(HDF5_LIB) -DOPS_MPI -DOPS_2D -DCPU -D$(LEVEL) -lops_seq -lops_hdf5_seq hilemms_ops.cpp
$(MPICPP) -c $(MPIFLAGS) $(OPS_INC) $(MPLB_INC) $(HDF5_INC) $(OPS_LIB) $(HDF5_LIB) -DOPS_MPI -DOPS_2D -DCPU -D$(LEVEL) -lops_seq -lops_hdf5_seq model.cpp
$(MPICPP) -c $(MPIFLAGS) $(OPS_INC) $(MPLB_INC) $(HDF5_INC) $(OPS_LIB) $(HDF5_LIB) -DOPS_MPI -DOPS_2D -DCPU -D$(LEVEL) -lops_seq -lops_hdf5_seq scheme.cpp
$(MPICPP) -c $(MPIFLAGS) $(OPS_INC) $(MPLB_INC) $(HDF5_INC) $(OPS_LIB) $(HDF5_LIB) -DOPS_MPI -DCPU -D$(LEVEL) -lops_seq -lops_hdf5_seq type.cpp
@ar crf libmplb2d_mpi.a boundary.o configuration.o evolution.o flowfield.o hilemms_ops.o model.o scheme.o type.o
# Old pre-processor 2D
setupdomain: Makefile setup_comput_domain.cpp scheme.cpp model.cpp boundary.cpp Case_Setup.cpp $(OPS_INSTALL_PATH)/lib/libops_seq.a
$(CPP) $(CPPFLAGS) $(OPS_INC) $(HDF5_INC) $(OPS_LIB) -DOPS_2D type.cpp boundary.cpp setup_comput_domain.cpp scheme.cpp model.cpp -lops_seq -lops_hdf5_seq $(HDF5_LIB) -o setup_comput_domain
# Old pre-processor 3D
setupdomain3D: Makefile setup_comput_domain.cpp scheme.cpp model.cpp boundary.cpp $(OPS_INSTALL_PATH)/lib/libops_seq.a
$(CPP) $(CPPFLAGS) $(OPS_INC) $(HDF5_INC) $(OPS_LIB) -DOPS_3D type.cpp boundary.cpp setup_comput_domain.cpp scheme.cpp model.cpp -lops_seq -lops_hdf5_seq $(HDF5_LIB) -o setup_comput_domain
#
# mpi version
#
SEQ_KERNELS = lbm2d_seq_kernels.cpp
lbm2d_mpi: Makefile type_ops.cpp boundary_ops.cpp scheme_ops.cpp flowfield_ops.cpp model_ops.cpp evolution_ops.cpp lbm2d_ops.cpp $(OPS_INSTALL_PATH)/lib/libops_mpi.a
$(MPICPP) $(MPIFLAGS) -DOPS_MPI $(OPS_INC) $(HDF5_INC) $(OPS_LIB) type_ops.cpp boundary_ops.cpp scheme_ops.cpp flowfield_ops.cpp model_ops.cpp evolution_ops.cpp lbm2d_ops.cpp -I. ./MPI/$(SEQ_KERNELS) -lops_mpi $(HDF5_LIB) -o lbm2d_mpi
#
# OpenMP version
#
OMP_KERNELS = lbm2d_omp_kernels.cpp
lbm2d_mpi_openmp: Makefile type_ops.cpp boundary_ops.cpp scheme_ops.cpp flowfield_ops.cpp model_ops.cpp lbm2d_ops.cpp $(OPS_INSTALL_PATH)/lib/libops_mpi.a
$(MPICPP) $(OMPFLAGS) $(MPIFLAGS) -DOPS_MPI $(OPS_INC) $(HDF5_INC) $(OPS_LIB) type_ops.cpp boundary_ops.cpp scheme_ops.cpp flowfield_ops.cpp model_ops.cpp lbm2d_ops.cpp -I. ./MPI_OpenMP/$(OMP_KERNELS) -lops_mpi $(HDF5_LIB) -o lbm2d_mpi_openmp
lbm2d_openmp: Makefile type_ops.cpp boundary_ops.cpp scheme_ops.cpp flowfield_ops.cpp model_ops.cpp evolution_ops.cpp lbm2d_ops.cpp $(OPS_INSTALL_PATH)/lib/libops_seq.a
$(MPICPP) $(OMPFLAGS) $(HDF5) $(MPIFLAGS) $(OPS_INC) $(HDF5_INC) $(OPS_LIB) type_ops.cpp boundary_ops.cpp scheme_ops.cpp flowfield_ops.cpp model_ops.cpp evolution_ops.cpp lbm2d_ops.cpp -I. ./MPI_OpenMP/$(OMP_KERNELS) -lops_seq $(HDF5_LIB) -o lbm2d_openmp
#
# Sequential version
#
lbm2d_seq: Makefile type_ops.cpp boundary_ops.cpp scheme_ops.cpp flowfield_ops.cpp model_ops.cpp evolution_ops.cpp lbm2d_ops.cpp $(OPS_INSTALL_PATH)/lib/libops_seq.a
$(MPICPP) $(MPIFLAGS) $(OPS_INC) $(HDF5_INC) $(OPS_LIB) -I. ./MPI/$(SEQ_KERNELS) type_ops.cpp boundary_ops.cpp scheme_ops.cpp flowfield_ops.cpp model_ops.cpp evolution_ops.cpp lbm2d_ops.cpp -lops_seq $(HDF5_LIB) -o lbm2d_seq
#
# CUDA version
# $(OMPFLAGS)
lbm2d_cuda: Makefile ./CUDA/lbm2d_kernels_cu.o type_ops.cpp boundary_ops.cpp scheme_ops.cpp flowfield_ops.cpp model_ops.cpp evolution_ops.cpp lbm2d_ops.cpp $(OPS_INSTALL_PATH)/lib/libops_cuda.a
$(MPICPP) $(OMPFLAGS) $(MPIFLAGS) $(CUDA_INC) $(HDF5_INC) $(OPS_INC) $(OPS_LIB) $(CUDA_LIB) type_ops.cpp boundary_ops.cpp scheme_ops.cpp flowfield_ops.cpp model_ops.cpp evolution_ops.cpp lbm2d_ops.cpp ./CUDA/lbm2d_kernels_cu.o -lcudart -lops_cuda $(HDF5_LIB) -o lbm2d_cuda
lbm2d_mpi_cuda: Makefile ./CUDA/lbm2d_kernels_mpi_cu.o type_ops.cpp boundary_ops.cpp scheme_ops.cpp flowfield_ops.cpp model_ops.cpp lbm2d_ops.cpp $(OPS_INSTALL_PATH)/lib/libops_mpi_cuda.a
$(MPICPP) $(OMPFLAGS) $(CPPFLAGS) -DOPS_MPI $(CUDA_INC) $(OPS_INC) $(OPS_LIB) $(CUDA_LIB) type_ops.cpp boundary_ops.cpp scheme_ops.cpp flowfield_ops.cpp model_ops.cpp lbm2d_ops.cpp ./CUDA/lbm2d_kernels_mpi_cu.o -lcudart -lops_mpi_cuda $(HDF5_LIB) -o lbm2d_mpi_cuda
./CUDA/lbm2d_kernels_cu.o: ./CUDA/lbm2d_kernels.cu Makefile
$(NVCC) $(VAR) $(INC) $(NVCC_FLAGS) $(CODE_GEN_CUDA) $(OPS_INC) -I. -c -o ./CUDA/lbm2d_kernels_cu.o ./CUDA/lbm2d_kernels.cu
./CUDA/lbm2d_kernels_mpi_cu.o: ./CUDA/lbm2d_kernels.cu Makefile
$(NVCC) $(VAR) $(INC) $(NVCC_FLAGS) -DOPS_MPI $(CODE_GEN_CUDA) $(OPS_INC) -I. -I$(MPI_INSTALL_PATH)/include \
-DMPICH_IGNORE_CXX_SEEK -c -o ./CUDA/lbm2d_kernels_mpi_cu.o ./CUDA/lbm2d_kernels.cu
openacc_obj_list = $(shell find OpenACC/ -name "*.cpp" | sed s/\\.cpp/\\.o/g | tr "\n" " ")
openacc_c_obj_list = $(shell find OpenACC/ -name "*_c.c" | sed s/\\.c/\\.o/g | tr "\n" " ")
openacc_mpi_obj_list = $(shell find OpenACC/ -name "*.cpp" | sed s/\\.cpp/\\_mpi.o/g | tr "\n" " ")
openacc_mpi_c_obj_list = $(shell find OpenACC/ -name "*_c.c" | sed s/\\.c/\\_mpi.o/g | tr "\n" " ")
./OpenACC/lbm2d_kernels.o: ./OpenACC/lbm2d_kernels.cpp Makefile
$(CPP) -acc -ta=tesla:cc35 -ta=tesla:maxregcount:64 $(CPPFLAGS) $(CUDA_ALIGN_FLAG) $(OPS_INC) $(CUDA_INC) \
-I. OpenACC/lbm2d_kernels.cpp -c -o OpenACC/lbm2d_kernels.o
./OpenACC/lbm2d_kernels_mpi.o: ./OpenACC/lbm2d_kernels.cpp Makefile
$(MPICPP) -acc -ta=tesla:cc35 -ta=tesla:maxregcount:64 -DOPS_MPI $(MPIFLAGS) $(CUDA_ALIGN_FLAG) $(OPS_INC) $(CUDA_INC) \
-I. OpenACC/lbm2d_kernels.cpp -c -o OpenACC/lbm2d_kernels_mpi.o
./OpenACC/%_mpi.o: ./OpenACC/%.cpp ./OpenACC/%_c.c Makefile
@echo "Compiling kernel" $*
$(MPICC) -acc -ta=tesla:cc35 -ta=tesla:maxregcount:64 $(MPIFLAGS) -DOPS_MPI $(CUDA_ALIGN_FLAG) $(OPS_INC) $(CUDA_INC) \
-I. OpenACC/$*_c.c -c -o OpenACC/$*_c_mpi.o
$(MPICPP) -acc -ta=tesla:cc35 -ta=tesla:maxregcount:64 $(MPIFLAGS) -DOPS_MPI $(CUDA_ALIGN_FLAG) $(OPS_INC) $(CUDA_INC) \
-I. OpenACC/$*.cpp -c -o OpenACC/$*_mpi.o
./OpenACC/%.o: ./OpenACC/%.cpp ./OpenACC/%_c.c Makefile
@echo "Compiling kernel" $*
$(CC) -acc -ta=tesla:cc35 -ta=tesla:maxregcount:64 $(CCFLAGS) $(CUDA_ALIGN_FLAG) $(OPS_INC) $(CUDA_INC) \
-I. OpenACC/$*_c.c -c -o OpenACC/$*_c.o
$(CPP) -acc -ta=tesla:cc35 -ta=tesla:maxregcount:64 $(CPPFLAGS) $(CUDA_ALIGN_FLAG) $(OPS_INC) $(CUDA_INC) \
-I. OpenACC/$*.cpp -c -o OpenACC/$*.o
lbm2d_openacc: $(openacc_obj_list) ./OpenACC/lbm2d_kernels.o type_ops.cpp boundary_ops.cpp scheme_ops.cpp flowfield_ops.cpp model_ops.cpp lbm2d_ops.cpp Makefile $(OPS_INSTALL_PATH)/lib/libops_cuda.a
$(MPICPP) -acc -ta=tesla:cc35 $(MPIFLAGS) $(OPS_INC) $(OPS_LIB) -DOPS_MPI $(CUDA_INC) $(CUDA_LIB) \
type_ops.cpp boundary_ops.cpp scheme_ops.cpp flowfield_ops.cpp model_ops.cpp lbm2d_ops.cpp -I. $(openacc_obj_list) $(openacc_c_obj_list) -lcudart -lops_cuda $(HDF5_LIB -o lbm2d_openacc
lbm2d_mpi_openacc: $(openacc_mpi_obj_list) ./OpenACC/lbm2d_kernels_mpi.o type_ops.cpp boundary_ops.cpp scheme_ops.cpp flowfield_ops.cpp model_ops.cpp lbm2d_ops.cpp Makefile $(OPS_INSTALL_PATH)/lib/libops_mpi_cuda.a
$(MPICPP) -acc -ta=tesla:cc35 $(MPIFLAGS) $(OPS_INC) $(OPS_LIB) $(CUDA_INC) $(CUDA_LIB) -DOPS_MPI \
type_ops.cpp boundary_ops.cpp scheme_ops.cpp flowfield_ops.cpp model_ops.cpp lbm2d_ops.cpp -I. $(openacc_mpi_obj_list) $(openacc_mpi_c_obj_list) -lcudart -lops_mpi_cuda $(HDF5_LIB -o lbm2d_mpi_openacc
#
#OpenCL version
#
lbm2d_opencl: ./OpenCL/lbm2d_opencl_kernels.o lbm2d_ops.cpp lbm2d_kernel.h lbm2d_print_kernel.h lbm2d_copy_kernel.h Makefile $(OPS_INSTALL_PATH)/lib/libops_opencl.a
$(CPP) $(MPIFLAGS) $(OPS_INC) $(OPS_LIB) $(OPENCL_LIB) lbm2d_ops.cpp ./OpenCL/lbm2d_opencl_kernels.o -lops_opencl -o lbm2d_opencl
lbm2d_mpi_opencl: ./OpenCL/lbm2d_mpi_opencl_kernels.o lbm2d_ops.cpp lbm2d_kernel.h lbm2d_print_kernel.h lbm2d_copy_kernel.h Makefile $(OPS_INSTALL_PATH)/lib/libops_mpi_opencl.a
$(MPICPP) $(MPIFLAGS) -DOPS_MPI $(OPS_INC) $(OPS_LIB) $(OPENCL_LIB) lbm2d_ops.cpp ./OpenCL/lbm2d_opencl_kernels.o -lops_mpi_opencl -o lbm2d_mpi_opencl
./OpenCL/lbm2d_opencl_kernels.o: ./OpenCL/lbm2d_opencl_kernels.cpp Makefile
$(CPP) $(CPPFLAGS) $(VAR) $(OPENCL_INC) $(INC) $(OPS_INC) -I. \
-c -o ./OpenCL/lbm2d_opencl_kernels.o ./OpenCL/lbm2d_opencl_kernels.cpp
./OpenCL/lbm2d_mpi_opencl_kernels.o: ./OpenCL/lbm2d_opencl_kernels.cpp Makefile
$(MPICPP) $(CPPFLAGS) -DOPS_MPI $(VAR) $(OPENCL_INC) $(INC) $(OPS_INC) -I. \
-c -o ./OpenCL/lbm2d_opencl_kernels.o ./OpenCL/lbm2d_opencl_kernels.cpp
#
#error
#
openacc_error:
@echo "lbm2d_mpi_openacc cannot be built with $(OPS_COMPILER) compilers .. requires an OpenACC compiler (e.g. pgi)"
#
# cleanup
#
clean:
rm -f lbm2d_dev_seq lbm2d_seq lbm2d_dev_seq lbm2d_mpi lbm2d_dev_mpi lbm2d_openmp lbm2d_mpi_openmp lbm2d_cuda lbm2d_mpi_cuda lbm2d_openacc lbm2d_mpi_openacc lbm2d_opencl lbm2d_mpi_opencl ./CUDA/*.o ./OpenACC/*.o *.o lbm2d_opencl lbm3d_dev_seq lbm3d_seq lbm3d_dev_seq lbm3d_mpi lbm3d_hilemms_dev_seq lbm3d_dev_mpi./OpenCL/*.o *.o