diff --git a/.github/workflows/build-and-push-images.yml b/.github/workflows/build-and-push-images.yml index b6b26f4..a8941ab 100644 --- a/.github/workflows/build-and-push-images.yml +++ b/.github/workflows/build-and-push-images.yml @@ -42,7 +42,7 @@ jobs: uses: lewagon/wait-on-check-action@v1.3.1 with: ref: ${{ github.ref }} - check-name: 'build base image' + check-name: "build base image" repo-token: ${{ secrets.GITHUB_TOKEN }} wait-interval: 10 @@ -74,8 +74,8 @@ jobs: - name: push developer-cpu image run: docker push ${{ env.REGISTRY }}/${{ env.IMAGE_NAME_PREFIX }}/developer-cpu:latest - build-cuda-sm70: - name: build cuda-sm70 images + build-cuda-sm89: + name: build cuda-sm89 images if: github.event_name != 'pull_request' runs-on: ubuntu-latest steps: @@ -83,7 +83,7 @@ jobs: uses: lewagon/wait-on-check-action@v1.3.1 with: ref: ${{ github.ref }} - check-name: 'build base image' + check-name: "build base image" repo-token: ${{ secrets.GITHUB_TOKEN }} wait-interval: 10 @@ -97,29 +97,29 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: build cuda-sm70 image + - name: build cuda-sm89 image run: | - docker-compose build --build-arg cuda_arch_sm=70 cuda + docker-compose build --build-arg cuda_arch_sm=89 cuda docker image tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME_PREFIX }}/cuda:latest \ - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME_PREFIX }}/cuda-sm70:latest + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME_PREFIX }}/cuda-sm89:latest - - name: push cuda-sm70 image - run: docker push ${{ env.REGISTRY }}/${{ env.IMAGE_NAME_PREFIX }}/cuda-sm70:latest + - name: push cuda-sm89 image + run: docker push ${{ env.REGISTRY }}/${{ env.IMAGE_NAME_PREFIX }}/cuda-sm89:latest - - name: build cuda-tpls-sm70 image + - name: build cuda-tpls-sm89 image run: | - docker-compose build --build-arg cuda_arch_sm=70 cuda-tpls + docker-compose build --build-arg cuda_arch_sm=89 cuda-tpls docker image tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME_PREFIX }}/cuda-tpls:latest \ - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME_PREFIX }}/cuda-tpls-sm70:latest + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME_PREFIX }}/cuda-tpls-sm89:latest - - name: push cuda-tpls-sm70 image - run: docker push ${{ env.REGISTRY }}/${{ env.IMAGE_NAME_PREFIX }}/cuda-tpls-sm70:latest + - name: push cuda-tpls-sm89 image + run: docker push ${{ env.REGISTRY }}/${{ env.IMAGE_NAME_PREFIX }}/cuda-tpls-sm89:latest - - name: build developer-cuda-sm70 image + - name: build developer-cuda-sm89 image run: | - docker-compose build --build-arg base_image=cuda-tpls-sm70 developer-cuda + docker-compose build --build-arg base_image=cuda-tpls-sm89 developer-cuda docker image tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME_PREFIX }}/developer-cuda:latest \ - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME_PREFIX }}/developer-cuda-sm70:latest + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME_PREFIX }}/developer-cuda-sm89:latest - - name: push developer-cuda-sm70 image - run: docker push ${{ env.REGISTRY }}/${{ env.IMAGE_NAME_PREFIX }}/developer-cuda-sm70:latest + - name: push developer-cuda-sm89 image + run: docker push ${{ env.REGISTRY }}/${{ env.IMAGE_NAME_PREFIX }}/developer-cuda-sm89:latest diff --git a/README.md b/README.md index aedd7f6..12c3da4 100644 --- a/README.md +++ b/README.md @@ -18,38 +18,38 @@ Both types are available in the following configurations: - [`cpu`](https://github.com/mfem/containers/pkgs/container/containers%2Fcpu) - OpenMPI 4.1.2 - - hypre 2.27 + - hypre 2.31.0 -- [`cuda-sm70`](https://github.com/mfem/containers/pkgs/container/containers%2Fcuda-sm70) - - CUDA toolkit 11.8 +- [`cuda-sm89`](https://github.com/mfem/containers/pkgs/container/containers%2Fcuda-sm89) + - CUDA toolkit 12.9 - OpenMPI 4.1.2 - - hypre 2.27 + - hypre 2.31.0 - [`cpu-tpls`](https://github.com/mfem/containers/pkgs/container/containers%2Fcpu-tpls) - OpenMPI 4.1.2 - - hypre 2.27.0 + - hypre 2.31.0 - SuperLU_dist 8.2.1 - PETSc 3.20.2 -- [`cuda-tpls-sm70`](https://github.com/mfem/containers/pkgs/container/containers%2Fcuda-tpls-sm70) - - CUDA toolkit 11.8 +- [`cuda-tpls-sm89`](https://github.com/mfem/containers/pkgs/container/containers%2Fcuda-tpls-sm89) + - CUDA toolkit 12.9 - OpenMPI 4.1.2 - - hypre 2.27.0 + - hypre 2.31.0 - AmgX 8.2.1 - [`developer-cpu`](https://github.com/mfem/containers/pkgs/container/containers%2Fdeveloper-cpu) - extension of `cpu-tpls` that includes a development environment with VSCode server and GLVis - see the MFEM [AWS tutorial](https://mfem.org/tutorial/docker) for details - -- [`developer-cuda-sm70`](https://github.com/mfem/containers/pkgs/container/containers%2Fdeveloper-cuda-sm70) - - extension of `cuda-tpls-sm70` that includes a development environment with VSCode server and GLVis + +- [`developer-cuda-sm89`](https://github.com/mfem/containers/pkgs/container/containers%2Fdeveloper-cuda-sm89) + - extension of `cuda-tpls-sm89` that includes a development environment with VSCode server and GLVis - see the MFEM [AWS tutorial](https://mfem.org/tutorial/docker) for details -Note that the `cuda` images require the host has the +Note that the `cuda` images require the host has the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html) to installed and configured. Note also that the `cuda` images have MFEM and its third-party libraries configured for -CUDA `sm_70`. You can create your own image that support a different compute capability with +CUDA `sm_89`. You can create your own image that support a different compute capability (e.g. sm_80) with `docker-compose` as follows: ``` git clone git@github.com:mfem/containers.git @@ -60,7 +60,7 @@ docker-compose build --build-arg cuda_arch_sm=80 cuda-tpls && docker image tag c We recommend starting the container with: ``` -docker run --cap-add=SYS_PTRACE -ti --gpus all ghcr.io/mfem/containers/cuda-tpls-sm70:latest /bin/bash +docker run --rm --cap-add=SYS_PTRACE --runtime=nvidia --gpus all -ti ghcr.io/mfem/containers/cuda-tpls-sm89:latest bash ``` which puts you in the home directory for the `euler` user, enables access to all GPUs, and lets you explore the broad selection of examples or write your own. diff --git a/base/Dockerfile b/base/Dockerfile index 33561dc..b2927c8 100644 --- a/base/Dockerfile +++ b/base/Dockerfile @@ -1,21 +1,36 @@ -FROM ubuntu:22.04 +FROM ubuntu:24.04 ARG num_jobs RUN apt-get update && apt-get install -y \ - curl git git-lfs clang clang-tools clangd cmake autoconf \ - automake gdb git libffi-dev zlib1g-dev python3-pip \ - libssl-dev xz-utils pkgconf gfortran openmpi-bin libopenmpi-dev \ - wget libarchive-tools htop libmetis-dev libparmetis-dev libopenblas-dev + curl git git-lfs clangd cmake autoconf lsb-release \ + automake gdb git libffi-dev zlib1g-dev python3-pip libzstd-dev \ + libssl-dev xz-utils pkgconf gfortran openmpi-bin libopenmpi-dev \ + wget libarchive-tools htop libmetis-dev libparmetis-dev libopenblas-dev -RUN update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++ 100 && \ - update-alternatives --install /usr/bin/cc cc /usr/bin/clang 100 +RUN curl -L https://apt.llvm.org/llvm-snapshot.gpg.key > /etc/apt/trusted.gpg.d/apt.llvm.org.asc &&\ + echo "deb http://apt.llvm.org/$(lsb_release -sc)/ llvm-toolchain-$(lsb_release -sc) main" | tee /etc/apt/sources.list.d/apt.llvm.org.list + +RUN apt-get update && apt-get install -y clang-19 libclang-19-dev clangd-19 + +RUN update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++-19 100 && \ + update-alternatives --install /usr/bin/cc cc /usr/bin/clang-19 100 && \ + update-alternatives --install /usr/bin/clang clang /usr/bin/clang-19 100 && \ + update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-19 100 && \ + update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-19 100 ENV OMPI_CC=clang ENV OMPI_CXX=clang++ ENV LD_LIBRARY_PATH=/usr/local/lib +RUN curl -L https://github.com/EnzymeAD/Enzyme/archive/refs/tags/v0.0.186.tar.gz > enzyme.tar.gz && \ + tar -xzf enzyme.tar.gz && \ + cd Enzyme-0.0.186/enzyme && \ + mkdir build && cd build && \ + cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local -DLLVM_DIR=/usr/lib/llvm-19 && \ + make -j ${num_jobs} && make install + RUN useradd --create-home --shell /bin/bash euler USER euler -WORKDIR /home/euler \ No newline at end of file +WORKDIR /home/euler diff --git a/cpu-tpls/user.mk b/cpu-tpls/user.mk index 850f802..4843d96 100644 --- a/cpu-tpls/user.mk +++ b/cpu-tpls/user.mk @@ -1,370 +1,43 @@ -# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced -# at the Lawrence Livermore National Laboratory. All Rights reserved. See files -# LICENSE and NOTICE for details. LLNL-CODE-806117. -# -# This file is part of the MFEM library. For more information and source code -# availability visit https://mfem.org. -# -# MFEM is free software; you can redistribute it and/or modify it under the -# terms of the BSD-3 license. We welcome feedback and contributions, see file -# CONTRIBUTING.md for details. - -# This file describes the default MFEM build options. -# -# See the file INSTALL for description of these options. You can -# customize them below, or copy this file to user.mk and modify it. - - -# Some choices below are based on the OS type: -NOTMAC := $(subst Darwin,,$(shell uname -s)) - -ETAGS_BIN = $(shell command -v etags 2> /dev/null) -EGREP_BIN = $(shell command -v egrep 2> /dev/null) +VERBOSE = NO CXX = clang++ -MPICXX = mpicxx +MPICXX = OMPI_CXX=$(CXX) mpicxx -BASE_FLAGS = -std=c++11 +BASE_FLAGS = -std=c++17 OPTIM_FLAGS = -O3 $(BASE_FLAGS) -DEBUG_FLAGS = -g $(XCOMPILER)-Wall $(BASE_FLAGS) - -# Prefixes for passing flags to the compiler and linker when using CXX or MPICXX -CXX_XCOMPILER = -CXX_XLINKER = -Wl, - -# Destination location of make install -# PREFIX = $(HOME)/mfem -PREFIX = ./mfem -# Install program -INSTALL = /usr/bin/install STATIC = NO SHARED = YES -# CUDA configuration options -# -# If you set MFEM_USE_ENZYME=YES, CUDA_CXX has to be configured to use cuda with -# clang as its host compiler. -CUDA_CXX = nvcc -CUDA_ARCH = sm_60 -CUDA_FLAGS = -x=cu --expt-extended-lambda -arch=$(CUDA_ARCH) -# Prefixes for passing flags to the host compiler and linker when using CUDA_CXX -CUDA_XCOMPILER = -Xcompiler= -CUDA_XLINKER = -Xlinker= - -# HIP configuration options -HIP_CXX = hipcc -# The HIP_ARCH option specifies the AMD GPU processor, similar to CUDA_ARCH. For -# example: gfx600 (tahiti), gfx700 (kaveri), gfx701 (hawaii), gfx801 (carrizo), -# gfx900, gfx1010, etc. -HIP_ARCH = gfx900 -HIP_FLAGS = --amdgpu-target=$(HIP_ARCH) -HIP_XCOMPILER = -HIP_XLINKER = -Wl, - -# Flags for generating dependencies. -DEP_FLAGS = -MM -MT - -ifneq ($(NOTMAC),) - AR = ar - ARFLAGS = crv - RANLIB = ranlib - PICFLAG = $(XCOMPILER)-fPIC - SO_EXT = so - SO_VER = so.$(MFEM_VERSION_STRING) - BUILD_SOFLAGS = -shared $(XLINKER)-soname,libmfem.$(SO_VER) - BUILD_RPATH = $(XLINKER)-rpath,$(BUILD_REAL_DIR) - INSTALL_SOFLAGS = $(BUILD_SOFLAGS) - INSTALL_RPATH = $(XLINKER)-rpath,@MFEM_LIB_DIR@ -else - # Silence "has no symbols" warnings on Mac OS X - AR = ar - ARFLAGS = Scrv - RANLIB = ranlib -no_warning_for_no_symbols - PICFLAG = $(XCOMPILER)-fPIC - SO_EXT = dylib - SO_VER = $(MFEM_VERSION_STRING).dylib - MAKE_SOFLAGS = $(XLINKER)-dylib,-install_name,$(1)/libmfem.$(SO_VER),\ - -compatibility_version,$(MFEM_VERSION_STRING),\ - -current_version,$(MFEM_VERSION_STRING),\ - -undefined,dynamic_lookup - BUILD_SOFLAGS = $(subst $1 ,,$(call MAKE_SOFLAGS,$(BUILD_REAL_DIR))) - BUILD_RPATH = $(XLINKER)-undefined,dynamic_lookup - INSTALL_SOFLAGS = $(subst $1 ,,$(call MAKE_SOFLAGS,$(MFEM_LIB_DIR))) - INSTALL_RPATH = $(XLINKER)-undefined,dynamic_lookup - # Silence unused command line argument warnings when generating dependencies - # with mpicxx and clang - DEP_FLAGS := -Wno-unused-command-line-argument $(DEP_FLAGS) -endif - -# Set CXXFLAGS to overwrite the default selection of DEBUG_FLAGS/OPTIM_FLAGS -# CXXFLAGS = -O3 -march=native - -# Optional extra compile flags, in addition to CXXFLAGS: -# CPPFLAGS = - -# Library configurations: -# Note: symbols of the form @VAR@ will be replaced by $(VAR) in derived -# variables, like MFEM_FLAGS, defined in config.mk. - -# Command used to launch MPI jobs -MFEM_MPIEXEC = mpirun -MFEM_MPIEXEC_NP = -np -# Number of mpi tasks for parallel jobs -MFEM_MPI_NP = 4 - -# MFEM configuration options: YES/NO values, which are exported to config.mk and -# config.hpp. The values below are the defaults for generating the actual values -# in config.mk and config.hpp. - MFEM_USE_MPI = YES MFEM_USE_METIS = $(MFEM_USE_MPI) MFEM_USE_METIS_5 = YES MFEM_DEBUG = NO -MFEM_USE_EXCEPTIONS = NO -MFEM_USE_ZLIB = NO MFEM_USE_LIBUNWIND = NO -MFEM_USE_LAPACK = NO -MFEM_THREAD_SAFE = NO -MFEM_USE_OPENMP = NO -MFEM_USE_LEGACY_OPENMP = NO MFEM_USE_MEMALLOC = YES MFEM_TIMER_TYPE = $(if $(NOTMAC),2,4) -MFEM_USE_SUNDIALS = NO -MFEM_USE_MESQUITE = NO -MFEM_USE_SUITESPARSE = NO +MFEM_USE_ENZYME = YES +MFEM_USE_CUDA = NO MFEM_USE_SUPERLU = YES -MFEM_USE_SUPERLU5 = NO -MFEM_USE_MUMPS = NO -MFEM_USE_STRUMPACK = NO -MFEM_USE_GINKGO = NO -MFEM_USE_AMGX = NO -MFEM_USE_GNUTLS = NO -MFEM_USE_NETCDF = NO MFEM_USE_PETSC = YES -MFEM_USE_SLEPC = NO -MFEM_USE_MPFR = NO -MFEM_USE_SIDRE = NO -MFEM_USE_FMS = NO -MFEM_USE_CONDUIT = NO -MFEM_USE_PUMI = NO -MFEM_USE_HIOP = NO -MFEM_USE_GSLIB = NO -MFEM_USE_CUDA = NO -MFEM_USE_HIP = NO -MFEM_USE_RAJA = NO -MFEM_USE_OCCA = NO -MFEM_USE_CEED = NO -MFEM_USE_CALIPER = NO -MFEM_USE_ALGOIM = NO -MFEM_USE_UMPIRE = NO -MFEM_USE_SIMD = NO -MFEM_USE_ADIOS2 = NO -MFEM_USE_MKL_CPARDISO = NO -MFEM_USE_MOONOLITH = NO -MFEM_USE_ADFORWARD = NO -MFEM_USE_CODIPACK = NO -MFEM_USE_BENCHMARK = NO -MFEM_USE_PARELAG = NO -MFEM_USE_ENZYME = NO - -# MPI library compile and link flags -# These settings are used only when building MFEM with MPI + HIP -ifeq ($(MFEM_USE_MPI)$(MFEM_USE_HIP),YESYES) - # We determine MPI_DIR assuming $(MPICXX) is in $(MPI_DIR)/bin - MPI_DIR := $(patsubst %/,%,$(dir $(shell which $(MPICXX)))) - MPI_DIR := $(patsubst %/,%,$(dir $(MPI_DIR))) - MPI_OPT = -I$(MPI_DIR)/include - MPI_LIB = -L$(MPI_DIR)/lib $(XLINKER)-rpath,$(MPI_DIR)/lib -lmpi -endif - -# ROCM/HIP directory such that ROCM/HIP libraries like rocsparse and rocrand are -# found in $(HIP_DIR)/lib, usually as links. Typically, this directoory is of -# the form /opt/rocm-X.Y.Z which is called ROCM_PATH by hipconfig. -ifeq ($(MFEM_USE_HIP),YES) - HIP_DIR := $(patsubst %/,%,$(dir $(shell which $(HIP_CXX)))) - HIP_DIR := $(patsubst %/,%,$(dir $(HIP_DIR))) - ifeq (,$(wildcard $(HIP_DIR)/lib/librocsparse.*)) - HIP_DIR := $(shell hipconfig --rocmpath 2> /dev/null) - ifeq (,$(wildcard $(HIP_DIR)/lib/librocsparse.*)) - $(error Unable to determine HIP_DIR. Please set it manually.) - endif - endif -endif - -# Compile and link options for zlib. -ZLIB_DIR = -ZLIB_OPT = $(if $(ZLIB_DIR),-I$(ZLIB_DIR)/include) -ZLIB_LIB = $(if $(ZLIB_DIR),$(ZLIB_RPATH) -L$(ZLIB_DIR)/lib ,)-lz -ZLIB_RPATH = $(XLINKER)-rpath,$(ZLIB_DIR)/lib - -LIBUNWIND_OPT = -g -LIBUNWIND_LIB = $(if $(NOTMAC),-lunwind -ldl,) -# HYPRE library configuration (needed to build the parallel version) HYPRE_DIR = /usr/local HYPRE_OPT = -I$(HYPRE_DIR)/include HYPRE_LIB = -L$(HYPRE_DIR)/lib -lHYPRE -ifeq (YES,$(MFEM_USE_CUDA)) - # This is only necessary when hypre is built with cuda: - HYPRE_LIB += -lcusparse -lcurand -endif -ifeq (YES,$(MFEM_USE_HIP)) - # This is only necessary when hypre is built with hip: - HYPRE_LIB += -L$(HIP_DIR)/lib $(XLINKER)-rpath,$(HIP_DIR)/lib\ - -lrocsparse -lrocrand -endif - -# METIS library configuration -ifeq ($(MFEM_USE_SUPERLU)$(MFEM_USE_STRUMPACK)$(MFEM_USE_MUMPS),NONONO) - ifeq ($(MFEM_USE_METIS_5),NO) - METIS_DIR = - METIS_OPT = - METIS_LIB = -L/usr/lib/x86_64-linux-gnu -lmetis - else - METIS_DIR = - METIS_OPT = - METIS_LIB = -L/usr/lib/x86_64-linux-gnu -lmetis - endif -else - # ParMETIS: currently needed by SuperLU or STRUMPACK. We assume that METIS 5 - # (included with ParMETIS) is installed in the same location. - # Starting with STRUMPACK v2.2.0, ParMETIS is an optional dependency while - # METIS is still required. - METIS_DIR = - METIS_OPT = - METIS_LIB = -L/usr/lib/x86_64-linux-gnu -lmetis -lparmetis - MFEM_USE_METIS_5 = YES -endif - -# LAPACK library configuration -LAPACK_OPT = -LAPACK_LIB = $(if $(NOTMAC),-llapack -lblas,-framework Accelerate) - -# OpenMP configuration -OPENMP_OPT = $(XCOMPILER)-fopenmp -OPENMP_LIB = - -# Used when MFEM_TIMER_TYPE = 2 -POSIX_CLOCKS_LIB = -lrt - -# SUNDIALS library configuration -# For sundials_nvecmpiplusx and nvecparallel remember to build with MPI_ENABLE=ON -# and modify cmake variables for hypre for sundials -SUNDIALS_DIR = @MFEM_DIR@/../sundials-5.0.0/instdir -SUNDIALS_OPT = -I$(SUNDIALS_DIR)/include -SUNDIALS_LIBDIR = $(wildcard $(SUNDIALS_DIR)/lib*) -SUNDIALS_LIB = $(XLINKER)-rpath,$(SUNDIALS_LIBDIR) -L$(SUNDIALS_LIBDIR)\ - -lsundials_arkode -lsundials_cvodes -lsundials_nvecserial -lsundials_kinsol - -ifeq ($(MFEM_USE_MPI),YES) - SUNDIALS_LIB += -lsundials_nvecparallel -lsundials_nvecmpiplusx -endif -ifeq ($(MFEM_USE_CUDA),YES) - SUNDIALS_LIB += -lsundials_nveccuda -endif -# If SUNDIALS was built with KLU: -# MFEM_USE_SUITESPARSE = YES - -# MESQUITE library configuration -MESQUITE_DIR = @MFEM_DIR@/../mesquite-2.99 -MESQUITE_OPT = -I$(MESQUITE_DIR)/include -MESQUITE_LIB = -L$(MESQUITE_DIR)/lib -lmesquite - -# SuiteSparse library configuration -LIB_RT = $(if $(NOTMAC),-lrt,) -SUITESPARSE_DIR = @MFEM_DIR@/../SuiteSparse -SUITESPARSE_OPT = -I$(SUITESPARSE_DIR)/include -SUITESPARSE_LIB = $(XLINKER)-rpath,$(SUITESPARSE_DIR)/lib\ - -L$(SUITESPARSE_DIR)/lib -lklu -lbtf -lumfpack -lcholmod -lcolamd -lamd -lcamd\ - -lccolamd -lsuitesparseconfig $(LIB_RT) $(METIS_LIB) $(LAPACK_LIB) - -# SuperLU library configuration -ifeq ($(MFEM_USE_SUPERLU5),YES) - SUPERLU_DIR = @MFEM_DIR@/../SuperLU_DIST_5.1.0 - SUPERLU_OPT = -I$(SUPERLU_DIR)/include - SUPERLU_LIB = $(XLINKER)-rpath,$(SUPERLU_DIR)/lib -L$(SUPERLU_DIR)/lib\ - -lsuperlu_dist_5.1.0 -else - SUPERLU_DIR = /usr/local - SUPERLU_OPT = - SUPERLU_LIB = $(XLINKER)-rpath,$(SUPERLU_DIR)/lib -L$(SUPERLU_DIR)/lib\ - -lsuperlu_dist -lblas -endif - -# SCOTCH library configuration (required by STRUMPACK <= v2.1.0, optional in -# STRUMPACK >= v2.2.0) -SCOTCH_DIR = @MFEM_DIR@/../scotch_6.0.4 -SCOTCH_OPT = -I$(SCOTCH_DIR)/include -SCOTCH_LIB = -L$(SCOTCH_DIR)/lib -lptscotch -lptscotcherr -lscotch -lscotcherr\ - -lpthread - -# SCALAPACK library configuration (required by STRUMPACK and MUMPS) -SCALAPACK_DIR = @MFEM_DIR@/../scalapack-2.0.2 -SCALAPACK_OPT = -I$(SCALAPACK_DIR)/SRC -SCALAPACK_LIB = -L$(SCALAPACK_DIR)/lib -lscalapack $(LAPACK_LIB) -# MPI Fortran library, needed e.g. by STRUMPACK or MUMPS -# MPICH: -MPI_FORTRAN_LIB = -lmpifort -# OpenMPI: -# MPI_FORTRAN_LIB = -lmpi_mpifh -# Additional Fortan library: -# MPI_FORTRAN_LIB += -lgfortran +METIS_DIR = +METIS_OPT = +METIS_LIB = -L/usr/lib/x86_64-linux-gnu -lmetis -lparmetis -# MUMPS library configuration -MUMPS_DIR = @MFEM_DIR@/../MUMPS_5.2.0 -MUMPS_OPT = -I$(MUMPS_DIR)/include -MUMPS_LIB = $(XLINKER)-rpath,$(MUMPS_DIR)/lib -L$(MUMPS_DIR)/lib -ldmumps\ - -lmumps_common -lpord $(SCALAPACK_LIB) $(LAPACK_LIB) $(MPI_FORTRAN_LIB) +ENZYME_DIR = /usr/local -# STRUMPACK library configuration -STRUMPACK_DIR = @MFEM_DIR@/../STRUMPACK-build -STRUMPACK_OPT = -I$(STRUMPACK_DIR)/include $(SCOTCH_OPT) -# If STRUMPACK was build with OpenMP support, the following may be need: -# STRUMPACK_OPT += $(OPENMP_OPT) -STRUMPACK_LIB = -L$(STRUMPACK_DIR)/lib -lstrumpack $(MPI_FORTRAN_LIB)\ - $(SCOTCH_LIB) $(SCALAPACK_LIB) - -# Ginkgo library configuration (currently not needed) -GINKGO_DIR = @MFEM_DIR@/../ginkgo/install -GINKGO_BUILD_TYPE=Release -ifeq ($(MFEM_USE_GINKGO),YES) - BASE_FLAGS = -std=c++14 -endif -GINKGO_OPT = -isystem $(GINKGO_DIR)/include -GINKGO_LIB_DIR = $(sort $(dir $(wildcard $(GINKGO_DIR)/lib*/libginkgo*.a $(GINKGO_DIR)/lib*/libginkgo*.so $(GINKGO_DIR)/lib*/libginkgo*.dylib $(GINKGO_DIR)/lib*/libginkgo*.dll))) -ALL_GINKGO_LIBS_DEBUG = $(notdir $(basename $(wildcard $(GINKGO_DIR)/lib*/libginkgo*d.a $(GINKGO_DIR)/lib*/libginkgo*d.so $(GINKGO_DIR)/lib*/libginkgo*d.dylib $(GINKGO_DIR)/lib*/libginkgo*d.dll))) -ALL_GINKGO_LIBS = $(notdir $(basename $(wildcard $(GINKGO_DIR)/lib*/libginkgo*.a $(GINKGO_DIR)/lib*/libginkgo*.so $(GINKGO_DIR)/lib*/libginkgo*.dylib $(GINKGO_DIR)/lib*/libginkgo*.dll))) -ALL_GINKGO_LIBS_RELEASE = $(filter-out $(ALL_GINKGO_LIBS_DEBUG),$(ALL_GINKGO_LIBS)) -GINKGO_LINK = $(subst libginkgo,-lginkgo,$(ALL_GINKGO_LIBS_RELEASE)) -ifeq ($(GINKGO_BUILD_TYPE),Debug) - ifneq (,$(ALL_GINKGO_LIBS_DEBUG)) - GINKGO_LINK = $(subst libginkgo,-lginkgo,$(ALL_GINKGO_LIBS_DEBUG)) - endif -else -endif -GINKGO_LIB = $(XLINKER)-rpath,$(GINKGO_LIB_DIR) -L$(GINKGO_LIB_DIR) $(GINKGO_LINK) - -# AmgX library configuration -AMGX_DIR = @MFEM_DIR@/../amgx -AMGX_OPT = -I$(AMGX_DIR)/include -AMGX_LIB = -lcusparse -lcusolver -lcublas -lnvToolsExt -L$(AMGX_DIR)/lib -lamgx - -# GnuTLS library configuration -GNUTLS_OPT = -GNUTLS_LIB = -lgnutls - -# NetCDF library configuration -NETCDF_DIR = $(HOME)/local -HDF5_DIR = $(HOME)/local -NETCDF_OPT = -I$(NETCDF_DIR)/include -I$(HDF5_DIR)/include $(ZLIB_OPT) -NETCDF_LIB = $(XLINKER)-rpath,$(NETCDF_DIR)/lib -L$(NETCDF_DIR)/lib\ - $(XLINKER)-rpath,$(HDF5_DIR)/lib -L$(HDF5_DIR)/lib\ - -lnetcdf -lhdf5_hl -lhdf5 $(ZLIB_LIB) +SUPERLU_DIR = /usr/local +SUPERLU_OPT = +SUPERLU_LIB = $(XLINKER)-rpath,$(SUPERLU_DIR)/lib -L$(SUPERLU_DIR)/lib\ + -lsuperlu_dist -lblas # PETSc library configuration (version greater or equal to 3.8 or the dev branch) -PETSC_ARCH := +PETSC_ARCH := PETSC_DIR := /usr/local PETSC_VARS := $(PETSC_DIR)/lib/petsc/conf/petscvariables PETSC_FOUND := $(if $(wildcard $(PETSC_VARS)),YES,) @@ -377,171 +50,3 @@ ifeq ($(PETSC_FOUND),YES) -L$(abspath $(PETSC_DIR))/lib -lpetsc\ $(subst $(CXX_XLINKER),$(XLINKER),$(PETSC_DEP)) endif - -SLEPC_DIR := $(MFEM_DIR)/../slepc -SLEPC_VARS := $(SLEPC_DIR)/lib/slepc/conf/slepc_variables -SLEPC_FOUND := $(if $(wildcard $(SLEPC_VARS)),YES,) -SLEPC_INC_VAR = SLEPC_INCLUDE -SLEPC_LIB_VAR = SLEPC_EXTERNAL_LIB -ifeq ($(SLEPC_FOUND),YES) - SLEPC_OPT := $(shell sed -n "s/$(SLEPC_INC_VAR) *= *//p" $(SLEPC_VARS)) - # Some additional external libraries might be defined in this file - -include ${SLEPC_DIR}/${PETSC_ARCH}/lib/slepc/conf/slepcvariables - SLEPC_DEP := $(shell sed -n "s/$(SLEPC_LIB_VAR) *= *//p" $(SLEPC_VARS)) - SLEPC_LIB = $(XLINKER)-rpath,$(abspath $(SLEPC_DIR))/$(PETSC_ARCH)/lib\ - -L$(abspath $(SLEPC_DIR))/$(PETSC_ARCH)/lib -lslepc\ - $(subst $(CXX_XLINKER),$(XLINKER),$(SLEPC_DEP)) -endif - -ifeq ($(MFEM_USE_MOONOLITH),YES) - include $(MOONOLITH_DIR)/config/moonolith-config.makefile - MOONOLITH_LIB=$(MOONOLITH_LIBRARIES) -endif - -# MPFR library configuration -MPFR_OPT = -MPFR_LIB = -lmpfr - -# FMS and required libraries configuration -FMS_DIR = $(MFEM_DIR)/../fms -FMS_OPT = -I$(FMS_DIR)/include -FMS_LIB = -Wl,-rpath,$(FMS_DIR)/lib -L$(FMS_DIR)/lib -lfms - -# Conduit and required libraries configuration -CONDUIT_DIR = @MFEM_DIR@/../conduit -CONDUIT_OPT = -I$(CONDUIT_DIR)/include/conduit -CONDUIT_LIB = \ - $(XLINKER)-rpath,$(CONDUIT_DIR)/lib -L$(CONDUIT_DIR)/lib \ - -lconduit -lconduit_relay -lconduit_blueprint -ldl - -# Check if Conduit was built with hdf5 support, by looking -# for the relay hdf5 header -CONDUIT_HDF5_HEADER=$(CONDUIT_DIR)/include/conduit/conduit_relay_hdf5.hpp -ifneq (,$(wildcard $(CONDUIT_HDF5_HEADER))) - CONDUIT_OPT += -I$(HDF5_DIR)/include - CONDUIT_LIB += $(XLINKER)-rpath,$(HDF5_DIR)/lib -L$(HDF5_DIR)/lib \ - -lhdf5 $(ZLIB_LIB) -endif - -# Sidre and required libraries configuration -# Be sure to check the HDF5_DIR (set above) is correct -SIDRE_DIR = @MFEM_DIR@/../axom -SIDRE_OPT = -I$(SIDRE_DIR)/include -I$(CONDUIT_DIR)/include/conduit\ - -I$(HDF5_DIR)/include -SIDRE_LIB = \ - $(XLINKER)-rpath,$(SIDRE_DIR)/lib -L$(SIDRE_DIR)/lib \ - $(XLINKER)-rpath,$(CONDUIT_DIR)/lib -L$(CONDUIT_DIR)/lib \ - $(XLINKER)-rpath,$(HDF5_DIR)/lib -L$(HDF5_DIR)/lib \ - -laxom -lconduit -lconduit_relay -lconduit_blueprint -lhdf5 $(ZLIB_LIB) -ldl - -# PUMI -# Note that PUMI_DIR is needed -- it is used to check for gmi_sim.h -PUMI_DIR = @MFEM_DIR@/../pumi-2.1.0 -PUMI_OPT = -I$(PUMI_DIR)/include -PUMI_LIB = -L$(PUMI_DIR)/lib -lpumi -lcrv -lma -lmds -lapf -lpcu -lgmi -lparma\ - -llion -lmth -lapf_zoltan -lspr - -# HIOP -HIOP_DIR = @MFEM_DIR@/../hiop/install -HIOP_OPT = -I$(HIOP_DIR)/include -HIOP_LIB = -L$(HIOP_DIR)/lib -lhiop $(LAPACK_LIB) - -# CoDiPack -CODIPACK_DIR = @MFEM_DIR@/../CoDiPack -CODIPACK_OPT = -I$(CODIPACK_DIR) -CODIPACK_LIB = - -# GSLIB library -GSLIB_DIR = @MFEM_DIR@/../gslib/build -GSLIB_OPT = -I$(GSLIB_DIR)/include -GSLIB_LIB = -L$(GSLIB_DIR)/lib -lgs - -# CUDA library configuration -CUDA_OPT = -CUDA_LIB = -lcusparse - -# HIP library configuration -HIP_OPT = -HIP_LIB = -L$(HIP_DIR)/lib $(XLINKER)-rpath,$(HIP_DIR)/lib -lhipsparse - -# OCCA library configuration -OCCA_DIR = @MFEM_DIR@/../occa -OCCA_OPT = -I$(OCCA_DIR)/include -OCCA_LIB = $(XLINKER)-rpath,$(OCCA_DIR)/lib -L$(OCCA_DIR)/lib -locca - -# CALIPER library configuration -CALIPER_DIR = @MFEM_DIR@/../caliper -CALIPER_OPT = -I$(CALIPER_DIR)/include -CALIPER_LIB = $(XLINKER)-rpath,$(CALIPER_DIR)/lib64 -L$(CALIPER_DIR)/lib64 -lcaliper - -# BLITZ library configuration -BLITZ_DIR = @MFEM_DIR@/../blitz -BLITZ_OPT = -I$(BLITZ_DIR)/include -BLITZ_LIB = $(XLINKER)-rpath,$(BLITZ_DIR)/lib -L$(BLITZ_DIR)/lib -lblitz - -# ALGOIM library configuration -ALGOIM_DIR = @MFEM_DIR@/../algoim -ALGOIM_OPT = -I$(ALGOIM_DIR)/src $(BLITZ_OPT) -ALGOIM_LIB = $(BLITZ_LIB) - -# BENCHMARK library configuration -BENCHMARK_DIR = @MFEM_DIR@/../google-benchmark -BENCHMARK_OPT = -I$(BENCHMARK_DIR)/include -BENCHMARK_LIB = -L$(BENCHMARK_DIR)/lib -lbenchmark -lpthread - -# libCEED library configuration -CEED_DIR ?= @MFEM_DIR@/../libCEED -CEED_OPT = -I$(CEED_DIR)/include -CEED_LIB = $(XLINKER)-rpath,$(CEED_DIR)/lib -L$(CEED_DIR)/lib -lceed - -# RAJA library configuration -RAJA_DIR = @MFEM_DIR@/../raja -RAJA_OPT = -I$(RAJA_DIR)/include -ifdef CUB_DIR - RAJA_OPT += -I$(CUB_DIR) -endif -ifdef CAMP_DIR - RAJA_OPT += -I$(CAMP_DIR)/include -endif -RAJA_LIB = $(XLINKER)-rpath,$(RAJA_DIR)/lib -L$(RAJA_DIR)/lib -lRAJA - -# UMPIRE library configuration -UMPIRE_DIR = @MFEM_DIR@/../umpire -UMPIRE_OPT = -I$(UMPIRE_DIR)/include $(if $(CAMP_DIR), -I$(CAMP_DIR)/include) -UMPIRE_LIB = -L$(UMPIRE_DIR)/lib -lumpire - -# MKL CPardiso library configuration -MKL_CPARDISO_DIR ?= -MKL_MPI_WRAPPER ?= mkl_blacs_mpich_lp64 -MKL_LIBRARY_SUBDIR ?= lib -MKL_CPARDISO_OPT = -I$(MKL_CPARDISO_DIR)/include -MKL_CPARDISO_LIB = $(XLINKER)-rpath,$(MKL_CPARDISO_DIR)/$(MKL_LIBRARY_SUBDIR)\ - -L$(MKL_CPARDISO_DIR)/$(MKL_LIBRARY_SUBDIR) -l$(MKL_MPI_WRAPPER)\ - -lmkl_intel_lp64 -lmkl_sequential -lmkl_core - -# PARELAG library configuration -PARELAG_DIR = @MFEM_DIR@/../parelag -PARELAG_OPT = -I$(PARELAG_DIR)/src -I$(PARELAG_DIR)/build/src -PARELAG_LIB = -L$(PARELAG_DIR)/build/src -lParELAG - -# Enzyme configuration - -# If you want to enable automatic differentiation at compile time, use the -# options below, adapted to your configuration. To be more flexible, we -# recommend using the Enzyme plugin during link time optimization. One option is -# to add your options to the global compiler/linker flags like -# -# BASE_FLAGS += -flto -# CXX_XLINKER += -fuse-ld=lld -Wl,--lto-legacy-pass-manager\ -# -Wl,-mllvm=-load=$(ENZYME_DIR)/LLDEnzyme-$(ENZYME_VERSION).so -Wl, -# -ENZYME_DIR ?= @MFEM_DIR@/../enzyme -ENZYME_VERSION ?= 14 -ENZYME_OPT = -fno-experimental-new-pass-manager -Xclang -load -Xclang $(ENZYME_DIR)/ClangEnzyme-$(ENZYME_VERSION).so -ENZYME_LIB = "" - -# If YES, enable some informational messages -VERBOSE = NO - -# Optional build tag -MFEM_BUILD_TAG = $(shell uname -snm) \ No newline at end of file diff --git a/cpu/user.mk b/cpu/user.mk index b0d5806..54f03fb 100644 --- a/cpu/user.mk +++ b/cpu/user.mk @@ -1,547 +1,29 @@ -# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced -# at the Lawrence Livermore National Laboratory. All Rights reserved. See files -# LICENSE and NOTICE for details. LLNL-CODE-806117. -# -# This file is part of the MFEM library. For more information and source code -# availability visit https://mfem.org. -# -# MFEM is free software; you can redistribute it and/or modify it under the -# terms of the BSD-3 license. We welcome feedback and contributions, see file -# CONTRIBUTING.md for details. - -# This file describes the default MFEM build options. -# -# See the file INSTALL for description of these options. You can -# customize them below, or copy this file to user.mk and modify it. - - -# Some choices below are based on the OS type: -NOTMAC := $(subst Darwin,,$(shell uname -s)) - -ETAGS_BIN = $(shell command -v etags 2> /dev/null) -EGREP_BIN = $(shell command -v egrep 2> /dev/null) +VERBOSE = NO CXX = clang++ -MPICXX = mpicxx +MPICXX = OMPI_CXX=$(CXX) mpicxx -BASE_FLAGS = -std=c++11 +BASE_FLAGS = -std=c++17 OPTIM_FLAGS = -O3 $(BASE_FLAGS) -DEBUG_FLAGS = -g $(XCOMPILER)-Wall $(BASE_FLAGS) - -# Prefixes for passing flags to the compiler and linker when using CXX or MPICXX -CXX_XCOMPILER = -CXX_XLINKER = -Wl, - -# Destination location of make install -# PREFIX = $(HOME)/mfem -PREFIX = ./mfem -# Install program -INSTALL = /usr/bin/install STATIC = NO SHARED = YES -# CUDA configuration options -# -# If you set MFEM_USE_ENZYME=YES, CUDA_CXX has to be configured to use cuda with -# clang as its host compiler. -CUDA_CXX = nvcc -CUDA_ARCH = sm_60 -CUDA_FLAGS = -x=cu --expt-extended-lambda -arch=$(CUDA_ARCH) -# Prefixes for passing flags to the host compiler and linker when using CUDA_CXX -CUDA_XCOMPILER = -Xcompiler= -CUDA_XLINKER = -Xlinker= - -# HIP configuration options -HIP_CXX = hipcc -# The HIP_ARCH option specifies the AMD GPU processor, similar to CUDA_ARCH. For -# example: gfx600 (tahiti), gfx700 (kaveri), gfx701 (hawaii), gfx801 (carrizo), -# gfx900, gfx1010, etc. -HIP_ARCH = gfx900 -HIP_FLAGS = --amdgpu-target=$(HIP_ARCH) -HIP_XCOMPILER = -HIP_XLINKER = -Wl, - -# Flags for generating dependencies. -DEP_FLAGS = -MM -MT - -ifneq ($(NOTMAC),) - AR = ar - ARFLAGS = crv - RANLIB = ranlib - PICFLAG = $(XCOMPILER)-fPIC - SO_EXT = so - SO_VER = so.$(MFEM_VERSION_STRING) - BUILD_SOFLAGS = -shared $(XLINKER)-soname,libmfem.$(SO_VER) - BUILD_RPATH = $(XLINKER)-rpath,$(BUILD_REAL_DIR) - INSTALL_SOFLAGS = $(BUILD_SOFLAGS) - INSTALL_RPATH = $(XLINKER)-rpath,@MFEM_LIB_DIR@ -else - # Silence "has no symbols" warnings on Mac OS X - AR = ar - ARFLAGS = Scrv - RANLIB = ranlib -no_warning_for_no_symbols - PICFLAG = $(XCOMPILER)-fPIC - SO_EXT = dylib - SO_VER = $(MFEM_VERSION_STRING).dylib - MAKE_SOFLAGS = $(XLINKER)-dylib,-install_name,$(1)/libmfem.$(SO_VER),\ - -compatibility_version,$(MFEM_VERSION_STRING),\ - -current_version,$(MFEM_VERSION_STRING),\ - -undefined,dynamic_lookup - BUILD_SOFLAGS = $(subst $1 ,,$(call MAKE_SOFLAGS,$(BUILD_REAL_DIR))) - BUILD_RPATH = $(XLINKER)-undefined,dynamic_lookup - INSTALL_SOFLAGS = $(subst $1 ,,$(call MAKE_SOFLAGS,$(MFEM_LIB_DIR))) - INSTALL_RPATH = $(XLINKER)-undefined,dynamic_lookup - # Silence unused command line argument warnings when generating dependencies - # with mpicxx and clang - DEP_FLAGS := -Wno-unused-command-line-argument $(DEP_FLAGS) -endif - -# Set CXXFLAGS to overwrite the default selection of DEBUG_FLAGS/OPTIM_FLAGS -# CXXFLAGS = -O3 -march=native - -# Optional extra compile flags, in addition to CXXFLAGS: -# CPPFLAGS = - -# Library configurations: -# Note: symbols of the form @VAR@ will be replaced by $(VAR) in derived -# variables, like MFEM_FLAGS, defined in config.mk. - -# Command used to launch MPI jobs -MFEM_MPIEXEC = mpirun -MFEM_MPIEXEC_NP = -np -# Number of mpi tasks for parallel jobs -MFEM_MPI_NP = 4 - -# MFEM configuration options: YES/NO values, which are exported to config.mk and -# config.hpp. The values below are the defaults for generating the actual values -# in config.mk and config.hpp. - MFEM_USE_MPI = YES MFEM_USE_METIS = $(MFEM_USE_MPI) MFEM_USE_METIS_5 = YES -MFEM_DEBUG = YES -MFEM_USE_EXCEPTIONS = NO -MFEM_USE_ZLIB = NO +MFEM_DEBUG = NO MFEM_USE_LIBUNWIND = NO -MFEM_USE_LAPACK = NO -MFEM_THREAD_SAFE = NO -MFEM_USE_OPENMP = NO -MFEM_USE_LEGACY_OPENMP = NO MFEM_USE_MEMALLOC = YES MFEM_TIMER_TYPE = $(if $(NOTMAC),2,4) -MFEM_USE_SUNDIALS = NO -MFEM_USE_MESQUITE = NO -MFEM_USE_SUITESPARSE = NO -MFEM_USE_SUPERLU = NO -MFEM_USE_SUPERLU5 = NO -MFEM_USE_MUMPS = NO -MFEM_USE_STRUMPACK = NO -MFEM_USE_GINKGO = NO -MFEM_USE_AMGX = NO -MFEM_USE_GNUTLS = NO -MFEM_USE_NETCDF = NO -MFEM_USE_PETSC = NO -MFEM_USE_SLEPC = NO -MFEM_USE_MPFR = NO -MFEM_USE_SIDRE = NO -MFEM_USE_FMS = NO -MFEM_USE_CONDUIT = NO -MFEM_USE_PUMI = NO -MFEM_USE_HIOP = NO -MFEM_USE_GSLIB = NO -MFEM_USE_CUDA = NO -MFEM_USE_HIP = NO -MFEM_USE_RAJA = NO -MFEM_USE_OCCA = NO -MFEM_USE_CEED = NO -MFEM_USE_CALIPER = NO -MFEM_USE_ALGOIM = NO -MFEM_USE_UMPIRE = NO -MFEM_USE_SIMD = NO -MFEM_USE_ADIOS2 = NO -MFEM_USE_MKL_CPARDISO = NO -MFEM_USE_MOONOLITH = NO -MFEM_USE_ADFORWARD = NO -MFEM_USE_CODIPACK = NO -MFEM_USE_BENCHMARK = NO -MFEM_USE_PARELAG = NO -MFEM_USE_ENZYME = NO - -# MPI library compile and link flags -# These settings are used only when building MFEM with MPI + HIP -ifeq ($(MFEM_USE_MPI)$(MFEM_USE_HIP),YESYES) - # We determine MPI_DIR assuming $(MPICXX) is in $(MPI_DIR)/bin - MPI_DIR := $(patsubst %/,%,$(dir $(shell which $(MPICXX)))) - MPI_DIR := $(patsubst %/,%,$(dir $(MPI_DIR))) - MPI_OPT = -I$(MPI_DIR)/include - MPI_LIB = -L$(MPI_DIR)/lib $(XLINKER)-rpath,$(MPI_DIR)/lib -lmpi -endif +MFEM_USE_ENZYME = YES -# ROCM/HIP directory such that ROCM/HIP libraries like rocsparse and rocrand are -# found in $(HIP_DIR)/lib, usually as links. Typically, this directoory is of -# the form /opt/rocm-X.Y.Z which is called ROCM_PATH by hipconfig. -ifeq ($(MFEM_USE_HIP),YES) - HIP_DIR := $(patsubst %/,%,$(dir $(shell which $(HIP_CXX)))) - HIP_DIR := $(patsubst %/,%,$(dir $(HIP_DIR))) - ifeq (,$(wildcard $(HIP_DIR)/lib/librocsparse.*)) - HIP_DIR := $(shell hipconfig --rocmpath 2> /dev/null) - ifeq (,$(wildcard $(HIP_DIR)/lib/librocsparse.*)) - $(error Unable to determine HIP_DIR. Please set it manually.) - endif - endif -endif - -# Compile and link options for zlib. -ZLIB_DIR = -ZLIB_OPT = $(if $(ZLIB_DIR),-I$(ZLIB_DIR)/include) -ZLIB_LIB = $(if $(ZLIB_DIR),$(ZLIB_RPATH) -L$(ZLIB_DIR)/lib ,)-lz -ZLIB_RPATH = $(XLINKER)-rpath,$(ZLIB_DIR)/lib - -LIBUNWIND_OPT = -g -LIBUNWIND_LIB = $(if $(NOTMAC),-lunwind -ldl,) - -# HYPRE library configuration (needed to build the parallel version) HYPRE_DIR = /usr/local HYPRE_OPT = -I$(HYPRE_DIR)/include HYPRE_LIB = -L$(HYPRE_DIR)/lib -lHYPRE -ifeq (YES,$(MFEM_USE_CUDA)) - # This is only necessary when hypre is built with cuda: - HYPRE_LIB += -lcusparse -lcurand -endif -ifeq (YES,$(MFEM_USE_HIP)) - # This is only necessary when hypre is built with hip: - HYPRE_LIB += -L$(HIP_DIR)/lib $(XLINKER)-rpath,$(HIP_DIR)/lib\ - -lrocsparse -lrocrand -endif - -# METIS library configuration -ifeq ($(MFEM_USE_SUPERLU)$(MFEM_USE_STRUMPACK)$(MFEM_USE_MUMPS),NONONO) - ifeq ($(MFEM_USE_METIS_5),NO) - METIS_DIR = - METIS_OPT = - METIS_LIB = -L/usr/lib/x86_64-linux-gnu -lmetis - else - METIS_DIR = - METIS_OPT = - METIS_LIB = -L/usr/lib/x86_64-linux-gnu -lmetis - endif -else - # ParMETIS: currently needed by SuperLU or STRUMPACK. We assume that METIS 5 - # (included with ParMETIS) is installed in the same location. - # Starting with STRUMPACK v2.2.0, ParMETIS is an optional dependency while - # METIS is still required. - METIS_DIR = - METIS_OPT = - METIS_LIB = -L/usr/lib/x86_64-linux-gnu -lmetis -lparmetis - MFEM_USE_METIS_5 = YES -endif - -# LAPACK library configuration -LAPACK_OPT = -LAPACK_LIB = $(if $(NOTMAC),-llapack -lblas,-framework Accelerate) - -# OpenMP configuration -OPENMP_OPT = $(XCOMPILER)-fopenmp -OPENMP_LIB = - -# Used when MFEM_TIMER_TYPE = 2 -POSIX_CLOCKS_LIB = -lrt - -# SUNDIALS library configuration -# For sundials_nvecmpiplusx and nvecparallel remember to build with MPI_ENABLE=ON -# and modify cmake variables for hypre for sundials -SUNDIALS_DIR = @MFEM_DIR@/../sundials-5.0.0/instdir -SUNDIALS_OPT = -I$(SUNDIALS_DIR)/include -SUNDIALS_LIBDIR = $(wildcard $(SUNDIALS_DIR)/lib*) -SUNDIALS_LIB = $(XLINKER)-rpath,$(SUNDIALS_LIBDIR) -L$(SUNDIALS_LIBDIR)\ - -lsundials_arkode -lsundials_cvodes -lsundials_nvecserial -lsundials_kinsol - -ifeq ($(MFEM_USE_MPI),YES) - SUNDIALS_LIB += -lsundials_nvecparallel -lsundials_nvecmpiplusx -endif -ifeq ($(MFEM_USE_CUDA),YES) - SUNDIALS_LIB += -lsundials_nveccuda -endif -# If SUNDIALS was built with KLU: -# MFEM_USE_SUITESPARSE = YES - -# MESQUITE library configuration -MESQUITE_DIR = @MFEM_DIR@/../mesquite-2.99 -MESQUITE_OPT = -I$(MESQUITE_DIR)/include -MESQUITE_LIB = -L$(MESQUITE_DIR)/lib -lmesquite - -# SuiteSparse library configuration -LIB_RT = $(if $(NOTMAC),-lrt,) -SUITESPARSE_DIR = @MFEM_DIR@/../SuiteSparse -SUITESPARSE_OPT = -I$(SUITESPARSE_DIR)/include -SUITESPARSE_LIB = $(XLINKER)-rpath,$(SUITESPARSE_DIR)/lib\ - -L$(SUITESPARSE_DIR)/lib -lklu -lbtf -lumfpack -lcholmod -lcolamd -lamd -lcamd\ - -lccolamd -lsuitesparseconfig $(LIB_RT) $(METIS_LIB) $(LAPACK_LIB) - -# SuperLU library configuration -ifeq ($(MFEM_USE_SUPERLU5),YES) - SUPERLU_DIR = @MFEM_DIR@/../SuperLU_DIST_5.1.0 - SUPERLU_OPT = -I$(SUPERLU_DIR)/include - SUPERLU_LIB = $(XLINKER)-rpath,$(SUPERLU_DIR)/lib -L$(SUPERLU_DIR)/lib\ - -lsuperlu_dist_5.1.0 -else - SUPERLU_DIR = @MFEM_DIR@/../SuperLU_DIST_6.3.1 - SUPERLU_OPT = -I$(SUPERLU_DIR)/include - SUPERLU_LIB = $(XLINKER)-rpath,$(SUPERLU_DIR)/lib64 -L$(SUPERLU_DIR)/lib64\ - -lsuperlu_dist -lblas -endif - -# SCOTCH library configuration (required by STRUMPACK <= v2.1.0, optional in -# STRUMPACK >= v2.2.0) -SCOTCH_DIR = @MFEM_DIR@/../scotch_6.0.4 -SCOTCH_OPT = -I$(SCOTCH_DIR)/include -SCOTCH_LIB = -L$(SCOTCH_DIR)/lib -lptscotch -lptscotcherr -lscotch -lscotcherr\ - -lpthread - -# SCALAPACK library configuration (required by STRUMPACK and MUMPS) -SCALAPACK_DIR = @MFEM_DIR@/../scalapack-2.0.2 -SCALAPACK_OPT = -I$(SCALAPACK_DIR)/SRC -SCALAPACK_LIB = -L$(SCALAPACK_DIR)/lib -lscalapack $(LAPACK_LIB) - -# MPI Fortran library, needed e.g. by STRUMPACK or MUMPS -# MPICH: -MPI_FORTRAN_LIB = -lmpifort -# OpenMPI: -# MPI_FORTRAN_LIB = -lmpi_mpifh -# Additional Fortan library: -# MPI_FORTRAN_LIB += -lgfortran - -# MUMPS library configuration -MUMPS_DIR = @MFEM_DIR@/../MUMPS_5.2.0 -MUMPS_OPT = -I$(MUMPS_DIR)/include -MUMPS_LIB = $(XLINKER)-rpath,$(MUMPS_DIR)/lib -L$(MUMPS_DIR)/lib -ldmumps\ - -lmumps_common -lpord $(SCALAPACK_LIB) $(LAPACK_LIB) $(MPI_FORTRAN_LIB) -# STRUMPACK library configuration -STRUMPACK_DIR = @MFEM_DIR@/../STRUMPACK-build -STRUMPACK_OPT = -I$(STRUMPACK_DIR)/include $(SCOTCH_OPT) -# If STRUMPACK was build with OpenMP support, the following may be need: -# STRUMPACK_OPT += $(OPENMP_OPT) -STRUMPACK_LIB = -L$(STRUMPACK_DIR)/lib -lstrumpack $(MPI_FORTRAN_LIB)\ - $(SCOTCH_LIB) $(SCALAPACK_LIB) - -# Ginkgo library configuration (currently not needed) -GINKGO_DIR = @MFEM_DIR@/../ginkgo/install -GINKGO_BUILD_TYPE=Release -ifeq ($(MFEM_USE_GINKGO),YES) - BASE_FLAGS = -std=c++14 -endif -GINKGO_OPT = -isystem $(GINKGO_DIR)/include -GINKGO_LIB_DIR = $(sort $(dir $(wildcard $(GINKGO_DIR)/lib*/libginkgo*.a $(GINKGO_DIR)/lib*/libginkgo*.so $(GINKGO_DIR)/lib*/libginkgo*.dylib $(GINKGO_DIR)/lib*/libginkgo*.dll))) -ALL_GINKGO_LIBS_DEBUG = $(notdir $(basename $(wildcard $(GINKGO_DIR)/lib*/libginkgo*d.a $(GINKGO_DIR)/lib*/libginkgo*d.so $(GINKGO_DIR)/lib*/libginkgo*d.dylib $(GINKGO_DIR)/lib*/libginkgo*d.dll))) -ALL_GINKGO_LIBS = $(notdir $(basename $(wildcard $(GINKGO_DIR)/lib*/libginkgo*.a $(GINKGO_DIR)/lib*/libginkgo*.so $(GINKGO_DIR)/lib*/libginkgo*.dylib $(GINKGO_DIR)/lib*/libginkgo*.dll))) -ALL_GINKGO_LIBS_RELEASE = $(filter-out $(ALL_GINKGO_LIBS_DEBUG),$(ALL_GINKGO_LIBS)) -GINKGO_LINK = $(subst libginkgo,-lginkgo,$(ALL_GINKGO_LIBS_RELEASE)) -ifeq ($(GINKGO_BUILD_TYPE),Debug) - ifneq (,$(ALL_GINKGO_LIBS_DEBUG)) - GINKGO_LINK = $(subst libginkgo,-lginkgo,$(ALL_GINKGO_LIBS_DEBUG)) - endif -else -endif -GINKGO_LIB = $(XLINKER)-rpath,$(GINKGO_LIB_DIR) -L$(GINKGO_LIB_DIR) $(GINKGO_LINK) - -# AmgX library configuration -AMGX_DIR = @MFEM_DIR@/../amgx -AMGX_OPT = -I$(AMGX_DIR)/include -AMGX_LIB = -lcusparse -lcusolver -lcublas -lnvToolsExt -L$(AMGX_DIR)/lib -lamgx - -# GnuTLS library configuration -GNUTLS_OPT = -GNUTLS_LIB = -lgnutls - -# NetCDF library configuration -NETCDF_DIR = $(HOME)/local -HDF5_DIR = $(HOME)/local -NETCDF_OPT = -I$(NETCDF_DIR)/include -I$(HDF5_DIR)/include $(ZLIB_OPT) -NETCDF_LIB = $(XLINKER)-rpath,$(NETCDF_DIR)/lib -L$(NETCDF_DIR)/lib\ - $(XLINKER)-rpath,$(HDF5_DIR)/lib -L$(HDF5_DIR)/lib\ - -lnetcdf -lhdf5_hl -lhdf5 $(ZLIB_LIB) - -# PETSc library configuration (version greater or equal to 3.8 or the dev branch) -PETSC_ARCH := arch-linux2-c-debug -PETSC_DIR := $(MFEM_DIR)/../petsc/$(PETSC_ARCH) -PETSC_VARS := $(PETSC_DIR)/lib/petsc/conf/petscvariables -PETSC_FOUND := $(if $(wildcard $(PETSC_VARS)),YES,) -PETSC_INC_VAR = PETSC_CC_INCLUDES -PETSC_LIB_VAR = PETSC_EXTERNAL_LIB_BASIC -ifeq ($(PETSC_FOUND),YES) - PETSC_OPT := $(shell sed -n "s/$(PETSC_INC_VAR) = *//p" $(PETSC_VARS)) - PETSC_DEP := $(shell sed -n "s/$(PETSC_LIB_VAR) = *//p" $(PETSC_VARS)) - PETSC_LIB = $(XLINKER)-rpath,$(abspath $(PETSC_DIR))/lib\ - -L$(abspath $(PETSC_DIR))/lib -lpetsc\ - $(subst $(CXX_XLINKER),$(XLINKER),$(PETSC_DEP)) -endif - -SLEPC_DIR := $(MFEM_DIR)/../slepc -SLEPC_VARS := $(SLEPC_DIR)/lib/slepc/conf/slepc_variables -SLEPC_FOUND := $(if $(wildcard $(SLEPC_VARS)),YES,) -SLEPC_INC_VAR = SLEPC_INCLUDE -SLEPC_LIB_VAR = SLEPC_EXTERNAL_LIB -ifeq ($(SLEPC_FOUND),YES) - SLEPC_OPT := $(shell sed -n "s/$(SLEPC_INC_VAR) *= *//p" $(SLEPC_VARS)) - # Some additional external libraries might be defined in this file - -include ${SLEPC_DIR}/${PETSC_ARCH}/lib/slepc/conf/slepcvariables - SLEPC_DEP := $(shell sed -n "s/$(SLEPC_LIB_VAR) *= *//p" $(SLEPC_VARS)) - SLEPC_LIB = $(XLINKER)-rpath,$(abspath $(SLEPC_DIR))/$(PETSC_ARCH)/lib\ - -L$(abspath $(SLEPC_DIR))/$(PETSC_ARCH)/lib -lslepc\ - $(subst $(CXX_XLINKER),$(XLINKER),$(SLEPC_DEP)) -endif - -ifeq ($(MFEM_USE_MOONOLITH),YES) - include $(MOONOLITH_DIR)/config/moonolith-config.makefile - MOONOLITH_LIB=$(MOONOLITH_LIBRARIES) -endif - -# MPFR library configuration -MPFR_OPT = -MPFR_LIB = -lmpfr - -# FMS and required libraries configuration -FMS_DIR = $(MFEM_DIR)/../fms -FMS_OPT = -I$(FMS_DIR)/include -FMS_LIB = -Wl,-rpath,$(FMS_DIR)/lib -L$(FMS_DIR)/lib -lfms - -# Conduit and required libraries configuration -CONDUIT_DIR = @MFEM_DIR@/../conduit -CONDUIT_OPT = -I$(CONDUIT_DIR)/include/conduit -CONDUIT_LIB = \ - $(XLINKER)-rpath,$(CONDUIT_DIR)/lib -L$(CONDUIT_DIR)/lib \ - -lconduit -lconduit_relay -lconduit_blueprint -ldl - -# Check if Conduit was built with hdf5 support, by looking -# for the relay hdf5 header -CONDUIT_HDF5_HEADER=$(CONDUIT_DIR)/include/conduit/conduit_relay_hdf5.hpp -ifneq (,$(wildcard $(CONDUIT_HDF5_HEADER))) - CONDUIT_OPT += -I$(HDF5_DIR)/include - CONDUIT_LIB += $(XLINKER)-rpath,$(HDF5_DIR)/lib -L$(HDF5_DIR)/lib \ - -lhdf5 $(ZLIB_LIB) -endif - -# Sidre and required libraries configuration -# Be sure to check the HDF5_DIR (set above) is correct -SIDRE_DIR = @MFEM_DIR@/../axom -SIDRE_OPT = -I$(SIDRE_DIR)/include -I$(CONDUIT_DIR)/include/conduit\ - -I$(HDF5_DIR)/include -SIDRE_LIB = \ - $(XLINKER)-rpath,$(SIDRE_DIR)/lib -L$(SIDRE_DIR)/lib \ - $(XLINKER)-rpath,$(CONDUIT_DIR)/lib -L$(CONDUIT_DIR)/lib \ - $(XLINKER)-rpath,$(HDF5_DIR)/lib -L$(HDF5_DIR)/lib \ - -laxom -lconduit -lconduit_relay -lconduit_blueprint -lhdf5 $(ZLIB_LIB) -ldl - -# PUMI -# Note that PUMI_DIR is needed -- it is used to check for gmi_sim.h -PUMI_DIR = @MFEM_DIR@/../pumi-2.1.0 -PUMI_OPT = -I$(PUMI_DIR)/include -PUMI_LIB = -L$(PUMI_DIR)/lib -lpumi -lcrv -lma -lmds -lapf -lpcu -lgmi -lparma\ - -llion -lmth -lapf_zoltan -lspr - -# HIOP -HIOP_DIR = @MFEM_DIR@/../hiop/install -HIOP_OPT = -I$(HIOP_DIR)/include -HIOP_LIB = -L$(HIOP_DIR)/lib -lhiop $(LAPACK_LIB) - -# CoDiPack -CODIPACK_DIR = @MFEM_DIR@/../CoDiPack -CODIPACK_OPT = -I$(CODIPACK_DIR) -CODIPACK_LIB = - -# GSLIB library -GSLIB_DIR = @MFEM_DIR@/../gslib/build -GSLIB_OPT = -I$(GSLIB_DIR)/include -GSLIB_LIB = -L$(GSLIB_DIR)/lib -lgs - -# CUDA library configuration -CUDA_OPT = -CUDA_LIB = -lcusparse - -# HIP library configuration -HIP_OPT = -HIP_LIB = -L$(HIP_DIR)/lib $(XLINKER)-rpath,$(HIP_DIR)/lib -lhipsparse - -# OCCA library configuration -OCCA_DIR = @MFEM_DIR@/../occa -OCCA_OPT = -I$(OCCA_DIR)/include -OCCA_LIB = $(XLINKER)-rpath,$(OCCA_DIR)/lib -L$(OCCA_DIR)/lib -locca - -# CALIPER library configuration -CALIPER_DIR = @MFEM_DIR@/../caliper -CALIPER_OPT = -I$(CALIPER_DIR)/include -CALIPER_LIB = $(XLINKER)-rpath,$(CALIPER_DIR)/lib64 -L$(CALIPER_DIR)/lib64 -lcaliper - -# BLITZ library configuration -BLITZ_DIR = @MFEM_DIR@/../blitz -BLITZ_OPT = -I$(BLITZ_DIR)/include -BLITZ_LIB = $(XLINKER)-rpath,$(BLITZ_DIR)/lib -L$(BLITZ_DIR)/lib -lblitz - -# ALGOIM library configuration -ALGOIM_DIR = @MFEM_DIR@/../algoim -ALGOIM_OPT = -I$(ALGOIM_DIR)/src $(BLITZ_OPT) -ALGOIM_LIB = $(BLITZ_LIB) - -# BENCHMARK library configuration -BENCHMARK_DIR = @MFEM_DIR@/../google-benchmark -BENCHMARK_OPT = -I$(BENCHMARK_DIR)/include -BENCHMARK_LIB = -L$(BENCHMARK_DIR)/lib -lbenchmark -lpthread - -# libCEED library configuration -CEED_DIR ?= @MFEM_DIR@/../libCEED -CEED_OPT = -I$(CEED_DIR)/include -CEED_LIB = $(XLINKER)-rpath,$(CEED_DIR)/lib -L$(CEED_DIR)/lib -lceed - -# RAJA library configuration -RAJA_DIR = @MFEM_DIR@/../raja -RAJA_OPT = -I$(RAJA_DIR)/include -ifdef CUB_DIR - RAJA_OPT += -I$(CUB_DIR) -endif -ifdef CAMP_DIR - RAJA_OPT += -I$(CAMP_DIR)/include -endif -RAJA_LIB = $(XLINKER)-rpath,$(RAJA_DIR)/lib -L$(RAJA_DIR)/lib -lRAJA - -# UMPIRE library configuration -UMPIRE_DIR = @MFEM_DIR@/../umpire -UMPIRE_OPT = -I$(UMPIRE_DIR)/include $(if $(CAMP_DIR), -I$(CAMP_DIR)/include) -UMPIRE_LIB = -L$(UMPIRE_DIR)/lib -lumpire - -# MKL CPardiso library configuration -MKL_CPARDISO_DIR ?= -MKL_MPI_WRAPPER ?= mkl_blacs_mpich_lp64 -MKL_LIBRARY_SUBDIR ?= lib -MKL_CPARDISO_OPT = -I$(MKL_CPARDISO_DIR)/include -MKL_CPARDISO_LIB = $(XLINKER)-rpath,$(MKL_CPARDISO_DIR)/$(MKL_LIBRARY_SUBDIR)\ - -L$(MKL_CPARDISO_DIR)/$(MKL_LIBRARY_SUBDIR) -l$(MKL_MPI_WRAPPER)\ - -lmkl_intel_lp64 -lmkl_sequential -lmkl_core - -# PARELAG library configuration -PARELAG_DIR = @MFEM_DIR@/../parelag -PARELAG_OPT = -I$(PARELAG_DIR)/src -I$(PARELAG_DIR)/build/src -PARELAG_LIB = -L$(PARELAG_DIR)/build/src -lParELAG - -# Enzyme configuration - -# If you want to enable automatic differentiation at compile time, use the -# options below, adapted to your configuration. To be more flexible, we -# recommend using the Enzyme plugin during link time optimization. One option is -# to add your options to the global compiler/linker flags like -# -# BASE_FLAGS += -flto -# CXX_XLINKER += -fuse-ld=lld -Wl,--lto-legacy-pass-manager\ -# -Wl,-mllvm=-load=$(ENZYME_DIR)/LLDEnzyme-$(ENZYME_VERSION).so -Wl, -# -ENZYME_DIR ?= @MFEM_DIR@/../enzyme -ENZYME_VERSION ?= 14 -ENZYME_OPT = -fno-experimental-new-pass-manager -Xclang -load -Xclang $(ENZYME_DIR)/ClangEnzyme-$(ENZYME_VERSION).so -ENZYME_LIB = "" - -# If YES, enable some informational messages -VERBOSE = NO +METIS_DIR = +METIS_OPT = +METIS_LIB = -L/usr/lib/x86_64-linux-gnu -lmetis -lparmetis -# Optional build tag -MFEM_BUILD_TAG = $(shell uname -snm) \ No newline at end of file +ENZYME_DIR = /usr/local diff --git a/cuda-tpls/Dockerfile b/cuda-tpls/Dockerfile deleted file mode 100644 index c3a6b14..0000000 --- a/cuda-tpls/Dockerfile +++ /dev/null @@ -1,30 +0,0 @@ -FROM ghcr.io/mfem/containers/cuda:latest - -ARG num_jobs -ARG cuda_arch_sm - -USER root - -WORKDIR /opt/archives - -RUN git clone -b v2.4.0 --single-branch --depth=1 https://github.com/NVIDIA/AMGX.git && \ - cd AMGX && \ - git submodule update --init --recursive -COPY --chown=euler:euler amgx_disable_examples.patch /opt/archives/AMGX/amgx_disable_examples.patch -RUN cd AMGX && git apply amgx_disable_examples.patch && \ - mkdir build && cd build && \ - cmake .. \ - -DCMAKE_INSTALL_PREFIX=/usr/local \ - -DBUILD_SHARED_LIBS=ON \ - -DCUDA_ARCH=${cuda_arch_sm} \ - -DCMAKE_BUILD_TYPE=Release && \ - make -j ${num_jobs} && make install - -USER euler -WORKDIR /home/euler - -ENV PATH=$PATH:/usr/local/cuda/bin - -COPY --chown=euler:euler user.mk mfem/config/user.mk -RUN sed -i 's/CUDA_ARCH_SM_PLACEHOLDER/sm_${cuda_arch_sm}/' mfem/config/user.mk -RUN cd mfem && make config && make -j ${num_jobs} && cd examples && make ex1 && make ex1p \ No newline at end of file diff --git a/cuda-tpls/amgx_disable_examples.patch b/cuda-tpls/amgx_disable_examples.patch deleted file mode 100644 index f3daae9..0000000 --- a/cuda-tpls/amgx_disable_examples.patch +++ /dev/null @@ -1,32 +0,0 @@ -diff --git a/CMakeLists.txt b/CMakeLists.txt -index 7bc0f1b..b6576fc 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -314,5 +314,5 @@ install(TARGETS amgxsh DESTINATION "lib") - #export(TARGETS amgxsh FILE ${CMAKE_CURRENT_SOURCE_DIR}/amgxsh.cmake) - - # build examples --add_subdirectory(examples) -+# add_subdirectory(examples) - -diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt -index b7fbef7..5d2b0f5 100644 ---- a/src/CMakeLists.txt -+++ b/src/CMakeLists.txt -@@ -72,6 +72,10 @@ install(FILES - - set(AMGX_INCLUDES ../include) - -+ -+option(ENABLE_TESTS "Enable tests" OFF) -+if(ENABLE_TESTS) -+ - set(tests_all ${tests_all} testframework.cu test_utils.cu unit_test.cu) - - add_library(amgx_tests_libs OBJECT ${tests_all}) -@@ -118,4 +122,4 @@ endif(MPI_FOUND) - if(NOT ${CMAKE_BUILD_TYPE} MATCHES "Release") - install(TARGETS amgx_tests_launcher DESTINATION "lib/tests") - endif(NOT ${CMAKE_BUILD_TYPE} MATCHES "Release") -- -+endif() diff --git a/cuda-tpls/user.mk b/cuda-tpls/user.mk deleted file mode 100644 index f24543f..0000000 --- a/cuda-tpls/user.mk +++ /dev/null @@ -1,547 +0,0 @@ -# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced -# at the Lawrence Livermore National Laboratory. All Rights reserved. See files -# LICENSE and NOTICE for details. LLNL-CODE-806117. -# -# This file is part of the MFEM library. For more information and source code -# availability visit https://mfem.org. -# -# MFEM is free software; you can redistribute it and/or modify it under the -# terms of the BSD-3 license. We welcome feedback and contributions, see file -# CONTRIBUTING.md for details. - -# This file describes the default MFEM build options. -# -# See the file INSTALL for description of these options. You can -# customize them below, or copy this file to user.mk and modify it. - - -# Some choices below are based on the OS type: -NOTMAC := $(subst Darwin,,$(shell uname -s)) - -ETAGS_BIN = $(shell command -v etags 2> /dev/null) -EGREP_BIN = $(shell command -v egrep 2> /dev/null) - -CXX = clang++ -MPICXX = mpicxx - -BASE_FLAGS = -std=c++11 -OPTIM_FLAGS = -O3 $(BASE_FLAGS) -DEBUG_FLAGS = -g $(XCOMPILER)-Wall $(BASE_FLAGS) - -# Prefixes for passing flags to the compiler and linker when using CXX or MPICXX -CXX_XCOMPILER = -CXX_XLINKER = -Wl, - -# Destination location of make install -# PREFIX = $(HOME)/mfem -PREFIX = ./mfem -# Install program -INSTALL = /usr/bin/install - -STATIC = NO -SHARED = YES - -# CUDA configuration options -# -# If you set MFEM_USE_ENZYME=YES, CUDA_CXX has to be configured to use cuda with -# clang as its host compiler. -CUDA_CXX = nvcc -CUDA_ARCH = CUDA_ARCH_SM_PLACEHOLDER -CUDA_FLAGS = -x=cu --expt-extended-lambda -arch=$(CUDA_ARCH) -# Prefixes for passing flags to the host compiler and linker when using CUDA_CXX -CUDA_XCOMPILER = -Xcompiler= -CUDA_XLINKER = -Xlinker= - -# HIP configuration options -HIP_CXX = hipcc -# The HIP_ARCH option specifies the AMD GPU processor, similar to CUDA_ARCH. For -# example: gfx600 (tahiti), gfx700 (kaveri), gfx701 (hawaii), gfx801 (carrizo), -# gfx900, gfx1010, etc. -HIP_ARCH = gfx900 -HIP_FLAGS = --amdgpu-target=$(HIP_ARCH) -HIP_XCOMPILER = -HIP_XLINKER = -Wl, - -# Flags for generating dependencies. -DEP_FLAGS = -MM -MT - -ifneq ($(NOTMAC),) - AR = ar - ARFLAGS = crv - RANLIB = ranlib - PICFLAG = $(XCOMPILER)-fPIC - SO_EXT = so - SO_VER = so.$(MFEM_VERSION_STRING) - BUILD_SOFLAGS = -shared $(XLINKER)-soname,libmfem.$(SO_VER) - BUILD_RPATH = $(XLINKER)-rpath,$(BUILD_REAL_DIR) - INSTALL_SOFLAGS = $(BUILD_SOFLAGS) - INSTALL_RPATH = $(XLINKER)-rpath,@MFEM_LIB_DIR@ -else - # Silence "has no symbols" warnings on Mac OS X - AR = ar - ARFLAGS = Scrv - RANLIB = ranlib -no_warning_for_no_symbols - PICFLAG = $(XCOMPILER)-fPIC - SO_EXT = dylib - SO_VER = $(MFEM_VERSION_STRING).dylib - MAKE_SOFLAGS = $(XLINKER)-dylib,-install_name,$(1)/libmfem.$(SO_VER),\ - -compatibility_version,$(MFEM_VERSION_STRING),\ - -current_version,$(MFEM_VERSION_STRING),\ - -undefined,dynamic_lookup - BUILD_SOFLAGS = $(subst $1 ,,$(call MAKE_SOFLAGS,$(BUILD_REAL_DIR))) - BUILD_RPATH = $(XLINKER)-undefined,dynamic_lookup - INSTALL_SOFLAGS = $(subst $1 ,,$(call MAKE_SOFLAGS,$(MFEM_LIB_DIR))) - INSTALL_RPATH = $(XLINKER)-undefined,dynamic_lookup - # Silence unused command line argument warnings when generating dependencies - # with mpicxx and clang - DEP_FLAGS := -Wno-unused-command-line-argument $(DEP_FLAGS) -endif - -# Set CXXFLAGS to overwrite the default selection of DEBUG_FLAGS/OPTIM_FLAGS -# CXXFLAGS = -O3 -march=native - -# Optional extra compile flags, in addition to CXXFLAGS: -# CPPFLAGS = - -# Library configurations: -# Note: symbols of the form @VAR@ will be replaced by $(VAR) in derived -# variables, like MFEM_FLAGS, defined in config.mk. - -# Command used to launch MPI jobs -MFEM_MPIEXEC = mpirun -MFEM_MPIEXEC_NP = -np -# Number of mpi tasks for parallel jobs -MFEM_MPI_NP = 4 - -# MFEM configuration options: YES/NO values, which are exported to config.mk and -# config.hpp. The values below are the defaults for generating the actual values -# in config.mk and config.hpp. - -MFEM_USE_MPI = YES -MFEM_USE_METIS = $(MFEM_USE_MPI) -MFEM_USE_METIS_5 = YES -MFEM_DEBUG = NO -MFEM_USE_EXCEPTIONS = NO -MFEM_USE_ZLIB = NO -MFEM_USE_LIBUNWIND = NO -MFEM_USE_LAPACK = NO -MFEM_THREAD_SAFE = NO -MFEM_USE_OPENMP = NO -MFEM_USE_LEGACY_OPENMP = NO -MFEM_USE_MEMALLOC = YES -MFEM_TIMER_TYPE = $(if $(NOTMAC),2,4) -MFEM_USE_SUNDIALS = NO -MFEM_USE_MESQUITE = NO -MFEM_USE_SUITESPARSE = NO -MFEM_USE_SUPERLU = NO -MFEM_USE_SUPERLU5 = NO -MFEM_USE_MUMPS = NO -MFEM_USE_STRUMPACK = NO -MFEM_USE_GINKGO = NO -MFEM_USE_AMGX = YES -MFEM_USE_GNUTLS = NO -MFEM_USE_NETCDF = NO -MFEM_USE_PETSC = NO -MFEM_USE_SLEPC = NO -MFEM_USE_MPFR = NO -MFEM_USE_SIDRE = NO -MFEM_USE_FMS = NO -MFEM_USE_CONDUIT = NO -MFEM_USE_PUMI = NO -MFEM_USE_HIOP = NO -MFEM_USE_GSLIB = NO -MFEM_USE_CUDA = YES -MFEM_USE_HIP = NO -MFEM_USE_RAJA = NO -MFEM_USE_OCCA = NO -MFEM_USE_CEED = NO -MFEM_USE_CALIPER = NO -MFEM_USE_ALGOIM = NO -MFEM_USE_UMPIRE = NO -MFEM_USE_SIMD = NO -MFEM_USE_ADIOS2 = NO -MFEM_USE_MKL_CPARDISO = NO -MFEM_USE_MOONOLITH = NO -MFEM_USE_ADFORWARD = NO -MFEM_USE_CODIPACK = NO -MFEM_USE_BENCHMARK = NO -MFEM_USE_PARELAG = NO -MFEM_USE_ENZYME = NO - -# MPI library compile and link flags -# These settings are used only when building MFEM with MPI + HIP -ifeq ($(MFEM_USE_MPI)$(MFEM_USE_HIP),YESYES) - # We determine MPI_DIR assuming $(MPICXX) is in $(MPI_DIR)/bin - MPI_DIR := $(patsubst %/,%,$(dir $(shell which $(MPICXX)))) - MPI_DIR := $(patsubst %/,%,$(dir $(MPI_DIR))) - MPI_OPT = -I$(MPI_DIR)/include - MPI_LIB = -L$(MPI_DIR)/lib $(XLINKER)-rpath,$(MPI_DIR)/lib -lmpi -endif - -# ROCM/HIP directory such that ROCM/HIP libraries like rocsparse and rocrand are -# found in $(HIP_DIR)/lib, usually as links. Typically, this directoory is of -# the form /opt/rocm-X.Y.Z which is called ROCM_PATH by hipconfig. -ifeq ($(MFEM_USE_HIP),YES) - HIP_DIR := $(patsubst %/,%,$(dir $(shell which $(HIP_CXX)))) - HIP_DIR := $(patsubst %/,%,$(dir $(HIP_DIR))) - ifeq (,$(wildcard $(HIP_DIR)/lib/librocsparse.*)) - HIP_DIR := $(shell hipconfig --rocmpath 2> /dev/null) - ifeq (,$(wildcard $(HIP_DIR)/lib/librocsparse.*)) - $(error Unable to determine HIP_DIR. Please set it manually.) - endif - endif -endif - -# Compile and link options for zlib. -ZLIB_DIR = -ZLIB_OPT = $(if $(ZLIB_DIR),-I$(ZLIB_DIR)/include) -ZLIB_LIB = $(if $(ZLIB_DIR),$(ZLIB_RPATH) -L$(ZLIB_DIR)/lib ,)-lz -ZLIB_RPATH = $(XLINKER)-rpath,$(ZLIB_DIR)/lib - -LIBUNWIND_OPT = -g -LIBUNWIND_LIB = $(if $(NOTMAC),-lunwind -ldl,) - -# HYPRE library configuration (needed to build the parallel version) -HYPRE_DIR = /usr/local -HYPRE_OPT = -I$(HYPRE_DIR)/include -HYPRE_LIB = -L$(HYPRE_DIR)/lib -lHYPRE -ifeq (YES,$(MFEM_USE_CUDA)) - # This is only necessary when hypre is built with cuda: - HYPRE_LIB += -lcusparse -lcurand -endif -ifeq (YES,$(MFEM_USE_HIP)) - # This is only necessary when hypre is built with hip: - HYPRE_LIB += -L$(HIP_DIR)/lib $(XLINKER)-rpath,$(HIP_DIR)/lib\ - -lrocsparse -lrocrand -endif - -# METIS library configuration -ifeq ($(MFEM_USE_SUPERLU)$(MFEM_USE_STRUMPACK)$(MFEM_USE_MUMPS),NONONO) - ifeq ($(MFEM_USE_METIS_5),NO) - METIS_DIR = /usr/local - METIS_OPT = - METIS_LIB = -L$(METIS_DIR) -lmetis - else - METIS_DIR = /usr/local - METIS_OPT = -I$(METIS_DIR)/include - METIS_LIB = -L$(METIS_DIR)/lib -lmetis - endif -else - # ParMETIS: currently needed by SuperLU or STRUMPACK. We assume that METIS 5 - # (included with ParMETIS) is installed in the same location. - # Starting with STRUMPACK v2.2.0, ParMETIS is an optional dependency while - # METIS is still required. - METIS_DIR = @MFEM_DIR@/../parmetis-4.0.3 - METIS_OPT = -I$(METIS_DIR)/include - METIS_LIB = -L$(METIS_DIR)/lib -lparmetis -lmetis - MFEM_USE_METIS_5 = YES -endif - -# LAPACK library configuration -LAPACK_OPT = -LAPACK_LIB = $(if $(NOTMAC),-llapack -lblas,-framework Accelerate) - -# OpenMP configuration -OPENMP_OPT = $(XCOMPILER)-fopenmp -OPENMP_LIB = - -# Used when MFEM_TIMER_TYPE = 2 -POSIX_CLOCKS_LIB = -lrt - -# SUNDIALS library configuration -# For sundials_nvecmpiplusx and nvecparallel remember to build with MPI_ENABLE=ON -# and modify cmake variables for hypre for sundials -SUNDIALS_DIR = @MFEM_DIR@/../sundials-5.0.0/instdir -SUNDIALS_OPT = -I$(SUNDIALS_DIR)/include -SUNDIALS_LIBDIR = $(wildcard $(SUNDIALS_DIR)/lib*) -SUNDIALS_LIB = $(XLINKER)-rpath,$(SUNDIALS_LIBDIR) -L$(SUNDIALS_LIBDIR)\ - -lsundials_arkode -lsundials_cvodes -lsundials_nvecserial -lsundials_kinsol - -ifeq ($(MFEM_USE_MPI),YES) - SUNDIALS_LIB += -lsundials_nvecparallel -lsundials_nvecmpiplusx -endif -ifeq ($(MFEM_USE_CUDA),YES) - SUNDIALS_LIB += -lsundials_nveccuda -endif -# If SUNDIALS was built with KLU: -# MFEM_USE_SUITESPARSE = YES - -# MESQUITE library configuration -MESQUITE_DIR = @MFEM_DIR@/../mesquite-2.99 -MESQUITE_OPT = -I$(MESQUITE_DIR)/include -MESQUITE_LIB = -L$(MESQUITE_DIR)/lib -lmesquite - -# SuiteSparse library configuration -LIB_RT = $(if $(NOTMAC),-lrt,) -SUITESPARSE_DIR = @MFEM_DIR@/../SuiteSparse -SUITESPARSE_OPT = -I$(SUITESPARSE_DIR)/include -SUITESPARSE_LIB = $(XLINKER)-rpath,$(SUITESPARSE_DIR)/lib\ - -L$(SUITESPARSE_DIR)/lib -lklu -lbtf -lumfpack -lcholmod -lcolamd -lamd -lcamd\ - -lccolamd -lsuitesparseconfig $(LIB_RT) $(METIS_LIB) $(LAPACK_LIB) - -# SuperLU library configuration -ifeq ($(MFEM_USE_SUPERLU5),YES) - SUPERLU_DIR = @MFEM_DIR@/../SuperLU_DIST_5.1.0 - SUPERLU_OPT = -I$(SUPERLU_DIR)/include - SUPERLU_LIB = $(XLINKER)-rpath,$(SUPERLU_DIR)/lib -L$(SUPERLU_DIR)/lib\ - -lsuperlu_dist_5.1.0 -else - SUPERLU_DIR = @MFEM_DIR@/../SuperLU_DIST_6.3.1 - SUPERLU_OPT = -I$(SUPERLU_DIR)/include - SUPERLU_LIB = $(XLINKER)-rpath,$(SUPERLU_DIR)/lib64 -L$(SUPERLU_DIR)/lib64\ - -lsuperlu_dist -lblas -endif - -# SCOTCH library configuration (required by STRUMPACK <= v2.1.0, optional in -# STRUMPACK >= v2.2.0) -SCOTCH_DIR = @MFEM_DIR@/../scotch_6.0.4 -SCOTCH_OPT = -I$(SCOTCH_DIR)/include -SCOTCH_LIB = -L$(SCOTCH_DIR)/lib -lptscotch -lptscotcherr -lscotch -lscotcherr\ - -lpthread - -# SCALAPACK library configuration (required by STRUMPACK and MUMPS) -SCALAPACK_DIR = @MFEM_DIR@/../scalapack-2.0.2 -SCALAPACK_OPT = -I$(SCALAPACK_DIR)/SRC -SCALAPACK_LIB = -L$(SCALAPACK_DIR)/lib -lscalapack $(LAPACK_LIB) - -# MPI Fortran library, needed e.g. by STRUMPACK or MUMPS -# MPICH: -MPI_FORTRAN_LIB = -lmpifort -# OpenMPI: -# MPI_FORTRAN_LIB = -lmpi_mpifh -# Additional Fortan library: -# MPI_FORTRAN_LIB += -lgfortran - -# MUMPS library configuration -MUMPS_DIR = @MFEM_DIR@/../MUMPS_5.2.0 -MUMPS_OPT = -I$(MUMPS_DIR)/include -MUMPS_LIB = $(XLINKER)-rpath,$(MUMPS_DIR)/lib -L$(MUMPS_DIR)/lib -ldmumps\ - -lmumps_common -lpord $(SCALAPACK_LIB) $(LAPACK_LIB) $(MPI_FORTRAN_LIB) - -# STRUMPACK library configuration -STRUMPACK_DIR = @MFEM_DIR@/../STRUMPACK-build -STRUMPACK_OPT = -I$(STRUMPACK_DIR)/include $(SCOTCH_OPT) -# If STRUMPACK was build with OpenMP support, the following may be need: -# STRUMPACK_OPT += $(OPENMP_OPT) -STRUMPACK_LIB = -L$(STRUMPACK_DIR)/lib -lstrumpack $(MPI_FORTRAN_LIB)\ - $(SCOTCH_LIB) $(SCALAPACK_LIB) - -# Ginkgo library configuration (currently not needed) -GINKGO_DIR = @MFEM_DIR@/../ginkgo/install -GINKGO_BUILD_TYPE=Release -ifeq ($(MFEM_USE_GINKGO),YES) - BASE_FLAGS = -std=c++14 -endif -GINKGO_OPT = -isystem $(GINKGO_DIR)/include -GINKGO_LIB_DIR = $(sort $(dir $(wildcard $(GINKGO_DIR)/lib*/libginkgo*.a $(GINKGO_DIR)/lib*/libginkgo*.so $(GINKGO_DIR)/lib*/libginkgo*.dylib $(GINKGO_DIR)/lib*/libginkgo*.dll))) -ALL_GINKGO_LIBS_DEBUG = $(notdir $(basename $(wildcard $(GINKGO_DIR)/lib*/libginkgo*d.a $(GINKGO_DIR)/lib*/libginkgo*d.so $(GINKGO_DIR)/lib*/libginkgo*d.dylib $(GINKGO_DIR)/lib*/libginkgo*d.dll))) -ALL_GINKGO_LIBS = $(notdir $(basename $(wildcard $(GINKGO_DIR)/lib*/libginkgo*.a $(GINKGO_DIR)/lib*/libginkgo*.so $(GINKGO_DIR)/lib*/libginkgo*.dylib $(GINKGO_DIR)/lib*/libginkgo*.dll))) -ALL_GINKGO_LIBS_RELEASE = $(filter-out $(ALL_GINKGO_LIBS_DEBUG),$(ALL_GINKGO_LIBS)) -GINKGO_LINK = $(subst libginkgo,-lginkgo,$(ALL_GINKGO_LIBS_RELEASE)) -ifeq ($(GINKGO_BUILD_TYPE),Debug) - ifneq (,$(ALL_GINKGO_LIBS_DEBUG)) - GINKGO_LINK = $(subst libginkgo,-lginkgo,$(ALL_GINKGO_LIBS_DEBUG)) - endif -else -endif -GINKGO_LIB = $(XLINKER)-rpath,$(GINKGO_LIB_DIR) -L$(GINKGO_LIB_DIR) $(GINKGO_LINK) - -# AmgX library configuration -AMGX_DIR = -AMGX_OPT = -AMGX_LIB = -lcusparse -lcusolver -lcublas -lnvToolsExt -lamgx - -# GnuTLS library configuration -GNUTLS_OPT = -GNUTLS_LIB = -lgnutls - -# NetCDF library configuration -NETCDF_DIR = $(HOME)/local -HDF5_DIR = $(HOME)/local -NETCDF_OPT = -I$(NETCDF_DIR)/include -I$(HDF5_DIR)/include $(ZLIB_OPT) -NETCDF_LIB = $(XLINKER)-rpath,$(NETCDF_DIR)/lib -L$(NETCDF_DIR)/lib\ - $(XLINKER)-rpath,$(HDF5_DIR)/lib -L$(HDF5_DIR)/lib\ - -lnetcdf -lhdf5_hl -lhdf5 $(ZLIB_LIB) - -# PETSc library configuration (version greater or equal to 3.8 or the dev branch) -PETSC_ARCH := arch-linux2-c-debug -PETSC_DIR := $(MFEM_DIR)/../petsc/$(PETSC_ARCH) -PETSC_VARS := $(PETSC_DIR)/lib/petsc/conf/petscvariables -PETSC_FOUND := $(if $(wildcard $(PETSC_VARS)),YES,) -PETSC_INC_VAR = PETSC_CC_INCLUDES -PETSC_LIB_VAR = PETSC_EXTERNAL_LIB_BASIC -ifeq ($(PETSC_FOUND),YES) - PETSC_OPT := $(shell sed -n "s/$(PETSC_INC_VAR) = *//p" $(PETSC_VARS)) - PETSC_DEP := $(shell sed -n "s/$(PETSC_LIB_VAR) = *//p" $(PETSC_VARS)) - PETSC_LIB = $(XLINKER)-rpath,$(abspath $(PETSC_DIR))/lib\ - -L$(abspath $(PETSC_DIR))/lib -lpetsc\ - $(subst $(CXX_XLINKER),$(XLINKER),$(PETSC_DEP)) -endif - -SLEPC_DIR := $(MFEM_DIR)/../slepc -SLEPC_VARS := $(SLEPC_DIR)/lib/slepc/conf/slepc_variables -SLEPC_FOUND := $(if $(wildcard $(SLEPC_VARS)),YES,) -SLEPC_INC_VAR = SLEPC_INCLUDE -SLEPC_LIB_VAR = SLEPC_EXTERNAL_LIB -ifeq ($(SLEPC_FOUND),YES) - SLEPC_OPT := $(shell sed -n "s/$(SLEPC_INC_VAR) *= *//p" $(SLEPC_VARS)) - # Some additional external libraries might be defined in this file - -include ${SLEPC_DIR}/${PETSC_ARCH}/lib/slepc/conf/slepcvariables - SLEPC_DEP := $(shell sed -n "s/$(SLEPC_LIB_VAR) *= *//p" $(SLEPC_VARS)) - SLEPC_LIB = $(XLINKER)-rpath,$(abspath $(SLEPC_DIR))/$(PETSC_ARCH)/lib\ - -L$(abspath $(SLEPC_DIR))/$(PETSC_ARCH)/lib -lslepc\ - $(subst $(CXX_XLINKER),$(XLINKER),$(SLEPC_DEP)) -endif - -ifeq ($(MFEM_USE_MOONOLITH),YES) - include $(MOONOLITH_DIR)/config/moonolith-config.makefile - MOONOLITH_LIB=$(MOONOLITH_LIBRARIES) -endif - -# MPFR library configuration -MPFR_OPT = -MPFR_LIB = -lmpfr - -# FMS and required libraries configuration -FMS_DIR = $(MFEM_DIR)/../fms -FMS_OPT = -I$(FMS_DIR)/include -FMS_LIB = -Wl,-rpath,$(FMS_DIR)/lib -L$(FMS_DIR)/lib -lfms - -# Conduit and required libraries configuration -CONDUIT_DIR = @MFEM_DIR@/../conduit -CONDUIT_OPT = -I$(CONDUIT_DIR)/include/conduit -CONDUIT_LIB = \ - $(XLINKER)-rpath,$(CONDUIT_DIR)/lib -L$(CONDUIT_DIR)/lib \ - -lconduit -lconduit_relay -lconduit_blueprint -ldl - -# Check if Conduit was built with hdf5 support, by looking -# for the relay hdf5 header -CONDUIT_HDF5_HEADER=$(CONDUIT_DIR)/include/conduit/conduit_relay_hdf5.hpp -ifneq (,$(wildcard $(CONDUIT_HDF5_HEADER))) - CONDUIT_OPT += -I$(HDF5_DIR)/include - CONDUIT_LIB += $(XLINKER)-rpath,$(HDF5_DIR)/lib -L$(HDF5_DIR)/lib \ - -lhdf5 $(ZLIB_LIB) -endif - -# Sidre and required libraries configuration -# Be sure to check the HDF5_DIR (set above) is correct -SIDRE_DIR = @MFEM_DIR@/../axom -SIDRE_OPT = -I$(SIDRE_DIR)/include -I$(CONDUIT_DIR)/include/conduit\ - -I$(HDF5_DIR)/include -SIDRE_LIB = \ - $(XLINKER)-rpath,$(SIDRE_DIR)/lib -L$(SIDRE_DIR)/lib \ - $(XLINKER)-rpath,$(CONDUIT_DIR)/lib -L$(CONDUIT_DIR)/lib \ - $(XLINKER)-rpath,$(HDF5_DIR)/lib -L$(HDF5_DIR)/lib \ - -laxom -lconduit -lconduit_relay -lconduit_blueprint -lhdf5 $(ZLIB_LIB) -ldl - -# PUMI -# Note that PUMI_DIR is needed -- it is used to check for gmi_sim.h -PUMI_DIR = @MFEM_DIR@/../pumi-2.1.0 -PUMI_OPT = -I$(PUMI_DIR)/include -PUMI_LIB = -L$(PUMI_DIR)/lib -lpumi -lcrv -lma -lmds -lapf -lpcu -lgmi -lparma\ - -llion -lmth -lapf_zoltan -lspr - -# HIOP -HIOP_DIR = @MFEM_DIR@/../hiop/install -HIOP_OPT = -I$(HIOP_DIR)/include -HIOP_LIB = -L$(HIOP_DIR)/lib -lhiop $(LAPACK_LIB) - -# CoDiPack -CODIPACK_DIR = @MFEM_DIR@/../CoDiPack -CODIPACK_OPT = -I$(CODIPACK_DIR) -CODIPACK_LIB = - -# GSLIB library -GSLIB_DIR = @MFEM_DIR@/../gslib/build -GSLIB_OPT = -I$(GSLIB_DIR)/include -GSLIB_LIB = -L$(GSLIB_DIR)/lib -lgs - -# CUDA library configuration -CUDA_OPT = -CUDA_LIB = -lcusparse - -# HIP library configuration -HIP_OPT = -HIP_LIB = -L$(HIP_DIR)/lib $(XLINKER)-rpath,$(HIP_DIR)/lib -lhipsparse - -# OCCA library configuration -OCCA_DIR = @MFEM_DIR@/../occa -OCCA_OPT = -I$(OCCA_DIR)/include -OCCA_LIB = $(XLINKER)-rpath,$(OCCA_DIR)/lib -L$(OCCA_DIR)/lib -locca - -# CALIPER library configuration -CALIPER_DIR = @MFEM_DIR@/../caliper -CALIPER_OPT = -I$(CALIPER_DIR)/include -CALIPER_LIB = $(XLINKER)-rpath,$(CALIPER_DIR)/lib64 -L$(CALIPER_DIR)/lib64 -lcaliper - -# BLITZ library configuration -BLITZ_DIR = @MFEM_DIR@/../blitz -BLITZ_OPT = -I$(BLITZ_DIR)/include -BLITZ_LIB = $(XLINKER)-rpath,$(BLITZ_DIR)/lib -L$(BLITZ_DIR)/lib -lblitz - -# ALGOIM library configuration -ALGOIM_DIR = @MFEM_DIR@/../algoim -ALGOIM_OPT = -I$(ALGOIM_DIR)/src $(BLITZ_OPT) -ALGOIM_LIB = $(BLITZ_LIB) - -# BENCHMARK library configuration -BENCHMARK_DIR = @MFEM_DIR@/../google-benchmark -BENCHMARK_OPT = -I$(BENCHMARK_DIR)/include -BENCHMARK_LIB = -L$(BENCHMARK_DIR)/lib -lbenchmark -lpthread - -# libCEED library configuration -CEED_DIR ?= @MFEM_DIR@/../libCEED -CEED_OPT = -I$(CEED_DIR)/include -CEED_LIB = $(XLINKER)-rpath,$(CEED_DIR)/lib -L$(CEED_DIR)/lib -lceed - -# RAJA library configuration -RAJA_DIR = @MFEM_DIR@/../raja -RAJA_OPT = -I$(RAJA_DIR)/include -ifdef CUB_DIR - RAJA_OPT += -I$(CUB_DIR) -endif -ifdef CAMP_DIR - RAJA_OPT += -I$(CAMP_DIR)/include -endif -RAJA_LIB = $(XLINKER)-rpath,$(RAJA_DIR)/lib -L$(RAJA_DIR)/lib -lRAJA - -# UMPIRE library configuration -UMPIRE_DIR = @MFEM_DIR@/../umpire -UMPIRE_OPT = -I$(UMPIRE_DIR)/include $(if $(CAMP_DIR), -I$(CAMP_DIR)/include) -UMPIRE_LIB = -L$(UMPIRE_DIR)/lib -lumpire - -# MKL CPardiso library configuration -MKL_CPARDISO_DIR ?= -MKL_MPI_WRAPPER ?= mkl_blacs_mpich_lp64 -MKL_LIBRARY_SUBDIR ?= lib -MKL_CPARDISO_OPT = -I$(MKL_CPARDISO_DIR)/include -MKL_CPARDISO_LIB = $(XLINKER)-rpath,$(MKL_CPARDISO_DIR)/$(MKL_LIBRARY_SUBDIR)\ - -L$(MKL_CPARDISO_DIR)/$(MKL_LIBRARY_SUBDIR) -l$(MKL_MPI_WRAPPER)\ - -lmkl_intel_lp64 -lmkl_sequential -lmkl_core - -# PARELAG library configuration -PARELAG_DIR = @MFEM_DIR@/../parelag -PARELAG_OPT = -I$(PARELAG_DIR)/src -I$(PARELAG_DIR)/build/src -PARELAG_LIB = -L$(PARELAG_DIR)/build/src -lParELAG - -# Enzyme configuration - -# If you want to enable automatic differentiation at compile time, use the -# options below, adapted to your configuration. To be more flexible, we -# recommend using the Enzyme plugin during link time optimization. One option is -# to add your options to the global compiler/linker flags like -# -# BASE_FLAGS += -flto -# CXX_XLINKER += -fuse-ld=lld -Wl,--lto-legacy-pass-manager\ -# -Wl,-mllvm=-load=$(ENZYME_DIR)/LLDEnzyme-$(ENZYME_VERSION).so -Wl, -# -ENZYME_DIR ?= @MFEM_DIR@/../enzyme -ENZYME_VERSION ?= 14 -ENZYME_OPT = -fno-experimental-new-pass-manager -Xclang -load -Xclang $(ENZYME_DIR)/ClangEnzyme-$(ENZYME_VERSION).so -ENZYME_LIB = "" - -# If YES, enable some informational messages -VERBOSE = NO - -# Optional build tag -MFEM_BUILD_TAG = $(shell uname -snm) \ No newline at end of file diff --git a/cuda/Dockerfile b/cuda/Dockerfile index bd334a1..2f8a7b1 100644 --- a/cuda/Dockerfile +++ b/cuda/Dockerfile @@ -8,10 +8,10 @@ USER root WORKDIR /opt/archives -RUN wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb \ +RUN wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/cuda-keyring_1.1-1_all.deb \ && dpkg -i cuda-keyring_1.1-1_all.deb \ && apt-get update \ - && apt-get install -y cuda-toolkit-11-8 nvidia-container-toolkit + && apt-get install -y cuda-toolkit-12-9 # hypre RUN curl -L https://github.com/hypre-space/hypre/archive/refs/tags/v${hypre_version}.tar.gz > /opt/archives/hypre-v${hypre_version}.tar.gz @@ -28,4 +28,4 @@ ENV PATH=$PATH:/usr/local/cuda/bin RUN git clone --depth=1 https://github.com/mfem/mfem.git mfem COPY --chown=euler:euler user.mk mfem/config/user.mk RUN sed -i 's/CUDA_ARCH_SM_PLACEHOLDER/sm_${cuda_arch_sm}/' mfem/config/user.mk -RUN cd mfem && make config && make -j ${num_jobs} && cd examples && make ex1 && make ex1p \ No newline at end of file +RUN cd mfem && make config && make -j ${num_jobs} && cd examples && make ex1 && make ex1p diff --git a/cuda/user.mk b/cuda/user.mk index 9516df6..ebd8606 100644 --- a/cuda/user.mk +++ b/cuda/user.mk @@ -1,547 +1,33 @@ -# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced -# at the Lawrence Livermore National Laboratory. All Rights reserved. See files -# LICENSE and NOTICE for details. LLNL-CODE-806117. -# -# This file is part of the MFEM library. For more information and source code -# availability visit https://mfem.org. -# -# MFEM is free software; you can redistribute it and/or modify it under the -# terms of the BSD-3 license. We welcome feedback and contributions, see file -# CONTRIBUTING.md for details. - -# This file describes the default MFEM build options. -# -# See the file INSTALL for description of these options. You can -# customize them below, or copy this file to user.mk and modify it. - - -# Some choices below are based on the OS type: -NOTMAC := $(subst Darwin,,$(shell uname -s)) - -ETAGS_BIN = $(shell command -v etags 2> /dev/null) -EGREP_BIN = $(shell command -v egrep 2> /dev/null) +VERBOSE = NO CXX = clang++ -MPICXX = mpicxx +MPICXX = OMPI_CXX=$(CXX) mpicxx -BASE_FLAGS = -std=c++11 +BASE_FLAGS = -std=c++17 OPTIM_FLAGS = -O3 $(BASE_FLAGS) -DEBUG_FLAGS = -g $(XCOMPILER)-Wall $(BASE_FLAGS) - -# Prefixes for passing flags to the compiler and linker when using CXX or MPICXX -CXX_XCOMPILER = -CXX_XLINKER = -Wl, - -# Destination location of make install -# PREFIX = $(HOME)/mfem -PREFIX = ./mfem -# Install program -INSTALL = /usr/bin/install STATIC = NO SHARED = YES -# CUDA configuration options -# -# If you set MFEM_USE_ENZYME=YES, CUDA_CXX has to be configured to use cuda with -# clang as its host compiler. -CUDA_CXX = nvcc +CUDA_CXX = $(MPICXX) CUDA_ARCH = CUDA_ARCH_SM_PLACEHOLDER -CUDA_FLAGS = -x=cu --expt-extended-lambda -arch=$(CUDA_ARCH) -# Prefixes for passing flags to the host compiler and linker when using CUDA_CXX -CUDA_XCOMPILER = -Xcompiler= -CUDA_XLINKER = -Xlinker= - -# HIP configuration options -HIP_CXX = hipcc -# The HIP_ARCH option specifies the AMD GPU processor, similar to CUDA_ARCH. For -# example: gfx600 (tahiti), gfx700 (kaveri), gfx701 (hawaii), gfx801 (carrizo), -# gfx900, gfx1010, etc. -HIP_ARCH = gfx900 -HIP_FLAGS = --amdgpu-target=$(HIP_ARCH) -HIP_XCOMPILER = -HIP_XLINKER = -Wl, - -# Flags for generating dependencies. -DEP_FLAGS = -MM -MT - -ifneq ($(NOTMAC),) - AR = ar - ARFLAGS = crv - RANLIB = ranlib - PICFLAG = $(XCOMPILER)-fPIC - SO_EXT = so - SO_VER = so.$(MFEM_VERSION_STRING) - BUILD_SOFLAGS = -shared $(XLINKER)-soname,libmfem.$(SO_VER) - BUILD_RPATH = $(XLINKER)-rpath,$(BUILD_REAL_DIR) - INSTALL_SOFLAGS = $(BUILD_SOFLAGS) - INSTALL_RPATH = $(XLINKER)-rpath,@MFEM_LIB_DIR@ -else - # Silence "has no symbols" warnings on Mac OS X - AR = ar - ARFLAGS = Scrv - RANLIB = ranlib -no_warning_for_no_symbols - PICFLAG = $(XCOMPILER)-fPIC - SO_EXT = dylib - SO_VER = $(MFEM_VERSION_STRING).dylib - MAKE_SOFLAGS = $(XLINKER)-dylib,-install_name,$(1)/libmfem.$(SO_VER),\ - -compatibility_version,$(MFEM_VERSION_STRING),\ - -current_version,$(MFEM_VERSION_STRING),\ - -undefined,dynamic_lookup - BUILD_SOFLAGS = $(subst $1 ,,$(call MAKE_SOFLAGS,$(BUILD_REAL_DIR))) - BUILD_RPATH = $(XLINKER)-undefined,dynamic_lookup - INSTALL_SOFLAGS = $(subst $1 ,,$(call MAKE_SOFLAGS,$(MFEM_LIB_DIR))) - INSTALL_RPATH = $(XLINKER)-undefined,dynamic_lookup - # Silence unused command line argument warnings when generating dependencies - # with mpicxx and clang - DEP_FLAGS := -Wno-unused-command-line-argument $(DEP_FLAGS) -endif - -# Set CXXFLAGS to overwrite the default selection of DEBUG_FLAGS/OPTIM_FLAGS -# CXXFLAGS = -O3 -march=native - -# Optional extra compile flags, in addition to CXXFLAGS: -# CPPFLAGS = - -# Library configurations: -# Note: symbols of the form @VAR@ will be replaced by $(VAR) in derived -# variables, like MFEM_FLAGS, defined in config.mk. - -# Command used to launch MPI jobs -MFEM_MPIEXEC = mpirun -MFEM_MPIEXEC_NP = -np -# Number of mpi tasks for parallel jobs -MFEM_MPI_NP = 4 - -# MFEM configuration options: YES/NO values, which are exported to config.mk and -# config.hpp. The values below are the defaults for generating the actual values -# in config.mk and config.hpp. MFEM_USE_MPI = YES MFEM_USE_METIS = $(MFEM_USE_MPI) MFEM_USE_METIS_5 = YES -MFEM_DEBUG = YES -MFEM_USE_EXCEPTIONS = NO -MFEM_USE_ZLIB = NO +MFEM_DEBUG = NO MFEM_USE_LIBUNWIND = NO -MFEM_USE_LAPACK = NO -MFEM_THREAD_SAFE = NO -MFEM_USE_OPENMP = NO -MFEM_USE_LEGACY_OPENMP = NO MFEM_USE_MEMALLOC = YES MFEM_TIMER_TYPE = $(if $(NOTMAC),2,4) -MFEM_USE_SUNDIALS = NO -MFEM_USE_MESQUITE = NO -MFEM_USE_SUITESPARSE = NO -MFEM_USE_SUPERLU = NO -MFEM_USE_SUPERLU5 = NO -MFEM_USE_MUMPS = NO -MFEM_USE_STRUMPACK = NO -MFEM_USE_GINKGO = NO -MFEM_USE_AMGX = NO -MFEM_USE_GNUTLS = NO -MFEM_USE_NETCDF = NO -MFEM_USE_PETSC = NO -MFEM_USE_SLEPC = NO -MFEM_USE_MPFR = NO -MFEM_USE_SIDRE = NO -MFEM_USE_FMS = NO -MFEM_USE_CONDUIT = NO -MFEM_USE_PUMI = NO -MFEM_USE_HIOP = NO -MFEM_USE_GSLIB = NO -MFEM_USE_CUDA = YES -MFEM_USE_HIP = NO -MFEM_USE_RAJA = NO -MFEM_USE_OCCA = NO -MFEM_USE_CEED = NO -MFEM_USE_CALIPER = NO -MFEM_USE_ALGOIM = NO -MFEM_USE_UMPIRE = NO -MFEM_USE_SIMD = NO -MFEM_USE_ADIOS2 = NO -MFEM_USE_MKL_CPARDISO = NO -MFEM_USE_MOONOLITH = NO -MFEM_USE_ADFORWARD = NO -MFEM_USE_CODIPACK = NO -MFEM_USE_BENCHMARK = NO -MFEM_USE_PARELAG = NO -MFEM_USE_ENZYME = NO - -# MPI library compile and link flags -# These settings are used only when building MFEM with MPI + HIP -ifeq ($(MFEM_USE_MPI)$(MFEM_USE_HIP),YESYES) - # We determine MPI_DIR assuming $(MPICXX) is in $(MPI_DIR)/bin - MPI_DIR := $(patsubst %/,%,$(dir $(shell which $(MPICXX)))) - MPI_DIR := $(patsubst %/,%,$(dir $(MPI_DIR))) - MPI_OPT = -I$(MPI_DIR)/include - MPI_LIB = -L$(MPI_DIR)/lib $(XLINKER)-rpath,$(MPI_DIR)/lib -lmpi -endif +MFEM_USE_ENZYME = YES +MFEM_USE_CUDA = YES -# ROCM/HIP directory such that ROCM/HIP libraries like rocsparse and rocrand are -# found in $(HIP_DIR)/lib, usually as links. Typically, this directoory is of -# the form /opt/rocm-X.Y.Z which is called ROCM_PATH by hipconfig. -ifeq ($(MFEM_USE_HIP),YES) - HIP_DIR := $(patsubst %/,%,$(dir $(shell which $(HIP_CXX)))) - HIP_DIR := $(patsubst %/,%,$(dir $(HIP_DIR))) - ifeq (,$(wildcard $(HIP_DIR)/lib/librocsparse.*)) - HIP_DIR := $(shell hipconfig --rocmpath 2> /dev/null) - ifeq (,$(wildcard $(HIP_DIR)/lib/librocsparse.*)) - $(error Unable to determine HIP_DIR. Please set it manually.) - endif - endif -endif - -# Compile and link options for zlib. -ZLIB_DIR = -ZLIB_OPT = $(if $(ZLIB_DIR),-I$(ZLIB_DIR)/include) -ZLIB_LIB = $(if $(ZLIB_DIR),$(ZLIB_RPATH) -L$(ZLIB_DIR)/lib ,)-lz -ZLIB_RPATH = $(XLINKER)-rpath,$(ZLIB_DIR)/lib - -LIBUNWIND_OPT = -g -LIBUNWIND_LIB = $(if $(NOTMAC),-lunwind -ldl,) - -# HYPRE library configuration (needed to build the parallel version) HYPRE_DIR = /usr/local HYPRE_OPT = -I$(HYPRE_DIR)/include HYPRE_LIB = -L$(HYPRE_DIR)/lib -lHYPRE -ifeq (YES,$(MFEM_USE_CUDA)) - # This is only necessary when hypre is built with cuda: - HYPRE_LIB += -lcusparse -lcurand -endif -ifeq (YES,$(MFEM_USE_HIP)) - # This is only necessary when hypre is built with hip: - HYPRE_LIB += -L$(HIP_DIR)/lib $(XLINKER)-rpath,$(HIP_DIR)/lib\ - -lrocsparse -lrocrand -endif - -# METIS library configuration -ifeq ($(MFEM_USE_SUPERLU)$(MFEM_USE_STRUMPACK)$(MFEM_USE_MUMPS),NONONO) - ifeq ($(MFEM_USE_METIS_5),NO) - METIS_DIR = /usr/local - METIS_OPT = - METIS_LIB = -L$(METIS_DIR) -lmetis - else - METIS_DIR = /usr/local - METIS_OPT = -I$(METIS_DIR)/include - METIS_LIB = -L$(METIS_DIR)/lib -lmetis - endif -else - # ParMETIS: currently needed by SuperLU or STRUMPACK. We assume that METIS 5 - # (included with ParMETIS) is installed in the same location. - # Starting with STRUMPACK v2.2.0, ParMETIS is an optional dependency while - # METIS is still required. - METIS_DIR = @MFEM_DIR@/../parmetis-4.0.3 - METIS_OPT = -I$(METIS_DIR)/include - METIS_LIB = -L$(METIS_DIR)/lib -lparmetis -lmetis - MFEM_USE_METIS_5 = YES -endif - -# LAPACK library configuration -LAPACK_OPT = -LAPACK_LIB = $(if $(NOTMAC),-llapack -lblas,-framework Accelerate) - -# OpenMP configuration -OPENMP_OPT = $(XCOMPILER)-fopenmp -OPENMP_LIB = - -# Used when MFEM_TIMER_TYPE = 2 -POSIX_CLOCKS_LIB = -lrt - -# SUNDIALS library configuration -# For sundials_nvecmpiplusx and nvecparallel remember to build with MPI_ENABLE=ON -# and modify cmake variables for hypre for sundials -SUNDIALS_DIR = @MFEM_DIR@/../sundials-5.0.0/instdir -SUNDIALS_OPT = -I$(SUNDIALS_DIR)/include -SUNDIALS_LIBDIR = $(wildcard $(SUNDIALS_DIR)/lib*) -SUNDIALS_LIB = $(XLINKER)-rpath,$(SUNDIALS_LIBDIR) -L$(SUNDIALS_LIBDIR)\ - -lsundials_arkode -lsundials_cvodes -lsundials_nvecserial -lsundials_kinsol - -ifeq ($(MFEM_USE_MPI),YES) - SUNDIALS_LIB += -lsundials_nvecparallel -lsundials_nvecmpiplusx -endif -ifeq ($(MFEM_USE_CUDA),YES) - SUNDIALS_LIB += -lsundials_nveccuda -endif -# If SUNDIALS was built with KLU: -# MFEM_USE_SUITESPARSE = YES - -# MESQUITE library configuration -MESQUITE_DIR = @MFEM_DIR@/../mesquite-2.99 -MESQUITE_OPT = -I$(MESQUITE_DIR)/include -MESQUITE_LIB = -L$(MESQUITE_DIR)/lib -lmesquite - -# SuiteSparse library configuration -LIB_RT = $(if $(NOTMAC),-lrt,) -SUITESPARSE_DIR = @MFEM_DIR@/../SuiteSparse -SUITESPARSE_OPT = -I$(SUITESPARSE_DIR)/include -SUITESPARSE_LIB = $(XLINKER)-rpath,$(SUITESPARSE_DIR)/lib\ - -L$(SUITESPARSE_DIR)/lib -lklu -lbtf -lumfpack -lcholmod -lcolamd -lamd -lcamd\ - -lccolamd -lsuitesparseconfig $(LIB_RT) $(METIS_LIB) $(LAPACK_LIB) - -# SuperLU library configuration -ifeq ($(MFEM_USE_SUPERLU5),YES) - SUPERLU_DIR = @MFEM_DIR@/../SuperLU_DIST_5.1.0 - SUPERLU_OPT = -I$(SUPERLU_DIR)/include - SUPERLU_LIB = $(XLINKER)-rpath,$(SUPERLU_DIR)/lib -L$(SUPERLU_DIR)/lib\ - -lsuperlu_dist_5.1.0 -else - SUPERLU_DIR = @MFEM_DIR@/../SuperLU_DIST_6.3.1 - SUPERLU_OPT = -I$(SUPERLU_DIR)/include - SUPERLU_LIB = $(XLINKER)-rpath,$(SUPERLU_DIR)/lib64 -L$(SUPERLU_DIR)/lib64\ - -lsuperlu_dist -lblas -endif - -# SCOTCH library configuration (required by STRUMPACK <= v2.1.0, optional in -# STRUMPACK >= v2.2.0) -SCOTCH_DIR = @MFEM_DIR@/../scotch_6.0.4 -SCOTCH_OPT = -I$(SCOTCH_DIR)/include -SCOTCH_LIB = -L$(SCOTCH_DIR)/lib -lptscotch -lptscotcherr -lscotch -lscotcherr\ - -lpthread - -# SCALAPACK library configuration (required by STRUMPACK and MUMPS) -SCALAPACK_DIR = @MFEM_DIR@/../scalapack-2.0.2 -SCALAPACK_OPT = -I$(SCALAPACK_DIR)/SRC -SCALAPACK_LIB = -L$(SCALAPACK_DIR)/lib -lscalapack $(LAPACK_LIB) - -# MPI Fortran library, needed e.g. by STRUMPACK or MUMPS -# MPICH: -MPI_FORTRAN_LIB = -lmpifort -# OpenMPI: -# MPI_FORTRAN_LIB = -lmpi_mpifh -# Additional Fortan library: -# MPI_FORTRAN_LIB += -lgfortran - -# MUMPS library configuration -MUMPS_DIR = @MFEM_DIR@/../MUMPS_5.2.0 -MUMPS_OPT = -I$(MUMPS_DIR)/include -MUMPS_LIB = $(XLINKER)-rpath,$(MUMPS_DIR)/lib -L$(MUMPS_DIR)/lib -ldmumps\ - -lmumps_common -lpord $(SCALAPACK_LIB) $(LAPACK_LIB) $(MPI_FORTRAN_LIB) -# STRUMPACK library configuration -STRUMPACK_DIR = @MFEM_DIR@/../STRUMPACK-build -STRUMPACK_OPT = -I$(STRUMPACK_DIR)/include $(SCOTCH_OPT) -# If STRUMPACK was build with OpenMP support, the following may be need: -# STRUMPACK_OPT += $(OPENMP_OPT) -STRUMPACK_LIB = -L$(STRUMPACK_DIR)/lib -lstrumpack $(MPI_FORTRAN_LIB)\ - $(SCOTCH_LIB) $(SCALAPACK_LIB) - -# Ginkgo library configuration (currently not needed) -GINKGO_DIR = @MFEM_DIR@/../ginkgo/install -GINKGO_BUILD_TYPE=Release -ifeq ($(MFEM_USE_GINKGO),YES) - BASE_FLAGS = -std=c++14 -endif -GINKGO_OPT = -isystem $(GINKGO_DIR)/include -GINKGO_LIB_DIR = $(sort $(dir $(wildcard $(GINKGO_DIR)/lib*/libginkgo*.a $(GINKGO_DIR)/lib*/libginkgo*.so $(GINKGO_DIR)/lib*/libginkgo*.dylib $(GINKGO_DIR)/lib*/libginkgo*.dll))) -ALL_GINKGO_LIBS_DEBUG = $(notdir $(basename $(wildcard $(GINKGO_DIR)/lib*/libginkgo*d.a $(GINKGO_DIR)/lib*/libginkgo*d.so $(GINKGO_DIR)/lib*/libginkgo*d.dylib $(GINKGO_DIR)/lib*/libginkgo*d.dll))) -ALL_GINKGO_LIBS = $(notdir $(basename $(wildcard $(GINKGO_DIR)/lib*/libginkgo*.a $(GINKGO_DIR)/lib*/libginkgo*.so $(GINKGO_DIR)/lib*/libginkgo*.dylib $(GINKGO_DIR)/lib*/libginkgo*.dll))) -ALL_GINKGO_LIBS_RELEASE = $(filter-out $(ALL_GINKGO_LIBS_DEBUG),$(ALL_GINKGO_LIBS)) -GINKGO_LINK = $(subst libginkgo,-lginkgo,$(ALL_GINKGO_LIBS_RELEASE)) -ifeq ($(GINKGO_BUILD_TYPE),Debug) - ifneq (,$(ALL_GINKGO_LIBS_DEBUG)) - GINKGO_LINK = $(subst libginkgo,-lginkgo,$(ALL_GINKGO_LIBS_DEBUG)) - endif -else -endif -GINKGO_LIB = $(XLINKER)-rpath,$(GINKGO_LIB_DIR) -L$(GINKGO_LIB_DIR) $(GINKGO_LINK) - -# AmgX library configuration -AMGX_DIR = @MFEM_DIR@/../amgx -AMGX_OPT = -I$(AMGX_DIR)/include -AMGX_LIB = -lcusparse -lcusolver -lcublas -lnvToolsExt -L$(AMGX_DIR)/lib -lamgx - -# GnuTLS library configuration -GNUTLS_OPT = -GNUTLS_LIB = -lgnutls - -# NetCDF library configuration -NETCDF_DIR = $(HOME)/local -HDF5_DIR = $(HOME)/local -NETCDF_OPT = -I$(NETCDF_DIR)/include -I$(HDF5_DIR)/include $(ZLIB_OPT) -NETCDF_LIB = $(XLINKER)-rpath,$(NETCDF_DIR)/lib -L$(NETCDF_DIR)/lib\ - $(XLINKER)-rpath,$(HDF5_DIR)/lib -L$(HDF5_DIR)/lib\ - -lnetcdf -lhdf5_hl -lhdf5 $(ZLIB_LIB) - -# PETSc library configuration (version greater or equal to 3.8 or the dev branch) -PETSC_ARCH := arch-linux2-c-debug -PETSC_DIR := $(MFEM_DIR)/../petsc/$(PETSC_ARCH) -PETSC_VARS := $(PETSC_DIR)/lib/petsc/conf/petscvariables -PETSC_FOUND := $(if $(wildcard $(PETSC_VARS)),YES,) -PETSC_INC_VAR = PETSC_CC_INCLUDES -PETSC_LIB_VAR = PETSC_EXTERNAL_LIB_BASIC -ifeq ($(PETSC_FOUND),YES) - PETSC_OPT := $(shell sed -n "s/$(PETSC_INC_VAR) = *//p" $(PETSC_VARS)) - PETSC_DEP := $(shell sed -n "s/$(PETSC_LIB_VAR) = *//p" $(PETSC_VARS)) - PETSC_LIB = $(XLINKER)-rpath,$(abspath $(PETSC_DIR))/lib\ - -L$(abspath $(PETSC_DIR))/lib -lpetsc\ - $(subst $(CXX_XLINKER),$(XLINKER),$(PETSC_DEP)) -endif - -SLEPC_DIR := $(MFEM_DIR)/../slepc -SLEPC_VARS := $(SLEPC_DIR)/lib/slepc/conf/slepc_variables -SLEPC_FOUND := $(if $(wildcard $(SLEPC_VARS)),YES,) -SLEPC_INC_VAR = SLEPC_INCLUDE -SLEPC_LIB_VAR = SLEPC_EXTERNAL_LIB -ifeq ($(SLEPC_FOUND),YES) - SLEPC_OPT := $(shell sed -n "s/$(SLEPC_INC_VAR) *= *//p" $(SLEPC_VARS)) - # Some additional external libraries might be defined in this file - -include ${SLEPC_DIR}/${PETSC_ARCH}/lib/slepc/conf/slepcvariables - SLEPC_DEP := $(shell sed -n "s/$(SLEPC_LIB_VAR) *= *//p" $(SLEPC_VARS)) - SLEPC_LIB = $(XLINKER)-rpath,$(abspath $(SLEPC_DIR))/$(PETSC_ARCH)/lib\ - -L$(abspath $(SLEPC_DIR))/$(PETSC_ARCH)/lib -lslepc\ - $(subst $(CXX_XLINKER),$(XLINKER),$(SLEPC_DEP)) -endif - -ifeq ($(MFEM_USE_MOONOLITH),YES) - include $(MOONOLITH_DIR)/config/moonolith-config.makefile - MOONOLITH_LIB=$(MOONOLITH_LIBRARIES) -endif - -# MPFR library configuration -MPFR_OPT = -MPFR_LIB = -lmpfr - -# FMS and required libraries configuration -FMS_DIR = $(MFEM_DIR)/../fms -FMS_OPT = -I$(FMS_DIR)/include -FMS_LIB = -Wl,-rpath,$(FMS_DIR)/lib -L$(FMS_DIR)/lib -lfms - -# Conduit and required libraries configuration -CONDUIT_DIR = @MFEM_DIR@/../conduit -CONDUIT_OPT = -I$(CONDUIT_DIR)/include/conduit -CONDUIT_LIB = \ - $(XLINKER)-rpath,$(CONDUIT_DIR)/lib -L$(CONDUIT_DIR)/lib \ - -lconduit -lconduit_relay -lconduit_blueprint -ldl - -# Check if Conduit was built with hdf5 support, by looking -# for the relay hdf5 header -CONDUIT_HDF5_HEADER=$(CONDUIT_DIR)/include/conduit/conduit_relay_hdf5.hpp -ifneq (,$(wildcard $(CONDUIT_HDF5_HEADER))) - CONDUIT_OPT += -I$(HDF5_DIR)/include - CONDUIT_LIB += $(XLINKER)-rpath,$(HDF5_DIR)/lib -L$(HDF5_DIR)/lib \ - -lhdf5 $(ZLIB_LIB) -endif - -# Sidre and required libraries configuration -# Be sure to check the HDF5_DIR (set above) is correct -SIDRE_DIR = @MFEM_DIR@/../axom -SIDRE_OPT = -I$(SIDRE_DIR)/include -I$(CONDUIT_DIR)/include/conduit\ - -I$(HDF5_DIR)/include -SIDRE_LIB = \ - $(XLINKER)-rpath,$(SIDRE_DIR)/lib -L$(SIDRE_DIR)/lib \ - $(XLINKER)-rpath,$(CONDUIT_DIR)/lib -L$(CONDUIT_DIR)/lib \ - $(XLINKER)-rpath,$(HDF5_DIR)/lib -L$(HDF5_DIR)/lib \ - -laxom -lconduit -lconduit_relay -lconduit_blueprint -lhdf5 $(ZLIB_LIB) -ldl - -# PUMI -# Note that PUMI_DIR is needed -- it is used to check for gmi_sim.h -PUMI_DIR = @MFEM_DIR@/../pumi-2.1.0 -PUMI_OPT = -I$(PUMI_DIR)/include -PUMI_LIB = -L$(PUMI_DIR)/lib -lpumi -lcrv -lma -lmds -lapf -lpcu -lgmi -lparma\ - -llion -lmth -lapf_zoltan -lspr - -# HIOP -HIOP_DIR = @MFEM_DIR@/../hiop/install -HIOP_OPT = -I$(HIOP_DIR)/include -HIOP_LIB = -L$(HIOP_DIR)/lib -lhiop $(LAPACK_LIB) - -# CoDiPack -CODIPACK_DIR = @MFEM_DIR@/../CoDiPack -CODIPACK_OPT = -I$(CODIPACK_DIR) -CODIPACK_LIB = - -# GSLIB library -GSLIB_DIR = @MFEM_DIR@/../gslib/build -GSLIB_OPT = -I$(GSLIB_DIR)/include -GSLIB_LIB = -L$(GSLIB_DIR)/lib -lgs - -# CUDA library configuration -CUDA_OPT = -CUDA_LIB = -lcusparse - -# HIP library configuration -HIP_OPT = -HIP_LIB = -L$(HIP_DIR)/lib $(XLINKER)-rpath,$(HIP_DIR)/lib -lhipsparse - -# OCCA library configuration -OCCA_DIR = @MFEM_DIR@/../occa -OCCA_OPT = -I$(OCCA_DIR)/include -OCCA_LIB = $(XLINKER)-rpath,$(OCCA_DIR)/lib -L$(OCCA_DIR)/lib -locca - -# CALIPER library configuration -CALIPER_DIR = @MFEM_DIR@/../caliper -CALIPER_OPT = -I$(CALIPER_DIR)/include -CALIPER_LIB = $(XLINKER)-rpath,$(CALIPER_DIR)/lib64 -L$(CALIPER_DIR)/lib64 -lcaliper - -# BLITZ library configuration -BLITZ_DIR = @MFEM_DIR@/../blitz -BLITZ_OPT = -I$(BLITZ_DIR)/include -BLITZ_LIB = $(XLINKER)-rpath,$(BLITZ_DIR)/lib -L$(BLITZ_DIR)/lib -lblitz - -# ALGOIM library configuration -ALGOIM_DIR = @MFEM_DIR@/../algoim -ALGOIM_OPT = -I$(ALGOIM_DIR)/src $(BLITZ_OPT) -ALGOIM_LIB = $(BLITZ_LIB) - -# BENCHMARK library configuration -BENCHMARK_DIR = @MFEM_DIR@/../google-benchmark -BENCHMARK_OPT = -I$(BENCHMARK_DIR)/include -BENCHMARK_LIB = -L$(BENCHMARK_DIR)/lib -lbenchmark -lpthread - -# libCEED library configuration -CEED_DIR ?= @MFEM_DIR@/../libCEED -CEED_OPT = -I$(CEED_DIR)/include -CEED_LIB = $(XLINKER)-rpath,$(CEED_DIR)/lib -L$(CEED_DIR)/lib -lceed - -# RAJA library configuration -RAJA_DIR = @MFEM_DIR@/../raja -RAJA_OPT = -I$(RAJA_DIR)/include -ifdef CUB_DIR - RAJA_OPT += -I$(CUB_DIR) -endif -ifdef CAMP_DIR - RAJA_OPT += -I$(CAMP_DIR)/include -endif -RAJA_LIB = $(XLINKER)-rpath,$(RAJA_DIR)/lib -L$(RAJA_DIR)/lib -lRAJA - -# UMPIRE library configuration -UMPIRE_DIR = @MFEM_DIR@/../umpire -UMPIRE_OPT = -I$(UMPIRE_DIR)/include $(if $(CAMP_DIR), -I$(CAMP_DIR)/include) -UMPIRE_LIB = -L$(UMPIRE_DIR)/lib -lumpire - -# MKL CPardiso library configuration -MKL_CPARDISO_DIR ?= -MKL_MPI_WRAPPER ?= mkl_blacs_mpich_lp64 -MKL_LIBRARY_SUBDIR ?= lib -MKL_CPARDISO_OPT = -I$(MKL_CPARDISO_DIR)/include -MKL_CPARDISO_LIB = $(XLINKER)-rpath,$(MKL_CPARDISO_DIR)/$(MKL_LIBRARY_SUBDIR)\ - -L$(MKL_CPARDISO_DIR)/$(MKL_LIBRARY_SUBDIR) -l$(MKL_MPI_WRAPPER)\ - -lmkl_intel_lp64 -lmkl_sequential -lmkl_core - -# PARELAG library configuration -PARELAG_DIR = @MFEM_DIR@/../parelag -PARELAG_OPT = -I$(PARELAG_DIR)/src -I$(PARELAG_DIR)/build/src -PARELAG_LIB = -L$(PARELAG_DIR)/build/src -lParELAG - -# Enzyme configuration - -# If you want to enable automatic differentiation at compile time, use the -# options below, adapted to your configuration. To be more flexible, we -# recommend using the Enzyme plugin during link time optimization. One option is -# to add your options to the global compiler/linker flags like -# -# BASE_FLAGS += -flto -# CXX_XLINKER += -fuse-ld=lld -Wl,--lto-legacy-pass-manager\ -# -Wl,-mllvm=-load=$(ENZYME_DIR)/LLDEnzyme-$(ENZYME_VERSION).so -Wl, -# -ENZYME_DIR ?= @MFEM_DIR@/../enzyme -ENZYME_VERSION ?= 14 -ENZYME_OPT = -fno-experimental-new-pass-manager -Xclang -load -Xclang $(ENZYME_DIR)/ClangEnzyme-$(ENZYME_VERSION).so -ENZYME_LIB = "" - -# If YES, enable some informational messages -VERBOSE = NO +METIS_DIR = +METIS_OPT = +METIS_LIB = -L/usr/lib/x86_64-linux-gnu -lmetis -lparmetis -# Optional build tag -MFEM_BUILD_TAG = $(shell uname -snm) \ No newline at end of file +ENZYME_DIR = /usr/local diff --git a/docker-compose.yaml b/docker-compose.yaml index 4a91ab1..4bc7d01 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,11 +1,9 @@ -version: '3' - services: base: build: context: base args: - num_jobs: 2 + num_jobs: 8 image: ghcr.io/mfem/containers/base:latest cpu: @@ -13,7 +11,7 @@ services: context: cpu args: num_jobs: 8 - hypre_version: 2.27.0 + hypre_version: 2.31.0 image: ghcr.io/mfem/containers/cpu:latest cpu-tpls: @@ -28,18 +26,10 @@ services: context: cuda args: num_jobs: 8 - cuda_arch_sm: 70 - hypre_version: 2.27.0 + cuda_arch_sm: 89 + hypre_version: 2.31.0 image: ghcr.io/mfem/containers/cuda:latest - cuda-tpls: - build: - context: cuda-tpls - args: - num_jobs: 8 - cuda_arch_sm: 70 - image: ghcr.io/mfem/containers/cuda-tpls:latest - developer-cpu: build: context: developer @@ -54,5 +44,5 @@ services: args: base_image: cuda-tpls num_jobs: 8 - cuda_arch_sm: 70 + cuda_arch_sm: 89 image: ghcr.io/mfem/containers/developer-cuda:latest