Merge pull request #71 from jacobmerson/dev

Building on summit
wrtobin · Apr 5, 2020 · 867f9c5 · 867f9c5
2 parents 0868fc3 + a9fe4c2
commit 867f9c5
Show file tree

Hide file tree

Showing 18 changed files with 265 additions and 79 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -39,9 +39,8 @@ if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Cray")
 elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "XL")
   # todo :  check for regular xl vs bg/q xl compiler
   #         currently assuming bg/q
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -qlanglvl=extended0x -O5 -qhot=level=0 -qsimd=auto -qmaxmem=-1 -qstrict -qstrict_induction -qreport")
-  # these definitions are BGQ specific, but for now the only place we use XL compiler is on BGQ
-  #add_definitions(-DBGQ)
+  #set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -qlanglvl=extended0x")
+  #set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -qlanglvl=extended0x -O5 -qhot=level=0 -qsimd=auto -qmaxmem=-1 -qstrict -qstrict_induction -qreport -Wall")
 elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Intel")
   # todo :  set intel-specific cxx flags
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")

diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt
@@ -26,7 +26,7 @@ endif()
 
 add_library(las_core STATIC ${las_core_sources})
 if(${CMAKE_VERSION} VERSION_GREATER "3.8.2")
-  target_compile_features(las_core PUBLIC cxx_std_11)
+  #target_compile_features(las_core PUBLIC cxx_std_11)
 endif()
 if(MPI_COMPILE_FLAGS)
   set_target_properties(las_core PROPERTIES COMPILE_FLAGS ${MPI_CXX_COMPILE_DEFINITIONS})

diff --git a/scripts/config_dcs.sh b/scripts/config_dcs.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+#mpicc=/gpfs/u/home/PASC/PASCmrsn/scratch/test_compile/mpicc
+#mpicxx=/gpfs/u/home/PASC/PASCmrsn/scratch/test_compile/mpicxx
+#mpif77=/gpfs/u/home/PASC/PASCmrsn/scratch/test_compile/mpif77
+export OMPI_CXX=xlc++_r
+
+  cmake /gpfs/u/home/PASC/PASCmrsn/barn/las/  \
+  -DCMAKE_C_COMPILER=`which mpicc` \
+  -DCMAKE_CXX_COMPILER=`which mpicxx` \
+  -DCMAKE_Fortran_COMPILER=`which mpif77` \
+  -DCMAKE_EXPORT_COMPILE_COMMANDS=0 \
+  -DSCOREC_DIR=/gpfs/u/home/PASC/PASCmrsn/scratch/dcs/install/core/lib/cmake/SCOREC/ \
+  -DCMAKE_BUILD_TYPE=Release \
+  -DCMAKE_INSTALL_PREFIX=/gpfs/u/home/PASC/PASCmrsn/scratch/dcs/install/las  \
+  -DPETSC_DIR="$PETSC_DIR" \
+  -DPETSC_ARCH="$PETSC_ARCH" \
+  -DBUILD_SPARSKIT=ON \
+  -DWITH_KOKKOS=FALSE \
+  -DBUILD_TESTS=FALSE \
+  -DCMAKE_CXX_FLAGS="-Ofast -std=c++11 -Wall" \
+  -DCMAKE_C_FLAGS="-Ofast -Wall" \
+  -DCMAKE_Fortran_FLAGS="-Ofast -Wall"
+  #-DCMAKE_C_FLAGS="-O5" \
+  #-DCMAKE_CXX_FLAGS="-O5 -sdt=c++11" \
+  #-DCMAKE_Fortran_FLAGS="-O5" \
diff --git a/scripts/config_erp.sh b/scripts/config_erp.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+PREFIX=/gpfs/u/home/PASC/PASCmrsn/scratch/install-erp/las
+
+  cmake /gpfs/u/home/PASC/PASCmrsn/barn/las/  \
+  -DCMAKE_C_COMPILER=`which mpicc` \
+  -DCMAKE_CXX_COMPILER=`which mpicxx` \
+  -DCMAKE_Fortran_COMPILER=`which mpif77` \
+  -DCMAKE_EXPORT_COMPILE_COMMANDS=0 \
+  -DSCOREC_DIR=/gpfs/u/home/PASC/PASCmrsn/scratch/install-erp/core/lib/cmake/SCOREC/ \
+  -DCMAKE_BUILD_TYPE=Release \
+  -DCMAKE_INSTALL_PREFIX="$PREFIX" \
+  -DPETSC_DIR="$PETSC_DIR" \
+  -DPETSC_ARCH="$PETSC_ARCH" \
+  -DBUILD_SPARSKIT=ON \
+  -DWITH_KOKKOS=FALSE \
+  -DBUILD_TESTS=FALSE
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
@@ -89,11 +89,11 @@ configure_file(lasConfig.h.in "${CMAKE_CURRENT_SOURCE_DIR}/lasConfig.h" @ONLY)
 set(las_headers ${las_headers} ${las_install} lasConfig.h)
 
 add_library(las STATIC ${las_sources})
-if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "XL")
+if("${HOST}" STREQUAL "q.ccni.rpi.edu")
   target_compile_definitions(las PUBLIC "-DBGQ")
 endif()
 if(${CMAKE_VERSION} VERSION_GREATER "3.8.2")
-  target_compile_features(las PUBLIC cxx_std_11)
+  #target_compile_features(las PUBLIC cxx_std_11)
 endif()
 if(CUDA_FOUND)
   set_target_properties(las PROPERTIES CUDA_SEPARABLE_COMPILATION ON)
@@ -131,7 +131,7 @@ function(add_backend backend headers libraries)
     ${CMAKE_CURRENT_BINARY_DIR}/capi/las_capi_${backend}.cc
     ${headers})
   if(${CMAKE_VERSION} VERSION_GREATER "3.8.2")
-    target_compile_features(${capi_lib} PUBLIC cxx_std_11)
+    #target_compile_features(${capi_lib} PUBLIC cxx_std_11)
   endif()
   target_include_directories(${capi_lib} PUBLIC
     $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>

diff --git a/src/las.h b/src/las.h
@@ -156,6 +156,14 @@ namespace las
      * @param cm The comm over which the vector is collective.
      */
     virtual Vec * create(unsigned lcl, unsigned bs, MPI_Comm cm) = 0;
+    /**
+     * Create a vector from an existing array
+     * @param data The array which the vector will use as data storage
+     * @param lcl The local number of rows (per-process in cm)
+     * @param bs The block size (should be the same over cm)
+     * @param cm The comm over which the vector is collective.
+     */
+    virtual Vec * create(scalar * data, unsigned lcl, unsigned bs, MPI_Comm cm)=0;
     virtual void destroy(Vec * v) = 0;
     /**
      * Create a vector suitable to act as the RHS vector to a
@@ -280,6 +288,30 @@ namespace las
   };
   template <class T>
   ScalarVecMult * getScalarVecMult();
+  class MatDiagonal
+  {
+    public:
+      virtual void exec(scalar s, Mat * m, Vec *& v) = 0;
+      virtual ~MatDiagonal() {}
+  };
+  template <class T>
+  MatDiagonal * getMatDiagonal();
+  class MatDiagonalInverse
+  {
+    public:
+      virtual void exec(scalar s, Mat * m, Vec *& v) = 0;
+      virtual ~MatDiagonalInverse() {}
+  };
+  template <class T>
+  MatDiagonalInverse * getMatDiagonalInverse();
+  class HadamardProduct
+  {
+    public:
+      virtual void exec(Vec * v1, Vec * v2, Vec * v3) = 0;
+      virtual ~HadamardProduct() {}
+  };
+  template <class T>
+  HadamardProduct * getHadamardProduct();
   /*
    * Finalize routines which must be called on a matrix when switching from
    * add mode to set mode

diff --git a/src/lasDense.cc b/src/lasDense.cc
@@ -11,8 +11,10 @@ namespace las
       lasVec * vb = getLASVec(b);
       int nr = dns->getNumRows();
       int nc = dns->getNumCols();
+#ifndef NDEBUG
       int la = va->size();
       int lb = vb->size();
+#endif
       assert(nc == la && "Matrix columns and lhs vector length must match");
       assert(nr == lb && "Matrix rows and rhs vector lenght must match");
       for(int rw = 0; rw < nr; ++rw)
@@ -29,7 +31,9 @@ namespace las
       dnsMat * y = getDnsMat(ym);
       int x_nr = x->getNumRows();
       int x_nc = x->getNumCols();
+#ifndef NDEBUG
       int y_nr = y->getNumRows();
+#endif
       int y_nc = y->getNumCols();
       assert(x_nc == y_nr && "Matrix X cols must equal matrix Y rows");
       Sparsity * dnsty = createDensity(x_nr,y_nc);

diff --git a/src/lasDense_impl.h b/src/lasDense_impl.h
@@ -99,6 +99,10 @@ namespace las
     {
       return createVector(lcl);
     }
+    virtual Vec * create(scalar * data, unsigned lcl, unsigned, MPI_Comm)
+    {
+      return createVector(data, lcl);
+    }
     virtual void destroy(Vec * v)
     {
       destroyVector(v);

diff --git a/src/lasInline.h b/src/lasInline.h
@@ -5,8 +5,8 @@
 #define LAS_INLINE inline
 #elif defined(__INTEL_COMPILER)
 #define LAS_INLINE __forceinline
-#elif defined(__xlc__)
-#define LAS_INLINE __attribute__((always_inline))
+#elif defined(__xlc__) || defined(__ibmxl__)
+#define LAS_INLINE inline __attribute__((always_inline))
 #elif defined(__GNUC__) || defined(__GNUG__)
 #define LAS_INLINE inline
 //__attribute__((always_inline))

diff --git a/src/lasPETSc_impl.h b/src/lasPETSc_impl.h
@@ -230,6 +230,15 @@ namespace las
     CHKERRABORT(LAS_COMM_WORLD, ierr);
     return reinterpret_cast<las::Vec*>(v);
   }
+  LAS_INLINE las::Vec * createPetscVector(scalar * data, unsigned l, unsigned bs, MPI_Comm cm = LAS_COMM_WORLD)
+  {
+    ::Vec * v = new ::Vec;
+    PetscErrorCode ierr = VecCreateMPIWithArray(cm, bs, l,PETSC_DECIDE,data, v);
+    CHKERRABORT(LAS_COMM_WORLD, ierr);
+    ierr = VecSetOption(*v,VEC_IGNORE_NEGATIVE_INDICES,PETSC_TRUE);
+    CHKERRABORT(LAS_COMM_WORLD, ierr);
+    return reinterpret_cast<las::Vec*>(v);
+  }
   LAS_INLINE void destroyPetscVec(las::Vec * v)
   {
     PetscErrorCode ierr = VecDestroy(getPetscVec(v));
@@ -255,6 +264,10 @@ namespace las
     {
       return createPetscVector(lcl,bs,cm);
     }
+    virtual Vec * create(scalar * data, unsigned lcl, unsigned bs, MPI_Comm cm)
+    {
+      return createPetscVector(data, lcl,bs,cm);
+    }
     virtual Vec * createRHS(Mat * m)
     {
       return createRHSVec(m);
@@ -490,6 +503,7 @@ namespace las
       if (c == nullptr)
       {
         PetscErrorCode ierr = ::MatScale(*getPetscMat(a), s);
+        CHKERRABORT(LAS_COMM_WORLD, ierr);
       }
       else
       {

diff --git a/src/lasSparse.cc b/src/lasSparse.cc
@@ -5,30 +5,6 @@ namespace las
   class sparseMatVec : public MatVecMult
   {
     public:
-      /*
-    void exec(Mat * x, Vec * a, Vec * b)
-    {
-      csrMat * cm = getCSRMat(x);
-      CSR * csr = cm->getCSR();
-      lasVec * sa = getLASVec(a);
-      lasVec * sb = getLASVec(b);
-      int nr = csr->getNumRows();
-      int nc = csr->getNumCols();
-      int la = sa->size();
-      int lb = sb->size();
-      assert(nc == la && "Matrix columns and lhs vector length must match");
-      assert(nr == lb && "Matrix rows and rhs vector length must match");
-      for (int rw = 0; rw < nr; ++rw)
-      {
-        double val = 0;
-        for (int cl = 0; cl < nc; ++cl)
-        {
-          val += (*cm)(rw, cl) * (*sa)[cl];
-        }
-        (*sb)[rw] = val;
-      }
-    }
-    */
     void exec(Mat * x, Vec * a, Vec * b)
     {
       csrMat * cm = getCSRMat(x);
@@ -39,9 +15,11 @@ namespace las
       lasVec * sa = getLASVec(a);
       lasVec * sb = getLASVec(b);
       int nr = csr->getNumRows();
+#ifndef NDEBUG
       int nc = csr->getNumCols();
       int la = sa->size();
       int lb = sb->size();
+#endif
       double val;
       assert(nc == la && "Matrix columns and lhs vector length must match");
       assert(nr == lb && "Matrix rows and rhs vector length must match");
@@ -67,7 +45,9 @@ namespace las
       CSR * y_csr = cy->getCSR();
       int x_nr = x_csr->getNumRows();
       int x_nc = x_csr->getNumCols();
+#ifndef NDEBUG
       int y_nr = y_csr->getNumRows();
+#endif
       int y_nc = y_csr->getNumCols();
       assert(x_nc == y_nr && "Matrix X cols must equal Matrix Y rows");
       CSRBuilder bld(x_nr, x_nr);
@@ -247,6 +227,67 @@ namespace las
       v3 = reinterpret_cast<Vec *>(sv3);
     }
   };
+  class sparseMatDiagonal : public MatDiagonal
+  {
+      void exec(scalar s, Mat * m, Vec *& v)
+      {
+        csrMat * cm = getCSRMat(m);
+        CSR * csr = cm->getCSR();
+        lasVec * sv;
+        assert(csr->getNumCols() == csr->getNumRows());
+        if (v)
+        {
+            destroyVector(v);
+            sv = reinterpret_cast<lasVec *>(createVector(csr->getNumRows()));
+        }
+        else
+        {
+          sv = reinterpret_cast<lasVec *>(createVector(csr->getNumRows()));
+        }
+        for(int i=0; i<csr->getNumRows(); ++i) {
+          (*sv)[i] = s*(*cm)(i,i);
+        }
+        v = reinterpret_cast<Vec *>(sv);
+      }
+  };
+  class sparseMatDiagonalInverse : public MatDiagonalInverse
+  {
+      void exec(scalar s, Mat * m, Vec *& v)
+      {
+        csrMat * cm = getCSRMat(m);
+        CSR * csr = cm->getCSR();
+        lasVec * sv;
+        assert(csr->getNumCols() == csr->getNumRows());
+        if (v)
+        {
+            destroyVector(v);
+            sv = reinterpret_cast<lasVec *>(createVector(csr->getNumRows()));
+        }
+        else
+        {
+          sv = reinterpret_cast<lasVec *>(createVector(csr->getNumRows()));
+        }
+        for(int i=0; i<csr->getNumRows(); ++i) {
+          (*sv)[i] = s/(*cm)(i,i);
+        }
+        v = reinterpret_cast<Vec *>(sv);
+      }
+  };
+  class sparseHadamardProduct : public HadamardProduct
+  {
+      void exec(Vec * v1, Vec * v2, Vec * v3)
+      {
+        lasVec * sv1 = getLASVec(v1);
+        lasVec * sv2 = getLASVec(v2);
+        lasVec * sv3 = getLASVec(v3);
+        assert(sv1->size() == sv2->size());
+        assert(sv1->size() == sv3->size());
+        for(int i=0; i<sv1->size(); ++i)
+        {
+          (*sv3)[i] = (*sv1)[i]*(*sv2)[i];
+        }
+      };
+  };
   template <>
   MatVecMult * getMatVecMult<sparse>()
   {
@@ -282,4 +323,58 @@ namespace las
     if (vva == nullptr) vva = new sparseVecVecAdd;
     return vva;
   }
+  template <>
+  LasCreateMat * getMatBuilder<sparse>(int)
+  {
+    static csrMatBuilder * mb = nullptr;
+    if(mb == nullptr)
+      mb = new csrMatBuilder;
+    return mb;
+  }
+  template <>
+  LasCreateVec * getVecBuilder<sparse>(int)
+  {
+    static csrVecBuilder * vb = nullptr;
+    if(vb == nullptr)
+      vb = new csrVecBuilder;
+    return vb;
+  }
+  template <>
+  LasOps<sparse> * getLASOps()
+  {
+    static sparse * ops = nullptr;
+    if(ops == nullptr)
+      ops = new sparse;
+    return ops;
+  }
+  template <>
+  void finalizeMatrix<sparse>(Mat * mat){};
+  template <>
+  void finalizeVector<sparse>(Vec * vec){};
+  template <>
+  void destroySparsity<sparse>(Sparsity * sprs)
+  {
+    delete reinterpret_cast<CSR*>(sprs);
+  }
+  template <>
+  MatDiagonal * getMatDiagonal<sparse>()
+  {
+    static sparseMatDiagonal * dia = nullptr;
+    if (dia == nullptr) dia = new sparseMatDiagonal;
+    return dia;
+  }
+  template <>
+  MatDiagonalInverse * getMatDiagonalInverse<sparse>()
+  {
+    static sparseMatDiagonalInverse * dia = nullptr;
+    if (dia == nullptr) dia = new sparseMatDiagonalInverse;
+    return dia;
+  }
+  template <>
+  HadamardProduct * getHadamardProduct<sparse>()
+  {
+    static sparseHadamardProduct * hp = nullptr;
+    if (hp == nullptr) hp = new sparseHadamardProduct;
+    return hp;
+  }
 }  // namespace las