Skip to content

Commit

Permalink
PCU2 using PCU naming, changed PCUHandle to PCU_t
Browse files Browse the repository at this point in the history
  • Loading branch information
@flagdanger
flagdanger committed Jul 5, 2024
1 parent 0dd10d1 commit 6e9a875
Show file tree
Hide file tree
Showing 24 changed files with 395 additions and 406 deletions.
1 change: 0 additions & 1 deletion mds/apfMDS.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@
*******************************************************************************/

#include <PCU2.h>
#include <PCUObj.h>
#include <lionPrint.h>
#include "apfMDS.h"
Expand Down
6 changes: 3 additions & 3 deletions mds/apfMDS.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ class Field;

/** \brief a map from global ids to vertex objects */
typedef std::map<long, MeshEntity*> GlobalToVert;
typedef struct PCUHandle PCUHandle;
typedef struct PCU_t PCU_t;


/** \brief create an empty MDS part
Expand Down Expand Up @@ -205,11 +205,11 @@ int getMdsIndex(Mesh2* in, MeshEntity* e);
MeshEntity* getMdsEntity(Mesh2* in, int dimension, int index);

Mesh2* loadMdsFromCGNS(gmi_model* g, const char* filename, CGNSBCMap& cgnsBCMap);
Mesh2* loadMdsFromCGNS2(PCUHandle h, gmi_model* g, const char* filename, CGNSBCMap& cgnsBCMap);
Mesh2* loadMdsFromCGNS2(PCU_t h, gmi_model* g, const char* filename, CGNSBCMap& cgnsBCMap);

// names of mesh data to read from file: (VERTEX, VelocityX; CellCentre, Pressure)
Mesh2* loadMdsFromCGNS(gmi_model* g, const char* filename, CGNSBCMap& cgnsBCMap, const std::vector<std::pair<std::string, std::string>>& meshData);
Mesh2* loadMdsFromCGNS2(PCUHandle h, gmi_model* g, const char* filename, CGNSBCMap& cgnsBCMap, const std::vector<std::pair<std::string, std::string>>& meshData);
Mesh2* loadMdsFromCGNS2(PCU_t h, gmi_model* g, const char* filename, CGNSBCMap& cgnsBCMap, const std::vector<std::pair<std::string, std::string>>& meshData);


int gmshMajorVersion(const char* filename);
Expand Down
76 changes: 38 additions & 38 deletions mds/mdsCGNS.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
#include "apfShape.h"
#include "gmi.h" /* this is for gmi_getline... */
#include <lionPrint.h>
#include <PCU2.h>
#include <PCU.h>
#include <apf.h>
#include <apfConvert.h>
#include "apfFieldData.h"
Expand Down Expand Up @@ -200,14 +200,14 @@ struct MeshDataGroup
};

template <typename Arg, typename... Args>
void DebugParallelPrinter(PCUHandle h, std::ostream &out, Arg &&arg, Args &&... args)
void DebugParallelPrinter(PCU_t h, std::ostream &out, Arg &&arg, Args &&... args)
{
// if constexpr (debugOutput) // probably will not get away with c++17
if (debugOutput)
{
for (int i = 0; i < PCU_Comm_Peers2(h); i++)
for (int i = 0; i < PCU_Comm_Peers(h); i++)
{
if (i == PCU_Comm_Self2(h))
if (i == PCU_Comm_Self(h))
{
out << "Rank [" << i << "] " << std::forward<Arg>(arg);
//((out << ", " << std::forward<Args>(args)), ...); // probably will not get away with c++17
Expand All @@ -223,15 +223,15 @@ void DebugParallelPrinter(PCUHandle h, std::ostream &out, Arg &&arg, Args &&...
}

template <typename... Args>
void Kill(PCUHandle h, const int fid, Args &&... args)
void Kill(PCU_t h, const int fid, Args &&... args)
{
DebugParallelPrinter(h, std::cout, "***** CGNS ERROR", args...);

if (PCU_Comm_Initialized2(h))
if (PCU_Comm_Initialized(h))
{
cgp_close(fid);
// Finalize the MPI environment.
PCU_Comm_Free2(h);
PCU_Comm_Free(h);
MPI_Finalize();
cgp_error_exit();
exit(EXIT_FAILURE);
Expand All @@ -244,15 +244,15 @@ void Kill(PCUHandle h, const int fid, Args &&... args)
}
}

void Kill(PCUHandle h, const int fid)
void Kill(PCU_t h, const int fid)
{
DebugParallelPrinter(h, std::cout, "***** CGNS ERROR");

if (PCU_Comm_Initialized2(h))
if (PCU_Comm_Initialized(h))
{
cgp_close(fid);
// Finalize the MPI environment.
PCU_Comm_Free2(h);
PCU_Comm_Free(h);
MPI_Finalize();
cgp_error_exit();
exit(EXIT_FAILURE);
Expand All @@ -265,7 +265,7 @@ void Kill(PCUHandle h, const int fid)
}
}

auto ReadCGNSCoords(PCUHandle h, int cgid, int base, int zone, int ncoords, int nverts, const std::vector<cgsize_t> &, const apf::GlobalToVert &globalToVert)
auto ReadCGNSCoords(PCU_t h, int cgid, int base, int zone, int ncoords, int nverts, const std::vector<cgsize_t> &, const apf::GlobalToVert &globalToVert)
{
// Read min required as defined by consecutive range
// make one based as ReadElements makes zero based
Expand Down Expand Up @@ -332,14 +332,14 @@ auto ReadCGNSCoords(PCUHandle h, int cgid, int base, int zone, int ncoords, int
return points;
}

void SimpleElementPartition(PCUHandle h, std::vector<cgsize_t> &numberToReadPerProc, std::vector<cgsize_t> &startingIndex, int el_start /* one based */, int el_end, int numElements)
void SimpleElementPartition(PCU_t h, std::vector<cgsize_t> &numberToReadPerProc, std::vector<cgsize_t> &startingIndex, int el_start /* one based */, int el_end, int numElements)
{
numberToReadPerProc.resize(PCU_Comm_Peers2(h), 0);
numberToReadPerProc.resize(PCU_Comm_Peers(h), 0);
const cgsize_t blockSize = static_cast<cgsize_t>(
std::floor(static_cast<double>(numElements) /
static_cast<double>(PCU_Comm_Peers2(h))));
static_cast<double>(PCU_Comm_Peers(h))));

DebugParallelPrinter(h, std::cout, "BlockSize", blockSize, "numElements", numElements, "comms Size", PCU_Comm_Peers2(h));
DebugParallelPrinter(h, std::cout, "BlockSize", blockSize, "numElements", numElements, "comms Size", PCU_Comm_Peers(h));

cgsize_t counter = 0;
if (blockSize == 0 && numElements > 0)
Expand All @@ -349,7 +349,7 @@ void SimpleElementPartition(PCUHandle h, std::vector<cgsize_t> &numberToReadPerP
bool keepGoing = true;
while (keepGoing)
{
for (int p = 0; p < PCU_Comm_Peers2(h) && keepGoing; p++)
for (int p = 0; p < PCU_Comm_Peers(h) && keepGoing; p++)
{
if (counter + blockSize <= numElements)
{
Expand All @@ -368,9 +368,9 @@ void SimpleElementPartition(PCUHandle h, std::vector<cgsize_t> &numberToReadPerP
}
DebugParallelPrinter(h, std::cout, "Sanity check: Counter", counter, "numElements", numElements);

DebugParallelPrinter(h, std::cout, "numberToReadPerProc for rank", PCU_Comm_Self2(h), "is:", numberToReadPerProc[PCU_Comm_Self2(h)]);
DebugParallelPrinter(h, std::cout, "numberToReadPerProc for rank", PCU_Comm_Self(h), "is:", numberToReadPerProc[PCU_Comm_Self(h)]);

startingIndex.resize(PCU_Comm_Peers2(h), 0);
startingIndex.resize(PCU_Comm_Peers(h), 0);
startingIndex[0] = el_start;
for (std::size_t i = 1; i < numberToReadPerProc.size(); i++)
{
Expand All @@ -381,33 +381,33 @@ void SimpleElementPartition(PCUHandle h, std::vector<cgsize_t> &numberToReadPerP
DebugParallelPrinter(h, std::cout, "Element start, end, numElements ", el_start,
el_end, numElements);

DebugParallelPrinter(h, std::cout, "startingIndex for rank", PCU_Comm_Self2(h), "is:", startingIndex[PCU_Comm_Self2(h)]);
DebugParallelPrinter(h, std::cout, "startingIndex for rank", PCU_Comm_Self(h), "is:", startingIndex[PCU_Comm_Self(h)]);

DebugParallelPrinter(h, std::cout, "Returning from SimpleElementPartition \n");
}

using Pair = std::pair<cgsize_t, cgsize_t>;
using LocalElementRanges = std::vector<Pair>; // one based

auto ReadElements(PCUHandle h, int cgid, int base, int zone, int section, int el_start /* one based */, int el_end, int numElements, int verticesPerElement, LocalElementRanges &localElementRanges)
auto ReadElements(PCU_t h, int cgid, int base, int zone, int section, int el_start /* one based */, int el_end, int numElements, int verticesPerElement, LocalElementRanges &localElementRanges)
{
std::vector<cgsize_t> numberToReadPerProc;
std::vector<cgsize_t> startingIndex;
SimpleElementPartition(h, numberToReadPerProc, startingIndex, el_start, el_end, numElements);

std::vector<cgsize_t> vertexIDs;
if (numberToReadPerProc[PCU_Comm_Self2(h)] > 0)
vertexIDs.resize(numberToReadPerProc[PCU_Comm_Self2(h)] * verticesPerElement,
if (numberToReadPerProc[PCU_Comm_Self(h)] > 0)
vertexIDs.resize(numberToReadPerProc[PCU_Comm_Self(h)] * verticesPerElement,
-1234567);

const auto start = startingIndex[PCU_Comm_Self2(h)];
const auto end = startingIndex[PCU_Comm_Self2h(h)] + numberToReadPerProc[PCU_Comm_Self2(h)] - 1;
DebugParallelPrinter(h, std::cout, "Range", start, "to", end, numberToReadPerProc[PCU_Comm_Self2(h)]);
const auto start = startingIndex[PCU_Comm_Self(h)];
const auto end = startingIndex[PCU_Comm_Self(h)] + numberToReadPerProc[PCU_Comm_Self(h)] - 1;
DebugParallelPrinter(h, std::cout, "Range", start, "to", end, numberToReadPerProc[PCU_Comm_Self(h)]);
//
cgp_elements_read_data(cgid, base, zone, section, start,
end, vertexIDs.data());

if (numberToReadPerProc[PCU_Comm_Self2(h)] > 0)
if (numberToReadPerProc[PCU_Comm_Self(h)] > 0)
{
localElementRanges.push_back(std::make_pair(start, end));
}
Expand All @@ -419,7 +419,7 @@ auto ReadElements(PCUHandle h, int cgid, int base, int zone, int section, int el
i = i - 1;
}

return std::make_tuple(vertexIDs, numberToReadPerProc[PCU_Comm_Self2(h)]);
return std::make_tuple(vertexIDs, numberToReadPerProc[PCU_Comm_Self(h)]);
}

struct CGNSBCMeta
Expand Down Expand Up @@ -856,7 +856,7 @@ struct BCInfo
}
}; // namespace

void ReadBCInfo(PCUHandle h, const int cgid, const int base, const int zone, const int nBocos, const int physDim, const int cellDim, const int nsections, std::vector<BCInfo> &bcInfos, const apf::GlobalToVert &globalToVert)
void ReadBCInfo(PCU_t h, const int cgid, const int base, const int zone, const int nBocos, const int physDim, const int cellDim, const int nsections, std::vector<BCInfo> &bcInfos, const apf::GlobalToVert &globalToVert)
{
// Read the BCS.
std::vector<CGNSBCMeta> bcMetas(nBocos);
Expand Down Expand Up @@ -1049,12 +1049,12 @@ void ReadBCInfo(PCUHandle h, const int cgid, const int base, const int zone, con
}
}

apf::Mesh2 *DoIt(PCUHandle h, gmi_model *g, const std::string &fname, apf::CGNSBCMap &cgnsBCMap, const std::vector<std::pair<std::string, std::string>> &readMeshData)
apf::Mesh2 *DoIt(PCU_t h, gmi_model *g, const std::string &fname, apf::CGNSBCMap &cgnsBCMap, const std::vector<std::pair<std::string, std::string>> &readMeshData)
{
static_assert(std::is_same<cgsize_t, int>::value, "cgsize_t not compiled as int");

int cgid = -1;
auto comm = PCU_Get_Comm2(h);
auto comm = PCU_Get_Comm(h);
cgp_mpi_comm(comm);
cgp_pio_mode(CGNS_ENUMV(CGP_INDEPENDENT));
cgp_open(fname.c_str(), CGNS_ENUMV(CG_MODE_READ), &cgid);
Expand Down Expand Up @@ -1632,7 +1632,7 @@ apf::Mesh2 *DoIt(PCUHandle h, gmi_model *g, const std::string &fname, apf::CGNSB
}

// free up memory
if (PCU_Comm_Initialized2(h))
if (PCU_Comm_Initialized(h))
cgp_close(cgid);
else
cg_close(cgid);
Expand Down Expand Up @@ -1711,7 +1711,7 @@ apf::Mesh2 *DoIt(PCUHandle h, gmi_model *g, const std::string &fname, apf::CGNSB
return mesh;
} // namespace

apf::Mesh2 *DoIt(PCUHandle h, gmi_model *g, const std::string &fname, apf::CGNSBCMap &cgnsBCMap)
apf::Mesh2 *DoIt(PCU_t h, gmi_model *g, const std::string &fname, apf::CGNSBCMap &cgnsBCMap)
{
std::vector<std::pair<std::string, std::string>> meshData;
return DoIt(h, g, fname, cgnsBCMap, meshData);
Expand All @@ -1725,11 +1725,11 @@ namespace apf
// caller needs to bring up and pull down mpi/pcu: mpi/pcu is required and assumed.
Mesh2 *loadMdsFromCGNS(gmi_model *g, const char *fname, apf::CGNSBCMap &cgnsBCMap, const std::vector<std::pair<std::string, std::string>> &meshData)
{
PCUHandle h = PCU_Get_Global_Handle();
return loadMdsFromCGNS2(h, g, fname, cgnsBCMap, meshData);
PCU_t h = PCU_Get_Global_Handle();
return loadMdsFromCGNS(h, g, fname, cgnsBCMap, meshData);
}

Mesh2 *loadMdsFromCGNS2(PCUHandle h, gmi_model *g, const char *fname, apf::CGNSBCMap &cgnsBCMap, const std::vector<std::pair<std::string, std::string>> &meshData)
Mesh2 *loadMdsFromCGNS(PCU_t h, gmi_model *g, const char *fname, apf::CGNSBCMap &cgnsBCMap, const std::vector<std::pair<std::string, std::string>> &meshData)
{
#ifdef HAVE_CGNS
Mesh2 *m = DoIt(h, g, fname, cgnsBCMap, meshData);
Expand All @@ -1746,11 +1746,11 @@ Mesh2 *loadMdsFromCGNS2(PCUHandle h, gmi_model *g, const char *fname, apf::CGNSB
// caller needs to bring up and pull down mpi/pcu: mpi/pcu is required and assumed.
Mesh2 *loadMdsFromCGNS(gmi_model *g, const char *fname, apf::CGNSBCMap &cgnsBCMap)
{
PCUHandle h = PCU_Get_Global_Handle();
return loadMdsFromCGNS2(h, g, fname, cgnsBCMap);
PCU_t h = PCU_Get_Global_Handle();
return loadMdsFromCGNS(h, g, fname, cgnsBCMap);
}

Mesh2 *loadMdsFromCGNS2(PCUHandle h, gmi_model *g, const char *fname, apf::CGNSBCMap &cgnsBCMap)
Mesh2 *loadMdsFromCGNS(PCU_t h, gmi_model *g, const char *fname, apf::CGNSBCMap &cgnsBCMap)
{
#ifdef HAVE_CGNS
Mesh2 *m = DoIt(h, g, fname, cgnsBCMap);
Expand Down
38 changes: 19 additions & 19 deletions mds/mds_apf.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
#include "mds_apf.h"
#include <stdlib.h>
#include <pcu_util.h>
#include <PCU2.h>
#include <PCU.h>

struct mds_apf* mds_apf_create(struct gmi_model* model, int d,
mds_id cap[MDS_TYPES])
Expand Down Expand Up @@ -148,17 +148,17 @@ int mds_model_id(struct mds_apf* m, struct gmi_ent* model)
return gmi_tag(m->user_model, model);
}

static void downs_to_copy(PCUHandle h, struct mds_set* s,
static void downs_to_copy(PCU_t h, struct mds_set* s,
struct mds_copy c)
{
int i;
PCU_COMM_PACK2(h, c.p, c.e);
PCU_COMM_PACK(h, c.p, c.e);
for (i = 0; i < s->n; ++i)
PCU_COMM_PACK2(h, c.p, s->e[i]);
PCU_COMM_PACK(h, c.p, s->e[i]);
}

static void downs_to_copies(
PCUHandle h, struct mds* m, mds_id e, struct mds_copies* c)
PCU_t h, struct mds* m, mds_id e, struct mds_copies* c)
{
int i;
struct mds_set s;
Expand Down Expand Up @@ -219,19 +219,19 @@ static void rotate_set(struct mds_set* in, int r, struct mds_set* out)
out->e[i] = in->e[(i + r) % in->n];
}

static int recv_down_copies(PCUHandle h, struct mds_net* net, struct mds* m)
static int recv_down_copies(PCU_t h, struct mds_net* net, struct mds* m)
{
mds_id e;
struct mds_set s;
struct mds_set rs;
struct mds_set s2;
int i;
int from = PCU_Comm_Sender2(h);
PCU_COMM_UNPACK2(h, e);
int from = PCU_Comm_Sender(h);
PCU_COMM_UNPACK(h, e);
mds_get_adjacent(m, e, mds_dim[mds_type(e)] - 1, &s);
rs.n = s.n;
for (i = 0; i < s.n; ++i)
PCU_COMM_UNPACK2(h, rs.e[i]);
PCU_COMM_UNPACK(h, rs.e[i]);
if (compare_copy_sets(net, &s, from, &rs))
return 0;
for (i = -s.n; i < s.n; ++i) {
Expand All @@ -251,55 +251,55 @@ static int copy_less(struct mds_copy a, struct mds_copy b)
return a.e < b.e;
}

static int owns_copies(PCUHandle h, mds_id e, struct mds_copies* c)
static int owns_copies(PCU_t h, mds_id e, struct mds_copies* c)
{
int i;
struct mds_copy mc;
mc.e = e;
mc.p = PCU_Comm_Self2(h);
mc.p = PCU_Comm_Self(h);
for (i = 0; i < c->n; ++i)
if (copy_less(c->c[i], mc))
return 0;
return 1;
}

static int align_copies(PCUHandle h, struct mds_net* net, struct mds* m)
static int align_copies(PCU_t h, struct mds_net* net, struct mds* m)
{
int d;
mds_id e;
struct mds_copies* c;
int did_change = 0;
for (d = 1; d < m->d; ++d){
PCU_Comm_Begin2(h);
PCU_Comm_Begin(h);
for (e = mds_begin(m, d); e != MDS_NONE; e = mds_next(m, e)) {
c = mds_get_copies(net, e);
if (!c)
continue;
if (owns_copies(h, e, c))
downs_to_copies(h, m, e, c);
}
PCU_Comm_Send2(h);
while (PCU_Comm_Receive2(h))
PCU_Comm_Send(h);
while (PCU_Comm_Receive(h))
if (recv_down_copies(h, net, m))
did_change = 1;
}
return PCU_Or2(h, did_change);
return PCU_Or(h, did_change);
}


int mds_align_matches(PCUHandle h, struct mds_apf* m)
int mds_align_matches(PCU_t h, struct mds_apf* m)
{
return align_copies(h, &m->matches, &m->mds);
}

// seol
int mds_align_ghosts(PCUHandle h, struct mds_apf* m)
int mds_align_ghosts(PCU_t h, struct mds_apf* m)
{
return align_copies(h, &m->ghosts, &m->mds);
}


int mds_align_remotes(PCUHandle h, struct mds_apf* m)
int mds_align_remotes(PCU_t h, struct mds_apf* m)
{
return align_copies(h, &m->remotes, &m->mds);
}
Expand Down
Loading

0 comments on commit 6e9a875

Please sign in to comment.