a bunch of work to make mesh partitions

.github/workflows/CI.yml

@@ -18,7 +18,7 @@ jobs:
       fail-fast: false
       matrix:
         version:
-          - '1.12.0-beta3'
+          - '1.12'
           - '1.11'
           - '1.10'
         os:

Project.toml

@@ -1,7 +1,7 @@
 name = "Exodus"
 uuid = "f57ae99e-f805-4780-bdca-96e224be1e5a"
 authors = ["cmhamel <[email protected]>"]
-version = "0.14.1"
+version = "0.14.2"
 
 [deps]
 DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
@@ -25,7 +25,6 @@ MPI = "0.20"
 Meshes = "0.53"
 PartitionedArrays = "0.5"
 Test = "1"
-TestSetExtensions = "2"
 Unitful = "1"
 julia = "1"
 
@@ -35,8 +34,7 @@ MPI = "da04e1cc-30fd-572f-bb4f-1f8673147195"
 Meshes = "eacbb407-ea5a-433e-ab97-5258b1ca43fa"
 PartitionedArrays = "5a9dfac6-5c52-46f7-8278-5e2210713be9"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
-TestSetExtensions = "98d24dd4-01ad-11ea-1b02-c9a08f80db04"
 Unitful = "1986cc42-f94f-5a68-af5c-568840ba703d"
 
 [targets]
-test = ["Aqua", "MPI", "Meshes", "PartitionedArrays", "Test", "TestSetExtensions", "Unitful"]
+test = ["Aqua", "MPI", "Meshes", "PartitionedArrays", "Test", "Unitful"]

ext/ExodusPartitionedArraysExt.jl

@@ -3,6 +3,15 @@ module ExodusPartitionedArraysExt
 using Exodus
 using PartitionedArrays
 
+# New sketch
+
+# 1. Read in node cmaps
+# 2. Get node to elem connectivity
+# 3. Also get elem and id maps
+# 4. Loop over node to elem conn and 
+#    get all unique ids
+
+
 # Some helpers for IO
 function Exodus.ExodusDatabase(ranks, mesh_file::String)
   # first open nemesis file to get number of procs
@@ -16,8 +25,9 @@ function Exodus.ExodusDatabase(ranks, mesh_file::String)
     init_global = Exodus.InitializationGlobal(nem)
 
     n_nodes_global = Exodus.num_nodes(init_global)
+    n_elems_global = Exodus.num_elements(init_global)
 
-    return num_proc, n_nodes_global
+    return num_proc, n_nodes_global, n_elems_global
   end
 
   @info "Reading exodus files"
@@ -36,6 +46,36 @@ function Exodus.close(exos::V) where V <: AbstractArray{<:ExodusDatabase}
   end
 end
 
+function PartitionedArrays.partition_from_color(
+  ranks, exos,
+  global_elem_to_color, 
+  global_node_to_color
+)
+  tuple_of_arrays(map(exos, ranks) do exo, rank
+    node_map = read_id_map(exo, NodeMap)
+    node_procs = global_node_to_color[node_map]
+
+    own_nodes = findall(x -> x == rank, node_procs)
+    ghost_nodes = findall(x -> x != rank, node_procs)
+
+    new_own_nodes = convert.(Int64, node_map[own_nodes])
+    own = OwnIndices(length(global_node_to_color), rank, new_own_nodes)
+
+    new_ghost_nodes = convert.(Int64, node_map[ghost_nodes])
+    ghost_procs = global_node_to_color[new_ghost_nodes]
+    ghost = GhostIndices(length(global_node_to_color), new_ghost_nodes, ghost_procs)
+
+    dof_own_and_ghost = OwnAndGhostIndices(own, ghost, global_node_to_color)
+
+    elem_map = convert.(Int64, read_id_map(exo, ElementMap))
+    own = OwnIndices(length(global_elem_to_color), rank, elem_map)
+    ghost = GhostIndices(length(global_elem_to_color), Int64[], Int64[])
+    elem_own_and_ghost = OwnAndGhostIndices(own, ghost, global_elem_to_color)
+
+    dof_own_and_ghost, elem_own_and_ghost
+  end)
+end
+
 # PArrays overrides
 # Bug in this currently
 # function PartitionedArrays.OwnAndGhostIndices(ranks, exos, inits, global_to_color)
@@ -44,6 +84,9 @@ end
 #     internal_node_ids, internal_proc_ids = Exodus.read_internal_nodes_and_procs(rank, exo)
 # 		ghost_node_ids, ghost_proc_ids = Exodus.read_ghost_nodes_and_procs(rank, exo)
 
+#     ghost_node_ids = unique(ghost_node_ids)
+#     # ghost_proc_ids = 
+
 #     own_indices = OwnIndices(n_nodes_global, rank, internal_node_ids)
 #     ghost_indices = GhostIndices(n_nodes_global, ghost_node_ids, ghost_proc_ids)
 
@@ -52,24 +95,73 @@ end
 #   return indices
 # end
 
-# dumb for now since each proc has to read each mesh part
-function PartitionedArrays.partition_from_color(ranks, file_name::String, global_to_color)
-  parts = partition_from_color(ranks, global_to_color)
-  exos, inits = ExodusDatabase(ranks, file_name)
+# function PartitionedArrays.OwnAndGhostIndices(exos, global_to_color)
+#   n_nodes_global = length(global_to_color)
+#   map(1:length(exos), exos) do rank, exo
+#     internal_node_ids, _ = Exodus.read_internal_nodes_and_procs(rank, exo)
+#     ghost_node_ids, ghost_proc_ids = Exodus.read_ghost_nodes_and_procs(rank, exo)
+#     own_indices = OwnIndices(n_nodes_global, rank, internal_node_ids)
+#     ghost_indices = GhostIndices(n_nodes_global, ghost_node_ids, ghost_proc_ids)
+#     return OwnAndGhostIndices(own_indices, ghost_indices, global_to_color)
+#   end
+# end
 
-  # below doesn't work
-  # parts = OwnAndGhostIndices(ranks, exos, inits, global_to_color)
+# # dumb for now since each proc has to read each mesh part
+# function PartitionedArrays.partition_from_color(ranks, file_name::String, global_to_color)
+#   parts = partition_from_color(ranks, global_to_color)
+#   exos, inits = ExodusDatabase(ranks, file_name)
+#   # return OwnAndGhostIndices(ranks, exos, inits, global_to_color)
+#   # below doesn't work
+#   # parts = OwnAndGhostIndices(ranks, exos, inits, global_to_color)
+
+#   # now update ghost nodes
+#   node_procs = map(ranks, exos) do rank, exo
+#     ghost_nodes, ghost_procs = Exodus.read_ghost_nodes_and_procs(rank, exo)
+#   end
+#   ghost_nodes, ghost_procs = tuple_of_arrays(node_procs)
 
-  # now update ghost nodes
-  node_procs = map(ranks, exos) do rank, exo
-    ghost_nodes, ghost_procs = Exodus.read_ghost_nodes_and_procs(rank, exo)
-  end
-  ghost_nodes, ghost_procs = tuple_of_arrays(node_procs)
+#   parts = map(parts, ghost_nodes, ghost_procs) do part, gids, owners
+#     replace_ghost(part, gids, owners)
+#   end
+#   return parts
+
+#   # # now update ghost elems
+#   # out = map(exos, ranks) do exo, rank
+#   #   ghost_elems, ghost_procs = Exodus.read_ghost_elements_and_procs(rank, exo)
+#   # end
+#   # ghost_elems, ghost_procs = tuple_of_arrays(out)
+
+#   # parts = map(ghost_elems, ghost_procs, parts) do ge, gp, part
+#   #   replace_ghost(part, ge, gp)
+#   # end
+#   # return parts
+# end
 
-  parts = map(parts, ghost_nodes, ghost_procs) do part, gids, owners
-    replace_ghost(part, gids, owners)
-  end
-  return parts
-end
+# function PartitionedArrays.partition_from_color(ranks, file_name::String)
+#   n_procs = length(ranks) |> Int32
+#   global_nodes = Exodus.collect_global_node_numberings(file_name, n_procs)
+
+#   # open exo files
+#   exos = map(ranks) do rank
+#     ExodusDatabase(file_name * ".$(n_procs).$(lpad(rank - 1, Exodus.exodus_pad(n_procs), '0'))", "r")
+#   end
+
+#   # create element partition
+#   # TODO use actual element numbering in partition
+#   num_elems = map(exos) do exo
+#     Exodus.initialization(exo).num_elements
+#   end
+
+#   element_parts = variable_partition(num_elems, sum(num_elems))
+
+#   # create node partition
+#   # TODO modify to have dofs as well
+#   num_nodes = map(ranks) do rank
+#     filter(x -> x == rank, global_nodes) |> length
+#   end
+
+#   dof_parts = variable_partition(num_nodes, sum(num_nodes))
+#   return element_parts, dof_parts
+# end
 
 end # module

src/Decomp.jl

@@ -39,12 +39,17 @@ nem_spread_error(cmd::Cmd) = throw(NemSpreadException(cmd))
 """
 $(TYPEDSIGNATURES)
 """
-function nem_slice(file_name::String, n_procs::I) where I <: Integer
+function nem_slice(file_name::String, n_procs::I; use_nodal=false) where I <: Integer
   nem_file = file_name * ".nem"
   dir_name = dirname(file_name) * "/" # TODO this will be an issue for windows
+  if use_nodal
+    decomp_type = "-n"
+  else
+    decomp_type = "-e"
+  end
   stdout_file = joinpath(dir_name, "decomp.log")
   stderr_file = joinpath(dir_name, "decomp_err.log")
-  nem_slice_cmd = String["-e", "-S", "-l", "inertial", "-c", "-o",
+  nem_slice_cmd = String[decomp_type, "-S", "-l", "inertial", "-c", "-o",
                          "$(abspath(nem_file))", "-m", 
                          "mesh=$n_procs", "$file_name"]
 
@@ -96,7 +101,11 @@ end
 """
 $(TYPEDSIGNATURES)
 """
-function decomp(file_name::String, n_procs::I) where I <: Integer
+function decomp(
+  file_name::String, 
+  n_procs::I;
+  use_nodal=false
+) where I <: Integer
   @assert !Sys.iswindows() "This method is not supported on Windows"
   @assert isfile(file_name) "File $file_name not found in decomp. Can't proceed."
 
@@ -113,7 +122,7 @@ function decomp(file_name::String, n_procs::I) where I <: Integer
   end
 
   # nem slice first
-  nem_slice(file_name_abs, n_procs)
+  nem_slice(file_name_abs, n_procs; use_nodal=use_nodal)
   # now nem spread
   nem_spread(file_name_abs, n_procs)
 

src/ExodusTypes.jl

@@ -633,6 +633,21 @@ function ExodusDatabase{M, I, B, F}(
   )
 end
 
+
+"""
+$(TYPEDSIGNATURES)
+Method that safely does stuff by ensuring
+the file closes no matter what.
+"""
+function ExodusDatabase(f::Function, file_name::String, mode::String)
+  exo = ExodusDatabase(file_name, mode)
+  try
+    return f(exo)
+  finally
+    close(exo)
+  end
+end
+
 function _juliac_safe_rpad(s::AbstractString, n::Integer)
   len = ncodeunits(s)
   padlen = max(0, n - len)