diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index b26bf003..fa272286 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -124,8 +124,8 @@ jobs: uses: actions/checkout@v3 with: repository: ORNL-MDF/Finch - # This is after the ability to extract data bounds with an input MPI comm was added - ref: 928d56776550e1a546a93197f713fc4d7e1a618d + # This is after the capability to extract multilayer temperature data was added + ref: 9f79a05ca6ff4fe31525161859e074ea881fc2b6 path: finch - name: Build Finch if: ${{ matrix.heat_transfer == 'Finch' }} @@ -173,7 +173,7 @@ jobs: if: ${{ matrix.heat_transfer == 'Finch' }} working-directory: utilities/Finch run: | - $HOME/exaca/bin/Finch-ExaCA inputs_small.json ../../examples/Inp_SmallFinch.json + $HOME/exaca/bin/Finch-ExaCA ../../examples/Inp_SmallFinch.json HIP: defaults: diff --git a/README.md b/README.md index 89fcaad7..a8169e3b 100644 --- a/README.md +++ b/README.md @@ -219,11 +219,12 @@ Alternatively, the `Finch-ExaCA` executable can be used for coupled Finch-ExaCA * `Inp_SmallFinch.json`: simulates melting and solidification of a small melt pool segment at a coarse resolution * `Inp_Finch.json`: simulates melting and solidification of a small melt pool segment at a fine resolution, repeated for 3 layers * `Inp_FinchTranslate.json`: simulates melting and solidification of the small melt pool segment at the fine resolution translated in space to form 3 overlapping segments - -For coupled Finch-ExaCA runs, caution should be taken such that the cell size given in the CA input file matches that given in the Finch input file. Additionally, `scan_path_file` in the Finch input file (e.g. for `Inp_Finch.json` this is `examples/single_line/inputs_small_refined.json` in the Finch repository) should be modified to represent a global path name. Run by calling the created executable with a Finch input file and an ExaCA input file on the command line, with the Finch input file listed first: + * `Inp_FinchVariedLayers.json`: performs 3 Finch simulations of simulates melting and solidification of a small melt pool segment at the fine resolution, where each simulation uses a unique value for laser absorption, then uses ExaCA to simulate the solidification microstructure +For simulation using the `Finch-ExaCA` executable Finch-ExaCA, both the Finch and ExaCA inputs are given in a combined input file given on the command line (see `examples/README.md` for more details on the format of this file, and the Finch README for the required Finch inputs). Alternatively, if the Finch inputs are not present in the combined Finch-ExaCA input file, and the combined simulation requires a single Finch heat transport simulation, separate input files for Finch and ExaCA can be given on the command line with the Finch input file listed first: ``` mpiexec -n 1 ./build/install/bin/Finch-ExaCA $PATH_TO_FINCH/examples/single_line/inputs_small.json examples/Inp_SmallFinch.json ``` +For coupled Finch-ExaCA runs, caution should be taken such that the cell size given in the CA input file matches that used for heat transport as given in the Finch input file or the Finch-ExaCA combined input file. Additionally, `scan_path_file` in the Finch input file or in the Finch-ExaCA combined input file (e.g. for `Inp_Finch.json` this is `examples/single_line/inputs_small_refined.json` in the Finch repository) should be modified to represent an accurate file path. ## Output and post-processing analysis diff --git a/bin/run.cpp b/bin/run.cpp index 5cdb1463..fb984388 100644 --- a/bin/run.cpp +++ b/bin/run.cpp @@ -45,6 +45,13 @@ int main(int argc, char *argv[]) { std::string input_file = argv[1]; // Read input file Inputs inputs(id, input_file); + std::string simulation_type = inputs.simulation_type; + // Full domain solidification - all cells initially liquid or active and end up solid + bool full_domain_solidification; + if ((simulation_type == "Directional") || (simulation_type == "SingleGrain")) + full_domain_solidification = true; + else + full_domain_solidification = false; // Setup local and global grids, decomposing domain (needed to construct temperature) Grid grid(inputs.simulation_type, id, np, inputs.domain.number_of_layers, inputs.domain, inputs.substrate, @@ -52,7 +59,102 @@ int main(int argc, char *argv[]) { // Temperature fields characterized by data in this structure Temperature temperature(grid, inputs.temperature, inputs.print); - runExaCA(id, np, inputs, timers, grid, temperature); + // Material response function + InterfacialResponseFunction irf(inputs.domain.deltat, grid.deltax, inputs.irf); + + // Read temperature data if necessary. For the spot problem, store spot melt data as if it were read from a + // file + if (simulation_type == "FromFile") + temperature.readTemperatureData(id, grid, 0); + else if (simulation_type == "Spot") + temperature.storeSpotData(id, grid, irf.freezingRange(), inputs.domain.deltat, + inputs.domain.spot_radius); + + // Initialize the temperature fields for the simulation type of interest. These are either simple + // unidirectional fields, or more complex data stored in a view in the temperature struct + if (full_domain_solidification) + temperature.initialize(id, simulation_type, grid, inputs.domain.deltat); + else + temperature.initialize(0, id, grid, irf.freezingRange(), inputs.domain.deltat); + MPI_Barrier(MPI_COMM_WORLD); + + // Initialize grain orientations + Orientation orientation(id, inputs.grain_orientation_file, false, inputs.rng_seed, + inputs.irf.num_phases, irf.solidificationTransformation()); + MPI_Barrier(MPI_COMM_WORLD); + + // Initialize cell types, grain IDs, and layer IDs + CellData celldata(grid, inputs.substrate, inputs.print.store_melt_pool_edge); + if (simulation_type == "Directional") + celldata.initSubstrate_Directional(id, grid, inputs.rng_seed); + else if (simulation_type == "SingleGrain") + celldata.initSubstrate_SingleGrain(id, grid); + else + celldata.initSubstrate_BaseplatePowder(id, grid, inputs.rng_seed); + MPI_Barrier(MPI_COMM_WORLD); + + // Variables characterizing the active cell region within each rank's grid, including buffers for ghost node + // data (fixed size) and the steering vector/steering vector size on host/device + Interface interface(id, grid.domain_size, inputs.substrate.init_oct_size); + // Initialize octahedra for initial active cells, if necessary for this problem type + if (full_domain_solidification) + createOctahedra_NoRemelt(grid, celldata, temperature, orientation, interface); + MPI_Barrier(MPI_COMM_WORLD); + + // Nucleation data structure, containing views of nuclei locations, time steps, and ids, and nucleation + // event counters - initialized with an estimate on the number of nuclei in the layer Without knowing + // estimated_nuclei_this_rank_this_layer yet, initialize nucleation data structures to estimated sizes, + // resize inside of placeNuclei when the number of nuclei per rank is known + int estimated_nuclei_this_rank_this_layer = + inputs.nucleation.n_max * pow(grid.deltax, 3) * grid.domain_size; + Nucleation nucleation(estimated_nuclei_this_rank_this_layer, inputs.nucleation, + celldata.num_prior_nuclei); + // Fill in nucleation data structures, and assign nucleation undercooling values to potential nucleation + // events Potential nucleation grains are only associated with liquid cells in layer 0 - they will be + // initialized for each successive layer when layer 0 is complete + nucleation.placeNuclei(simulation_type, temperature, irf, inputs.rng_seed, 0, grid, id, + inputs.domain.deltat); + + // Initialize printing struct from inputs + Print print(grid, np, inputs.print); + + // End of initialization + timers.stopInit(); + MPI_Barrier(MPI_COMM_WORLD); + + int cycle = 0; + timers.startRun(); + + // Run ExaCA to model solidification of each layer + for (int layernumber = 0; layernumber < grid.number_of_layers; layernumber++) { + timers.startLayer(); + runExaCALayer(id, np, layernumber, cycle, inputs, timers, grid, temperature, irf, orientation, celldata, + interface, nucleation, print, simulation_type, full_domain_solidification); + + if (layernumber != grid.number_of_layers - 1) { + // Initialize new temperature field data for layer "layernumber + 1" + // TODO: reorganize these temperature functions calls into a temperature.init_next_layer as done + // with the substrate If the next layer's temperature data isn't already stored, it should be read + if ((simulation_type == "FromFile") && (inputs.temperature.layerwise_temp_read)) + temperature.readTemperatureData(id, grid, layernumber + 1); + MPI_Barrier(MPI_COMM_WORLD); + + // Initialize next layer of the simulation + initExaCALayer(id, layernumber, cycle, simulation_type, inputs, grid, irf, temperature, celldata, + interface, nucleation); + MPI_Barrier(MPI_COMM_WORLD); + timers.stopLayer(layernumber); + } + else { + MPI_Barrier(MPI_COMM_WORLD); + timers.stopLayer(); + } + } + timers.stopRun(); + MPI_Barrier(MPI_COMM_WORLD); + + // Print ExaCA end-of-run data + finalizeExaCA(id, np, cycle, inputs, timers, grid, temperature, orientation, celldata, interface, print); } } // Finalize Kokkos diff --git a/bin/runCoupled.cpp b/bin/runCoupled.cpp index 21ccbba2..850f8e7c 100644 --- a/bin/runCoupled.cpp +++ b/bin/runCoupled.cpp @@ -14,57 +14,107 @@ #include #include -void runCoupled(int id, int np, Finch::Inputs finch_inputs, Inputs exaca_inputs) { - // Set up Finch +// Run Finch and store solidification event data for use by ExaCA. Either called for a specific layer, or for a series +// of layers where the return view stores time-temperature histories for layers l=0,1,2... where first_value[l] is the +// index of the first event and last_value[l]-1 is the index of the last event associated with layer l +Kokkos::View +getFinchData(const int, const int, const int first_finch_simulation, const int num_finch_simulations, + const int number_of_layers, Inputs exaca_inputs, std::array &exaca_low_corner, + std::array &exaca_high_corner, std::vector &first_value_finch, + std::vector &last_value_finch, std::string top_level_input_file_finch) { + using exec_space = Kokkos::DefaultExecutionSpace; using memory_space = exec_space::memory_space; + Kokkos::View input_temperature_data( + Kokkos::ViewAllocateWithoutInitializing("finch_temperature_data"), 0, 0); + // Run Finch, storing solidification data, domain bounds, and the locations in the view where each layer's data + // starts and ends + const int last_finch_simulation = first_finch_simulation + num_finch_simulations; + for (int finch_input_file_num = first_finch_simulation; finch_input_file_num < last_finch_simulation; + finch_input_file_num++) { + // Setup Finch simulation + // If there more more Finch simulations to be done than input files available, loop back around + const int finch_input_file_num_modulo = finch_input_file_num % (exaca_inputs.temperature.temp_files_in_series); + Finch::Inputs finch_inputs(MPI_COMM_WORLD, top_level_input_file_finch, finch_input_file_num_modulo); - // Create timers - Timers timers(id); - timers.startHeatTransfer(); - - // initialize a moving beam - Finch::MovingBeam beam(finch_inputs.source.scan_path_file); - - // Define boundary condition details. - std::array bc_types = {"adiabatic", "adiabatic", "adiabatic", - "adiabatic", "adiabatic", "adiabatic"}; - - // create the global mesh - Finch::Grid finch_grid(MPI_COMM_WORLD, finch_inputs.space.cell_size, - finch_inputs.space.global_low_corner, finch_inputs.space.global_high_corner, - finch_inputs.space.ranks_per_dim, bc_types, - finch_inputs.space.initial_temperature); - - // Run Finch heat transfer - Finch::Layer app(finch_inputs, finch_grid); - auto fd = Finch::createSolver(finch_inputs, finch_grid); - app.run(exec_space(), finch_inputs, finch_grid, beam, fd); - - // Start of ExaCA init time. - timers.stopHeatTransfer(); - timers.startInit(); - - // Setup local and global grids, decomposing domain (needed to construct temperature) - // ExaCA either uses the same bounds as Finch, or a bounding box that only includes the regions that underwent - // melting and solidification - std::array exaca_low_corner, exaca_high_corner; - if (exaca_inputs.temperature.trim_unmelted_region) { - exaca_low_corner = app.getLowerSolidificationDataBounds(MPI_COMM_WORLD); - exaca_high_corner = app.getUpperSolidificationDataBounds(MPI_COMM_WORLD); - } - else { - exaca_low_corner = finch_inputs.space.global_low_corner; - exaca_high_corner = finch_inputs.space.global_high_corner; + // initialize a moving beam + Finch::MovingBeam beam(finch_inputs.source.scan_path_file); + + // Define boundary condition details. + std::array bc_types = {"adiabatic", "adiabatic", "adiabatic", + "adiabatic", "adiabatic", "adiabatic"}; + + // create the global mesh + Finch::Grid finch_grid(MPI_COMM_WORLD, finch_inputs.space.cell_size, + finch_inputs.space.global_low_corner, + finch_inputs.space.global_high_corner, finch_inputs.space.ranks_per_dim, + bc_types, finch_inputs.space.initial_temperature); + + // Run Finch heat transfer + Finch::Layer app(finch_inputs, finch_grid); + auto fd = Finch::createSolver(finch_inputs, finch_grid); + app.run(exec_space(), finch_inputs, finch_grid, beam, fd); + + // x,y,z bounds of Finch simulation domain from this layer + std::array finch_low_corner_layer = finch_inputs.space.global_low_corner; + std::array finch_high_corner_layer = finch_inputs.space.global_high_corner; + + // x,y,z bounds of Finch solidification data for this layer + std::array data_low_corner_layer = app.getLowerSolidificationDataBounds(MPI_COMM_WORLD); + std::array data_high_corner_layer = app.getUpperSolidificationDataBounds(MPI_COMM_WORLD); + + // x,y,z bounds used by ExaCA - either the Finch bounds of the solidification data bounds + std::array exaca_low_corner_layer, exaca_high_corner_layer; + if (exaca_inputs.temperature.trim_unmelted_region_xy) { + exaca_low_corner_layer[0] = data_low_corner_layer[0]; + exaca_high_corner_layer[0] = data_high_corner_layer[0]; + exaca_low_corner_layer[1] = data_low_corner_layer[1]; + exaca_high_corner_layer[1] = data_high_corner_layer[1]; + } + else { + exaca_low_corner_layer[0] = finch_low_corner_layer[0]; + exaca_high_corner_layer[0] = finch_high_corner_layer[0]; + exaca_low_corner_layer[1] = finch_low_corner_layer[1]; + exaca_high_corner_layer[1] = finch_high_corner_layer[1]; + } + if (exaca_inputs.temperature.trim_unmelted_region_z) { + exaca_low_corner_layer[2] = data_low_corner_layer[2]; + exaca_high_corner_layer[2] = data_low_corner_layer[2]; + } + else { + exaca_low_corner_layer[2] = finch_low_corner_layer[2]; + exaca_high_corner_layer[2] = finch_high_corner_layer[2]; + } + + // If first layer, ExaCA bounds are equal to this layer's bounds + if (finch_input_file_num == 0) { + for (int i = 0; i < 3; i++) { + exaca_low_corner[i] = exaca_low_corner_layer[i]; + exaca_high_corner[i] = exaca_high_corner_layer[i]; + } + } + else { + // If on a later layer, ExaCA bounds must span bounds of all individual layers + for (int i = 0; i < 3; i++) { + exaca_low_corner[i] = std::min(exaca_low_corner_layer[i], exaca_low_corner[i]); + exaca_high_corner[i] = std::max(exaca_high_corner_layer[i], exaca_high_corner[i]); + } + } + + // Append this layer's solidification data to input_temperature_data + app.appendSolidificationData(input_temperature_data, first_value_finch, last_value_finch, finch_input_file_num, + num_finch_simulations); + // If performing multiple finch simulations during initialization, fill first/last_value_finch with values for + // repeated data + if (num_finch_simulations > 1) { + for (int repeated_layer = num_finch_simulations; repeated_layer < number_of_layers; repeated_layer++) { + const int repeated_file = repeated_layer % num_finch_simulations; + first_value_finch[repeated_layer] = first_value_finch[repeated_file]; + last_value_finch[repeated_layer] = last_value_finch[repeated_file]; + } + } } - Grid exaca_grid(id, np, exaca_inputs.domain, exaca_inputs.substrate, exaca_inputs.temperature, - finch_inputs.space.cell_size, exaca_low_corner, exaca_high_corner); - // Temperature fields characterized by data in this structure - Temperature temperature(id, np, exaca_grid, exaca_inputs.temperature, exaca_inputs.print, - app.getSolidificationData()); - - // Now run ExaCA - runExaCA(id, np, exaca_inputs, timers, exaca_grid, temperature); + return input_temperature_data; } int main(int argc, char *argv[]) { @@ -77,6 +127,8 @@ int main(int argc, char *argv[]) { MPI_Comm_size(MPI_COMM_WORLD, &np); // Get individual process ID MPI_Comm_rank(MPI_COMM_WORLD, &id); + using exec_space = Kokkos::DefaultExecutionSpace; + using memory_space = exec_space::memory_space; if (id == 0) { std::cout << "ExaCA version: " << version() << " \nExaCA commit: " << gitCommitHash() @@ -84,21 +136,137 @@ int main(int argc, char *argv[]) { Kokkos::DefaultExecutionSpace().print_configuration(std::cout); std::cout << "Number of MPI ranks = " << np << std::endl; } - if (argc < 3) { - throw std::runtime_error("Error: Must provide path to Finch and ExaCA input files on the command line."); + std::string top_level_input_file_finch, top_level_input_file_exaca; + if (argc < 2) + throw std::runtime_error("Error: Must provide path to input file on the command line."); + else if (argc == 2) { + // Should contain both Finch and ExaCA inputs + top_level_input_file_finch = argv[1]; + top_level_input_file_exaca = argv[1]; } else { - std::string finch_input_file = argv[1]; - std::string exaca_input_file = argv[2]; + // Finch inputs in first file, ExaCA inputs in second file + top_level_input_file_finch = argv[1]; + top_level_input_file_exaca = argv[2]; + } + // Initialize ExaCA simulation inputs using ExaCA section of top level input file + Inputs inputs(id, top_level_input_file_exaca); + + // Create timers + Timers timers(id); + + // ExaCA either uses the same bounds as Finch, or a bounding box that only includes the regions that underwent + // melting and solidification + std::array exaca_low_corner, exaca_high_corner; + // Should Finch simulations for all layers be performed and time-temperature history data stored prior to + // running ExaCA, or should each Finch simulation be run one at a time between ExaCA-simulated layers? + int num_finch_simulations; + if (inputs.temperature.layerwise_temp_read) + num_finch_simulations = 1; + else + num_finch_simulations = inputs.temperature.temp_files_in_series; + // Store values denoting which portions of input_temperature_data belong to which simulated layer in Finch + std::vector first_value_finch(inputs.domain.number_of_layers), + last_value_finch(inputs.domain.number_of_layers); + // View for time-temperature history data from Finch - run either just the first layer, or run all Finch + // simulations and store results + timers.startHeatTransfer(); + Kokkos::View input_temperature_data = + getFinchData(id, np, 0, num_finch_simulations, inputs.domain.number_of_layers, inputs, exaca_low_corner, + exaca_high_corner, first_value_finch, last_value_finch, top_level_input_file_finch); + MPI_Barrier(MPI_COMM_WORLD); + timers.stopHeatTransfer(); + + // Start of ExaCA init time. + timers.startInit(); + // Initialize ExaCA grid, using bounds of input Finch data + Grid grid(id, np, inputs.domain, inputs.substrate, inputs.temperature, inputs.domain.deltax, exaca_low_corner, + exaca_high_corner); + + // Material response function + InterfacialResponseFunction irf(inputs.domain.deltat, grid.deltax, inputs.irf); + + // Temperature fields characterized by data in this structure - rearrange data in input_temperature_data to + // match ExaCA's domain decomposition. Store which temperature data is associated with which layers + Temperature temperature(id, np, grid, inputs.temperature, inputs.print, input_temperature_data, + first_value_finch, last_value_finch); + temperature.initialize(0, id, grid, irf.freezingRange(), inputs.domain.deltat); + MPI_Barrier(MPI_COMM_WORLD); + + // Initialize grain orientations + Orientation orientation(id, inputs.grain_orientation_file, false, inputs.rng_seed, + inputs.irf.num_phases, irf.solidificationTransformation()); + MPI_Barrier(MPI_COMM_WORLD); - // Setup Finch simulation - Finch::Inputs finch_inputs(MPI_COMM_WORLD, finch_input_file); + // Initialize cell types, grain IDs, and layer IDs + CellData celldata(grid, inputs.substrate, inputs.print.store_melt_pool_edge); + celldata.initSubstrate_BaseplatePowder(id, grid, inputs.rng_seed); + MPI_Barrier(MPI_COMM_WORLD); - // Setup ExaCA simulation details - Inputs inputs(id, exaca_input_file); + // Variables characterizing the active cell region within each rank's grid, including buffers for ghost node + // data (fixed size) and the steering vector/steering vector size on host/device + Interface interface(id, grid.domain_size, inputs.substrate.init_oct_size); + MPI_Barrier(MPI_COMM_WORLD); - runCoupled(id, np, finch_inputs, inputs); + // Nucleation data structure, containing views of nuclei locations, time steps, and ids, and nucleation event + // counters - initialized with an estimate on the number of nuclei in the layer Without knowing + // estimated_nuclei_this_rank_this_layer yet, initialize nucleation data structures to estimated sizes, resize + // inside of placeNuclei when the number of nuclei per rank is known + int estimated_nuclei_this_rank_this_layer = inputs.nucleation.n_max * pow(grid.deltax, 3) * grid.domain_size; + Nucleation nucleation(estimated_nuclei_this_rank_this_layer, inputs.nucleation); + // Fill in nucleation data structures, and assign nucleation undercooling values to potential nucleation events + // Potential nucleation grains are only associated with liquid cells in layer 0 - they will be initialized for + // each successive layer when layer 0 is complete + nucleation.placeNuclei("FromFinch", temperature, irf, inputs.rng_seed, 0, grid, id, inputs.domain.deltat); + + // Initialize printing struct from inputs + Print print(grid, np, inputs.print); + + // End of initialization + timers.stopInit(); + MPI_Barrier(MPI_COMM_WORLD); + + int cycle = 0; + timers.startRun(); + + // Run ExaCA to model solidification of each layer + for (int layernumber = 0; layernumber < grid.number_of_layers; layernumber++) { + timers.startLayer(); + runExaCALayer(id, np, layernumber, cycle, inputs, timers, grid, temperature, irf, orientation, celldata, + interface, nucleation, print, "FromFinch", false); + + if (layernumber != grid.number_of_layers - 1) { + + // Initialize new temperature field data for layer "layernumber + 1" - this is either stored in the + // temperature struct, or comes from a new Finch simulation + if (inputs.temperature.layerwise_temp_read) { + timers.startHeatTransfer(); + input_temperature_data = + getFinchData(id, np, layernumber + 1, num_finch_simulations, inputs.domain.number_of_layers, + inputs, exaca_low_corner, exaca_high_corner, first_value_finch, last_value_finch, + top_level_input_file_finch); + timers.stopHeatTransfer(); + temperature.copyTemperatureData(id, np, layernumber + 1, grid, input_temperature_data, + first_value_finch, last_value_finch); + } + MPI_Barrier(MPI_COMM_WORLD); + + // Initialize next layer of the simulation + initExaCALayer(id, layernumber, cycle, "FromFinch", inputs, grid, irf, temperature, celldata, interface, + nucleation); + MPI_Barrier(MPI_COMM_WORLD); + timers.stopLayer(layernumber); + } + else { + MPI_Barrier(MPI_COMM_WORLD); + timers.stopLayer(); + } } + timers.stopRun(); + MPI_Barrier(MPI_COMM_WORLD); + + // Print ExaCA end-of-run data + finalizeExaCA(id, np, cycle, inputs, timers, grid, temperature, orientation, celldata, interface, print); } // Finalize Kokkos & MPI diff --git a/examples/Inp_Finch.json b/examples/Inp_Finch.json index a8845822..c5e7ae65 100644 --- a/examples/Inp_Finch.json +++ b/examples/Inp_Finch.json @@ -1,32 +1,68 @@ { - "SimulationType": "FromFinch", - "MaterialFileName": "Inconel625.json", - "GrainOrientationFile": "GrainOrientationVectors.csv", - "RandomSeed": 0, - "Domain": { - "CellSize": 3, - "TimeStep": 0.08, - "NumberOfLayers": 3, - "LayerOffset": 8 - }, - "Nucleation": { - "Density": 100, - "MeanUndercooling": 5, - "StDev": 0.5 - }, - "TemperatureData": { - "TrimUnmeltedRegion": true - }, - "Substrate": { - "MeanBaseplateGrainSize": 50 - }, - "Printing": { - "PathToOutput": "./", - "OutputFile": "TestProblemFinch", - "PrintBinary": true, - "PrintExaConstitSize": 0, - "Interlayer": { - "Fields": ["LayerID", "GrainMisorientation"] - } - } + "Finch": { + "time": { + "Co": 0.125, + "start_time": 0.0, + "end_time": 0.0015, + "total_output_steps": 2, + "total_monitor_steps": 10 + }, + "space": { + "initial_temperature": 300.0, + "cell_size": 3e-6, + "global_low_corner": [-210e-6, -180e-6, -180e-6], + "global_high_corner": [300e-6, 180e-6, 0.0], + "ranks_per_dim": [2, 2, 1] + }, + "properties": { + "density": 7500.0, + "specific_heat": 750.0, + "thermal_conductivity": 25.0, + "latent_heat": 2e5, + "solidus": 1410.0, + "liquidus": 1620.0 + }, + "source": { + "absorption": 0.3, + "two_sigma": [60e-6, 60e-6, 60e-6], + "scan_path_file": "../Finch/examples/single_line/scan_path_small.txt" + }, + "sampling": { + "type": "solidification_data", + "format": "default", + "directory_name": "single_line_small_exaca" + } + }, + "ExaCA": { + "SimulationType": "FromFinch", + "MaterialFileName": "Inconel625.json", + "GrainOrientationFile": "GrainOrientationVectors.csv", + "RandomSeed": 0, + "Domain": { + "CellSize": 3, + "TimeStep": 0.08, + "NumberOfLayers": 3, + "LayerOffset": 8 + }, + "Nucleation": { + "Density": 100, + "MeanUndercooling": 5, + "StDev": 0.5 + }, + "TemperatureData": { + "TrimUnmeltedRegion": true + }, + "Substrate": { + "MeanBaseplateGrainSize": 50 + }, + "Printing": { + "PathToOutput": "./", + "OutputFile": "TestProblemFinch", + "PrintBinary": true, + "PrintExaConstitSize": 0, + "Interlayer": { + "Fields": ["GrainID", "LayerID", "GrainMisorientation"] + } + } + } } diff --git a/examples/Inp_FinchTranslate.json b/examples/Inp_FinchTranslate.json index 058bd81c..667cceea 100644 --- a/examples/Inp_FinchTranslate.json +++ b/examples/Inp_FinchTranslate.json @@ -1,36 +1,72 @@ { - "SimulationType": "FromFinch", - "MaterialFileName": "Inconel625.json", - "GrainOrientationFile": "GrainOrientationVectors.csv", - "RandomSeed": 0, - "Domain": { - "CellSize": 3, - "TimeStep": 0.08, - "NumberOfLayers": 1, - "LayerOffset": 8 - }, - "Nucleation": { - "Density": 100, - "MeanUndercooling": 5, - "StDev": 0.5 - }, - "Substrate": { - "MeanBaseplateGrainSize": 50 - }, - "TemperatureData": { - "TranslationCount": 2, - "OffsetDirection": "Y", - "SpatialOffset": 40, - "TemporalOffset": 1600, - "AlternatingDirection": true - }, - "Printing": { - "PathToOutput": "./", - "OutputFile": "TestProblemFinchTranslate", - "PrintBinary": true, - "PrintExaConstitSize": 0, - "Interlayer": { - "Fields": ["CritTimeStep", "LayerID", "GrainMisorientation"] - } - } + "Finch": { + "time": { + "Co": 0.125, + "start_time": 0.0, + "end_time": 0.0015, + "total_output_steps": 2, + "total_monitor_steps": 10 + }, + "space": { + "initial_temperature": 300.0, + "cell_size": 3e-6, + "global_low_corner": [-210e-6, -180e-6, -180e-6], + "global_high_corner": [300e-6, 180e-6, 0.0], + "ranks_per_dim": [2, 2, 1] + }, + "properties": { + "density": 7500.0, + "specific_heat": 750.0, + "thermal_conductivity": 25.0, + "latent_heat": 2e5, + "solidus": 1410.0, + "liquidus": 1620.0 + }, + "source": { + "absorption": 0.3, + "two_sigma": [60e-6, 60e-6, 60e-6], + "scan_path_file": "../Finch/examples/single_line/scan_path_small.txt" + }, + "sampling": { + "type": "solidification_data", + "format": "default", + "directory_name": "single_line_small_exaca" + } + }, + "ExaCA": { + "SimulationType": "FromFinch", + "MaterialFileName": "Inconel625.json", + "GrainOrientationFile": "GrainOrientationVectors.csv", + "RandomSeed": 0, + "Domain": { + "CellSize": 3, + "TimeStep": 0.08, + "NumberOfLayers": 1, + "LayerOffset": 8 + }, + "Nucleation": { + "Density": 100, + "MeanUndercooling": 5, + "StDev": 0.5 + }, + "Substrate": { + "MeanBaseplateGrainSize": 50 + }, + "TemperatureData": { + "TranslationCount": 2, + "OffsetDirection": "Y", + "SpatialOffset": 40, + "TemporalOffset": 1600, + "AlternatingDirection": true + }, + "Printing": { + "PathToOutput": "./", + "OutputFile": "TestProblemFinchTranslate", + "PrintBinary": true, + "PrintExaConstitSize": 0, + "Interlayer": { + "Fields": ["CritTimeStep", "LayerID", "GrainMisorientation"] + } + } + } } diff --git a/examples/Inp_FinchVariedLayers.json b/examples/Inp_FinchVariedLayers.json new file mode 100644 index 00000000..36356bb1 --- /dev/null +++ b/examples/Inp_FinchVariedLayers.json @@ -0,0 +1,87 @@ +{ + "Finch": { + "time": { + "Co": 0.125, + "start_time": 0.0, + "end_time": 0.0015, + "total_output_steps": 2, + "total_monitor_steps": 10 + }, + "space": { + "initial_temperature": 300.0, + "cell_size": 3e-6, + "global_low_corner": [-210e-6, -180e-6, -180e-6], + "global_high_corner": [300e-6, 180e-6, 0.0], + "ranks_per_dim": [2, 2, 1] + }, + "properties": { + "density": 7500.0, + "specific_heat": 750.0, + "thermal_conductivity": 25.0, + "latent_heat": 2e5, + "solidus": 1410.0, + "liquidus": 1620.0 + }, + "sampling": { + "type": "solidification_data", + "format": "default", + "directory_name": "single_line_small_exaca" + }, + "layers": [ + { + "source": { + "absorption": 0.4, + "two_sigma": [60e-6, 60e-6, 60e-6], + "scan_path_file": "../Finch/examples/single_line/scan_path_small.txt" + } + }, + { + "source": { + "absorption": 0.3, + "two_sigma": [60e-6, 60e-6, 60e-6], + "scan_path_file": "../Finch/examples/single_line/scan_path_small.txt" + } + }, + { + "source": { + "absorption": 0.2, + "two_sigma": [60e-6, 60e-6, 60e-6], + "scan_path_file": "../Finch/examples/single_line/scan_path_small.txt" + } + } + ] + }, + "ExaCA": { + "SimulationType": "FromFinch", + "MaterialFileName": "Inconel625.json", + "GrainOrientationFile": "GrainOrientationVectors.csv", + "RandomSeed": 0, + "Domain": { + "CellSize": 3, + "TimeStep": 0.08, + "NumberOfLayers": 3, + "LayerOffset": 8 + }, + "Nucleation": { + "Density": 100, + "MeanUndercooling": 5, + "StDev": 0.5 + }, + "TemperatureData": { + "TrimUnmeltedRegion": true, + "LayerwiseTempRead": false + }, + "Substrate": { + "MeanBaseplateGrainSize": 50 + }, + "Printing": { + "PathToOutput": "./", + "OutputFile": "TestProblemFinchVariedLayers", + "PrintBinary": true, + "PrintExaConstitSize": 0, + "Interlayer": { + "Fields": ["GrainID", "LayerID", "GrainMisorientation"] + } + } + } +} diff --git a/examples/Inp_SmallFinch.json b/examples/Inp_SmallFinch.json index 98736701..b7662945 100644 --- a/examples/Inp_SmallFinch.json +++ b/examples/Inp_SmallFinch.json @@ -1,29 +1,65 @@ { - "SimulationType": "FromFinch", - "MaterialFileName": "Inconel625.json", - "GrainOrientationFile": "GrainOrientationVectors.csv", - "RandomSeed": 0, - "Domain": { - "CellSize": 10, - "TimeStep": 0.2, - "NumberOfLayers": 1, - "LayerOffset": 8 - }, - "Nucleation": { - "Density": 100, - "MeanUndercooling": 5, - "StDev": 0.5 - }, - "Substrate": { - "MeanBaseplateGrainSize": 75 - }, - "Printing": { - "PathToOutput": "./", - "OutputFile": "TestProblemSmallFinch", - "PrintBinary": true, - "PrintExaConstitSize": 0, - "Interlayer": { - "Fields": ["LayerID", "GrainMisorientation"] - } - } + "Finch": { + "time": { + "Co": 0.125, + "start_time": 0.0, + "end_time": 0.0015, + "total_output_steps": 2, + "total_monitor_steps": 10 + }, + "space": { + "initial_temperature": 300.0, + "cell_size": 10e-6, + "global_low_corner": [-2e-4, -2e-4, -2e-4], + "global_high_corner": [3e-4, 2e-4, 0.0], + "ranks_per_dim": [1, 1, 1] + }, + "properties": { + "density": 7500.0, + "specific_heat": 750.0, + "thermal_conductivity": 25.0, + "latent_heat": 2e5, + "solidus": 1410.0, + "liquidus": 1620.0 + }, + "source": { + "absorption": 0.3, + "two_sigma": [60e-6, 60e-6, 60e-6], + "scan_path_file": "scan_path_small.txt" + }, + "sampling": { + "type": "solidification_data", + "format": "default", + "directory_name": "single_line_small" + } + }, + "ExaCA": { + "SimulationType": "FromFinch", + "MaterialFileName": "Inconel625.json", + "GrainOrientationFile": "GrainOrientationVectors.csv", + "RandomSeed": 0, + "Domain": { + "CellSize": 10, + "TimeStep": 0.2, + "NumberOfLayers": 1, + "LayerOffset": 8 + }, + "Nucleation": { + "Density": 100, + "MeanUndercooling": 5, + "StDev": 0.5 + }, + "Substrate": { + "MeanBaseplateGrainSize": 75 + }, + "Printing": { + "PathToOutput": "./", + "OutputFile": "TestProblemSmallFinch", + "PrintBinary": true, + "PrintExaConstitSize": 0, + "Interlayer": { + "Fields": ["LayerID", "GrainMisorientation"] + } + } + } } diff --git a/examples/README.md b/examples/README.md index 7cdcd481..3756e6c6 100644 --- a/examples/README.md +++ b/examples/README.md @@ -11,12 +11,12 @@ ExaCA currently can model five types of problems: * The top surface (the largest Z coordinate in a file) is assumed to be flat. Additionally, if multiple temperature files are being used (for example, a scan pattern consisting of 10 layers of repeating even and odd file data), the Z coordinate corresponding to this flat top surface should be the same for all files. * Alternatively, if a time-temperature history file has the extension `.catemp`, it will be parsed as a binary string. The binary form for these files does not contain commas, newlines, nor a header, but consists of sequential x,y,z,tm,tl,cr,x,y,z,tm,tl,cr... data as double precision values (little endian). This is often a significantly smaller file size than the standard format, and will be faster to read during initialization. * The M is no longer required in the problem type, as problem type R is now the same as RM (it now includes multiple melting and solidification events per cell) -* Problem type `FromFinch` uses time-temperature history data generated from Finch, and requires Finch (and as a result, Cabana) as a dependency and two input files: a Finch input file and an ExaCA input file (in that order). Running using this problem type requires use of the `Finch-ExaCA` executable - currently, multilayer simulations are supported, but only using data from a single finch run (i.e., time-temperature history data for every layer of the ExaCA simulation is the same) +* Problem type `FromFinch` uses time-temperature history data generated from Finch, and requires Finch (and as a result, Cabana) as a dependency as well as an input file containing inputs for both codes. Running using this problem type requires use of the `Finch-ExaCA` executable. # ExaCA top level input files -The .json files in the examples subdirectory are provided on the command line to let ExaCA know which problem is being simulated. The json files contain different input sections, with certain inputs only used for certain problem types. For optional inputs, the default value used is noted. +The .json files in the examples subdirectory are provided on the command line to let ExaCA know which problem is being simulated. The json files contain different input sections, with certain inputs only used for certain problem types. For optional inputs, the default value used is noted. There are two types of input files for ExaCA: "singular" (ExaCA) input files and "combined" (Finch-ExaCA). For combined input files, the top level input sections are `Finch` and `ExaCA`, with subsections `SimulationType`, `MaterialFileName`, `GrainOrientationsFile`, `RandomSeed`, `Domain`, `Nucleation`, `TemperatureData`, `Substrate`, and `Printing` in the ExaCA portion of the file and subsections as given in the Finch README for the Finch portion of the file. Note that for the Finch section of these files, inputs that vary on different layers (for multilayer simulations) can be given in a `layers` list; see `examples/Inp_FinchVariedLayers.json` for an example of this. For singular input files, the aforemenioned ExaCA subsections are top level inputs. Also note that the file path to the Finch scan path file(s) given in the input file may need to be adjusted depending on the location of your Finch build and associated input files. -## Top level inputs +## Top level inputs (individual input files) or ExaCA level inputs (combined input files) |Input | Details | |------------------------|---------| | SimulationType | @@ -58,7 +58,7 @@ The .json files in the examples subdirectory are provided on the command line to |--------------|-------------------------|---------| |G | Directional, Spot, SingleGrain | Thermal gradient in the build (+Z) directions, in K/m |R | Directional, Spot, SingleGrain | Cooling rate (uniform across the domain), in K/s -|LayerwiseTempRead | FromFile | If set to true, the appropriate temperature data will be read during each layer's initialization, stored temporarily, and discarded. If set to false, temperature data for all layers will be read and stored during code initialization, and initialization of each layer will be performed using this stored temperature data. Setting this to true is only recommended if a large quantity of temperature data is read by ExaCA (for example, a 10 layer simulation where each layer's temperature data comes from a different file). +|LayerwiseTempRead | FromFile, FromFinch | If set to true, the appropriate temperature data will be read (FromFile) or generated (FromFinch) during each layer's initialization, stored temporarily, and discarded. If set to false, temperature data for all layers will be read/generated and stored during code initialization, and initialization of each layer will be performed using this stored temperature data. Setting this to true is only recommended if a large quantity of temperature data is read by ExaCA (for example, a 10 layer simulation where each layer's temperature data comes from a different file or Finch simulation). |TemperatureFiles | FromFile | List of files corresponding to each layer's temperature data, in the form ["filename1.csv","filename2.csv",...]. If the number of entries is less than NumberOfLayers, the list is repeated. Note that if the Z coordinate of the top surface for each data set has the layer offset applied, LayerOffset in the "Domain" section of the input file should be set to 0, to avoid offsetting the layers twice. |InitUndercooling | Directional, SingleGrain | For SingleGrain, this is the undercooling at the location of the seeded grain. For problem type Directional, this is an optional argument (defaulting to zero) for the initial undercooling at the domain's bottom surface |TrimUnmeltedRegion | FromFinch | Should ExaCA use only the bounds of the region in the Finch data that underwent solidification? ("true" or "false"). Selecting "false" will give ExaCA the same simulation bounds as Finch, which will allocate a larger domain and may be slower to simulate; however, this may be desirable in the case of a series of simulations that should all use the same fixed XYZ bounds diff --git a/src/CAinputdata.hpp b/src/CAinputdata.hpp index a9f30c92..af533be2 100644 --- a/src/CAinputdata.hpp +++ b/src/CAinputdata.hpp @@ -77,7 +77,7 @@ struct TemperatureInputs { double G = 0, R = 0; double init_undercooling = 0.0; // Used for FromFinch and FromFile problem types with translated temperature data - bool trim_unmelted_region = false; + bool trim_unmelted_region_xy = false, trim_unmelted_region_z = false; bool use_fixed_x_bounds = false, use_fixed_y_bounds = false; std::vector temperature_x_bounds = {std::numeric_limits::lowest(), std::numeric_limits::max()}; diff --git a/src/CAinputs.hpp b/src/CAinputs.hpp index c9f19fb8..550b40f3 100644 --- a/src/CAinputs.hpp +++ b/src/CAinputs.hpp @@ -42,8 +42,8 @@ struct Inputs { : file_name(input_file) { // Open and read JSON input file - std::ifstream input_data_stream(input_file); - nlohmann::json input_data = nlohmann::json::parse(input_data_stream); + bool contains_finch_inputs; + nlohmann::json input_data = getInputDataObject(input_file, contains_finch_inputs); // General inputs simulation_type = input_data["SimulationType"]; @@ -144,15 +144,60 @@ struct Inputs { // Temperature inputs: if ((simulation_type == "FromFile") || (simulation_type == "FromFinch")) { - if (simulation_type == "FromFile") { - if ((input_data["TemperatureData"].contains("HeatTransferCellSize")) && (id == 0)) - std::cout << "Note: Heat transport data cell size is no longer an input used in ExaCA, temperature " - "data must be at the same resolution as the CA cell size" - << std::endl; - // Read all temperature files at once (default), or one at a time? - if (input_data["TemperatureData"].contains("LayerwiseTempRead")) { - temperature.layerwise_temp_read = input_data["TemperatureData"]["LayerwiseTempRead"]; + // Read/store all temperature files at once (default), or one at a time? + if (input_data["TemperatureData"].contains("LayerwiseTempRead")) + temperature.layerwise_temp_read = input_data["TemperatureData"]["LayerwiseTempRead"]; + // Specified region in X and Y for simulation + if (input_data["TemperatureData"].contains("XRegion")) { + temperature.temperature_x_bounds[0] = input_data["TemperatureData"]["XRegion"][0]; + temperature.temperature_x_bounds[1] = input_data["TemperatureData"]["XRegion"][1]; + temperature.use_fixed_x_bounds = true; + } + if (input_data["TemperatureData"].contains("YRegion")) { + temperature.temperature_y_bounds[0] = input_data["TemperatureData"]["YRegion"][0]; + temperature.temperature_y_bounds[1] = input_data["TemperatureData"]["YRegion"][1]; + temperature.use_fixed_y_bounds = true; + } + if ((input_data["TemperatureData"].contains("HeatTransferCellSize")) && (id == 0)) + std::cout << "Note: Heat transport data cell size is no longer an input used in ExaCA, temperature " + "data must be at the same resolution as the CA cell size" + << std::endl; + // Optional translation/mirroring of input temperature data + if (input_data["TemperatureData"].contains("TranslationCount")) { + temperature.number_of_copies = input_data["TemperatureData"]["TranslationCount"]; + // Include the original in the total number of copies (number of translations + 1) + temperature.number_of_copies++; + std::string offset_direction = input_data["TemperatureData"]["OffsetDirection"]; + // Offsets are given in cells (for consistency with layer height) and microseconds (for consistency + // with time step) but stored in meters and seconds for consistency with how cell size and time step + // are treated + if (offset_direction == "X") { + // X offset from input file, Y defaults to 0 + temperature.x_offset = input_data["TemperatureData"]["SpatialOffset"]; + } + else if (offset_direction == "Y") { + // Y offset from input file, X defaults to 0 + temperature.y_offset = input_data["TemperatureData"]["SpatialOffset"]; + } + else + throw std::runtime_error("Error: OffsetDirection must be either X or Y"); + temperature.x_offset = temperature.x_offset * domain.deltax; + temperature.y_offset = temperature.y_offset * domain.deltax; + temperature.temporal_offset = input_data["TemperatureData"]["TemporalOffset"]; + temperature.temporal_offset = temperature.temporal_offset * pow(10, -6); + + // For the direction not used to offset copies of the temperature data, should the data be mirrored + // on every other pass (i.e., bidirectional scanning of a laser)? + bool alternating_direction = input_data["TemperatureData"]["AlternatingDirection"]; + if (alternating_direction) { + // False (default) for offset direction, true for other direction + if (offset_direction == "X") + temperature.mirror_y = true; + else + temperature.mirror_x = true; } + } + if (simulation_type == "FromFile") { // Get the paths/number of/names of the temperature data files used temperature.temp_files_in_series = input_data["TemperatureData"]["TemperatureFiles"].size(); if (temperature.temp_files_in_series == 0) @@ -163,61 +208,41 @@ struct Inputs { temperature.temp_paths.push_back(input_data["TemperatureData"]["TemperatureFiles"][filename]); } } - // See if temperature data translation instructions are given - if (input_data.contains("TemperatureData")) { + else if (simulation_type == "FromFinch") { + // For FromFinch problems, get the number of finch input files (temp_files_in_series) and, if a list of + // input files are given in the top level input file, store these (temp_paths) + if (contains_finch_inputs) { + // Parse Finch section of the input file for the list of auxiliary files governing Finch simulations + std::ifstream input_data_stream(input_file); + nlohmann::json input_data_full = nlohmann::json::parse(input_data_stream); + if (input_data_full["Finch"].contains("layers")) { + // Number of Finch input files + temperature.temp_files_in_series = input_data_full["Finch"]["layers"].size(); + } + else { + // Finch data in the top level input file only + temperature.temp_files_in_series = 1; + } + } + else { + // Only one Finch input file - was given on command line + temperature.temp_files_in_series = 1; + } // Option to trim bounds of Finch data around the region that underwent solidification. If not given, // defaults to false // As an alternative to the TrimUnmeltedRegion option, fixed X or Y bounds can be given to trim the // input data - if ((input_data["TemperatureData"].contains("TrimUnmeltedRegion")) && (simulation_type == "FromFinch")) - temperature.trim_unmelted_region = input_data["TemperatureData"]["TrimUnmeltedRegion"]; - else { - if (input_data["TemperatureData"].contains("XRegion")) { - temperature.temperature_x_bounds[0] = input_data["TemperatureData"]["XRegion"][0]; - temperature.temperature_x_bounds[1] = input_data["TemperatureData"]["XRegion"][1]; - temperature.use_fixed_x_bounds = true; - } - if (input_data["TemperatureData"].contains("YRegion")) { - temperature.temperature_y_bounds[0] = input_data["TemperatureData"]["YRegion"][0]; - temperature.temperature_y_bounds[1] = input_data["TemperatureData"]["YRegion"][1]; - temperature.use_fixed_y_bounds = true; - } - } - // Optional translation/mirroring of input temperature data - if (input_data["TemperatureData"].contains("TranslationCount")) { - temperature.number_of_copies = input_data["TemperatureData"]["TranslationCount"]; - // Include the original in the total number of copies (number of translations + 1) - temperature.number_of_copies++; - std::string offset_direction = input_data["TemperatureData"]["OffsetDirection"]; - // Offsets are given in cells (for consistency with layer height) and microseconds (for consistency - // with time step) but stored in meters and seconds for consistency with how cell size and time step - // are treated - if (offset_direction == "X") { - // X offset from input file, Y defaults to 0 - temperature.x_offset = input_data["TemperatureData"]["SpatialOffset"]; - } - else if (offset_direction == "Y") { - // Y offset from input file, X defaults to 0 - temperature.y_offset = input_data["TemperatureData"]["SpatialOffset"]; - } - else - throw std::runtime_error("Error: OffsetDirection must be either X or Y"); - temperature.x_offset = temperature.x_offset * domain.deltax; - temperature.y_offset = temperature.y_offset * domain.deltax; - temperature.temporal_offset = input_data["TemperatureData"]["TemporalOffset"]; - temperature.temporal_offset = temperature.temporal_offset * pow(10, -6); - - // For the direction not used to offset copies of the temperature data, should the data be mirrored - // on every other pass (i.e., bidirectional scanning of a laser)? - bool alternating_direction = input_data["TemperatureData"]["AlternatingDirection"]; - if (alternating_direction) { - // False (default) for offset direction, true for other direction - if (offset_direction == "X") - temperature.mirror_y = true; - else - temperature.mirror_x = true; - } + if ((input_data["TemperatureData"].contains("TrimUnmeltedRegion"))) { + if (id == 0) + std::cout << "Warning: TrimUnmeltedRegion option is deprecated and will be removed in a future " + "release, please use TrimUnmeltedRegionLateral or TrimUnmeltedRegionBuild options" + << std::endl; + temperature.trim_unmelted_region_xy = input_data["TemperatureData"]["TrimUnmeltedRegion"]; } + if ((input_data["TemperatureData"].contains("TrimUnmeltedRegionLateral"))) + temperature.trim_unmelted_region_xy = input_data["TemperatureData"]["TrimUnmeltedRegionLateral"]; + if ((input_data["TemperatureData"].contains("TrimUnmeltedRegionBuild"))) + temperature.trim_unmelted_region_z = input_data["TemperatureData"]["TrimUnmeltedRegionBuild"]; } } else { @@ -485,6 +510,23 @@ struct Inputs { } } + // Get input file object from either nested ExaCA structure or the input file top level. Also store whether the file + // contains a Finch data section to parse + nlohmann::json getInputDataObject(std::string input_file, bool &contains_finch_inputs) { + std::ifstream input_data_stream(input_file); + nlohmann::json input_data_raw = nlohmann::json::parse(input_data_stream); + // Check if data is within a top level "ExaCA" structure and parse JSON object for ExaCA + contains_finch_inputs = false; + if (input_data_raw.contains("ExaCA")) { + nlohmann::json input_data = input_data_raw["ExaCA"]; + if (input_data_raw.contains("Finch")) + contains_finch_inputs = true; + return input_data; + } + else + return input_data_raw; + } + // Check that all given fields are valid, and all required fields are given void checkPrintFieldValidity(nlohmann::json input_data, const int id) { const int num_valid_print_fields = print.print_field_label.size(); diff --git a/src/CAtemperature.hpp b/src/CAtemperature.hpp index d3e6bcb3..977e33fa 100644 --- a/src/CAtemperature.hpp +++ b/src/CAtemperature.hpp @@ -125,7 +125,8 @@ struct Temperature { // Constructor using in-memory temperature data from external source (input_temperature_data) Temperature(const int id, const int np, const Grid &grid, TemperatureInputs inputs, PrintInputs print_inputs, - view_type_coupled input_temperature_data, const int est_num_temperature_data_points = 100000) + view_type_coupled input_temperature_data, std::vector first_value_finch, + std::vector last_value_finch) : liquidus_time_list_host( view_type_int_host(Kokkos::ViewAllocateWithoutInitializing("liquidus_time_list"), 2 * grid.domain_size)) , liquidus_cell_list( @@ -137,7 +138,8 @@ struct Temperature { , number_of_solidification_events(view_type_int_host( Kokkos::ViewAllocateWithoutInitializing("number_of_solidification_events"), grid.domain_size)) , raw_temperature_data(view_type_double_2d_host(Kokkos::ViewAllocateWithoutInitializing("raw_temperature_data"), - est_num_temperature_data_points, num_temperature_components)) + input_temperature_data.extent(0) * (inputs.number_of_copies), + num_temperature_components)) , first_value(view_type_int_host(Kokkos::ViewAllocateWithoutInitializing("first_value"), grid.number_of_layers)) , last_value(view_type_int_host(Kokkos::ViewAllocateWithoutInitializing("last_value"), grid.number_of_layers)) , layer_id_all_layers(view_type_short(Kokkos::ViewAllocateWithoutInitializing("layer_id_all_layers"), @@ -150,7 +152,7 @@ struct Temperature { view_type_float(Kokkos::ViewAllocateWithoutInitializing("undercooling_current_all_layers"), 1)) , _inputs(inputs) { - copyTemperatureData(id, np, grid, input_temperature_data); + copyTemperatureData(id, np, 0, grid, input_temperature_data, first_value_finch, last_value_finch); initOptionalViews(grid, print_inputs); Kokkos::deep_copy(layer_id_all_layers, -1); layer_id = getLayerSubview(layer_id_all_layers, grid.layer_range); @@ -208,79 +210,114 @@ struct Temperature { } // Copy data from external source onto the appropriate MPI ranks - void copyTemperatureData(const int id, const int np, const Grid &grid, view_type_coupled input_temperature_data, - const int resize_padding = 100) { + void copyTemperatureData(const int id, const int np, const int first_layer_init, const Grid &grid, + view_type_coupled input_temperature_data, std::vector first_value_finch, + std::vector last_value_finch, const int resize_padding = 100) { // Take first num_temperature_components columns of input_temperature_data - int finch_data_size = input_temperature_data.extent(0); int finch_temp_components = input_temperature_data.extent(1); - std::cout << "Rank " << id << " has " << finch_data_size << " events from the Finch simulation" << std::endl; - - // First, store data with Y coordinates in bounds for this rank in raw_temperature_data - int temperature_point_counter = 0; - extractTemperatureData(input_temperature_data, raw_temperature_data, temperature_point_counter, grid); - - // Communication pattern - sending to right, receiving from left - int left, right; - if (id == 0) - left = np - 1; - else - left = id - 1; - if (id == np - 1) - right = 0; + // Number of layers of Finch data to copy into temperature struct at once + int num_layers_init; + if (_inputs.layerwise_temp_read) + num_layers_init = 1; else - right = id + 1; - - // Send and recieve data so each rank parses all finch data points - int send_data_size = finch_data_size; - for (int i = 0; i < np - 1; i++) { - // Get size for sending/receiving - int recv_data_size; - MPI_Request send_request_size, recv_request_size; - MPI_Isend(&send_data_size, 1, MPI_INT, right, 0, MPI_COMM_WORLD, &send_request_size); - MPI_Irecv(&recv_data_size, 1, MPI_INT, left, 0, MPI_COMM_WORLD, &recv_request_size); - MPI_Wait(&send_request_size, MPI_STATUS_IGNORE); - MPI_Wait(&recv_request_size, MPI_STATUS_IGNORE); - // Allocate view for received data - view_type_coupled finch_data_recv(Kokkos::ViewAllocateWithoutInitializing("finch_data_recv"), - recv_data_size, finch_temp_components); - - // Send data to the right, recieve data from the left - if needed, increase size of data stored on this rank - // to accomodate received data - MPI_Request send_request_data, recv_request_data; - MPI_Isend(input_temperature_data.data(), send_data_size * finch_temp_components, MPI_DOUBLE, right, 1, - MPI_COMM_WORLD, &send_request_data); - MPI_Irecv(finch_data_recv.data(), recv_data_size * finch_temp_components, MPI_DOUBLE, left, 1, - MPI_COMM_WORLD, &recv_request_data); - int current_size = raw_temperature_data.extent(0); - if (temperature_point_counter + _inputs.number_of_copies * recv_data_size >= current_size) - Kokkos::resize(raw_temperature_data, - temperature_point_counter + _inputs.number_of_copies * recv_data_size + resize_padding, - num_temperature_components); - MPI_Wait(&send_request_data, MPI_STATUS_IGNORE); - MPI_Wait(&recv_request_data, MPI_STATUS_IGNORE); - - // Unpack the appropriate received data into raw_temperature_data - extractTemperatureData(finch_data_recv, raw_temperature_data, temperature_point_counter, grid); - - // Replace send buffer with the received data - input_temperature_data = finch_data_recv; - send_data_size = recv_data_size; + num_layers_init = _inputs.temp_files_in_series; + int last_layer_init = first_layer_init + num_layers_init; + int temperature_point_counter = 0; + for (int layer = first_layer_init; layer < last_layer_init; layer++) { + + // input_temperature_data contains data for one or multiple Finch simulations of various layers - which are + // tracked via the first_value_finch and last_value_finch vectors. The first_value and last_value vectors + // track, on each ExaCA MPI rank, which portions of the data are associated with which layer + const int layer_start = first_value_finch[layer]; + const int layer_end = last_value_finch[layer]; + std::pair finch_layer_range = std::make_pair(layer_start, layer_end); + auto input_temperature_data_layer = Kokkos::subview(input_temperature_data, finch_layer_range, Kokkos::ALL); + + // First, store data with Y coordinates in bounds for this rank in raw_temperature_data + const int temperature_point_counter_start = temperature_point_counter; + extractTemperatureData(input_temperature_data_layer, raw_temperature_data, temperature_point_counter, grid); + + // Communication pattern - sending to right, receiving from left + int left, right; + if (id == 0) + left = np - 1; + else + left = id - 1; + if (id == np - 1) + right = 0; + else + right = id + 1; + + int finch_data_size = input_temperature_data_layer.extent(0); + std::cout << "Rank " << id << " has " << finch_data_size << " events from Finch simulation number " << layer + << ", corresponding to events " << layer_start << " through " << layer_end - 1 << std::endl; + // Send and recieve data so each rank parses all finch data points + view_type_coupled finch_data_send(Kokkos::ViewAllocateWithoutInitializing("finch_data_send"), + layer_end - layer_start, finch_temp_components); + Kokkos::deep_copy(finch_data_send, input_temperature_data_layer); + int send_data_size = finch_data_size; + for (int i = 0; i < np - 1; i++) { + // Get size for sending/receiving + int recv_data_size; + MPI_Request send_request_size, recv_request_size; + MPI_Isend(&send_data_size, 1, MPI_INT, right, 0, MPI_COMM_WORLD, &send_request_size); + MPI_Irecv(&recv_data_size, 1, MPI_INT, left, 0, MPI_COMM_WORLD, &recv_request_size); + MPI_Wait(&send_request_size, MPI_STATUS_IGNORE); + MPI_Wait(&recv_request_size, MPI_STATUS_IGNORE); + // Allocate view for received data + view_type_coupled finch_data_recv(Kokkos::ViewAllocateWithoutInitializing("finch_data_recv"), + recv_data_size, finch_temp_components); + + // Send data to the right, recieve data from the left - if needed, increase size of data stored on this + // rank to accomodate received data + MPI_Request send_request_data, recv_request_data; + MPI_Isend(finch_data_send.data(), send_data_size * finch_temp_components, MPI_DOUBLE, right, 1, + MPI_COMM_WORLD, &send_request_data); + MPI_Irecv(finch_data_recv.data(), recv_data_size * finch_temp_components, MPI_DOUBLE, left, 1, + MPI_COMM_WORLD, &recv_request_data); + int current_size = raw_temperature_data.extent(0); + if (temperature_point_counter + _inputs.number_of_copies * recv_data_size >= current_size) + Kokkos::resize(raw_temperature_data, + temperature_point_counter + _inputs.number_of_copies * recv_data_size + + resize_padding, + num_temperature_components); + MPI_Wait(&send_request_data, MPI_STATUS_IGNORE); + MPI_Wait(&recv_request_data, MPI_STATUS_IGNORE); + + // Unpack the appropriate received data into raw_temperature_data + extractTemperatureData(finch_data_recv, raw_temperature_data, temperature_point_counter, grid); + + // Replace send buffer with the received data + Kokkos::realloc(finch_data_send, recv_data_size, finch_temp_components); + Kokkos::deep_copy(finch_data_send, finch_data_recv); + send_data_size = recv_data_size; + } + first_value(layer) = temperature_point_counter_start; + last_value(layer) = temperature_point_counter; + std::cout << "Rank " << id << " has " << temperature_point_counter - temperature_point_counter_start + << " events to simulate with ExaCA from Finch simulation number " << layer << std::endl; } // Resize with the number of temperature data points on this rank now known Kokkos::resize(raw_temperature_data, temperature_point_counter, num_temperature_components); - for (int n = 0; n < grid.number_of_layers; n++) { - first_value(n) = 0; - last_value(n) = temperature_point_counter; - } - std::cout << "Rank " << id << " has " << temperature_point_counter << " events to simulate with ExaCA" - << std::endl; + + // If reading multiple files during initialization, associated each layer with the appropriate portion of the + // Finch data + if (!(_inputs.layerwise_temp_read)) + fillFirstLastValue(grid.number_of_layers, _inputs.temp_files_in_series); } // Get temperature data from Finch and store as raw temperatures. template void extractTemperatureData(const SrcViewType &temp_src, DstViewType &temp_dst, int &temp_count, const Grid grid) { int data_size = temp_src.extent(0); + // Ensure that dst view has sufficient capacity to store values from temp_src + int max_num_pts_extracted = data_size * _inputs.number_of_copies; + int current_dst_view_size = temp_dst.extent(0); + int free_space_dst_view = temp_dst.extent(0) - temp_count; + if (max_num_pts_extracted > free_space_dst_view) + Kokkos::resize(temp_dst, current_dst_view_size + max_num_pts_extracted - free_space_dst_view, + num_temperature_components); for (int n = 0; n < data_size; n++) { for (int l = 0; l < _inputs.number_of_copies; l++) { // Consider each translated point in either X or Y, optionally mirroring the point in the non-translated @@ -417,6 +454,24 @@ struct Temperature { } } + // For simulations where temperature data is read during ExaCA initialization and reused for multiple layers, fill + // out first_value and last_value to associate each layer with the correct temperature data + void fillFirstLastValue(const int number_of_layers, const int temp_files_in_series) { + for (int layer_read_count = temp_files_in_series; layer_read_count < number_of_layers; layer_read_count++) { + if (temp_files_in_series == 1) { + // All layers have the same temperature data + first_value(layer_read_count) = first_value(layer_read_count - 1); + last_value(layer_read_count) = last_value(layer_read_count - 1); + } + else { + // All layers have different temperature data but in a repeating pattern + int repeated_file = layer_read_count % temp_files_in_series; + first_value(layer_read_count) = first_value(repeated_file); + last_value(layer_read_count) = last_value(repeated_file); + } + } + } + // Read in temperature data from files, stored in the host view raw_temperature_data, with the appropriate MPI ranks // storing the appropriate data void readTemperatureData(int id, const Grid &grid, int layernumber) { @@ -457,24 +512,8 @@ struct Temperature { } // End loop over all files read for all layers Kokkos::resize(raw_temperature_data, number_of_temperature_data_points, num_temperature_components); // Determine start values for each layer's data within raw_temperature_data, if all layers were read - if (!(_inputs.layerwise_temp_read)) { - if (grid.number_of_layers > _inputs.temp_files_in_series) { - for (int layer_read_count = _inputs.temp_files_in_series; layer_read_count < grid.number_of_layers; - layer_read_count++) { - if (_inputs.temp_files_in_series == 1) { - // All layers have the same temperature data - first_value(layer_read_count) = first_value(layer_read_count - 1); - last_value(layer_read_count) = last_value(layer_read_count - 1); - } - else { - // All layers have different temperature data but in a repeating pattern - int repeated_file = (layer_read_count) % _inputs.temp_files_in_series; - first_value(layer_read_count) = first_value(repeated_file); - last_value(layer_read_count) = last_value(repeated_file); - } - } - } - } + if (!(_inputs.layerwise_temp_read)) + fillFirstLastValue(grid.number_of_layers, _inputs.temp_files_in_series); } // Initialize temperature data with a fixed thermal gradient in Z (can also be zero) for Directional/SingleGrain @@ -813,6 +852,18 @@ struct Temperature { }); Kokkos::fence(); MPI_Barrier(MPI_COMM_WORLD); + + // For the optionally stored views, reset solidification event counter and get the subview associated with the + // undercooling field for the next layer + if (store_undercooling_current) + undercooling_current = getLayerSubview(undercooling_current_all_layers, grid.layer_range); + if (store_undercooling_solidification_start) + undercooling_solidification_start = + getLayerSubview(undercooling_solidification_start_all_layers, grid.layer_range); + if (store_solidification_event_counter) { + Kokkos::realloc(solidification_event_counter, grid.domain_size); + Kokkos::deep_copy(solidification_event_counter, 0); + } if (id == 0) { std::cout << "Layer " << layernumber << " temperature field is from Z = " << grid.z_layer_bottom << " through " << grid.nz_layer + grid.z_layer_bottom - 1 << " of the global domain" << std::endl; @@ -905,20 +956,6 @@ struct Temperature { solidification_event_counter(index)++; } - // For the optionally stored views, reset solidification event counter and get the subview associated with the - // undercooling field for the next layer - void resetLayerEventsUndercooling(const Grid &grid) { - if (store_undercooling_current) - undercooling_current = getLayerSubview(undercooling_current_all_layers, grid.layer_range); - if (store_undercooling_solidification_start) - undercooling_solidification_start = - getLayerSubview(undercooling_solidification_start_all_layers, grid.layer_range); - if (store_solidification_event_counter) { - Kokkos::realloc(solidification_event_counter, grid.domain_size); - Kokkos::deep_copy(solidification_event_counter, 0); - } - } - // Get undercooling of an active cell (init_undercooling is undercooling of domains with a uniform undercooling and // no cooling rate) KOKKOS_INLINE_FUNCTION diff --git a/src/runCA.hpp b/src/runCA.hpp index c5afd158..5bbb1b07 100644 --- a/src/runCA.hpp +++ b/src/runCA.hpp @@ -13,194 +13,120 @@ #include #include +// Run ExaCA using thermal data for a given problem type template -void runExaCA(int id, int np, Inputs inputs, Timers timers, Grid grid, Temperature temperature) { - - // Run on the default space. - using memory_space = MemorySpace; - - std::string simulation_type = inputs.simulation_type; - // Full domain solidification - all cells initially liquid or active and end up solid - bool full_domain_solidification; - if ((simulation_type == "Directional") || (simulation_type == "SingleGrain")) - full_domain_solidification = true; - else - full_domain_solidification = false; - - // Material response function - InterfacialResponseFunction irf(inputs.domain.deltat, grid.deltax, inputs.irf); - - // Read temperature data if necessary. For the spot problem, store spot melt data as if it were read from a file - if (simulation_type == "FromFile") - temperature.readTemperatureData(id, grid, 0); - else if (simulation_type == "Spot") - temperature.storeSpotData(id, grid, irf.freezingRange(), inputs.domain.deltat, inputs.domain.spot_radius); - - // Initialize the temperature fields for the simulation type of interest. These are either simple unidirectional - // fields, or more complex data stored in a view in the temperature struct - if (full_domain_solidification) - temperature.initialize(id, simulation_type, grid, inputs.domain.deltat); - else - temperature.initialize(0, id, grid, irf.freezingRange(), inputs.domain.deltat); - MPI_Barrier(MPI_COMM_WORLD); - - // Initialize grain orientations - Orientation orientation(id, inputs.grain_orientation_file, false, inputs.rng_seed, - inputs.irf.num_phases, irf.solidificationTransformation()); - MPI_Barrier(MPI_COMM_WORLD); - - // Initialize cell types, grain IDs, and layer IDs - CellData celldata(grid, inputs.substrate, inputs.print.store_melt_pool_edge); - if (simulation_type == "Directional") - celldata.initSubstrate_Directional(id, grid, inputs.rng_seed); - else if (simulation_type == "SingleGrain") - celldata.initSubstrate_SingleGrain(id, grid); - else - celldata.initSubstrate_BaseplatePowder(id, grid, inputs.rng_seed); - MPI_Barrier(MPI_COMM_WORLD); - - // Variables characterizing the active cell region within each rank's grid, including buffers for ghost node data - // (fixed size) and the steering vector/steering vector size on host/device - Interface interface(id, grid.domain_size, inputs.substrate.init_oct_size); - // Initialize octahedra for initial active cells, if necessary for this problem type - if (full_domain_solidification) - createOctahedra_NoRemelt(grid, celldata, temperature, orientation, interface); - MPI_Barrier(MPI_COMM_WORLD); - - // Nucleation data structure, containing views of nuclei locations, time steps, and ids, and nucleation event - // counters - initialized with an estimate on the number of nuclei in the layer Without knowing - // estimated_nuclei_this_rank_this_layer yet, initialize nucleation data structures to estimated sizes, resize - // inside of placeNuclei when the number of nuclei per rank is known - int estimated_nuclei_this_rank_this_layer = inputs.nucleation.n_max * pow(grid.deltax, 3) * grid.domain_size; - Nucleation nucleation(estimated_nuclei_this_rank_this_layer, inputs.nucleation, - celldata.num_prior_nuclei); - // Fill in nucleation data structures, and assign nucleation undercooling values to potential nucleation events - // Potential nucleation grains are only associated with liquid cells in layer 0 - they will be initialized for each - // successive layer when layer 0 is complete - nucleation.placeNuclei(simulation_type, temperature, irf, inputs.rng_seed, 0, grid, id, inputs.domain.deltat); - - // Initialize printing struct from inputs - Print print(grid, np, inputs.print); - - // End of initialization - timers.stopInit(); - MPI_Barrier(MPI_COMM_WORLD); - - int cycle = 0; - timers.startRun(); - for (int layernumber = 0; layernumber < grid.number_of_layers; layernumber++) { - - int x_switch = 0; - timers.startLayer(); - - // Loop continues until all liquid cells claimed by solid grains, and no solid cells undergo remelting - do { - - // Start of time step - optional print current state of ExaCA simulation (up to and including the current - // layer's data) - print.printIntralayer(id, np, layernumber, inputs.domain.deltat, cycle, grid, celldata, temperature, - interface, orientation); - cycle++; - - // Cells with a successful nucleation event are marked and added to a steering vector, later dealt with in - // cellCapture - timers.startNucleation(); - nucleation.nucleateGrain(cycle, grid, celldata, interface); - timers.stopNucleation(); - - // Melt cells above the liquidus, activate cells at the solid-liquid interface below the liquidus - if (!full_domain_solidification) { - timers.startMeltAct(); - remeltActivateCells(cycle, grid, irf, celldata, temperature, interface); - timers.stopMeltAct(); - } - - // Create steering vector of cells that are active and undercooled on this time step - timers.startSV(); - fillSteeringVector(cycle, grid, celldata, temperature, interface); - timers.stopSV(); - - // Iterate over the steering vector to perform active cell creation and capture operations, and if needed, - // melting of cells that have gone above the liquidus. Also places halo cell data into send buffers, later - // checking the MPI buffers to ensure that all appropriate interface updates in the halo regions were - // recorded - timers.startCapture(); - cellCapture(cycle, np, grid, irf, celldata, temperature, interface, orientation); - checkBuffers(id, cycle, grid, celldata, interface, orientation.n_grain_orientations); - timers.stopCapture(); - - if (np > 1) { - // Update ghost nodes - timers.startComm(); - haloUpdate(cycle, id, grid, celldata, interface, orientation); - timers.stopComm(); - } - - // Check on progress of solidification simulation of the current layer, setting x_switch = 1 if complete - if ((cycle % 1000 == 0) && (simulation_type != "SingleGrain")) { - intermediateOutputAndCheck(id, np, cycle, grid, nucleation.successful_nucleation_counter, x_switch, - nucleation, celldata, temperature, inputs.simulation_type, layernumber, - orientation, print, inputs.domain.deltat, interface); - } - else if (simulation_type == "SingleGrain") { - intermediateOutputAndCheck(id, cycle, grid, x_switch, celldata.cell_type); - } - - } while (x_switch == 0); - - // Reset intralayer print counter and print time series file for previous layer's intralayer data (if needed) - print.resetIntralayer(id, layernumber); - - if (layernumber != grid.number_of_layers - 1) { - MPI_Barrier(MPI_COMM_WORLD); - - // Optional print current state of ExaCA - print.printInterlayer(id, np, layernumber, inputs.domain.deltat, cycle, grid, celldata, temperature, - interface, orientation); - - // Determine new active cell domain size and offset from bottom of global domain - grid.initNextLayer(id, simulation_type, layernumber + 1); - - // Initialize new temperature field data for layer "layernumber + 1" - // TODO: reorganize these temperature functions calls into a temperature.init_next_layer as done with the - // substrate - // If the next layer's temperature data isn't already stored, it should be read - if ((simulation_type == "FromFile") && (inputs.temperature.layerwise_temp_read)) - temperature.readTemperatureData(id, grid, layernumber + 1); - MPI_Barrier(MPI_COMM_WORLD); - // Initialize next layer's temperature data - temperature.initialize(layernumber + 1, id, grid, irf.freezingRange(0), inputs.domain.deltat); - - // Reset solidification event counter of all cells to zeros for the next layer, resizing to number of cells - // associated with the next layer, and get the subview for undercooling - temperature.resetLayerEventsUndercooling(grid); - // Resize and zero all view data relating to the active region from the last layer, in preparation for the - // next layer - interface.initNextLayer(grid.domain_size); - MPI_Barrier(MPI_COMM_WORLD); - - // Sets up views, powder layer (if necessary), and cell types for the next layer of a multilayer problem - celldata.initNextLayer(layernumber + 1, id, grid, inputs.rng_seed); - - // Initialize potential nucleation event data for next layer "layernumber + 1" - // Views containing nucleation data will be resized to the possible number of nuclei on a given MPI rank for - // the next layer - nucleation.resetNucleiCounters(); // start counters at 0 - nucleation.placeNuclei(simulation_type, temperature, irf, inputs.rng_seed, layernumber + 1, grid, id, - inputs.domain.deltat); - - x_switch = 0; - MPI_Barrier(MPI_COMM_WORLD); - cycle = 0; - timers.stopLayer(layernumber); +void runExaCALayer(int id, int np, int layernumber, int &cycle, Inputs inputs, Timers &timers, Grid &grid, + Temperature temperature, InterfacialResponseFunction irf, + Orientation &orientation, CellData &celldata, + Interface &interface, Nucleation &nucleation, Print &print, + std::string simulation_type, bool full_domain_solidification = true) { + + int x_switch = 0; + + // Loop continues until all liquid cells claimed by solid grains, and no solid cells undergo remelting + do { + + // Start of time step - optional print current state of ExaCA simulation (up to and including the current + // layer's data) + print.printIntralayer(id, np, layernumber, inputs.domain.deltat, cycle, grid, celldata, temperature, interface, + orientation); + cycle++; + + // Cells with a successful nucleation event are marked and added to a steering vector, later dealt with in + // cellCapture + timers.startNucleation(); + nucleation.nucleateGrain(cycle, grid, celldata, interface); + timers.stopNucleation(); + + // Melt cells above the liquidus, activate cells at the solid-liquid interface below the liquidus + if (!full_domain_solidification) { + timers.startMeltAct(); + remeltActivateCells(cycle, grid, irf, celldata, temperature, interface); + timers.stopMeltAct(); } - else { - MPI_Barrier(MPI_COMM_WORLD); - timers.stopLayer(); + + // Create steering vector of cells that are active and undercooled on this time step + timers.startSV(); + fillSteeringVector(cycle, grid, celldata, temperature, interface); + timers.stopSV(); + + // Iterate over the steering vector to perform active cell creation and capture operations, and if needed, + // melting of cells that have gone above the liquidus. Also places halo cell data into send buffers, later + // checking the MPI buffers to ensure that all appropriate interface updates in the halo regions were + // recorded + timers.startCapture(); + cellCapture(cycle, np, grid, irf, celldata, temperature, interface, orientation); + checkBuffers(id, cycle, grid, celldata, interface, orientation.n_grain_orientations); + timers.stopCapture(); + + if (np > 1) { + // Update ghost nodes + timers.startComm(); + haloUpdate(cycle, id, grid, celldata, interface, orientation); + timers.stopComm(); + } + + // Check on progress of solidification simulation of the current layer, setting x_switch = 1 if complete + if ((cycle % 1000 == 0) && (simulation_type != "SingleGrain")) { + intermediateOutputAndCheck(id, np, cycle, grid, nucleation.successful_nucleation_counter, x_switch, + nucleation, celldata, temperature, inputs.simulation_type, layernumber, + orientation, print, inputs.domain.deltat, interface); + } + else if (simulation_type == "SingleGrain") { + intermediateOutputAndCheck(id, cycle, grid, x_switch, celldata.cell_type); } + + } while (x_switch == 0); + + // Reset intralayer print counter and print time series file for previous layer's intralayer data (if needed) + print.resetIntralayer(id, layernumber); + + if (layernumber != grid.number_of_layers - 1) { + MPI_Barrier(MPI_COMM_WORLD); + + // Optional print current state of ExaCA + print.printInterlayer(id, np, layernumber, inputs.domain.deltat, cycle, grid, celldata, temperature, interface, + orientation); + + // If more layers to simulate, determine new active cell domain size and offset from bottom of global domain + grid.initNextLayer(id, simulation_type, layernumber + 1); } - timers.stopRun(); - MPI_Barrier(MPI_COMM_WORLD); +} + +// Initialize structs for the next layer of a multilayer ExaCA simulation +template +void initExaCALayer(int id, int layernumber, int &cycle, std::string simulation_type, Inputs inputs, Grid &grid, + InterfacialResponseFunction irf, Temperature &temperature, + CellData &celldata, Interface &interface, + Nucleation &nucleation) { + + // Initialize next layer's temperature data + temperature.initialize(layernumber + 1, id, grid, irf.freezingRange(), inputs.domain.deltat); + + // Resize and zero all view data relating to the active region from the last layer, in preparation for the + // next layer + interface.initNextLayer(grid.domain_size); + + // Sets up views, powder layer (if necessary), and cell types for the next layer of a multilayer problem + celldata.initNextLayer(layernumber + 1, id, grid, inputs.rng_seed); + + // Initialize potential nucleation event data for next layer "layernumber + 1" + // Views containing nucleation data will be resized to the possible number of nuclei on a given MPI rank for + // the next layer + nucleation.resetNucleiCounters(); // start counters at 0 + nucleation.placeNuclei(simulation_type, temperature, irf, inputs.rng_seed, layernumber + 1, grid, id, + inputs.domain.deltat); + + // Reset the time step counter to 0 + cycle = 0; +} + +// Finalize timer values, print end-of-run output to the console/files, and print the log file +template +void finalizeExaCA(int id, int np, int cycle, Inputs inputs, Timers timers, Grid &grid, + Temperature temperature, Orientation &orientation, + CellData &celldata, Interface &interface, Print &print) { + timers.startOutput(); // Collect and print specified final fields to output files diff --git a/unit_test/tstTemperature.hpp b/unit_test/tstTemperature.hpp index 0261b7fd..6481cade 100644 --- a/unit_test/tstTemperature.hpp +++ b/unit_test/tstTemperature.hpp @@ -298,56 +298,53 @@ void testInit_UnidirectionalGradient(const std::string simulation_type, const do // Check temperature initialization for translated/mirrored FromFile or FromFinch data template void checkTemperatureResults(Inputs &inputs, Grid &grid, Temperature &temperature, - const int number_of_copies, const int id, const int np, const bool mirror_x) { - const int expected_num_data_points = grid.nx * number_of_copies * np; - std::vector expected_data_point_found(expected_num_data_points, false); - for (int n = 0; n < expected_num_data_points; n++) { - // X,Y,Z the cell would be placed at - const int coord_x = temperature.getTempCoordX(n, grid.x_min, grid.deltax); - const int coord_y = temperature.getTempCoordY(n, grid.y_min, grid.deltax, grid.y_offset); - const int coord_z = temperature.getTempCoordZ(n, grid.deltax, grid.layer_height, 0, grid.z_min_layer); - const int index = grid.get1DIndex(coord_x, coord_y, coord_z); - if (!expected_data_point_found[index]) - expected_data_point_found[index] = true; - else { - std::string error = "Error: Rank " + std::to_string(id) + " has more than one data point at location (" + - std::to_string(coord_x) + "," + std::to_string(coord_y) + "," + std::to_string(coord_z); - throw std::runtime_error(error); + const int number_of_copies, const int, const int np, const bool mirror_x, + const int num_layers_with_points) { + + // Should have equal numbers of points associated with each layer - if initializing only 1 layer at a time, the data + // from the other layers should not have been stored + const int expected_num_data_points = grid.nx * number_of_copies * np * num_layers_with_points; + EXPECT_EQ(temperature.raw_temperature_data.extent(0), expected_num_data_points); + std::vector expected_num_points_per_cell(grid.domain_size, 0); + for (int layernumber = 0; layernumber < num_layers_with_points; layernumber++) { + for (int n = 0; n < grid.nx * number_of_copies * np; n++) { + int point_num = layernumber * grid.nx * number_of_copies * np + n; + // X,Y,Z the cell would be placed at + const int coord_x = temperature.getTempCoordX(point_num, grid.x_min, grid.deltax); + const int coord_y = temperature.getTempCoordY(point_num, grid.y_min, grid.deltax, grid.y_offset); + const int coord_z = + temperature.getTempCoordZ(point_num, grid.deltax, grid.layer_height, layernumber, grid.z_min_layer); + const int index = grid.get1DIndex(coord_x, coord_y, coord_z); + expected_num_points_per_cell[index]++; + // Make sure the cell is associated with the correct temperature data from "input_temperature_data" + // TM for even numbered translations (or when mirror_x is false) is equal to the X coordinate plus any + // applied temporal offset in Y related to its Y coordinate. If mirrored, odd numbered translations will use + // (nx - coord_x) in place of coord_x in these calculations. TL should be (1 * layernumber) larger than TM, + // and the cooling rate is proportional to 100 times the MPI rank the point was initalized on (in turn equal + // to its Z coordinate) + const bool x_coord_mirrored = ((mirror_x) && (coord_y % 2 == 1)); + double loc_along_scan; + if (x_coord_mirrored) + loc_along_scan = static_cast((grid.nx - 1) - coord_x); + else + loc_along_scan = static_cast(coord_x); + const double expected_tm = + loc_along_scan + static_cast(coord_y) * inputs.temperature.temporal_offset; + const double expected_tl = expected_tm + 1.0 * (layernumber + 1); + const double expected_cr = 100.0 * static_cast(coord_z + 1); + EXPECT_DOUBLE_EQ(expected_tm, temperature.raw_temperature_data(point_num, 3)); + EXPECT_DOUBLE_EQ(expected_tl, temperature.raw_temperature_data(point_num, 4)); + EXPECT_DOUBLE_EQ(expected_cr, temperature.raw_temperature_data(point_num, 5)); } - // Make sure the cell is associated with the correct temperature data from "input_temperature_data" - // TM for even numbered translations (or when mirror_x is false) is equal to the X coordinate plus any applied - // temporal offset in Y related to its Y coordinate. If mirrored, odd numbered translations will use (nx - - // coord_x) in place of coord_x in these calculations. TL should be one larger than TM, and the cooling rate is - // proportional to 100 times the MPI rank the point was initalized on (in turn equal to its Z coordinate) - const bool x_coord_mirrored = ((mirror_x) && (coord_y % 2 == 1)); - double loc_along_scan; - if (x_coord_mirrored) - loc_along_scan = static_cast((grid.nx - 1) - coord_x); - else - loc_along_scan = static_cast(coord_x); - const double expected_tm = loc_along_scan + static_cast(coord_y) * inputs.temperature.temporal_offset; - const double expected_tl = expected_tm + 1.0; - const double expected_cr = 100.0 * static_cast(coord_z + 1); - EXPECT_DOUBLE_EQ(expected_tm, temperature.raw_temperature_data(n, 3)); - EXPECT_DOUBLE_EQ(expected_tl, temperature.raw_temperature_data(n, 4)); - EXPECT_DOUBLE_EQ(expected_cr, temperature.raw_temperature_data(n, 5)); } - // Make sure one data point was present in each expected cell - for (int n = 0; n < expected_num_data_points; n++) { - const int coord_x = temperature.getTempCoordX(n, grid.x_min, grid.deltax); - const int coord_y = temperature.getTempCoordY(n, grid.y_min, grid.deltax, grid.y_offset); - const int coord_z = temperature.getTempCoordZ(n, grid.deltax, grid.layer_height, 0, grid.z_min_layer); - const int index = grid.get1DIndex(coord_x, coord_y, coord_z); - if (!expected_data_point_found[index]) { - if (!expected_data_point_found[index]) - expected_data_point_found[index] = true; - else { - std::string error = "Error: Rank " + std::to_string(id) + - " has more than one data point at location (" + std::to_string(coord_x) + "," + - std::to_string(coord_y) + "," + std::to_string(coord_z); - throw std::runtime_error(error); - } + // Make sure one data point was present in each expected cell per each layer of data stored. With number_of_copies = + // 1, data will only be present at Y = 0 locally, and each copy is translated 1 cell in Y + for (int index = 0; index < grid.domain_size; index++) { + const int coord_y = grid.getCoordY(index); + const int coord_z = grid.getCoordZ(index); + if ((coord_y < number_of_copies) && (coord_z < np)) { + EXPECT_EQ(expected_num_points_per_cell[index], num_layers_with_points); } } } @@ -384,7 +381,8 @@ void initTestGrid(const int id, const int np, Grid &grid) { } // Test storing temperature data from Finch on the correct ExaCA ranks, optionally mirroring and translating the data -void testInitTemperatureFromFinch(const bool mirror_x, const int number_of_copies) { +void testInitTemperatureFromFinch(const bool mirror_x, const int number_of_copies, const bool layerwise_temp_init, + const int number_of_layers) { using memory_space = TEST_MEMSPACE; using view_type_coupled = Kokkos::View; @@ -397,35 +395,52 @@ void testInitTemperatureFromFinch(const bool mirror_x, const int number_of_copie // Default inputs struct - manually set temperature values for test Inputs inputs; + inputs.temperature.layerwise_temp_read = layerwise_temp_init; + inputs.temperature.temp_files_in_series = number_of_layers; // Default grid struct - manually set up domain for test Grid grid; initTestInputs(mirror_x, number_of_copies, inputs); initTestGrid(id, np, grid); - - // Create dummy Finch temperature data stored on various ranks, to be mapped to ExaCA ranks - view_type_coupled input_temperature_data(Kokkos::ViewAllocateWithoutInitializing("FinchData"), grid.nx * np, 6); - - // Each rank has Finch data at a given Z location, at all relevant X values and at every third Y coordinate (so each - // ExaCA rank after rearranging the data will only own data from one of the Y coordinates). Number of data points - // proportional to the number of MPI ranks + grid.number_of_layers = number_of_layers; + + // Create dummy Finch temperature data stored on various ranks, to be mapped to ExaCA ranks. Each layer has nx * np + // locations with data + const int locations_with_data = np * grid.nx * number_of_layers; + int num_layers_with_points; + if (layerwise_temp_init) + num_layers_with_points = 1; + else + num_layers_with_points = number_of_layers; + view_type_coupled input_temperature_data(Kokkos::ViewAllocateWithoutInitializing("FinchData"), locations_with_data, + 6); + std::vector first_value, last_value; int finch_counter = 0; - for (int coord_x = 0; coord_x < grid.nx; coord_x++) { - for (int coord_y = 0; coord_y < np; coord_y++) { - input_temperature_data(finch_counter, 0) = grid.x_min + grid.deltax * coord_x; - input_temperature_data(finch_counter, 1) = grid.y_min + 3.0 * coord_y * grid.deltax; - input_temperature_data(finch_counter, 2) = grid.z_min + id * grid.deltax; - input_temperature_data(finch_counter, 3) = static_cast(coord_x); - input_temperature_data(finch_counter, 4) = static_cast(coord_x) + 1.0; - input_temperature_data(finch_counter, 5) = 100.0 * (id + 1); - finch_counter++; + for (int layernumber = 0; layernumber < num_layers_with_points; layernumber++) { + first_value.push_back(finch_counter); + // Each rank has Finch data at a given Z location, at all relevant X values and at every third Y coordinate (so + // each ExaCA rank after rearranging the data will only own data from one of the Y coordinates). Number of data + // points proportional to the number of MPI ranks and varies based on the layer number. Liquidus times are + // offset from the melting times by a value proportional to the layer number + for (int coord_x = 0; coord_x < grid.nx; coord_x++) { + for (int coord_y = 0; coord_y < np; coord_y++) { + input_temperature_data(finch_counter, 0) = grid.x_min + grid.deltax * coord_x; + input_temperature_data(finch_counter, 1) = grid.y_min + 3.0 * coord_y * grid.deltax; + input_temperature_data(finch_counter, 2) = grid.z_min + id * grid.deltax; + input_temperature_data(finch_counter, 3) = static_cast(coord_x); + input_temperature_data(finch_counter, 4) = static_cast(coord_x) + 1.0 * (layernumber + 1); + input_temperature_data(finch_counter, 5) = 100.0 * (id + 1); + finch_counter++; + } } + last_value.push_back(finch_counter); } // Construct temperature views - Temperature temperature(id, np, grid, inputs.temperature, inputs.print, input_temperature_data); + Temperature temperature(id, np, grid, inputs.temperature, inputs.print, input_temperature_data, first_value, + last_value); // Check that the right ranks have the right temperature data - checkTemperatureResults(inputs, grid, temperature, number_of_copies, id, np, mirror_x); + checkTemperatureResults(inputs, grid, temperature, number_of_copies, id, np, mirror_x, num_layers_with_points); } // Test storing temperature data from a file on the correct ExaCA ranks, using the same process at the FromFinch test @@ -475,7 +490,7 @@ void testInitTemperatureFromFile(const bool mirror_x, const int number_of_copies temperature.readTemperatureData(id, grid, 0); // Check that the right ranks have the right temperature data - checkTemperatureResults(inputs, grid, temperature, number_of_copies, id, np, mirror_x); + checkTemperatureResults(inputs, grid, temperature, number_of_copies, id, np, mirror_x, grid.number_of_layers); } //---------------------------------------------------------------------------// @@ -498,9 +513,17 @@ TEST(TEST_CATEGORY, temperature) { testInit_UnidirectionalGradient("Directional", 1000000, 2); testInit_UnidirectionalGradient("SingleGrain", 0, 2); testInit_UnidirectionalGradient("SingleGrain", 1000000, 2); - testInitTemperatureFromFinch(false, 1); - testInitTemperatureFromFinch(false, 3); - testInitTemperatureFromFinch(true, 3); + // Coupled Finch-ExaCA simulation init tests: mirror_x (true/false), number_of_copies, layerwise_temp_init + // (true/false), number_of_layers + std::vector mirror_x_vals = {false, false, true, false, false}; + std::vector number_of_copies_vals = {1, 3, 3, 1, 1}; + std::vector layerwise_temp_init_vals = {false, false, false, false, true}; + std::vector number_of_layers_finch_vals = {1, 1, 1, 2, 2}; + num_vals = mirror_x_vals.size(); + for (int test_count = 0; test_count < num_vals; test_count++) { + testInitTemperatureFromFinch(mirror_x_vals[test_count], number_of_copies_vals[test_count], + layerwise_temp_init_vals[test_count], number_of_layers_finch_vals[test_count]); + } testInitTemperatureFromFile(false, 1); testInitTemperatureFromFile(false, 3); testInitTemperatureFromFile(true, 3); diff --git a/unit_test/tstUpdate.hpp b/unit_test/tstUpdate.hpp index de13b4ae..96533d00 100644 --- a/unit_test/tstUpdate.hpp +++ b/unit_test/tstUpdate.hpp @@ -44,11 +44,59 @@ void testSmallDirS() { inputs.temperature); // Temperature fields characterized by data in this structure Temperature temperature(grid, inputs.temperature, inputs.print); + temperature.initialize(id, "Directional", grid, inputs.domain.deltat); + MPI_Barrier(MPI_COMM_WORLD); // Run SmallDirS problem and check volume fraction of nucleated grains with 1% tolerance of expected value (to // account for the non-deterministic nature of the cell capture) - runExaCA(id, np, inputs, timers, grid, temperature); + // Material response function + InterfacialResponseFunction irf(inputs.domain.deltat, grid.deltax, inputs.irf); + + // Initialize grain orientations + Orientation orientation(id, inputs.grain_orientation_file, false); + MPI_Barrier(MPI_COMM_WORLD); + + // Initialize cell types, grain IDs, and layer IDs + CellData celldata(grid, inputs.substrate, inputs.print.store_melt_pool_edge); + celldata.initSubstrate_Directional(id, grid, inputs.rng_seed); + MPI_Barrier(MPI_COMM_WORLD); + + // Variables characterizing the active cell region within each rank's grid, including buffers for ghost node data + // (fixed size) and the steering vector/steering vector size on host/device + Interface interface(id, grid.domain_size, inputs.substrate.init_oct_size); + // Initialize octahedron for active cells + createOctahedra_NoRemelt(grid, celldata, temperature, orientation, interface); + MPI_Barrier(MPI_COMM_WORLD); + // Nucleation data structure, containing views of nuclei locations, time steps, and ids, and nucleation event + // counters - initialized with an estimate on the number of nuclei in the layer Without knowing + // estimated_nuclei_this_rank_this_layer yet, initialize nucleation data structures to estimated sizes, resize + // inside of placeNuclei when the number of nuclei per rank is known + int estimated_nuclei_this_rank_this_layer = inputs.nucleation.n_max * pow(grid.deltax, 3) * grid.domain_size; + Nucleation nucleation(estimated_nuclei_this_rank_this_layer, inputs.nucleation); + // Fill in nucleation data structures, and assign nucleation undercooling values to potential nucleation events + // Potential nucleation grains are only associated with liquid cells in layer 0 - they will be initialized for each + // successive layer when layer 0 is complete + nucleation.placeNuclei("Directional", temperature, irf, inputs.rng_seed, 0, grid, id, inputs.domain.deltat); + + // Initialize printing struct from inputs + Print print(grid, np, inputs.print); + + // End of initialization + timers.stopInit(); + MPI_Barrier(MPI_COMM_WORLD); + + int cycle = 0; + timers.startRun(); + + runExaCALayer(id, np, 0, cycle, inputs, timers, grid, temperature, irf, orientation, celldata, interface, + nucleation, print, "Directional"); + + timers.stopRun(); + MPI_Barrier(MPI_COMM_WORLD); + + // Print ExaCA end-of-run data + finalizeExaCA(id, np, cycle, inputs, timers, grid, temperature, orientation, celldata, interface, print); // MPI barrier to ensure that log file has been written MPI_Barrier(MPI_COMM_WORLD); std::string log_file = "TestProblemSmallDirS.json"; @@ -76,9 +124,53 @@ void testSmallEquiaxedGrain() { inputs.temperature); // Temperature fields characterized by data in this structure Temperature temperature(grid, inputs.temperature, inputs.print); + temperature.initialize(id, "SingleGrain", grid, inputs.domain.deltat); // Run Small equiaxed grain problem and check time step at which the grain reaches the domain edge - runExaCA(id, np, inputs, timers, grid, temperature); + // Material response function + InterfacialResponseFunction irf(inputs.domain.deltat, grid.deltax, inputs.irf); + + // Initialize grain orientations + Orientation orientation(id, inputs.grain_orientation_file, false); + MPI_Barrier(MPI_COMM_WORLD); + + // Initialize cell types, grain IDs, and layer IDs + CellData celldata(grid, inputs.substrate, inputs.print.store_melt_pool_edge); + celldata.initSubstrate_SingleGrain(id, grid); + MPI_Barrier(MPI_COMM_WORLD); + + // Variables characterizing the active cell region within each rank's grid, including buffers for ghost node data + // (fixed size) and the steering vector/steering vector size on host/device + Interface interface(id, grid.domain_size, inputs.substrate.init_oct_size); + // Initialize octahedron for active cell + createOctahedra_NoRemelt(grid, celldata, temperature, orientation, interface); + MPI_Barrier(MPI_COMM_WORLD); + + // Nucleation data structure, containing views of nuclei locations, time steps, and ids, and nucleation event + // counters - initialized with an estimate on the number of nuclei in the layer Without knowing + // estimated_nuclei_this_rank_this_layer yet, initialize nucleation data structures to estimated sizes, resize + // inside of placeNuclei when the number of nuclei per rank is known + int estimated_nuclei_this_rank_this_layer = 0; + Nucleation nucleation(estimated_nuclei_this_rank_this_layer, inputs.nucleation); + + // Initialize printing struct from inputs + Print print(grid, np, inputs.print); + + // End of initialization + timers.stopInit(); + MPI_Barrier(MPI_COMM_WORLD); + + int cycle = 0; + timers.startRun(); + + runExaCALayer(id, np, 0, cycle, inputs, timers, grid, temperature, irf, orientation, celldata, interface, + nucleation, print, "SingleGrain"); + + timers.stopRun(); + MPI_Barrier(MPI_COMM_WORLD); + + // Print ExaCA end-of-run data + finalizeExaCA(id, np, cycle, inputs, timers, grid, temperature, orientation, celldata, interface, print); // MPI barrier to ensure that log file has been written MPI_Barrier(MPI_COMM_WORLD);