diff --git a/utils/profiling/cmp2exp.yaml b/utils/profiling/cmp2exp.yaml new file mode 100644 index 000000000..21b88713b --- /dev/null +++ b/utils/profiling/cmp2exp.yaml @@ -0,0 +1,49 @@ +oops log: ['exp1.log','exp2.log'] +plot type: bars +bar attributes: + colors: ['b','r'] + labels: ['Exp1','Exp2'] + width: 0.35 +methods: +- oops::Covariance::SABER::multiply +- oops::Geometry::Geometry +#- oops::GetValues::GetValues +#- oops::GetValues::finalizeAD +- oops::GetValues::process +- oops::GetValues::processAD +- oops::GetValues::processTL +#- oops::Increment::Increment +#- oops::Increment::axpy +#- oops::Increment::dot_product_with +#- oops::Increment::fromFieldSet +#- oops::Increment::operator= +#- oops::Increment::print +- oops::Increment::write +#- oops::Increment::toFieldSet +#- oops::LinearObsOper::AIRS AQUA::simulateObsAD +#- oops::LinearObsOper::CRIS-FSR NOAA-20::setTrajectory +#- oops::LinearObsOper::CRIS-FSR NOAA-20::simulateObsAD +#- oops::LinearObsOper::CRIS-FSR NPP::setTrajectory +#- oops::LinearObsOper::CRIS-FSR NPP::simulateObsAD +#- oops::LinearObsOper::IASI METOP-B::setTrajectory +#- oops::LinearObsOper::IASI METOP-B::simulateObsAD +#- oops::LinearObsOper::IASI METOP-C::setTrajectory +#- oops::LinearObsOper::IASI METOP-C::simulateObsAD +#- oops::LinearObsOper::ssmis_f17::simulateObsAD +#- oops::LinearVariableChange::changeVarAD +#- oops::LinearVariableChange::changeVarTL +#- oops::ObsAuxControl::ObsAuxControl +#- oops::ObsDataVector::print +#- oops::ObsOperator::AIRS AQUA::simulateObs +#- oops::ObsOperator::CRIS-FSR NOAA-20::simulateObs +#- oops::ObsOperator::CRIS-FSR NPP::simulateObs +#- oops::ObsOperator::IASI METOP-B::simulateObs +#- oops::ObsOperator::IASI METOP-C::simulateObs +- oops::ObsSpace::ObsSpace +- oops::ObsSpace::save +- oops::State::State +#- oops::State::print +#- oops::State::write +- oops::UnstructuredInterpolator::apply +- oops::UnstructuredInterpolator::applyAD +#- oops::VariableChange::changeVar diff --git a/utils/profiling/oops_app_profile.py b/utils/profiling/oops_app_profile.py new file mode 100644 index 000000000..4f7485a9a --- /dev/null +++ b/utils/profiling/oops_app_profile.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 + +import argparse +import yaml +import matplotlib.pyplot as plt +import re +import numpy as np + +def read_timers(file_path): + + # Define a regex to extract parallel timing statistics + pattern = re.compile(r"OOPS_STATS\s+(.+?)\s+:\s+(\d+\.\d+)") + + # Read and parse the file + runtimes = {} + in_parallel_section = False + with open(file_path, "r") as file: + for line in file: + if "---------------------------------- Parallel Timing Statistics" in line: + if in_parallel_section: + break # Stop parsing at the end of the section + else: + in_parallel_section = True + continue + + if in_parallel_section and "OOPS_STATS" in line: + match = pattern.search(line) + if match: + key, runtime = match.groups() + runtimes[key.strip()] = float(runtime) / 1000 # Convert ms to seconds + + return runtimes + +def makepie(runtimes,methods): + + # Ensure there is data to plot + if runtimes: + # Sort by runtime in descending order +# if nosort==True: +# print ("not being sorted") +# sorted_runtimes = runtimes.items() +# else: +# sorted_runtimes = sorted(runtimes.items(), key=lambda item: item[1], reverse=True) + sorted_runtimes = sorted(runtimes.items(), key=lambda item: item[1], reverse=True) + + # Calculate cumulative runtime and determine the cutoff for the top 90% + total_runtime = sum(runtime for _, runtime in sorted_runtimes) + cumulative_runtime = 0 + top_runtimes = [] + other_runtime = 0 + # Convert sorted_runtimes to a dictionary + sorted_runtimes_dict = {key: runtime for key, runtime in sorted_runtimes} + + # print the sorted_runtimes_dict one key/value pair per line + for key, value in sorted_runtimes_dict.items(): + print(f"{key}: {value}") + + total_runtime_value = sorted_runtimes_dict['util::Timers::Total'] + + for key, runtime in sorted_runtimes: + if key in methods: + print(f"---------- key: {key}, runtime: {runtime}") + top_runtimes.append((key, runtime)) + cumulative_runtime += runtime + + other_runtime = total_runtime_value - cumulative_runtime + + print(f"Total runtime: {total_runtime_value}") + print(f"Top runtime: {cumulative_runtime}") + print(f"Other runtime: {other_runtime}") + + # Add the "Other" category + if other_runtime > 0: + top_runtimes.append(("Other", other_runtime)) + + # Prepare data for plotting + labels, sizes = zip(*top_runtimes) + + # Set figure dimensions + plt.figure(figsize=(15, 10)) + + # Generate Pie Chart with labels on the side + wedges, texts, autotexts = plt.pie(sizes, autopct='%1.1f%%', startangle=140) + for i, text in enumerate(autotexts): + text.set_text(f'{sizes[i]:.2f}') + if labels[i] == "Other": + wedges[i].set_alpha(0.5) + + # Add a legend with the labels + plt.legend(wedges, labels, title="Categories") + plt.axis('equal') # Equal aspect ratio ensures that pie is drawn as a circle. + + plt.title("Parallel Runtime Distribution from OOPS Profiling Data", fontsize=16, fontweight='bold') + plt.xlabel("Runtime (s)", fontsize=14) + else: + print("No timing data found in the Parallel Timing Statistics section.") + +def makebars(runtimes,methods,ax,width,offset,color,case): + + # Ensure there is data to plot + if runtimes: + # Sort by runtime in descending order + sorted_runtimes = runtimes.items() + + # Calculate cumulative runtime and determine the cutoff for the top 90% + total_runtime = sum(runtime for _, runtime in sorted_runtimes) + cumulative_runtime = 0 + top_runtimes = [] + other_runtime = 0 + # Convert sorted_runtimes to a dictionary + sorted_runtimes_dict = {key: runtime for key, runtime in sorted_runtimes} + + # print the sorted_runtimes_dict one key/value pair per line + for key, value in sorted_runtimes_dict.items(): + print(f"{key}: {value}") + + total_runtime_value = sorted_runtimes_dict['util::Timers::Total'] + + for key, runtime in sorted_runtimes: + if key in methods: + print(f"---------- key: {key}, runtime: {runtime}") + top_runtimes.append((key, runtime)) + cumulative_runtime += runtime + + other_runtime = total_runtime_value - cumulative_runtime + + print(f"Total runtime: {total_runtime_value}") + print(f"Top runtime: {cumulative_runtime}") + print(f"Other runtime: {other_runtime}") + + # Add the "Other" category + if other_runtime > 0: + top_runtimes.append(("Other", other_runtime)) + + # Prepare data for plotting + labels, sizes = zip(*top_runtimes) + + # Generate Pie Chart with labels on the side + ind = np.arange(len(labels)) + bars = ax.barh(ind+offset,sizes,width,color=color,label=case+f': {total_runtime_value:.1f}s') + + ax.set(yticks=ind+offset, yticklabels=labels, ylim=[2*width - 1, len(labels)]) + ax.legend() + ax.set_title("Parallel Runtime Distribution from OOPS Profiling Data", fontsize=16, fontweight='bold') + ax.set_xlabel("Runtime (s)", fontsize=14) + + # annotate bars with percentage times + for i, this in enumerate(top_runtimes): + p = 100 * this[1] / total_runtime_value + ax.text(this[1] + 0.005*total_runtime_value, i+offset, f'{p:.1f}%\n', + color=color, fontweight='bold', fontsize=8, ha='left', va='top') + else: + print("No timing data found in the Parallel Timing Statistics section.") + +# Parse command-line arguments +parser = argparse.ArgumentParser(description='Process profiling data from a YAML configuration file.') +parser.add_argument('config', type=str, help='Path to the YAML configuration file') +args = parser.parse_args() + +# Load the YAML configuration file +with open(args.config, 'r') as yaml_file: + config = yaml.safe_load(yaml_file) + +# Extract the file path from the configuration +file_path = config['oops log'] +methods = config['methods'] +plt_type = config['plot type'] + +if plt_type == 'pie': + # make pie chart + for file in file_path: + runtimes = read_timers(file) + makepie(runtimes,methods) + +else: + # read relevant additional attributes + # make bar plots + colors = config['bar attributes']['colors'] + labels = config['bar attributes']['labels'] + width = config['bar attributes']['width'] + fig, ax = plt.subplots(figsize=(12, 8), gridspec_kw={'width_ratios': [2], 'height_ratios': [1]}) + for i, file in enumerate(file_path): + runtimes = read_timers(file) + offset = i*width + makebars(runtimes,methods,ax,width,offset,colors[i],labels[i]) + plt.subplots_adjust(left=0.3) # labels tend to be long + +plt.show() + diff --git a/utils/profiling/profile.yaml b/utils/profiling/profile.yaml new file mode 100644 index 000000000..dd68a49ed --- /dev/null +++ b/utils/profiling/profile.yaml @@ -0,0 +1,36 @@ +oops log: ['oops_stats.txt'] +plot type: pie +methods: +- oops::Covariance::SABER::multiply +- oops::State::State +- oops::Increment::diff +- oops::Geometry::Geometry +- oops::State::write +- oops::Model::step +- oops::State::read +- oops::Increment::write +- oops::ObsSpace::ObsSpace +- oops::ObsSpace::save +- oops::UnstructuredInterpolator::apply +- oops::LinearVariableChange::changeVarTraj +#- oops::Covariance::SABER::Constructor +#- oops::Increment::Increment +#- oops::GetValues::processTL +#- oops::Increment::read +#- oops::GetValues::GetValues +#- oops::GetValues::processAD +#- oops::LinearVariableChange::changeVarTL +#- oops::LinearVariableChange::changeVarAD +#- oops::Increment::operator= +#- oops::UnstructuredInterpolator::applyAD +#- oops::VariableChange::changeVar +#- oops::Increment::operator+=(State, Increment) +#- oops::Parameters::deserialize +#- oops::GetValues::fillGeoVaLsTL +#- oops::GetValues::process +#- oops::ObsVector::dot_product +#- oops::Increment::toFieldSet +#- oops::GetValues::fillGeoVaLsAD +#- oops::State::toFieldSet +#- oops::Increment::fromFieldSet +#