Skip to content

Generate GPU Component runtime summary and rank-channel variance #4

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
prosenjitdhole wants to merge 1 commit into
base: users/oyazdanb/gemm_test
Choose a base branch
from
Open

Generate GPU Component runtime summary and rank-channel variance #4

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
145 changes: 138 additions & 7 deletions scripts/gemm_analysis/create_embeded_html_report.py
View file
Open in desktop
Copy link
Collaborator

@oyazdanb oyazdanb Dec 5, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

rebase your branch. This file has a new version that is on the main. Put your changes on top of that.

Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@

#!/usr/bin/env python3
"""
Create a self-contained HTML report comparing two experiment sweeps.
Expand Down Expand Up @@ -68,6 +69,13 @@ def parse_args():
default=None,
help='Label for second sweep (default: directory name)'
)

parser.add_argument(
'--variance',
type=str,
default='channel',
help='Label for second sweep (default: directory name)'
)

parser.add_argument(
'--output',
Expand All @@ -78,7 +86,7 @@ def parse_args():

return parser.parse_args()

def get_plot_images(sweep_path):
def get_plot_images(sweep_path, variance):
"""Get paths to all plot images for a sweep"""
plots_dir = sweep_path / "tracelens_analysis" / "plots"

Expand All @@ -88,9 +96,23 @@ def get_plot_images(sweep_path):
'ranks': plots_dir / 'variance_by_ranks_boxplot.png',
'violin': plots_dir / 'variance_violin_combined.png',
'interaction': plots_dir / 'variance_thread_channel_interaction.png',
'gpu_summary' : plots_dir / 'comparison_summary.png',
'busy_time' : plots_dir / f'busy_time_{variance}_variance.png',
'computation_time' : plots_dir / f'computation_time_{variance}_variance.png',
'exposed_comm_time' : plots_dir / f'exposed_comm_time_{variance}_variance.png',
'exposed_memcpy_time' : plots_dir / f'exposed_memcpy_time_{variance}_variance.png',
'idle_time' : plots_dir / f'idle_time_{variance}_variance.png',
'total_comm_time' : plots_dir / f'total_comm_time_{variance}_variance.png',
'total_memcpy_time' : plots_dir / f'total_memcpy_time_{variance}_variance.png',
'total_time' : plots_dir / f'total_time_{variance}_variance.png',
'k0' : plots_dir / 'Kernel_0.png',
'k1' : plots_dir / 'Kernel_1.png',
'k2' : plots_dir / 'Kernel_2.png',
'k3' : plots_dir / 'Kernel_3.png',
}

def create_html_report(sweep1_path, sweep2_path, label1, label2, output_path):

def create_html_report(sweep1_path, sweep2_path, label1, label2, variance, output_path):
"""Create HTML report comparing two sweeps"""

# Get sweep names from paths if labels not provided
Expand All @@ -99,9 +121,10 @@ def create_html_report(sweep1_path, sweep2_path, label1, label2, output_path):
if label2 is None:
label2 = sweep2_path.name


# Get image paths for both sweeps
images_sweep1 = get_plot_images(sweep1_path)
images_sweep2 = get_plot_images(sweep2_path)
images_sweep1 = get_plot_images(sweep1_path, variance=variance)
images_sweep2 = get_plot_images(sweep2_path, variance=variance)

# Convert images to base64
print("Converting images to base64...")
Expand Down Expand Up @@ -349,6 +372,116 @@ def create_html_report(sweep1_path, sweep2_path, label1, label2, output_path):

<hr>

<h2> GPU Component Time Summary </h2>
<table class="comparison-table">
<tr>
<th>{label1}</th>
<th>{label2}</th>
</tr>
<tr>
<td><img src="{image_data.get('gpu_summary_sweep1', '')}" alt="Summary Sweep 1"></td>
<td><img src="{image_data.get('gpu_summary_sweep2', '')}" alt="Summary Sweep 2"></td>
</tr>
</table>

<hr>

<h2> GPU Component Time over {variance} </h2>

<table>
<tr>
<th>Component</th>
<th>{label1}</th>
<th>{label2}</th>
</tr>


<tr>
<td>Busy Time</td>
<td><img src="{image_data.get('busy_time_sweep1', '')}" alt="busy_time Sweep 1"></td>
<td><img src="{image_data.get('busy_time_sweep2', '')}" alt="busy_time Sweep 2"></td>
</tr>

<tr>
<td>Computation Time</td>
<td><img src="{image_data.get('computation_time_sweep1', '')}" alt="computation_time Sweep 1"></td>
<td><img src="{image_data.get('computation_time_sweep2', '')}" alt="computation_time Sweep 2"></td>
</tr>

<tr>
<td>Exposed Comm Time</td>
<td><img src="{image_data.get('exposed_comm_time_sweep1', '')}" alt="exposed_comm_time Sweep 1"></td>
<td><img src="{image_data.get('exposed_comm_time_sweep2', '')}" alt="exposed_comm_time Sweep 2"></td>
</tr>

<tr>
<td>Exposed Memcpy Time</td>
<td><img src="{image_data.get('exposed_memcpy_time_sweep1', '')}" alt="exposed_memcpy_time Sweep 1"></td>
<td><img src="{image_data.get('exposed_memcpy_time_sweep2', '')}" alt="exposed_memcpy_time Sweep 2"></td>
</tr>

<tr>
<td>Idle Time</td>
<td><img src="{image_data.get('idle_time_sweep1', '')}" alt="idle_time Sweep 1"></td>
<td><img src="{image_data.get('idle_time_sweep2', '')}" alt="idle_time Sweep 2"></td>
</tr>

<tr>
<td>Total Comm Time</td>
<td><img src="{image_data.get('total_comm_time_sweep1', '')}" alt="total_comm_time Sweep 1"></td>
<td><img src="{image_data.get('total_comm_time_sweep2', '')}" alt="total_comm_time Sweep 2"></td>
</tr>

<tr>
<td>Total Memcpy Time</td>
<td><img src="{image_data.get('total_memcpy_time_sweep1', '')}" alt="total_memcpy_time Sweep 1"></td>
<td><img src="{image_data.get('total_memcpy_time_sweep2', '')}" alt="total_memcpy_time Sweep 2"></td>
</tr>

<tr>
<td>Total Time</td>
<td><img src="{image_data.get('total_time_sweep1', '')}" alt="total_time Sweep 1"></td>
<td><img src="{image_data.get('total_time_sweep2', '')}" alt="total_time Sweep 2"></td>
</tr>

</table>

<hr>

<table>

<tr>
<th>Index</th>
<th>{label1}</th>
<th>{label2}</th>
</tr>

<tr>
<td>0</td>
<td><img src="{image_data.get('k0_sweep1', '')}" alt="Kernel 0 Sweep 1"></td>
<td><img src="{image_data.get('k0_sweep2', '')}" alt="Kernel 0 Sweep 2"></td>
</tr>

<tr>
<td>1</td>
<td><img src="{image_data.get('k1_sweep1', '')}" alt="Kernel 1 Sweep 1"></td>
<td><img src="{image_data.get('k1_sweep2', '')}" alt="Kernel 1 Sweep 2"></td>
</tr>

<tr>
<td>2</td>
<td><img src="{image_data.get('k2_sweep1', '')}" alt="Kernel 2 Sweep 1"></td>
<td><img src="{image_data.get('k2_sweep2', '')}" alt="Kernel 2 Sweep 2"></td>
</tr>

<tr>
<td>3</td>
<td><img src="{image_data.get('k3_sweep1', '')}" alt="Kernel 3 Sweep 1"></td>
<td><img src="{image_data.get('k3_sweep2', '')}" alt="Kernel 3 Sweep 2"></td>
</tr>

<hr>

<div class="data-section">
<h2>Data Files Information</h2>

Expand Down Expand Up @@ -416,6 +549,7 @@ def main():
args.sweep2,
args.label1,
args.label2,
args.variance,
args.output
)

Expand All @@ -424,6 +558,3 @@ def main():

if __name__ == "__main__":
exit(main())



14 changes: 9 additions & 5 deletions scripts/gemm_analysis/generate_comparison_report.py
View file
Open in desktop
Copy link
Collaborator

@oyazdanb oyazdanb Dec 5, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

rebase it to the main. The updated versions of these files are in the main. Put your changes on top of them.

Original file line number Diff line number Diff line change
Expand Up @@ -50,14 +50,18 @@ def get_thread_and_type_values_over_ranks_with_mean_channel(all_result) :
print("Done computing geomeans across channels.")
return mean_result

def plot_mean_result(output_dir, rank_length, mean_result) :
def plot_mean_result(output_dir, rank_length, mean_result, threads) :
bar_width = 0.35
x_pos = np.arange(rank_length)
for type, type_info in mean_result.items() :
output_file = output_dir / f"{type}.png"
output_file = output_dir / f"{type}_rank_variance.png"
plt.figure()
plt.bar(x_pos - (bar_width/2), type_info['256'], bar_width, label="256", color='r')
plt.bar(x_pos + (bar_width/2), type_info['512'], bar_width, label="512", color='b')

if(len(threads) > 1) :
plt.bar(x_pos - (bar_width/2), type_info[str(threads[0])], bar_width, label="256", color='r')
plt.bar(x_pos + (bar_width/2), type_info[str(threads[1])], bar_width, label="512", color='b')
else :
plt.bar(x_pos, type_info[str(threads[0])], bar_width, label="256", color='b')
plt.ylabel("Time")
plt.xlabel("Rank")
plt.title(type)
Expand Down Expand Up @@ -165,7 +169,7 @@ def main():

all_results = process_comparison_data(base_path=base_path, channel=channels, thread=thread_configs, rank=ranks)
mean_results = get_thread_and_type_values_over_ranks_with_mean_channel(all_results)
plot_mean_result(output_dir, len(ranks), mean_results)
plot_mean_result(output_dir, len(ranks), mean_results, threads=thread_configs)

if __name__ == "__main__":
main()
Expand Down
Loading