diff --git a/scripts/tracelens_single_config/add_collective_comparison.py b/scripts/tracelens_single_config/add_collective_comparison.py new file mode 100644 index 0000000..b72dba1 --- /dev/null +++ b/scripts/tracelens_single_config/add_collective_comparison.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 +import pandas as pd +import argparse +from openpyxl.styles import Color +from openpyxl.formatting.rule import ColorScaleRule + + +def add_collective_comparison_sheets(input_path, output_path): + """ + Add comparison sheets to the combined collective reports. + This function will create comparison sheets for the combined collective reports. + The comparison sheets will contain the comparison of the baseline and test data. + TODO : Later we need to generalize for n runs and get rid of hardcoded data labels + """ + print(f"Loading: {input_path}") + + xl = pd.ExcelFile(input_path) + + with pd.ExcelWriter(output_path, engine="openpyxl") as writer: + # Copy only summary sheets + for sheet_name in xl.sheet_names: + # Only keep sheets with 'summary' in the name + if "summary" not in sheet_name.lower(): + print(f" Skip {sheet_name} (keeping only summary sheets)") + continue + df = pd.read_excel(input_path, sheet_name=sheet_name) + df.to_excel(writer, sheet_name=sheet_name, index=False) + print(f" Copied {sheet_name}") + + # Process summary sheets for comparison + for sheet_name in ["nccl_summary_implicit_sync", "nccl_summary_long"]: + if sheet_name not in xl.sheet_names: + continue + + df = pd.read_excel(input_path, sheet_name=sheet_name) + + # Separate baseline and test + baseline_df = df[df["source"] == "baseline"].copy() + test_df = df[df["source"] == "test"].copy() + + if len(baseline_df) == 0 or len(test_df) == 0: + print(f" Skip {sheet_name} - missing data") + continue + + # Create comparison dataframe + comparison = pd.DataFrame() + + # Identify key columns for grouping + group_cols = ["Collective name", "dtype", "In msg nelems"] + if not all(col in baseline_df.columns for col in group_cols): + group_cols = ["Collective name"] + + # Group and compare + baseline_grouped = baseline_df.groupby(group_cols, as_index=False) + test_grouped = test_df.groupby(group_cols, as_index=False) + + for name, base_group in baseline_grouped: + # Find matching test group + if isinstance(name, tuple): + mask = pd.Series([True] * len(test_df), index=test_df.index) + for col, val in zip(group_cols, name): + mask = mask & (test_df[col] == val) + else: + mask = test_df[group_cols[0]] == name + + sale_group = test_df.loc[mask] + + if len(sale_group) == 0: + continue + + # Create comparison row + comp_row = {} + + # Copy grouping columns + if isinstance(name, tuple): + for col, val in zip(group_cols, name): + comp_row[col] = val + else: + comp_row[group_cols[0]] = name + + # Compare numeric columns + numeric_cols = [ + "comm_latency_mean", + "algo bw (GB/s)_mean", + "bus bw (GB/s)_mean", + "Total comm latency (ms)", + "count", + ] + + for col in numeric_cols: + if col not in base_group.columns or col not in sale_group.columns: + continue + + base_val = base_group[col].values[0] + sale_val = sale_group[col].values[0] + + comp_row[f"baseline_{col}"] = base_val + comp_row[f"test_{col}"] = sale_val + comp_row[f"diff_{col}"] = sale_val - base_val + + # For latency/time: positive percent_change means faster (less time) + # For bandwidth: positive percent_change means better (more bandwidth) + if "latency" in col.lower() or "time" in col.lower(): + # Lower is better - positive when test is faster + pct_change = ( + (base_val - sale_val) / base_val * 100 + if base_val != 0 + else 0 + ) + comp_row[f"percent_change_{col}"] = pct_change + elif "bw" in col.lower() or "bandwidth" in col.lower(): + # Higher is better - positive when test is better + pct_change = ( + (sale_val - base_val) / base_val * 100 + if base_val != 0 + else 0 + ) + comp_row[f"percent_change_{col}"] = pct_change + + comp_row[f"ratio_{col}"] = ( + sale_val / base_val if base_val != 0 else 0 + ) + + comparison = pd.concat( + [comparison, pd.DataFrame([comp_row])], ignore_index=True + ) + + # Write comparison sheet (shorten name to fit Excel's 31 char limit) + # Replace 'nccl_summary_' with 'nccl_' and '_comparison' with '_cmp' + comparison_sheet_name = ( + sheet_name.replace("nccl_summary_", "nccl_") + "_cmp" + ) + comparison.to_excel(writer, sheet_name=comparison_sheet_name, index=False) + print(f" Added {comparison_sheet_name}") + + # Add conditional formatting to percent_change columns + print(f" Applying conditional formatting to {comparison_sheet_name}...") + + ws = writer.sheets[comparison_sheet_name] + + # Format all percent_change columns with color scale + for col_idx, col in enumerate(comparison.columns, start=1): + if "percent_change" in col: + # Convert column index to Excel letter (A, B, C, ...) + if col_idx <= 26: + col_letter = chr(64 + col_idx) + else: + col_letter = chr(64 + (col_idx // 26)) + chr( + 64 + (col_idx % 26) + ) + + data_range = f"{col_letter}2:{col_letter}{len(comparison)+1}" + + # Color scale: red (min/negative) -> white (0) -> green (max/positive) + ws.conditional_formatting.add( + data_range, + ColorScaleRule( + start_type="min", + start_color="F8696B", # Red + mid_type="num", + mid_value=0, + mid_color="FFFFFF", # White + end_type="max", + end_color="63BE7B", # Green + ), + ) + + print(f" Formatted {col}") + + print(f"\nSaved: {output_path}") + print("\nNew comparison sheets added") + print("percent_change interpretation:") + print(" For latency/time: Positive = faster (less time)") + print(" For bandwidth: Positive = better (more bandwidth)") + return 0 + + +def main(): + parser = argparse.ArgumentParser( + description="Add comparison sheets to combined collective reports" + ) + parser.add_argument( + "--input", required=True, help="Input combined collective Excel file" + ) + parser.add_argument( + "--output", required=True, help="Output Excel file with comparison sheets" + ) + + args = parser.parse_args() + + return add_collective_comparison_sheets(args.input, args.output) + + +if __name__ == "__main__": + exit(main()) diff --git a/scripts/tracelens_single_config/add_comparison_sheets.py b/scripts/tracelens_single_config/add_comparison_sheets.py new file mode 100644 index 0000000..7c7c473 --- /dev/null +++ b/scripts/tracelens_single_config/add_comparison_sheets.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 +import pandas as pd +import argparse +from openpyxl.styles import Color +from openpyxl.formatting.rule import ColorScaleRule + + +def add_comparison_sheets(input_path, output_path): + """ + Create comparison sheets for the combined excel file of individual reports. + """ + print(f"Loading: {input_path}") + + xl = pd.ExcelFile(input_path) + + with pd.ExcelWriter(output_path, engine="openpyxl") as writer: + # Copy all original sheets + for sheet_name in xl.sheet_names: + df = pd.read_excel(input_path, sheet_name=sheet_name) + df.to_excel(writer, sheet_name=sheet_name, index=False) + print(f" Copied {sheet_name}") + + # Add comparison sheets + all_combined = pd.read_excel(input_path, sheet_name="All_Ranks_Combined") + + # Comparison 1: Side-by-side by rank + baseline_data = all_combined[all_combined["source"] == "baseline"] + test_data = all_combined[all_combined["source"] == "test"] + + comparison_by_rank = pd.DataFrame() + for rank in sorted(baseline_data["rank"].unique()): + base_rank = baseline_data[baseline_data["rank"] == rank].set_index("type") + sale_rank = test_data[test_data["rank"] == rank].set_index("type") + + for metric_type in base_rank.index: + if metric_type in sale_rank.index: + base_time = base_rank.loc[metric_type, "time ms"] + sale_time = sale_rank.loc[metric_type, "time ms"] + ratio_val = sale_time / base_time if base_time != 0 else 0 + # Percentage change: positive when test is faster (takes less time) + pct_change = ( + (base_time - sale_time) / base_time * 100 + if base_time != 0 + else 0 + ) + + # Determine if better or worse + if pct_change > 1: + status = "Better" + elif pct_change < -1: + status = "Worse" + else: + status = "Similar" + + comparison_by_rank = pd.concat( + [ + comparison_by_rank, + pd.DataFrame( + { + "rank": [rank], + "type": [metric_type], + "baseline_time_ms": [base_time], + "test_time_ms": [sale_time], + "diff_time_ms": [sale_time - base_time], + "percent_change": [pct_change], + "status": [status], + "ratio": [ratio_val], + "baseline_percent": [ + base_rank.loc[metric_type, "percent"] + ], + "test_percent": [ + sale_rank.loc[metric_type, "percent"] + ], + "diff_percent": [ + sale_rank.loc[metric_type, "percent"] + - base_rank.loc[metric_type, "percent"] + ], + } + ), + ], + ignore_index=True, + ) + + comparison_by_rank.to_excel( + writer, sheet_name="Comparison_By_Rank", index=False + ) + print(f" Added Comparison_By_Rank") + + # Comparison 2: Summary comparison + summary = pd.read_excel(input_path, sheet_name="Summary") + baseline_summary = summary[summary["source"] == "baseline"].set_index("type") + test_summary = summary[summary["source"] == "test"].set_index("type") + + summary_comparison = pd.DataFrame() + for metric_type in baseline_summary.index: + if metric_type in test_summary.index: + base_time = baseline_summary.loc[metric_type, "time ms"] + sale_time = test_summary.loc[metric_type, "time ms"] + ratio_val = sale_time / base_time if base_time != 0 else 0 + # Percentage change: positive when test is faster (takes less time) + pct_change = ( + (base_time - sale_time) / base_time * 100 if base_time != 0 else 0 + ) + + summary_comparison = pd.concat( + [ + summary_comparison, + pd.DataFrame( + { + "type": [metric_type], + "baseline_time_ms": [base_time], + "test_time_ms": [sale_time], + "diff_time_ms": [sale_time - base_time], + "percent_change": [pct_change], + "ratio": [ratio_val], + "baseline_percent": [ + baseline_summary.loc[metric_type, "percent"] + ], + "test_percent": [ + test_summary.loc[metric_type, "percent"] + ], + "diff_percent": [ + test_summary.loc[metric_type, "percent"] + - baseline_summary.loc[metric_type, "percent"] + ], + } + ), + ], + ignore_index=True, + ) + + summary_comparison.to_excel( + writer, sheet_name="Summary_Comparison", index=False + ) + print(f" Added Summary_Comparison") + + # Add conditional formatting to percent_change columns + print("\n Applying conditional formatting...") + + # Create color scale: Red (negative) -> White (0) -> Green (positive) + + # Format Comparison_By_Rank + ws_rank = writer.sheets["Comparison_By_Rank"] + # Find percent_change column + for col_idx, col in enumerate(comparison_by_rank.columns, start=1): + if col == "percent_change": + col_letter = chr(64 + col_idx) # Convert to Excel column letter + data_range = f"{col_letter}2:{col_letter}{len(comparison_by_rank)+1}" + # Color scale: red (min) -> white (0) -> green (max) + ws_rank.conditional_formatting.add( + data_range, + ColorScaleRule( + start_type="min", + start_color="F8696B", # Red + mid_type="num", + mid_value=0, + mid_color="FFFFFF", # White + end_type="max", + end_color="63BE7B", # Green + ), + ) + print(f" Formatted Comparison_By_Rank column {col}") + break + + # Format Summary_Comparison + ws_summary = writer.sheets["Summary_Comparison"] + for col_idx, col in enumerate(summary_comparison.columns, start=1): + if col == "percent_change": + col_letter = chr(64 + col_idx) + data_range = f"{col_letter}2:{col_letter}{len(summary_comparison)+1}" + # Color scale: red (min) -> white (0) -> green (max) + ws_summary.conditional_formatting.add( + data_range, + ColorScaleRule( + start_type="min", + start_color="F8696B", # Red + mid_type="num", + mid_value=0, + mid_color="FFFFFF", # White + end_type="max", + end_color="63BE7B", # Green + ), + ) + print(f" Formatted Summary_Comparison column {col}") + break + + print(f"\nSaved: {output_path}") + print("\nNew sheets:") + print(" Comparison_By_Rank - Side-by-side comparison for each rank") + print(" Summary_Comparison - Overall comparison") + return 0 + + +def main(): + parser = argparse.ArgumentParser( + description="Add comparison sheets to combined GPU timeline" + ) + parser.add_argument("--input", required=True, help="Input combined Excel file") + parser.add_argument( + "--output", required=True, help="Output Excel file with comparison sheets" + ) + + args = parser.parse_args() + + return add_comparison_sheets(args.input, args.output) + + +if __name__ == "__main__": + exit(main())