diff --git a/requirements.txt b/requirements.txt index 985670f..cb04fa8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,6 +14,10 @@ pyyaml>=6.0 # For analysis scripts matplotlib>=3.7.0 +panda +seaborn +openpyxl +nummpy # For hw_queue_eval (optional - install with: pip install -e ".[hw-queue]") # click>=8.0.0 diff --git a/scripts/tracelens_single_config/add_collective_comparison.py b/scripts/tracelens_single_config/add_collective_comparison.py index b72dba1..acfac65 100644 --- a/scripts/tracelens_single_config/add_collective_comparison.py +++ b/scripts/tracelens_single_config/add_collective_comparison.py @@ -5,7 +5,7 @@ from openpyxl.formatting.rule import ColorScaleRule -def add_collective_comparison_sheets(input_path, output_path): +def add_collective_comparison_sheets(input_path, output_path, baseline_label='baseline', test_label='test'): """ Add comparison sheets to the combined collective reports. This function will create comparison sheets for the combined collective reports. @@ -34,9 +34,19 @@ def add_collective_comparison_sheets(input_path, output_path): df = pd.read_excel(input_path, sheet_name=sheet_name) + # Get actual source values from the dataframe + sources = df['source'].unique() + # Determine which is baseline and which is test (baseline should be first) + if len(sources) >= 2: + actual_baseline = sources[0] + actual_test = sources[1] + else: + actual_baseline = baseline_label + actual_test = test_label + # Separate baseline and test - baseline_df = df[df["source"] == "baseline"].copy() - test_df = df[df["source"] == "test"].copy() + baseline_df = df[df["source"] == actual_baseline].copy() + test_df = df[df["source"] == actual_test].copy() if len(baseline_df) == 0 or len(test_df) == 0: print(f" Skip {sheet_name} - missing data") @@ -63,9 +73,9 @@ def add_collective_comparison_sheets(input_path, output_path): else: mask = test_df[group_cols[0]] == name - sale_group = test_df.loc[mask] + test_group = test_df.loc[mask] - if len(sale_group) == 0: + if len(test_group) == 0: continue # Create comparison row @@ -88,22 +98,22 @@ def add_collective_comparison_sheets(input_path, output_path): ] for col in numeric_cols: - if col not in base_group.columns or col not in sale_group.columns: + if col not in base_group.columns or col not in test_group.columns: continue base_val = base_group[col].values[0] - sale_val = sale_group[col].values[0] + test_val = test_group[col].values[0] - comp_row[f"baseline_{col}"] = base_val - comp_row[f"test_{col}"] = sale_val - comp_row[f"diff_{col}"] = sale_val - base_val + comp_row[f"{actual_baseline}_{col}"] = base_val + comp_row[f"{actual_test}_{col}"] = test_val + comp_row[f"diff_{col}"] = test_val - base_val # For latency/time: positive percent_change means faster (less time) # For bandwidth: positive percent_change means better (more bandwidth) if "latency" in col.lower() or "time" in col.lower(): # Lower is better - positive when test is faster pct_change = ( - (base_val - sale_val) / base_val * 100 + (base_val - test_val) / base_val * 100 if base_val != 0 else 0 ) @@ -111,14 +121,14 @@ def add_collective_comparison_sheets(input_path, output_path): elif "bw" in col.lower() or "bandwidth" in col.lower(): # Higher is better - positive when test is better pct_change = ( - (sale_val - base_val) / base_val * 100 + (test_val - base_val) / base_val * 100 if base_val != 0 else 0 ) comp_row[f"percent_change_{col}"] = pct_change comp_row[f"ratio_{col}"] = ( - sale_val / base_val if base_val != 0 else 0 + test_val / base_val if base_val != 0 else 0 ) comparison = pd.concat( @@ -185,10 +195,13 @@ def main(): parser.add_argument( "--output", required=True, help="Output Excel file with comparison sheets" ) + parser.add_argument('--baseline-label', default='baseline', help='Label for baseline data') + parser.add_argument('--test-label', default='test', help='Label for test data') args = parser.parse_args() - return add_collective_comparison_sheets(args.input, args.output) + return add_collective_comparison_sheets(args.input, args.output, args.baseline_label, args.test_label) + if __name__ == "__main__": diff --git a/scripts/tracelens_single_config/add_comparison_sheets.py b/scripts/tracelens_single_config/add_comparison_sheets.py index 7c7c473..c309ed3 100644 --- a/scripts/tracelens_single_config/add_comparison_sheets.py +++ b/scripts/tracelens_single_config/add_comparison_sheets.py @@ -5,7 +5,7 @@ from openpyxl.formatting.rule import ColorScaleRule -def add_comparison_sheets(input_path, output_path): +def add_comparison_sheets(input_path, output_path, baseline_label='baseline', test_label='test'): """ Create comparison sheets for the combined excel file of individual reports. """ @@ -23,23 +23,33 @@ def add_comparison_sheets(input_path, output_path): # Add comparison sheets all_combined = pd.read_excel(input_path, sheet_name="All_Ranks_Combined") + # Get actual source values from the dataframe + sources = all_combined['source'].unique() + # Determine which is baseline and which is test (baseline should be first) + if len(sources) >= 2: + actual_baseline = sources[0] + actual_test = sources[1] + else: + actual_baseline = baseline_label + actual_test = test_label + # Comparison 1: Side-by-side by rank - baseline_data = all_combined[all_combined["source"] == "baseline"] - test_data = all_combined[all_combined["source"] == "test"] + baseline_data = all_combined[all_combined["source"] == actual_baseline] + test_data = all_combined[all_combined["source"] == actual_test] comparison_by_rank = pd.DataFrame() for rank in sorted(baseline_data["rank"].unique()): base_rank = baseline_data[baseline_data["rank"] == rank].set_index("type") - sale_rank = test_data[test_data["rank"] == rank].set_index("type") + test_rank = test_data[test_data["rank"] == rank].set_index("type") for metric_type in base_rank.index: - if metric_type in sale_rank.index: + if metric_type in test_rank.index: base_time = base_rank.loc[metric_type, "time ms"] - sale_time = sale_rank.loc[metric_type, "time ms"] - ratio_val = sale_time / base_time if base_time != 0 else 0 + test_time = test_rank.loc[metric_type, "time ms"] + ratio_val = test_time / base_time if base_time != 0 else 0 # Percentage change: positive when test is faster (takes less time) pct_change = ( - (base_time - sale_time) / base_time * 100 + (base_time - test_time) / base_time * 100 if base_time != 0 else 0 ) @@ -59,20 +69,20 @@ def add_comparison_sheets(input_path, output_path): { "rank": [rank], "type": [metric_type], - "baseline_time_ms": [base_time], - "test_time_ms": [sale_time], - "diff_time_ms": [sale_time - base_time], + f"{baseline_label}_time_ms": [base_time], + f"{test_label}_time_ms": [test_time], + "diff_time_ms": [test_time - base_time], "percent_change": [pct_change], "status": [status], "ratio": [ratio_val], - "baseline_percent": [ + f"{baseline_label}_percent": [ base_rank.loc[metric_type, "percent"] ], - "test_percent": [ - sale_rank.loc[metric_type, "percent"] + f"{test_label}_percent": [ + test_rank.loc[metric_type, "percent"] ], "diff_percent": [ - sale_rank.loc[metric_type, "percent"] + test_rank.loc[metric_type, "percent"] - base_rank.loc[metric_type, "percent"] ], } @@ -88,18 +98,18 @@ def add_comparison_sheets(input_path, output_path): # Comparison 2: Summary comparison summary = pd.read_excel(input_path, sheet_name="Summary") - baseline_summary = summary[summary["source"] == "baseline"].set_index("type") - test_summary = summary[summary["source"] == "test"].set_index("type") + baseline_summary = summary[summary["source"] == actual_baseline].set_index("type") + test_summary = summary[summary["source"] == actual_test].set_index("type") summary_comparison = pd.DataFrame() for metric_type in baseline_summary.index: if metric_type in test_summary.index: base_time = baseline_summary.loc[metric_type, "time ms"] - sale_time = test_summary.loc[metric_type, "time ms"] - ratio_val = sale_time / base_time if base_time != 0 else 0 + test_time = test_summary.loc[metric_type, "time ms"] + ratio_val = test_time / base_time if base_time != 0 else 0 # Percentage change: positive when test is faster (takes less time) pct_change = ( - (base_time - sale_time) / base_time * 100 if base_time != 0 else 0 + (base_time - test_time) / base_time * 100 if base_time != 0 else 0 ) summary_comparison = pd.concat( @@ -108,15 +118,15 @@ def add_comparison_sheets(input_path, output_path): pd.DataFrame( { "type": [metric_type], - "baseline_time_ms": [base_time], - "test_time_ms": [sale_time], - "diff_time_ms": [sale_time - base_time], + f"{baseline_label}_time_ms": [base_time], + f"{test_label}_time_ms": [test_time], + "diff_time_ms": [test_time - base_time], "percent_change": [pct_change], "ratio": [ratio_val], - "baseline_percent": [ + f"{baseline_label}_percent": [ baseline_summary.loc[metric_type, "percent"] ], - "test_percent": [ + f"{test_label}_percent": [ test_summary.loc[metric_type, "percent"] ], "diff_percent": [ @@ -199,10 +209,12 @@ def main(): parser.add_argument( "--output", required=True, help="Output Excel file with comparison sheets" ) + parser.add_argument('--baseline-label', default='baseline', help='Label for baseline data') + parser.add_argument('--test-label', default='test', help='Label for test data') args = parser.parse_args() - return add_comparison_sheets(args.input, args.output) + return add_comparison_sheets(args.input, args.output, args.baseline_label, args.test_label) if __name__ == "__main__": diff --git a/scripts/tracelens_single_config/combine_reports.py b/scripts/tracelens_single_config/combine_reports.py index 1fc2eba..ea01d17 100644 --- a/scripts/tracelens_single_config/combine_reports.py +++ b/scripts/tracelens_single_config/combine_reports.py @@ -4,15 +4,18 @@ from pathlib import Path -def combine_collective_reports(baseline_path, test_path, output_path): +def combine_collective_reports(baseline_path, test_path, output_path, baseline_label = "baseline", test_label="test"): """ Combine two collective reports into a single Excel file by adding a source column to the data. """ + # Extract folder names from paths for labels + #baseline_label = Path(baseline_path).parent.parent.name # Get the config folder name + #test_label = Path(test_path).parent.parent.name # Get the config folder name - print(f"Loading baseline: {baseline_path}") + print(f"Loading baseline ({baseline_label}): {baseline_path}") baseline_xl = pd.ExcelFile(baseline_path) - print(f"Loading test: {test_path}") + print(f"Loading test ({test_label}): {test_path}") test_xl = pd.ExcelFile(test_path) print(f"\nBaseline sheets: {baseline_xl.sheet_names}") @@ -27,8 +30,8 @@ def combine_collective_reports(baseline_path, test_path, output_path): baseline_df = pd.read_excel(baseline_path, sheet_name=sheet_name) test_df = pd.read_excel(test_path, sheet_name=sheet_name) - baseline_df["source"] = "baseline" - test_df["source"] = "test" + baseline_df["source"] = baseline_label + test_df["source"] = test_label combined = pd.concat([baseline_df, test_df], ignore_index=True) @@ -49,13 +52,19 @@ def main(): parser.add_argument( "--test", required=True, help="Path to test collective_all_ranks.xlsx" ) + parser.add_argument( + "--baseline-label", default="baseline", help="Label for baseline data" + ) + parser.add_argument( + "--test-label", default="test", help="Label for test data" + ) parser.add_argument( "--output", required=True, help="Output path for combined Excel file" ) args = parser.parse_args() - return combine_collective_reports(args.baseline, args.test, args.output) + return combine_collective_reports(args.baseline, args.test, args.output, args.baseline_label, args.test_label) if __name__ == "__main__": diff --git a/scripts/tracelens_single_config/create_final_plots.py b/scripts/tracelens_single_config/create_final_plots.py index 006f6ab..a9074be 100644 --- a/scripts/tracelens_single_config/create_final_plots.py +++ b/scripts/tracelens_single_config/create_final_plots.py @@ -2,7 +2,7 @@ import matplotlib.pyplot as plt from pathlib import Path import seaborn as sns - +import numpy as np def plot_improvement_chart(df, output_path): fig, ax = plt.subplots(figsize=(10, 6)) @@ -33,26 +33,28 @@ def plot_improvement_chart(df, output_path): plt.close() -def plot_abs_time_comparison(df, output_path): +def plot_abs_time_comparison(df, output_path, labels): fig, ax = plt.subplots(figsize=(10, 6)) # Set up bar positions - x = range(len(df)) + x = np.arange(len(df)) width = 0.35 + colors = ["#3498db", "#e67e22"] + values = [] + for label in labels: + values.append(df[label]) # Create bars for Baseline and Test - bars1 = ax.bar( - [i - width / 2 for i in x], - df["Baseline"], - width, - label="Baseline", - color="#3498db", - ) - bars2 = ax.bar( - [i + width / 2 for i in x], df["Test"], width, label="Test", color="#e67e22" - ) - + for i,val in enumerate(values): + offset = (i - len(labels) / 2 + 0.5) * width + bars = ax.bar( + x + offset, + val, + width, + label=labels[i], + color=colors[i], + ) # Add horizontal grid lines only ax.xaxis.grid(True, linestyle="--", alpha=0.7, color="gray") ax.set_axisbelow(True) @@ -78,41 +80,31 @@ def plot_abs_time_comparison(df, output_path): plt.close() -def create_summary_charts(excel_path, output_path): +def create_summary_charts(excel_path, output_path, labels): # Read the Summary_Dashboard sheet df = pd.read_excel(excel_path, sheet_name="Summary_Dashboard") plot_improvement_chart(df, output_path) - plot_abs_time_comparison(df, output_path) + plot_abs_time_comparison(df, output_path, labels) # Create the horizontal bar chart -def plot_gpu_type_by_rank(total_time_df, output_path, title): +def plot_gpu_type_by_rank(total_time_df, output_path, title, labels): # Create the line plot fig, ax = plt.subplots(figsize=(12, 6)) - - # Plot baseline total_time by rank - ax.plot( - total_time_df["rank"], - total_time_df["baseline_time_ms"], - marker="o", - linewidth=2, - markersize=8, - color="#3498db", - label="Baseline", - ) - - # Plot Test (test) total_time by rank - ax.plot( - total_time_df["rank"], - total_time_df["test_time_ms"], - marker="s", - linewidth=2, - markersize=8, - color="#e67e22", - label="Test", - ) + colors = ["#3498db", "#e67e22"] + markers = ["o", "s"] + for i, label in enumerate(labels): + ax.plot( + total_time_df["rank"], + total_time_df[label + "_time_ms"], + marker=markers[i], + linewidth=2, + markersize=8, + color=colors[i], + label=label, + ) # Add horizontal grid lines only ax.yaxis.grid(True, linestyle="--", alpha=0.7, color="gray") @@ -129,14 +121,14 @@ def plot_gpu_type_by_rank(total_time_df, output_path, title): plt.close() -def create_gpu_time_accross_all_ranks(excel_path, output_path): +def create_gpu_time_accross_all_ranks(excel_path, output_path, labels): # Read the GPU_ByRank_Cmp sheet df = pd.read_excel(excel_path, sheet_name="GPU_ByRank_Cmp") # Filter for total_time rows only for type in ["total_time", "computation_time", "total_comm_time", "idle_time"]: total_time_df = df[df["type"] == type] - plot_gpu_type_by_rank(total_time_df, output_path / f"{type}_by_rank.png", type) + plot_gpu_type_by_rank(total_time_df, output_path / f"{type}_by_rank.png", type, labels) def plot_gpu_time_change_percentage_summaryby_rank(df, ax): @@ -178,60 +170,50 @@ def create_gpu_time_change_percentage_summaryby_rank(excel_path, output_path): plt.close() -def create_nccl_charts(excel_path, output_path): +def create_nccl_charts(excel_path, output_path, labels): # Read the NCCL_Charst sheet - df = pd.read_excel(excel_path, sheet_name="NCCL_ImplSync_Cmp") + df = pd.read_excel(excel_path, sheet_name="NCCL_ImplicitSyncCmp") df["label"] = df["Collective name"] + "\n" + df["In msg nelems"].astype(str) - x = range(len(df)) - + x = np.arange(len(df)) + colors = ["#3498db", "#e67e22"] plot_item = { "NCCL Communication Latency": { "x_label": "Collective Operation (Message Size)", "y_label": "Communication Latency (ms)", - "y_col_names": ["baseline_comm_latency_mean", "test_comm_latency_mean"], + "y_col": "comm_latency_mean" }, "NCCL Algorithm Bandwidth": { "x_label": "Collective Operation (Message Size)", "y_label": "Algorithm Bandwidth (GB/s)", - "y_col_names": [ - "baseline_algo bw (GB/s)_mean", - "test_algo bw (GB/s)_mean", - ], + "y_col": "algo bw (GB/s)_mean" }, "NCCL Bus Bandwidth": { "x_label": "Collective Operation (Message Size)", "y_label": "Bus Bandwidth (GB/s)", - "y_col_names": [ - "baseline_bus bw (GB/s)_mean", - "test_bus bw (GB/s)_mean", - ], + "y_col": "bus bw (GB/s)_mean" }, "NCCL Total Communication Latency": { "x_label": "Collective Operation (Message Size)", "y_label": "Total Communication Latency (ms)", - "y_col_names": [ - "baseline_Total comm latency (ms)", - "test_Total comm latency (ms)", - ], + "y_col": "Total comm latency (ms)" }, } for item in plot_item.keys(): fig, ax = plt.subplots(figsize=(14, 6)) width = 0.35 - bars1 = ax.bar( - [i - width / 2 for i in x], - df[plot_item[item]["y_col_names"][0]], - width, - label="Baseline", - color="#3498db", - ) - bars2 = ax.bar( - [i + width / 2 for i in x], - df[plot_item[item]["y_col_names"][1]], - width, - label="Test", - color="#e67e22", - ) + for i, label in enumerate(labels): + col_name = f"{label}_{plot_item[item]['y_col']}" + print(f"col_name: {col_name} {df.columns}") + if(col_name in df.columns): + values = df[col_name].values + offset = (i - len(labels) / 2 + 0.5) * width + bar = ax.bar( + x + offset, + values, + width, + label=label, + color=colors[i], + ) ax.yaxis.grid(True, linestyle="--", alpha=0.7, color="gray") ax.set_axisbelow(True) ax.set_xticks(x) @@ -309,6 +291,11 @@ def create_gpu_time_heatmap(excel_path, output_path): plt.savefig(output_path / "gpu_time_heatmap.png", dpi=150) plt.show() +def get_labels(excel_path): + # Read the Summary_Dashboard sheet + df = pd.read_excel(excel_path, sheet_name="Summary_Dashboard") + cols = df.columns.tolist() + return [cols[1], cols[2]] # for now just for two, later will be generalized for more than two def main(): import argparse @@ -317,7 +304,7 @@ def main(): description="Generate improvement chart from generated reports" ) parser.add_argument( - "--report-path", + "--input", type=Path, default="~/aorta/aorta_single_config/aorta/expt_compare/final_analysis_report.xlsx", help="Path to the input Excel file (should have Summary_Dashboard sheet)", @@ -330,17 +317,18 @@ def main(): ) args = parser.parse_args() - output_path = args.output if args.output else args.report_path.parent / "plots" + output_path = args.output if args.output else args.input.parent / "plots" output_path.mkdir(exist_ok=True, parents=True) - create_summary_charts(args.report_path, output_path) + labels = get_labels(args.input) + create_summary_charts(args.input, output_path, labels) print(f"Summary charts saved to: {args.output}") - create_gpu_time_heatmap(args.report_path, output_path) + create_gpu_time_heatmap(args.input, output_path) print(f"GPU time heatmap saved to: {output_path}") - create_gpu_time_accross_all_ranks(args.report_path, output_path) + create_gpu_time_accross_all_ranks(args.input, output_path, labels) print(f"GPU time across all runs saved to: {output_path}") - create_gpu_time_change_percentage_summaryby_rank(args.report_path, output_path) + create_gpu_time_change_percentage_summaryby_rank(args.input, output_path) print(f"GPU time change percentage summary by rank saved to: {output_path}") - create_nccl_charts(args.report_path, output_path) + create_nccl_charts(args.input, output_path, labels) print(f"NCCL communication charts saved to: {output_path}") diff --git a/scripts/tracelens_single_config/create_final_report.py b/scripts/tracelens_single_config/create_final_report.py index 1a1f8d8..09cd382 100644 --- a/scripts/tracelens_single_config/create_final_report.py +++ b/scripts/tracelens_single_config/create_final_report.py @@ -62,7 +62,7 @@ def add_excel_table(worksheet, table_name, start_row=1): def create_final_report( - gpu_combined, gpu_comparison, coll_combined, coll_comparison, output_file + gpu_combined, gpu_comparison, coll_combined, coll_comparison, output_file,baseline_label='Baseline', test_label='Test' ): """Create comprehensive report with all data.""" @@ -127,17 +127,26 @@ def create_final_report( # Read collective comparison coll_comp_xl = pd.ExcelFile(coll_comparison) - coll_cmp_mapping = { - "nccl_implicit_sync_cmp": "NCCL_ImplSync_Cmp", - "nccl_long_cmp": "NCCL_Long_Cmp", - } for sheet_name in coll_comp_xl.sheet_names: - if "_cmp" in sheet_name: - df = pd.read_excel(coll_comparison, sheet_name=sheet_name) - new_name = coll_cmp_mapping.get(sheet_name, f"NCCL_{sheet_name}") - df.to_excel(writer, sheet_name=new_name, index=False) + df = pd.read_excel(coll_comparison, sheet_name=sheet_name) + + # Determine appropriate naming + if 'nccl' in sheet_name.lower(): + if '_cmp' in sheet_name or 'comparison' in sheet_name.lower(): + new_name = f"NCCL_{sheet_name.replace('nccl_', '').title().replace('_', '')}" + else: + new_name = f"NCCL_{sheet_name}" + else: + new_name = sheet_name + + df.to_excel(writer, sheet_name=new_name, index=False) + + if '_cmp' in sheet_name.lower() or 'comparison' in sheet_name.lower(): comparison_sheets.append(new_name) - print(f" Added {new_name}") + else: + raw_sheets.append(new_name) + + print(f" Added {new_name}") # === CREATE SUMMARY DASHBOARD === print("\nCreating Summary Dashboard...") @@ -147,25 +156,34 @@ def create_final_report( # Create dashboard data dashboard_data = { - "Metric": [], - "Baseline": [], - "Test": [], - "Improvement (%)": [], - "Status": [], + 'Metric': [], + baseline_label: [], + test_label: [], + 'Improvement (%)': [], + 'Status': [] } # Add GPU metrics - for _, row in gpu_summary.iterrows(): - metric_type = row["type"] - dashboard_data["Metric"].append(f"GPU_{metric_type}") - dashboard_data["Baseline"].append(round(row["baseline_time_ms"], 2)) - dashboard_data["Test"].append(round(row["test_time_ms"], 2)) - dashboard_data["Improvement (%)"].append(round(row["percent_change"], 2)) - dashboard_data["Status"].append( - "Better" - if row["percent_change"] > 0 - else "Worse" if row["percent_change"] < -1 else "Similar" - ) + # Find the actual column names (they may be config-specific like '32cu_512threads_time_ms') + time_cols = [col for col in gpu_summary.columns if 'time_ms' in col and 'diff' not in col and 'percent' not in col] + if len(time_cols) >= 2: + baseline_col = time_cols[0] + test_col = time_cols[1] + else: + # Fallback to default names + baseline_col = 'baseline_time_ms' if 'baseline_time_ms' in gpu_summary.columns else time_cols[0] if time_cols else None + test_col = 'test_time_ms' if 'test_time_ms' in gpu_summary.columns else time_cols[1] if len(time_cols) > 1 else None + + if baseline_col and test_col: + for _, row in gpu_summary.iterrows(): + metric_type = row['type'] + dashboard_data['Metric'].append(f"GPU_{metric_type}") + dashboard_data[baseline_label].append(round(row[baseline_col], 2)) + dashboard_data[test_label].append(round(row[test_col], 2)) + dashboard_data['Improvement (%)'].append(round(row['percent_change'], 2) if 'percent_change' in row else 0) + + pct_val = row['percent_change'] if 'percent_change' in row else 0 + dashboard_data['Status'].append('Better' if pct_val > 0 else 'Worse' if pct_val < -1 else 'Similar') dashboard_df = pd.DataFrame(dashboard_data) dashboard_df.to_excel(writer, sheet_name="Summary_Dashboard", index=False) @@ -291,6 +309,10 @@ def main(): "--coll-comparison", required=True, help="Path to collective comparison file" ) parser.add_argument("--output", required=True, help="Output path for final report") + parser.add_argument('--baseline-label', default='Baseline', + help='Label for baseline configuration') + parser.add_argument('--test-label', default='Test', + help='Label for test configuration') args = parser.parse_args() @@ -312,6 +334,10 @@ def main(): args.coll_combined, args.coll_comparison, args.output, + #args.baseline_label.replace('_', ' '), + #args.test_label.replace('_', ' ') + args.baseline_label, + args.test_label ) return 0 diff --git a/scripts/tracelens_single_config/run_full_analysis.py b/scripts/tracelens_single_config/run_full_analysis.py index 5385ec4..163fe69 100644 --- a/scripts/tracelens_single_config/run_full_analysis.py +++ b/scripts/tracelens_single_config/run_full_analysis.py @@ -54,8 +54,12 @@ def process_gpu_timeline(reports_dir): return run_command(cmd, "Processing GPU timeline") -def combine_reports(baseline_file, test_file, output_file): +def combine_reports(baseline_file, test_file, output_file, baseline_label=None, test_label=None): """Combine baseline and test reports.""" + if(baseline_label is None or test_label is None) : + print("WARN : Baseline and test label are not provided, using default labels baseline and test") + baseline_label = "baseline" + test_label = "test" script_path = Path(__file__).parent / "combine_reports.py" cmd = [ "python3", @@ -64,6 +68,10 @@ def combine_reports(baseline_file, test_file, output_file): baseline_file, "--test", test_file, + "--baseline-label", + baseline_label, + "--test-label", + test_label, "--output", output_file, ] @@ -71,56 +79,51 @@ def combine_reports(baseline_file, test_file, output_file): return run_command(cmd, f"Combining reports to {output_file}") -def add_comparison_sheets(input_file, output_file): +def add_comparison_sheets(input_file, output_file, baseline_label=None, test_label=None): """Add comparison sheets for GPU timeline.""" script_path = Path(__file__).parent / "add_comparison_sheets.py" - cmd = ["python3", str(script_path), "--input", input_file, "--output", output_file] - + cmd = ["python3", str(script_path), + "--input", input_file, + "--output", output_file] + if baseline_label: + cmd.extend(["--baseline-label", baseline_label]) + if test_label: + cmd.extend(["--test-label", test_label]) return run_command(cmd, "Adding GPU timeline comparison sheets") -def add_collective_comparison(input_file, output_file): +def add_collective_comparison(input_file, output_file, baseline_label=None, test_label=None): """Add comparison sheets for collective operations.""" script_path = Path(__file__).parent / "add_collective_comparison.py" - cmd = ["python3", str(script_path), "--input", input_file, "--output", output_file] - + cmd = ["python3", str(script_path), + "--input", input_file, + "--output", output_file] + if baseline_label: + cmd.extend(["--baseline-label", baseline_label]) + if test_label: + cmd.extend(["--test-label", test_label]) return run_command(cmd, "Adding collective comparison sheets") def create_final_report( - gpu_combined, gpu_comparison, coll_combined, coll_comparison, output_file + gpu_combined, gpu_comparison, coll_combined, coll_comparison, output_file,baseline_label=None, test_label=None ): """Create comprehensive final report with all data.""" script_path = Path(__file__).parent / "create_final_report.py" cmd = [ - "python3", - str(script_path), - "--gpu-combined", - gpu_combined, - "--gpu-comparison", - gpu_comparison, - "--coll-combined", - coll_combined, - "--coll-comparison", - coll_comparison, - "--output", - output_file, + "python3", str(script_path), + "--gpu-combined", gpu_combined, + "--gpu-comparison", gpu_comparison, + "--coll-combined", coll_combined, + "--coll-comparison", coll_comparison, + "--output", output_file, ] + if baseline_label: + cmd.extend(["--baseline-label", baseline_label]) + if test_label: + cmd.extend(["--test-label", test_label]) - if run_command(cmd, "Creating comprehensive final report"): - plot_script_path = Path(__file__).parent / "create_final_plots.py" - cmd = ["python3", str(plot_script_path), "--report-path", output_file] - if run_command(cmd, "Creating final plots"): - html_script_path = Path(__file__).parent / "create_final_html.py" - cmd = [ - "python3", - str(html_script_path), - "--plot-files-directory", - str(Path(output_file).parent / "plots"), - ] - if run_command(cmd, "Creating final HTML"): - return True - return False + return run_command(cmd, "Creating comprehensive final report") def main(): @@ -191,6 +194,9 @@ def main(): action="store_true", help="Create comprehensive final report with tables and hidden raw data", ) + parser.add_argument( + "--generate-plots", action="store_true", help="Generate visualization plots and HTML report from final report", + ) parser.add_argument( "--all", action="store_true", @@ -204,6 +210,7 @@ def main(): args.gpu_timeline = True args.collective = True args.final_report = True + args.generate_plots = True # Validate inputs baseline_path = Path(args.baseline) @@ -265,10 +272,18 @@ def main(): print("Run without --skip-tracelens flag first") return 1 + # Extract config labels from paths + baseline_label = baseline_path.name # e.g., "56cu_256threads" + test_label = test_path.name # e.g., "37cu_384threads" + + if(baseline_label == test_label) : + print(f"Same baseline and test label : {baseline_label}, reverting to default labels baseline and test") # Step 2: GPU Timeline Comparison if args.gpu_timeline: print("\n" + "=" * 80) print("STEP 2: GPU Timeline Comparison") + print(f" Baseline: {baseline_label}") + print(f" Test: {test_label}") print("=" * 80) # Process GPU timelines @@ -281,11 +296,11 @@ def main(): ) return 1 - print("\nProcessing baseline GPU timeline...") + print("\nProcessing baseline GPU timeline ({baseline_label})...") if not process_gpu_timeline(str(baseline_reports)): return 1 - print("\nProcessing test GPU timeline...") + print("\nProcessing test GPU timeline ({test_label})...") if not process_gpu_timeline(str(test_reports)): return 1 @@ -294,12 +309,12 @@ def main(): test_gpu = test_analysis / "gpu_timeline_summary_mean.xlsx" combined_gpu = output_path / "gpu_timeline_combined.xlsx" - if not combine_reports(str(baseline_gpu), str(test_gpu), str(combined_gpu)): + if not combine_reports(str(baseline_gpu), str(test_gpu), str(combined_gpu), baseline_label, test_label): return 1 # Add comparison sheets gpu_comparison = output_path / "gpu_timeline_comparison.xlsx" - if not add_comparison_sheets(str(combined_gpu), str(gpu_comparison)): + if not add_comparison_sheets(str(combined_gpu), str(gpu_comparison), baseline_label, test_label): return 1 print(f"\nGPU timeline comparison saved to: {gpu_comparison}") @@ -308,6 +323,8 @@ def main(): if args.collective: print("\n" + "=" * 80) print("STEP 3: Collective/NCCL Comparison") + print(f" Baseline: {baseline_label}") + print(f" Test: {test_label}") print("=" * 80) baseline_collective = ( @@ -326,14 +343,15 @@ def main(): # Combine collective reports combined_collective = output_path / "collective_combined.xlsx" if not combine_reports( - str(baseline_collective), str(test_collective), str(combined_collective) + str(baseline_collective), str(test_collective), str(combined_collective), baseline_label, test_label ): return 1 # Add collective comparison collective_comparison = output_path / "collective_comparison.xlsx" if not add_collective_comparison( - str(combined_collective), str(collective_comparison) + str(combined_collective), str(collective_comparison), + baseline_label, test_label ): return 1 @@ -357,6 +375,8 @@ def main(): str(collective_combined), str(collective_comparison), str(final_report), + baseline_label, + test_label ): return 1 @@ -367,6 +387,43 @@ def main(): print(" - All data formatted as Excel tables with filters") print(" - Color coding applied (green=better, red=worse)") + # Step 5: Generate visualization plots + if args.generate_plots and args.final_report: + print("\n" + "="*80) + print("STEP 5: Generating Visualization Plots") + print("="*80) + + final_report = output_path / "final_analysis_report.xlsx" + plots_dir = output_path / "plots" + + if final_report.exists(): + script_path = Path(__file__).parent / "create_final_plots.py" + cmd = ["python3", str(script_path), + "--input", str(final_report), + "--output", str(plots_dir)] + + # The script generates visualization plots + if run_command(cmd, "Generating visualization plots and HTML report"): + print(f"\nOutput saved to: {plots_dir}/") + print("\n Generated plots:") + print(" - Percentage Change Overview") + print(" - Absolute Time Comparison") + print(" - Performance Heatmap by Rank") + print(" - Total Execution Time by Rank") + print(" - Time Breakdown by Rank") + print(" - Percentage Breakdown by Rank") + print(" - NCCL/Collective Metrics") + html_script_path = Path(__file__).parent / "create_final_html.py" + output_html_path = output_path / "performance_analysis_report.html" + cmd = ["python3", str(html_script_path), + "--plot-files-directory", str(plots_dir), + "--output-html", str(output_html_path)] + if run_command(cmd, "Creating final HTML report"): + print(f"\nHTML Report saved to: {output_html_path}") + print(" - Open in browser to view complete report") + print(" - Print to PDF: Ctrl+P (or Cmd+P on Mac)") + else: + print(" Final report not found, skipping plot generation") # Summary print("\n" + "=" * 80) print("ANALYSIS COMPLETE!") @@ -375,10 +432,19 @@ def main(): files = list(output_path.glob("*.xlsx")) if files: - print("\nGenerated files:") + print("\nGenerated Excel files:") for f in sorted(files): print(f" - {f.name}") + if args.generate_plots: + plots_dir = output_path / "plots" + if plots_dir.exists(): + plot_files = list(plots_dir.glob("*.png")) + if plot_files: + print("\nGenerated plots:") + for f in sorted(plot_files): + print(f" - plots/{f.name}") + print("\nAnalysis pipeline completed successfully!") return 0