diff --git a/scripts/tracelens_single_config/create_final_report.py b/scripts/tracelens_single_config/create_final_report.py new file mode 100644 index 0000000..1a1f8d8 --- /dev/null +++ b/scripts/tracelens_single_config/create_final_report.py @@ -0,0 +1,321 @@ +#!/usr/bin/env python3 +""" +Create final comprehensive report with combined and comparison data. +Raw data sheets are hidden and all data is formatted as Excel tables. +""" +import pandas as pd +import argparse +from pathlib import Path +from openpyxl import load_workbook +from openpyxl.worksheet.table import Table, TableStyleInfo +from openpyxl.styles import Color +from openpyxl.formatting.rule import ColorScaleRule + + +def get_column_letter(col_num): + """Convert column number to Excel column letter.""" + result = "" + while col_num > 0: + col_num -= 1 + result = chr(65 + (col_num % 26)) + result + col_num //= 26 + return result + + +def add_excel_table(worksheet, table_name, start_row=1): + """Convert worksheet data to Excel table format.""" + # Find data range + max_row = worksheet.max_row + max_col = worksheet.max_column + + if max_row <= start_row: + return # No data + + # Ensure all column headers are strings + for col_idx in range(1, max_col + 1): + cell = worksheet.cell(row=start_row, column=col_idx) + if cell.value is not None and not isinstance(cell.value, str): + cell.value = str(cell.value) + + # Create table reference using proper column letter conversion + start_cell = f"A{start_row}" + end_col_letter = get_column_letter(max_col) + end_cell = f"{end_col_letter}{max_row}" + table_ref = f"{start_cell}:{end_cell}" + + # Create table with style + try: + tab = Table(displayName=table_name, ref=table_ref) + style = TableStyleInfo( + name="TableStyleMedium2", + showFirstColumn=False, + showLastColumn=False, + showRowStripes=True, + showColumnStripes=False, + ) + tab.tableStyleInfo = style + + # Add table to worksheet + worksheet.add_table(tab) + except Exception as e: + print(f" Warning: Could not create table {table_name}: {e}") + + +def create_final_report( + gpu_combined, gpu_comparison, coll_combined, coll_comparison, output_file +): + """Create comprehensive report with all data.""" + + print("Creating comprehensive final report...") + print(f" Output: {output_file}") + + # Track sheet info for hiding/organizing + raw_sheets = [] + comparison_sheets = [] + summary_sheets = [] + + with pd.ExcelWriter(output_file, engine="openpyxl") as writer: + + # === GPU TIMELINE SHEETS === + print("\nAdding GPU Timeline sheets...") + + # Read GPU combined (raw data) + gpu_comb_xl = pd.ExcelFile(gpu_combined) + sheet_mapping = { + "Summary": "GPU_Summary_Raw", + "All_Ranks_Combined": "GPU_AllRanks_Raw", + "Per_Rank_Time_ms": "GPU_Time_Raw", + "Per_Rank_Percent": "GPU_Pct_Raw", + } + for sheet_name in gpu_comb_xl.sheet_names: + df = pd.read_excel(gpu_combined, sheet_name=sheet_name) + new_name = sheet_mapping.get(sheet_name, f"GPU_{sheet_name}_Raw") + df.to_excel(writer, sheet_name=new_name, index=False) + raw_sheets.append(new_name) + print(f" Added {new_name} (will be hidden)") + + # Read GPU comparison + gpu_comp_xl = pd.ExcelFile(gpu_comparison) + comp_mapping = { + "Summary_Comparison": "GPU_Summary_Cmp", + "Comparison_By_Rank": "GPU_ByRank_Cmp", + } + for sheet_name in gpu_comp_xl.sheet_names: + if "Comparison" in sheet_name: + df = pd.read_excel(gpu_comparison, sheet_name=sheet_name) + new_name = comp_mapping.get(sheet_name, f"GPU_{sheet_name}") + df.to_excel(writer, sheet_name=new_name, index=False) + comparison_sheets.append(new_name) + print(f" Added {new_name}") + + # === COLLECTIVE SHEETS === + print("\nAdding Collective/NCCL sheets...") + + # Read collective combined (raw data for hidden sheets) + coll_comb_xl = pd.ExcelFile(coll_combined) + coll_mapping = { + "nccl_summary_implicit_sync": "NCCL_ImplSync_Raw", + "nccl_summary_long": "NCCL_Long_Raw", + } + for sheet_name in coll_comb_xl.sheet_names: + if "summary" in sheet_name.lower(): + df = pd.read_excel(coll_combined, sheet_name=sheet_name) + new_name = coll_mapping.get(sheet_name, f"NCCL_{sheet_name}_Raw") + df.to_excel(writer, sheet_name=new_name, index=False) + raw_sheets.append(new_name) + print(f" Added {new_name} (will be hidden)") + + # Read collective comparison + coll_comp_xl = pd.ExcelFile(coll_comparison) + coll_cmp_mapping = { + "nccl_implicit_sync_cmp": "NCCL_ImplSync_Cmp", + "nccl_long_cmp": "NCCL_Long_Cmp", + } + for sheet_name in coll_comp_xl.sheet_names: + if "_cmp" in sheet_name: + df = pd.read_excel(coll_comparison, sheet_name=sheet_name) + new_name = coll_cmp_mapping.get(sheet_name, f"NCCL_{sheet_name}") + df.to_excel(writer, sheet_name=new_name, index=False) + comparison_sheets.append(new_name) + print(f" Added {new_name}") + + # === CREATE SUMMARY DASHBOARD === + print("\nCreating Summary Dashboard...") + + # Read key metrics for dashboard + gpu_summary = pd.read_excel(gpu_comparison, sheet_name="Summary_Comparison") + + # Create dashboard data + dashboard_data = { + "Metric": [], + "Baseline": [], + "Test": [], + "Improvement (%)": [], + "Status": [], + } + + # Add GPU metrics + for _, row in gpu_summary.iterrows(): + metric_type = row["type"] + dashboard_data["Metric"].append(f"GPU_{metric_type}") + dashboard_data["Baseline"].append(round(row["baseline_time_ms"], 2)) + dashboard_data["Test"].append(round(row["test_time_ms"], 2)) + dashboard_data["Improvement (%)"].append(round(row["percent_change"], 2)) + dashboard_data["Status"].append( + "Better" + if row["percent_change"] > 0 + else "Worse" if row["percent_change"] < -1 else "Similar" + ) + + dashboard_df = pd.DataFrame(dashboard_data) + dashboard_df.to_excel(writer, sheet_name="Summary_Dashboard", index=False) + summary_sheets.append("Summary_Dashboard") + print(f" Added Summary_Dashboard") + + # Now modify the workbook to hide sheets and add tables + print("\nApplying formatting...") + wb = load_workbook(output_file) + + # Hide raw data sheets + for sheet_name in raw_sheets: + if sheet_name in wb.sheetnames: + wb[sheet_name].sheet_state = "hidden" + print(f" Hidden: {sheet_name}") + + # Convert all sheets to tables + for sheet_name in wb.sheetnames: + ws = wb[sheet_name] + + # Skip if sheet is empty + if ws.max_row <= 1: + continue + + # Create unique table name from sheet name (remove special chars) + table_name = ( + sheet_name.replace(" ", "_") + .replace("-", "_") + .replace("(", "") + .replace(")", "") + ) + # Ensure name starts with letter and is max 255 chars + if not table_name[0].isalpha(): + table_name = "Tbl_" + table_name + table_name = table_name[:255] + + add_excel_table(ws, table_name) + print(f" Converted to table: {sheet_name}") + + # Add conditional formatting for percent_change columns + if "Cmp" in sheet_name or "Comparison" in sheet_name: + # Find percent_change columns + for col_idx in range(1, ws.max_column + 1): + cell_value = ws.cell(row=1, column=col_idx).value + if cell_value and "percent_change" in str(cell_value): + col_letter = get_column_letter(col_idx) + data_range = f"{col_letter}2:{col_letter}{ws.max_row}" + + # Apply color scale: red (min/negative) -> white (0) -> green (max/positive) + try: + ws.conditional_formatting.add( + data_range, + ColorScaleRule( + start_type="min", + start_color="F8696B", # Red + mid_type="num", + mid_value=0, + mid_color="FFFFFF", # White + end_type="max", + end_color="63BE7B", # Green + ), + ) + print( + f" Applied color scale to {sheet_name} column {cell_value}" + ) + except Exception as e: + print( + f" Warning: Could not apply formatting to {cell_value}: {e}" + ) + + # Move Summary Dashboard to first position + if "Summary_Dashboard" in wb.sheetnames: + dashboard_sheet = wb["Summary_Dashboard"] + wb.move_sheet(dashboard_sheet, offset=-(len(wb.sheetnames) - 1)) + wb.active = 0 # Set dashboard as active sheet + print("\n Moved Summary_Dashboard to first position") + + # Save workbook + wb.save(output_file) + print(f"\nFinal report saved: {output_file}") + + # Report structure + print("\nReport Structure:") + print(" Visible Sheets (Analysis):") + print(f" - Summary_Dashboard") + for sheet in comparison_sheets: + print(f" - {sheet}") + print("\n Hidden Sheets (Raw Data):") + for sheet in raw_sheets: + print(f" - {sheet}") + print("\n All data formatted as Excel tables with filters") + print(" Percent change columns are color-coded (green=better, red=worse)") + print( + "\nUsers can unhide raw data sheets in Excel: Right-click any sheet tab → Unhide" + ) + + +def main(): + parser = argparse.ArgumentParser( + description="Create final comprehensive report with all data", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Example: + python create_final_report.py \\ + --gpu-combined gpu_timeline_combined.xlsx \\ + --gpu-comparison gpu_timeline_comparison.xlsx \\ + --coll-combined collective_combined.xlsx \\ + --coll-comparison collective_comparison.xlsx \\ + --output final_analysis_report.xlsx + """, + ) + + parser.add_argument( + "--gpu-combined", required=True, help="Path to GPU timeline combined file" + ) + parser.add_argument( + "--gpu-comparison", required=True, help="Path to GPU timeline comparison file" + ) + parser.add_argument( + "--coll-combined", required=True, help="Path to collective combined file" + ) + parser.add_argument( + "--coll-comparison", required=True, help="Path to collective comparison file" + ) + parser.add_argument("--output", required=True, help="Output path for final report") + + args = parser.parse_args() + + # Validate inputs + for file_arg in [ + "gpu_combined", + "gpu_comparison", + "coll_combined", + "coll_comparison", + ]: + file_path = getattr(args, file_arg) + if not Path(file_path).exists(): + print(f"Error: File not found: {file_path}") + return 1 + + create_final_report( + args.gpu_combined, + args.gpu_comparison, + args.coll_combined, + args.coll_comparison, + args.output, + ) + + return 0 + + +if __name__ == "__main__": + exit(main()) diff --git a/scripts/tracelens_single_config/run_full_analysis.py b/scripts/tracelens_single_config/run_full_analysis.py new file mode 100644 index 0000000..5385ec4 --- /dev/null +++ b/scripts/tracelens_single_config/run_full_analysis.py @@ -0,0 +1,387 @@ +#!/usr/bin/env python3 +""" +Master script for complete TraceLens analysis pipeline. +Runs analysis on baseline and test traces, then performs all comparisons. +""" +import argparse +import subprocess +import os +import sys +from pathlib import Path + + +def run_command(cmd, description): + """Execute a command and handle errors.""" + print(f"\n{'='*80}") + print(f"{description}") + print(f"{'='*80}") + print(f"Command: {' '.join(cmd)}") + + result = subprocess.run(cmd, capture_output=True, text=True) + + if result.returncode != 0: + print(f"Error: {description} failed!") + print(f"Stderr: {result.stderr}") + return False + + print(result.stdout) + return True + + +def run_tracelens_analysis( + trace_dir, output_name, individual_only=False, collective_only=False +): + """Run TraceLens analysis on a single trace directory.""" + print(f"\nAnalyzing: {trace_dir}") + + # Build command + script_path = Path(__file__).parent / "run_tracelens_single_config.sh" + cmd = ["bash", str(script_path), trace_dir] + + if individual_only: + cmd.append("--individual-only") + elif collective_only: + cmd.append("--collective-only") + + return run_command(cmd, f"TraceLens analysis for {output_name}") + + +def process_gpu_timeline(reports_dir): + """Process GPU timeline from individual reports.""" + script_path = Path(__file__).parent / "process_gpu_timeline.py" + cmd = ["python3", str(script_path), "--reports-dir", reports_dir] + + return run_command(cmd, "Processing GPU timeline") + + +def combine_reports(baseline_file, test_file, output_file): + """Combine baseline and test reports.""" + script_path = Path(__file__).parent / "combine_reports.py" + cmd = [ + "python3", + str(script_path), + "--baseline", + baseline_file, + "--test", + test_file, + "--output", + output_file, + ] + + return run_command(cmd, f"Combining reports to {output_file}") + + +def add_comparison_sheets(input_file, output_file): + """Add comparison sheets for GPU timeline.""" + script_path = Path(__file__).parent / "add_comparison_sheets.py" + cmd = ["python3", str(script_path), "--input", input_file, "--output", output_file] + + return run_command(cmd, "Adding GPU timeline comparison sheets") + + +def add_collective_comparison(input_file, output_file): + """Add comparison sheets for collective operations.""" + script_path = Path(__file__).parent / "add_collective_comparison.py" + cmd = ["python3", str(script_path), "--input", input_file, "--output", output_file] + + return run_command(cmd, "Adding collective comparison sheets") + + +def create_final_report( + gpu_combined, gpu_comparison, coll_combined, coll_comparison, output_file +): + """Create comprehensive final report with all data.""" + script_path = Path(__file__).parent / "create_final_report.py" + cmd = [ + "python3", + str(script_path), + "--gpu-combined", + gpu_combined, + "--gpu-comparison", + gpu_comparison, + "--coll-combined", + coll_combined, + "--coll-comparison", + coll_comparison, + "--output", + output_file, + ] + + if run_command(cmd, "Creating comprehensive final report"): + plot_script_path = Path(__file__).parent / "create_final_plots.py" + cmd = ["python3", str(plot_script_path), "--report-path", output_file] + if run_command(cmd, "Creating final plots"): + html_script_path = Path(__file__).parent / "create_final_html.py" + cmd = [ + "python3", + str(html_script_path), + "--plot-files-directory", + str(Path(output_file).parent / "plots"), + ] + if run_command(cmd, "Creating final HTML"): + return True + return False + + +def main(): + parser = argparse.ArgumentParser( + description="Complete TraceLens analysis pipeline with comparisons", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Full analysis with everything including final report + python run_full_analysis.py \\ + --baseline /path/to/baseline/traces \\ + --test /path/to/test/traces \\ + --output /path/to/output \\ + --all + + # Only GPU timeline comparison + python run_full_analysis.py \\ + --baseline /path/to/baseline \\ + --test /path/to/test \\ + --output /path/to/output \\ + --gpu-timeline + + # Create final report (skip TraceLens if already done) + python run_full_analysis.py \\ + --baseline /path/to/baseline \\ + --test /path/to/test \\ + --output /path/to/output \\ + --gpu-timeline --collective --final-report \\ + --skip-tracelens + """, + ) + + # Required arguments + parser.add_argument( + "--baseline", required=True, help="Path to baseline trace directory" + ) + parser.add_argument("--test", required=True, help="Path to test trace directory") + parser.add_argument( + "--output", required=True, help="Output directory for comparison results" + ) + + # Analysis options + parser.add_argument( + "--skip-tracelens", + action="store_true", + help="Skip TraceLens report generation (if already done)", + ) + parser.add_argument( + "--individual-only", + action="store_true", + help="Generate only individual reports", + ) + parser.add_argument( + "--collective-only", + action="store_true", + help="Generate only collective reports", + ) + + # Comparison options + parser.add_argument( + "--gpu-timeline", action="store_true", help="Perform GPU timeline comparison" + ) + parser.add_argument( + "--collective", action="store_true", help="Perform collective/NCCL comparison" + ) + parser.add_argument( + "--final-report", + action="store_true", + help="Create comprehensive final report with tables and hidden raw data", + ) + parser.add_argument( + "--all", + action="store_true", + help="Perform all analyses and comparisons including final report", + ) + + args = parser.parse_args() + + # Handle --all flag + if args.all: + args.gpu_timeline = True + args.collective = True + args.final_report = True + + # Validate inputs + baseline_path = Path(args.baseline) + test_path = Path(args.test) + output_path = Path(args.output) + + if not baseline_path.exists(): + print(f"Error: Baseline path not found: {args.baseline}") + return 1 + + if not test_path.exists(): + print(f"Error: Test path not found: {args.test}") + return 1 + + # Create output directory + output_path.mkdir(parents=True, exist_ok=True) + + print("\n" + "=" * 80) + print("TRACELENS FULL ANALYSIS PIPELINE") + print("=" * 80) + print(f"Baseline: {args.baseline}") + print(f"Test: {args.test}") + print(f"Output: {args.output}") + print(f"Options:") + print(f" Skip TraceLens: {args.skip_tracelens}") + print(f" GPU timeline: {args.gpu_timeline}") + print(f" Collective: {args.collective}") + print(f" Final report: {args.final_report}") + + # Step 1: Run TraceLens analysis on both directories + if not args.skip_tracelens: + print("\n" + "=" * 80) + print("STEP 1: Running TraceLens Analysis") + print("=" * 80) + + if not run_tracelens_analysis( + args.baseline, "baseline", args.individual_only, args.collective_only + ): + return 1 + + if not run_tracelens_analysis( + args.test, "test", args.individual_only, args.collective_only + ): + return 1 + else: + print("\nSkipping TraceLens report generation (--skip-tracelens flag)") + + # Determine analysis directories + baseline_analysis = baseline_path / "tracelens_analysis" + test_analysis = test_path / "tracelens_analysis" + + if not baseline_analysis.exists(): + print(f"Error: Baseline analysis not found: {baseline_analysis}") + print("Run without --skip-tracelens flag first") + return 1 + + if not test_analysis.exists(): + print(f"Error: Test analysis not found: {test_analysis}") + print("Run without --skip-tracelens flag first") + return 1 + + # Step 2: GPU Timeline Comparison + if args.gpu_timeline: + print("\n" + "=" * 80) + print("STEP 2: GPU Timeline Comparison") + print("=" * 80) + + # Process GPU timelines + baseline_reports = baseline_analysis / "individual_reports" + test_reports = test_analysis / "individual_reports" + + if not baseline_reports.exists() or not test_reports.exists(): + print( + "Error: Individual reports not found. Run without --individual-only flag" + ) + return 1 + + print("\nProcessing baseline GPU timeline...") + if not process_gpu_timeline(str(baseline_reports)): + return 1 + + print("\nProcessing test GPU timeline...") + if not process_gpu_timeline(str(test_reports)): + return 1 + + # Combine GPU timeline summaries + baseline_gpu = baseline_analysis / "gpu_timeline_summary_mean.xlsx" + test_gpu = test_analysis / "gpu_timeline_summary_mean.xlsx" + combined_gpu = output_path / "gpu_timeline_combined.xlsx" + + if not combine_reports(str(baseline_gpu), str(test_gpu), str(combined_gpu)): + return 1 + + # Add comparison sheets + gpu_comparison = output_path / "gpu_timeline_comparison.xlsx" + if not add_comparison_sheets(str(combined_gpu), str(gpu_comparison)): + return 1 + + print(f"\nGPU timeline comparison saved to: {gpu_comparison}") + + # Step 3: Collective Comparison + if args.collective: + print("\n" + "=" * 80) + print("STEP 3: Collective/NCCL Comparison") + print("=" * 80) + + baseline_collective = ( + baseline_analysis / "collective_reports" / "collective_all_ranks.xlsx" + ) + test_collective = ( + test_analysis / "collective_reports" / "collective_all_ranks.xlsx" + ) + + if not baseline_collective.exists() or not test_collective.exists(): + print( + "Error: Collective reports not found. Run without --collective-only flag" + ) + return 1 + + # Combine collective reports + combined_collective = output_path / "collective_combined.xlsx" + if not combine_reports( + str(baseline_collective), str(test_collective), str(combined_collective) + ): + return 1 + + # Add collective comparison + collective_comparison = output_path / "collective_comparison.xlsx" + if not add_collective_comparison( + str(combined_collective), str(collective_comparison) + ): + return 1 + + print(f"\nCollective comparison saved to: {collective_comparison}") + + # Step 4: Create final comprehensive report + if args.final_report and args.gpu_timeline and args.collective: + print("\n" + "=" * 80) + print("STEP 4: Creating Final Comprehensive Report") + print("=" * 80) + + gpu_combined = output_path / "gpu_timeline_combined.xlsx" + gpu_comparison = output_path / "gpu_timeline_comparison.xlsx" + collective_combined = output_path / "collective_combined.xlsx" + collective_comparison = output_path / "collective_comparison.xlsx" + final_report = output_path / "final_analysis_report.xlsx" + + if not create_final_report( + str(gpu_combined), + str(gpu_comparison), + str(collective_combined), + str(collective_comparison), + str(final_report), + ): + return 1 + + print(f"\nFinal comprehensive report saved to: {final_report}") + print(" - Summary Dashboard as first sheet") + print(" - All comparison sheets visible") + print(" - Raw data sheets hidden (can be unhidden in Excel)") + print(" - All data formatted as Excel tables with filters") + print(" - Color coding applied (green=better, red=worse)") + + # Summary + print("\n" + "=" * 80) + print("ANALYSIS COMPLETE!") + print("=" * 80) + print(f"\nResults saved to: {output_path}") + + files = list(output_path.glob("*.xlsx")) + if files: + print("\nGenerated files:") + for f in sorted(files): + print(f" - {f.name}") + + print("\nAnalysis pipeline completed successfully!") + return 0 + + +if __name__ == "__main__": + sys.exit(main())