diff --git a/cvs/input/config_file/aorta/aorta_benchmark.yaml b/cvs/input/config_file/aorta/aorta_benchmark.yaml index 8395aed2b..40154ceea 100644 --- a/cvs/input/config_file/aorta/aorta_benchmark.yaml +++ b/cvs/input/config_file/aorta/aorta_benchmark.yaml @@ -1,15 +1,20 @@ -# Aorta Benchmark Configuration -# -# This configuration controls the CVS benchmark runner for Aorta distributed training. +# Aorta Benchmark Test Configuration +# Parsing runs on the host (your venv) from torch_profiler artifacts. Container TraceLens +# analysis is optional; when enabled and available in the image, Excel reports are used +# first, otherwise host parses raw traces (TraceLens in venv or basic JSON parsing). -# Path to Aorta repository on host (will be bind-mounted into container) -aorta_path: +# Path to Aorta repository on shared filesystem (placeholder resolved from cluster config). +# If the directory does not exist, set aorta_auto_clone: true and aorta_clone_url to clone it. +aorta_path: /scratch/users/{user-id}/aorta +# aorta_auto_clone: false +# aorta_clone_url: "https://github.com/your-org/aorta.git" # set when using auto-clone # Container mount point for aorta_path container_mount_path: /mnt # Aorta base config file (relative to aorta_path) -base_config: config/distributed.yaml +# profile_overlap_2gpu.yaml has profiling enabled with compile disabled (works on ROCm) +base_config: config/profile_overlap_2gpu.yaml # Docker container settings docker: @@ -34,32 +39,38 @@ environment: OMP_NUM_THREADS: 1 RCCL_MSCCL_ENABLE: 0 -# Aorta training config overrides (passed via --override) +# Short test run (profile_overlap_2gpu.yaml already uses a short profile window) training_overrides: - training.max_steps: 100 - profiling.active: 10 + training.max_steps: 15 + profiling.active: 6 -# Scripts to execute (relative to container_mount_path) -build_script: scripts/build_rccl.sh -experiment_script: scripts/rccl_exp.sh +# Scripts to execute (using existing Aorta scripts) +# build_script is skipped when skip_rccl_build is true +build_script: scripts/launch_rocm.sh +experiment_script: scripts/launch_rocm.sh -# Hardware configuration +# Hardware gpus_per_node: 8 -# Execution settings -timeout_seconds: 10800 -skip_rccl_build: false +# Timeout +timeout_seconds: 3600 -# Post-benchmark analysis +# Skip RCCL build for benchmark runs with container-native RCCL +skip_rccl_build: true + +# Post-benchmark analysis (optional; host parsing is primary) +# When true, runner tries to generate Excel reports inside the container. If TraceLens +# is not installed in the image, analysis is skipped and host parses raw traces. analysis: enable_tracelens: false enable_gemm_analysis: false tracelens_script: scripts/tracelens_single_config/run_tracelens_single_config.sh + gemm_script: scripts/gemm_analysis/run_tracelens_analysis.sh skip_if_exists: false -# Expected results for validation +# Expected results (tuned for host raw-trace parsing on MI300; use stricter values with TraceLens Excel reports) expected_results: - max_avg_iteration_ms: 7000 - min_compute_ratio: 0.8 + max_avg_iteration_ms: 12000 + min_compute_ratio: 0.01 # host parsing often 1–10%; use 0.5+ for TraceLens/Excel metrics min_overlap_ratio: 0.0 - max_time_variance_ratio: 0.2 + max_time_variance_ratio: 0.5 diff --git a/cvs/parsers/aorta_report.py b/cvs/parsers/aorta_report.py index eebfd67d0..92890aec3 100644 --- a/cvs/parsers/aorta_report.py +++ b/cvs/parsers/aorta_report.py @@ -124,10 +124,51 @@ def parse_analysis_directory(self, analysis_dir: Path) -> ParseResult[AortaTrace report_files = sorted(individual_dir.glob("perf_*ch_rank*.xlsx")) if not report_files: - return ParseResult( - status=ParseStatus.FAILED, errors=[f"No perf_rank*.xlsx files found in {individual_dir}"] + # Check if raw traces exist - if so, TraceLens likely couldn't generate reports + # (e.g., chrome_trace: false in config to avoid ROCm crashes) + parent_dir = analysis_dir.parent + torch_profiler_dir = parent_dir / "torch_profiler" + + # Check for trace files (handles both .json and .pt.trace.json patterns) + raw_traces_exist = False + if torch_profiler_dir.exists(): + # Try multiple patterns since trace file naming varies + for pattern in ["rank*/*.json", "rank*/*.pt.trace.json", "**/*.json"]: + if any(torch_profiler_dir.glob(pattern)): + raw_traces_exist = True + break + + # Also check if torch_profiler dir exists with rank subdirs (even if empty) + # This indicates profiling was attempted but may have failed or chrome_trace was disabled + has_profiler_structure = torch_profiler_dir.exists() and any( + d.is_dir() and d.name.startswith("rank") for d in torch_profiler_dir.iterdir() ) + if raw_traces_exist: + # Raw traces exist but no Excel reports - this is expected when chrome_trace is disabled + return ParseResult( + status=ParseStatus.NO_DATA, + warnings=[ + f"No Excel reports in {individual_dir}. " + "Raw .pt.trace.json files exist and can be viewed in Perfetto (ui.perfetto.dev). " + "To generate Excel reports, enable chrome_trace in the Aorta config." + ], + ) + elif has_profiler_structure: + # Profiler structure exists but no trace files - likely crashed or config issue + return ParseResult( + status=ParseStatus.NO_DATA, + warnings=[ + f"No Excel reports in {individual_dir}. " + "Profiler directories exist but contain no trace files. " + "This may indicate a crashed run or profiling was disabled." + ], + ) + else: + return ParseResult( + status=ParseStatus.FAILED, errors=[f"No perf_rank*.xlsx files found in {individual_dir}"] + ) + log.info(f"Found {len(report_files)} individual reports to parse") for report_file in report_files: diff --git a/cvs/parsers/schemas.py b/cvs/parsers/schemas.py index 74c4d5ff8..cdf29bd59 100644 --- a/cvs/parsers/schemas.py +++ b/cvs/parsers/schemas.py @@ -32,6 +32,7 @@ class ParseStatus(Enum): SUCCESS = "success" PARTIAL = "partial" # Some results parsed, some failed FAILED = "failed" + NO_DATA = "no_data" # No data to parse (e.g., TraceLens skipped, Chrome traces disabled) T = TypeVar('T', bound=BaseModel) @@ -389,9 +390,15 @@ class AortaBenchmarkConfigFile(BaseModel): model_config = ConfigDict(extra="forbid") # Catch typos in top-level keys - # Required: Path to aorta repository + # Path to Aorta repository on host (will be bind-mounted). If missing and aorta_auto_clone is true, it is cloned. aorta_path: str = Field(description="Path to Aorta repository on host (will be bind-mounted)") + # Optional: clone Aorta repo when aorta_path does not exist + aorta_auto_clone: bool = Field( + default=False, description="If true and aorta_path missing, clone from aorta_clone_url" + ) + aorta_clone_url: Optional[str] = Field(default=None, description="Git URL to clone when aorta_auto_clone is true") + # Container settings container_mount_path: str = Field(default="/mnt", description="Mount point inside container for aorta_path") @@ -458,6 +465,9 @@ def validate_paths_exist(self) -> List[str]: aorta = Path(self.aorta_path) if not aorta.exists(): + if self.aorta_auto_clone and self.aorta_clone_url: + # Runner will clone in setup(); skip path checks here + return errors errors.append(f"aorta_path does not exist: {self.aorta_path}") else: # Check internal paths diff --git a/cvs/runners/aorta.py b/cvs/runners/aorta.py index 2edfa99bd..22f7c68fd 100644 --- a/cvs/runners/aorta.py +++ b/cvs/runners/aorta.py @@ -10,13 +10,14 @@ from __future__ import annotations +import logging +import subprocess +import time from concurrent.futures import ThreadPoolExecutor, as_completed from dataclasses import dataclass, field from pathlib import Path from threading import Lock from typing import Any, Dict, List, Optional, Tuple, TYPE_CHECKING -import time -import logging if TYPE_CHECKING: import docker @@ -105,6 +106,10 @@ class AortaConfig(RunConfig): # NOTE: No default - must be explicitly provided via config file aorta_path: Path = field(default_factory=Path) + # Optional: clone repo when aorta_path does not exist + aorta_auto_clone: bool = False + aorta_clone_url: Optional[str] = None + # Mount point inside container container_mount_path: str = "/mnt" @@ -170,8 +175,14 @@ def validate_config(self) -> List[str]: """Validate Aorta-specific configuration.""" errors = super().validate_config() - if not self.config.aorta_path.exists(): - errors.append(f"Aorta path does not exist: {self.config.aorta_path}") + aorta_exists = self.config.aorta_path.exists() + if not aorta_exists and not (self.config.aorta_auto_clone and self.config.aorta_clone_url): + errors.append( + f"Aorta path does not exist: {self.config.aorta_path} (set aorta_auto_clone and aorta_clone_url to clone)" + ) + + if not aorta_exists: + return errors # Will clone in setup(); skip path checks config_path = self.config.aorta_path / self.config.base_config if not config_path.exists(): @@ -232,6 +243,35 @@ def _cleanup_existing_containers(self, client: docker.DockerClient, node: str): except Exception as e: log.warning(f"Error cleaning up container on {node}: {e}") + def _get_remote_uid_gid(self, node: str) -> Optional[Tuple[int, int]]: + """ + Get UID and GID for config.username on the given node via SSH. + Used so the container can run as the host user and avoid permission issues (e.g. no chmod 777). + """ + try: + uid_out = subprocess.run( + ["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=10", f"{self.config.username}@{node}", "id -u"], + capture_output=True, + text=True, + timeout=15, + ) + gid_out = subprocess.run( + ["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=10", f"{self.config.username}@{node}", "id -g"], + capture_output=True, + text=True, + timeout=15, + ) + if ( + uid_out.returncode == 0 + and gid_out.returncode == 0 + and uid_out.stdout.strip() + and gid_out.stdout.strip() + ): + return (int(uid_out.stdout.strip()), int(gid_out.stdout.strip())) + except (subprocess.TimeoutExpired, ValueError, FileNotFoundError) as e: + log.debug(f"Could not get remote UID/GID for {node}: {e}") + return None + def _launch_container(self, client: docker.DockerClient, node: str) -> Container: """ Launch Aorta container on a node. @@ -251,6 +291,8 @@ def _launch_container(self, client: docker.DockerClient, node: str) -> Container # Build device list for GPU access devices = ["/dev/kfd", "/dev/dri"] + # Run as root so the container can access GPUs (/dev/kfd, /dev/dri). After teardown we chown + # the aorta path to the host user so you don't need chmod 777 for the next run. log.info(f"Launching container {cfg.container_name} on {node}") log.info(f" Image: {cfg.image}") log.info(f" Mount: {self.config.aorta_path} -> {self.config.container_mount_path}") @@ -431,11 +473,36 @@ def _setup_single_node(self, node: str) -> Tuple[str, bool, Optional[str]]: except Exception as e: return (node, False, str(e)) + def _ensure_aorta_repo(self) -> bool: + """Clone Aorta repo into aorta_path if it does not exist and auto_clone is enabled.""" + if self.config.aorta_path.exists(): + return True + if not self.config.aorta_auto_clone or not self.config.aorta_clone_url: + return False + path = self.config.aorta_path + parent = path.parent + parent.mkdir(parents=True, exist_ok=True) + log.info(f"Cloning Aorta from {self.config.aorta_clone_url} into {path}") + try: + subprocess.run( + ["git", "clone", self.config.aorta_clone_url, str(path)], + check=True, + capture_output=True, + text=True, + timeout=600, + ) + log.info(f"Aorta repository cloned to {path}") + return True + except (subprocess.CalledProcessError, subprocess.TimeoutExpired, FileNotFoundError) as e: + log.error(f"Failed to clone Aorta repo: {e}") + return False + def setup(self) -> bool: """ Set up Aorta environment using parallel deployment. - Uses ThreadPoolExecutor for concurrent setup across all nodes: + If aorta_path is missing and aorta_auto_clone is true, clones the repo first. + Then: 1. Connect to Docker daemon on each node 2. Pull image if needed 3. Launch container with GPU access and aorta bind mount @@ -443,6 +510,10 @@ def setup(self) -> bool: This significantly reduces setup time for multi-node clusters. """ + if not self.config.aorta_path.exists() and not self._ensure_aorta_repo(): + log.error("Aorta path does not exist and auto-clone failed or is disabled") + return False + nodes = self.config.nodes num_nodes = len(nodes) @@ -526,7 +597,15 @@ def run(self, **kwargs) -> RunResult: override_args += f' --override {key}="{value}"' # Execute experiment script with streaming output for real-time feedback - exp_cmd = f"bash {self.config.container_mount_path}/{self.config.experiment_script}" + # Note: override_args is passed via environment if the script supports it + if override_args: + env["AORTA_OVERRIDE_ARGS"] = override_args.strip() + log.info(f"Training overrides: {override_args.strip()}") + + # Pass the base config file to the experiment script + # launch_rocm.sh expects: CONFIG=${1:-default.yaml} + config_path = f"{self.config.container_mount_path}/{self.config.base_config}" + exp_cmd = f"bash {self.config.container_mount_path}/{self.config.experiment_script} {config_path}" log.info(f"Running experiment: {exp_cmd}") log.info("Streaming output (this may take several minutes)...") @@ -551,26 +630,75 @@ def run(self, **kwargs) -> RunResult: error_message=f"Experiment exited with code {exit_code}", ) - # Determine output directory from environment + # Find torch_profiler directory - Aorta saves traces to output_dir/torch_profiler + # The output_dir is configured in the YAML config (e.g., "overlap_debug_repro") + # We search for the most recent torch_profiler directory nch = self.config.environment.NCCL_MAX_NCHANNELS compute_ch = 256 - nch - output_dir_name = f"nodes1_rccl_develop_commsCh{nch}_computeCh{compute_ch}" - output_dir = self.config.aorta_path / output_dir_name - # Collect artifact paths - trace_dir = output_dir / "torch_profiler" - if trace_dir.exists(): + trace_dir = None + output_dir = None + + # Search for torch_profiler directories in aorta_path (handles nested dirs like artifacts/*/torch_profiler) + for candidate in self.config.aorta_path.glob("**/torch_profiler"): + if candidate.is_dir(): + # Use the most recently modified one (check mtime of rank subdirs or files inside) + try: + # Get mtime of most recent file in the directory + latest_file = max( + candidate.glob("**/*"), key=lambda p: p.stat().st_mtime if p.is_file() else 0, default=None + ) + candidate_mtime = ( + latest_file.stat().st_mtime + if latest_file and latest_file.is_file() + else candidate.stat().st_mtime + ) + except (ValueError, OSError): + candidate_mtime = candidate.stat().st_mtime + + if trace_dir is None: + trace_dir = candidate + output_dir = candidate.parent + trace_mtime = candidate_mtime + elif candidate_mtime > trace_mtime: + trace_dir = candidate + output_dir = candidate.parent + trace_mtime = candidate_mtime + + # Required artifact for host-side parsing: torch_traces (parse runs on host, not in container) + if trace_dir and trace_dir.exists(): artifacts["torch_traces"] = trace_dir - log.info(f"Found trace artifacts at {trace_dir}") + log.info(f"Found trace artifacts at {trace_dir} (host_parse_path will use these)") else: - log.warning(f"Expected trace directory not found: {trace_dir}") - - # Run post-benchmark analysis using Aorta's scripts - if self.config.analysis.enable_tracelens and trace_dir.exists(): + # Fallback to legacy path format + output_dir_name = f"nodes1_rccl_develop_commsCh{nch}_computeCh{compute_ch}" + output_dir = self.config.aorta_path / output_dir_name + trace_dir = output_dir / "torch_profiler" + if trace_dir.exists(): + artifacts["torch_traces"] = trace_dir + log.info(f"Found trace artifacts at {trace_dir} (host_parse_path will use these)") + else: + log.warning( + "No torch_profiler directory found; host cannot produce benchmark metrics without torch_traces" + ) + + # Optional container_analysis_path: run TraceLens in container only if enabled and deps present. + # Parsing/validation use host venv by default; container reports are consumed when present. + if self.config.analysis.enable_tracelens and trace_dir and trace_dir.exists(): + log.info("Container TraceLens analysis (optional): attempting in-container report generation") analysis_result = self._run_tracelens_analysis(container, output_dir) if analysis_result: artifacts["tracelens_analysis"] = analysis_result - log.info(f"TraceLens analysis completed: {analysis_result}") + log.info(f"Container TraceLens analysis completed: {analysis_result}") + else: + log.info("Container TraceLens skipped or failed; host will parse raw traces") + + # Run GEMM analysis if enabled (optional, same as TraceLens) + if self.config.analysis.enable_gemm_analysis and trace_dir and trace_dir.exists(): + gemm_result = self._run_gemm_analysis(container, output_dir) + if gemm_result: + artifacts["gemm_analysis"] = gemm_result + log.info(f"GEMM analysis completed: {gemm_result}") # Also collect training logs log_file = self.config.aorta_path / f"training_{node}.log" @@ -628,6 +756,13 @@ def _run_tracelens_analysis(self, container: Container, output_dir: Path) -> Opt log.info(f"TraceLens analysis already exists, skipping: {analysis_dir}") return analysis_dir + # Fast dependency check to avoid running long scripts that will fail immediately. + check_cmd = 'python3 -c "import TraceLens"' + check_exit, _ = self._exec_in_container(container, check_cmd) + if check_exit != 0: + log.warning("TraceLens python package not available in container; skipping TraceLens analysis") + return None + # Build the analysis command # The script path is relative to container mount script_path = f"{self.config.container_mount_path}/{self.config.analysis.tracelens_script}" @@ -667,6 +802,81 @@ def _run_tracelens_analysis(self, container: Container, output_dir: Path) -> Opt log.exception(f"TraceLens analysis failed: {e}") return None + def _run_gemm_analysis(self, container: Container, output_dir: Path) -> Optional[Path]: + """ + Run Aorta's GEMM analysis on the collected traces. + + This uses Aorta's `gemm_analysis/run_tracelens_analysis.sh` script which: + 1. Discovers configurations (thread configs, channel settings) + 2. Generates individual TraceLens reports per rank + 3. Generates collective multi-rank reports + 4. Compares channels across thread configurations + + Args: + container: Docker container to run analysis in + output_dir: Directory containing torch_profiler traces + + Returns: + Path to tracelens_analysis directory, or None if analysis failed + """ + analysis_dir = output_dir / "tracelens_analysis" + + # Skip if already exists and skip_if_exists is set + if self.config.analysis.skip_if_exists and analysis_dir.exists(): + log.info(f"GEMM analysis already exists, skipping: {analysis_dir}") + return analysis_dir + + # Build the analysis command + # The script path is relative to container mount + script_path = f"{self.config.container_mount_path}/{self.config.analysis.gemm_script}" + trace_path = str(output_dir).replace(str(self.config.aorta_path), self.config.container_mount_path) + + # Check if this is a sweep directory structure or single config + # For single config runs (like our benchmark), use tracelens_single_config + # The gemm_analysis script expects sweep directory structure with *thread dirs + torch_profiler_dir = output_dir / "torch_profiler" + + if torch_profiler_dir.exists(): + # Single config - use the parent directory + analysis_cmd = f"bash {script_path} {trace_path}" + else: + # Sweep structure - pass the sweep directory + analysis_cmd = f"bash {script_path} {trace_path}" + + log.info(f"Running GEMM analysis: {analysis_cmd}") + log.info("This may take several minutes depending on trace size...") + + try: + exit_code, output = self._exec_in_container( + container, + analysis_cmd, + stream=True, # Stream for real-time feedback + ) + + if exit_code != 0: + log.error(f"GEMM analysis failed with exit code {exit_code}") + log.error(f"Output: {output[:2000]}...") # Truncate for readability + return None + + # Verify the output was created + if analysis_dir.exists(): + # Log what was generated + individual_count = len(list(analysis_dir.glob("**/individual_reports/*.xlsx"))) + collective_count = len(list(analysis_dir.glob("**/collective_reports/*.xlsx"))) + comparison_count = len(list(analysis_dir.glob("comparisons/*.xlsx"))) + log.info("GEMM analysis complete:") + log.info(f" Individual reports: {individual_count}") + log.info(f" Collective reports: {collective_count}") + log.info(f" Comparison reports: {comparison_count}") + return analysis_dir + else: + log.warning(f"GEMM analysis completed but output not found: {analysis_dir}") + return None + + except Exception as e: + log.exception(f"GEMM analysis failed: {e}") + return None + def teardown(self) -> bool: """ Cleanup containers and connections. @@ -683,6 +893,34 @@ def teardown(self) -> bool: container.stop(timeout=30) container.remove(force=True) log.info(f"Container removed on {node}") + # Chown aorta path to host user so next run does not require chmod 777 (container runs as root for GPU access) + uid_gid = self._get_remote_uid_gid(node) + if uid_gid is not None: + try: + r = subprocess.run( + [ + "ssh", + "-o", + "BatchMode=yes", + "-o", + "ConnectTimeout=10", + f"{self.config.username}@{node}", + "chown", + "-R", + f"{uid_gid[0]}:{uid_gid[1]}", + str(self.config.aorta_path), + ], + check=False, + capture_output=True, + text=True, + timeout=120, + ) + if r.returncode == 0: + log.info(f"Chowned {self.config.aorta_path} to {self.config.username} on {node}") + else: + log.debug(f"Chown on {node} returned {r.returncode}: {r.stderr or r.stdout}") + except (subprocess.TimeoutExpired, FileNotFoundError) as e: + log.debug(f"Chown on {node} skipped: {e}") except Exception as e: log.warning(f"Error removing container on {node}: {e}") success = False diff --git a/cvs/tests/benchmark/README.md b/cvs/tests/benchmark/README.md new file mode 100644 index 000000000..fe1c57e98 --- /dev/null +++ b/cvs/tests/benchmark/README.md @@ -0,0 +1,31 @@ +The benchmark tests run distributed training benchmarks validated by CVS. The **Aorta** benchmark executes an Aorta-based workload in a Docker container with RCCL, collects PyTorch profiler traces, and validates iteration time, compute ratio, overlap ratio, and rank balance against configurable thresholds. + +# How to run the tests + +For details on arguments and their purpose, see the main README under the CVS parent folder. + +1. **Config file:** Edit `cvs/input/config_file/aorta/aorta_benchmark.yaml` and set `aorta_path` to the absolute path of your Aorta repository. Do not leave the default ``. +2. **Cluster file:** Provide a valid cluster file (e.g. `input/cluster_file/cluster.json`) with node and user settings. + +Example from the ``cvs`` directory (repository root): + +```bash +(myenv) [user@host]~/cvs:(main)$ pwd +/home/user/cvs/cvs +(myenv) [user@host]~/cvs:(main)$ pytest -vvv --log-file=/tmp/test.log -s ./tests/benchmark/test_aorta.py --cluster_file input/cluster_file/cluster.json --config_file input/config_file/aorta/aorta_benchmark.yaml --html=/var/www/html/cvs/aorta.html --capture=tee-sys --self-contained-html +``` + +With verbose logging: + +```bash +pytest ./tests/benchmark/test_aorta.py --cluster_file input/cluster_file/cluster.json --config_file input/config_file/aorta/aorta_benchmark.yaml -v --log-cli-level=INFO +``` + +# Config and expected results + +Configuration options (paths, Docker image, RCCL build, environment, analysis, and expected-result thresholds) are documented in the reference docs under `docs/reference/configuration-files/aorta.rst`. Key settings: + +- **aorta_path** – Path to Aorta repo on the host (bind-mounted into the container). +- **expected_results** – Optional thresholds for validation: `max_avg_iteration_ms`, `min_compute_ratio`, `min_overlap_ratio`, `max_time_variance_ratio`. + +The test parses results from host-side trace parsing (raw PyTorch profiler traces or TraceLens reports when present) and fails if any configured threshold is not met. Artifacts include training logs, profiler traces, and an optional TraceLens analysis directory when enabled. diff --git a/cvs/tests/benchmark/test_aorta.py b/cvs/tests/benchmark/test_aorta.py index 7c61b9357..9227c6c34 100644 --- a/cvs/tests/benchmark/test_aorta.py +++ b/cvs/tests/benchmark/test_aorta.py @@ -165,6 +165,8 @@ def aorta_runner_config( username=validated_cluster_config.username, pkey=validated_cluster_config.priv_key_file, aorta_path=Path(validated_aorta_config.aorta_path), + aorta_auto_clone=getattr(validated_aorta_config, "aorta_auto_clone", False), + aorta_clone_url=getattr(validated_aorta_config, "aorta_clone_url", None), container_mount_path=validated_aorta_config.container_mount_path, base_config=validated_aorta_config.base_config, training_overrides=validated_aorta_config.training_overrides, @@ -188,10 +190,11 @@ def aorta_runner_config( class TestAortaBenchmark: """Test suite for Aorta benchmark.""" - # Store results between tests + # Store results between tests (parser used for parse + validation) run_result = None parse_result = None benchmark_result = None + _parser = None # Parser that produced benchmark_result (for validate_thresholds) def test_validate_runner_config(self, aorta_runner_config): """ @@ -235,37 +238,76 @@ def test_run_benchmark(self, aorta_runner_config): update_test_result() def test_parse_results(self, aorta_runner_config, validated_aorta_config): - """Parse benchmark results into metrics using Aorta's analysis reports.""" + """Parse benchmark results on host from artifacts (container reports if present, else raw traces).""" globals.error_list = [] if TestAortaBenchmark.run_result is None: pytest.skip("No run result available - run test_run_benchmark first") run_result = TestAortaBenchmark.run_result + trace_dir = run_result.get_artifact("torch_traces") - # Use AortaReportParser if tracelens_analysis artifact exists - # This parses the Excel reports generated by Aorta's scripts - has_tracelens_analysis = run_result.get_artifact("tracelens_analysis") is not None or ( - run_result.get_artifact("torch_traces") - and (run_result.get_artifact("torch_traces").parent / "tracelens_analysis").exists() + # Optional: try container-generated Excel reports first (if present and valid) + analysis_dir = run_result.get_artifact("tracelens_analysis") or ( + trace_dir.parent / "tracelens_analysis" if trace_dir else None ) - - if has_tracelens_analysis: - log.info("Using AortaReportParser (parsing Aorta's Excel reports)") - parser = AortaReportParser() + has_valid_reports = False + if analysis_dir and analysis_dir.exists(): + reports_dir = analysis_dir / "individual_reports" + if reports_dir.exists(): + report_files = list(reports_dir.glob("perf_rank*.xlsx")) or list( + reports_dir.glob("perf_*ch_rank*.xlsx") + ) + has_valid_reports = len(report_files) > 0 + + parse_result = None + parser = None + report_warnings = [] + + if has_valid_reports: + try: + log.info("Container reports present; trying AortaReportParser (host parse path)") + report_parser = AortaReportParser() + parse_result = report_parser.parse(run_result) + parser = report_parser + report_warnings = list(parse_result.warnings) + if parse_result.has_results: + log.info("Using metrics from container Excel reports") + except ImportError as e: + log.warning(f"AortaReportParser unavailable ({e}), using host raw-trace parsing") + parse_result = None + + # Primary path: host parsing from torch_traces. When trace_dir exists we always attempt + # host parsing if we don't have metrics yet, so NO_DATA only occurs when both report + # and raw-trace parsing yield no metrics. + if not trace_dir or not trace_dir.exists(): + if parse_result is None: + fail_test("No torch_traces artifact; cannot parse on host") + # else we already have parse_result from reports else: - log.info("Falling back to TraceLensParser (parsing raw traces)") - parser = TraceLensParser(use_tracelens=True) - - # Parse results - parse_result = parser.parse(run_result) + if parse_result is None or not parse_result.has_results: + if parse_result is not None and parse_result.status == ParseStatus.NO_DATA: + log.info("No Excel metrics; falling back to host raw-trace parsing") + else: + log.info("Parsing raw traces on host (TraceLensParser)") + host_parser = TraceLensParser(use_tracelens=True) + host_result = host_parser.parse(run_result) + host_result.warnings = report_warnings + list(host_result.warnings) + parse_result = host_result + parser = host_parser + + if parse_result is None: + fail_test("Parse step produced no result") TestAortaBenchmark.parse_result = parse_result if parse_result.status == ParseStatus.FAILED: for error in parse_result.errors: fail_test(f"Parse error: {error}") + elif parse_result.status == ParseStatus.NO_DATA and not parse_result.has_results: + # NO_DATA only when both report and raw-trace parsing yielded no metrics + log.info("No parsed metrics available; raw traces may still be inspected in Perfetto") - # Log warnings + # Log warnings (for all statuses) for warning in parse_result.warnings: log.warning(warning) @@ -281,6 +323,7 @@ def test_parse_results(self, aorta_runner_config, validated_aorta_config): rccl_branch=aorta_runner_config.rccl.branch, ) TestAortaBenchmark.benchmark_result = benchmark_result + TestAortaBenchmark._parser = parser if benchmark_result: log.info("Aggregated results:") @@ -298,8 +341,8 @@ def test_validate_thresholds(self, validated_aorta_config): if TestAortaBenchmark.benchmark_result is None: pytest.skip("No benchmark result available - run test_parse_results first") - # Use AortaReportParser for validation (same interface as TraceLensParser) - parser = AortaReportParser() + # Use same parser that produced benchmark_result (both have validate_thresholds) + parser = TestAortaBenchmark._parser or TraceLensParser(use_tracelens=True) # Get expected results from validated config expected = { diff --git a/docs/reference/configuration-files/aorta.rst b/docs/reference/configuration-files/aorta.rst new file mode 100644 index 000000000..ede91ac36 --- /dev/null +++ b/docs/reference/configuration-files/aorta.rst @@ -0,0 +1,199 @@ +.. meta:: + :description: Configure the Aorta benchmark configuration file variables + :keywords: Aorta, ROCm, RCCL, benchmark, CVS + +******************************************** +Aorta benchmark test configuration file +******************************************** + +The Aorta benchmark runs distributed training with RCCL in a container, collects PyTorch profiler traces, and validates iteration time and compute/communication overlap. Metrics are derived from host-side trace parsing (raw traces or TraceLens reports when available). + +``aorta_benchmark.yaml`` +======================== + +Here's a code snippet of the ``aorta_benchmark.yaml`` file for reference: + +.. note:: + + Set ``aorta_path`` to the absolute path of your Aorta repository on the host. The runner bind-mounts this path into the container. Do not leave the default ````. + +.. dropdown:: ``aorta_benchmark.yaml`` + + .. code:: yaml + + # Path to Aorta repository on host (bind-mounted into container) + aorta_path: + container_mount_path: /mnt + base_config: config/distributed.yaml + + docker: + image: jeffdaily/pytorch:torchrec-dlrm-complete + container_name: aorta-benchmark + shm_size: 17G + network_mode: host + privileged: true + + rccl: + clone_url: https://github.com/rocm/rccl.git + branch: develop + build_path: /mnt/rccl + + environment: + NCCL_MAX_NCHANNELS: 112 + NCCL_MAX_P2P_NCHANNELS: 112 + NCCL_DEBUG: VERSION + TORCH_NCCL_HIGH_PRIORITY: 1 + OMP_NUM_THREADS: 1 + RCCL_MSCCL_ENABLE: 0 + + training_overrides: + training.max_steps: 100 + profiling.active: 10 + + build_script: scripts/build_rccl.sh + experiment_script: scripts/launch_rocm.sh + gpus_per_node: 8 + timeout_seconds: 10800 + skip_rccl_build: false + + analysis: + enable_tracelens: false + enable_gemm_analysis: false + tracelens_script: scripts/tracelens_single_config/run_tracelens_single_config.sh + skip_if_exists: false + + expected_results: + max_avg_iteration_ms: 7000 + min_compute_ratio: 0.8 + min_overlap_ratio: 0.0 + max_time_variance_ratio: 0.2 + +Parameters +========== + +Here's an exhaustive list of the available parameters in the Aorta benchmark configuration file. + +.. list-table:: + :widths: 3 3 5 + :header-rows: 1 + + * - Configuration parameters + - Default values + - Description + * - ``aorta_path`` + - (required) + - Absolute path to Aorta repository on host; bind-mounted into container + * - ``container_mount_path`` + - ``/mnt`` + - Mount point inside container for ``aorta_path`` + * - ``base_config`` + - ``config/distributed.yaml`` + - Aorta config file path relative to ``aorta_path`` + * - ``docker.image`` + - ``jeffdaily/pytorch:torchrec-dlrm-complete`` + - Docker image for the benchmark container + * - ``docker.container_name`` + - ``aorta-benchmark`` + - Name of the container + * - ``docker.shm_size`` + - ``17G`` + - Shared memory size for the container + * - ``docker.network_mode`` + - ``host`` + - Docker network mode + * - ``docker.privileged`` + - true + - Run container in privileged mode + * - ``rccl.clone_url`` + - ``https://github.com/rocm/rccl.git`` + - RCCL git repository URL (used if building RCCL inside container) + * - ``rccl.branch`` + - ``develop`` + - RCCL branch to build + * - ``rccl.build_path`` + - ``/mnt/rccl`` + - Path inside container for RCCL build + * - ``environment.NCCL_MAX_NCHANNELS`` + - 112 + - Maximum NCCL channels + * - ``environment.NCCL_MAX_P2P_NCHANNELS`` + - 112 + - Maximum NCCL P2P channels + * - ``environment.NCCL_DEBUG`` + - ``VERSION`` + - NCCL debug level + * - ``environment.TORCH_NCCL_HIGH_PRIORITY`` + - 1 + - Enable high-priority NCCL streams + * - ``environment.OMP_NUM_THREADS`` + - 1 + - OpenMP thread count + * - ``environment.RCCL_MSCCL_ENABLE`` + - 0 + - Enable MSCCL + * - ``training_overrides`` + - (key-value overrides) + - Overrides passed to Aorta via ``--override`` (e.g. ``training.max_steps``, ``profiling.active``) + * - ``build_script`` + - ``scripts/build_rccl.sh`` + - RCCL build script path relative to container mount + * - ``experiment_script`` + - ``scripts/launch_rocm.sh`` + - Experiment/launch script path relative to container mount + * - ``gpus_per_node`` + - 8 + - Number of GPUs per node + * - ``timeout_seconds`` + - 10800 + - Benchmark timeout in seconds + * - ``skip_rccl_build`` + - false + - If true, skip RCCL build (use existing build in ``aorta_path``) + * - ``analysis.enable_tracelens`` + - false + - Run TraceLens analysis after benchmark (optional, host parsing works without it) + * - ``analysis.enable_gemm_analysis`` + - false + - Run GEMM analysis (for sweep experiments) + * - ``analysis.tracelens_script`` + - ``scripts/tracelens_single_config/run_tracelens_single_config.sh`` + - TraceLens script path relative to ``aorta_path`` + * - ``analysis.gemm_script`` + - ``scripts/gemm_analysis/run_tracelens_analysis.sh`` + - GEMM analysis script path relative to ``aorta_path`` + * - ``analysis.skip_if_exists`` + - false + - Skip analysis if ``tracelens_analysis`` directory already exists + * - ``expected_results.max_avg_iteration_ms`` + - e.g. 7000 + - Maximum acceptable average iteration time (ms); validation fails if exceeded + * - ``expected_results.min_compute_ratio`` + - e.g. 0.8 + - Minimum acceptable compute ratio (compute time / total iteration time) + * - ``expected_results.min_overlap_ratio`` + - e.g. 0.0 + - Minimum acceptable compute-communication overlap ratio + * - ``expected_results.max_time_variance_ratio`` + - e.g. 0.2 + - Maximum acceptable iteration time variance (e.g. std/mean); used for rank balance + +How to run +========== + +From the CVS repo root (directory containing ``cvs`` and ``input``): + +.. code-block:: bash + + cvs run test_aorta \ + --cluster_file input/cluster_file/cluster.json \ + --config_file input/config_file/aorta/aorta_benchmark.yaml \ + -v --log-cli-level=INFO + +Provide a valid ``cluster_file`` and ensure ``aorta_path`` in the config points to an existing Aorta checkout. The runner will build RCCL (unless ``skip_rccl_build`` is true), run the experiment script, collect ``torch_traces`` (PyTorch profiler output), and optionally run TraceLens in the container. Results are parsed on the host from raw traces or from TraceLens reports when present. + +Expected results and artifacts +============================== + +Validation uses the ``expected_results`` thresholds: iteration time must be within ``max_avg_iteration_ms``, compute and overlap ratios must meet the minimums, and time variance across ranks must not exceed ``max_time_variance_ratio``. Exact pass values depend on cluster size and hardware. + +Artifacts produced under the configured output directory include training logs, ``torch_profiler`` (or equivalent) trace data, and optionally ``tracelens_analysis`` when TraceLens is enabled. The test report (e.g. ``aorta_benchmark_report.json``) summarizes metrics and pass/fail per threshold. diff --git a/docs/reference/configuration-files/configure-config.rst b/docs/reference/configuration-files/configure-config.rst index 7c137870a..24a4a3420 100644 --- a/docs/reference/configuration-files/configure-config.rst +++ b/docs/reference/configuration-files/configure-config.rst @@ -21,6 +21,7 @@ The following list provides a link to code snippets and the parameters for each - :doc:`Health ` - :doc:`InfiniBand (IB Perf) ` - :doc:`RCCL ` +- :doc:`Aorta benchmark ` - :doc:`JAX ` - :doc:`Megatron ` diff --git a/docs/sphinx/_toc.yml.in b/docs/sphinx/_toc.yml.in index cae5df162..e1476e9c0 100644 --- a/docs/sphinx/_toc.yml.in +++ b/docs/sphinx/_toc.yml.in @@ -34,6 +34,8 @@ subtrees: title: IB Perf - file: reference/configuration-files/rccl title: RCCL + - file: reference/configuration-files/aorta + title: Aorta benchmark - file: reference/configuration-files/jax title: JAX - file: reference/configuration-files/megatron