Skip to content

Commit c76fbea

Browse files
committed
Update
[ghstack-poisoned]
2 parents 21812dd + efa086d commit c76fbea

487 files changed

Lines changed: 33527 additions & 1991 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.ci/docker/build.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ case "${IMAGE_NAME}" in
8888
LINTRUNNER=""
8989
GCC_VERSION=11
9090
CUDA_WINDOWS_CROSS_COMPILE=yes
91-
CUDA_VERSION=13.0
91+
CUDA_VERSION=13.2
9292
SKIP_PYTORCH=yes
9393
;;
9494
executorch-ubuntu-24.04-gcc14)

‎.ci/docker/common/install_cuda.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,10 @@
1010

1111
set -ex
1212

13-
# CUDA version must be specified (e.g., 13.0)
13+
# CUDA version must be specified (e.g., 13.2)
1414
CUDA_VERSION="${CUDA_VERSION:?CUDA_VERSION must be set}"
1515

16-
# Convert version format (e.g., 13.0 -> 13-0 for package names)
16+
# Convert version format (e.g., 13.2 -> 13-2 for package names)
1717
CUDA_VERSION_DASH=$(echo "${CUDA_VERSION}" | tr '.' '-')
1818

1919
# Add NVIDIA package repository

‎.ci/docker/common/install_cuda_windows_cross_compile.sh‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,7 @@ INSTALL_DIR="${WINDOWS_CUDA_INSTALL_DIR:-/opt/cuda-windows}"
1414
# Mapping of CUDA versions to their corresponding driver versions for Windows installers
1515
# Source: https://developer.nvidia.com/cuda-toolkit-archive
1616
declare -A CUDA_DRIVER_MAP=(
17-
["12.8"]="12.8.1:572.61"
18-
["12.9"]="12.9.1:576.57"
19-
["13.0"]="13.0.2:"
17+
["13.2"]="13.2.2:"
2018
)
2119

2220
install_mingw() {

‎.ci/scripts/export_model_artifact.sh‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -716,9 +716,14 @@ if [ "$MODEL_NAME" = "muse_glimmer" ]; then
716716
# executor drives, plus the vision encoder and forward_from_embeddings.
717717
OFFGRAPH_FLAG=""
718718
if [ "$MODE" = "dflash-image-batching" ]; then
719-
OFFGRAPH_FLAG="--use-offgraph-kv-cache"
719+
# Every static forward_{N} and draft_forward_{N} is its own AOTI
720+
# compile and autotune, which the 11-vCPU runner cannot fit in the job
721+
# timeout at the default sizes. These still cover static and dynamic
722+
# target steps and multi-owner drafting and verification.
723+
OFFGRAPH_FLAG="--use-offgraph-kv-cache --forward-sizes 1,4 --max-draft-owners 2"
720724
fi
721725
EXPORT_START_SECONDS=$SECONDS
726+
PYTHONUNBUFFERED=1 \
722727
TMPDIR="$INDUCTOR_TMPDIR" \
723728
TORCHINDUCTOR_CACHE_DIR="$INDUCTOR_CACHE" \
724729
python -m executorch.examples.models.muse_glimmer.export.export_dflash \

‎.ci/scripts/test-cuda-build.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77

88
set -exu
99

10-
CUDA_VERSION=${1:-"13.0"}
10+
CUDA_VERSION=${1:-"13.2"}
1111

1212
echo "=== Testing ExecuTorch CUDA ${CUDA_VERSION} Build ==="
1313

‎.ci/scripts/test_espressif_host.sh‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
#!/usr/bin/env bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
# Builds the ESP-NN delegate for the host, plain and with ASan/UBSan, and checks
9+
# fully delegated programs against the bit-exact Python reference on both. The
10+
# sanitized build also turns on the delegate's non-default options. Needs no
11+
# ESP-IDF toolchain or emulator.
12+
13+
set -euo pipefail
14+
15+
et_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
16+
build_dir="${ESPRESSIF_HOST_BUILD_DIR:-${et_root}/cmake-out-espnn-host}"
17+
cd "${et_root}"
18+
19+
for variant in plain sanitized; do
20+
sanitize=OFF
21+
if [[ "${variant}" == sanitized ]]; then
22+
sanitize=ON
23+
fi
24+
options=(-DESPNN_HOST_SANITIZE="${sanitize}"
25+
-DESPNN_DELEGATE_SCRATCH_IN_ARENA="${sanitize}"
26+
-DESPNN_DELEGATE_PROFILE="${sanitize}")
27+
cmake -S backends/espressif/test/host -B "${build_dir}/${variant}" -G Ninja \
28+
-DCMAKE_BUILD_TYPE=RelWithDebInfo \
29+
-DPYTHON_EXECUTABLE="$(command -v python)" "${options[@]}"
30+
cmake --build "${build_dir}/${variant}"
31+
ESPNN_HOST_RUNNER="${build_dir}/${variant}/espnn_host_runner" \
32+
ESPNN_HOST_INTERPRETER="${build_dir}/${variant}/espnn_host_interpreter" \
33+
python -m pytest -v --tb=short backends/espressif/test/test_host_runtime.py \
34+
backends/espressif/test/test_host_interpreter.py
35+
done

‎.ci/scripts/test_native_llm_runner.py‎

Lines changed: 50 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -5,34 +5,49 @@
55
# LICENSE file in the root directory of this source tree.
66

77
"""Compare GGML LLM runner prefill and decoding with a tiny eager HF Llama,
8-
including a prompt longer than the method's token input."""
8+
including a prompt longer than the method's token input, for in-graph and
9+
off-graph KV cache packages."""
910

1011
import argparse
1112
import base64
13+
import copy
1214
import re
1315
import subprocess
1416
import tempfile
1517
from pathlib import Path
1618

1719
import torch
18-
from executorch.backends.native.examples.hf_llm.export_hf_llm import LastTokenLM
20+
from executorch.backends.native.examples.hf_llm.export_hf_llm import (
21+
LastTokenLM,
22+
OffGraphLastTokenLM,
23+
)
1924
from executorch.exir.native import to_native
25+
from executorch.extension.llm.cache.hf_attention import register_offgraph_attention
2026
from transformers import LlamaConfig, LlamaForCausalLM
2127

2228

2329
def export_package(
24-
model: LlamaForCausalLM, max_seq_length: int, max_tokens: int, path: Path
30+
wrapper: torch.nn.Module,
31+
max_tokens: int,
32+
path: Path,
33+
metadata: dict | None = None,
34+
selected: bool = False,
2535
) -> None:
26-
"""A last-token package whose method takes at most `max_tokens` tokens."""
27-
wrapper = LastTokenLM(model, max_seq_length).eval()
36+
"""A package of `wrapper` whose method takes at most `max_tokens` tokens,
37+
and with `selected` the token indices whose logits to return."""
2838
seq = torch.export.Dim("seq", min=1, max=max_tokens)
39+
inputs = [torch.zeros((1, 6), dtype=torch.long), torch.arange(6)]
40+
dynamic_shapes = {"input_ids": {1: seq}, "cache_position": {0: seq}}
41+
if selected:
42+
# Trace two rows: export specializes a size of 1.
43+
inputs.append(torch.tensor([0, 5]))
44+
dynamic_shapes["logits_to_keep"] = {
45+
0: torch.export.Dim("rows", min=1, max=max_tokens)
46+
}
2947
exported = torch.export.export(
30-
wrapper,
31-
(torch.zeros((1, 6), dtype=torch.long), torch.arange(6)),
32-
dynamic_shapes={"input_ids": {1: seq}, "cache_position": {0: seq}},
33-
strict=True,
48+
wrapper, tuple(inputs), dynamic_shapes=dynamic_shapes, strict=True
3449
)
35-
to_native(exported).save(str(path))
50+
to_native(exported, metadata=metadata).save(str(path))
3651

3752

3853
def check_runner(
@@ -121,12 +136,34 @@ def main() -> None:
121136
f"{base64.b64encode(bytes([i])).decode()} {i}\n" for i in range(256)
122137
)
123138
)
139+
in_graph = LastTokenLM(model, max_seq_length).eval()
124140
full = root / "llama.ptn"
125-
export_package(model, max_seq_length, max_seq_length - 1, full)
141+
export_package(in_graph, max_seq_length - 1, full)
126142
# A 20-token prompt prefills in three calls of at most 7 tokens.
127143
chunked = root / "llama_chunked.ptn"
128-
export_package(model, max_seq_length, 7, chunked)
129-
for package, prompt in ((full, "abc"), (chunked, "the quick brown fox")):
144+
export_package(in_graph, 7, chunked)
145+
# The runner supplies the KV cache, sized from the package's metadata.
146+
register_offgraph_attention("offgraph")
147+
offgraph_model = copy.deepcopy(model)
148+
offgraph_model.config._attn_implementation = "offgraph"
149+
offgraph = OffGraphLastTokenLM(offgraph_model, max_seq_length).eval()
150+
offgraph_chunked = root / "llama_offgraph_chunked.ptn"
151+
export_package(offgraph, 7, offgraph_chunked, offgraph.metadata())
152+
# A batching export; the runner selects the last token's logits.
153+
selected = OffGraphLastTokenLM(
154+
offgraph_model, max_seq_length, selected=True
155+
).eval()
156+
selected_chunked = root / "llama_selected_chunked.ptn"
157+
export_package(
158+
selected, 7, selected_chunked, selected.metadata(), selected=True
159+
)
160+
long_prompt = "the quick brown fox"
161+
for package, prompt in (
162+
(full, "abc"),
163+
(chunked, long_prompt),
164+
(offgraph_chunked, long_prompt),
165+
(selected_chunked, long_prompt),
166+
):
130167
check_runner(
131168
args.runner,
132169
model,

‎.ci/scripts/tests/test_cu134_dependencies.py‎

Lines changed: 31 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,12 @@
66

77
import ast
88
import importlib.util
9+
import io
910
import os
1011
import subprocess
1112
import sys
1213
import unittest
14+
from contextlib import redirect_stderr
1315
from pathlib import Path
1416
from unittest.mock import patch
1517

@@ -70,7 +72,7 @@ def test_all_install_steps_preserve_exact_cu134_selection(self):
7072
"torch==2.15.0.dev20260921+cu134",
7173
"torchvision==0.30.0.dev20260922+cu134",
7274
"torchaudio==2.11.0.dev20260922+cu134",
73-
f"torchao=={self.installer.CU134_TORCHAO_NIGHTLY_VERSION}+{ao_variant}",
75+
f"torchao=={self.installer.TORCHAO_NIGHTLY_VERSION}+{ao_variant}",
7476
}
7577
for index, command in enumerate(commands):
7678
required = (
@@ -103,7 +105,7 @@ def test_all_install_steps_preserve_exact_cu134_selection(self):
103105
)
104106

105107
def test_other_cuda_trains_keep_existing_pins(self):
106-
for cuda in ((13, 0), (13, 2)):
108+
for cuda in ((13, 2),):
107109
for machine in ("x86_64", "aarch64"):
108110
with self.subTest(cuda=cuda, machine=machine):
109111
core, local, domains, examples = self.install_commands(
@@ -118,6 +120,27 @@ def test_other_cuda_trains_keep_existing_pins(self):
118120
self.assertFalse(any("==" in arg for arg in local))
119121
self.assertFalse(any("==" in arg for arg in examples))
120122

123+
def test_cuda_13_0_falls_back_to_cpu_with_a_warning(self):
124+
self.assertNotIn((13, 0), self.utils.SUPPORTED_CUDA_VERSIONS)
125+
nvcc = subprocess.CompletedProcess(
126+
[], 0, stdout="Cuda compilation tools, release 13.0, V13.0.88"
127+
)
128+
for cache in (
129+
self.utils._get_cuda_version,
130+
self.utils.determine_torch_url,
131+
):
132+
cache.cache_clear()
133+
self.addCleanup(cache.cache_clear)
134+
stderr = io.StringIO()
135+
with (
136+
patch.object(self.utils.subprocess, "run", return_value=nvcc),
137+
patch.object(self.utils.platform, "system", return_value="Linux"),
138+
redirect_stderr(stderr),
139+
):
140+
url = self.utils.determine_torch_url(self.installer.TORCH_URL_BASE)
141+
self.assertEqual(url, f"{self.installer.TORCH_URL_BASE}/cpu")
142+
self.assertIn("CUDA 13.0 is installed but not supported", stderr.getvalue())
143+
121144
def test_source_pinned_torch_is_not_replaced(self):
122145
for cuda in ((13, 2), (13, 4)):
123146
with self.subTest(cuda=cuda):
@@ -159,7 +182,7 @@ def test_cpu_release_dependencies_are_opt_in(self):
159182
)
160183

161184
def test_cpu_release_option_keeps_cuda_pins(self):
162-
for cuda in ((13, 0), (13, 2), (13, 4)):
185+
for cuda in ((13, 2), (13, 4)):
163186
for machine in ("x86_64", "aarch64"):
164187
with self.subTest(cuda=cuda, machine=machine):
165188
self.assertEqual(
@@ -408,15 +431,11 @@ def test_cu134_keeps_explicit_torchao_source_build(self):
408431
f"{source_version}+git{source_commit[:7]}", metadata.specifier
409432
)
410433

411-
def test_wheel_torchao_bound_matches_selected_train(self):
412-
for cuda in ((13, 4), (13, 2), None):
413-
version = (
414-
self.installer.CU134_TORCHAO_NIGHTLY_VERSION
415-
if cuda == (13, 4)
416-
else self.installer.TORCHAO_NIGHTLY_VERSION
417-
)
418-
major, minor = (int(part) for part in version.split(".")[:2])
419-
expected = f"torchao>={version},<{major}.{minor + 1}"
434+
def test_wheel_torchao_bound_is_the_same_for_every_train(self):
435+
version = self.installer.TORCHAO_NIGHTLY_VERSION
436+
major, minor = (int(part) for part in version.split(".")[:2])
437+
expected = f"torchao>={version},<{major}.{minor + 1}"
438+
for cuda in (*self.utils.SUPPORTED_CUDA_VERSIONS, None):
420439
self.utils.determine_torch_url.cache_clear()
421440
with (
422441
patch.object(

‎.ci/scripts/tests/test_cuda_workflow.py‎

Lines changed: 45 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
# LICENSE file in the root directory of this source tree.
66

77
import os
8+
import re
89
import subprocess
910
import sys
1011
import unittest
@@ -17,6 +18,13 @@
1718
WORKFLOW = yaml.safe_load((ROOT / ".github" / "workflows" / "cuda.yml").read_text())
1819

1920

21+
def _supported_cuda_versions():
22+
import runpy
23+
24+
versions = runpy.run_path(str(ROOT / "install_utils.py"))["SUPPORTED_CUDA_VERSIONS"]
25+
return [f"{major}.{minor}" for major, minor in versions]
26+
27+
2028
def _all_keys(value):
2129
if isinstance(value, dict):
2230
for key, child in value.items():
@@ -32,11 +40,44 @@ def _model_quant(entry):
3240

3341

3442
class CudaWorkflowTest(unittest.TestCase):
35-
def test_build_matrix_preserves_existing_cuda_versions(self):
43+
def test_build_matrix_covers_supported_cuda_versions(self):
3644
job = WORKFLOW["jobs"]["test-cuda-builds"]
37-
self.assertEqual(job["strategy"]["matrix"]["cuda-version"], ["13.0", "13.4"])
45+
self.assertEqual(
46+
job["strategy"]["matrix"]["cuda-version"], _supported_cuda_versions()
47+
)
3848
self.assertEqual(job["with"]["gpu-arch-version"], "${{ matrix.cuda-version }}")
3949

50+
def test_every_cuda_job_uses_a_supported_version(self):
51+
supported = _supported_cuda_versions()
52+
checked = 0
53+
for workflow in ("cuda.yml", "cuda-windows.yml"):
54+
jobs = yaml.safe_load(
55+
(ROOT / ".github" / "workflows" / workflow).read_text()
56+
)["jobs"]
57+
for name, job in jobs.items():
58+
with_ = job.get("with", {})
59+
version = with_.get("gpu-arch-version")
60+
if with_.get("gpu-arch-type") != "cuda" or "${{" in str(version):
61+
continue
62+
checked += 1
63+
with self.subTest(workflow=workflow, job=name):
64+
self.assertIn(version, supported)
65+
self.assertGreater(checked, 0)
66+
67+
def test_windows_export_image_matches_the_export_job(self):
68+
build = (ROOT / ".ci" / "docker" / "build.sh").read_text()
69+
match = re.search(
70+
r"executorch-ubuntu-22\.04-cuda-windows\)(.*?);;", build, re.S
71+
)
72+
self.assertIsNotNone(match)
73+
image_version = re.search(r"CUDA_VERSION=(\S+)", match.group(1)).group(1)
74+
self.assertIn(image_version, _supported_cuda_versions())
75+
jobs = yaml.safe_load(
76+
(ROOT / ".github" / "workflows" / "cuda-windows.yml").read_text()
77+
)["jobs"]
78+
export = jobs["export-model-cuda-windows-artifact"]["with"]
79+
self.assertEqual(export["gpu-arch-version"], image_version)
80+
4081
def test_cuda_builds_take_the_node_driver_on_an_unpinned_v3(self):
4182
# This job used to pin linux_job_v2 to a test-infra SHA because that was
4283
# the only ref carrying driver-version / driver-download-url, and it
@@ -56,7 +97,7 @@ def test_cuda_runtime_update_precedes_build_and_propagates_failure(self):
5697
source() { printf 'BUILD %s\n' "$*"; printf 'LIBRARY PATH %s\n' "$LD_LIBRARY_PATH"; }
5798
"""
5899
for version, conda_status in (
59-
(version, status) for version in ("13.0", "13.4") for status in (0, 1)
100+
(version, status) for version in ("13.2", "13.4") for status in (0, 1)
60101
):
61102
with self.subTest(version=version, conda_status=conda_status):
62103
result = subprocess.run(
@@ -120,7 +161,7 @@ def assert_close(actual, expected):
120161
"""
121162
for expected, invalid_result, succeeds in (
122163
("13.4", "0", True),
123-
("13.0", "0", False),
164+
("13.2", "0", False),
124165
("13.4", "1", False),
125166
):
126167
with self.subTest(expected=expected, invalid_result=invalid_result):

0 commit comments

Comments
 (0)