diff --git a/.ci/docker/build.sh b/.ci/docker/build.sh index e6122c498ce..7c4a80044e4 100755 --- a/.ci/docker/build.sh +++ b/.ci/docker/build.sh @@ -92,18 +92,6 @@ esac TORCH_VERSION=$(cat ci_commit_pins/pytorch.txt) BUILD_DOCS=1 -# Pull channel + spec/url helpers out of torch_pin.py so install_pytorch.sh -# (which runs inside the docker build, where torch_pin.py isn't available) -# can decide between wheel install (test/release) and source build (nightly). -# Self-hosted runners often have python3 but not the unversioned python alias. -PYTHON_BIN=$(command -v python3 || command -v python) -TORCH_PIN_HELPERS=$(cd ../.. && "$PYTHON_BIN" -c "from torch_pin import CHANNEL, torch_spec, torchaudio_spec, torchvision_spec, torch_index_url_base; print(CHANNEL); print(torch_spec()); print(torchaudio_spec()); print(torchvision_spec()); print(torch_index_url_base())") -TORCH_CHANNEL=$(echo "${TORCH_PIN_HELPERS}" | sed -n '1p') -TORCH_SPEC=$(echo "${TORCH_PIN_HELPERS}" | sed -n '2p') -TORCHAUDIO_SPEC=$(echo "${TORCH_PIN_HELPERS}" | sed -n '3p') -TORCHVISION_SPEC=$(echo "${TORCH_PIN_HELPERS}" | sed -n '4p') -TORCH_INDEX_URL=$(echo "${TORCH_PIN_HELPERS}" | sed -n '5p') - # Copy requirements-lintrunner.txt from root to here cp ../../requirements-lintrunner.txt ./ @@ -116,11 +104,6 @@ docker build \ --build-arg "PYTHON_VERSION=${PYTHON_VERSION}" \ --build-arg "MINICONDA_VERSION=${MINICONDA_VERSION}" \ --build-arg "TORCH_VERSION=${TORCH_VERSION}" \ - --build-arg "TORCH_CHANNEL=${TORCH_CHANNEL}" \ - --build-arg "TORCH_SPEC=${TORCH_SPEC}" \ - --build-arg "TORCHAUDIO_SPEC=${TORCHAUDIO_SPEC}" \ - --build-arg "TORCHVISION_SPEC=${TORCHVISION_SPEC}" \ - --build-arg "TORCH_INDEX_URL=${TORCH_INDEX_URL}" \ --build-arg "BUCK2_VERSION=${BUCK2_VERSION}" \ --build-arg "LINTRUNNER=${LINTRUNNER:-}" \ --build-arg "BUILD_DOCS=${BUILD_DOCS}" \ diff --git a/.ci/docker/ci_commit_pins/pytorch.txt b/.ci/docker/ci_commit_pins/pytorch.txt index 0932a9ef6b8..f6e39a63b92 100644 --- a/.ci/docker/ci_commit_pins/pytorch.txt +++ b/.ci/docker/ci_commit_pins/pytorch.txt @@ -1 +1 @@ -release/2.11 +release/2.11 \ No newline at end of file diff --git a/.ci/docker/common/install_pytorch.sh b/.ci/docker/common/install_pytorch.sh index ddf2f21baa9..548a24f885d 100755 --- a/.ci/docker/common/install_pytorch.sh +++ b/.ci/docker/common/install_pytorch.sh @@ -17,24 +17,6 @@ install_domains() { } install_pytorch_and_domains() { - if [ "${TORCH_CHANNEL}" != "nightly" ]; then - # Test/release: install the published wheels directly. The specs and URL - # are passed in as docker build args (computed from torch_pin.py by - # .ci/docker/build.sh). RC wheels at /whl/test/ get re-uploaded under the - # same version, so use --no-cache-dir there to avoid stale cache hits. - local cache_flag="" - if [ "${TORCH_CHANNEL}" = "test" ]; then - cache_flag="--no-cache-dir" - fi - pip_install --force-reinstall ${cache_flag} \ - "${TORCH_SPEC}" "${TORCHVISION_SPEC}" "${TORCHAUDIO_SPEC}" \ - --index-url "${TORCH_INDEX_URL}/cpu" - return - fi - - # Nightly: build pytorch from source against the pinned SHA in pytorch.txt - # so we catch upstream regressions, then install audio/vision from the - # commits that pytorch itself pins. git clone https://github.com/pytorch/pytorch.git # Fetch the target commit @@ -45,19 +27,14 @@ install_pytorch_and_domains() { chown -R ci-user . export _GLIBCXX_USE_CXX11_ABI=1 - # PyTorch's FindARM.cmake hard-fails when the SVE+BF16 compile probe - # doesn't pass ā€” gcc-11 in this image is too old to accept the combined - # NEON/SVE/bfloat16 intrinsics the probe exercises. Executorch's aarch64 - # runtime targets (phones, embedded) don't use SVE, so bypass the check. - export BUILD_IGNORE_SVE_UNAVAILABLE=1 # Then build and install PyTorch conda_run python setup.py bdist_wheel pip_install "$(echo dist/*.whl)" - # Defer to PyTorch's own pinned audio/vision commits. - TORCHAUDIO_VERSION=$(cat .github/ci_commit_pins/audio.txt) + # Grab the pinned audio and vision commits from PyTorch + TORCHAUDIO_VERSION=release/2.11 export TORCHAUDIO_VERSION - TORCHVISION_VERSION=$(cat .github/ci_commit_pins/vision.txt) + TORCHVISION_VERSION=release/0.26 export TORCHVISION_VERSION install_domains diff --git a/.ci/docker/ubuntu/Dockerfile b/.ci/docker/ubuntu/Dockerfile index 98268d49675..0e2d7e48eb9 100644 --- a/.ci/docker/ubuntu/Dockerfile +++ b/.ci/docker/ubuntu/Dockerfile @@ -64,11 +64,6 @@ ENV SCCACHE_S3_KEY_PREFIX executorch ENV SCCACHE_REGION us-east-1 ARG TORCH_VERSION -ARG TORCH_CHANNEL -ARG TORCH_SPEC -ARG TORCHAUDIO_SPEC -ARG TORCHVISION_SPEC -ARG TORCH_INDEX_URL ARG SKIP_PYTORCH COPY ./common/install_pytorch.sh install_pytorch.sh COPY ./common/utils.sh utils.sh diff --git a/.ci/scripts/test_model_e2e.sh b/.ci/scripts/test_model_e2e.sh index 97711de2713..1678b0a4fbb 100755 --- a/.ci/scripts/test_model_e2e.sh +++ b/.ci/scripts/test_model_e2e.sh @@ -272,10 +272,7 @@ elif [[ "$MODEL_NAME" == *whisper* ]] || [ "$MODEL_NAME" = "voxtral_realtime" ]; fi fi pip install datasets soundfile - # We pushd'd into EXECUTORCH_ROOT above, so torch_pin is importable here. - TORCHCODEC_PKG=$(python -c "from torch_pin import torchcodec_spec; print(torchcodec_spec())") - TORCHCODEC_INDEX=$(python -c "from torch_pin import torch_index_url_base; print(torch_index_url_base())") - pip install "$TORCHCODEC_PKG" --extra-index-url "${TORCHCODEC_INDEX}/cpu" + pip install torchcodec==0.11.0 --extra-index-url https://download.pytorch.org/whl/test/cpu python -c "from datasets import load_dataset;import soundfile as sf;sample = load_dataset('distil-whisper/librispeech_long', 'clean', split='validation')[0]['audio'];sf.write('${MODEL_DIR}/$AUDIO_FILE', sample['array'][:sample['sampling_rate']*30], sample['sampling_rate'])" fi diff --git a/.ci/scripts/test_wheel_package_qnn.sh b/.ci/scripts/test_wheel_package_qnn.sh index f44fafadb58..763bd8733c1 100644 --- a/.ci/scripts/test_wheel_package_qnn.sh +++ b/.ci/scripts/test_wheel_package_qnn.sh @@ -150,26 +150,25 @@ run_core_tests () { echo "=== [$LABEL] Installing wheel & deps ===" "$PIPBIN" install --upgrade pip "$PIPBIN" install "$WHEEL_FILE" - # runpy.run_path uses a relative path, so the caller must run this script - # from the executorch repo root (where torch_pin.py lives). - TORCH_SPEC=$( + TORCH_VERSION=$( "$PYBIN" - <<'PY' import runpy module_vars = runpy.run_path("torch_pin.py") -print(module_vars["torch_spec"]()) +print(module_vars["TORCH_VERSION"]) PY ) - TORCH_INDEX=$( - "$PYBIN" - <<'PY' -import runpy -module_vars = runpy.run_path("torch_pin.py") -print(module_vars["torch_index_url_base"]()) -PY -) - echo "=== [$LABEL] Install $TORCH_SPEC from ${TORCH_INDEX}/cpu ===" - # Install torch based on the pinned PyTorch version from the channel index. - "$PIPBIN" install "$TORCH_SPEC" --index-url "${TORCH_INDEX}/cpu" +# NIGHTLY_VERSION=$( +# "$PYBIN" - <<'PY' +# import runpy +# module_vars = runpy.run_path("torch_pin.py") +# print(module_vars["NIGHTLY_VERSION"]) +# PY +# ) + echo "=== [$LABEL] Install torch==${TORCH_VERSION} ===" + + # Install torch based on the pinned PyTorch version, preferring the PyTorch test index + "$PIPBIN" install torch=="${TORCH_VERSION}" --extra-index-url "https://download.pytorch.org/whl/test" "$PIPBIN" install wheel # Install torchao based on the pinned commit from third-party/ao submodule diff --git a/.ci/scripts/tests/test_torch_pin.py b/.ci/scripts/tests/test_torch_pin.py deleted file mode 100644 index 6c475aeaa05..00000000000 --- a/.ci/scripts/tests/test_torch_pin.py +++ /dev/null @@ -1,54 +0,0 @@ -import importlib - -import pytest - - -@pytest.fixture -def pin(): - """Yield a fresh import of torch_pin so tests can mutate CHANNEL safely.""" - import torch_pin - - yield torch_pin - importlib.reload(torch_pin) - - -@pytest.mark.parametrize( - "channel, expected_torch, expected_url", - [ - ( - "nightly", - "torch=={TORCH_VERSION}.{NIGHTLY_VERSION}", - "https://download.pytorch.org/whl/nightly", - ), - ("test", "torch=={TORCH_VERSION}", "https://download.pytorch.org/whl/test"), - ("release", "torch=={TORCH_VERSION}", "https://download.pytorch.org/whl"), - ], -) -def test_channel_resolution(pin, channel, expected_torch, expected_url): - pin.CHANNEL = channel - expected = expected_torch.format( - TORCH_VERSION=pin.TORCH_VERSION, NIGHTLY_VERSION=pin.NIGHTLY_VERSION - ) - assert pin.torch_spec() == expected - assert pin.torch_index_url_base() == expected_url - - -def test_all_specs_share_nightly_suffix(pin): - pin.CHANNEL = "nightly" - suffix = f".{pin.NIGHTLY_VERSION}" - assert pin.torch_spec().endswith(suffix) - assert pin.torchaudio_spec().endswith(suffix) - assert pin.torchcodec_spec().endswith(suffix) - assert pin.torchvision_spec().endswith(suffix) - - -def test_specs_drop_suffix_off_nightly(pin): - pin.CHANNEL = "test" - assert pin.torch_spec() == f"torch=={pin.TORCH_VERSION}" - assert pin.torchaudio_spec() == f"torchaudio=={pin.TORCHAUDIO_VERSION}" - assert pin.torchcodec_spec() == f"torchcodec=={pin.TORCHCODEC_VERSION}" - assert pin.torchvision_spec() == f"torchvision=={pin.TORCHVISION_VERSION}" - - -def test_torch_branch_derived_from_version(pin): - assert pin.torch_branch() == f"release/{pin.TORCH_VERSION.rsplit('.', 1)[0]}" diff --git a/.ci/scripts/utils.sh b/.ci/scripts/utils.sh index 12e7f3d2067..86e54b478ef 100644 --- a/.ci/scripts/utils.sh +++ b/.ci/scripts/utils.sh @@ -53,7 +53,7 @@ dedupe_macos_loader_path_rpaths() { pushd .. torch_lib_dir=$(python -c "import importlib.util; print(importlib.util.find_spec('torch').submodule_search_locations[0])")/lib popd - + if [[ -z "${torch_lib_dir}" || ! -d "${torch_lib_dir}" ]]; then return fi @@ -89,30 +89,6 @@ install_domains() { } install_pytorch_and_domains() { - # CWD is the executorch repo root, where torch_pin.py lives. - TORCH_CHANNEL=$(python -c "from torch_pin import CHANNEL; print(CHANNEL)") - - if [ "${TORCH_CHANNEL}" != "nightly" ]; then - # Test/release: install the published wheels directly from torch_pin.py's - # channel index, skipping the source-build path entirely. RC wheels at - # /whl/test/ get re-uploaded under the same version, so use --no-cache-dir - # there to avoid stale cache hits. - local torch_spec=$(python -c "from torch_pin import torch_spec; print(torch_spec())") - local torchvision_spec=$(python -c "from torch_pin import torchvision_spec; print(torchvision_spec())") - local torchaudio_spec=$(python -c "from torch_pin import torchaudio_spec; print(torchaudio_spec())") - local torch_index_url=$(python -c "from torch_pin import torch_index_url_base; print(torch_index_url_base())") - local cache_flag="" - if [ "${TORCH_CHANNEL}" = "test" ]; then - cache_flag="--no-cache-dir" - fi - pip install --force-reinstall ${cache_flag} \ - "${torch_spec}" "${torchvision_spec}" "${torchaudio_spec}" \ - --index-url "${torch_index_url}/cpu" - return - fi - - # Nightly: source-build pytorch from the pinned SHA so CI catches upstream - # regressions; pytorch's own audio/vision pins drive those installs. pushd .ci/docker || return TORCH_VERSION=$(cat ci_commit_pins/pytorch.txt) popd || return @@ -164,10 +140,10 @@ install_pytorch_and_domains() { fi dedupe_macos_loader_path_rpaths - # We're on the nightly path here; defer to PyTorch's own pinned commits. - TORCHAUDIO_VERSION=$(cat .github/ci_commit_pins/audio.txt) + # Grab the pinned audio and vision commits from PyTorch + TORCHAUDIO_VERSION=release/2.11 export TORCHAUDIO_VERSION - TORCHVISION_VERSION=$(cat .github/ci_commit_pins/vision.txt) + TORCHVISION_VERSION=release/0.26 export TORCHVISION_VERSION install_domains @@ -242,21 +218,17 @@ download_stories_model_artifacts() { } do_not_use_nightly_on_ci() { - # Sanity check that prevents accidentally landing a PR that pins to PyTorch - # nightly without exercising the source-build path (see #6564). - # - # For CHANNEL=nightly, CI source-builds pytorch from the SHA in pytorch.txt, - # so the installed torch shows up as e.g. 2.13.0a0+gitc8a648d ā€” assert that. - # For CHANNEL=test/release, we install published wheels by design (e.g. - # 2.11.0), so the +git assertion doesn't apply. - TORCH_CHANNEL=$(python -c "from torch_pin import CHANNEL; print(CHANNEL)") - if [ "${TORCH_CHANNEL}" != "nightly" ]; then - return 0 - fi - + # An assert to make sure that we are not using PyTorch nightly on CI to prevent + # regression as documented in https://github.com/pytorch/executorch/pull/6564 TORCH_VERSION=$(pip list | grep -w 'torch ' | awk -F ' ' {'print $2'} | tr -d '\n') + + # The version of PyTorch building from source looks like 2.6.0a0+gitc8a648d that + # includes the commit while nightly (2.6.0.dev20241019+cpu) or release (2.6.0) + # won't have that. Note that we couldn't check for the exact commit from the pin + # ci_commit_pins/pytorch.txt here because the value will be different when running + # this on PyTorch CI if [[ "${TORCH_VERSION}" != *"+git"* ]]; then - echo "Unexpected torch version. Expected binary built from source for CHANNEL=nightly, got ${TORCH_VERSION}" + echo "Unexpected torch version. Expected binary built from source, got ${TORCH_VERSION}" exit 1 fi } diff --git a/.github/scripts/update_pytorch_pin.py b/.github/scripts/update_pytorch_pin.py index 9f2698917b2..dbc48552d9b 100644 --- a/.github/scripts/update_pytorch_pin.py +++ b/.github/scripts/update_pytorch_pin.py @@ -8,12 +8,6 @@ import urllib.request from pathlib import Path -# torch_pin.py lives at the repo root. Locate it relative to this script so -# the import works regardless of where the script is invoked from. -_REPO_ROOT = Path(__file__).resolve().parents[2] -sys.path.insert(0, str(_REPO_ROOT)) -from torch_pin import CHANNEL, NIGHTLY_VERSION, torch_branch - def parse_nightly_version(nightly_version): """ @@ -33,6 +27,23 @@ def parse_nightly_version(nightly_version): return f"{year}-{month}-{day}" +def get_torch_nightly_version(): + """ + Read NIGHTLY_VERSION from torch_pin.py. + + Returns: + NIGHTLY_VERSION string + """ + with open("torch_pin.py", "r") as f: + content = f.read() + + match = re.search(r'NIGHTLY_VERSION\s*=\s*["\']([^"\']+)["\']', content) + if not match: + raise ValueError("Could not find NIGHTLY_VERSION in torch_pin.py") + + return match.group(1) + + def get_commit_hash_for_nightly(date_str): """ Fetch commit hash from PyTorch nightly branch for a given date. @@ -80,16 +91,17 @@ def extract_hash_from_title(title): return match.group(1) -def update_pytorch_pin(ref): +def update_pytorch_pin(commit_hash): """ - Update .ci/docker/ci_commit_pins/pytorch.txt with the new ref. + Update .ci/docker/ci_commit_pins/pytorch.txt with the new commit hash. Args: - ref: Either a commit SHA (nightly) or a branch name (test/release). + commit_hash: Commit hash to write """ - pin_file = _REPO_ROOT / ".ci/docker/ci_commit_pins/pytorch.txt" - pin_file.write_text(f"{ref}\n") - print(f"Updated {pin_file} with ref: {ref}") + pin_file = ".ci/docker/ci_commit_pins/pytorch.txt" + with open(pin_file, "w") as f: + f.write(f"{commit_hash}\n") + print(f"Updated {pin_file} with commit hash: {commit_hash}") def should_skip_file(filename): @@ -106,20 +118,18 @@ def should_skip_file(filename): return filename in skip_files -def fetch_file_content(ref, file_path): +def fetch_file_content(commit_hash, file_path): """ Fetch file content from GitHub API. Args: - ref: Commit SHA or branch name to fetch from + commit_hash: Commit hash to fetch from file_path: File path in the repository Returns: File content as bytes """ - api_url = ( - f"https://api.github.com/repos/pytorch/pytorch/contents/{file_path}?ref={ref}" - ) + api_url = f"https://api.github.com/repos/pytorch/pytorch/contents/{file_path}?ref={commit_hash}" req = urllib.request.Request(api_url) req.add_header("Accept", "application/vnd.github.v3+json") @@ -136,7 +146,7 @@ def fetch_file_content(ref, file_path): raise -def sync_directory(et_dir, pt_path, ref): +def sync_directory(et_dir, pt_path, commit_hash): """ Sync files from PyTorch to ExecuTorch using GitHub API. Only syncs files that already exist in ExecuTorch - does not add new files. @@ -144,7 +154,7 @@ def sync_directory(et_dir, pt_path, ref): Args: et_dir: ExecuTorch directory path pt_path: PyTorch directory path in the repository (e.g., "c10") - ref: Commit SHA or branch name to fetch from + commit_hash: Commit hash to fetch from Returns: Number of files grafted @@ -171,12 +181,12 @@ def sync_directory(et_dir, pt_path, ref): # Fetch content from PyTorch and compare try: - pt_content = fetch_file_content(ref, pt_file_path) + pt_content = fetch_file_content(commit_hash, pt_file_path) et_content = et_file.read_bytes() if pt_content != et_content: print(f"āš ļø Difference detected in {rel_path}") - print(f"šŸ“‹ Grafting from PyTorch ref {ref}...") + print(f"šŸ“‹ Grafting from PyTorch commit {commit_hash}...") et_file.write_bytes(pt_content) print(f"āœ… Grafted {et_file}") @@ -191,34 +201,37 @@ def sync_directory(et_dir, pt_path, ref): return files_grafted -def sync_c10_directories(ref): +def sync_c10_directories(commit_hash): """ Sync c10 and torch/headeronly directories from PyTorch to ExecuTorch using GitHub API. Args: - ref: PyTorch commit SHA or branch name to sync from + commit_hash: PyTorch commit hash to sync from Returns: Total number of files grafted """ print("\nšŸ”„ Syncing c10 directories from PyTorch via GitHub API...") + # Get repository root + repo_root = Path.cwd() + # Define directory pairs to sync (from check_c10_sync.sh) # Format: (executorch_dir, pytorch_path_in_repo) dir_pairs = [ ( - _REPO_ROOT / "runtime/core/portable_type/c10/c10", + repo_root / "runtime/core/portable_type/c10/c10", "c10", ), ( - _REPO_ROOT / "runtime/core/portable_type/c10/torch/headeronly", + repo_root / "runtime/core/portable_type/c10/torch/headeronly", "torch/headeronly", ), ] total_grafted = 0 for et_dir, pt_path in dir_pairs: - files_grafted = sync_directory(et_dir, pt_path, ref) + files_grafted = sync_directory(et_dir, pt_path, commit_hash) total_grafted += files_grafted if total_grafted > 0: @@ -231,26 +244,27 @@ def sync_c10_directories(ref): def main(): try: - print(f"CHANNEL: {CHANNEL}") - if CHANNEL == "nightly": - # Nightly pins to an immutable SHA looked up by date. - print(f"Found NIGHTLY_VERSION: {NIGHTLY_VERSION}") - date_str = parse_nightly_version(NIGHTLY_VERSION) - print(f"Parsed date: {date_str}") - pin_ref = get_commit_hash_for_nightly(date_str) - else: - # For test/release, pin to the branch name so CI picks up - # cherry-picks / security patches as they land on the branch. - pin_ref = torch_branch() - print(f"Pin ref: {pin_ref}") + # Read NIGHTLY_VERSION from torch_pin.py + nightly_version = get_torch_nightly_version() + print(f"Found NIGHTLY_VERSION: {nightly_version}") + + # Parse to date string + date_str = parse_nightly_version(nightly_version) + print(f"Parsed date: {date_str}") + + # Fetch commit hash from PyTorch nightly branch + commit_hash = get_commit_hash_for_nightly(date_str) + print(f"Found commit hash: {commit_hash}") # Update the pin file - update_pytorch_pin(pin_ref) + update_pytorch_pin(commit_hash) - # Sync c10 directories from PyTorch (ref param accepts branches too) - sync_c10_directories(pin_ref) + # Sync c10 directories from PyTorch + sync_c10_directories(commit_hash) - print("\nāœ… Successfully updated PyTorch pin and synced c10 directories!") + print( + "\nāœ… Successfully updated PyTorch commit pin and synced c10 directories!" + ) except Exception as e: print(f"Error: {e}", file=sys.stderr) diff --git a/.github/workflows/weekly-pytorch-pin-bump.yml b/.github/workflows/weekly-pytorch-pin-bump.yml index ba8f48505d5..30579c77701 100644 --- a/.github/workflows/weekly-pytorch-pin-bump.yml +++ b/.github/workflows/weekly-pytorch-pin-bump.yml @@ -22,46 +22,29 @@ jobs: with: python-version: '3.11' - - name: Check torch_pin channel - id: channel - run: | - CHANNEL=$(python -c "from torch_pin import CHANNEL; print(CHANNEL)") - echo "channel=${CHANNEL}" >> "$GITHUB_OUTPUT" - if [ "${CHANNEL}" != "nightly" ]; then - echo "torch_pin.py CHANNEL is '${CHANNEL}'; weekly nightly bump only runs when CHANNEL == 'nightly'." - fi - - name: Determine nightly version - if: steps.channel.outputs.channel == 'nightly' id: nightly run: | NIGHTLY_DATE=$(date -u -d 'yesterday' '+%Y%m%d') NIGHTLY_VERSION="dev${NIGHTLY_DATE}" echo "version=${NIGHTLY_VERSION}" >> "$GITHUB_OUTPUT" + - name: Read current TORCH_VERSION + id: torch + run: | + TORCH_VERSION=$(python -c "exec(open('torch_pin.py').read()); print(TORCH_VERSION)") + echo "version=${TORCH_VERSION}" >> "$GITHUB_OUTPUT" + - name: Update torch_pin.py with new NIGHTLY_VERSION - if: steps.channel.outputs.channel == 'nightly' - env: - NIGHTLY_VERSION: ${{ steps.nightly.outputs.version }} run: | - python - <<'PY' - import os, pathlib, re - p = pathlib.Path('torch_pin.py') - p.write_text(re.sub( - r'^NIGHTLY_VERSION\s*=\s*".*"$', - f'NIGHTLY_VERSION = "{os.environ["NIGHTLY_VERSION"]}"', - p.read_text(), - count=1, - flags=re.MULTILINE, - )) - PY + printf 'TORCH_VERSION = "%s"\nNIGHTLY_VERSION = "%s"\n' \ + "${{ steps.torch.outputs.version }}" \ + "${{ steps.nightly.outputs.version }}" > torch_pin.py - name: Run pin bump script - if: steps.channel.outputs.channel == 'nightly' run: python .github/scripts/update_pytorch_pin.py - name: Create branch and PR - if: steps.channel.outputs.channel == 'nightly' env: GH_TOKEN: ${{ secrets.UPDATEBOT_TOKEN }} run: | diff --git a/examples/models/moshi/mimi/install_requirements.sh b/examples/models/moshi/mimi/install_requirements.sh index eb8fe96ed05..9fc12f64bc9 100755 --- a/examples/models/moshi/mimi/install_requirements.sh +++ b/examples/models/moshi/mimi/install_requirements.sh @@ -7,16 +7,10 @@ set -x -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -# torch_pin lives at the executorch repo root. -cd "$SCRIPT_DIR/../../../.." - -TORCHCODEC_PKG=$(python -c "from torch_pin import torchcodec_spec; print(torchcodec_spec())") -TORCHCODEC_INDEX=$(python -c "from torch_pin import torch_index_url_base; print(torch_index_url_base())") - sudo apt install ffmpeg -y -pip install "$TORCHCODEC_PKG" --extra-index-url "${TORCHCODEC_INDEX}/cpu" +pip install torchcodec==0.11.0 --extra-index-url https://download.pytorch.org/whl/test/cpu pip install moshi==0.2.11 pip install bitsandbytes soundfile einops # Run llama2/install requirements for torchao deps +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) bash "$SCRIPT_DIR"/../../llama/install_requirements.sh diff --git a/install_executorch.py b/install_executorch.py index d305a06bd28..140a1163020 100644 --- a/install_executorch.py +++ b/install_executorch.py @@ -174,11 +174,7 @@ def _parse_args() -> argparse.Namespace: parser.add_argument( "--use-pt-pinned-commit", action="store_true", - help="install plain `torch` (whatever pip resolves by default; CI " - "uses this when torch is already built from source against the " - "pinned ref in pytorch.txt). Without this flag, install the specific " - "pinned version from the channel selected in torch_pin.py " - "(nightly / test / release).", + help="build from the pinned PyTorch commit instead of nightly", ) parser.add_argument( "--editable", @@ -221,14 +217,13 @@ def main(args): return check_and_update_submodules() - # By default install the specific pinned version from the channel selected - # in torch_pin.py. With --use-pt-pinned-commit, install plain `torch` (pip's - # default resolution); CI uses this when torch is already built from source - # against the pinned ref in pytorch.txt. - install_pinned_version = not args.use_pt_pinned_commit + # This option is used in CI to make sure that PyTorch build from the pinned commit + # is used instead of nightly. CI jobs wouldn't be able to catch regression from the + # latest PT commit otherwise + use_pytorch_nightly = not args.use_pt_pinned_commit # Step 1: Install core dependencies first - install_requirements(install_pinned_version) + install_requirements(use_pytorch_nightly) # Step 2: Install core package package_spec = "." @@ -253,7 +248,7 @@ def main(args): # Step 3: Extra (optional) packages that is only useful for running examples. if not args.minimal: - install_optional_example_requirements(install_pinned_version) + install_optional_example_requirements(use_pytorch_nightly) if __name__ == "__main__": diff --git a/install_requirements.py b/install_requirements.py index 1e8ab5c2d6f..b30068cbdb8 100644 --- a/install_requirements.py +++ b/install_requirements.py @@ -12,18 +12,9 @@ from install_utils import determine_torch_url, is_intel_mac_os, python_is_compatible -from torch_pin import ( - CHANNEL, - torch_index_url_base, - torch_spec, - torchaudio_spec, - torchvision_spec, -) - -# Only RC wheels at /whl/test/ get re-uploaded under the same version, so -# pip's local cache can serve stale content. Nightly and release wheels are -# immutable per their identifier. -_NO_CACHE_DIR_FLAG = ["--no-cache-dir"] if CHANNEL == "test" else [] +# The pip repository that hosts nightly torch packages. +# This will be dynamically set based on CUDA availability and CUDA backend enabled/disabled. +TORCH_URL_BASE = "https://download.pytorch.org/whl/test" # Since ExecuTorch often uses main-branch features of pytorch, only the nightly # pip versions will have the required features. @@ -32,18 +23,17 @@ # NIGHTLY_VERSION, you should re-run this script to install the necessary # package versions. # -# NOTE: If you change torch_pin.py, the pre-commit hook runs -# .github/scripts/update_pytorch_pin.py to refresh -# .ci/docker/ci_commit_pins/pytorch.txt and the c10 grafted headers. -# If you bypass the hook, run that script manually. +# NOTE: If you're changing, make the corresponding change in .ci/docker/ci_commit_pins/pytorch.txt +# by picking the hash from the same date in +# https://hud.pytorch.org/hud/pytorch/pytorch/nightly/ @lint-ignore # # NOTE: If you're changing, make the corresponding supported CUDA versions in # SUPPORTED_CUDA_VERSIONS in install_utils.py if needed. -def install_requirements(install_pinned_version): - # No prebuilt wheels are available for Intel macOS, regardless of channel. - if install_pinned_version and is_intel_mac_os(): +def install_requirements(use_pytorch_nightly): + # Skip pip install on Intel macOS if using nightly. + if use_pytorch_nightly and is_intel_mac_os(): print( "ERROR: Prebuilt PyTorch wheels are no longer available for Intel-based macOS.\n" "Please build from source by following https://docs.pytorch.org/executorch/main/using-executorch-building-from-source.html", @@ -52,26 +42,25 @@ def install_requirements(install_pinned_version): sys.exit(1) # Determine the appropriate PyTorch URL based on CUDA delegate status - torch_url = determine_torch_url(torch_index_url_base()) + torch_url = determine_torch_url(TORCH_URL_BASE) # pip packages needed by exir. TORCH_PACKAGE = [ - # Default: install the specific pinned version from the channel selected - # in torch_pin.py. With --use-pt-pinned-commit, pass plain "torch" and - # let pip resolve its default (CI's source-build is already installed). - (torch_spec() if install_pinned_version else "torch"), + # Setting use_pytorch_nightly to false to test the pinned PyTorch commit. Note + # that we don't need to set any version number there because they have already + # been installed on CI before this step, so pip won't reinstall them + ("torch==2.11.0" if use_pytorch_nightly else "torch"), ] # Install the requirements for core ExecuTorch package. - # `--extra-index-url` tells pip to look for package versions on the - # provided URL if they aren't available on the default URL. + # `--extra-index-url` tells pip to look for package + # versions on the provided URL if they aren't available on the default URL. subprocess.run( [ sys.executable, "-m", "pip", "install", - *_NO_CACHE_DIR_FLAG, "-r", "requirements-dev.txt", *TORCH_PACKAGE, @@ -117,14 +106,14 @@ def install_requirements(install_pinned_version): ) -def install_optional_example_requirements(install_pinned_version): +def install_optional_example_requirements(use_pytorch_nightly): # Determine the appropriate PyTorch URL based on CUDA delegate status - torch_url = determine_torch_url(torch_index_url_base()) + torch_url = determine_torch_url(TORCH_URL_BASE) print("Installing torch domain libraries") DOMAIN_LIBRARIES = [ - (torchvision_spec() if install_pinned_version else "torchvision"), - (torchaudio_spec() if install_pinned_version else "torchaudio"), + ("torchvision==0.26.0" if use_pytorch_nightly else "torchvision"), + ("torchaudio==2.11.0" if use_pytorch_nightly else "torchaudio"), ] # Then install domain libraries subprocess.run( @@ -133,7 +122,6 @@ def install_optional_example_requirements(install_pinned_version): "-m", "pip", "install", - *_NO_CACHE_DIR_FLAG, *DOMAIN_LIBRARIES, "--extra-index-url", torch_url, @@ -164,11 +152,7 @@ def main(args): parser.add_argument( "--use-pt-pinned-commit", action="store_true", - help="install plain `torch` (whatever pip resolves by default; CI " - "uses this when torch is already built from source against the " - "pinned ref in pytorch.txt). Without this flag, install the specific " - "pinned version from the channel selected in torch_pin.py " - "(nightly / test / release).", + help="build from the pinned PyTorch commit instead of nightly", ) parser.add_argument( "--example", @@ -176,10 +160,10 @@ def main(args): help="Also installs required packages for running example scripts.", ) args = parser.parse_args(args) - install_pinned_version = not bool(args.use_pt_pinned_commit) - install_requirements(install_pinned_version) + use_pytorch_nightly = not bool(args.use_pt_pinned_commit) + install_requirements(use_pytorch_nightly) if args.example: - install_optional_example_requirements(install_pinned_version) + install_optional_example_requirements(use_pytorch_nightly) if __name__ == "__main__": diff --git a/torch_pin.py b/torch_pin.py index 856a67c1990..3575d9a376d 100644 --- a/torch_pin.py +++ b/torch_pin.py @@ -1,59 +1,2 @@ -# CHANNEL selects the wheel source for torch and its domain libraries. -# "nightly" ā€” dev builds from /whl/nightly. NIGHTLY_VERSION is appended to -# every package spec, and CI source-builds pytorch from the -# pinned SHA in pytorch.txt to catch upstream regressions. -# "test" ā€” release candidates from /whl/test. -# "release" ā€” stable releases from /whl. -# For "test" and "release", NIGHTLY_VERSION is ignored and CI installs the -# published wheels directly (no source build). -# -# Example ā€” pinning to a 2.12 release candidate when nightly is broken: -# 1. Set CHANNEL = "test". -# 2. Set the four version constants to the RC's major.minor.patch -# (look up matching versions on https://download.pytorch.org/whl/test/). -# 3. Re-run install_requirements.sh; commit. The pre-commit hook calls -# .github/scripts/update_pytorch_pin.py, which writes torch_branch() -# (e.g. "release/2.12") into .ci/docker/ci_commit_pins/pytorch.txt and -# re-syncs grafted c10 headers. -CHANNEL = "test" - TORCH_VERSION = "2.11.0" -TORCHAUDIO_VERSION = "2.11.0" -TORCHCODEC_VERSION = "0.11.0" -TORCHVISION_VERSION = "0.26.0" - -NIGHTLY_VERSION = "dev20260318" - - -def _spec(name: str, version: str) -> str: - if CHANNEL == "nightly": - return f"{name}=={version}.{NIGHTLY_VERSION}" - return f"{name}=={version}" - - -def torch_spec() -> str: - return _spec("torch", TORCH_VERSION) - - -def torchaudio_spec() -> str: - return _spec("torchaudio", TORCHAUDIO_VERSION) - - -def torchcodec_spec() -> str: - return _spec("torchcodec", TORCHCODEC_VERSION) - - -def torchvision_spec() -> str: - return _spec("torchvision", TORCHVISION_VERSION) - - -def torch_index_url_base() -> str: - if CHANNEL == "release": - return "https://download.pytorch.org/whl" - return f"https://download.pytorch.org/whl/{CHANNEL}" - - -def torch_branch() -> str: - # PyTorch uses "release/M.N" branches; derive from the pinned version. - # Used by update_pytorch_pin.py to write into pytorch.txt for test/release. - return f"release/{TORCH_VERSION.rsplit('.', 1)[0]}" +# NIGHTLY_VERSION = "dev20260318" Temporarily pinning to stable release candidate. Revert https://github.com/pytorch/executorch/pull/18287