Skip to content

Commit 477d278

Browse files
GasoonjiaGasoonjia
authored
Fix CUDA CI: HF writes fail on read-only /mnt/hf_cache (HF_TOKEN + writable HF_HOME) (#20782)
## Summary Since **pytorch/pytorch#188654** (landed **2026-07-03**), the OSDC runner fleet mounts a shared, **read-only** HuggingFace cache at `/mnt/hf_cache` and defaults `HF_HOME` to it (to share a pre-seeded, S3-backed model cache). executorch's CUDA jobs were never adapted, so every HF **write** now lands on the read-only mount and fails: ``` OSError: [Errno 30] Read-only file system: '/mnt/hf_cache/...' ``` This broke the CUDA CI (`export-model-cuda-artifact`, `benchmark-cuda`, `export-model-cuda-windows-artifact`, `test-cuda-pybind`) starting Jul 3. pytorch/pytorch updated its own workflows in the same PR (use the read-only mount only in `TRANSFORMERS_OFFLINE` mode, otherwise download into a writable `${RUNNER_TEMP}/hf_cache`); this PR does the equivalent for executorch.
1 parent ce7ff11 commit 477d278

3 files changed

Lines changed: 41 additions & 8 deletions

File tree

‎.github/workflows/cuda-perf.yml‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -144,6 +144,13 @@ jobs:
144144
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
145145
script: |
146146
set -eux
147+
148+
# OSDC mounts HF_HOME read-only at /mnt/hf_cache; redirect to a writable dir
149+
# (RUNNER_TEMP, or /tmp when RUNNER_TEMP isn't writable inside the container).
150+
export HF_HOME="${RUNNER_TEMP:-/tmp}/hf_cache"
151+
mkdir -p "${HF_HOME}" 2>/dev/null || export HF_HOME=/tmp/hf_cache
152+
mkdir -p "${HF_HOME}"
153+
147154
echo "::group::Setup ExecuTorch"
148155
# OSDC runners can't reach the public PyPI CDN that download.pytorch.org's
149156
# transitive deps resolve to. Pre-install torch's pure-python deps from the
@@ -160,8 +167,7 @@ jobs:
160167
161168
echo "::group::Setup Huggingface"
162169
pip install -U "huggingface_hub[cli]>=1.2.1,<2.0" accelerate "optimum~=2.0.0" "transformers==5.0.0rc1"
163-
HF_AUTH_TOKEN="$(printf '%s' "$SECRET_EXECUTORCH_HF_TOKEN" | tr -d '\r\n')"
164-
hf auth login --token "$HF_AUTH_TOKEN"
170+
export HF_TOKEN="$(printf '%s' "$SECRET_EXECUTORCH_HF_TOKEN" | tr -d '\r\n')"
165171
OPTIMUM_ET_VERSION=$(cat .ci/docker/ci_commit_pins/optimum-executorch.txt)
166172
pip install --no-deps git+https://github.com/huggingface/optimum-executorch.git@${OPTIMUM_ET_VERSION}
167173
echo "::endgroup::"
@@ -219,6 +225,13 @@ jobs:
219225
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
220226
script: |
221227
set -eux
228+
229+
# OSDC mounts HF_HOME read-only at /mnt/hf_cache; redirect to a writable dir
230+
# (RUNNER_TEMP, or /tmp when RUNNER_TEMP isn't writable inside the container).
231+
export HF_HOME="${RUNNER_TEMP:-/tmp}/hf_cache"
232+
mkdir -p "${HF_HOME}" 2>/dev/null || export HF_HOME=/tmp/hf_cache
233+
mkdir -p "${HF_HOME}"
234+
222235
echo "::group::Setup environment"
223236
# OSDC runners can't reach the public PyPI CDN that download.pytorch.org's
224237
# transitive deps resolve to. Pre-install torch's pure-python deps from the

‎.github/workflows/cuda-windows.yml‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,12 @@ jobs:
9999
script: |
100100
set -eux
101101
102+
# OSDC mounts HF_HOME read-only at /mnt/hf_cache; redirect to a writable dir
103+
# (RUNNER_TEMP, or /tmp when RUNNER_TEMP isn't writable inside the container).
104+
export HF_HOME="${RUNNER_TEMP:-/tmp}/hf_cache"
105+
mkdir -p "${HF_HOME}" 2>/dev/null || export HF_HOME=/tmp/hf_cache
106+
mkdir -p "${HF_HOME}"
107+
102108
echo "::group::Fix libstdc++ GLIBCXX version"
103109
# The executorch pybindings require GLIBCXX_3.4.30 which conda's libstdc++ doesn't have.
104110
# Replace conda's libstdc++ with the system version to fix ImportError.
@@ -126,8 +132,7 @@ jobs:
126132
if [ "${{ matrix.model_name }}" != "dinov2-small-imagenet1k-1-layer" ]; then
127133
echo "::group::Setup Huggingface"
128134
pip install -U "huggingface_hub[cli]>=1.2.1,<2.0" accelerate "optimum~=2.0.0" "transformers==5.0.0rc1"
129-
HF_AUTH_TOKEN="$(printf '%s' "$SECRET_EXECUTORCH_HF_TOKEN" | tr -d '\r\n')"
130-
hf auth login --token "$HF_AUTH_TOKEN"
135+
export HF_TOKEN="$(printf '%s' "$SECRET_EXECUTORCH_HF_TOKEN" | tr -d '\r\n')"
131136
OPTIMUM_ET_VERSION=$(cat .ci/docker/ci_commit_pins/optimum-executorch.txt)
132137
pip install --no-deps git+https://github.com/huggingface/optimum-executorch.git@${OPTIMUM_ET_VERSION}
133138
echo "::endgroup::"

‎.github/workflows/cuda.yml‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -388,6 +388,12 @@ jobs:
388388
script: |
389389
set -eux
390390
391+
# OSDC mounts HF_HOME read-only at /mnt/hf_cache; redirect to a writable dir
392+
# (RUNNER_TEMP, or /tmp when RUNNER_TEMP isn't writable inside the container).
393+
export HF_HOME="${RUNNER_TEMP:-/tmp}/hf_cache"
394+
mkdir -p "${HF_HOME}" 2>/dev/null || export HF_HOME=/tmp/hf_cache
395+
mkdir -p "${HF_HOME}"
396+
391397
echo "::group::Setup ExecuTorch"
392398
# OSDC runners can't reach the public PyPI CDN that download.pytorch.org's
393399
# transitive deps resolve to. Pre-install torch's pure-python deps from the
@@ -406,8 +412,7 @@ jobs:
406412
if [ "${{ matrix.model.name }}" != "parakeet-tdt" ] && [ "${{ matrix.model.name }}" != "dinov2-small-imagenet1k-1-layer" ]; then
407413
echo "::group::Setup Huggingface"
408414
pip install -U "huggingface_hub[cli]>=1.2.1,<2.0" accelerate "optimum~=2.0.0" "transformers==5.0.0rc1"
409-
HF_AUTH_TOKEN="$(printf '%s' "$SECRET_EXECUTORCH_HF_TOKEN" | tr -d '\r\n')"
410-
hf auth login --token "$HF_AUTH_TOKEN"
415+
export HF_TOKEN="$(printf '%s' "$SECRET_EXECUTORCH_HF_TOKEN" | tr -d '\r\n')"
411416
OPTIMUM_ET_VERSION=$(cat .ci/docker/ci_commit_pins/optimum-executorch.txt)
412417
pip install --no-deps git+https://github.com/huggingface/optimum-executorch.git@${OPTIMUM_ET_VERSION}
413418
echo "::endgroup::"
@@ -554,6 +559,11 @@ jobs:
554559
# fsspec is pinned to satisfy datasets' fsspec[http]<=2025.3.0 so the later
555560
# examples install doesn't try to downgrade it from the public CDN.
556561
pip install filelock typing-extensions "setuptools<82" sympy networkx jinja2 "fsspec[http]<=2025.3.0" numpy pillow
562+
# OSDC mounts HF_HOME read-only at /mnt/hf_cache; redirect to a writable dir
563+
# (RUNNER_TEMP, or /tmp when RUNNER_TEMP isn't writable inside the container).
564+
export HF_HOME="${RUNNER_TEMP:-/tmp}/hf_cache"
565+
mkdir -p "${HF_HOME}" 2>/dev/null || export HF_HOME=/tmp/hf_cache
566+
mkdir -p "${HF_HOME}"
557567
source .ci/scripts/test_model_e2e.sh cuda "${{ matrix.model.repo }}/${{ matrix.model.name }}" "${{ matrix.quant }}" "${RUNNER_ARTIFACT_DIR}"
558568
559569
test-cuda-pybind:
@@ -605,6 +615,12 @@ jobs:
605615
script: |
606616
set -eux
607617
618+
# OSDC mounts HF_HOME read-only at /mnt/hf_cache; redirect to a writable dir
619+
# (RUNNER_TEMP, or /tmp when RUNNER_TEMP isn't writable inside the container).
620+
export HF_HOME="${RUNNER_TEMP:-/tmp}/hf_cache"
621+
mkdir -p "${HF_HOME}" 2>/dev/null || export HF_HOME=/tmp/hf_cache
622+
mkdir -p "${HF_HOME}"
623+
608624
echo "::group::Setup ExecuTorch"
609625
# Disable MKL to avoid duplicate target error when conda has multiple MKL installations
610626
export USE_MKL=OFF
@@ -626,8 +642,7 @@ jobs:
626642
627643
echo "::group::Setup Huggingface"
628644
pip install -U "huggingface_hub[cli]>=1.2.1,<2.0"
629-
HF_AUTH_TOKEN="$(printf '%s' "$SECRET_EXECUTORCH_HF_TOKEN" | tr -d '\r\n')"
630-
hf auth login --token "$HF_AUTH_TOKEN"
645+
export HF_TOKEN="$(printf '%s' "$SECRET_EXECUTORCH_HF_TOKEN" | tr -d '\r\n')"
631646
echo "::endgroup::"
632647
633648
echo "::group::Install optimum-executorch"

0 commit comments

Comments
 (0)