Skip to content

Commit c951e79

Browse files
authored
Build Native wheels in ExecuTorch release CI
Summary: Add dedicated Linux x86_64 and aarch64 wheel workflows for `executorch-native`, sharing ExecuTorch's release/nightly triggers, Python versions, and `BUILD_VERSION`. Native gets its own workflows because the reusable PyTorch workflow keys its concurrency group and artifact names by repository, so two packages from one repository cannot share a workflow run. Build and repair the standalone GGML/Vulkan wheel, then export PTN fixtures and test GGML plus the Python binding suite in a fresh environment containing only Native and NumPy. Nova's smoke test is disabled for Native because it requires a pinned torch dependency; the post-build script performs a stricter clean-environment test instead. The Native wheels are not uploaded yet: the `executorch-native` workflows pass a non-`push` trigger event to the reusable workflow, so repaired wheels stay as workflow artifacts until the package is registered for PyPI, the nightly index, and release promotion. This is a separate commit above the Native wheel and Python bindings changes. Authored with AI assistance from Codex. Reviewed By: SS-JIA Differential Revision: D126830796 Pull Request resolved: #24255
1 parent cfc04be commit c951e79

12 files changed

Lines changed: 542 additions & 4 deletions
Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
# All rights reserved.
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
import importlib.util
8+
import os
9+
import unittest
10+
from pathlib import Path
11+
from unittest.mock import patch
12+
13+
14+
ROOT = Path(__file__).resolve().parents[3]
15+
if ROOT.name == "oss":
16+
ROOT = ROOT.parent
17+
SPEC = importlib.util.spec_from_file_location(
18+
"wheel_version", ROOT / "backends/native/python/wheel_version.py"
19+
)
20+
assert SPEC is not None and SPEC.loader is not None
21+
VERSION = importlib.util.module_from_spec(SPEC)
22+
SPEC.loader.exec_module(VERSION)
23+
24+
25+
class NativeWheelVersionTest(unittest.TestCase):
26+
def test_invalid_metadata_configuration(self) -> None:
27+
for field, settings in (
28+
("dependencies", {}),
29+
("version", {"input": "version.txt"}),
30+
):
31+
with self.subTest(field=field, settings=settings):
32+
with self.assertRaisesRegex(
33+
ValueError, "Only version metadata without settings is supported"
34+
):
35+
VERSION.dynamic_metadata(field, settings)
36+
37+
def test_build_version_overrides_source_version(self) -> None:
38+
for version in ("1.6.0", "1.6.0rc2", "1.7.0.dev20261021"):
39+
with self.subTest(version=version), patch.dict(
40+
os.environ, {"BUILD_VERSION": f" {version}\n"}
41+
):
42+
self.assertEqual(VERSION.dynamic_metadata("version", {}), version)
43+
44+
def test_local_build_uses_version_file(self) -> None:
45+
for version in ("", " \n"):
46+
with self.subTest(version=version), patch.dict(
47+
os.environ, {"BUILD_VERSION": version}
48+
):
49+
self.assertEqual(
50+
VERSION.dynamic_metadata("version", {}),
51+
(ROOT / "version.txt").read_text().strip(),
52+
)
53+
54+
55+
if __name__ == "__main__":
56+
unittest.main()

‎.ci/scripts/wheel/envvar_native.sh‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
# All rights reserved.
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
export NATIVE_WHEEL_ROOT="${GITHUB_WORKSPACE}/${REPOSITORY}"
8+
export NATIVE_WHEEL_DEPS="${GITHUB_WORKSPACE}/native-wheel-deps"
9+
export CMAKE_PREFIX_PATH="${NATIVE_WHEEL_DEPS}/libzip:${NATIVE_WHEEL_DEPS}:${NATIVE_WHEEL_DEPS}/shaderc${CMAKE_PREFIX_PATH:+:${CMAKE_PREFIX_PATH}}"
10+
export CMAKE_BUILD_PARALLEL_LEVEL="${CMAKE_BUILD_PARALLEL_LEVEL:-4}"
11+
export FLATC_EXECUTABLE="${NATIVE_WHEEL_DEPS}/flatbuffers/flatc"
12+
export CMAKE_ARGS="${CMAKE_ARGS:-} -DFLATC_EXECUTABLE=${FLATC_EXECUTABLE} -DGLSLC_PATH=${NATIVE_WHEEL_DEPS}/shaderc/bin/glslc -DEXECUTORCH_NATIVE_EXTRA_LICENSES=${NATIVE_WHEEL_DEPS}/libzip-1.11.4/LICENSE"
13+
export PIP_EXTRA_INDEX_URL="${PIP_EXTRA_INDEX_URL:+${PIP_EXTRA_INDEX_URL} }https://pypi.org/simple"
14+
15+
# Nova sources this hook before invoking setup.py, allowing a subdirectory build.
16+
if [[ "$(pwd)" == "${NATIVE_WHEEL_ROOT}" ]]; then
17+
cd "${NATIVE_WHEEL_ROOT}/backends/native/python" || exit 1
18+
fi

‎.ci/scripts/wheel/export_native.py‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
# All rights reserved.
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
"""Generate Native wheel fixtures with the exporter from this checkout."""
8+
9+
import argparse
10+
from pathlib import Path
11+
12+
import numpy as np
13+
import torch
14+
from executorch.exir.native import to_native
15+
16+
17+
class Add(torch.nn.Module):
18+
def forward(self, value: torch.Tensor) -> torch.Tensor:
19+
return value + value
20+
21+
22+
def main() -> None:
23+
parser = argparse.ArgumentParser(description=__doc__)
24+
parser.add_argument("--output", type=Path, required=True)
25+
args = parser.parse_args()
26+
args.output.mkdir(parents=True, exist_ok=True)
27+
torch.manual_seed(0)
28+
conv_bn = torch.nn.Sequential(
29+
torch.nn.Conv2d(3, 4, kernel_size=3, padding=1),
30+
torch.nn.BatchNorm2d(4),
31+
torch.nn.ReLU(),
32+
)
33+
with torch.no_grad():
34+
conv_bn[1].running_mean.copy_(torch.linspace(-1, 1, 4))
35+
conv_bn[1].running_var.copy_(torch.linspace(0.5, 1.5, 4))
36+
conv_bn[1].weight.copy_(torch.linspace(0.8, 1.2, 4))
37+
conv_bn[1].bias.copy_(torch.linspace(-0.2, 0.2, 4))
38+
models = {
39+
"add": (Add(), torch.arange(1, 7, dtype=torch.float32).reshape(1, 6)),
40+
"conv_bn": (
41+
conv_bn,
42+
torch.randn(1, 3, 8, 8),
43+
),
44+
}
45+
for name, (model, value) in models.items():
46+
model.eval()
47+
with torch.no_grad():
48+
expected = model(value)
49+
to_native(torch.export.export(model, (value,))).save(
50+
str(args.output / f"{name}.ptn")
51+
)
52+
np.savez(
53+
args.output / f"{name}.npz",
54+
input=value.numpy(),
55+
expected=expected.numpy(),
56+
)
57+
58+
59+
if __name__ == "__main__":
60+
main()
Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
set -euxo pipefail
9+
10+
cd "${NATIVE_WHEEL_ROOT}/backends/native/python"
11+
wheels=(dist/executorch_native-*.whl)
12+
[[ ${#wheels[@]} == 1 && -f "${wheels[0]}" ]]
13+
python -m auditwheel repair "${wheels[0]}" --wheel-dir repaired
14+
rm "${wheels[0]}"
15+
mv repaired/*.whl dist/
16+
wheels=(dist/executorch_native-*.whl)
17+
[[ ${#wheels[@]} == 1 && -f "${wheels[0]}" ]]
18+
19+
# Only the export environment needs PyTorch and the source checkout.
20+
torchao_version=$(cd "${NATIVE_WHEEL_ROOT}" && python -c \
21+
'from install_requirements import TORCHAO_NIGHTLY_VERSION; print(TORCHAO_NIGHTLY_VERSION)')
22+
python -m pip install torch "torchao==${torchao_version}" numpy flatbuffers packaging pyyaml ruamel.yaml tabulate \
23+
--index-url https://download.pytorch.org/whl/test/cpu \
24+
--extra-index-url https://download.pytorch.org/whl/nightly/cpu \
25+
--extra-index-url https://pypi.org/simple
26+
mkdir -p "${NATIVE_WHEEL_DEPS}/export-source"
27+
ln -s "${NATIVE_WHEEL_ROOT}" "${NATIVE_WHEEL_DEPS}/export-source/executorch"
28+
PYTHONPATH="${NATIVE_WHEEL_DEPS}/export-source" python \
29+
"${NATIVE_WHEEL_ROOT}/.ci/scripts/wheel/export_native.py" \
30+
--output "${NATIVE_WHEEL_DEPS}/fixtures"
31+
python "${NATIVE_WHEEL_ROOT}/.ci/scripts/wheel/test_native.py" \
32+
--wheel "${wheels[0]}" --source "${NATIVE_WHEEL_ROOT}" \
33+
--fixtures "${NATIVE_WHEEL_DEPS}/fixtures"
34+
35+
# Nova's artifact steps also consume dist at the repository root.
36+
mkdir -p "${NATIVE_WHEEL_ROOT}/dist"
37+
cp "${wheels[0]}" "${NATIVE_WHEEL_ROOT}/dist/"
Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
set -euxo pipefail
9+
10+
python -m pip install build 'setuptools>=77' 'cmake>=3.26,<4' ninja auditwheel patchelf
11+
python -m unittest discover -s "${NATIVE_WHEEL_ROOT}/.ci/scripts/tests" \
12+
-p test_native_wheel_version.py
13+
mkdir -p "${NATIVE_WHEEL_DEPS}"
14+
conda create -y -p "${NATIVE_WHEEL_DEPS}/shaderc" -c conda-forge shaderc zlib
15+
16+
curl --fail --location --retry 3 https://libzip.org/download/libzip-1.11.4.tar.xz \
17+
--output "${NATIVE_WHEEL_DEPS}/libzip.tar.xz"
18+
tar -xJf "${NATIVE_WHEEL_DEPS}/libzip.tar.xz" -C "${NATIVE_WHEEL_DEPS}"
19+
cmake -S "${NATIVE_WHEEL_DEPS}/libzip-1.11.4" -B "${NATIVE_WHEEL_DEPS}/libzip-build" \
20+
-DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX="${NATIVE_WHEEL_DEPS}/libzip" \
21+
-DCMAKE_POSITION_INDEPENDENT_CODE=ON -DBUILD_SHARED_LIBS=OFF \
22+
-DBUILD_TOOLS=OFF -DBUILD_REGRESS=OFF -DBUILD_EXAMPLES=OFF -DBUILD_DOC=OFF \
23+
-DENABLE_BZIP2=OFF -DENABLE_LZMA=OFF -DENABLE_ZSTD=OFF \
24+
-DENABLE_OPENSSL=OFF -DENABLE_GNUTLS=OFF -DENABLE_MBEDTLS=OFF
25+
cmake --build "${NATIVE_WHEEL_DEPS}/libzip-build" --target install --parallel
26+
27+
cmake -S "${NATIVE_WHEEL_ROOT}/third-party/flatbuffers" \
28+
-B "${NATIVE_WHEEL_DEPS}/flatbuffers" \
29+
-DCMAKE_BUILD_TYPE=Release -DFLATBUFFERS_BUILD_TESTS=OFF
30+
cmake --build "${NATIVE_WHEEL_DEPS}/flatbuffers" --target flatc --parallel

‎.ci/scripts/wheel/test_native.py‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
# All rights reserved.
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
"""Install a repaired Native wheel into a clean environment and test GGML."""
8+
9+
import argparse
10+
import os
11+
import shutil
12+
import subprocess
13+
import tempfile
14+
import venv
15+
from pathlib import Path
16+
17+
18+
def main() -> None:
19+
parser = argparse.ArgumentParser(description=__doc__)
20+
parser.add_argument("--wheel", type=Path, required=True)
21+
parser.add_argument("--source", type=Path, required=True)
22+
parser.add_argument("--fixtures", type=Path, required=True)
23+
args = parser.parse_args()
24+
wheel = args.wheel.resolve()
25+
source = args.source.resolve()
26+
fixtures = args.fixtures.resolve()
27+
with tempfile.TemporaryDirectory(prefix="native-wheel-test-") as directory:
28+
work = Path(directory)
29+
venv.create(work / "venv", with_pip=True)
30+
python = work / "venv/bin/python"
31+
env = os.environ.copy()
32+
for name in ("PYTHONPATH", "PYTHONHOME", "LD_LIBRARY_PATH", "LD_PRELOAD"):
33+
env.pop(name, None)
34+
env["PYTHONNOUSERSITE"] = "1"
35+
subprocess.run(
36+
[str(python), "-m", "pip", "install", str(wheel)],
37+
cwd=work,
38+
env=env,
39+
check=True,
40+
timeout=600,
41+
)
42+
subprocess.run(
43+
[str(python), "-m", "pip", "check"],
44+
cwd=work,
45+
env=env,
46+
check=True,
47+
timeout=60,
48+
)
49+
shutil.copytree(fixtures, work / "fixtures")
50+
shutil.copyfile(
51+
source / "backends/native/pybindings/test/test_bindings.py",
52+
work / "test_bindings.py",
53+
)
54+
subprocess.run(
55+
[
56+
str(python),
57+
"-I",
58+
"-c",
59+
"""
60+
import importlib.metadata
61+
import importlib.util
62+
from pathlib import Path
63+
import numpy as np
64+
from executorch_native import Model
65+
66+
for name in ('torch', 'executorch'):
67+
assert importlib.util.find_spec(name) is None, name
68+
installed = {d.metadata['Name'].lower() for d in importlib.metadata.distributions()}
69+
assert installed <= {'pip', 'setuptools', 'executorch-native', 'numpy'}, installed
70+
for name in ('add', 'conv_bn'):
71+
package = Path('fixtures') / f'{name}.ptn'
72+
with np.load(package.with_suffix('.npz')) as data:
73+
session = Model.load(package).create_session('ggml')
74+
(actual,) = session.run([data['input']])
75+
np.testing.assert_allclose(actual, data['expected'], rtol=1e-4, atol=1e-4)
76+
print(f'PASS: {package.name} with GGML')
77+
""",
78+
],
79+
cwd=work,
80+
env=env,
81+
check=True,
82+
timeout=120,
83+
)
84+
env["NATIVE_TEST_PACKAGE"] = str(work / "fixtures/add.ptn")
85+
env["NATIVE_TEST_ENGINE"] = "ggml"
86+
subprocess.run(
87+
[str(python), "-m", "unittest", "-v", "test_bindings"],
88+
cwd=work,
89+
env=env,
90+
check=True,
91+
timeout=300,
92+
)
93+
print(f"PASS: clean install of {wheel.name}")
94+
95+
96+
if __name__ == "__main__":
97+
main()
Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
# From https://github.com/pytorch/test-infra/wiki/Using-Nova-Reusable-Build-Workflows
2+
name: Build Aarch64 Linux Native Wheels
3+
4+
on:
5+
pull_request:
6+
paths:
7+
- .ci/**/*
8+
- .github/workflows/build-wheels-native-aarch64-linux.yml
9+
- '**/CMakeLists.txt'
10+
- backends/native/**/*
11+
- backends/vulkan/**/*
12+
- backends/transforms/**/*
13+
- exir/**/*
14+
- schema/**/*
15+
- examples/**/*
16+
- version.txt
17+
- .gitmodules
18+
- pyproject.toml
19+
- setup.py
20+
- install_utils.py
21+
- tools/cmake/**/*
22+
push:
23+
branches:
24+
- nightly
25+
- release/*
26+
tags:
27+
# NOTE: Binary build pipelines should only get triggered on release candidate builds
28+
# Release candidate tags look like: v1.11.0-rc1
29+
- v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+
30+
- ciflow/binaries/*
31+
workflow_dispatch:
32+
33+
concurrency:
34+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ github.ref_type == 'branch' && github.sha }}-${{ github.event_name == 'workflow_dispatch' }}
35+
cancel-in-progress: true
36+
37+
jobs:
38+
generate-matrix:
39+
uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main
40+
with:
41+
package-type: wheel
42+
os: linux-aarch64
43+
test-infra-repository: pytorch/test-infra
44+
test-infra-ref: main
45+
runner-fleet: osdc
46+
with-cuda: disabled
47+
with-rocm: disabled
48+
python-versions: '["3.11", "3.12", "3.13", "3.14", "3.15"]'
49+
50+
build:
51+
needs: generate-matrix
52+
permissions:
53+
id-token: write
54+
contents: read
55+
name: executorch-native
56+
uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main
57+
with:
58+
repository: pytorch/executorch
59+
ref: ""
60+
test-infra-repository: pytorch/test-infra
61+
test-infra-ref: main
62+
build-matrix: ${{ needs.generate-matrix.outputs.matrix }}
63+
submodules: recursive
64+
env-var-script: .ci/scripts/wheel/envvar_native.sh
65+
pre-script: ../../../.ci/scripts/wheel/pre_build_native.sh
66+
post-script: ../../../.ci/scripts/wheel/post_build_native.sh
67+
package-name: executorch-native
68+
# The post-build script installs the wheel in a clean environment and
69+
# tests it; Nova's smoke test requires a pinned torch dependency.
70+
run-smoke-test: false
71+
# Only "push" uploads; executorch-native is not published yet.
72+
trigger-event: development
73+
architecture: aarch64

0 commit comments

Comments
 (0)