Skip to content

Commit 8fd5caf

Browse files
shoumikhinJacobSzwejbka
shoumikhin
and
JacobSzwejbka
committed
Trim the minimal wheel's declared dependencies to the export-only set
The opt-in minimal wheel build (`EXECUTORCH_BUILD_MINIMAL=1`) previously trimmed the packaged files but not the declared dependencies. Because `pyproject.toml` `[project.dependencies]` is static, a normal `pip install` of the minimal wheel still pulled `coremltools`, `scikit-learn`, `pandas`, `hydra-core`, `omegaconf` and the test-only deps (`expecttest`, `hypothesis`, `kgb`, `parameterized`), which is most of the real install size. The CI smoke test only passed because it used `pip install --no-deps`. Make `dependencies` dynamic in `pyproject.toml` and compute `install_requires` in `setup.py` per build mode. The full wheel keeps the existing set unchanged; the minimal wheel declares the subset of that set which `executorch.exir` needs to export a `.pte`: `flatbuffers`, `numpy`, `packaging`, `pyyaml`, `ruamel.yaml`, `sympy`, `tabulate`, `typing-extensions`. `torch` stays consumer-provided and `mpmath` comes transitively via `sympy`. The minimal set is derived from the full set so version pins and markers cannot drift. Also make the minimal build skip the non-minimal cmake targets explicitly, drop `--no-deps` from the smoke test so dependency resolution is exercised, and assert the built wheel METADATA declares exactly the slim set and none of the heavy deps. Document the slim dependency set in `README-wheel.md`. Co-authored-by: Jacob Szwejbka <20804483+JacobSzwejbka@users.noreply.github.com>
1 parent 6322992 commit 8fd5caf

4 files changed

Lines changed: 178 additions & 59 deletions

File tree

‎.ci/scripts/test_minimal_wheel.sh‎

Lines changed: 72 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ WHEEL_FILE="$(find "${REPO_ROOT}/dist" -maxdepth 1 -name 'executorch-*.whl' | he
4040
test -n "${WHEEL_FILE}"
4141

4242
python - "${WHEEL_FILE}" <<'PY'
43+
import re
4344
import sys
4445
import zipfile
4546
@@ -66,27 +67,89 @@ extensions = [
6667
]
6768
if extensions:
6869
raise AssertionError(f"{wheel_file} unexpectedly contains extensions: {extensions}")
70+
71+
# The minimal wheel must declare only the slim runtime dependency set: none of
72+
# the heavy full-wheel deps, and all of the minimal ones.
73+
metadata_name = next(
74+
(name for name in names if name.endswith(".dist-info/METADATA")), None
75+
)
76+
if metadata_name is None:
77+
raise AssertionError(f"{wheel_file} has no METADATA")
78+
with zipfile.ZipFile(wheel_file) as wheel:
79+
metadata_text = wheel.read(metadata_name).decode("utf-8")
80+
81+
82+
def _dist_name(requirement):
83+
name = re.split(r"[ ;\[<>=!~(]", requirement.strip(), maxsplit=1)[0]
84+
return re.sub(r"[-_.]+", "-", name).lower()
85+
86+
87+
# Only the core (non-extra) Requires-Dist entries define what a plain
88+
# "pip install" pulls; ignore the optional extras (cortex_m, vgf, ...).
89+
declared = {
90+
_dist_name(line.split(":", 1)[1])
91+
for line in metadata_text.splitlines()
92+
if line.startswith("Requires-Dist:") and "extra==" not in line.replace(" ", "")
93+
}
94+
# Heavy / non-minimal deps that must never appear in the minimal wheel. mpmath
95+
# (pulled transitively via sympy) and torch (consumer-provided) are listed too so
96+
# re-adding them directly to the minimal set is caught even though both are absent
97+
# today.
98+
heavy = {
99+
"coremltools",
100+
"scikit-learn",
101+
"pandas",
102+
"hydra-core",
103+
"omegaconf",
104+
"expecttest",
105+
"hypothesis",
106+
"kgb",
107+
"parameterized",
108+
"pytorch-tokenizers",
109+
"mpmath",
110+
"torch",
111+
}
112+
leaked = sorted(declared & heavy)
113+
if leaked:
114+
raise AssertionError(f"{wheel_file} METADATA declares non-minimal deps: {leaked}")
115+
# Exact match: the minimal wheel must declare these and nothing else, so any
116+
# unexpected direct dependency (not just the enumerated heavy ones) is caught.
117+
expected = {
118+
"flatbuffers",
119+
"numpy",
120+
"packaging",
121+
"pyyaml",
122+
"ruamel-yaml",
123+
"sympy",
124+
"tabulate",
125+
"typing-extensions",
126+
}
127+
if declared != expected:
128+
raise AssertionError(
129+
f"{wheel_file} minimal deps mismatch: "
130+
f"unexpected={sorted(declared - expected)} missing={sorted(expected - declared)}"
131+
)
69132
PY
70133

71134
deactivate
72135

73136
"${PYTHON_EXECUTABLE}" -m venv "${TEST_VENV}"
74137
source "${TEST_VENV}/bin/activate"
75138
python -m pip install --upgrade pip
139+
# torch and torchvision are needed to export a model but are intentionally not
140+
# declared as wheel dependencies (consumers are expected to bring their own).
76141
python -m pip install \
77-
"flatbuffers" \
78-
"numpy>=2.0.0" \
79-
"packaging" \
80-
"pyyaml" \
81-
"ruamel.yaml" \
82-
"sympy" \
83-
"tabulate" \
84142
"torch" \
85143
"torchvision" \
86-
"typing-extensions>=4.10.0" \
87144
--index-url https://download.pytorch.org/whl/cpu \
88145
--extra-index-url https://pypi.org/simple
89-
python -m pip install --no-deps "${WHEEL_FILE}"
146+
# Install the minimal wheel WITHOUT --no-deps so pip resolves its declared
147+
# dependencies: this proves the slim set is correct and resolvable, and that
148+
# none of the heavy full-wheel deps get pulled in.
149+
python -m pip install \
150+
"${WHEEL_FILE}" \
151+
--index-url https://download.pytorch.org/whl/cpu \
152+
--extra-index-url https://pypi.org/simple
90153

91154
python - <<'PY'
92155
from pathlib import Path

‎README-wheel.md‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,10 @@ To build a minimal wheel from source, set
1212
`EXECUTORCH_BUILD_MINIMAL=1` when running `pip wheel` or `pip install`.
1313
That wheel contains the Python EXIR export path and `flatc` for `.pte`
1414
serialization, but omits runtime pybindings, kernels, backend packages, headers,
15-
examples, and devtools.
15+
examples, and devtools. It also declares only the Python dependencies the export
16+
path needs (no `coremltools`, `pandas`, `scikit-learn`, `hydra-core`, or
17+
`omegaconf`), so a normal install stays small. The wheel is still platform
18+
specific because it ships `flatc`.
1619

1720
The prebuilt `executorch.runtime` module included in this package provides a way
1821
to run ExecuTorch `.pte` files, with some restrictions:

‎pyproject.toml‎

Lines changed: 8 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,9 @@ name = "executorch"
1616
dynamic = [
1717
# setup.py will set the version.
1818
'version',
19+
# setup.py sets dependencies, which vary by build mode (the
20+
# EXECUTORCH_BUILD_MINIMAL wheel declares a slimmer runtime set).
21+
'dependencies',
1922
]
2023
description = "On-device AI across mobile, embedded and edge for PyTorch"
2124
readme = "README-wheel.md"
@@ -51,30 +54,11 @@ classifiers = [
5154
]
5255

5356
requires-python = ">=3.10,<3.14"
54-
dependencies=[
55-
"expecttest",
56-
"flatbuffers",
57-
"hypothesis",
58-
"kgb",
59-
"mpmath==1.3.0",
60-
"numpy>=2.0.0; python_version >= '3.10'",
61-
"packaging",
62-
"pandas>=2.2.2; python_version >= '3.10'",
63-
"parameterized",
64-
"pytorch-tokenizers",
65-
"pyyaml",
66-
"ruamel.yaml",
67-
"sympy",
68-
"tabulate",
69-
# See also third-party/TARGETS for buck's typing-extensions version.
70-
"typing-extensions>=4.10.0",
71-
# Keep this version in sync with: ./backends/apple/coreml/scripts/install_requirements.sh
72-
"coremltools==9.0; platform_system == 'Darwin' or platform_system == 'Linux'",
73-
# scikit-learn is used to support palettization in the coreml backend
74-
"scikit-learn==1.7.1",
75-
"hydra-core>=1.3.0",
76-
"omegaconf>=2.3.0",
77-
]
57+
58+
# Runtime dependencies are declared dynamically (see `dynamic` above) and
59+
# computed in setup.py, so the EXECUTORCH_BUILD_MINIMAL wheel can ship a slimmer
60+
# set than the full wheel. See `_base_dependencies()` / `_minimal_dependencies()`
61+
# in setup.py.
7862

7963
[project.optional-dependencies]
8064
cortex_m = [

‎setup.py‎

Lines changed: 94 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -165,6 +165,68 @@ def _minimal_packages() -> List[str]:
165165
)
166166

167167

168+
def _base_dependencies() -> List[str]:
169+
"""Runtime dependencies for the full wheel.
170+
171+
Declared here rather than in pyproject.toml (where `dependencies` is marked
172+
dynamic) so the minimal build can ship a slimmer set. Keep in sync with the
173+
project's runtime needs.
174+
"""
175+
return [
176+
"expecttest",
177+
"flatbuffers",
178+
"hypothesis",
179+
"kgb",
180+
"mpmath==1.3.0",
181+
"numpy>=2.0.0; python_version >= '3.10'",
182+
"packaging",
183+
"pandas>=2.2.2; python_version >= '3.10'",
184+
"parameterized",
185+
"pytorch-tokenizers",
186+
"pyyaml",
187+
"ruamel.yaml",
188+
"sympy",
189+
"tabulate",
190+
# See also third-party/TARGETS for buck's typing-extensions version.
191+
"typing-extensions>=4.10.0",
192+
# Keep this version in sync with: ./backends/apple/coreml/scripts/install_requirements.sh
193+
"coremltools==9.0; platform_system == 'Darwin' or platform_system == 'Linux'",
194+
# scikit-learn is used to support palettization in the coreml backend.
195+
"scikit-learn==1.7.1",
196+
"hydra-core>=1.3.0",
197+
"omegaconf>=2.3.0",
198+
]
199+
200+
201+
def _minimal_dependencies() -> List[str]:
202+
"""Runtime dependencies for the minimal (AOT export only) wheel.
203+
204+
Derived as the subset of _base_dependencies() that executorch.exir needs to
205+
lower and serialize a .pte, so version pins and markers stay in sync with the
206+
full set. torch is intentionally absent from both (consumers bring their own)
207+
and mpmath comes transitively via sympy. Keep the name set below in sync with
208+
the `expected` set in .ci/scripts/test_minimal_wheel.sh.
209+
"""
210+
keep = {
211+
"flatbuffers",
212+
"numpy",
213+
"packaging",
214+
"pyyaml",
215+
"ruamel-yaml",
216+
"sympy",
217+
"tabulate",
218+
"typing-extensions",
219+
}
220+
221+
def _name(dep: str) -> str:
222+
# PEP 503 normalized distribution name, e.g. "ruamel.yaml" -> "ruamel-yaml".
223+
return re.sub(
224+
r"[-_.]+", "-", re.split(r"[ ;\[<>=!~(]", dep, maxsplit=1)[0]
225+
).lower()
226+
227+
return [dep for dep in _base_dependencies() if _name(dep) in keep]
228+
229+
168230
class Version:
169231
"""Static strings that describe the version of the pip package."""
170232

@@ -880,41 +942,45 @@ def run(self): # noqa C901
880942
]
881943

882944
if minimal_build:
945+
# The minimal wheel only needs flatc. Every other target is gated off
946+
# by _minimal_cmake_flags(), so skip the entire non-minimal target
947+
# list explicitly rather than relying on each flag being OFF.
883948
cmake_build_args += ["--target", "flatbuffers_ep"]
884-
elif cmake_cache.is_enabled("EXECUTORCH_BUILD_PYBIND"):
885-
cmake_build_args += ["--target", "portable_lib"]
886-
cmake_build_args += ["--target", "data_loader"]
887-
cmake_build_args += ["--target", "selective_build"]
949+
else:
950+
if cmake_cache.is_enabled("EXECUTORCH_BUILD_PYBIND"):
951+
cmake_build_args += ["--target", "portable_lib"]
952+
cmake_build_args += ["--target", "data_loader"]
953+
cmake_build_args += ["--target", "selective_build"]
888954

889-
if cmake_cache.is_enabled("EXECUTORCH_BUILD_EXTENSION_LLM_RUNNER"):
890-
cmake_build_args += ["--target", "_llm_runner"]
955+
if cmake_cache.is_enabled("EXECUTORCH_BUILD_EXTENSION_LLM_RUNNER"):
956+
cmake_build_args += ["--target", "_llm_runner"]
891957

892-
if cmake_cache.is_enabled("EXECUTORCH_BUILD_CUDA"):
893-
cmake_build_args += ["--target", "aoti_cuda_backend"]
894-
cmake_build_args += ["--target", "aoti_common_shims_slim"]
958+
if cmake_cache.is_enabled("EXECUTORCH_BUILD_CUDA"):
959+
cmake_build_args += ["--target", "aoti_cuda_backend"]
960+
cmake_build_args += ["--target", "aoti_common_shims_slim"]
895961

896-
if cmake_cache.is_enabled("EXECUTORCH_BUILD_EXTENSION_MODULE"):
897-
cmake_build_args += ["--target", "extension_module"]
962+
if cmake_cache.is_enabled("EXECUTORCH_BUILD_EXTENSION_MODULE"):
963+
cmake_build_args += ["--target", "extension_module"]
898964

899-
if cmake_cache.is_enabled("EXECUTORCH_BUILD_EXTENSION_TRAINING"):
900-
cmake_build_args += ["--target", "_training_lib"]
965+
if cmake_cache.is_enabled("EXECUTORCH_BUILD_EXTENSION_TRAINING"):
966+
cmake_build_args += ["--target", "_training_lib"]
901967

902-
if cmake_cache.is_enabled("EXECUTORCH_BUILD_COREML"):
903-
cmake_build_args += ["--target", "executorchcoreml"]
968+
if cmake_cache.is_enabled("EXECUTORCH_BUILD_COREML"):
969+
cmake_build_args += ["--target", "executorchcoreml"]
904970

905-
if cmake_cache.is_enabled("EXECUTORCH_BUILD_MLX"):
906-
cmake_build_args += ["--target", "mlxdelegate"]
971+
if cmake_cache.is_enabled("EXECUTORCH_BUILD_MLX"):
972+
cmake_build_args += ["--target", "mlxdelegate"]
907973

908-
if cmake_cache.is_enabled("EXECUTORCH_BUILD_KERNELS_LLM_AOT"):
909-
cmake_build_args += ["--target", "custom_ops_aot_lib"]
910-
cmake_build_args += ["--target", "quantized_ops_aot_lib"]
974+
if cmake_cache.is_enabled("EXECUTORCH_BUILD_KERNELS_LLM_AOT"):
975+
cmake_build_args += ["--target", "custom_ops_aot_lib"]
976+
cmake_build_args += ["--target", "quantized_ops_aot_lib"]
911977

912-
if cmake_cache.is_enabled("EXECUTORCH_BUILD_QNN"):
913-
cmake_build_args += ["--target", "qnn_executorch_backend"]
914-
cmake_build_args += ["--target", "PyQnnManagerAdaptor"]
978+
if cmake_cache.is_enabled("EXECUTORCH_BUILD_QNN"):
979+
cmake_build_args += ["--target", "qnn_executorch_backend"]
980+
cmake_build_args += ["--target", "PyQnnManagerAdaptor"]
915981

916-
if cmake_cache.is_enabled("EXECUTORCH_BUILD_OPENVINO"):
917-
cmake_build_args += ["--target", "openvino_backend"]
982+
if cmake_cache.is_enabled("EXECUTORCH_BUILD_OPENVINO"):
983+
cmake_build_args += ["--target", "openvino_backend"]
918984

919985
# Set PYTHONPATH to the location of the pip package.
920986
os.environ["PYTHONPATH"] = (
@@ -931,6 +997,9 @@ def run(self): # noqa C901
931997
setup_kwargs = {}
932998
if _is_minimal_build():
933999
setup_kwargs["packages"] = _minimal_packages()
1000+
setup_kwargs["install_requires"] = _minimal_dependencies()
1001+
else:
1002+
setup_kwargs["install_requires"] = _base_dependencies()
9341003

9351004

9361005
setup(

0 commit comments

Comments
 (0)