From 26524c9507d7912d80b17639d3f82ade3890835e Mon Sep 17 00:00:00 2001 From: timrid <6593626+timrid@users.noreply.github.com> Date: Wed, 26 Nov 2025 19:51:49 +0100 Subject: [PATCH 1/7] add support for "*" in extractPackages --- product/gradle-plugin/src/main/kotlin/PythonTasks.kt | 4 +++- product/runtime/src/main/python/java/android/importer.py | 7 +++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/product/gradle-plugin/src/main/kotlin/PythonTasks.kt b/product/gradle-plugin/src/main/kotlin/PythonTasks.kt index cf55aa2bd2..2b62a64415 100644 --- a/product/gradle-plugin/src/main/kotlin/PythonTasks.kt +++ b/product/gradle-plugin/src/main/kotlin/PythonTasks.kt @@ -319,7 +319,9 @@ internal class TaskBuilder( // Exclude .py files which have a corresponding .pyc, unless unless they’re // included in extractPackages. val excludePy = { fte: FileTreeElement -> - if (fte.name.endsWith(".py") && + if ("*" in python.extractPackages) { + false + } else if (fte.name.endsWith(".py") && File(fte.file.parent, fte.name + "c").exists() ) { ! python.extractPackages.any { diff --git a/product/runtime/src/main/python/java/android/importer.py b/product/runtime/src/main/python/java/android/importer.py index e2abc36acc..5534e7e962 100644 --- a/product/runtime/src/main/python/java/android/importer.py +++ b/product/runtime/src/main/python/java/android/importer.py @@ -534,8 +534,11 @@ def extract_so(self, path): def extract_dir(self, zip_dir, recursive=True): dotted_dir = zip_dir.replace("/", ".") - extract_package = any((dotted_dir == ep) or dotted_dir.startswith(ep + ".") - for ep in self.extract_packages) + if "*" in self.extract_packages: + extract_package = True + else: + extract_package = any((dotted_dir == ep) or dotted_dir.startswith(ep + ".") + for ep in self.extract_packages) for filename in self.listdir(zip_dir): zip_path = join(zip_dir, filename) From d9be57a4b2b264e9dff907f7f09a593b03e6a4ac Mon Sep 17 00:00:00 2001 From: timrid <6593626+timrid@users.noreply.github.com> Date: Wed, 26 Nov 2025 19:51:49 +0100 Subject: [PATCH 2/7] add docs --- product/runtime/docs/sphinx/android.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/product/runtime/docs/sphinx/android.rst b/product/runtime/docs/sphinx/android.rst index 4a81725b8f..0a8e9029b1 100644 --- a/product/runtime/docs/sphinx/android.rst +++ b/product/runtime/docs/sphinx/android.rst @@ -408,6 +408,15 @@ can declare them like this:: Each extracted file will slightly slow down your app's startup, so this setting should be used on the deepest possible package. +But for development purposes, it is sometimes useful to extract all packages. This is +possible using a wildcard:: + + chaquopy { + defaultConfig { + extractPackages("*") + } + } + .. _android-data: Data files From 27b67cc4e609191c60f54bb3ee51bc9c8a9af528 Mon Sep 17 00:00:00 2001 From: timrid <6593626+timrid@users.noreply.github.com> Date: Wed, 26 Nov 2025 19:51:49 +0100 Subject: [PATCH 3/7] Extract all on startup and not only on import --- product/runtime/docs/sphinx/android.rst | 11 +++++++---- .../runtime/src/main/python/java/android/importer.py | 4 ++++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/product/runtime/docs/sphinx/android.rst b/product/runtime/docs/sphinx/android.rst index 0a8e9029b1..8fbe93f908 100644 --- a/product/runtime/docs/sphinx/android.rst +++ b/product/runtime/docs/sphinx/android.rst @@ -405,11 +405,12 @@ can declare them like this:: } } -Each extracted file will slightly slow down your app's startup, so this setting should be -used on the deepest possible package. +The files are extracted when they are imported for the first time. Each extracted file will +slightly slow down your app's startup, so this setting should be used on the deepest possible +package. -But for development purposes, it is sometimes useful to extract all packages. This is -possible using a wildcard:: +For development purposes, it is sometimes useful to extract all packages. This is possible +using a wildcard:: chaquopy { defaultConfig { @@ -417,6 +418,8 @@ possible using a wildcard:: } } +If the wildcard is used, all packages are imported before your app's code starts running. + .. _android-data: Data files diff --git a/product/runtime/src/main/python/java/android/importer.py b/product/runtime/src/main/python/java/android/importer.py index 5534e7e962..6d1af71643 100644 --- a/product/runtime/src/main/python/java/android/importer.py +++ b/product/runtime/src/main/python/java/android/importer.py @@ -86,6 +86,10 @@ def hook(path): # read by addsitedir below. finder.extract_dir("", recursive=False) + # Extract all Python packages if requested. + if "*" in build_json["extract_packages"]: + finder.extract_dir("") + # Extract data files from top-level directories which aren't Python packages. for name in finder.listdir(""): if finder.isdir(name) and \ From e862f700ac9f4ceeda04ebf4c493e6e6474ecdec Mon Sep 17 00:00:00 2001 From: timrid <6593626+timrid@users.noreply.github.com> Date: Wed, 26 Nov 2025 19:51:49 +0100 Subject: [PATCH 4/7] Improvements from code review --- .../src/main/kotlin/PythonTasks.kt | 6 ++---- product/runtime/docs/sphinx/android.rst | 13 +++++++------ .../runtime/docs/sphinx/changes/1424.added.rst | 1 + .../src/main/python/java/android/importer.py | 17 ++++++----------- 4 files changed, 16 insertions(+), 21 deletions(-) create mode 100644 product/runtime/docs/sphinx/changes/1424.added.rst diff --git a/product/gradle-plugin/src/main/kotlin/PythonTasks.kt b/product/gradle-plugin/src/main/kotlin/PythonTasks.kt index 2b62a64415..c398de192f 100644 --- a/product/gradle-plugin/src/main/kotlin/PythonTasks.kt +++ b/product/gradle-plugin/src/main/kotlin/PythonTasks.kt @@ -319,13 +319,11 @@ internal class TaskBuilder( // Exclude .py files which have a corresponding .pyc, unless unless they’re // included in extractPackages. val excludePy = { fte: FileTreeElement -> - if ("*" in python.extractPackages) { - false - } else if (fte.name.endsWith(".py") && + if (fte.name.endsWith(".py") && File(fte.file.parent, fte.name + "c").exists() ) { ! python.extractPackages.any { - fte.path.replace("/", ".").startsWith(it + ".") + it == "*" || fte.path.replace("/", ".").startsWith(it + ".") } } else false } diff --git a/product/runtime/docs/sphinx/android.rst b/product/runtime/docs/sphinx/android.rst index 8fbe93f908..a5da630730 100644 --- a/product/runtime/docs/sphinx/android.rst +++ b/product/runtime/docs/sphinx/android.rst @@ -405,12 +405,12 @@ can declare them like this:: } } -The files are extracted when they are imported for the first time. Each extracted file will -slightly slow down your app's startup, so this setting should be used on the deepest possible -package. +The files are extracted when the package is imported for the first time. Each extracted +file will slightly slow down your app's startup, so this setting should be used on the +deepest possible package. -For development purposes, it is sometimes useful to extract all packages. This is possible -using a wildcard:: +For development purposes, it is sometimes useful to extract all packages. This is +possible using a wildcard:: chaquopy { defaultConfig { @@ -418,7 +418,8 @@ using a wildcard:: } } -If the wildcard is used, all packages are imported before your app's code starts running. +If the wildcard is used, all packages are imported before your app's code starts +running. .. _android-data: diff --git a/product/runtime/docs/sphinx/changes/1424.added.rst b/product/runtime/docs/sphinx/changes/1424.added.rst new file mode 100644 index 0000000000..acb686d0ed --- /dev/null +++ b/product/runtime/docs/sphinx/changes/1424.added.rst @@ -0,0 +1 @@ +Added the wildcard `*` to :ref:`extractPackages` to extract all packages at startup. \ No newline at end of file diff --git a/product/runtime/src/main/python/java/android/importer.py b/product/runtime/src/main/python/java/android/importer.py index 6d1af71643..929c0b1b97 100644 --- a/product/runtime/src/main/python/java/android/importer.py +++ b/product/runtime/src/main/python/java/android/importer.py @@ -83,12 +83,8 @@ def hook(path): .format(entry, type(finder).__name__)) # Extract data files from the root directory. This includes .pth files, which will be - # read by addsitedir below. - finder.extract_dir("", recursive=False) - - # Extract all Python packages if requested. - if "*" in build_json["extract_packages"]: - finder.extract_dir("") + # read by addsitedir below. If requested, also extract all Python packages. + finder.extract_dir("", recursive="*" in build_json["extract_packages"]) # Extract data files from top-level directories which aren't Python packages. for name in finder.listdir(""): @@ -538,11 +534,10 @@ def extract_so(self, path): def extract_dir(self, zip_dir, recursive=True): dotted_dir = zip_dir.replace("/", ".") - if "*" in self.extract_packages: - extract_package = True - else: - extract_package = any((dotted_dir == ep) or dotted_dir.startswith(ep + ".") - for ep in self.extract_packages) + extract_package = any( + ep in (dotted_dir, "*") or dotted_dir.startswith(ep + ".") + for ep in self.extract_packages + ) for filename in self.listdir(zip_dir): zip_path = join(zip_dir, filename) From b961da783fac06bcd8190160dfa04233cbfaf18c Mon Sep 17 00:00:00 2001 From: timrid <6593626+timrid@users.noreply.github.com> Date: Wed, 26 Nov 2025 20:37:05 +0100 Subject: [PATCH 5/7] add integration test --- .../src/test/integration/test_gradle_plugin.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/product/gradle-plugin/src/test/integration/test_gradle_plugin.py b/product/gradle-plugin/src/test/integration/test_gradle_plugin.py index 189b52d0c4..f438668c6b 100644 --- a/product/gradle-plugin/src/test/integration/test_gradle_plugin.py +++ b/product/gradle-plugin/src/test/integration/test_gradle_plugin.py @@ -193,7 +193,7 @@ def checkZip(self, zip_filename, files, *, pyc=False, extract_packages=[], with self.subTest(f=f): filename, attrs = f if isinstance(f, tuple) else (f, {}) if pyc and filename.endswith(".py"): - if any(filename.startswith(ep.replace(".", "/") + "/") + if any(ep == "*" or filename.startswith(ep.replace(".", "/") + "/") for ep in extract_packages): expected_files.append(filename) filename += "c" @@ -675,6 +675,11 @@ def test_variant_merge(self): app=["common/__init__.py", "red/__init__.py", "blue/__init__.py"], variants={"red-debug": dict(extract_packages=["common"]), "blue-debug": dict(extract_packages=["common", "blue"])}) + + def test_wildcard(self): + self.RunGradle("base", "ExtractPackages/wildcard", + app=["common/__init__.py", "red/__init__.py", "blue/__init__.py"], + extract_packages=["common", "*"]) class Pyc(GradleTestCase): From 46a0650ffac62ee566f1052db167440806e123f0 Mon Sep 17 00:00:00 2001 From: timrid <6593626+timrid@users.noreply.github.com> Date: Thu, 27 Nov 2025 20:16:02 +0100 Subject: [PATCH 6/7] added missing files for integration tests --- .../ExtractPackages/wildcard/app/build.gradle | 20 +++++++++++++++++++ .../app/src/main/python/pkg1/__init__.py | 0 .../app/src/main/python/pkg2/__init__.py | 0 .../app/src/main/python/pkg3/__init__.py | 0 .../wildcard/app/src/main/python/pkg3/mod1.py | 0 .../src/main/python/pkg3/sub_pkg/__init__.py | 0 .../app/src/main/python/pkg3/sub_pkg/mod2.py | 0 .../test/integration/test_gradle_plugin.py | 12 +++++++++-- 8 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 product/gradle-plugin/src/test/integration/data/ExtractPackages/wildcard/app/build.gradle create mode 100644 product/gradle-plugin/src/test/integration/data/ExtractPackages/wildcard/app/src/main/python/pkg1/__init__.py create mode 100644 product/gradle-plugin/src/test/integration/data/ExtractPackages/wildcard/app/src/main/python/pkg2/__init__.py create mode 100644 product/gradle-plugin/src/test/integration/data/ExtractPackages/wildcard/app/src/main/python/pkg3/__init__.py create mode 100644 product/gradle-plugin/src/test/integration/data/ExtractPackages/wildcard/app/src/main/python/pkg3/mod1.py create mode 100644 product/gradle-plugin/src/test/integration/data/ExtractPackages/wildcard/app/src/main/python/pkg3/sub_pkg/__init__.py create mode 100644 product/gradle-plugin/src/test/integration/data/ExtractPackages/wildcard/app/src/main/python/pkg3/sub_pkg/mod2.py diff --git a/product/gradle-plugin/src/test/integration/data/ExtractPackages/wildcard/app/build.gradle b/product/gradle-plugin/src/test/integration/data/ExtractPackages/wildcard/app/build.gradle new file mode 100644 index 0000000000..1c34e0d0db --- /dev/null +++ b/product/gradle-plugin/src/test/integration/data/ExtractPackages/wildcard/app/build.gradle @@ -0,0 +1,20 @@ +apply plugin: 'com.android.application' +apply plugin: 'com.chaquo.python' + +android { + namespace "com.chaquo.python.test" + compileSdk 31 + defaultConfig { + applicationId "com.chaquo.python.test" + minSdk 24 + targetSdk 31 + versionCode 1 + versionName "0.0.1" + python { + extractPackages "pkg1", "*" + } + ndk { + abiFilters "x86" + } + } +} diff --git a/product/gradle-plugin/src/test/integration/data/ExtractPackages/wildcard/app/src/main/python/pkg1/__init__.py b/product/gradle-plugin/src/test/integration/data/ExtractPackages/wildcard/app/src/main/python/pkg1/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/product/gradle-plugin/src/test/integration/data/ExtractPackages/wildcard/app/src/main/python/pkg2/__init__.py b/product/gradle-plugin/src/test/integration/data/ExtractPackages/wildcard/app/src/main/python/pkg2/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/product/gradle-plugin/src/test/integration/data/ExtractPackages/wildcard/app/src/main/python/pkg3/__init__.py b/product/gradle-plugin/src/test/integration/data/ExtractPackages/wildcard/app/src/main/python/pkg3/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/product/gradle-plugin/src/test/integration/data/ExtractPackages/wildcard/app/src/main/python/pkg3/mod1.py b/product/gradle-plugin/src/test/integration/data/ExtractPackages/wildcard/app/src/main/python/pkg3/mod1.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/product/gradle-plugin/src/test/integration/data/ExtractPackages/wildcard/app/src/main/python/pkg3/sub_pkg/__init__.py b/product/gradle-plugin/src/test/integration/data/ExtractPackages/wildcard/app/src/main/python/pkg3/sub_pkg/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/product/gradle-plugin/src/test/integration/data/ExtractPackages/wildcard/app/src/main/python/pkg3/sub_pkg/mod2.py b/product/gradle-plugin/src/test/integration/data/ExtractPackages/wildcard/app/src/main/python/pkg3/sub_pkg/mod2.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/product/gradle-plugin/src/test/integration/test_gradle_plugin.py b/product/gradle-plugin/src/test/integration/test_gradle_plugin.py index f438668c6b..fb1de4c77d 100644 --- a/product/gradle-plugin/src/test/integration/test_gradle_plugin.py +++ b/product/gradle-plugin/src/test/integration/test_gradle_plugin.py @@ -677,9 +677,17 @@ def test_variant_merge(self): "blue-debug": dict(extract_packages=["common", "blue"])}) def test_wildcard(self): + PY_FILES = [ + "pkg1/__init__.py", + "pkg2/__init__.py", + "pkg3/__init__.py", + "pkg3/mod1.py", + "pkg3/sub_pkg/__init__.py", + "pkg3/sub_pkg/mod2.py", + ] self.RunGradle("base", "ExtractPackages/wildcard", - app=["common/__init__.py", "red/__init__.py", "blue/__init__.py"], - extract_packages=["common", "*"]) + app=PY_FILES, + extract_packages=["pkg1", "*"]) class Pyc(GradleTestCase): From 4931e169d6729be1f823050c06b4528209312875 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Sun, 30 Nov 2025 16:04:21 +0000 Subject: [PATCH 7/7] Use correct change note category, remove trailing whitespace --- .../gradle-plugin/src/test/integration/test_gradle_plugin.py | 2 +- .../docs/sphinx/changes/{1424.added.rst => 1424.feature.rst} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename product/runtime/docs/sphinx/changes/{1424.added.rst => 1424.feature.rst} (100%) diff --git a/product/gradle-plugin/src/test/integration/test_gradle_plugin.py b/product/gradle-plugin/src/test/integration/test_gradle_plugin.py index fb1de4c77d..77966ee3c1 100644 --- a/product/gradle-plugin/src/test/integration/test_gradle_plugin.py +++ b/product/gradle-plugin/src/test/integration/test_gradle_plugin.py @@ -675,7 +675,7 @@ def test_variant_merge(self): app=["common/__init__.py", "red/__init__.py", "blue/__init__.py"], variants={"red-debug": dict(extract_packages=["common"]), "blue-debug": dict(extract_packages=["common", "blue"])}) - + def test_wildcard(self): PY_FILES = [ "pkg1/__init__.py", diff --git a/product/runtime/docs/sphinx/changes/1424.added.rst b/product/runtime/docs/sphinx/changes/1424.feature.rst similarity index 100% rename from product/runtime/docs/sphinx/changes/1424.added.rst rename to product/runtime/docs/sphinx/changes/1424.feature.rst