From 55076664958581de770f027fe440bc217947388f Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Wed, 15 Jan 2025 15:42:13 +0000 Subject: [PATCH] Enable python external package loading by default. --- src/pyodide/BUILD.bazel | 36 ----------------------------- src/pyodide/pyodide_extra.capnp | 5 ---- src/pyodide/python-entrypoint.js | 9 -------- src/workerd/api/BUILD.bazel | 3 --- src/workerd/api/pyodide/pyodide.c++ | 14 +++++++++++ src/workerd/api/pyodide/pyodide.h | 2 ++ src/workerd/server/BUILD.bazel | 1 - src/workerd/server/workerd-api.c++ | 16 ++++--------- 8 files changed, 20 insertions(+), 66 deletions(-) delete mode 100644 src/pyodide/pyodide_extra.capnp delete mode 100644 src/pyodide/python-entrypoint.js diff --git a/src/pyodide/BUILD.bazel b/src/pyodide/BUILD.bazel index 27007613677..25abfb9dfda 100644 --- a/src/pyodide/BUILD.bazel +++ b/src/pyodide/BUILD.bazel @@ -20,48 +20,12 @@ capnp_embed( deps = ["pyodide_packages_archive"], ) -copy_file( - name = "python_entrypoint_file", - src = "python-entrypoint.js", - out = "generated/python-entrypoint.js", -) - -capnp_embed( - name = "python_entrypoint_file_embed", - src = "generated/python-entrypoint.js", - deps = ["python_entrypoint_file"], -) - -copy_file( - name = "pyodide_extra_capnp_file", - src = "pyodide_extra.capnp", - out = "generated/pyodide_extra.capnp", -) - -capnp_embed( - name = "pyodide_extra_file_embed", - src = "generated/pyodide_extra.capnp", - deps = ["pyodide_extra_capnp_file"], -) - capnp_embed( name = "pyodide_lock_file_embed", src = "generated/pyodide-lock.json", deps = ["pyodide-lock.js@rule"], ) -cc_capnp_library( - name = "pyodide_extra_capnp", - srcs = ["generated/pyodide_extra.capnp"], - visibility = ["//visibility:public"], - deps = [ - ":pyodide_extra_file_embed", - ":pyodide_lock_file_embed", - ":pyodide_packages_archive_embed", - ":python_entrypoint_file_embed", - ], -) - copy_file( name = "pyodide.asm.wasm@rule", src = "@pyodide//:pyodide/pyodide.asm.wasm", diff --git a/src/pyodide/pyodide_extra.capnp b/src/pyodide/pyodide_extra.capnp deleted file mode 100644 index b0d8c30cab5..00000000000 --- a/src/pyodide/pyodide_extra.capnp +++ /dev/null @@ -1,5 +0,0 @@ -@0x96c290dbf479ac0c; - -const pythonEntrypoint :Text = embed "python-entrypoint.js"; -const pyodidePackagesTar :Data = embed "pyodide_packages.tar"; -const pyodideLock :Text = embed "pyodide-lock.json"; diff --git a/src/pyodide/python-entrypoint.js b/src/pyodide/python-entrypoint.js deleted file mode 100644 index 66829110256..00000000000 --- a/src/pyodide/python-entrypoint.js +++ /dev/null @@ -1,9 +0,0 @@ -// The entrypoint to a worker has to be an ES6 module (well, ignoring service -// workers). For Python workers, we use this file as the ES6 entrypoint. -// -// This file is treated as part of the user bundle and cannot import internal -// modules. Any imports from this file must be user-visible. To deal with this, -// we delegate the implementation to `python-entrypoint-helper` which is a -// BUILTIN module that can see our INTERNAL modules. - -export { default } from 'pyodide:python-entrypoint-helper'; diff --git a/src/workerd/api/BUILD.bazel b/src/workerd/api/BUILD.bazel index 61625d9121a..4f87c04e80c 100644 --- a/src/workerd/api/BUILD.bazel +++ b/src/workerd/api/BUILD.bazel @@ -85,7 +85,6 @@ wd_cc_library( hdrs = [ "modules.h", "rtti.h", - "//src/pyodide:generated/pyodide_extra.capnp.h", "//src/workerd/server:workerd.capnp.h", ], visibility = ["//visibility:public"], @@ -96,7 +95,6 @@ wd_cc_library( ":pyodide", ":r2", "//src/pyodide", - "//src/pyodide:pyodide_extra_capnp", "//src/workerd/api/node", "//src/workerd/io", "//src/workerd/jsg:rtti", @@ -140,7 +138,6 @@ wd_cc_library( implementation_deps = ["//src/workerd/util:string-buffer"], visibility = ["//visibility:public"], deps = [ - "//src/pyodide:pyodide_extra_capnp", "//src/workerd/io", "//src/workerd/jsg", "//src/workerd/server:workerd_capnp", diff --git a/src/workerd/api/pyodide/pyodide.c++ b/src/workerd/api/pyodide/pyodide.c++ index 32dd0d92337..78744031934 100644 --- a/src/workerd/api/pyodide/pyodide.c++ +++ b/src/workerd/api/pyodide/pyodide.c++ @@ -521,4 +521,18 @@ bool hasPythonModules(capnp::List::Reader module return false; } +kj::StringPtr getPythonEntrypoint() { + return R"JS_SCRIPT( +// The entrypoint to a worker has to be an ES6 module (well, ignoring service +// workers). For Python workers, we use this file as the ES6 entrypoint. +// +// This file is treated as part of the user bundle and cannot import internal +// modules. Any imports from this file must be user-visible. To deal with this, +// we delegate the implementation to `python-entrypoint-helper` which is a +// BUILTIN module that can see our INTERNAL modules. + +export { default } from 'pyodide:python-entrypoint-helper'; + )JS_SCRIPT"_kj; +} + } // namespace workerd::api::pyodide diff --git a/src/workerd/api/pyodide/pyodide.h b/src/workerd/api/pyodide/pyodide.h index 3f932809e62..04bd2af0d33 100644 --- a/src/workerd/api/pyodide/pyodide.h +++ b/src/workerd/api/pyodide/pyodide.h @@ -424,6 +424,8 @@ jsg::Ref makePyodideMetadataReader(Worker::Reader conf, bool hasPythonModules(capnp::List::Reader modules); +kj::StringPtr getPythonEntrypoint(); + #define EW_PYODIDE_ISOLATE_TYPES \ api::pyodide::ReadOnlyBuffer, api::pyodide::PyodideMetadataReader, \ api::pyodide::ArtifactBundler, api::pyodide::DiskCache, \ diff --git a/src/workerd/server/BUILD.bazel b/src/workerd/server/BUILD.bazel index 58037da488b..acf952c3544 100644 --- a/src/workerd/server/BUILD.bazel +++ b/src/workerd/server/BUILD.bazel @@ -51,7 +51,6 @@ wd_cc_binary( ":v8-platform-impl", ":workerd-meta_capnp", ":workerd_capnp", - "//src/pyodide:pyodide_extra_capnp", "//src/rust/cxx-integration", "//src/workerd/util:autogate", "//src/workerd/util:perfetto", diff --git a/src/workerd/server/workerd-api.c++ b/src/workerd/server/workerd-api.c++ index b574b84ecb5..cda8d93f96f 100644 --- a/src/workerd/server/workerd-api.c++ +++ b/src/workerd/server/workerd-api.c++ @@ -46,8 +46,6 @@ #include #include -#include - #include #include #include @@ -553,7 +551,7 @@ void WorkerdApi::compileModules(jsg::Lock& lockParam, auto mainModule = confModules.begin(); capnp::MallocMessageBuilder message; auto module = message.getRoot(); - module.setEsModule(PYTHON_ENTRYPOINT); + module.setEsModule(getPythonEntrypoint()); auto info = tryCompileModule(lockParam, module, modules->getObserver(), featureFlags); auto path = kj::Path::parse(mainModule->getName()); modules->add(path, kj::mv(KJ_REQUIRE_NONNULL(info))); @@ -565,14 +563,8 @@ void WorkerdApi::compileModules(jsg::Lock& lockParam, jsg::ModuleRegistry::Type::INTERNAL); // Inject packages tar file - if (featureFlags.getPythonExternalPackages()) { - modules->addBuiltinModule("pyodide-internal:packages_tar_reader", "export default { }"_kj, - workerd::jsg::ModuleRegistry::Type::INTERNAL, {}); - } else { - modules->addBuiltinModule("pyodide-internal:packages_tar_reader", - jsg::alloc(PYODIDE_PACKAGES_TAR.get()), - workerd::jsg::ModuleRegistry::Type::INTERNAL); - } + modules->addBuiltinModule("pyodide-internal:packages_tar_reader", "export default { }"_kj, + workerd::jsg::ModuleRegistry::Type::INTERNAL, {}); // Inject artifact bundler. modules->addBuiltinModule("pyodide-internal:artifacts", @@ -967,7 +959,7 @@ kj::Own WorkerdApi::initializeBundleModuleRegistry KJ_REQUIRE(featureFlags.getPythonWorkers(), "The python_workers compatibility flag is required to use Python."); hasPythonModules = true; - bundleBuilder.addEsmModule(def.getName(), kj::str(PYTHON_ENTRYPOINT).releaseArray()); + bundleBuilder.addEsmModule(def.getName(), kj::str(getPythonEntrypoint()).releaseArray()); break; } case config::Worker::Module::PYTHON_REQUIREMENT: {