Merge pull request #3328 from cloudflare/dominik/remove-python-lock

Removes usage of embedded PYODIDE_LOCK in C++ code.
cloudflare · Jan 14, 2025 · f15ac0f · f15ac0f
2 parents 8cb7ffd + 29612f9
commit f15ac0f
Show file tree

Hide file tree

Showing 6 changed files with 27 additions and 6 deletions.
diff --git a/src/workerd/api/BUILD.bazel b/src/workerd/api/BUILD.bazel
@@ -136,7 +136,6 @@ wd_cc_library(
     hdrs = [
         "pyodide/pyodide.h",
         "pyodide/setup-emscripten.h",
-        "//src/pyodide:generated/pyodide_extra.capnp.h",
     ],
     implementation_deps = ["//src/workerd/util:string-buffer"],
     visibility = ["//visibility:public"],

diff --git a/src/workerd/api/pyodide/pyodide.h b/src/workerd/api/pyodide/pyodide.h
@@ -13,8 +13,6 @@
 #include <workerd/server/workerd.capnp.h>
 #include <workerd/util/autogate.h>
 
-#include <pyodide/generated/pyodide_extra.capnp.h>
-
 #include <capnp/serialize.h>
 #include <kj/array.h>
 #include <kj/common.h>

diff --git a/src/workerd/server/tests/python/BUILD.bazel b/src/workerd/server/tests/python/BUILD.bazel
@@ -30,6 +30,7 @@ py_wd_test("subdirectory")
 
 py_wd_test(
     "sdk",
+    # TODO(EW-8976): Disabled in dev due to test getting stuck
     python_flags = ["0.26.0a2"],
     tags = [
         # TODO(someday): Fix asan failure for this, see https://github.com/cloudflare/workerd/pull/3140#discussion_r1858273318

diff --git a/src/workerd/server/workerd-api.c++ b/src/workerd/server/workerd-api.c++
@@ -46,6 +46,8 @@
 #include <workerd/util/thread-scopes.h>
 #include <workerd/util/use-perfetto-categories.h>
 
+#include <pyodide/generated/pyodide_extra.capnp.h>
+
 #include <kj/compat/http.h>
 #include <kj/compat/tls.h>
 #include <kj/compat/url.h>
@@ -157,6 +159,7 @@ void writePyodideBundleFileToDisk(const kj::Maybe<kj::Own<const kj::Directory>>&
     replacer->commit();
   }
 }
+}  // namespace
 
 kj::Maybe<jsg::Bundle::Reader> fetchPyodideBundle(
     const api::pyodide::PythonConfig& pyConfig, kj::StringPtr version) {
@@ -213,7 +216,6 @@ kj::Maybe<jsg::Bundle::Reader> fetchPyodideBundle(
   KJ_LOG(INFO, "Loaded Pyodide package from internet");
   return pyConfig.pyodideBundleManager.getPyodideBundle(version);
 }
-}  // namespace
 
 struct WorkerdApi::Impl final {
   kj::Own<CompatibilityFlags::Reader> features;

diff --git a/src/workerd/server/workerd-api.h b/src/workerd/server/workerd-api.h
@@ -265,4 +265,7 @@ class WorkerdApi final: public Worker::Api {
       capnp::List<config::Extension>::Reader extensions) const;
 };
 
+kj::Maybe<jsg::Bundle::Reader> fetchPyodideBundle(
+    const api::pyodide::PythonConfig& pyConfig, kj::StringPtr version);
+
 }  // namespace workerd::server
diff --git a/src/workerd/server/workerd.c++ b/src/workerd/server/workerd.c++
@@ -3,8 +3,10 @@
 //     https://opensource.org/licenses/Apache-2.0
 
 #include "server.h"
+#include "workerd-api.h"
 
 #include <workerd/io/compatibility-date.capnp.h>
+#include <workerd/io/compatibility-date.h>
 #include <workerd/io/supported-compatibility-date.capnp.h>
 #include <workerd/jsg/setup.h>
 #include <workerd/rust/cxx-integration/lib.rs.h>
@@ -15,7 +17,6 @@
 
 #include <fcntl.h>
 #include <openssl/rand.h>
-#include <pyodide/generated/pyodide_extra.capnp.h>
 #include <sys/stat.h>
 
 #include <capnp/dynamic.h>
@@ -807,7 +808,24 @@ class CliMain final: public SchemaFileImpl::ErrorReporter {
         context, getVersionString(), "Outputs the package lock file used by Pyodide.");
     return builder
         .callAfterParsing([]() -> kj::MainBuilder::Validity {
-      printf("%s\n", PYODIDE_LOCK->cStr());
+      static const PythonConfig config{
+        .packageDiskCacheRoot = kj::none,
+        .pyodideDiskCacheRoot = kj::none,
+        .createSnapshot = false,
+        .createBaselineSnapshot = false,
+      };
+
+      capnp::MallocMessageBuilder message;
+      // TODO(EW-8977): Implement option to specify python worker flags.
+      auto features = message.getRoot<CompatibilityFlags>();
+      features.setPythonWorkers(true);
+      auto pythonRelease = KJ_ASSERT_NONNULL(getPythonSnapshotRelease(features));
+      auto version = getPythonBundleName(pythonRelease);
+      KJ_ASSERT_NONNULL(fetchPyodideBundle(config, version), "Failed to get Pyodide bundle");
+
+      auto lock = KJ_ASSERT_NONNULL(config.pyodideBundleManager.getPyodideLock(pythonRelease));
+
+      printf("%s\n", lock.cStr());
       fflush(stdout);
       return true;
     }).build();