From e8b4f58c8feee1fa7e55c828d16b12b9e5563b5b Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Thu, 16 Jan 2025 11:48:36 +0100 Subject: [PATCH 01/11] Cap boto3 in release-python-runtime job (#3344) --- .github/workflows/release-python-runtime.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release-python-runtime.yml b/.github/workflows/release-python-runtime.yml index 9d75513c945..d70a577adc5 100644 --- a/.github/workflows/release-python-runtime.yml +++ b/.github/workflows/release-python-runtime.yml @@ -46,5 +46,7 @@ jobs: R2_ACCESS_KEY_ID: ${{ secrets.PYODIDE_CAPNP_R2_ACCESS_KEY_ID }} R2_SECRET_ACCESS_KEY: ${{ secrets.PYODIDE_CAPNP_R2_SECRET_ACCESS_KEY }} run: | - pip install boto3 + # boto3 v1.36.0 fails with: + # NotImplemented error occurred in CreateMultipartUpload operation: Header 'x-amz-checksum-algorithm' with value 'CRC32' not implemented + pip install 'boto3<1.36.0' python3 src/pyodide/upload_bundle.py pyodide.capnp.bin pyodide_${{ inputs.pyodide }}_${{ inputs.pyodideRevision }}_${{ inputs.backport }}.capnp.bin From ebf021c01d9d4e7183304ab0d9c9c6da685e7287 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Thu, 19 Dec 2024 11:41:36 +0100 Subject: [PATCH 02/11] Add compatibility date for Pyodide 0.27.1 --- src/workerd/io/compatibility-date.capnp | 7 ++++++ src/workerd/server/tests/python/BUILD.bazel | 23 ++++++++++++++++++- .../server/tests/python/import_tests.bzl | 3 ++- .../server/tests/python/py_wd_test.bzl | 6 +++-- 4 files changed, 35 insertions(+), 4 deletions(-) diff --git a/src/workerd/io/compatibility-date.capnp b/src/workerd/io/compatibility-date.capnp index 3572bf2aa5e..4342107465c 100644 --- a/src/workerd/io/compatibility-date.capnp +++ b/src/workerd/io/compatibility-date.capnp @@ -679,4 +679,11 @@ struct CompatibilityFlags @0x8f8c1b68151b6cef { $compatDisableFlag("cache_no_cache_disabled") $experimental; # Enables the use of cache: no-cache in the fetch api. + + pythonWorkers20250116 @71 :Bool + $compatEnableFlag("python_workers_20250116") + $experimental + $pythonSnapshotRelease(pyodide = "0.27.1", pyodideRevision = "2025-01-16", + packages = "2024-12-18", backport = 0, + baselineSnapshotHash = "TODO"); } diff --git a/src/workerd/server/tests/python/BUILD.bazel b/src/workerd/server/tests/python/BUILD.bazel index 450650b77f7..bfa7eb38bbd 100644 --- a/src/workerd/server/tests/python/BUILD.bazel +++ b/src/workerd/server/tests/python/BUILD.bazel @@ -38,4 +38,25 @@ py_wd_test( ], ) -gen_import_tests(PYODIDE_IMPORTS_TO_TEST) +gen_import_tests( + PYODIDE_IMPORTS_TO_TEST, + # TODO: Micropip version mismatch for 0.27.0 + pkg_python_versions = { + "micropip": [ + "0.26.0a2", + "development", + ], + "langchain-core": [ + "0.26.0a2", + "development", + ], + "langchain_openai": [ + "0.26.0a2", + "development", + ], + "langsmith": [ + "0.26.0a2", + "development", + ], + }, +) diff --git a/src/workerd/server/tests/python/import_tests.bzl b/src/workerd/server/tests/python/import_tests.bzl index 5409cd1d624..d17f30c7899 100644 --- a/src/workerd/server/tests/python/import_tests.bzl +++ b/src/workerd/server/tests/python/import_tests.bzl @@ -32,7 +32,7 @@ def generate_wd_test_file(requirement): return WD_FILE_TEMPLATE.format(requirement, requirement) # to_test is a dictionary from library name to list of imports -def gen_import_tests(to_test): +def gen_import_tests(to_test, pkg_python_versions = {}): for lib in to_test.keys(): prefix = "import/" + lib worker_py_fname = prefix + "/worker.py" @@ -52,6 +52,7 @@ def gen_import_tests(to_test): name = prefix, directory = lib, src = wd_test_fname, + python_flags = pkg_python_versions.get(lib, "all"), args = ["--experimental", "--pyodide-package-disk-cache-dir", "../all_pyodide_wheels"], data = [worker_py_fname, "@all_pyodide_wheels//:whls"], size = "enormous", diff --git a/src/workerd/server/tests/python/py_wd_test.bzl b/src/workerd/server/tests/python/py_wd_test.bzl index c646724dc5f..572979b7ebe 100644 --- a/src/workerd/server/tests/python/py_wd_test.bzl +++ b/src/workerd/server/tests/python/py_wd_test.bzl @@ -2,7 +2,8 @@ load("@bazel_skylib//rules:expand_template.bzl", "expand_template") load("//:build/wd_test.bzl", "wd_test") FEATURE_FLAGS = { - "0.26.0a2": ["python_workers"], + "0.26.0a2": [], + "0.27.1": ["python_workers_20250116"], "development": ["python_workers_development", "python_external_packages"], } @@ -37,7 +38,8 @@ def py_wd_test( name_flag = name + "_" + python_flag templated_src = name_flag.replace("/", "-") + "@template" templated_src = "/".join(src.split("/")[:-1] + [templated_src]) - feature_flags_txt = ",".join(['"{}"'.format(flag) for flag in FEATURE_FLAGS[python_flag]]) + flags = FEATURE_FLAGS[python_flag] + ["python_workers"] + feature_flags_txt = ",".join(['"{}"'.format(flag) for flag in flags]) expand_template( name = name_flag + "@rule", out = templated_src, From 703415f3df113f06ccc17bf8dcd70c1ece81449a Mon Sep 17 00:00:00 2001 From: Yagiz Nizipli Date: Thu, 16 Jan 2025 10:14:39 -0500 Subject: [PATCH 03/11] add rest of node:net api (#3339) --- src/node/net.ts | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/node/net.ts b/src/node/net.ts index 89bfab07b41..197acc2f2d4 100644 --- a/src/node/net.ts +++ b/src/node/net.ts @@ -184,6 +184,18 @@ type SocketClass = SocketType & { prototype: SocketClass; }; +export function BlockList(): void { + throw new Error('BlockList is not implemented'); +} + +export function SocketAddress(): void { + throw new Error('SocketAddress is not implemented'); +} + +export function Server(): void { + throw new Error('Server is not implemented'); +} + export const Socket = function Socket( this: SocketClass, options?: SocketOptions @@ -1343,6 +1355,10 @@ export function connect(...args: unknown[]): SocketClass { export const createConnection = connect; +export function createServer(): void { + throw new Error('createServer() is not implemented'); +} + export function getDefaultAutoSelectFamily(): boolean { // This is the only value we support. return false; @@ -1384,10 +1400,14 @@ export function isIPv6(input: unknown): boolean { } export default { + BlockList, + SocketAddress, Stream: Socket, + Server, Socket, connect, createConnection, + createServer, getDefaultAutoSelectFamily, setDefaultAutoSelectFamily, getDefaultAutoSelectFamilyAttemptTimeout, From c2a63234c77c7d6c7c0d9e33df1a0db2a5ef2841 Mon Sep 17 00:00:00 2001 From: Yagiz Nizipli Date: Thu, 16 Jan 2025 11:22:37 -0500 Subject: [PATCH 04/11] update node:net to enable mongodb and mysql usage (#3340) Co-authored-by: Victor Berchet --- src/node/net.ts | 18 ++++++++---------- src/workerd/api/node/tests/net-nodejs-test.js | 4 +++- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/node/net.ts b/src/node/net.ts index 197acc2f2d4..cceaa065022 100644 --- a/src/node/net.ts +++ b/src/node/net.ts @@ -769,11 +769,13 @@ Socket.prototype.setNoDelay = function ( Socket.prototype.setKeepAlive = function ( this: SocketClass, - enable?: boolean, + _enable?: boolean, _initialDelay?: number ): SocketClass { - if (!enable) return this; - throw new ERR_INVALID_ARG_VALUE('enable', enable, 'is not supported'); + // Ignore this for now. + // This is used by services like mySQL. + // TODO(soon): Investigate supporting this. + return this; }; // @ts-expect-error TS2322 Intentionally no-op @@ -960,13 +962,9 @@ function initializeConnection( let { port = 0 } = options; if (autoSelectFamily != null) { - // We don't support this option. If the value is falsy, we can safely ignore it. - // If the value is truthy, we'll throw an error. - throw new ERR_INVALID_ARG_VALUE( - 'options.autoSelectFamily', - autoSelectFamily, - 'is not supported' - ); + // We don't support this option. + // We shouldn't throw this because services like mongodb depends on it. + // TODO(soon): Investigate supporting this. } if (typeof port !== 'number' && typeof port !== 'string') { diff --git a/src/workerd/api/node/tests/net-nodejs-test.js b/src/workerd/api/node/tests/net-nodejs-test.js index 8718cb7446e..b3b5ede1734 100644 --- a/src/workerd/api/node/tests/net-nodejs-test.js +++ b/src/workerd/api/node/tests/net-nodejs-test.js @@ -443,10 +443,12 @@ export const testNetConnectImmediateFinish = { // something different than the original Node.js test export const testNetConnectKeepAlive = { async test() { + // Test that setKeepAlive call does not throw. throws(() => new net.Socket({ keepAlive: true })); const c = new net.Socket(); c.setKeepAlive(false); - throws(() => c.setKeepAlive(true)); + c.setKeepAlive(true); + // throws(() => c.setKeepAlive(true)); }, }; From 5377f9f0ae734267cb54c35086687e13513a19c5 Mon Sep 17 00:00:00 2001 From: Felix Hanau Date: Wed, 15 Jan 2025 22:29:11 -0500 Subject: [PATCH 05/11] 01-15-25 Roll deps - With latest rules_rust, -Cembed-bitcode=n is set automatically if LTO is disabled, no need to add it manually - Rust crate data is no longer passed at compile time, so use compile_data in cxxbridge-cmd build file --- .bazelrc | 7 ++----- WORKSPACE | 2 +- build/deps/build_deps.jsonc | 6 +----- build/deps/gen/dep_aspect_bazel_lib.bzl | 8 ++++---- build/deps/gen/dep_aspect_rules_js.bzl | 8 ++++---- build/deps/gen/dep_aspect_rules_ts.bzl | 8 ++++---- build/deps/gen/dep_buildifier_darwin_amd64.bzl | 6 +++--- build/deps/gen/dep_buildifier_darwin_arm64.bzl | 6 +++--- build/deps/gen/dep_buildifier_linux_amd64.bzl | 6 +++--- build/deps/gen/dep_buildifier_linux_arm64.bzl | 6 +++--- build/deps/gen/dep_buildifier_windows_amd64.bzl | 6 +++--- build/deps/gen/dep_capnp_cpp.bzl | 8 ++++---- build/deps/gen/dep_cargo_bazel_linux_arm64.bzl | 6 +++--- build/deps/gen/dep_cargo_bazel_linux_x64.bzl | 6 +++--- build/deps/gen/dep_cargo_bazel_macos_arm64.bzl | 6 +++--- build/deps/gen/dep_cargo_bazel_macos_x64.bzl | 6 +++--- build/deps/gen/dep_cargo_bazel_win_x64.bzl | 6 +++--- build/deps/gen/dep_cxxbridge_cmd.bzl | 8 ++++---- build/deps/gen/dep_rules_rust.bzl | 6 +++--- build/deps/gen/dep_simdutf.bzl | 6 +++--- deps/rust/BUILD.cxxbridge-cmd | 2 +- 21 files changed, 61 insertions(+), 68 deletions(-) diff --git a/.bazelrc b/.bazelrc index 9df5288d047..91dae7dc4cd 100644 --- a/.bazelrc +++ b/.bazelrc @@ -156,13 +156,10 @@ build --cxxopt="-fbracket-depth=512" --host_cxxopt="-fbracket-depth=512" build --@capnp-cpp//src/capnp:gen_rust=True # Additional Rust flags (see https://doc.rust-lang.org/rustc/codegen-options/index.html) -# Need to disable debug-assertions for fastbuild, should be off automatically for opt. As long as -# rust-level LTO is not enabled, LLVM bitcode is not needed so -C embed-bitcode=n provides a free -# improvement to build speed and Rust code size. +# Need to disable debug-assertions for fastbuild, should be off automatically for opt. # Using extra_rustc_flag for non-exec flags so they can be overwritten selectively. -build --@rules_rust//:extra_rustc_flag=-Cembed-bitcode=n build --@rules_rust//:extra_rustc_flag=-Cdebug-assertions=n -build --@rules_rust//:extra_exec_rustc_flags=-C,debug-assertions=n,-C,embed-bitcode=n +build --@rules_rust//:extra_exec_rustc_flags=-Cdebug-assertions=n # We default to not enabling debug assertions and unwinding for Rust. For debug builds this is not # the right setting, but unfortunately we can't set that directly. diff --git a/WORKSPACE b/WORKSPACE index b4330515343..a33bc7ab487 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -191,7 +191,7 @@ rust_register_toolchains( # Add support for macOS rosetta "aarch64-unknown-linux-gnu", ], - versions = ["1.82.0"], # LLVM 19 + versions = ["1.83.0"], # LLVM 19 ) load("@rules_rust//crate_universe:repositories.bzl", "crate_universe_dependencies") diff --git a/build/deps/build_deps.jsonc b/build/deps/build_deps.jsonc index 6d6631ce3d4..b3fa61ad5fb 100644 --- a/build/deps/build_deps.jsonc +++ b/build/deps/build_deps.jsonc @@ -116,11 +116,7 @@ "type": "github_release", "owner": "aspect-build", "repo": "rules_js", - "file_regex": "^rules_js-", - // TODO(cleanup): On latest version, Windows build fails with `tar.exe: Error opening archive: - // Can't initialize filter; unable to run program "gzip -d"`, likely due to - // https://github.com/aspect-build/rules_js/pull/1905/files. Develop a workaround. - "freeze_version": "v2.0.1" + "file_regex": "^rules_js-" }, { "name": "aspect_rules_ts", diff --git a/build/deps/gen/dep_aspect_bazel_lib.bzl b/build/deps/gen/dep_aspect_bazel_lib.bzl index cade4a37fea..8d8a670b16a 100644 --- a/build/deps/gen/dep_aspect_bazel_lib.bzl +++ b/build/deps/gen/dep_aspect_bazel_lib.bzl @@ -2,10 +2,10 @@ load("@//:build/http.bzl", "http_archive") -TAG_NAME = "v2.9.4" -URL = "https://github.com/bazel-contrib/bazel-lib/releases/download/v2.9.4/bazel-lib-v2.9.4.tar.gz" -STRIP_PREFIX = "bazel-lib-2.9.4" -SHA256 = "349aabd3c2b96caeda6181eb0ae1f14f2a1d9f3cd3c8b05d57f709ceb12e9fb3" +TAG_NAME = "v2.11.0" +URL = "https://github.com/bazel-contrib/bazel-lib/releases/download/v2.11.0/bazel-lib-v2.11.0.tar.gz" +STRIP_PREFIX = "bazel-lib-2.11.0" +SHA256 = "c96db69dd2714a37f3298338a1a42b27e3a2696c3b36dd4441b9bf7a1a12bee0" TYPE = "tgz" def dep_aspect_bazel_lib(): diff --git a/build/deps/gen/dep_aspect_rules_js.bzl b/build/deps/gen/dep_aspect_rules_js.bzl index b8bcf064c8e..5c0f2b868af 100644 --- a/build/deps/gen/dep_aspect_rules_js.bzl +++ b/build/deps/gen/dep_aspect_rules_js.bzl @@ -2,10 +2,10 @@ load("@//:build/http.bzl", "http_archive") -TAG_NAME = "v2.0.1" -URL = "https://github.com/aspect-build/rules_js/releases/download/v2.0.1/rules_js-v2.0.1.tar.gz" -STRIP_PREFIX = "rules_js-2.0.1" -SHA256 = "4cab6898f0ff8048e32640cce06a47aa4b92b2fb330d055940f95f24c8ebb868" +TAG_NAME = "v2.1.2" +URL = "https://github.com/aspect-build/rules_js/releases/download/v2.1.2/rules_js-v2.1.2.tar.gz" +STRIP_PREFIX = "rules_js-2.1.2" +SHA256 = "fbc34d815a0cc52183a1a26732fc0329e26774a51abbe0f26fc9fd2dab6133b4" TYPE = "tgz" def dep_aspect_rules_js(): diff --git a/build/deps/gen/dep_aspect_rules_ts.bzl b/build/deps/gen/dep_aspect_rules_ts.bzl index 7c291427228..6e7ffe99bb5 100644 --- a/build/deps/gen/dep_aspect_rules_ts.bzl +++ b/build/deps/gen/dep_aspect_rules_ts.bzl @@ -2,10 +2,10 @@ load("@//:build/http.bzl", "http_archive") -TAG_NAME = "v3.3.2" -URL = "https://github.com/aspect-build/rules_ts/releases/download/v3.3.2/rules_ts-v3.3.2.tar.gz" -STRIP_PREFIX = "rules_ts-3.3.2" -SHA256 = "cff3137b043ff6bf1a2542fd9691dc762432370cd39eb4bb0756d288de52067d" +TAG_NAME = "v3.4.0" +URL = "https://github.com/aspect-build/rules_ts/releases/download/v3.4.0/rules_ts-v3.4.0.tar.gz" +STRIP_PREFIX = "rules_ts-3.4.0" +SHA256 = "013a10b2b457add73b081780e604778eb50a141709f9194298f97761acdcc169" TYPE = "tgz" def dep_aspect_rules_ts(): diff --git a/build/deps/gen/dep_buildifier_darwin_amd64.bzl b/build/deps/gen/dep_buildifier_darwin_amd64.bzl index b030e191c69..785845d7d8f 100644 --- a/build/deps/gen/dep_buildifier_darwin_amd64.bzl +++ b/build/deps/gen/dep_buildifier_darwin_amd64.bzl @@ -2,9 +2,9 @@ load("@//:build/http.bzl", "http_file") -TAG_NAME = "v7.3.1" -URL = "https://github.com/bazelbuild/buildtools/releases/download/v7.3.1/buildifier-darwin-amd64" -SHA256 = "375f823103d01620aaec20a0c29c6cbca99f4fd0725ae30b93655c6704f44d71" +TAG_NAME = "v8.0.1" +URL = "https://github.com/bazelbuild/buildtools/releases/download/v8.0.1/buildifier-darwin-amd64" +SHA256 = "802b013211dbcf91e3c0658ba33ecb3932ef5a6f6764a0b13efcec4e2df04c83" def dep_buildifier_darwin_amd64(): http_file( diff --git a/build/deps/gen/dep_buildifier_darwin_arm64.bzl b/build/deps/gen/dep_buildifier_darwin_arm64.bzl index 7201ac3d5b5..7da7830040b 100644 --- a/build/deps/gen/dep_buildifier_darwin_arm64.bzl +++ b/build/deps/gen/dep_buildifier_darwin_arm64.bzl @@ -2,9 +2,9 @@ load("@//:build/http.bzl", "http_file") -TAG_NAME = "v7.3.1" -URL = "https://github.com/bazelbuild/buildtools/releases/download/v7.3.1/buildifier-darwin-arm64" -SHA256 = "5a6afc6ac7a09f5455ba0b89bd99d5ae23b4174dc5dc9d6c0ed5ce8caac3f813" +TAG_NAME = "v8.0.1" +URL = "https://github.com/bazelbuild/buildtools/releases/download/v8.0.1/buildifier-darwin-arm64" +SHA256 = "833e2afc331b9ad8f6b038ad3d69ceeaf97651900bf2a3a45f54f42cafe0bfd3" def dep_buildifier_darwin_arm64(): http_file( diff --git a/build/deps/gen/dep_buildifier_linux_amd64.bzl b/build/deps/gen/dep_buildifier_linux_amd64.bzl index 04bc02f4aa4..388c733070d 100644 --- a/build/deps/gen/dep_buildifier_linux_amd64.bzl +++ b/build/deps/gen/dep_buildifier_linux_amd64.bzl @@ -2,9 +2,9 @@ load("@//:build/http.bzl", "http_file") -TAG_NAME = "v7.3.1" -URL = "https://github.com/bazelbuild/buildtools/releases/download/v7.3.1/buildifier-linux-amd64" -SHA256 = "5474cc5128a74e806783d54081f581662c4be8ae65022f557e9281ed5dc88009" +TAG_NAME = "v8.0.1" +URL = "https://github.com/bazelbuild/buildtools/releases/download/v8.0.1/buildifier-linux-amd64" +SHA256 = "1976053ed4decd6dd93170885b4387eddc76ec70dc2697b2e91a9af83269418a" def dep_buildifier_linux_amd64(): http_file( diff --git a/build/deps/gen/dep_buildifier_linux_arm64.bzl b/build/deps/gen/dep_buildifier_linux_arm64.bzl index da1cf660f63..3979da6362a 100644 --- a/build/deps/gen/dep_buildifier_linux_arm64.bzl +++ b/build/deps/gen/dep_buildifier_linux_arm64.bzl @@ -2,9 +2,9 @@ load("@//:build/http.bzl", "http_file") -TAG_NAME = "v7.3.1" -URL = "https://github.com/bazelbuild/buildtools/releases/download/v7.3.1/buildifier-linux-arm64" -SHA256 = "0bf86c4bfffaf4f08eed77bde5b2082e4ae5039a11e2e8b03984c173c34a561c" +TAG_NAME = "v8.0.1" +URL = "https://github.com/bazelbuild/buildtools/releases/download/v8.0.1/buildifier-linux-arm64" +SHA256 = "93078c57763493bdc2914ed340544500b8f3497341a62e90f00e9e184c4d9c2c" def dep_buildifier_linux_arm64(): http_file( diff --git a/build/deps/gen/dep_buildifier_windows_amd64.bzl b/build/deps/gen/dep_buildifier_windows_amd64.bzl index 05187b27b35..ff56daf57e2 100644 --- a/build/deps/gen/dep_buildifier_windows_amd64.bzl +++ b/build/deps/gen/dep_buildifier_windows_amd64.bzl @@ -2,9 +2,9 @@ load("@//:build/http.bzl", "http_file") -TAG_NAME = "v7.3.1" -URL = "https://github.com/bazelbuild/buildtools/releases/download/v7.3.1/buildifier-windows-amd64.exe" -SHA256 = "370cd576075ad29930a82f5de132f1a1de4084c784a82514bd4da80c85acf4a8" +TAG_NAME = "v8.0.1" +URL = "https://github.com/bazelbuild/buildtools/releases/download/v8.0.1/buildifier-windows-amd64.exe" +SHA256 = "6edc9247e6d42d27fb67b9509bb795d159a12468faa89e9f290dcadc26571c31" def dep_buildifier_windows_amd64(): http_file( diff --git a/build/deps/gen/dep_capnp_cpp.bzl b/build/deps/gen/dep_capnp_cpp.bzl index 4eb88898c5f..bde1155122b 100644 --- a/build/deps/gen/dep_capnp_cpp.bzl +++ b/build/deps/gen/dep_capnp_cpp.bzl @@ -2,11 +2,11 @@ load("@//:build/http.bzl", "http_archive") -URL = "https://github.com/capnproto/capnproto/tarball/1c676b2df7f97220607591a38c28ce7e4a968ad4" -STRIP_PREFIX = "capnproto-capnproto-1c676b2/c++" -SHA256 = "9281b860a778c9427c55be647d1247d9f8373ab9023e671505881bf2172eba04" +URL = "https://github.com/capnproto/capnproto/tarball/f600d249496c55289fa07fd6a21cadeeb340edb9" +STRIP_PREFIX = "capnproto-capnproto-f600d24/c++" +SHA256 = "1d96cd31ad93eaf8194ba51d4a009a4f08332ad643740c4b49c081a515a27a60" TYPE = "tgz" -COMMIT = "1c676b2df7f97220607591a38c28ce7e4a968ad4" +COMMIT = "f600d249496c55289fa07fd6a21cadeeb340edb9" def dep_capnp_cpp(): http_archive( diff --git a/build/deps/gen/dep_cargo_bazel_linux_arm64.bzl b/build/deps/gen/dep_cargo_bazel_linux_arm64.bzl index 66fadd723fc..9025d43659c 100644 --- a/build/deps/gen/dep_cargo_bazel_linux_arm64.bzl +++ b/build/deps/gen/dep_cargo_bazel_linux_arm64.bzl @@ -2,9 +2,9 @@ load("@//:build/http.bzl", "http_file") -TAG_NAME = "0.54.1" -URL = "https://github.com/bazelbuild/rules_rust/releases/download/0.54.1/cargo-bazel-aarch64-unknown-linux-gnu" -SHA256 = "3792feb084bd43b9a7a9cd75be86ee9910b46db59360d6b29c9cca2f8889a0aa" +TAG_NAME = "0.56.0" +URL = "https://github.com/bazelbuild/rules_rust/releases/download/0.56.0/cargo-bazel-aarch64-unknown-linux-gnu" +SHA256 = "f3a35b137221d4627ba0ed62f775c4d3cbb45b6db839ae47f2ebfd48abe44a91" def dep_cargo_bazel_linux_arm64(): http_file( diff --git a/build/deps/gen/dep_cargo_bazel_linux_x64.bzl b/build/deps/gen/dep_cargo_bazel_linux_x64.bzl index 35ccef58ebe..7e1cf0490ce 100644 --- a/build/deps/gen/dep_cargo_bazel_linux_x64.bzl +++ b/build/deps/gen/dep_cargo_bazel_linux_x64.bzl @@ -2,9 +2,9 @@ load("@//:build/http.bzl", "http_file") -TAG_NAME = "0.54.1" -URL = "https://github.com/bazelbuild/rules_rust/releases/download/0.54.1/cargo-bazel-x86_64-unknown-linux-gnu" -SHA256 = "2b9d07f34694f63f0cc704989ad6ec148ff8d126579832f4f4d88edea75875b2" +TAG_NAME = "0.56.0" +URL = "https://github.com/bazelbuild/rules_rust/releases/download/0.56.0/cargo-bazel-x86_64-unknown-linux-gnu" +SHA256 = "53418ae5457040e84009f7c69b070e1f12f10a9a3a3ef4f5d0829c66e5438ba1" def dep_cargo_bazel_linux_x64(): http_file( diff --git a/build/deps/gen/dep_cargo_bazel_macos_arm64.bzl b/build/deps/gen/dep_cargo_bazel_macos_arm64.bzl index 5c3ef7d09f4..7b7dfc7b546 100644 --- a/build/deps/gen/dep_cargo_bazel_macos_arm64.bzl +++ b/build/deps/gen/dep_cargo_bazel_macos_arm64.bzl @@ -2,9 +2,9 @@ load("@//:build/http.bzl", "http_file") -TAG_NAME = "0.54.1" -URL = "https://github.com/bazelbuild/rules_rust/releases/download/0.54.1/cargo-bazel-aarch64-apple-darwin" -SHA256 = "8204746334a17823bd6a54ce2c3821b0bdca96576700d568e2ca2bd8224dc0ea" +TAG_NAME = "0.56.0" +URL = "https://github.com/bazelbuild/rules_rust/releases/download/0.56.0/cargo-bazel-aarch64-apple-darwin" +SHA256 = "5143e347ec45e0fcc249f1de44f0dce227062579ad364a0b269b39f0ce49f1d8" def dep_cargo_bazel_macos_arm64(): http_file( diff --git a/build/deps/gen/dep_cargo_bazel_macos_x64.bzl b/build/deps/gen/dep_cargo_bazel_macos_x64.bzl index fad845ed359..f23ec641577 100644 --- a/build/deps/gen/dep_cargo_bazel_macos_x64.bzl +++ b/build/deps/gen/dep_cargo_bazel_macos_x64.bzl @@ -2,9 +2,9 @@ load("@//:build/http.bzl", "http_file") -TAG_NAME = "0.54.1" -URL = "https://github.com/bazelbuild/rules_rust/releases/download/0.54.1/cargo-bazel-x86_64-apple-darwin" -SHA256 = "2ee14b230d32c05415852b7a388b76e700c87c506459e5b31ced19d6c131b6d0" +TAG_NAME = "0.56.0" +URL = "https://github.com/bazelbuild/rules_rust/releases/download/0.56.0/cargo-bazel-x86_64-apple-darwin" +SHA256 = "9e8a92674771982f68031d0cc516ba08b34dd736c4f818d51349547f2136d012" def dep_cargo_bazel_macos_x64(): http_file( diff --git a/build/deps/gen/dep_cargo_bazel_win_x64.bzl b/build/deps/gen/dep_cargo_bazel_win_x64.bzl index 02ce5e19dc6..87ca52f7855 100644 --- a/build/deps/gen/dep_cargo_bazel_win_x64.bzl +++ b/build/deps/gen/dep_cargo_bazel_win_x64.bzl @@ -2,9 +2,9 @@ load("@//:build/http.bzl", "http_file") -TAG_NAME = "0.54.1" -URL = "https://github.com/bazelbuild/rules_rust/releases/download/0.54.1/cargo-bazel-x86_64-pc-windows-msvc.exe" -SHA256 = "609e894dc52576407afd5edf3b293eab2774e4fffc25ae212ab138556ff7306d" +TAG_NAME = "0.56.0" +URL = "https://github.com/bazelbuild/rules_rust/releases/download/0.56.0/cargo-bazel-x86_64-pc-windows-msvc.exe" +SHA256 = "5c62a1c42b71af6c2765dc5f9518eba22ed28276a65f7accf5a2bca0c5cfaf74" def dep_cargo_bazel_win_x64(): http_file( diff --git a/build/deps/gen/dep_cxxbridge_cmd.bzl b/build/deps/gen/dep_cxxbridge_cmd.bzl index 858301a7a93..8b1b0b3f5b0 100644 --- a/build/deps/gen/dep_cxxbridge_cmd.bzl +++ b/build/deps/gen/dep_cxxbridge_cmd.bzl @@ -2,11 +2,11 @@ load("@//:build/http.bzl", "http_archive") -URL = "https://crates.io/api/v1/crates/cxxbridge-cmd/1.0.131/download" -STRIP_PREFIX = "cxxbridge-cmd-1.0.131" -SHA256 = "5b12e3f7b0477bd2e469107b778d333171cae561a555e96157e67bbd9f42e54f" +URL = "https://crates.io/api/v1/crates/cxxbridge-cmd/1.0.136/download" +STRIP_PREFIX = "cxxbridge-cmd-1.0.136" +SHA256 = "6c33fd49f5d956a1b7ee5f7a9768d58580c6752838d92e39d0d56439efdedc35" TYPE = "tgz" -VERSION = "1.0.131" +VERSION = "1.0.136" def dep_cxxbridge_cmd(): http_archive( diff --git a/build/deps/gen/dep_rules_rust.bzl b/build/deps/gen/dep_rules_rust.bzl index 604519cd905..1b8f7fa819b 100644 --- a/build/deps/gen/dep_rules_rust.bzl +++ b/build/deps/gen/dep_rules_rust.bzl @@ -2,10 +2,10 @@ load("@//:build/http.bzl", "http_archive") -TAG_NAME = "0.54.1" -URL = "https://github.com/bazelbuild/rules_rust/releases/download/0.54.1/rules_rust-v0.54.1.tar.gz" +TAG_NAME = "0.56.0" +URL = "https://github.com/bazelbuild/rules_rust/releases/download/0.56.0/rules_rust-0.56.0.tar.gz" STRIP_PREFIX = "" -SHA256 = "af4f56caae50a99a68bfce39b141b509dd68548c8204b98ab7a1cafc94d5bb02" +SHA256 = "f1306aac0b258b790df01ad9abc6abb0df0b65416c74b4ef27f4aab298780a64" TYPE = "tgz" def dep_rules_rust(): diff --git a/build/deps/gen/dep_simdutf.bzl b/build/deps/gen/dep_simdutf.bzl index 941a8081757..b9ed5400e64 100644 --- a/build/deps/gen/dep_simdutf.bzl +++ b/build/deps/gen/dep_simdutf.bzl @@ -2,10 +2,10 @@ load("@//:build/http.bzl", "http_archive") -TAG_NAME = "v5.6.3" -URL = "https://github.com/simdutf/simdutf/releases/download/v5.6.3/singleheader.zip" +TAG_NAME = "v6.0.3" +URL = "https://github.com/simdutf/simdutf/releases/download/v6.0.3/singleheader.zip" STRIP_PREFIX = "" -SHA256 = "33e2968e0f14a72c970c4838e08e502438f37c0a856a9238d69d9161727ba4c2" +SHA256 = "0e5ba4bc981633bb024ee066833f733b7d3422bc6250f6a6b5bbc09121b782af" TYPE = "zip" def dep_simdutf(): diff --git a/deps/rust/BUILD.cxxbridge-cmd b/deps/rust/BUILD.cxxbridge-cmd index 8305de5c4a9..c5432ee6f50 100644 --- a/deps/rust/BUILD.cxxbridge-cmd +++ b/deps/rust/BUILD.cxxbridge-cmd @@ -5,7 +5,7 @@ load("@rules_rust//rust:defs.bzl", "rust_binary") rust_binary( name = "cxxbridge-cmd", srcs = glob(["src/**/*.rs"]), - data = ["src/gen/include/cxx.h"], + compile_data = ["src/gen/include/cxx.h"], edition = "2021", visibility = ["//visibility:public"], deps = [ From bcbe94f6038c25b578f2bf4c68f8eeb6f748442c Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Thu, 16 Jan 2025 17:59:51 +0000 Subject: [PATCH 06/11] Add missing python-fetch-individual-packages autogate check. --- src/workerd/server/workerd-api.c++ | 3 ++- src/workerd/util/autogate.c++ | 2 ++ src/workerd/util/autogate.h | 3 +++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/workerd/server/workerd-api.c++ b/src/workerd/server/workerd-api.c++ index b574b84ecb5..da2768139a7 100644 --- a/src/workerd/server/workerd-api.c++ +++ b/src/workerd/server/workerd-api.c++ @@ -565,7 +565,8 @@ void WorkerdApi::compileModules(jsg::Lock& lockParam, jsg::ModuleRegistry::Type::INTERNAL); // Inject packages tar file - if (featureFlags.getPythonExternalPackages()) { + if (featureFlags.getPythonExternalPackages() || + util::Autogate::isEnabled(util::AutogateKey::PYTHON_FETCH_INDIVIDUAL_PACKAGES)) { modules->addBuiltinModule("pyodide-internal:packages_tar_reader", "export default { }"_kj, workerd::jsg::ModuleRegistry::Type::INTERNAL, {}); } else { diff --git a/src/workerd/util/autogate.c++ b/src/workerd/util/autogate.c++ index c5264a9fa03..49e0bf03ab0 100644 --- a/src/workerd/util/autogate.c++ +++ b/src/workerd/util/autogate.c++ @@ -19,6 +19,8 @@ kj::StringPtr KJ_STRINGIFY(AutogateKey key) { return "test-workerd"_kj; case AutogateKey::STREAMING_TAIL_WORKERS: return "streaming-tail-workers"_kj; + case AutogateKey::PYTHON_FETCH_INDIVIDUAL_PACKAGES: + return "python-fetch-individual-packages"; case AutogateKey::NumOfKeys: KJ_FAIL_ASSERT("NumOfKeys should not be used in getName"); } diff --git a/src/workerd/util/autogate.h b/src/workerd/util/autogate.h index f44f2193147..51dc29e63d0 100644 --- a/src/workerd/util/autogate.h +++ b/src/workerd/util/autogate.h @@ -15,6 +15,9 @@ namespace workerd::util { enum class AutogateKey { TEST_WORKERD, STREAMING_TAIL_WORKERS, + // Fetches Python packages as individual bundles from GCS instead of using a single big bundle + // embedded in the binary + PYTHON_FETCH_INDIVIDUAL_PACKAGES, NumOfKeys // Reserved for iteration. }; From 0ae9812e66c83e556efc4acb7e753b40132e9ec2 Mon Sep 17 00:00:00 2001 From: Felix Hanau Date: Sun, 12 Jan 2025 20:37:33 -0500 Subject: [PATCH 07/11] [build] Do not have //src/workerd/io depend on src/cloudflare, src/node This was making io depend on the gen-compile-cache binary, which delayed building source files in io and creating a bottleneck. Drive-by: Fix LLVM 20 operator warnings --- src/workerd/api/BUILD.bazel | 2 ++ src/workerd/api/node/BUILD.bazel | 2 +- src/workerd/io/BUILD.bazel | 2 -- src/workerd/jsg/url.c++ | 2 +- src/workerd/server/BUILD.bazel | 2 ++ src/workerd/server/server-test.c++ | 2 +- src/workerd/tools/BUILD.bazel | 2 ++ 7 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/workerd/api/BUILD.bazel b/src/workerd/api/BUILD.bazel index 61625d9121a..700159290d7 100644 --- a/src/workerd/api/BUILD.bazel +++ b/src/workerd/api/BUILD.bazel @@ -95,6 +95,8 @@ wd_cc_library( ":memory-cache", ":pyodide", ":r2", + "//src/cloudflare", + "//src/node", "//src/pyodide", "//src/pyodide:pyodide_extra_capnp", "//src/workerd/api/node", diff --git a/src/workerd/api/node/BUILD.bazel b/src/workerd/api/node/BUILD.bazel index 1e348768f40..f168df6fdb5 100644 --- a/src/workerd/api/node/BUILD.bazel +++ b/src/workerd/api/node/BUILD.bazel @@ -1,5 +1,4 @@ load("@aspect_rules_js//js:defs.bzl", "js_binary") -load("@bazel_skylib//lib:selects.bzl", "selects") load("//:build/kj_test.bzl", "kj_test") load("//:build/wd_cc_library.bzl", "wd_cc_library") load("//:build/wd_test.bzl", "wd_test") @@ -29,6 +28,7 @@ wd_cc_library( visibility = ["//visibility:public"], deps = [ ":node-core", + "//src/node", "@capnp-cpp//src/kj/compat:kj-brotli", ], ) diff --git a/src/workerd/io/BUILD.bazel b/src/workerd/io/BUILD.bazel index 04032baafd7..f68bc8cd884 100644 --- a/src/workerd/io/BUILD.bazel +++ b/src/workerd/io/BUILD.bazel @@ -106,8 +106,6 @@ wd_cc_library( ":supported-compatibility-date_capnp", ":trace", ":worker-interface", - "//src/cloudflare", - "//src/node", "//src/workerd/api:analytics-engine_capnp", "//src/workerd/api:data-url", "//src/workerd/api:deferred-proxy", diff --git a/src/workerd/jsg/url.c++ b/src/workerd/jsg/url.c++ index b5fe9d79c26..c5791a69651 100644 --- a/src/workerd/jsg/url.c++ +++ b/src/workerd/jsg/url.c++ @@ -2301,6 +2301,6 @@ UrlPattern::UrlPattern(kj::Array components, bool ignoreCase) } // namespace workerd::jsg -const workerd::jsg::Url operator"" _url(const char* str, size_t size) { +const workerd::jsg::Url operator""_url(const char* str, size_t size) { return KJ_ASSERT_NONNULL(workerd::jsg::Url::tryParse(kj::ArrayPtr(str, size))); } diff --git a/src/workerd/server/BUILD.bazel b/src/workerd/server/BUILD.bazel index 58037da488b..9bbfcd29e3f 100644 --- a/src/workerd/server/BUILD.bazel +++ b/src/workerd/server/BUILD.bazel @@ -114,6 +114,8 @@ wd_cc_library( ":alarm-scheduler", ":workerd_capnp", "//deps/rust:runtime", + "//src/cloudflare", + "//src/node", "//src/workerd/api:html-rewriter", "//src/workerd/api:hyperdrive", "//src/workerd/api:memory-cache", diff --git a/src/workerd/server/server-test.c++ b/src/workerd/server/server-test.c++ index 7754ea18d09..1e79f723114 100644 --- a/src/workerd/server/server-test.c++ +++ b/src/workerd/server/server-test.c++ @@ -51,7 +51,7 @@ kj::Own parseConfig(kj::StringPtr text, kj::SourceLocati // This is intended to allow multi-line raw text to be specified conveniently using C++11 // `R"(blah)"` literal syntax, without the need to mess up indentation relative to the // surrounding code. -kj::String operator"" _blockquote(const char* str, size_t n) { +kj::String operator""_blockquote(const char* str, size_t n) { kj::StringPtr text(str, n); // Ignore a leading newline so that `R"(` can be placed on the line before the initial indent. diff --git a/src/workerd/tools/BUILD.bazel b/src/workerd/tools/BUILD.bazel index 2331e0eeee9..27ea9c42e9a 100644 --- a/src/workerd/tools/BUILD.bazel +++ b/src/workerd/tools/BUILD.bazel @@ -18,6 +18,8 @@ cc_ast_dump( "//conditions:default": [], }), deps = [ + "//src/cloudflare", + "//src/node", "//src/workerd/api:html-rewriter", "//src/workerd/api:hyperdrive", "//src/workerd/api:memory-cache", From e6a9cc560282d4c34807e885612910a79bdb7e83 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Fri, 17 Jan 2025 16:07:00 +0100 Subject: [PATCH 08/11] Clean up snapshot upload code (#3349) And fix workerd's --snapshot-to-disk functionality. We don't need this deferred upload stuff anymore so take it out. --- src/pyodide/internal/python.ts | 4 +- src/pyodide/internal/snapshot.ts | 60 ++----------------------- src/pyodide/python-entrypoint-helper.ts | 13 ------ 3 files changed, 6 insertions(+), 71 deletions(-) diff --git a/src/pyodide/internal/python.ts b/src/pyodide/internal/python.ts index baf8c1923a3..4104fe99944 100644 --- a/src/pyodide/internal/python.ts +++ b/src/pyodide/internal/python.ts @@ -9,7 +9,7 @@ import { import { SHOULD_RESTORE_SNAPSHOT, finishSnapshotSetup, - maybeSetupSnapshotUpload, + maybeCollectSnapshot, restoreSnapshot, preloadDynamicLibs, } from 'pyodide-internal:snapshot'; @@ -92,7 +92,7 @@ export async function loadPyodide( prepareWasmLinearMemory(Module) ); - maybeSetupSnapshotUpload(Module); + maybeCollectSnapshot(Module); // Mount worker files after doing snapshot upload so we ensure that data from the files is never // present in snapshot memory. mountWorkerFiles(Module); diff --git a/src/pyodide/internal/snapshot.ts b/src/pyodide/internal/snapshot.ts index e0c62e27195..fbb2279ae2c 100644 --- a/src/pyodide/internal/snapshot.ts +++ b/src/pyodide/internal/snapshot.ts @@ -26,10 +26,6 @@ let LOADED_BASELINE_SNAPSHOT: number; * `pyodide.loadPackage`. In trade we add memory snapshots here. */ -const TOP_LEVEL_SNAPSHOT = - ArtifactBundler.isEwValidating() || SHOULD_SNAPSHOT_TO_DISK; -const SHOULD_UPLOAD_SNAPSHOT = TOP_LEVEL_SNAPSHOT; - /** * Global variable for the memory snapshot. On the first run we stick a copy of * the linear memory here, on subsequent runs we can skip bootstrapping Python @@ -45,32 +41,6 @@ export let SHOULD_RESTORE_SNAPSHOT = false; */ let DSO_METADATA: any = {}; // TODO -/** - * Used to defer artifact upload. This is set during initialisation, but is executed during a - * request because an IO context is needed for the upload. - */ -let DEFERRED_UPLOAD_FUNCTION: (() => Promise) | undefined = undefined; - -export async function uploadArtifacts(): Promise { - if (DEFERRED_UPLOAD_FUNCTION) { - return await DEFERRED_UPLOAD_FUNCTION(); - } -} - -/** - * Used to hold the memory that needs to be uploaded for the validator. - */ -let MEMORY_TO_UPLOAD: ArtifactBundler.MemorySnapshotResult | undefined = - undefined; -function getMemoryToUpload(): ArtifactBundler.MemorySnapshotResult { - if (!MEMORY_TO_UPLOAD) { - throw new TypeError('Expected MEMORY_TO_UPLOAD to be set'); - } - const tmp = MEMORY_TO_UPLOAD; - MEMORY_TO_UPLOAD = undefined; - return tmp; -} - /** * Preload a dynamic library. * @@ -333,28 +303,6 @@ function makeLinearMemorySnapshot( }; } -function setUploadFunction( - snapshot: Uint8Array, - importedModulesList: string[] -): void { - if (snapshot.constructor.name !== 'Uint8Array') { - throw new TypeError('Expected TO_UPLOAD to be a Uint8Array'); - } - if (TOP_LEVEL_SNAPSHOT) { - MEMORY_TO_UPLOAD = { snapshot, importedModulesList }; - return; - } -} - -// TODO(later): Rename this to avoid `upload` nomenclature. -export function maybeSetupSnapshotUpload(Module: Module): void { - if (!SHOULD_UPLOAD_SNAPSHOT) { - return; - } - const { snapshot, importedModulesList } = makeLinearMemorySnapshot(Module); - setUploadFunction(snapshot, importedModulesList); -} - // "\x00snp" const SNAPSHOT_MAGIC = 0x706e7300; const CREATE_SNAPSHOT_VERSION = 2; @@ -465,11 +413,11 @@ export function finishSnapshotSetup(pyodide: Pyodide): void { } } -export function maybeStoreMemorySnapshot() { +export function maybeCollectSnapshot(Module: Module): void { if (ArtifactBundler.isEwValidating()) { - ArtifactBundler.storeMemorySnapshot(getMemoryToUpload()); + ArtifactBundler.storeMemorySnapshot(makeLinearMemorySnapshot(Module)); } else if (SHOULD_SNAPSHOT_TO_DISK) { - DiskCache.put('snapshot.bin', getMemoryToUpload().snapshot); - console.log('Saved snapshot to disk'); + const { snapshot } = makeLinearMemorySnapshot(Module); + DiskCache.put('snapshot.bin', snapshot); } } diff --git a/src/pyodide/python-entrypoint-helper.ts b/src/pyodide/python-entrypoint-helper.ts index 121b889472e..5aa8c42b0f3 100644 --- a/src/pyodide/python-entrypoint-helper.ts +++ b/src/pyodide/python-entrypoint-helper.ts @@ -2,10 +2,6 @@ // python-entrypoint.js USER module. import { loadPyodide } from 'pyodide-internal:python'; -import { - uploadArtifacts, - maybeStoreMemorySnapshot, -} from 'pyodide-internal:snapshot'; import { enterJaegerSpan } from 'pyodide-internal:jaeger'; import { TRANSITIVE_REQUIREMENTS, @@ -149,8 +145,6 @@ function makeHandler(pyHandlerName: string): Handler { } catch (e) { console.warn('Error in makeHandler'); reportError(e); - } finally { - args[2].waitUntil(uploadArtifacts()); } }; } @@ -180,13 +174,6 @@ try { } } } - /** - * Store the memory snapshot in the ArtifactBundler so that the validator can - * read it out. Needs to happen at the top level because the validator does - * not perform requests. In workerd, this will save a snapshot to disk if the - * `--python-save-snapshot` or `--python-save-baseline-snapshot` is passed. - */ - maybeStoreMemorySnapshot(); } catch (e) { console.warn('Error in top level in python-entrypoint-helper.js'); reportError(e); From 43139f0dd6ed880c314b45972be79dfeb3c3d223 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Thu, 16 Jan 2025 17:24:34 +0000 Subject: [PATCH 09/11] Fixes Python SDK tests when external package loading is enabled. --- src/workerd/server/tests/python/BUILD.bazel | 2 -- src/workerd/server/tests/python/sdk/proxy.js | 12 +++++++++ .../server/tests/python/sdk/sdk.wd-test | 27 ++++++++++++++++--- 3 files changed, 36 insertions(+), 5 deletions(-) create mode 100644 src/workerd/server/tests/python/sdk/proxy.js diff --git a/src/workerd/server/tests/python/BUILD.bazel b/src/workerd/server/tests/python/BUILD.bazel index bfa7eb38bbd..27e0872920b 100644 --- a/src/workerd/server/tests/python/BUILD.bazel +++ b/src/workerd/server/tests/python/BUILD.bazel @@ -30,8 +30,6 @@ py_wd_test("subdirectory") py_wd_test( "sdk", - # TODO(EW-8976): Disabled in dev due to test getting stuck - python_flags = ["0.26.0a2"], tags = [ # TODO(someday): Fix asan failure for this, see https://github.com/cloudflare/workerd/pull/3140#discussion_r1858273318 "no-asan", diff --git a/src/workerd/server/tests/python/sdk/proxy.js b/src/workerd/server/tests/python/sdk/proxy.js new file mode 100644 index 00000000000..f32b8b93d27 --- /dev/null +++ b/src/workerd/server/tests/python/sdk/proxy.js @@ -0,0 +1,12 @@ +export default { + async fetch(req, env) { + const url = new URL(req.url); + if (url.hostname == 'pyodide-packages.runtime-playground.workers.dev') { + return env.INTERNET.fetch(req); + } else if (url.hostname == 'example.com') { + return env.PYTHON.fetch(req); + } + + throw new Error('Invalid url: ' + url); + }, +}; diff --git a/src/workerd/server/tests/python/sdk/sdk.wd-test b/src/workerd/server/tests/python/sdk/sdk.wd-test index 7060d4ecc9c..3809929bf14 100644 --- a/src/workerd/server/tests/python/sdk/sdk.wd-test +++ b/src/workerd/server/tests/python/sdk/sdk.wd-test @@ -19,13 +19,34 @@ const server :Workerd.Worker = ( compatibilityFlags = [%PYTHON_FEATURE_FLAGS], ); +# We need this proxy so that internal requests made to fetch packages for Python Workers are sent +# out to the open internet and not to the python-server worker. +const jsChooserProxy :Workerd.Worker = ( + modules = [ + (name = "proxy.js", esModule = embed "proxy.js") + ], + bindings = [ + ( name = "PYTHON", service = "python-server" ), + ( name = "INTERNET", service = "external" ), + ], + compatibilityDate = "2024-10-01", +); + const unitTests :Workerd.Config = ( services = [ - ( name = "python-sdk", - worker = .python + ( name = "external", + network = ( + tlsOptions = (trustBrowserCas = true) + ) ), ( name = "internet", + worker = .jsChooserProxy + ), + ( name = "python-server", worker = .server - ) + ), + ( name = "python-sdk", + worker = .python + ), ], ); From 811fe745987ac1c0ef78b4f835ac2bd7d0a6b1a4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 17 Jan 2025 03:52:14 +0000 Subject: [PATCH 10/11] Bump jinja2 from 3.1.4 to 3.1.5 in /build/deps Bumps [jinja2](https://github.com/pallets/jinja) from 3.1.4 to 3.1.5. - [Release notes](https://github.com/pallets/jinja/releases) - [Changelog](https://github.com/pallets/jinja/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/jinja/compare/3.1.4...3.1.5) --- updated-dependencies: - dependency-name: jinja2 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- build/deps/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build/deps/requirements.txt b/build/deps/requirements.txt index b72528c5a8b..b5d38b32a2e 100644 --- a/build/deps/requirements.txt +++ b/build/deps/requirements.txt @@ -4,9 +4,9 @@ # # pip-compile --generate-hashes requirements.in # -jinja2==3.1.4 \ - --hash=sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369 \ - --hash=sha256:bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d +jinja2==3.1.5 \ + --hash=sha256:8fefff8dc3034e27bb80d67c671eb8a9bc424c0ef4c0826edbff304cceff43bb \ + --hash=sha256:aba0f4dc9ed8013c424088f68a5c226f7d6097ed89b246d7749c2ec4175c6adb # via -r requirements.in markupsafe==3.0.2 \ --hash=sha256:0bff5e0ae4ef2e1ae4fdf2dfd5b76c75e5c2fa4132d05fc1b0dabcd20c7e28c4 \ From c134deb3e2c6177c990ba0142d080242ef3531db Mon Sep 17 00:00:00 2001 From: Felix Hanau Date: Tue, 14 Jan 2025 21:19:25 -0500 Subject: [PATCH 11/11] [o11y] Adjust user span naming This is more descriptive: Websocket::constructor => websocket_open actor_subrequest => durable_object_subrequest hyperdrive_dev => hyperdrive_connect to match downstream name --- src/workerd/api/actor.c++ | 6 +++--- src/workerd/api/hyperdrive.c++ | 2 +- src/workerd/api/web-socket.c++ | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/workerd/api/actor.c++ b/src/workerd/api/actor.c++ index 397d350a099..8158b86f16b 100644 --- a/src/workerd/api/actor.c++ +++ b/src/workerd/api/actor.c++ @@ -40,7 +40,7 @@ class LocalActorOutgoingFactory final: public Fetcher::OutgoingFactory { }, {.inHouse = true, .wrapMetrics = true, - .operationName = kj::ConstString("actor_subrequest"_kjc)})); + .operationName = kj::ConstString("durable_object_subrequest"_kjc)})); } private: @@ -82,7 +82,7 @@ class GlobalActorOutgoingFactory final: public Fetcher::OutgoingFactory { }, {.inHouse = true, .wrapMetrics = true, - .operationName = kj::ConstString("actor_subrequest"_kjc)})); + .operationName = kj::ConstString("durable_object_subrequest"_kjc)})); } private: @@ -110,7 +110,7 @@ kj::Own ReplicaActorOutgoingFactory::newSingleUseClient( }, {.inHouse = true, .wrapMetrics = true, - .operationName = kj::ConstString("actor_subrequest"_kjc)})); + .operationName = kj::ConstString("durable_object_subrequest"_kjc)})); } jsg::Ref ColoLocalActorNamespace::get(kj::String actorId) { diff --git a/src/workerd/api/hyperdrive.c++ b/src/workerd/api/hyperdrive.c++ index d046a650483..41f143c927e 100644 --- a/src/workerd/api/hyperdrive.c++ +++ b/src/workerd/api/hyperdrive.c++ @@ -86,7 +86,7 @@ kj::String Hyperdrive::getConnectionString() { kj::Promise> Hyperdrive::connectToDb() { auto& context = IoContext::current(); auto service = - context.getSubrequestChannel(this->clientIndex, true, kj::none, "hyperdrive_dev"_kjc); + context.getSubrequestChannel(this->clientIndex, true, kj::none, "hyperdrive_connect"_kjc); kj::HttpHeaderTable headerTable; kj::HttpHeaders headers(headerTable); diff --git a/src/workerd/api/web-socket.c++ b/src/workerd/api/web-socket.c++ index 8d3d7fcf8f4..06a97c0e204 100644 --- a/src/workerd/api/web-socket.c++ +++ b/src/workerd/api/web-socket.c++ @@ -205,7 +205,7 @@ jsg::Ref WebSocket::constructor(jsg::Lock& js, urlRecord.fragment == kj::none, DOMSyntaxError, wsErr, "The url fragment must be empty."); kj::HttpHeaders headers(context.getHeaderTable()); - auto client = context.getHttpClient(0, false, kj::none, "WebSocket::constructor"_kjc); + auto client = context.getHttpClient(0, false, kj::none, "websocket_open"_kjc); // Set protocols header if necessary. KJ_IF_SOME(variant, protocols) {