From 0481b0fc2c534a8b3f07a67250af6e0ebcc7d95a Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Tue, 10 Dec 2024 17:55:14 +0000 Subject: [PATCH] Implements Response.replace_body and support for cf opts in Py fetch. --- src/pyodide/internal/workers.py | 49 +++++++++++++--- src/workerd/io/compatibility-date.capnp | 2 +- src/workerd/server/tests/python/sdk/worker.py | 57 ++++++++++++++++++- 3 files changed, 97 insertions(+), 11 deletions(-) diff --git a/src/pyodide/internal/workers.py b/src/pyodide/internal/workers.py index 210619c8da4..67c7606a964 100644 --- a/src/pyodide/internal/workers.py +++ b/src/pyodide/internal/workers.py @@ -5,7 +5,7 @@ from contextlib import ExitStack, contextmanager from enum import StrEnum from http import HTTPMethod, HTTPStatus -from typing import TypedDict, Unpack +from typing import Any, TypedDict, Unpack import js @@ -21,10 +21,32 @@ Headers = dict[str, str] | list[tuple[str, str]] +# https://developers.cloudflare.com/workers/runtime-apis/request/#the-cf-property-requestinitcfproperties +class RequestInitCfProperties(TypedDict, total=False): + apps: bool | None + cacheEverything: bool | None + cacheKey: str | None + cacheTags: list[str] | None + cacheTtl: int + cacheTtlByStatus: dict[str, int] + image: ( + Any | None + ) # TODO: https://developers.cloudflare.com/images/transform-images/transform-via-workers/ + mirage: bool | None + polish: str | None + resolveOverride: str | None + scrapeShield: bool | None + webp: bool | None + + +# This matches the Request options: +# https://developers.cloudflare.com/workers/runtime-apis/request/#options class FetchKwargs(TypedDict, total=False): headers: Headers | None body: "Body | None" method: HTTPMethod = HTTPMethod.GET + redirect: str | None + cf: RequestInitCfProperties | None # TODO: Pyodide's FetchResponse.headers returns a dict[str, str] which means @@ -65,6 +87,15 @@ async def formData(self) -> "FormData": except JsException as exc: raise _to_python_exception(exc) from exc + def replace_body(self, body: Body) -> "FetchResponse": + """ + Returns a new Response object with the same options (status, headers, etc) as + the original but with an updated body. + """ + b = body.js_object if isinstance(body, FormData) else body + js_resp = js.Response.new(b, self.js_response) + return FetchResponse(js_resp.url, js_resp) + async def fetch( resource: str, @@ -99,7 +130,7 @@ def __init__( self, body: Body, status: HTTPStatus | int = HTTPStatus.OK, - statusText="", + status_text="", headers: Headers = None, ): """ @@ -108,7 +139,7 @@ def __init__( Based on the JS API of the same name: https://developer.mozilla.org/en-US/docs/Web/API/Response/Response. """ - options = self._create_options(status, statusText, headers) + options = self._create_options(status, status_text, headers) # Initialise via the FetchResponse super-class which gives us access to # methods that we would ordinarily have to redeclare. @@ -119,13 +150,15 @@ def __init__( @staticmethod def _create_options( - status: HTTPStatus | int = HTTPStatus.OK, statusText="", headers: Headers = None + status: HTTPStatus | int = HTTPStatus.OK, + status_text="", + headers: Headers = None, ): options = { "status": status.value if isinstance(status, HTTPStatus) else status, } - if len(statusText) > 0: - options["statusText"] = statusText + if status_text: + options["statusText"] = status_text if headers: if isinstance(headers, list): # We should have a list[tuple[str, str]] @@ -154,10 +187,10 @@ def redirect(url: str, status: HTTPStatus | int = HTTPStatus.FOUND): def json( data: str | dict[str, str], status: HTTPStatus | int = HTTPStatus.OK, - statusText="", + status_text="", headers: Headers = None, ): - options = Response._create_options(status, statusText, headers) + options = Response._create_options(status, status_text, headers) with _manage_pyproxies() as pyproxies: try: return js.Response.json( diff --git a/src/workerd/io/compatibility-date.capnp b/src/workerd/io/compatibility-date.capnp index 9f765024abb..9c7aac02604 100644 --- a/src/workerd/io/compatibility-date.capnp +++ b/src/workerd/io/compatibility-date.capnp @@ -430,7 +430,7 @@ struct CompatibilityFlags @0x8f8c1b68151b6cef { pythonWorkers @43 :Bool $compatEnableFlag("python_workers") $pythonSnapshotRelease(pyodide = "0.26.0a2", pyodideRevision = "2024-03-01", - packages = "2024-03-01", backport = 11, + packages = "2024-03-01", backport = 12, baselineSnapshotHash = "d13ce2f4a0ade2e09047b469874dacf4d071ed3558fec4c26f8d0b99d95f77b5") $impliedByAfterDate(name = "pythonWorkersDevPyodide", date = "2000-01-01"); # Enables Python Workers. Access to this flag is not restricted, instead bundles containing diff --git a/src/workerd/server/tests/python/sdk/worker.py b/src/workerd/server/tests/python/sdk/worker.py index 506267722f1..edcc6119c50 100644 --- a/src/workerd/server/tests/python/sdk/worker.py +++ b/src/workerd/server/tests/python/sdk/worker.py @@ -1,11 +1,27 @@ +from contextlib import asynccontextmanager from http import HTTPMethod, HTTPStatus import js +import pyodide.http from cloudflare.workers import Blob, File, FormData, Response, fetch from pyodide.ffi import to_js +@asynccontextmanager +async def _mock_fetch(check): + async def mocked_fetch(original_fetch, url, opts): + check(url, opts) + return await original_fetch(url, opts) + + original_fetch = pyodide.http._jsfetch + pyodide.http._jsfetch = lambda url, opts: mocked_fetch(original_fetch, url, opts) + try: + yield + finally: + pyodide.http._jsfetch = original_fetch + + # Each path in this handler is its own test. The URLs that are being fetched # here are defined in server.py. async def on_fetch(request): @@ -51,8 +67,18 @@ async def on_fetch(request): elif request.url.endswith("/undefined_opts"): # This tests two things: # * `Response.redirect` static method - # * that other options can be passed into `fetch` - resp = await fetch("https://example.com/redirect", redirect="manual") + # * that other options can be passed into `fetch` (so that we can support + # new options without updating this code) + + # Mock pyodide.http._jsfetch to ensure `foobarbaz` gets passed in. + def fetch_check(url, opts): + assert opts.foobarbaz == 42 + + async with _mock_fetch(fetch_check): + resp = await fetch( + "https://example.com/redirect", redirect="manual", foobarbaz=42 + ) + return resp elif request.url.endswith("/response_inherited"): expected = "test123" @@ -89,6 +115,14 @@ async def on_fetch(request): assert data["blob.py"].content_type == "text/python" assert data["metadata"].name == "metadata.json" + return Response("success") + elif request.url.endswith("/cf_opts"): + resp = await fetch( + "http://example.com/redirect", + redirect="manual", + cf={"cacheTtl": 5, "cacheEverything": True, "cacheKey": "someCustomKey"}, + ) + assert resp.status == 301 return Response("success") else: resp = await fetch("https://example.com/sub") @@ -273,6 +307,23 @@ async def can_request_form_data_blob(env): assert text == "success" +async def replace_body_unit_tests(env): + response = Response("test", status=201, status_text="Created") + cloned = response.replace_body("other") + assert cloned.status == 201 + assert cloned.status_text == "Created" + t = await cloned.text() + assert t == "other" + + +async def can_use_cf_fetch_opts(env): + response = await env.SELF.fetch( + "http://example.com/cf_opts", + ) + text = await response.text() + assert text == "success" + + async def test(ctrl, env): await can_return_custom_fetch_response(env) await can_modify_response(env) @@ -287,3 +338,5 @@ async def test(ctrl, env): await form_data_unit_tests(env) await blob_unit_tests(env) await can_request_form_data_blob(env) + await replace_body_unit_tests(env) + await can_use_cf_fetch_opts(env)