Let geth choose the open port, pass it to fixtures

Author: fselmo

tests/core/providers/test_legacy_websocket_provider.py

@@ -1,19 +1,5 @@
 import pytest
-import asyncio
-from asyncio.exceptions import (
-    TimeoutError,
-)
-from threading import (
-    Thread,
-)
-
-from websockets.legacy.server import (
-    serve,
-)
 
-from tests.utils import (
-    wait_for_ws,
-)
 from web3 import (
     Web3,
 )
@@ -26,39 +12,6 @@
 )
 
 
-@pytest.fixture
-def start_websocket_server(open_port):
-    event_loop = asyncio.new_event_loop()
-
-    def run_server():
-        async def empty_server(websocket, path):
-            data = await websocket.recv()
-            await asyncio.sleep(0.02)
-            await websocket.send(data)
-
-        asyncio.set_event_loop(event_loop)
-        server = serve(empty_server, "127.0.0.1", open_port)
-        event_loop.run_until_complete(server)
-        event_loop.run_forever()
-
-    thd = Thread(target=run_server)
-    thd.start()
-    try:
-        yield
-    finally:
-        event_loop.call_soon_threadsafe(event_loop.stop)
-
-
-@pytest.fixture
-def w3(open_port, start_websocket_server):
-    # need new event loop as the one used by server is already running
-    event_loop = asyncio.new_event_loop()
-    endpoint_uri = f"ws://127.0.0.1:{open_port}"
-    event_loop.run_until_complete(wait_for_ws(endpoint_uri))
-    provider = LegacyWebSocketProvider(endpoint_uri, websocket_timeout=0.01)
-    return Web3(provider)
-
-
 def test_no_args():
     provider = LegacyWebSocketProvider()
     w3 = Web3(provider)
@@ -69,11 +22,6 @@ def test_no_args():
         w3.is_connected(show_traceback=True)
 
 
-def test_websocket_provider_timeout(w3):
-    with pytest.raises(TimeoutError):
-        w3.eth.accounts
-
-
 def test_restricted_websocket_kwargs():
     invalid_kwargs = {"uri": "ws://127.0.0.1:8546"}
     re_exc_message = f".*found: {set(invalid_kwargs)!r}*"

tests/integration/go_ethereum/conftest.py

@@ -4,12 +4,13 @@
 from pathlib import (
     Path,
 )
+import re
 import subprocess
+import time
 import zipfile
 
 from eth_utils import (
     is_dict,
-    to_text,
 )
 import pytest_asyncio
 
@@ -28,6 +29,9 @@
 from web3._utils.contract_sources.contract_data.storage_contract import (
     STORAGE_CONTRACT_DATA,
 )
+from web3.exceptions import (
+    Web3Exception,
+)
 
 from .utils import (
     kill_proc_gracefully,
@@ -36,6 +40,13 @@
 KEYFILE_PW = "web3py-test"
 GETH_FIXTURE_ZIP = "geth-1.15.5-fixture.zip"
 
+PORT_REGEX = re.compile(r"127\.0\.0\.1:(\d+)")
+INTERRUPT_REGEX = re.compile(r"interrupt, shutting down")
+
+
+class ShouldRestartGethProcess(Web3Exception):
+    pass
+
 
 @pytest.fixture
 def geth_binary():
@@ -133,36 +144,88 @@ def genesis_file(datadir):
     return genesis_file_path
 
 
+def wait_for_port(proc, timeout=10):
+    start = time.time()
+    port = None
+    wait_time = start + timeout
+
+    while time.time() < wait_time:
+        line = proc.stderr.readline()
+        if not line:
+            continue
+
+        if port is not None:
+            if INTERRUPT_REGEX.search(line):
+                raise ShouldRestartGethProcess(
+                    "Geth process interrupted after port assignment"
+                )
+        else:
+            if match := PORT_REGEX.search(line):
+                port = int(match.group(1))
+                if port == 80:
+                    port = None
+                else:
+                    # found port, but give a grace period for any interrupt called
+                    wait_time = time.time() + 0.1
+
+    if port is None:
+        raise TimeoutError(f"Did not find port in logs within {timeout} seconds")
+    return port
+
+
 @pytest.fixture
-def geth_process(geth_binary, datadir, genesis_file, geth_command_arguments):
-    init_datadir_command = (
+def start_geth_process_and_yield_port(
+    geth_binary, datadir, genesis_file, geth_command_arguments
+):
+    init_cmd = (
         geth_binary,
         "--datadir",
         str(datadir),
         "init",
         str(genesis_file),
     )
-    subprocess.check_output(
-        init_datadir_command,
-        stdin=subprocess.PIPE,
-        stderr=subprocess.PIPE,
-    )
-    proc = subprocess.Popen(
-        geth_command_arguments,
-        stdin=subprocess.PIPE,
-        stdout=subprocess.PIPE,
-        stderr=subprocess.PIPE,
-    )
+    subprocess.check_output(init_cmd, stdin=subprocess.PIPE, stderr=subprocess.PIPE)
+
+    tries = 2
+    proc = None
+    port = None
+
+    while tries > 0:
+        proc = subprocess.Popen(
+            geth_command_arguments,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+            bufsize=1,
+        )
+        try:
+            port = wait_for_port(proc)
+            break  # success
+        except ShouldRestartGethProcess:
+            kill_proc_gracefully(proc)
+            tries -= 1
+            if tries == 0:
+                raise RuntimeError("Failed to start Geth after retries")
+        except Exception:
+            kill_proc_gracefully(proc)
+            raise  # bubble up real errors
+
+    if port is None:
+        raise RuntimeError("Failed to obtain port after Geth start")
+
     try:
-        yield proc
+        yield port
     finally:
-        kill_proc_gracefully(proc)
-        output, errors = proc.communicate()
-        print(
-            "Geth Process Exited:\n"
-            f"stdout:{to_text(output)}\n\n"
-            f"stderr:{to_text(errors)}\n\n"
-        )
+        if proc:
+            kill_proc_gracefully(proc)
+            try:
+                output, errors = proc.communicate(timeout=5)
+            except subprocess.TimeoutExpired:
+                proc.kill()
+                output, errors = proc.communicate()
+            print(
+                "Geth Process Exited:\n" f"stdout:{output}\n\n" f"stderr:{errors}\n\n"
+            )
 
 
 @pytest.fixture

tests/integration/go_ethereum/test_goethereum_http.py

@@ -5,9 +5,6 @@
 )
 import pytest_asyncio
 
-from tests.utils import (
-    get_open_port,
-)
 from web3 import (
     AsyncWeb3,
     Web3,
@@ -32,29 +29,15 @@
     GoEthereumTxPoolModuleTest,
     GoEthereumWeb3ModuleTest,
 )
-from .utils import (
-    wait_for_aiohttp,
-    wait_for_http,
-)
-
-
-@pytest.fixture
-def rpc_port():
-    return get_open_port()
-
-
-@pytest.fixture
-def endpoint_uri(rpc_port):
-    return f"http://localhost:{rpc_port}"
 
 
-def _geth_command_arguments(rpc_port, base_geth_command_arguments, geth_version):
+def _geth_command_arguments(base_geth_command_arguments, geth_version):
     yield from base_geth_command_arguments
     if geth_version.major == 1:
         yield from (
             "--http",
             "--http.port",
-            rpc_port,
+            "0",
             "--http.api",
             "admin,debug,eth,net,web3,txpool",
             "--ipcdisable",
@@ -64,26 +47,28 @@ def _geth_command_arguments(rpc_port, base_geth_command_arguments, geth_version)
 
 
 @pytest.fixture
-def geth_command_arguments(rpc_port, base_geth_command_arguments, get_geth_version):
-    return _geth_command_arguments(
-        rpc_port, base_geth_command_arguments, get_geth_version
-    )
+def geth_command_arguments(base_geth_command_arguments, get_geth_version):
+    return _geth_command_arguments(base_geth_command_arguments, get_geth_version)
 
 
 @pytest.fixture
-def w3(geth_process, endpoint_uri):
-    wait_for_http(endpoint_uri)
-    return Web3(Web3.HTTPProvider(endpoint_uri, request_kwargs={"timeout": 10}))
+def w3(start_geth_process_and_yield_port):
+    port = start_geth_process_and_yield_port
+    _w3 = Web3(
+        Web3.HTTPProvider(f"http://127.0.0.1:{port}", request_kwargs={"timeout": 10})
+    )
+    return _w3
 
 
 @pytest.fixture
-def auto_w3(geth_process, endpoint_uri):
-    wait_for_http(endpoint_uri)
-
+def auto_w3(start_geth_process_and_yield_port, monkeypatch):
     from web3.auto import (
         w3,
     )
 
+    port = start_geth_process_and_yield_port
+    monkeypatch.setenv("WEB3_PROVIDER_URI", f"http://127.0.0.1:{port}")
+
     return w3
 
 
@@ -116,13 +101,7 @@ class TestGoEthereumDebugModuleTest(GoEthereumDebugModuleTest):
 
 
 class TestGoEthereumEthModuleTest(GoEthereumEthModuleTest):
-    def test_auto_provider_batching(
-        self,
-        auto_w3: "Web3",
-        monkeypatch,
-        endpoint_uri,
-    ) -> None:
-        monkeypatch.setenv("WEB3_PROVIDER_URI", endpoint_uri)
+    def test_auto_provider_batching(self, auto_w3: "Web3") -> None:
         # test that batch_requests doesn't error out when using the auto provider
         auto_w3.batch_requests()
 
@@ -139,11 +118,14 @@ class TestGoEthereumTxPoolModuleTest(GoEthereumTxPoolModuleTest):
 
 
 @pytest_asyncio.fixture
-async def async_w3(geth_process, endpoint_uri):
-    await wait_for_aiohttp(endpoint_uri)
+async def async_w3(start_geth_process_and_yield_port):
+    port = start_geth_process_and_yield_port
     _w3 = AsyncWeb3(
-        AsyncHTTPProvider(endpoint_uri, request_kwargs={"timeout": ClientTimeout(10)})
+        AsyncHTTPProvider(
+            f"http://127.0.0.1:{port}", request_kwargs={"timeout": ClientTimeout(10)}
+        )
     )
+    await _w3.provider.disconnect()
     return _w3
 
 

tests/integration/go_ethereum/test_goethereum_ipc.py

@@ -54,18 +54,19 @@ def geth_ipc_path(datadir):
 
 
 @pytest.fixture
-def auto_w3(geth_process, geth_ipc_path):
-    wait_for_socket(geth_ipc_path)
-
+def auto_w3(start_geth_process_and_yield_port, geth_ipc_path, monkeypatch):
     from web3.auto import (
         w3,
     )
 
+    wait_for_socket(geth_ipc_path)
+    monkeypatch.setenv("WEB3_PROVIDER_URI", f"file:///{geth_ipc_path}")
+
     return w3
 
 
 @pytest.fixture
-def w3(geth_process, geth_ipc_path):
+def w3(start_geth_process_and_yield_port, geth_ipc_path):
     wait_for_socket(geth_ipc_path)
     return Web3(Web3.IPCProvider(geth_ipc_path, timeout=10))
 
@@ -79,13 +80,7 @@ class TestGoEthereumDebugModuleTest(GoEthereumDebugModuleTest):
 
 
 class TestGoEthereumEthModuleTest(GoEthereumEthModuleTest):
-    def test_auto_provider_batching(
-        self,
-        auto_w3: "Web3",
-        monkeypatch,
-        geth_ipc_path,
-    ) -> None:
-        monkeypatch.setenv("WEB3_PROVIDER_URI", f"file:///{geth_ipc_path}")
+    def test_auto_provider_batching(self, auto_w3: "Web3") -> None:
         # test that batch_requests doesn't error out when using the auto provider
         auto_w3.batch_requests()
 
@@ -118,7 +113,7 @@ def test_admin_start_stop_ws(self, w3: "Web3") -> None:
 
 
 @pytest_asyncio.fixture
-async def async_w3(geth_process, geth_ipc_path):
+async def async_w3(start_geth_process_and_yield_port, geth_ipc_path):
     await wait_for_async_socket(geth_ipc_path)
     async with AsyncWeb3(AsyncIPCProvider(geth_ipc_path, request_timeout=10)) as _aw3:
         yield _aw3