libuv based event loop / server

.github/workflows/deploy-github-page.yml

@@ -33,7 +33,7 @@ jobs:
           environment-file: environment-wasm-build.yml
           init-shell: bash
           environment-name: xeus-python-wasm-build
-
+ 
       - name: Set ncpus
         run: echo "ncpus=$(nproc --all)" >> $GITHUB_ENV
 
@@ -47,6 +47,12 @@ jobs:
 
           set -eux
 
+
+          URL=https://github.com/DerThorsten/xeus-python-shell/archive/refs/heads/libuv.zip 
+          curl -L $URL -o xeus-python-shell-libuv.zip
+          unzip xeus-python-shell-libuv.zip
+
+
           export PREFIX=$MAMBA_ROOT_PREFIX/envs/xeus-python-wasm-host
           echo "PREFIX=$PREFIX" >> $GITHUB_ENV
 
@@ -60,13 +66,19 @@ jobs:
 
           emmake make -j${{ env.ncpus }} install
 
+
+
+
       - name: Jupyter Lite integration
         shell: bash -l {0}
         run: |
+          THIS_DIR=$(pwd)
           jupyter lite build \
               --XeusAddon.prefix=${{ env.PREFIX }} \
               --contents notebooks/ \
-              --output-dir dist
+              --output-dir dist \
+              --XeusAddon.mounts=$THIS_DIR/xeus-python-shell-libuv/xeus_python_shell/:/lib/python3.13/site-packages/xeus_python_shell/
+
 
       - name: Upload artifact
         uses: actions/upload-pages-artifact@v4

.github/workflows/main.yml

@@ -22,7 +22,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-22.04, ubuntu-24.04, macos-13, macos-14]
+        os: [ubuntu-22.04, ubuntu-24.04, macos-14]
         build_type: [static_build, shared_build]
 
     steps:
@@ -32,6 +32,27 @@ jobs:
         uses: mamba-org/setup-micromamba@v2
         with:
           environment-file: environment-dev.yml
+
+      - name: Install xeus-python-shell from GitHub
+        run: pip install git+https://github.com/DerThorsten/xeus-python-shell.git@libuv
+
+      - name: Checkout xeus-uv
+        uses: actions/checkout@v5
+        with:
+          repository: jupyter-xeus/xeus-uv
+          path: xeus-uv
+
+      - name: Build and install xeus-uv
+        run: |
+          mkdir xeus-uv/build
+          cd xeus-uv/build
+          cmake .. \
+            -D CMAKE_PREFIX_PATH=$CONDA_PREFIX \
+            -D CMAKE_INSTALL_PREFIX=$CONDA_PREFIX \
+            -D CMAKE_INSTALL_LIBDIR=lib
+          make -j 2 install
+          cd ../..
+          rm -rf xeus-uv
 
       - name: Make build directory
         run: mkdir build
@@ -93,10 +114,33 @@ jobs:
         uses: mamba-org/setup-micromamba@v2
         with:
           init-shell: cmd.exe
-          environment-file: environment-dev.yml
+          environment-file: environment-dev-win.yml
 
       - name: Make build directory
         run: mkdir build
+
+      - name: Install xeus-python-shell from GitHub
+        run: pip install git+https://github.com/DerThorsten/xeus-python-shell.git@libuv
+
+      - name: Checkout xeus-uv
+        uses: actions/checkout@v5
+        with:
+          repository: jupyter-xeus/xeus-uv
+          path: xeus-uv
+
+      - name: Build and install xeus-uv
+        shell: cmd /C call {0}
+        run: |
+          mkdir xeus-uv\build
+          cd xeus-uv\build
+          cmake .. ^
+            -GNinja ^
+            -DCMAKE_BUILD_TYPE=Release ^
+            -DCMAKE_PREFIX_PATH="%CONDA_PREFIX%\Library" ^
+            -DCMAKE_INSTALL_PREFIX="%CONDA_PREFIX%\Library"
+          ninja install
+          cd ../..
+          rm -rf xeus-uv
 
       - name: Static build option
         if: matrix.build_type == 'static_build'

CMakeLists.txt

@@ -79,7 +79,7 @@ set(pybind11_REQUIRED_VERSION 2.6.1)
 set(pybind11_json_REQUIRED_VERSION 0.2.8)
 set(pyjs_REQUIRED_VERSION 2.0.0)
 
-find_package(Python COMPONENTS Interpreter REQUIRED)
+# find_package(Python COMPONENTS Interpreter REQUIRED)
 
 if (NOT TARGET pybind11::headers)
     # Defaults to ON for cmake >= 3.18
@@ -105,8 +105,14 @@ else ()
     if (NOT TARGET xeus-zmq AND NOT TARGET xeus-zmq-static)
         find_package(xeus-zmq ${xeus-zmq_REQUIRED_VERSION} REQUIRED)
     endif ()
+
+    find_package(xeus-uv REQUIRED)
+    find_library(UVW_LIBRARY uvw)
+    find_path(UVW_INCLUDE_DIR uvw.hpp)
 endif()
 
+
+
 # Configuration
 # =============
 
@@ -163,6 +169,7 @@ set(XEUS_PYTHON_SRC
     src/xdebugpy_client.cpp
     src/xdisplay.cpp
     src/xdisplay.hpp
+    src/xhook.cpp
     src/xinput.cpp
     src/xinput.hpp
     src/xinspect.cpp
@@ -188,6 +195,7 @@ set(XEUS_PYTHON_HEADERS
     include/xeus-python/xinterpreter_raw.hpp
     include/xeus-python/xtraceback.hpp
     include/xeus-python/xutils.hpp
+    include/xeus-python/xhook.hpp
 )
 
 set(XPYTHON_SRC
@@ -317,7 +325,10 @@ macro(xpyt_create_target target_name src headers linkage output_name)
         set(XPYT_XEUS_TARGET xeus-zmq-static)
     endif ()
 
-    target_link_libraries(${target_name} PUBLIC ${XPYT_XEUS_TARGET} PRIVATE pybind11::pybind11 pybind11_json)
+    target_link_libraries(${target_name}
+        PUBLIC ${XPYT_XEUS_TARGET}  ${UVW_LIBRARY}
+        PRIVATE pybind11::pybind11 pybind11_json
+    )
     if (WIN32 OR CYGWIN)
         target_link_libraries(${target_name} PRIVATE ${PYTHON_LIBRARIES})
     elseif (APPLE)
@@ -371,7 +382,7 @@ endif ()
 
 if (XPYT_BUILD_XPYTHON_EXECUTABLE)
     add_executable(xpython ${XPYTHON_SRC})
-    target_link_libraries(xpython PRIVATE pybind11::embed)
+    target_link_libraries(xpython PRIVATE xeus-uv pybind11::embed )
 
     xpyt_set_common_options(xpython)
     xpyt_set_kernel_options(xpython)

environment-dev-win.yml

@@ -0,0 +1,29 @@
+name: xeus-python
+channels:
+  - conda-forge
+dependencies:
+  # Build dependencies
+  - cmake
+  - cxx-compiler
+  - ninja
+  - cmake < 4  # TODO: remove pin on CMake
+  # Host dependencies
+  - xeus-zmq>=3.1,<4.0
+  - nlohmann_json=3.12.0
+  - pybind11>=2.6.1,<3.0
+  - pybind11_json>=0.2.6,<0.3
+  # The debugger is not available with python 3.13 because
+  # of a spurious bug in debugpy
+  - python <3.13
+  #- xeus-python-shell>=0.6.3,<0.7 # we need an unreleased version
+  - debugpy>=1.6.5
+  - libuvw
+  - ipython
+  # Test dependencies
+  - pytest
+  - nbval
+  - ipympl
+  - jupyter_kernel_test<0.8
+  - doctest
+  - pluggy=1.3
+  - winloop

environment-dev.yml

@@ -15,8 +15,10 @@ dependencies:
   # The debugger is not available with python 3.13 because
   # of a spurious bug in debugpy
   - python <3.13
-  - xeus-python-shell>=0.6.3,<0.7
+  #- xeus-python-shell>=0.6.3,<0.7 # we need an unreleased version
   - debugpy>=1.6.5
+  - libuvw
+  - uvloop
   - ipython
   # Test dependencies
   - pytest

environment-wasm-build.yml

@@ -13,3 +13,5 @@ dependencies:
   - jupyterlite-core >0.6
   - jupyter_server
   - jupyterlite-xeus
+  - # widgets
+  - ipywidgets

environment-wasm-host.yml

@@ -12,9 +12,11 @@ dependencies:
   - numpy
   - xeus-lite
   - xeus
-  - xeus-python-shell>=0.6.3
   - pyjs >=2,<3
   - libpython
   - zstd
   - openssl
   - xz
+  - ipywidgets
+  # - xeus-python-shell>=0.6.3
+

include/xeus-python/xhook.hpp

@@ -0,0 +1,51 @@
+/***************************************************************************
+* Copyright (c) 2018, Martin Renou, Johan Mabille, Sylvain Corlay, and     *
+* Wolf Vollprecht                                                          *
+* Copyright (c) 2018, QuantStack
+*                                                                          *
+* Distributed under the terms of the BSD 3-Clause License.                 *
+*                                                                          *
+* The full license is in the file LICENSE, distributed with this software. *
+****************************************************************************/
+
+#ifndef XPYT_HOOK_HPP
+#define XPYT_HOOK_HPP
+
+// pybind11 code internally forces hidden visibility on all internal code, but
+// if non-hidden (and thus exported) code attempts to include a pybind type
+// this warning occurs:
+// 'xpyt::hook' declared with greater visibility than the type of its
+// field 'xpyt::hook::p_acquire' [-Wattributes]
+#ifdef __GNUC__
+    #pragma GCC diagnostic ignored "-Wattributes"
+#endif
+
+#include "xeus-python/xeus_python_config.hpp"
+#include "xeus-uv/xhook_base.hpp"
+
+#include "pybind11/embed.h"
+#include "pybind11/pybind11.h"
+
+namespace py = pybind11;
+
+namespace xpyt
+{
+    class XEUS_PYTHON_API hook : public xeus::xhook_base
+    {
+    public:
+
+        hook() = default;
+        virtual ~hook();
+
+    private:
+
+        void pre_hook_impl() override;
+        void post_hook_impl() override;
+        void run_impl(std::shared_ptr<uvw::loop> loop) override;
+
+        py::gil_scoped_acquire* p_acquire{ nullptr};
+    };
+
+}
+
+#endif

include/xeus-python/xinterpreter.hpp

@@ -93,14 +93,28 @@ namespace xpyt
         // is started, m_release_gil_at_startup has to be set to false to prevent
         // releasing it again in configure_impl().
         //
-        bool m_release_gil_at_startup = true;
+        bool m_release_gil_at_startup = false;
         gil_scoped_release_ptr m_release_gil = nullptr;
 
         bool m_redirect_output_enabled;
         bool m_redirect_display_enabled;
 
     private:
 
+        // helper methods:
+        void execute_request_impl_sync(send_reply_callback cb,
+                                  int execution_counter,
+                                  const std::string& code,
+                                  xeus::execute_request_config config,
+                                  nl::json user_expressions);
+
+        void execute_request_impl_async(send_reply_callback cb,
+                                  int execution_counter,
+                                  const std::string& code,
+                                  xeus::execute_request_config config,
+                                  nl::json user_expressions);
+
+
         virtual void instanciate_ipython_shell();
         virtual bool use_jedi_for_completion() const;
     };