[wip] limited api wheels

.github/workflows/wheels.yml

@@ -72,12 +72,12 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - os: macos-12
+          - os: macos-13
             name: mac-cpython
             cibw:
               build: "cp*"
 
-          - os: macos-12
+          - os: macos-13
             name: mac-pypy
             cibw:
               build: "pp*"

CMakeLists.txt

@@ -5,13 +5,13 @@ set(CMAKE_POLICY_DEFAULT_CMP0077 NEW)
 list(PREPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake)
 find_package(
   Python
-  COMPONENTS Interpreter Development.Module
+  COMPONENTS Interpreter Development.Module ${SKBUILD_SABI_COMPONENT}
   REQUIRED)
 
 # Python_SOABI isn't always right when cross-compiling
 # SKBUILD_SOABI seems to be
 if (DEFINED SKBUILD_SOABI AND NOT "${SKBUILD_SOABI}" STREQUAL "${Python_SOABI}")
-  message(WARNING "SKBUILD_SOABI=${SKBUILD_SOABI} != Python_SOABI=${Python_SOABI}; likely cross-compiling, using SOABI=${SKBUILD_SOABI} from scikit-build")
+  message(WARNING "SKBUILD_SOABI=${SKBUILD_SOABI} != Python_SOABI=${Python_SOABI}; likely cross-compiling or Limited API, using SOABI=${SKBUILD_SOABI} from scikit-build")
   set(Python_SOABI "${SKBUILD_SOABI}")
 endif()
 
@@ -404,9 +404,19 @@ endif()
 
 file(MAKE_DIRECTORY ${ZMQ_BACKEND_DEST})
 
+if(NOT "${SKBUILD_SABI_COMPONENT}" STREQUAL "")
+  # set stable API
+  # assume we are targeting >= current Python version
+  # this isn't required, but we can't seem to get `wheel.py-api`
+  # see https://github.com/scikit-build/scikit-build-core/issues/958
+  set(SABI_ARG "USE_SABI;${Python_VERSION_MAJOR}.${Python_VERSION_MINOR}")
+  message(STATUS "Building with stable API ${SABI_ARG} for ${Python_SOABI}")
+endif()
+
 python_add_library(
   ${ZMQ_EXT_NAME} MODULE
   WITH_SOABI
+  ${SABI_ARG}
   ${ZMQ_C}
 )
 

pyproject.toml

@@ -2,9 +2,10 @@
 [build-system]
 requires = [
   "cffi; implementation_name == 'pypy'",
-  "cython>=3.0.0; implementation_name != 'pypy'",
+  "cython>=3; implementation_name != 'pypy'",
+  "cython>=3.1.0a1; implementation_name != 'pypy' and python_version >= '3.11'",
   "packaging",
-  "scikit-build-core",
+  "scikit-build-core>=0.10",
 ]
 build-backend = "scikit_build_core.build"
 
@@ -56,7 +57,7 @@ wheel.license-files = ["licenses/LICENSE*"]
 # 3.15 is required by scikit-build-core
 cmake.version = ">=3.15"
 # only build/install the pyzmq component
-cmake.targets = ["pyzmq"]
+build.targets = ["pyzmq"]
 install.components = ["pyzmq"]
 
 [tool.ruff]
@@ -146,6 +147,7 @@ build-verbosity = "1"
 free-threaded-support = true
 test-requires = ["pytest>=6", "importlib_metadata"]
 test-command = "pytest -vsx {package}/tools/test_wheel.py"
+build-frontend = "build"
 
 [tool.cibuildwheel.linux]
 before-all = "bash tools/install_libzmq.sh"
@@ -194,14 +196,30 @@ repair-wheel-command = """\
 # mac-arm target is 10.15
 [[tool.cibuildwheel.overrides]]
 select = "*macos*{universal2,arm64}*"
-environment = { ZMQ_PREFIX = "/tmp/zmq", MACOSX_DEPLOYMENT_TARGET = "10.15" }
+inherit.environment = "append"
+"environment.MACOSX_DEPLOYMENT_TARGET" = "10.15"
 
 # manylinux2010 for (less) old cp37-9, pp37-8
 [[tool.cibuildwheel.overrides]]
 select = "cp3{7,8,9}-* pp3{7,8}-*"
 manylinux-x86_64-image = "manylinux2010"
 manylinux-i686-image = "manylinux2010"
 
+# build limited-api wheels for 3.11
+[[tool.cibuildwheel.overrides]]
+select = "cp311-*"
+inherit.config-settings = "append"
+config-settings."wheel.py-api" = "cp311"
+inherit.repair-wheel-command = "append"
+before-build = "pip install abi3audit"
+repair-wheel-command = "abi3audit --strict --report {wheel}"
+
+# for benchmarking, build limited cp312 as well
+[[tool.cibuildwheel.overrides]]
+select = "cp312-*"
+inherit.config-settings = "append"
+config-settings."wheel.py-api" = "cp312"
+
 # note: manylinux_2_28 builds are added
 # in .github/workflows/wheels.yml
 

tools/install_libzmq.sh

@@ -6,8 +6,9 @@ LIBZMQ_VERSION=$(python buildutils/bundle.py)
 PYZMQ_DIR="$PWD"
 LICENSE_DIR="$PYZMQ_DIR/licenses"
 test -d "$LICENSE_DIR" || mkdir "$LICENSE_DIR"
-
+SHLIB_EXT="so"
 if [[ "$(uname)" == "Darwin" ]]; then
+    SHLIB_EXT="dylib"
     # need LT_MULTI_MODULE or libtool will strip out
     # all multi-arch symbols at the last step
     export LT_MULTI_MODULE=1
@@ -41,6 +42,12 @@ if [[ "$(uname)" == "Darwin" ]]; then
 fi
 
 PREFIX="${ZMQ_PREFIX:-/usr/local}"
+
+if [ -f "$PREFIX/lib/libzmq.${SHLIB_EXT}" ]; then
+  echo "using $PREFIX/lib/libzmq.${SHLIB_EXT}"
+  exit 0
+fi
+
 # add rpath so auditwheel patches it
 export LDFLAGS="${LDFLAGS} -Wl,-rpath,$PREFIX/lib"
 

tools/wheel-requirements.txt

@@ -1,2 +1,3 @@
-cibuildwheel==2.20.*
-delvewheel==1.7.2; sys_platform == 'win32'
+abi3audit
+cibuildwheel==2.22.*
+delvewheel==1.9.0; sys_platform == 'win32'

zmq/backend/cython/_cpython.pxd

@@ -0,0 +1,10 @@
+# These should be cimported from cpython.bytes
+# but that has a transitive import of cpython.type
+# which currently conflicts with limited API
+cdef extern from "Python.h":
+  # cpython.bytes
+  char* PyBytes_AsString(object string) except NULL
+  bytes PyBytes_FromStringAndSize(char *v, Py_ssize_t len)
+  Py_ssize_t PyBytes_Size(object string) except -1
+  # cpython.exc
+  int PyErr_CheckSignals() except -1

zmq/backend/cython/_zmq.py

@@ -56,12 +56,15 @@
     size_t,
     sizeof,
 )
-from cython.cimports.cpython import (
-    PyBytes_AsString,
-    PyBytes_FromStringAndSize,
-    PyBytes_Size,
-    PyErr_CheckSignals,
-)
+
+# Cannot cimport these with Limited API yet
+# see https://github.com/cython/cython/issues/5634
+# from cython.cimports.cpython.bytes import (
+#     PyBytes_AsString,
+#     PyBytes_FromStringAndSize,
+#     PyBytes_Size,
+# )
+# from cython.cimports.cpython.exc import PyErr_CheckSignals
 from cython.cimports.libc.errno import EAGAIN, EINTR, ENAMETOOLONG, ENOENT, ENOTSOCK
 
 # cimports require Cython 3
@@ -70,6 +73,14 @@
 from cython.cimports.libc.stdio import stderr as cstderr
 from cython.cimports.libc.stdlib import free, malloc
 from cython.cimports.libc.string import memcpy
+
+# these should be from cython.cimports.cpython
+from cython.cimports.zmq.backend.cython._cpython import (
+    PyBytes_AsString,
+    PyBytes_FromStringAndSize,
+    PyBytes_Size,
+    PyErr_CheckSignals,
+)
 from cython.cimports.zmq.backend.cython._externs import (
     get_ipc_path_max_len,
     getpid,

zmq/utils/mutex.h

@@ -9,6 +9,8 @@
 
 #pragma once
 
+#include <stdlib.h>
+
 #if defined(_WIN32)
 #  include <windows.h>
 #else