Switch to pybind11 implementation

Author: trundev

meson.build

@@ -13,10 +13,18 @@ incdir_numpy = run_command(py,
   ],
   check: true
 ).stdout().strip()
+incdir_pybind11 = run_command(py,
+  [
+    '-c',
+    'import pybind11; print(pybind11.get_include())'
+  ],
+  check: true
+).stdout().strip()
 
 includes = include_directories(
   [
     incdir_numpy,
+    incdir_pybind11,
     'src'
   ]
 )
@@ -27,13 +35,17 @@ c_args = [
   '-DNPY_NO_DEPRECATED_API=NPY_2_0_API_VERSION',
   '-DNPY_TARGET_VERSION=NPY_2_0_API_VERSION',
 ]
+if get_option('buildtype') == 'debug'
+  # Avoid '_DEBUG' as it links to 'python*_d.lib'
+  c_args += ['-D__DEBUG']
+endif
 
 srcs = [
   'src/casts.cpp',
   'src/casts.h',
-  'src/dtype.c',
+  'src/dtype.cpp',
   'src/dtype.h',
-  'src/main.c',
+  'src/main.cpp',
 ]
 
 py.install_sources(
@@ -46,7 +58,7 @@ py.install_sources(
 )
 
 py.extension_module(
-  '_sample_dtypes_main',
+  '_dtypes_ext',
   srcs,
   install: true,
   subdir: 'sample_dtypes',

pyproject.toml

@@ -30,6 +30,7 @@ dev = [
 requires = [
     "meson",
     "meson-python",
+    "pybind11",
     "wheel",
     "numpy",
 ]

sample_dtypes/__init__.py

@@ -1,11 +1,15 @@
 """NumPy sample Dtypes"""
 
-from .scalar import SampleScalar
-from ._sample_dtypes_main import SampleDType
+from . import _dtypes_ext
+from .scalar import ScalarType
 
 __version__ = "0.0.0"  # Keep in sync with pyproject.toml:version
 __all__ = [
     "__version__",
     "SampleDType",
-    "SampleScalar",
+    "ScalarType",
 ]
+
+# At startup create NumPy DType, based on this scalar
+SampleDType = _dtypes_ext.init_dtype(ScalarType)
+del _dtypes_ext

sample_dtypes/scalar.py

@@ -4,85 +4,61 @@
 import numpy as np
 
 
-DEFAULT_SHAPE = (2,) * 2
-BYTES_BUF_ORDER = 'A'
+DEFAULT_SHAPE = tuple()
 
 
-def custom_as_ctypes_type(dtype: np.dtype):
-  """Wrapper around `numpy.ctypeslib.as_ctypes_type` that maps 'O' to py_object"""
-  if not dtype.hasobject:
-    return np.ctypeslib.as_ctypes_type(dtype)
-
-  if dtype.fields is None:
-    # Simple dtype or array
-    if dtype.subdtype:
-      # Handle sub-array types like ('O', 3)
-      base, shape = dtype.subdtype
-      new_base = custom_as_ctypes_type(base)
-      for dim in reversed(shape):
-        new_base = new_base * dim
-      return new_base
-    else:
-      # Simple object (non-objects are already handled)
-      assert dtype == np.dtype(object)
-      return ctypes.py_object
-
-  # Structured dtype
-  new_fields = []
-  for name, (field_dtype, *_) in dtype.fields.items():
-    new_field_dtype = custom_as_ctypes_type(field_dtype)
-    new_fields.append((name, new_field_dtype))
-
-  class CustomStruct(ctypes.Structure):
-    _pack_ = dtype.alignment
-    _fields_ = new_fields
-
-  return CustomStruct
-
-
-class SampleScalar:
-  _ndarr: np.ndarray
+class ScalarType:
+  shape: tuple[int, ...]
+  dtype: np.dtype
 
   def __init__(self, *, shape: tuple[int, ...] = DEFAULT_SHAPE, dtype=None):
-    print(f'TODO: scalar.py: SampleScalar.__init__({shape=}, {dtype=})')
-    self._ndarr = np.empty(shape, dtype=dtype)
+    self.shape = shape
+    self.dtype = np.dtype(dtype)
 
-  def copy(self) -> 'SampleScalar':
-    return type(self)(shape=self._ndarr.shape, dtype=self._ndarr.dtype)
+  def __repr__(self) -> str:
+    return f'{type(self).__name__}(shape={self.shape}, dtype={self.dtype})'
 
   @property
   def elsize(self) -> int:
-    return self._ndarr.nbytes
+    """Overall size of DType item (to initialize `PyArray_Descr::elsize`)"""
+    # Consider @cashed_property
+    return np.empty(self.shape, self.dtype).nbytes
 
   @property
   def alignment(self) -> int:
-    return self._ndarr.dtype.alignment
+    """Alignment of DType item (to initialize `PyArray_Descr::alignment`)"""
+    return self.dtype.alignment
 
   @property
   def hasobject(self) -> bool:
-    return self._ndarr.dtype.hasobject
+    """Whether DType contain pointers to Python objects
 
-  def is_compatible(self, other: 'SampleScalar') -> bool:
-    print(f'scalar.py: SampleScalar.is_compatible({other})')
-    return self._ndarr.shape == other._ndarr.shape
+    (adds `PyArray_Descr::flags` like `NPY_ITEM_HASOBJECT` and `NPY_NEEDS_INIT`)
+    """
+    return self.dtype.hasobject
 
-  def __repr__(self) -> str:
-    return f'{type(self).__name__}(id={hex(id(self))}, shape={self._ndarr.shape}, dtype={self._ndarr.dtype})'
+  def is_compatible(self, other: 'ScalarType') -> bool:
+    """Check if both objects are compatible for set/cast operations"""
+    return self.shape == other.shape
+
+  def _create(self) -> 'Scalar':
+    """Instantiate Scalar object, that includes actual data storage"""
+    return Scalar(shape=self.shape, dtype=self.dtype)
 
   def _get_np_view(self, dataptr: int) -> np.ndarray:
     """Create numpy array that uses `dataptr` memory block"""
     ct_array = ctypes.cast(
-        dataptr, ctypes.POINTER(custom_as_ctypes_type(self._ndarr.dtype))
+        dataptr, ctypes.POINTER(custom_as_ctypes_type(self.dtype))
     )
-    return np.ctypeslib.as_array(ct_array, shape=self._ndarr.shape)
+    return np.ctypeslib.as_array(ct_array, shape=self.shape)
 
   def _get_strided_np_view(
       self, dataptr: int, size: int, stride: int
   ) -> np.ndarray:
     """Create numpy array that uses strided `dataptr` memory block"""
     # Start with ctypes object to cover single SampleScalar
-    ct_base = custom_as_ctypes_type(self._ndarr.dtype)
-    for dim in reversed(self._ndarr.shape):
+    ct_base = custom_as_ctypes_type(self.dtype)
+    for dim in reversed(self.shape):
       ct_base = ct_base * dim
 
     # Single ctypes object to cover whole strided data-block
@@ -101,23 +77,27 @@ class PaddedStruct(ctypes.Structure):
     ct_array = ctypes.cast(dataptr, ctypes.POINTER(ct_base))
     return np.ctypeslib.as_array(ct_array, shape=())
 
-  def setitem(self, src: 'SampleScalar', dataptr: int) -> None:
+  def setitem(self, src: 'Scalar', dataptr: int) -> int:
     """Python `NPY_DT_setitem` implementation
 
     NOTE:
       `dataptr` is a buffer address, valid during execution of the function only
     """
-    if not self.is_compatible(src):
+    if not src.is_compatible(self):
       raise ValueError('Incompatible item value')
 
+    # TODO: Avoid private member access
     self._get_np_view(dataptr)[...] = src._ndarr
+    return 0  # -1 on failure
 
-  def getitem(self, dataptr: int) -> 'SampleScalar':
+  def getitem(self, dataptr: int) -> 'Scalar':
     """Python `NPY_DT_getitem` implementation
 
-    NOTE: See setitem()
+    NOTE:
+      `dataptr` is a buffer address, valid during execution of the function only
     """
-    new = self.copy()
+    new = self._create()
+    # TODO: Avoid private member access
     new._ndarr[...] = self._get_np_view(dataptr)
     return new
 
@@ -126,14 +106,13 @@ def clear_loop(self, data: int, size: int, stride: int) -> int:
 
     NOTE: See setitem()
     """
-    print(f'scalar.py: SampleScalar.clear_loop {data=}, {size=}, {stride=}')
     view = self._get_strided_np_view(data, size, stride)
     view[...] = 0  # Force dereference of all object-entries
     return 0
 
   def cast_loop(
       self,
-      other: 'SampleScalar',
+      other: 'ScalarType',
       data: tuple[int, int],
       dimensions: tuple[int],
       strides: tuple[int, int],
@@ -142,10 +121,47 @@ def cast_loop(
 
     NOTE: See setitem()
     """
-    print(
-        f'scalar.py: SampleScalar.cast_loop {other=}, {data=}, {dimensions=}, {strides=}'
-    )
     in_view = other._get_strided_np_view(data[0], dimensions[0], strides[0])
     out_view = self._get_strided_np_view(data[1], dimensions[0], strides[1])
     out_view[...] = in_view
     return 0
+
+
+class Scalar(ScalarType):
+  _ndarr: np.ndarray
+
+  def __init__(self, *args, **kwargs):
+    super().__init__(*args, **kwargs)
+    self._ndarr = np.empty(self.shape, dtype=self.dtype)
+
+
+def custom_as_ctypes_type(dtype: np.dtype):
+  """Wrapper around `numpy.ctypeslib.as_ctypes_type` that maps 'O' to py_object"""
+  if not dtype.hasobject:
+    return np.ctypeslib.as_ctypes_type(dtype)
+
+  if dtype.fields is None:
+    # Simple dtype or array
+    if dtype.subdtype:
+      # Handle sub-array types like ('O', 3)
+      base, shape = dtype.subdtype
+      new_base = custom_as_ctypes_type(base)
+      for dim in reversed(shape):
+        new_base = new_base * dim
+      return new_base
+    else:
+      # Simple object (non-objects are already handled)
+      assert dtype == np.dtype(object)
+      return ctypes.py_object
+
+  # Structured dtype
+  new_fields = []
+  for name, (field_dtype, *_) in dtype.fields.items():
+    new_field_dtype = custom_as_ctypes_type(field_dtype)
+    new_fields.append((name, new_field_dtype))
+
+  class CustomStruct(ctypes.Structure):
+    _pack_ = dtype.alignment
+    _fields_ = new_fields
+
+  return CustomStruct