feat: encode bytearray and memoryview buffers

CHANGELOG.md

@@ -2,6 +2,25 @@
 
 ## Unreleased
 
+## 0.3.0b5
+
+Binary buffer encode parity and actionable mapping-key errors.
+
+<!-- release-compatibility:start -->
+- Compatibility since `0.1.0b3`: 32 newly supported, 1 removed, 36 total support-status changes, and 0 shared canonical-wire changes.
+<!-- release-compatibility:end -->
+
+- Encode exact `bytearray` and C-contiguous `memoryview` values as standard padded base64.
+- Copy mutable and exported buffers before encoding; keep `bytes` and `bytearray` subclasses on
+  the explicit hook/refusal path, matching msgspec 0.21.1.
+- Preserve msgspec's `BufferError` refusal for non-contiguous memoryviews.
+- Point non-string mapping-key errors to `msgspec.to_builtins(..., str_keys=True)` without changing
+  the established mapping-key policy.
+- Record `bytearray`, `memoryview`, and `bytes` subclasses as separate executable support-matrix
+  entries.
+- Advance the performance guard to `v0.3.0b4`.
+- Preserve canonical bytes and token counts for existing locked payloads.
+
 ## 0.3.0b4
 
 Native encode parity for collection and binary projections.

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "msgspec-toon-native"
-version = "0.3.0-beta.4"
+version = "0.3.0-beta.5"
 edition = "2024"
 rust-version = "1.88"
 license = "MIT"

README.md

@@ -142,30 +142,36 @@ The encoder handles these Python values without `msgspec.to_builtins` or an `enc
 - `None`, `bool`, `int`, `float`, and `str`
 - `list`, `tuple`, `set`, and `frozenset`
 - dictionaries with string keys
-- exact `bytes` objects
+- exact `bytes` and `bytearray` objects, plus C-contiguous `memoryview` objects
 - `msgspec.Struct` instances
 - the msgspec-native scalar values in the next section
 
-The three projected types use the same value model as msgspec:
+The projected types use the same value model as msgspec:
 
 | Python value | Encoded TOON value | Decode boundary |
 |---|---|---|
 | `set` | array | Untyped decode returns `list`. |
 | `frozenset` | array | Untyped decode returns `list`. |
 | exact `bytes` | padded base64 string | Untyped decode returns `str`. |
+| exact `bytearray` | padded base64 string | Untyped decode returns `str`. |
+| C-contiguous `memoryview` | padded base64 string | Untyped decode returns `str`. |
 
 Set order is the current Python iteration order. This behavior matches the default msgspec encoder.
 The order can change between processes because sets are unordered.
 When output order must be stable, use a list.
 
-Typed decode does not reconstruct `set`, `frozenset`, or `bytes`. Decode the projected value first.
-Then use `msgspec.convert` when the application needs the original type.
+Typed decode does not reconstruct these projected container or binary types. Decode the projected
+value first. Then use `msgspec.convert` when the application needs the original type.
 
-The encoder refuses `bytearray`, `memoryview`, and subclasses of `bytes`.
-An `enc_hook` can convert each value to exact `bytes`:
+The encoder raises `BufferError` for a non-contiguous memoryview before it calls `enc_hook`.
+For a `bytes` or `bytearray` subclass, an `enc_hook` can convert the value to exact `bytes`:
 
 ```python
-toon.encode(memoryview(b"ab"), enc_hook=lambda value: bytes(value))
+class SemanticBytes(bytes):
+    pass
+
+
+toon.encode(SemanticBytes(b"ab"), enc_hook=lambda value: bytes(value))
 # b'YWI='
 ```
 
@@ -224,7 +230,7 @@ float string conversions as msgspec 0.21.1. Strict mode stays the default.
 Other multi-member unions remain explicit plan errors. Use a tagged Struct union for object
 variants. Use `object` or `Any` when the value shape is intentionally open.
 
-Non-string mapping keys are intentionally rejected. See the
+Non-string mapping keys are intentionally rejected. See the supported conversion route in the
 [mapping-key policy](https://github.com/goblinmode2700/msgspec-toon/blob/main/docs/mapping-key-policy.md).
 
 ## Why not wrap another TOON codec?

benches/GUARD_TAG

@@ -1 +1 @@
-v0.3.0b3
+v0.3.0b4

conformance/support_matrix.py

@@ -117,6 +117,10 @@ class Priority(enum.IntEnum):
     HIGH = 2
 
 
+class BytesSubclass(bytes):
+    pass
+
+
 @dataclass
 class PlainDataclass:
     x: int
@@ -625,6 +629,35 @@ class ArrayKeywordOnly(msgspec.Struct, array_like=True, kw_only=True):
         "so untyped decode returns str",
         round_trip=lambda: _projection_round_trip(b"ab"),
     ),
+    SupportEntry(
+        "exact bytearray encode projection",
+        2,
+        SUPPORTED,
+        lambda: toon.decode(toon.encode(bytearray(b"ab"))),
+        lambda: msgspec.json.decode(msgspec.json.encode(bytearray(b"ab"))),
+        "native encode copies the mutable buffer before applying msgspec-compatible padded base64; "
+        "untyped decode returns str",
+        round_trip=lambda: _projection_round_trip(bytearray(b"ab")),
+    ),
+    SupportEntry(
+        "memoryview encode projection",
+        2,
+        SUPPORTED,
+        lambda: toon.decode(toon.encode(memoryview(b"ab"))),
+        lambda: msgspec.json.decode(msgspec.json.encode(memoryview(b"ab"))),
+        "native encode copies a C-contiguous view before applying msgspec-compatible padded base64; "
+        "untyped decode returns str",
+        round_trip=lambda: _projection_round_trip(memoryview(b"ab")),
+    ),
+    SupportEntry(
+        "bytes subclasses",
+        2,
+        PARITY_REJECTS,
+        lambda: toon.encode(BytesSubclass(b"ab")),
+        lambda: msgspec.json.encode(BytesSubclass(b"ab")),
+        "both encoders refuse subclasses because the subtype can carry semantics not present in "
+        "exact bytes",
+    ),
     SupportEntry(
         "dataclasses",
         2,

docs/mapping-key-policy.md

@@ -13,5 +13,10 @@ executable support matrix has one explicit probe for this boundary. Therefore, t
 keeps the plan-construction rejection. It returns `TypePlanError` with code
 `unsupported_mapping_key` and a schema-only path. It does not decode to the wrong key type.
 
+Untyped encoding also rejects mappings with non-string keys instead of silently choosing a
+stringification and collision policy. Every root or nested `EncodeError` names
+`msgspec.to_builtins(..., str_keys=True)` as the supported conversion route for callers that choose
+msgspec's policy.
+
 This decision can change only after differential tests define conversion, collision, large-integer,
 and payload-safety behavior.

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "msgspec-toon"
-version = "0.3.0b4"
+version = "0.3.0b5"
 description = "A native TOON 4.1 codec with direct msgspec.Struct decoding"
 requires-python = ">=3.13"
 dependencies = ["msgspec==0.21.1"]

python/msgspec_toon/__init__.py

@@ -59,7 +59,7 @@ class _RawScalar(str):
 
 
 def _is_native_scalar(value: Any) -> bool:
-    return type(value) is bytes or isinstance(value, _NATIVE_SCALAR_TYPES)
+    return type(value) in (bytes, bytearray, memoryview) or isinstance(value, _NATIVE_SCALAR_TYPES)
 
 
 class _EncodeHook:

src/encode.rs

@@ -9,12 +9,12 @@
 use std::collections::HashMap;
 use std::sync::{Arc, Mutex};
 
-use pyo3::exceptions::PyAttributeError;
+use pyo3::exceptions::{PyAttributeError, PyBufferError};
 use pyo3::prelude::*;
 use pyo3::sync::critical_section::with_critical_section;
 use pyo3::types::{
-    PyBool, PyBytes, PyBytesMethods, PyDict, PyFloat, PyFrozenSet, PyInt, PyList, PySet, PyString,
-    PyTuple,
+    PyBool, PyByteArray, PyByteArrayMethods, PyBytes, PyBytesMethods, PyDict, PyFloat, PyFrozenSet,
+    PyInt, PyList, PySet, PyString, PyTuple,
 };
 
 use crate::limits::{MAX_NESTING_DEPTH, reserve_bytes};
@@ -26,6 +26,8 @@ use crate::writer::Writer;
 const CELL_WIDTH_ESTIMATE: usize = 10;
 
 const MAX_HOOK_DEPTH: usize = 8;
+const NON_STRING_KEY_ERROR: &str =
+    "object keys must be strings; convert with msgspec.to_builtins(..., str_keys=True)";
 
 pub struct EncodeContext {
     pub enc_hook: Option<Py<PyAny>>,
@@ -302,6 +304,11 @@ fn classify_fallback<'py>(
         let text = std::str::from_utf8(&encoded).expect("base64 output is ASCII");
         return Ok(Val::Str(PyString::new(py, text)));
     }
+    if let Some(bytes) = exact_buffer_bytes(ctx, py, obj)? {
+        let encoded = checked_base64(ctx, py, &bytes)?;
+        let text = std::str::from_utf8(&encoded).expect("base64 output is ASCII");
+        return Ok(Val::Str(PyString::new(py, text)));
+    }
     if obj.is_instance_of::<PySet>() || obj.is_instance_of::<PyFrozenSet>() {
         // Match msgspec's default encoder: a set is an array in its current
         // interpreter iteration order. Collect owned references, not a Python
@@ -404,7 +411,7 @@ fn object_pairs<'value, 'py>(
             let mut pairs = Vec::with_capacity(map.len());
             for (key, item) in map.iter() {
                 let Ok(key_text) = key.cast_into::<PyString>() else {
-                    return Err(encode_err(ctx, py, "object keys must be strings"));
+                    return Err(encode_err(ctx, py, NON_STRING_KEY_ERROR));
                 };
                 pairs.push((EntryText::Object(key_text), item));
             }
@@ -779,7 +786,7 @@ fn root_object_decision<'value, 'py>(
     let mut rows = (map.len() >= 2).then(|| Vec::with_capacity(map.len()));
     for (key, item) in map.iter() {
         let Ok(key_text) = key.cast_into::<PyString>() else {
-            return Err(encode_err(ctx, py, "object keys must be strings"));
+            return Err(encode_err(ctx, py, NON_STRING_KEY_ERROR));
         };
         if let Some(candidate_rows) = rows.as_mut() {
             if item.is_instance_of::<PyDict>() || item.is_instance(ctx.struct_base.bind(py))? {
@@ -1259,6 +1266,36 @@ fn exact_type(obj: &Bound<'_, PyAny>) -> *mut pyo3::ffi::PyTypeObject {
     unsafe { pyo3::ffi::Py_TYPE(obj.as_ptr()) }
 }
 
+/// Copy the two exact buffer-backed types that msgspec projects as binary
+/// values. Exact-type dispatch intentionally keeps `bytes` and `bytearray`
+/// subclasses on the hook/refusal path. msgspec accepts only C-contiguous
+/// memoryviews, so check that boundary before copying through `tobytes`.
+#[cold]
+#[inline(never)]
+fn exact_buffer_bytes(
+    ctx: &EncodeContext,
+    py: Python<'_>,
+    obj: &Bound<'_, PyAny>,
+) -> PyResult<Option<Vec<u8>>> {
+    let type_ptr = exact_type(obj);
+    if type_ptr == std::ptr::addr_of_mut!(pyo3::ffi::PyByteArray_Type) {
+        let value = obj.cast::<PyByteArray>()?;
+        check_binary_len(ctx, py, value.len())?;
+        return Ok(Some(value.to_vec()));
+    }
+    if type_ptr == std::ptr::addr_of_mut!(pyo3::ffi::PyMemoryView_Type) {
+        if !obj.getattr("c_contiguous")?.extract::<bool>()? {
+            return Err(PyBufferError::new_err(
+                "memoryview: underlying buffer is not C-contiguous",
+            ));
+        }
+        check_binary_len(ctx, py, obj.getattr("nbytes")?.extract::<usize>()?)?;
+        let copied = obj.call_method0("tobytes")?;
+        return Ok(Some(copied.cast::<PyBytes>()?.as_bytes().to_vec()));
+    }
+    Ok(None)
+}
+
 #[cold]
 #[inline(never)]
 fn raw_scalar<'py>(
@@ -1354,6 +1391,9 @@ fn write_scalar_fallback<'py>(
     if type_ptr == std::ptr::addr_of_mut!(pyo3::ffi::PyBytes_Type) {
         return write_bytes_scalar(ctx, py, writer, obj.cast::<PyBytes>()?);
     }
+    if let Some(bytes) = exact_buffer_bytes(ctx, py, obj)? {
+        return write_binary_scalar(ctx, py, writer, &bytes);
+    }
     // Subclass slow path.
     if obj.is_instance_of::<PyBool>() {
         writer.bytes(if obj.extract::<bool>()? {
@@ -1460,14 +1500,19 @@ fn base64_encode(input: &[u8]) -> Vec<u8> {
 #[cold]
 #[inline(never)]
 fn checked_base64(ctx: &EncodeContext, py: Python<'_>, input: &[u8]) -> PyResult<Vec<u8>> {
-    if input.len() > u32::MAX as usize {
+    check_binary_len(ctx, py, input.len())?;
+    Ok(base64_encode(input))
+}
+
+fn check_binary_len(ctx: &EncodeContext, py: Python<'_>, len: usize) -> PyResult<()> {
+    if len > u32::MAX as usize {
         return Err(encode_err(
             ctx,
             py,
             "bytes objects longer than 2**32 - 1 are not encodable",
         ));
     }
-    Ok(base64_encode(input))
+    Ok(())
 }
 
 #[cold]
@@ -1478,7 +1523,18 @@ fn write_bytes_scalar(
     writer: &mut Writer,
     value: &Bound<'_, PyBytes>,
 ) -> PyResult<()> {
-    let encoded = checked_base64(ctx, py, value.as_bytes())?;
+    write_binary_scalar(ctx, py, writer, value.as_bytes())
+}
+
+#[cold]
+#[inline(never)]
+fn write_binary_scalar(
+    ctx: &EncodeContext,
+    py: Python<'_>,
+    writer: &mut Writer,
+    value: &[u8],
+) -> PyResult<()> {
+    let encoded = checked_base64(ctx, py, value)?;
     let text = std::str::from_utf8(&encoded).expect("base64 output is ASCII");
     if needs_quote(text, ctx.delimiter) {
         write_quoted(writer, text);

tests/test_native_encode_types.py

@@ -28,6 +28,10 @@ class BytesSubclass(bytes):
     pass
 
 
+class BytearraySubclass(bytearray):
+    pass
+
+
 def projected(value: Any) -> Any:
     """The msgspec value model that TOON can represent on its wire."""
     return msgspec.to_builtins(value)
@@ -95,11 +99,60 @@ def test_bytes_are_scalars_inside_compact_arrays() -> None:
     assert toon.encode(values) == toon.encode([projected(value) for value in values])
 
 
+@pytest.mark.parametrize("factory", [bytearray, memoryview])
 @pytest.mark.parametrize(
-    "value",
-    [bytearray(b"ab"), memoryview(b"ab"), BytesSubclass(b"ab")],
+    "payload",
+    [b"", b"a", b"ab", b"abc", bytes(range(256))],
 )
-def test_refused_bytes_like_values_have_a_working_hook_route(value: Any) -> None:
+def test_buffer_values_encode_as_their_msgspec_projection(
+    factory: type[bytearray | memoryview], payload: bytes
+) -> None:
+    value = factory(payload)
+    expected = toon.encode(payload)
+    assert toon.encode(value) == expected
+    assert toon.Encoder().encode(value) == expected
+    assert toon.decode(expected) == projected(value)
+
+
+def test_non_contiguous_memoryview_matches_msgspec_refusal() -> None:
+    value = memoryview(b"abcdef")[::2]
+    with pytest.raises(BufferError, match="not C-contiguous"):
+        msgspec.to_builtins(value)
+    with pytest.raises(BufferError, match="not C-contiguous"):
+        toon.encode(value)
+
+
+@pytest.mark.parametrize("value", [bytearray(b"ab"), memoryview(b"ab")])
+def test_buffer_values_bypass_enc_hook(value: bytearray | memoryview) -> None:
+    calls: list[Any] = []
+    assert toon.encode(value, enc_hook=lambda item: calls.append(item) or "hooked") == toon.encode(
+        b"ab"
+    )
+    assert calls == []
+
+
+@pytest.mark.parametrize("value", [bytearray(b"ab"), memoryview(b"ab")])
+def test_buffer_values_work_inside_structs(value: bytearray | memoryview) -> None:
+    item = NativeTypesRow(set(), frozenset(), value)  # type: ignore[arg-type]
+    assert toon.encode(item) == toon.encode(projected(item))
+
+
+def test_buffer_values_are_scalars_inside_compact_arrays() -> None:
+    values = [bytearray(b"ab"), memoryview(b"cd")]
+    assert toon.encode(values) == toon.encode([projected(value) for value in values])
+
+
+@pytest.mark.parametrize("value", [BytesSubclass(b"ab"), BytearraySubclass(b"ab")])
+def test_binary_subclasses_remain_refused_with_a_working_hook_route(value: Any) -> None:
     with pytest.raises(msgspec.EncodeError, match=f"unsupported type: {type(value).__name__}"):
         toon.encode(value)
     assert toon.encode(value, enc_hook=lambda item: bytes(item)) == toon.encode(b"ab")
+
+
+@pytest.mark.parametrize("value", [{1: "a"}, {"outer": {1: "a"}}])
+def test_non_string_mapping_key_error_names_conversion_route(value: Any) -> None:
+    with pytest.raises(
+        msgspec.EncodeError,
+        match=r"msgspec\.to_builtins\(\.\.\., str_keys=True\)",
+    ):
+        toon.encode(value)