diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a15cbd..527f113 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,25 @@ ## Unreleased +## 0.3.0b5 + +Binary buffer encode parity and actionable mapping-key errors. + + +- Compatibility since `0.1.0b3`: 32 newly supported, 1 removed, 36 total support-status changes, and 0 shared canonical-wire changes. + + +- Encode exact `bytearray` and C-contiguous `memoryview` values as standard padded base64. +- Copy mutable and exported buffers before encoding; keep `bytes` and `bytearray` subclasses on + the explicit hook/refusal path, matching msgspec 0.21.1. +- Preserve msgspec's `BufferError` refusal for non-contiguous memoryviews. +- Point non-string mapping-key errors to `msgspec.to_builtins(..., str_keys=True)` without changing + the established mapping-key policy. +- Record `bytearray`, `memoryview`, and `bytes` subclasses as separate executable support-matrix + entries. +- Advance the performance guard to `v0.3.0b4`. +- Preserve canonical bytes and token counts for existing locked payloads. + ## 0.3.0b4 Native encode parity for collection and binary projections. diff --git a/Cargo.lock b/Cargo.lock index 8fc5eae..57af6b5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -28,7 +28,7 @@ checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "msgspec-toon-native" -version = "0.3.0-beta.4" +version = "0.3.0-beta.5" dependencies = [ "itoa", "memchr", diff --git a/Cargo.toml b/Cargo.toml index d55837e..bd68537 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "msgspec-toon-native" -version = "0.3.0-beta.4" +version = "0.3.0-beta.5" edition = "2024" rust-version = "1.88" license = "MIT" diff --git a/README.md b/README.md index f250d53..3a6dba0 100644 --- a/README.md +++ b/README.md @@ -142,30 +142,36 @@ The encoder handles these Python values without `msgspec.to_builtins` or an `enc - `None`, `bool`, `int`, `float`, and `str` - `list`, `tuple`, `set`, and `frozenset` - dictionaries with string keys -- exact `bytes` objects +- exact `bytes` and `bytearray` objects, plus C-contiguous `memoryview` objects - `msgspec.Struct` instances - the msgspec-native scalar values in the next section -The three projected types use the same value model as msgspec: +The projected types use the same value model as msgspec: | Python value | Encoded TOON value | Decode boundary | |---|---|---| | `set` | array | Untyped decode returns `list`. | | `frozenset` | array | Untyped decode returns `list`. | | exact `bytes` | padded base64 string | Untyped decode returns `str`. | +| exact `bytearray` | padded base64 string | Untyped decode returns `str`. | +| C-contiguous `memoryview` | padded base64 string | Untyped decode returns `str`. | Set order is the current Python iteration order. This behavior matches the default msgspec encoder. The order can change between processes because sets are unordered. When output order must be stable, use a list. -Typed decode does not reconstruct `set`, `frozenset`, or `bytes`. Decode the projected value first. -Then use `msgspec.convert` when the application needs the original type. +Typed decode does not reconstruct these projected container or binary types. Decode the projected +value first. Then use `msgspec.convert` when the application needs the original type. -The encoder refuses `bytearray`, `memoryview`, and subclasses of `bytes`. -An `enc_hook` can convert each value to exact `bytes`: +The encoder raises `BufferError` for a non-contiguous memoryview before it calls `enc_hook`. +For a `bytes` or `bytearray` subclass, an `enc_hook` can convert the value to exact `bytes`: ```python -toon.encode(memoryview(b"ab"), enc_hook=lambda value: bytes(value)) +class SemanticBytes(bytes): + pass + + +toon.encode(SemanticBytes(b"ab"), enc_hook=lambda value: bytes(value)) # b'YWI=' ``` @@ -224,7 +230,7 @@ float string conversions as msgspec 0.21.1. Strict mode stays the default. Other multi-member unions remain explicit plan errors. Use a tagged Struct union for object variants. Use `object` or `Any` when the value shape is intentionally open. -Non-string mapping keys are intentionally rejected. See the +Non-string mapping keys are intentionally rejected. See the supported conversion route in the [mapping-key policy](https://github.com/goblinmode2700/msgspec-toon/blob/main/docs/mapping-key-policy.md). ## Why not wrap another TOON codec? diff --git a/benches/GUARD_TAG b/benches/GUARD_TAG index 477b3e4..3620d0a 100644 --- a/benches/GUARD_TAG +++ b/benches/GUARD_TAG @@ -1 +1 @@ -v0.3.0b3 +v0.3.0b4 diff --git a/conformance/support_matrix.py b/conformance/support_matrix.py index 737c997..b8127aa 100644 --- a/conformance/support_matrix.py +++ b/conformance/support_matrix.py @@ -117,6 +117,10 @@ class Priority(enum.IntEnum): HIGH = 2 +class BytesSubclass(bytes): + pass + + @dataclass class PlainDataclass: x: int @@ -625,6 +629,35 @@ class ArrayKeywordOnly(msgspec.Struct, array_like=True, kw_only=True): "so untyped decode returns str", round_trip=lambda: _projection_round_trip(b"ab"), ), + SupportEntry( + "exact bytearray encode projection", + 2, + SUPPORTED, + lambda: toon.decode(toon.encode(bytearray(b"ab"))), + lambda: msgspec.json.decode(msgspec.json.encode(bytearray(b"ab"))), + "native encode copies the mutable buffer before applying msgspec-compatible padded base64; " + "untyped decode returns str", + round_trip=lambda: _projection_round_trip(bytearray(b"ab")), + ), + SupportEntry( + "memoryview encode projection", + 2, + SUPPORTED, + lambda: toon.decode(toon.encode(memoryview(b"ab"))), + lambda: msgspec.json.decode(msgspec.json.encode(memoryview(b"ab"))), + "native encode copies a C-contiguous view before applying msgspec-compatible padded base64; " + "untyped decode returns str", + round_trip=lambda: _projection_round_trip(memoryview(b"ab")), + ), + SupportEntry( + "bytes subclasses", + 2, + PARITY_REJECTS, + lambda: toon.encode(BytesSubclass(b"ab")), + lambda: msgspec.json.encode(BytesSubclass(b"ab")), + "both encoders refuse subclasses because the subtype can carry semantics not present in " + "exact bytes", + ), SupportEntry( "dataclasses", 2, diff --git a/docs/mapping-key-policy.md b/docs/mapping-key-policy.md index c50fcd8..d57be42 100644 --- a/docs/mapping-key-policy.md +++ b/docs/mapping-key-policy.md @@ -13,5 +13,10 @@ executable support matrix has one explicit probe for this boundary. Therefore, t keeps the plan-construction rejection. It returns `TypePlanError` with code `unsupported_mapping_key` and a schema-only path. It does not decode to the wrong key type. +Untyped encoding also rejects mappings with non-string keys instead of silently choosing a +stringification and collision policy. Every root or nested `EncodeError` names +`msgspec.to_builtins(..., str_keys=True)` as the supported conversion route for callers that choose +msgspec's policy. + This decision can change only after differential tests define conversion, collision, large-integer, and payload-safety behavior. diff --git a/pyproject.toml b/pyproject.toml index 578e814..e7481bf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "maturin" [project] name = "msgspec-toon" -version = "0.3.0b4" +version = "0.3.0b5" description = "A native TOON 4.1 codec with direct msgspec.Struct decoding" requires-python = ">=3.13" dependencies = ["msgspec==0.21.1"] diff --git a/python/msgspec_toon/__init__.py b/python/msgspec_toon/__init__.py index ea059b5..3c788ed 100644 --- a/python/msgspec_toon/__init__.py +++ b/python/msgspec_toon/__init__.py @@ -59,7 +59,7 @@ class _RawScalar(str): def _is_native_scalar(value: Any) -> bool: - return type(value) is bytes or isinstance(value, _NATIVE_SCALAR_TYPES) + return type(value) in (bytes, bytearray, memoryview) or isinstance(value, _NATIVE_SCALAR_TYPES) class _EncodeHook: diff --git a/src/encode.rs b/src/encode.rs index 9f3f2d6..7fb0e90 100644 --- a/src/encode.rs +++ b/src/encode.rs @@ -9,12 +9,12 @@ use std::collections::HashMap; use std::sync::{Arc, Mutex}; -use pyo3::exceptions::PyAttributeError; +use pyo3::exceptions::{PyAttributeError, PyBufferError}; use pyo3::prelude::*; use pyo3::sync::critical_section::with_critical_section; use pyo3::types::{ - PyBool, PyBytes, PyBytesMethods, PyDict, PyFloat, PyFrozenSet, PyInt, PyList, PySet, PyString, - PyTuple, + PyBool, PyByteArray, PyByteArrayMethods, PyBytes, PyBytesMethods, PyDict, PyFloat, PyFrozenSet, + PyInt, PyList, PySet, PyString, PyTuple, }; use crate::limits::{MAX_NESTING_DEPTH, reserve_bytes}; @@ -26,6 +26,8 @@ use crate::writer::Writer; const CELL_WIDTH_ESTIMATE: usize = 10; const MAX_HOOK_DEPTH: usize = 8; +const NON_STRING_KEY_ERROR: &str = + "object keys must be strings; convert with msgspec.to_builtins(..., str_keys=True)"; pub struct EncodeContext { pub enc_hook: Option>, @@ -302,6 +304,11 @@ fn classify_fallback<'py>( let text = std::str::from_utf8(&encoded).expect("base64 output is ASCII"); return Ok(Val::Str(PyString::new(py, text))); } + if let Some(bytes) = exact_buffer_bytes(ctx, py, obj)? { + let encoded = checked_base64(ctx, py, &bytes)?; + let text = std::str::from_utf8(&encoded).expect("base64 output is ASCII"); + return Ok(Val::Str(PyString::new(py, text))); + } if obj.is_instance_of::() || obj.is_instance_of::() { // Match msgspec's default encoder: a set is an array in its current // interpreter iteration order. Collect owned references, not a Python @@ -404,7 +411,7 @@ fn object_pairs<'value, 'py>( let mut pairs = Vec::with_capacity(map.len()); for (key, item) in map.iter() { let Ok(key_text) = key.cast_into::() else { - return Err(encode_err(ctx, py, "object keys must be strings")); + return Err(encode_err(ctx, py, NON_STRING_KEY_ERROR)); }; pairs.push((EntryText::Object(key_text), item)); } @@ -779,7 +786,7 @@ fn root_object_decision<'value, 'py>( let mut rows = (map.len() >= 2).then(|| Vec::with_capacity(map.len())); for (key, item) in map.iter() { let Ok(key_text) = key.cast_into::() else { - return Err(encode_err(ctx, py, "object keys must be strings")); + return Err(encode_err(ctx, py, NON_STRING_KEY_ERROR)); }; if let Some(candidate_rows) = rows.as_mut() { if item.is_instance_of::() || item.is_instance(ctx.struct_base.bind(py))? { @@ -1259,6 +1266,36 @@ fn exact_type(obj: &Bound<'_, PyAny>) -> *mut pyo3::ffi::PyTypeObject { unsafe { pyo3::ffi::Py_TYPE(obj.as_ptr()) } } +/// Copy the two exact buffer-backed types that msgspec projects as binary +/// values. Exact-type dispatch intentionally keeps `bytes` and `bytearray` +/// subclasses on the hook/refusal path. msgspec accepts only C-contiguous +/// memoryviews, so check that boundary before copying through `tobytes`. +#[cold] +#[inline(never)] +fn exact_buffer_bytes( + ctx: &EncodeContext, + py: Python<'_>, + obj: &Bound<'_, PyAny>, +) -> PyResult>> { + let type_ptr = exact_type(obj); + if type_ptr == std::ptr::addr_of_mut!(pyo3::ffi::PyByteArray_Type) { + let value = obj.cast::()?; + check_binary_len(ctx, py, value.len())?; + return Ok(Some(value.to_vec())); + } + if type_ptr == std::ptr::addr_of_mut!(pyo3::ffi::PyMemoryView_Type) { + if !obj.getattr("c_contiguous")?.extract::()? { + return Err(PyBufferError::new_err( + "memoryview: underlying buffer is not C-contiguous", + )); + } + check_binary_len(ctx, py, obj.getattr("nbytes")?.extract::()?)?; + let copied = obj.call_method0("tobytes")?; + return Ok(Some(copied.cast::()?.as_bytes().to_vec())); + } + Ok(None) +} + #[cold] #[inline(never)] fn raw_scalar<'py>( @@ -1354,6 +1391,9 @@ fn write_scalar_fallback<'py>( if type_ptr == std::ptr::addr_of_mut!(pyo3::ffi::PyBytes_Type) { return write_bytes_scalar(ctx, py, writer, obj.cast::()?); } + if let Some(bytes) = exact_buffer_bytes(ctx, py, obj)? { + return write_binary_scalar(ctx, py, writer, &bytes); + } // Subclass slow path. if obj.is_instance_of::() { writer.bytes(if obj.extract::()? { @@ -1460,14 +1500,19 @@ fn base64_encode(input: &[u8]) -> Vec { #[cold] #[inline(never)] fn checked_base64(ctx: &EncodeContext, py: Python<'_>, input: &[u8]) -> PyResult> { - if input.len() > u32::MAX as usize { + check_binary_len(ctx, py, input.len())?; + Ok(base64_encode(input)) +} + +fn check_binary_len(ctx: &EncodeContext, py: Python<'_>, len: usize) -> PyResult<()> { + if len > u32::MAX as usize { return Err(encode_err( ctx, py, "bytes objects longer than 2**32 - 1 are not encodable", )); } - Ok(base64_encode(input)) + Ok(()) } #[cold] @@ -1478,7 +1523,18 @@ fn write_bytes_scalar( writer: &mut Writer, value: &Bound<'_, PyBytes>, ) -> PyResult<()> { - let encoded = checked_base64(ctx, py, value.as_bytes())?; + write_binary_scalar(ctx, py, writer, value.as_bytes()) +} + +#[cold] +#[inline(never)] +fn write_binary_scalar( + ctx: &EncodeContext, + py: Python<'_>, + writer: &mut Writer, + value: &[u8], +) -> PyResult<()> { + let encoded = checked_base64(ctx, py, value)?; let text = std::str::from_utf8(&encoded).expect("base64 output is ASCII"); if needs_quote(text, ctx.delimiter) { write_quoted(writer, text); diff --git a/tests/test_native_encode_types.py b/tests/test_native_encode_types.py index 6f3c3f9..26eb697 100644 --- a/tests/test_native_encode_types.py +++ b/tests/test_native_encode_types.py @@ -28,6 +28,10 @@ class BytesSubclass(bytes): pass +class BytearraySubclass(bytearray): + pass + + def projected(value: Any) -> Any: """The msgspec value model that TOON can represent on its wire.""" return msgspec.to_builtins(value) @@ -95,11 +99,60 @@ def test_bytes_are_scalars_inside_compact_arrays() -> None: assert toon.encode(values) == toon.encode([projected(value) for value in values]) +@pytest.mark.parametrize("factory", [bytearray, memoryview]) @pytest.mark.parametrize( - "value", - [bytearray(b"ab"), memoryview(b"ab"), BytesSubclass(b"ab")], + "payload", + [b"", b"a", b"ab", b"abc", bytes(range(256))], ) -def test_refused_bytes_like_values_have_a_working_hook_route(value: Any) -> None: +def test_buffer_values_encode_as_their_msgspec_projection( + factory: type[bytearray | memoryview], payload: bytes +) -> None: + value = factory(payload) + expected = toon.encode(payload) + assert toon.encode(value) == expected + assert toon.Encoder().encode(value) == expected + assert toon.decode(expected) == projected(value) + + +def test_non_contiguous_memoryview_matches_msgspec_refusal() -> None: + value = memoryview(b"abcdef")[::2] + with pytest.raises(BufferError, match="not C-contiguous"): + msgspec.to_builtins(value) + with pytest.raises(BufferError, match="not C-contiguous"): + toon.encode(value) + + +@pytest.mark.parametrize("value", [bytearray(b"ab"), memoryview(b"ab")]) +def test_buffer_values_bypass_enc_hook(value: bytearray | memoryview) -> None: + calls: list[Any] = [] + assert toon.encode(value, enc_hook=lambda item: calls.append(item) or "hooked") == toon.encode( + b"ab" + ) + assert calls == [] + + +@pytest.mark.parametrize("value", [bytearray(b"ab"), memoryview(b"ab")]) +def test_buffer_values_work_inside_structs(value: bytearray | memoryview) -> None: + item = NativeTypesRow(set(), frozenset(), value) # type: ignore[arg-type] + assert toon.encode(item) == toon.encode(projected(item)) + + +def test_buffer_values_are_scalars_inside_compact_arrays() -> None: + values = [bytearray(b"ab"), memoryview(b"cd")] + assert toon.encode(values) == toon.encode([projected(value) for value in values]) + + +@pytest.mark.parametrize("value", [BytesSubclass(b"ab"), BytearraySubclass(b"ab")]) +def test_binary_subclasses_remain_refused_with_a_working_hook_route(value: Any) -> None: with pytest.raises(msgspec.EncodeError, match=f"unsupported type: {type(value).__name__}"): toon.encode(value) assert toon.encode(value, enc_hook=lambda item: bytes(item)) == toon.encode(b"ab") + + +@pytest.mark.parametrize("value", [{1: "a"}, {"outer": {1: "a"}}]) +def test_non_string_mapping_key_error_names_conversion_route(value: Any) -> None: + with pytest.raises( + msgspec.EncodeError, + match=r"msgspec\.to_builtins\(\.\.\., str_keys=True\)", + ): + toon.encode(value) diff --git a/tests/test_release_report.py b/tests/test_release_report.py index a940a73..07e7773 100644 --- a/tests/test_release_report.py +++ b/tests/test_release_report.py @@ -41,9 +41,15 @@ def test_current_compatibility_delta_records_support_changes( "after": "supported", }, {"feature": "bytes encode projection", "before": None, "after": "supported"}, + {"feature": "bytes subclasses", "before": None, "after": "parity_rejects"}, {"feature": "date", "before": None, "after": "supported"}, {"feature": "datetime", "before": "unsupported", "after": "supported"}, {"feature": "enum members", "before": "unsupported", "after": None}, + { + "feature": "exact bytearray encode projection", + "before": None, + "after": "supported", + }, {"feature": "fractional and exponent floats", "before": None, "after": "supported"}, {"feature": "frozenset encode projection", "before": None, "after": "supported"}, {"feature": "integer Enum", "before": None, "after": "supported"}, @@ -102,6 +108,11 @@ def test_current_compatibility_delta_records_support_changes( "before": None, "after": "supported", }, + { + "feature": "memoryview encode projection", + "before": None, + "after": "supported", + }, { "feature": "object and containers of object", "before": None, diff --git a/uv.lock b/uv.lock index d7f92fd..2fe0b30 100644 --- a/uv.lock +++ b/uv.lock @@ -239,7 +239,7 @@ wheels = [ [[package]] name = "msgspec-toon" -version = "0.3.0b4" +version = "0.3.0b5" source = { editable = "." } dependencies = [ { name = "msgspec" },