Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -155,26 +155,26 @@ The library includes specialized support for collections when working with:
1. Protocol Buffer message types
2. Primitive types (int, long, string, etc.)

When using `IList<T>` with these types, the library automatically handles serialization/deserialization without requiring wrapper message types. This is particularly useful for Protocol Buffers, where `RepeatedField<T>` typically cannot be serialized as a standalone entity.
When using `IList<T>` or `ISet<T>` with these types, the library automatically handles serialization/deserialization without requiring wrapper message types. This is particularly useful for Protocol Buffers, where `RepeatedField<T>` typically cannot be serialized as a standalone entity.

The serialization format varies depending on the element type:

#### Fixed-Size Types (int, long, etc.)

```
[4 bytes: List length][Contiguous array of serialized elements]
[4 bytes: Collection length][Contiguous array of serialized elements]
```

#### Variable-Size Types (string, protobuf messages)

```
[4 bytes: List length][For each element: [4 bytes: Element size][N bytes: Element data]]
[4 bytes: Collection length][For each element: [4 bytes: Element size][N bytes: Element data]]
```

Example types that work automatically with this support:

- Protocol Buffer message types: `IList<YourProtobufMessage>`
- Primitive types: `IList<int>`, `IList<long>`, `IList<string>`, etc.
- Protocol Buffer message types: `IList<YourProtobufMessage>`, `ISet<YourProtobufMessage>`
- Primitive types: `IList<int>`, `IList<long>`, `IList<string>`, `ISet<int>`, `ISet<string>`, etc.

## Merge Operators

Expand Down
99 changes: 99 additions & 0 deletions src/RocksDb.Extensions/FixedSizeCollectionSerializer.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
using System.Buffers;

namespace RocksDb.Extensions;

/// <summary>
/// Base class for serializing collections of fixed-size elements like primitive types (int, long, etc.)
/// where each element occupies the same number of bytes in memory.
/// </summary>
/// <remarks>
/// The serialized format consists of:
/// - 4 bytes: Collection length (number of elements)
/// - Remaining bytes: Contiguous array of serialized elements
/// </remarks>
/// <typeparam name="TCollection">The collection type (e.g., IList{T}, ISet{T})</typeparam>
/// <typeparam name="TElement">The element type</typeparam>
internal abstract class FixedSizeCollectionSerializer<TCollection, TElement> : ISerializer<TCollection>
where TCollection : ICollection<TElement>
{
private readonly ISerializer<TElement> _scalarSerializer;

protected FixedSizeCollectionSerializer(ISerializer<TElement> scalarSerializer)
{
_scalarSerializer = scalarSerializer;
}

/// <summary>
/// Creates a new collection instance with the specified capacity.
/// </summary>
protected abstract TCollection CreateCollection(int capacity);

/// <summary>
/// Adds an element to the collection.
/// </summary>
protected abstract void AddElement(TCollection collection, TElement element);

public bool TryCalculateSize(ref TCollection value, out int size)
{
size = sizeof(int); // size of the collection
if (value.Count == 0)
{
return true;
}

var referentialElement = value.First();
if (_scalarSerializer.TryCalculateSize(ref referentialElement, out var elementSize))
{
size += value.Count * elementSize;
return true;
}

return false;
}

public void WriteTo(ref TCollection value, ref Span<byte> span)
{
// Write the size of the collection
var slice = span.Slice(0, sizeof(int));
BitConverter.TryWriteBytes(slice, value.Count);

// Write the elements of the collection
int offset = sizeof(int);
var elementSize = (span.Length - offset) / value.Count;
Copy link

Copilot AI Dec 30, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Division by zero will occur when value.Count is 0. While TryCalculateSize handles empty collections correctly, WriteTo does not check if the collection is empty before dividing. This will cause a DivideByZeroException when serializing an empty collection. Add a check to return early when value.Count is 0.

Copilot uses AI. Check for mistakes.
foreach (var item in value)
{
var element = item;
slice = span.Slice(offset, elementSize);
_scalarSerializer.WriteTo(ref element, ref slice);
offset += elementSize;
}
}

public void WriteTo(ref TCollection value, IBufferWriter<byte> buffer)
{
throw new NotImplementedException();
}

public TCollection Deserialize(ReadOnlySpan<byte> buffer)
{
// Read the size of the collection
var slice = buffer.Slice(0, sizeof(int));
var size = BitConverter.ToInt32(slice);

var collection = CreateCollection(size);

// Read the elements of the collection
int offset = sizeof(int);
var elementSize = (buffer.Length - offset) / size;
Copy link

Copilot AI Dec 30, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Division by zero will occur when size is 0. The Deserialize method does not check if the deserialized size is 0 before dividing. This will cause a DivideByZeroException when deserializing an empty collection. Add a check to return an empty collection when size is 0.

Copilot uses AI. Check for mistakes.
for (int i = 0; i < size; i++)
{
slice = buffer.Slice(offset, elementSize);
var element = _scalarSerializer.Deserialize(slice);
AddElement(collection, element);
offset += elementSize;
}

return collection;
}
}

83 changes: 4 additions & 79 deletions src/RocksDb.Extensions/FixedSizeListSerializer.cs
Original file line number Diff line number Diff line change
@@ -1,87 +1,12 @@
using System.Buffers;

namespace RocksDb.Extensions;

/// <summary>
/// Serializes lists of fixed-size elements like primitive types (int, long, etc.) where each element
/// occupies the same number of bytes in memory. This implementation optimizes for performance by
/// pre-calculating buffer sizes based on element count.
/// </summary>
/// <remarks>
/// Use this serializer when working with lists of primitive types or structs where all elements
/// have identical size. The serialized format consists of:
/// - 4 bytes: List length (number of elements)
/// - Remaining bytes: Contiguous array of serialized elements
/// </remarks>
internal class FixedSizeListSerializer<T> : ISerializer<IList<T>>
internal class FixedSizeListSerializer<T> : FixedSizeCollectionSerializer<IList<T>, T>
{
private readonly ISerializer<T> _scalarSerializer;

public FixedSizeListSerializer(ISerializer<T> scalarSerializer)
{
_scalarSerializer = scalarSerializer;
}

public bool TryCalculateSize(ref IList<T> value, out int size)
public FixedSizeListSerializer(ISerializer<T> scalarSerializer) : base(scalarSerializer)
{
size = sizeof(int); // size of the list
if (value.Count == 0)
{
return true;
}

var referentialElement = value[0];
if (_scalarSerializer.TryCalculateSize(ref referentialElement, out var elementSize))
{
size += value.Count * elementSize;
return true;
}

return false;
}

public void WriteTo(ref IList<T> value, ref Span<byte> span)
{
// Write the size of the list
var slice = span.Slice(0, sizeof(int));
BitConverter.TryWriteBytes(slice, value.Count);

// Write the elements of the list
int offset = sizeof(int);
var elementSize = (span.Length - offset) / value.Count;
for (int i = 0; i < value.Count; i++)
{
var element = value[i];
slice = span.Slice(offset, elementSize);
_scalarSerializer.WriteTo(ref element, ref slice);
offset += elementSize;
}
}
protected override IList<T> CreateCollection(int capacity) => new List<T>(capacity);

public void WriteTo(ref IList<T> value, IBufferWriter<byte> buffer)
{
throw new NotImplementedException();
}

public IList<T> Deserialize(ReadOnlySpan<byte> buffer)
{
// Read the size of the list
var slice = buffer.Slice(0, sizeof(int));
var size = BitConverter.ToInt32(slice);

var list = new List<T>(size);

// Read the elements of the list
int offset = sizeof(int);
var elementSize = (buffer.Length - offset) / size;
for (int i = 0; i < size; i++)
{
slice = buffer.Slice(offset, elementSize);
var element = _scalarSerializer.Deserialize(slice);
list.Add(element);
offset += elementSize;
}

return list;
}
protected override void AddElement(IList<T> collection, T element) => collection.Add(element);
}
12 changes: 12 additions & 0 deletions src/RocksDb.Extensions/FixedSizeSetSerializer.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
namespace RocksDb.Extensions;

internal class FixedSizeSetSerializer<T> : FixedSizeCollectionSerializer<ISet<T>, T>
{
public FixedSizeSetSerializer(ISerializer<T> scalarSerializer) : base(scalarSerializer)
{
}

protected override ISet<T> CreateCollection(int capacity) => new HashSet<T>(capacity);

protected override void AddElement(ISet<T> collection, T element) => collection.Add(element);
}
20 changes: 20 additions & 0 deletions src/RocksDb.Extensions/RocksDbBuilder.cs
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,26 @@ private static ISerializer<T> CreateSerializer<T>(IReadOnlyList<ISerializerFacto
return (ISerializer<T>) Activator.CreateInstance(typeof(VariableSizeListSerializer<>).MakeGenericType(elementType), scalarSerializer)!;
}

if (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(ISet<>))
{
var elementType = type.GetGenericArguments()[0];

// Use reflection to call CreateSerializer method with generic type argument
// This is equivalent to calling CreateSerializer<elementType>(serializerFactories)
var scalarSerializer = typeof(RocksDbBuilder).GetMethod(nameof(CreateSerializer), BindingFlags.NonPublic | BindingFlags.Static)
?.MakeGenericMethod(elementType)
.Invoke(null, new object[] { serializerFactories });

if (elementType.IsPrimitive)
{
// Use fixed size set serializer for primitive types
return (ISerializer<T>) Activator.CreateInstance(typeof(FixedSizeSetSerializer<>).MakeGenericType(elementType), scalarSerializer)!;
}

// Use variable size set serializer for non-primitive types
return (ISerializer<T>) Activator.CreateInstance(typeof(VariableSizeSetSerializer<>).MakeGenericType(elementType), scalarSerializer)!;
}

// Handle CollectionOperation<T> for the ListMergeOperator
if (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(MergeOperators.CollectionOperation<>))
{
Expand Down
119 changes: 119 additions & 0 deletions src/RocksDb.Extensions/VariableSizeCollectionSerializer.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
using System.Buffers;

namespace RocksDb.Extensions;

/// <summary>
/// Base class for serializing collections containing variable-size elements like strings or complex objects
/// where each element may occupy a different number of bytes when serialized.
/// </summary>
/// <remarks>
/// The serialized format consists of:
/// - 4 bytes: Collection length (number of elements)
/// - For each element:
/// - 4 bytes: Size of the serialized element
/// - N bytes: Serialized element data
/// </remarks>
/// <typeparam name="TCollection">The collection type (e.g., IList{T}, ISet{T})</typeparam>
/// <typeparam name="TElement">The element type</typeparam>
internal abstract class VariableSizeCollectionSerializer<TCollection, TElement> : ISerializer<TCollection>
where TCollection : ICollection<TElement>
{
private readonly ISerializer<TElement> _scalarSerializer;

protected VariableSizeCollectionSerializer(ISerializer<TElement> scalarSerializer)
{
_scalarSerializer = scalarSerializer;
}

/// <summary>
/// Creates a new collection instance with the specified capacity.
/// </summary>
protected abstract TCollection CreateCollection(int capacity);

/// <summary>
/// Adds an element to the collection.
/// </summary>
protected abstract void AddElement(TCollection collection, TElement element);

public bool TryCalculateSize(ref TCollection value, out int size)
{
size = sizeof(int); // size of the collection
if (value.Count == 0)
{
return true;
}

foreach (var item in value)
{
var element = item;
if (_scalarSerializer.TryCalculateSize(ref element, out var elementSize))
{
size += sizeof(int);
size += elementSize;
}
else
{
// Element serializer can't calculate size, so we can't either
size = 0;
return false;
}
}

return true;
}

public void WriteTo(ref TCollection value, ref Span<byte> span)
{
// Write the size of the collection
var slice = span.Slice(0, sizeof(int));
BitConverter.TryWriteBytes(slice, value.Count);

// Write the elements of the collection
int offset = sizeof(int);
foreach (var item in value)
{
var element = item;
if (_scalarSerializer.TryCalculateSize(ref element, out var elementSize))
{
slice = span.Slice(offset, sizeof(int));
BitConverter.TryWriteBytes(slice, elementSize);
offset += sizeof(int);

slice = span.Slice(offset, elementSize);
_scalarSerializer.WriteTo(ref element, ref slice);
offset += elementSize;
}
}
}

public void WriteTo(ref TCollection value, IBufferWriter<byte> buffer)
{
throw new NotImplementedException();
}

public TCollection Deserialize(ReadOnlySpan<byte> buffer)
{
// Read the size of the collection
var slice = buffer.Slice(0, sizeof(int));
var size = BitConverter.ToInt32(slice);

var collection = CreateCollection(size);

// Read the elements of the collection
int offset = sizeof(int);
for (int i = 0; i < size; i++)
{
slice = buffer.Slice(offset, sizeof(int));
var elementSize = BitConverter.ToInt32(slice);
offset += sizeof(int);

slice = buffer.Slice(offset, elementSize);
var element = _scalarSerializer.Deserialize(slice);
AddElement(collection, element);
offset += elementSize;
}

return collection;
}
}

Loading