Add serialization for primitive lists and protobuf lists

README.md

@@ -62,7 +62,9 @@ var rocksDbBuilder = builder.Services.AddRocksDb(options =>
     options.SerializerFactories.Add(new SystemTextJsonSerializerFactory());
 });
 ```
+
 ### Register your store
+
 Before your store can be used, you need to register it with RocksDb. You can do this as follows:
 
 ```csharp
@@ -72,6 +74,7 @@ rocksDbBuilder.AddStore<string, User, UsersStore>("users-store");
 This registers an instance of `UsersStore` with RocksDb under the name "users-store".
 
 ### Use your store
+
 Once you have registered your store, you can use it to add, get, and remove data from RocksDb. For example:
 
 ```csharp
@@ -126,4 +129,40 @@ var rocksDbBuilder = builder.Services.AddRocksDb(options =>
 
 When this option is set to true, the existing database will be deleted on startup and a new one will be created. Note that all data in the existing database will be lost when this option is used.
 
-By default, the `DeleteExistingDatabaseOnStartup` option is set to false to preserve the current behavior of not automatically deleting the database. If you need to ensure a clean start for your application, set this option to true in your configuration.
+By default, the `DeleteExistingDatabaseOnStartup` option is set to false to preserve the current behavior of not automatically deleting the database. If you need to ensure a clean start for your application, set this option to true in your configuration.
+
+## Collections Support
+
+RocksDb.Extensions provides built-in support for collections across different serialization packages:
+
+### System.Text.Json and ProtoBufNet
+
+The `RocksDb.Extensions.System.Text.Json` and `RocksDb.Extensions.ProtoBufNet` packages support collections out of the box. You can use any collection type like `List<T>` or arrays without additional configuration.
+
+### Protocol Buffers and Primitive Types Support
+
+The library includes specialized support for collections when working with:
+
+1. Protocol Buffer message types
+2. Primitive types (int, long, string, etc.)
+
+When using `IList<T>` with these types, the library automatically handles serialization/deserialization without requiring wrapper message types. This is particularly useful for Protocol Buffers, where `RepeatedField<T>` typically cannot be serialized as a standalone entity.
+
+The serialization format varies depending on the element type:
+
+#### Fixed-Size Types (int, long, etc.)
+
+```
+[4 bytes: List length][Contiguous array of serialized elements]
+```
+
+#### Variable-Size Types (string, protobuf messages)
+
+```
+[4 bytes: List length][For each element: [4 bytes: Element size][N bytes: Element data]]
+```
+
+Example types that work automatically with this support:
+
+- Protocol Buffer message types: `IList<YourProtobufMessage>`
+- Primitive types: `IList<int>`, `IList<long>`, `IList<string>`, etc.

src/RocksDb.Extensions/FixedSizeListSerializer.cs

@@ -0,0 +1,87 @@
+using System.Buffers;
+
+namespace RocksDb.Extensions;
+
+/// <summary>
+/// Serializes lists of fixed-size elements like primitive types (int, long, etc.) where each element
+/// occupies the same number of bytes in memory. This implementation optimizes for performance by
+/// pre-calculating buffer sizes based on element count.
+/// </summary>
+/// <remarks>
+/// Use this serializer when working with lists of primitive types or structs where all elements
+/// have identical size. The serialized format consists of:
+/// - 4 bytes: List length (number of elements)
+/// - Remaining bytes: Contiguous array of serialized elements
+/// </remarks>
+internal class FixedSizeListSerializer<T> : ISerializer<IList<T>>
+{
+    private readonly ISerializer<T> _scalarSerializer;
+
+    public FixedSizeListSerializer(ISerializer<T> scalarSerializer)
+    {
+        _scalarSerializer = scalarSerializer;
+    }
+
+    public bool TryCalculateSize(ref IList<T> value, out int size)
+    {
+        size = sizeof(int); // size of the list
+        if (value.Count == 0)
+        {
+            return true;
+        }
+
+        var referentialElement = value[0];
+        if (_scalarSerializer.TryCalculateSize(ref referentialElement, out var elementSize))
+        {
+            size += value.Count * elementSize;
+            return true;
+        }
+
+        return false;
+    }
+
+    public void WriteTo(ref IList<T> value, ref Span<byte> span)
+    {
+        // Write the size of the list
+        var slice = span.Slice(0, sizeof(int));
+        BitConverter.TryWriteBytes(slice, value.Count);
+
+        // Write the elements of the list
+        int offset = sizeof(int);
+        var elementSize = (span.Length - offset) / value.Count;
+        for (int i = 0; i < value.Count; i++)
+        {
+            var element = value[i];
+            slice = span.Slice(offset, elementSize);
+            _scalarSerializer.WriteTo(ref element, ref slice);
+            offset += elementSize;
+        }
+    }
+
+    public void WriteTo(ref IList<T> value, IBufferWriter<byte> buffer)
+    {
+        throw new NotImplementedException();
+    }
+
+    public IList<T> Deserialize(ReadOnlySpan<byte> buffer)
+    {
+        // Read the size of the list
+        var slice = buffer.Slice(0, sizeof(int));
+        var size = BitConverter.ToInt32(slice);
+
+        var list = new List<T>(size);
+
+        // Read the elements of the list
+        int offset = sizeof(int);
+        var elementSize = (buffer.Length - offset) / size;
+        for (int i = 0; i < size; i++)
+        {
+            slice = buffer.Slice(offset, elementSize);
+            var element = _scalarSerializer.Deserialize(slice);
+            list.Add(element);
+            offset += elementSize;
+        }
+
+        return list;
+    }
+}

src/RocksDb.Extensions/RocksDbBuilder.cs

@@ -1,3 +1,4 @@
+using System.Reflection;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Options;
 
@@ -51,6 +52,26 @@ private static ISerializer<T> CreateSerializer<T>(IReadOnlyList<ISerializerFacto
                 return serializerFactory.CreateSerializer<T>();
             }
         }
+
+        if (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(IList<>))
+        {
+            var elementType = type.GetGenericArguments()[0];
+
+            // Use reflection to call CreateSerializer method with generic type argument
+            // This is equivalent to calling CreateSerializer<elementType>(serializerFactories)
+            var scalarSerializer = typeof(RocksDbBuilder).GetMethod(nameof(CreateSerializer), BindingFlags.NonPublic | BindingFlags.Static)
+                ?.MakeGenericMethod(elementType)
+                .Invoke(null, new object[] { serializerFactories });
+
+            if (elementType.IsPrimitive)
+            {
+                // Use fixed size list serializer for primitive types
+                return (ISerializer<T>) Activator.CreateInstance(typeof(FixedSizeListSerializer<>).MakeGenericType(elementType), scalarSerializer);
+            }
+
+            // Use variable size list serializer for non-primitive types
+            return (ISerializer<T>) Activator.CreateInstance(typeof(VariableSizeListSerializer<>).MakeGenericType(elementType), scalarSerializer);
+        }
 
         throw new InvalidOperationException($"Type {type.FullName} cannot be used as RocksDbStore key/value. " +
                                             $"Consider registering {nameof(ISerializerFactory)} that support this type.");

src/RocksDb.Extensions/VariableSizeListSerializer.cs

@@ -0,0 +1,100 @@
+using System.Buffers;
+
+namespace RocksDb.Extensions;
+
+/// <summary>
+/// Serializes lists containing variable-size elements like strings or complex objects where each element
+/// may occupy a different number of bytes when serialized.
+/// </summary>
+/// <remarks>
+/// Use this serializer for lists containing elements that may have different sizes (strings, nested objects, etc.).
+/// The serialized format consists of:
+/// - 4 bytes: List length (number of elements)
+/// - For each element:
+///   - 4 bytes: Size of the serialized element
+///   - N bytes: Serialized element data
+/// </remarks>
+internal class VariableSizeListSerializer<T> : ISerializer<IList<T>>
+{
+    private readonly ISerializer<T> _scalarSerializer;
+
+    public VariableSizeListSerializer(ISerializer<T> scalarSerializer)
+    {
+        _scalarSerializer = scalarSerializer;
+    }
+
+    public bool TryCalculateSize(ref IList<T> value, out int size)
+    {
+        size = sizeof(int); // size of the list
+        if (value.Count == 0)
+        {
+            return true;
+        }
+
+        for (int i = 0; i < value.Count; i++)
+        {
+            var element = value[i];
+            if (_scalarSerializer.TryCalculateSize(ref element, out var elementSize))
+            {
+                size += sizeof(int);
+                size += elementSize;
+            }
+        }
+
+        return true;
+    }
+
+    public void WriteTo(ref IList<T> value, ref Span<byte> span)
+    {
+        // Write the size of the list
+        var slice = span.Slice(0, sizeof(int));
+        BitConverter.TryWriteBytes(slice, value.Count);
+
+        // Write the elements of the list
+        int offset = sizeof(int);
+        for (int i = 0; i < value.Count; i++)
+        {
+            var element = value[i];
+            if (_scalarSerializer.TryCalculateSize(ref element, out var elementSize))
+            {
+                slice = span.Slice(offset, sizeof(int));
+                BitConverter.TryWriteBytes(slice, elementSize);
+                offset += sizeof(int);
+
+                slice = span.Slice(offset, elementSize);
+                _scalarSerializer.WriteTo(ref element, ref slice);
+                offset += elementSize;
+            }
+        }
+    }
+
+    public void WriteTo(ref IList<T> value, IBufferWriter<byte> buffer)
+    {
+        throw new NotImplementedException();
+    }
+
+    public IList<T> Deserialize(ReadOnlySpan<byte> buffer)
+    {
+        // Read the size of the list
+        var slice = buffer.Slice(0, sizeof(int));
+        var size = BitConverter.ToInt32(slice);
+
+        var list = new List<T>(size);
+
+        // Read the elements of the list
+        int offset = sizeof(int);
+        for (int i = 0; i < size; i++)
+        {
+            slice = buffer.Slice(offset, sizeof(int));
+            var elementSize = BitConverter.ToInt32(slice);
+            offset += sizeof(int);
+
+            slice = buffer.Slice(offset, elementSize);
+            var element = _scalarSerializer.Deserialize(slice);
+            list.Add(element);
+            offset += elementSize;
+        }
+
+        return list;
+    }
+}

test/RocksDb.Extensions.Tests/RocksDbStoreWithJsonSerializerTests.cs

@@ -12,7 +12,7 @@ public class RocksDbStoreWithJsonSerializerTests
     public void should_put_and_retrieve_data_from_store()
     {
         // Arrange
-        using var testFixture = CreateTestFixture();
+        using var testFixture = CreateTestFixture<ProtoNetCacheKey, ProtoNetCacheValue>();
 
         var store = testFixture.GetStore<RocksDbGenericStore<ProtoNetCacheKey, ProtoNetCacheValue>>();
         var cacheKey = new ProtoNetCacheKey
@@ -38,7 +38,7 @@ public void should_put_and_retrieve_data_from_store()
     public void should_put_and_remove_data_from_store()
     {
         // Arrange
-        using var testFixture = CreateTestFixture();
+        using var testFixture = CreateTestFixture<ProtoNetCacheKey, ProtoNetCacheValue>();
 
         var store = testFixture.GetStore<RocksDbGenericStore<ProtoNetCacheKey, ProtoNetCacheValue>>();
         var cacheKey = new ProtoNetCacheKey
@@ -64,7 +64,7 @@ public void should_put_and_remove_data_from_store()
     public void should_put_range_of_data_to_store()
     {
         // Arrange
-        using var testFixture = CreateTestFixture();
+        using var testFixture = CreateTestFixture<ProtoNetCacheKey, ProtoNetCacheValue>();
         var store = testFixture.GetStore<RocksDbGenericStore<ProtoNetCacheKey, ProtoNetCacheValue>>();
 
         // Act
@@ -90,7 +90,7 @@ public void should_put_range_of_data_to_store()
     public void should_put_range_of_data_to_store_when_key_is_derived_from_value()
     {
         // Arrange
-        using var testFixture = CreateTestFixture();
+        using var testFixture = CreateTestFixture<ProtoNetCacheKey, ProtoNetCacheValue>();
         var store = testFixture.GetStore<RocksDbGenericStore<ProtoNetCacheKey, ProtoNetCacheValue>>();
 
         // Act
@@ -108,17 +108,47 @@ public void should_put_range_of_data_to_store_when_key_is_derived_from_value()
             cacheValue.ShouldBeEquivalentTo(expectedCacheValue);
         }
     }
+
+    [Test]
+    public void should_put_and_retrieve_data_with_lists_from_store()
+    {
+        // Arrange
+        using var testFixture = CreateTestFixture<IList<ProtoNetCacheKey>, IList<ProtoNetCacheValue>>();
+        var store = testFixture.GetStore<RocksDbGenericStore<IList<ProtoNetCacheKey>, IList<ProtoNetCacheValue>>>();
+
+        // Act
+        var cacheKey = Enumerable.Range(0, 100)
+            .Select(x => new ProtoNetCacheKey 
+            { 
+                Id = x,
+            })
+            .ToList();
+
+        var cacheValue = Enumerable.Range(0, 100)
+            .Select(x => new ProtoNetCacheValue 
+            { 
+                Id = x, 
+                Value = $"value-{x}",
+            })
+            .ToList();
+
+        store.Put(cacheKey, cacheValue);
 
-    private static TestFixture CreateTestFixture()
+        store.HasKey(cacheKey).ShouldBeTrue();
+        store.TryGet(cacheKey, out var value).ShouldBeTrue();
+        value.ShouldBeEquivalentTo(cacheValue);
+    }
+
+    private static TestFixture CreateTestFixture<TKey, TValue>()
     {
         var testFixture = TestFixture.Create(rockDb =>
         {
-            _ = rockDb.AddStore<ProtoNetCacheKey, ProtoNetCacheValue, RocksDbGenericStore<ProtoNetCacheKey, ProtoNetCacheValue>>("my-store");
+            _ = rockDb.AddStore<TKey, TValue, RocksDbGenericStore<TKey, TValue>>("my-store");
         }, options =>
         {
             options.SerializerFactories.Clear();
             options.SerializerFactories.Add(new SystemTextJsonSerializerFactory());
         });
         return testFixture;
-    }
+    }    
 }