-
Notifications
You must be signed in to change notification settings - Fork 0
Add support for Merge operator #42
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Closed
Closed
Changes from all commits
Commits
Show all changes
19 commits
Select commit
Hold shift + click to select a range
6f77e2b
WIP
Havret 81f3746
wip
Havret 8563757
wip
Havret 0ba1404
WIP
Havret 36a7381
wip
Havret 1143ec4
wip
Havret da93637
wip
Havret 95dbd3a
wip
Havret 3b1398f
wip
Havret c985414
wip
Havret 63b97fc
wip
Havret bd32b5a
wip
Havret d2670f5
wip
Havret ee73d9f
wip
Havret ae6036c
wip
Havret 4cc4e27
wip
Havret 0d6565c
wip
Havret 47f24e1
wip
Havret 00fb780
wip
Havret File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| using System.ComponentModel; | ||
|
|
||
| namespace RocksDb.Extensions; | ||
|
|
||
| #pragma warning disable CS1591 | ||
|
|
||
| /// <summary> | ||
| /// This interface is not intended to be used directly by the clients of the library. | ||
| /// It provides merge operation support with a separate operand type. | ||
| /// </summary> | ||
| [EditorBrowsable(EditorBrowsableState.Never)] | ||
| public interface IMergeAccessor<TKey, TValue, in TOperand> : IRocksDbAccessor<TKey, TValue> | ||
| { | ||
| void Merge(TKey key, TOperand operand); | ||
| } | ||
|
|
||
| #pragma warning restore CS1591 |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,47 @@ | ||
| namespace RocksDb.Extensions; | ||
|
|
||
| /// <summary> | ||
| /// Defines a merge operator for RocksDB that enables atomic read-modify-write operations. | ||
| /// Merge operators allow efficient updates without requiring a separate read before write, | ||
| /// which is particularly useful for counters, list appends, set unions, and other accumulative operations. | ||
| /// </summary> | ||
| /// <typeparam name="TValue">The type of the value stored in the database.</typeparam> | ||
| /// <typeparam name="TOperand">The type of the merge operand (the delta/change to apply).</typeparam> | ||
| /// <remarks> | ||
| /// The separation of <typeparamref name="TValue"/> and <typeparamref name="TOperand"/> allows for flexible merge patterns: | ||
| /// <list type="bullet"> | ||
| /// <item><description>For counters: TValue=long, TOperand=long (same type)</description></item> | ||
| /// <item><description>For list append: TValue=IList<T>, TOperand=IList<T> (same type)</description></item> | ||
| /// <item><description>For list with add/remove: TValue=IList<T>, TOperand=CollectionOperation<T> (different types)</description></item> | ||
| /// </list> | ||
| /// </remarks> | ||
| public interface IMergeOperator<TValue, TOperand> | ||
| { | ||
| /// <summary> | ||
| /// Gets the name of the merge operator. This name is stored in the database | ||
| /// and must remain consistent across database opens. | ||
| /// </summary> | ||
| string Name { get; } | ||
|
|
||
| /// <summary> | ||
| /// Performs a full merge of the existing value with one or more operands. | ||
| /// Called when a Get operation encounters merge operands and needs to produce the final value. | ||
| /// </summary> | ||
| /// <param name="existingValue">The existing value in the database. For value types, this will be default if no value exists.</param> | ||
| /// <param name="operands">The span of merge operands to apply, in order.</param> | ||
| /// <returns>The merged value to store.</returns> | ||
| TValue FullMerge(TValue? existingValue, ReadOnlySpan<TOperand> operands); | ||
|
|
||
| /// <summary> | ||
| /// Performs a partial merge of multiple operands without the existing value. | ||
| /// Called during compaction to combine multiple merge operands into a single operand. | ||
| /// This is an optimization that reduces the number of operands that need to be stored. | ||
| /// </summary> | ||
| /// <param name="operands">The span of merge operands to combine, in order.</param> | ||
| /// <returns>The combined operand, or null if partial merge is not safe for these operands.</returns> | ||
| /// <remarks> | ||
| /// Return null when it's not safe to combine operands without knowing the existing value. | ||
| /// When null is returned, RocksDB will keep the operands separate and call FullMerge later. | ||
| /// </remarks> | ||
| TOperand? PartialMerge(ReadOnlySpan<TOperand> operands); | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,84 @@ | ||
| using System.Buffers; | ||
| using RocksDb.Extensions.MergeOperators; | ||
|
|
||
| namespace RocksDb.Extensions; | ||
|
|
||
| /// <summary> | ||
| /// Serializes CollectionOperation<T> which contains an operation type (Add/Remove) and a list of items. | ||
| /// </summary> | ||
| /// <remarks> | ||
| /// <para> | ||
| /// The serialized format consists of: | ||
| /// - 1 byte: Operation type (0 = Add, 1 = Remove) | ||
| /// - Remaining bytes: Serialized list using FixedSizeListSerializer (for primitives) or VariableSizeListSerializer (for complex types) | ||
| /// </para> | ||
| /// <para> | ||
| /// Space efficiency optimization: | ||
| /// - For primitive types (int, long, bool, etc.), uses FixedSizeListSerializer which stores: | ||
| /// - 4 bytes: list count | ||
| /// - N * elementSize bytes: all elements (no per-element size prefix) | ||
| /// Example: List<int> with 3 elements = 4 + (3 * 4) = 16 bytes | ||
| /// </para> | ||
| /// <para> | ||
| /// - For non-primitive types (strings, objects, protobuf messages), uses VariableSizeListSerializer which stores: | ||
| /// - 4 bytes: list count | ||
| /// - For each element: 4 bytes size prefix + element data | ||
| /// Example: List<string> with ["ab", "cde"] = 4 + (4+2) + (4+3) = 17 bytes | ||
| /// </para> | ||
| /// </remarks> | ||
| internal class ListOperationSerializer<T> : ISerializer<CollectionOperation<T>> | ||
| { | ||
| private readonly ISerializer<IList<T>> _listSerializer; | ||
|
|
||
| public ListOperationSerializer(ISerializer<T> itemSerializer) | ||
| { | ||
| // Use FixedSizeListSerializer for primitive types to avoid storing size for each element | ||
| // Use VariableSizeListSerializer for non-primitive types where elements may vary in size | ||
| _listSerializer = typeof(T).IsPrimitive | ||
| ? new FixedSizeListSerializer<T>(itemSerializer) | ||
| : new VariableSizeListSerializer<T>(itemSerializer); | ||
| } | ||
|
|
||
| public bool TryCalculateSize(ref CollectionOperation<T> value, out int size) | ||
| { | ||
| // 1 byte for operation type + size of the list | ||
| size = sizeof(byte); | ||
|
|
||
| var items = value.Items; | ||
| if (_listSerializer.TryCalculateSize(ref items, out var listSize)) | ||
| { | ||
| size += listSize; | ||
| return true; | ||
| } | ||
|
|
||
| return false; | ||
| } | ||
|
|
||
| public void WriteTo(ref CollectionOperation<T> value, ref Span<byte> span) | ||
| { | ||
| // Write operation type (1 byte) | ||
| span[0] = (byte)value.Type; | ||
|
|
||
| // Write the list using the list serializer | ||
| var listSpan = span.Slice(sizeof(byte)); | ||
| var items = value.Items; | ||
| _listSerializer.WriteTo(ref items, ref listSpan); | ||
| } | ||
|
|
||
| public void WriteTo(ref CollectionOperation<T> value, IBufferWriter<byte> buffer) | ||
| { | ||
| throw new NotImplementedException(); | ||
| } | ||
|
|
||
| public CollectionOperation<T> Deserialize(ReadOnlySpan<byte> buffer) | ||
| { | ||
| // Read operation type | ||
| var operationType = (OperationType)buffer[0]; | ||
|
|
||
| // Read the list using the list serializer | ||
| var listBuffer = buffer.Slice(sizeof(byte)); | ||
| var items = _listSerializer.Deserialize(listBuffer); | ||
|
|
||
| return new CollectionOperation<T>(operationType, items); | ||
| } | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,88 @@ | ||
| using System.Buffers; | ||
| using CommunityToolkit.HighPerformance.Buffers; | ||
|
|
||
| namespace RocksDb.Extensions; | ||
|
|
||
| internal class MergeAccessor<TKey, TValue, TOperand> : RocksDbAccessor<TKey, TValue>, IMergeAccessor<TKey, TValue, TOperand> | ||
| { | ||
| private readonly ISerializer<TOperand> _operandSerializer; | ||
|
|
||
| public MergeAccessor( | ||
| RocksDbContext db, | ||
| ColumnFamily columnFamily, | ||
| ISerializer<TKey> keySerializer, | ||
| ISerializer<TValue> valueSerializer, | ||
| ISerializer<TOperand> operandSerializer) : base(db, columnFamily, keySerializer, valueSerializer) | ||
| { | ||
| _operandSerializer = operandSerializer; | ||
| } | ||
|
|
||
| public void Merge(TKey key, TOperand operand) | ||
| { | ||
| byte[]? rentedKeyBuffer = null; | ||
| bool useSpanAsKey; | ||
| // ReSharper disable once AssignmentInConditionalExpression | ||
| Span<byte> keyBuffer = (useSpanAsKey = KeySerializer.TryCalculateSize(ref key, out var keySize)) | ||
| ? keySize < MaxStackSize | ||
| ? stackalloc byte[keySize] | ||
| : (rentedKeyBuffer = ArrayPool<byte>.Shared.Rent(keySize)).AsSpan(0, keySize) | ||
| : Span<byte>.Empty; | ||
|
|
||
| ReadOnlySpan<byte> keySpan = keyBuffer; | ||
| ArrayPoolBufferWriter<byte>? keyBufferWriter = null; | ||
Havret marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
|
||
| byte[]? rentedOperandBuffer = null; | ||
| bool useSpanAsOperand; | ||
| // ReSharper disable once AssignmentInConditionalExpression | ||
| Span<byte> operandBuffer = (useSpanAsOperand = _operandSerializer.TryCalculateSize(ref operand, out var operandSize)) | ||
| ? operandSize < MaxStackSize | ||
| ? stackalloc byte[operandSize] | ||
| : (rentedOperandBuffer = ArrayPool<byte>.Shared.Rent(operandSize)).AsSpan(0, operandSize) | ||
| : Span<byte>.Empty; | ||
|
|
||
|
|
||
| ReadOnlySpan<byte> operandSpan = operandBuffer; | ||
| ArrayPoolBufferWriter<byte>? operandBufferWriter = null; | ||
Havret marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
|
||
| try | ||
| { | ||
| if (useSpanAsKey) | ||
| { | ||
| KeySerializer.WriteTo(ref key, ref keyBuffer); | ||
| } | ||
| else | ||
| { | ||
| keyBufferWriter = new ArrayPoolBufferWriter<byte>(); | ||
| KeySerializer.WriteTo(ref key, keyBufferWriter); | ||
| keySpan = keyBufferWriter.WrittenSpan; | ||
| } | ||
|
|
||
| if (useSpanAsOperand) | ||
| { | ||
| _operandSerializer.WriteTo(ref operand, ref operandBuffer); | ||
| } | ||
| else | ||
| { | ||
| operandBufferWriter = new ArrayPoolBufferWriter<byte>(); | ||
| _operandSerializer.WriteTo(ref operand, operandBufferWriter); | ||
| operandSpan = operandBufferWriter.WrittenSpan; | ||
| } | ||
|
|
||
| RocksDbContext.Db.Merge(keySpan, operandSpan, ColumnFamily.Handle, RocksDbContext.WriteOptions); | ||
| } | ||
| finally | ||
| { | ||
| keyBufferWriter?.Dispose(); | ||
| operandBufferWriter?.Dispose(); | ||
| if (rentedKeyBuffer is not null) | ||
| { | ||
| ArrayPool<byte>.Shared.Return(rentedKeyBuffer); | ||
| } | ||
|
|
||
| if (rentedOperandBuffer is not null) | ||
| { | ||
| ArrayPool<byte>.Shared.Return(rentedOperandBuffer); | ||
| } | ||
| } | ||
| } | ||
| } | ||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The
WriteTo(ref ListOperation<T> value, IBufferWriter<byte> buffer)method throwsNotImplementedException, but this is a required method for the serializer interface. This will cause runtime failures when the buffer writer path is used instead of the span-based path. This method should be implemented to support variable-sized serialization.