Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

AVX2_decode UTF8 #30

Merged
merged 12 commits into from
Sep 5, 2024
Merged

AVX2_decode UTF8 #30

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
c3a470c
careful checking
Nick-Nuon Aug 20, 2024
ee26813
AVX2 port first draft
Nick-Nuon Aug 21, 2024
dc07f32
Added UTF8 tests: Issue511 failing
Nick-Nuon Aug 21, 2024
06ea437
Issue 511 fixed
Nick-Nuon Aug 21, 2024
d3a9457
DoomedPartialBufferAvx2UTF8 test pass
Nick-Nuon Aug 22, 2024
8485b4e
partial cleanup
Nick-Nuon Aug 25, 2024
7e61d59
Merging with main
Nick-Nuon Aug 27, 2024
5698ae8
Every tests pass
Nick-Nuon Aug 27, 2024
a4eae24
Cleanup
Nick-Nuon Aug 27, 2024
a9b933d
Added UTF16 support + tests + benchmarks
Nick-Nuon Aug 28, 2024
bf3e521
cleanup
Nick-Nuon Aug 28, 2024
a408bd8
Merge branch 'main' into AVX2_decode
Nick-Nuon Aug 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
109 changes: 109 additions & 0 deletions benchmark/Benchmark.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -344,6 +344,83 @@ public unsafe void RunSSEDecodingBenchmarkWithAllocUTF16(string[] data, int[] le
}
}

public unsafe void RunAVX2DecodingBenchmarkUTF8(string[] data, int[] lengths)
{
for (int i = 0; i < FileContent.Length; i++)
{
//string s = FileContent[i];
byte[] base64 = input[i];
byte[] dataoutput = output[i];
int bytesConsumed = 0;
int bytesWritten = 0;
SimdBase64.AVX2.Base64.DecodeFromBase64AVX2(base64.AsSpan(), dataoutput, out bytesConsumed, out bytesWritten, false);
if (bytesWritten != lengths[i])
{
Console.WriteLine($"Error: {bytesWritten} != {lengths[i]}");
#pragma warning disable CA2201
throw new Exception("Error");
}
}
}

public unsafe void RunAVX2DecodingBenchmarkUTF16(string[] data, int[] lengths)
{
for (int i = 0; i < FileContent.Length; i++)
{
string s = FileContent[i];
ReadOnlySpan<char> base64 = s.AsSpan();
byte[] dataoutput = output[i];
int bytesConsumed = 0;
int bytesWritten = 0;
SimdBase64.AVX2.Base64.DecodeFromBase64AVX2(base64, dataoutput, out bytesConsumed, out bytesWritten, false);
if (bytesWritten != lengths[i])
{
Console.WriteLine($"Error: {bytesWritten} != {lengths[i]}");
#pragma warning disable CA2201
throw new Exception("Error");
}
}
}



public unsafe void RunAVX2DecodingBenchmarkWithAllocUTF8(string[] data, int[] lengths)
{
for (int i = 0; i < FileContent.Length; i++)
{
byte[] base64 = input[i];
byte[] dataoutput = new byte[SimdBase64.Scalar.Base64.MaximalBinaryLengthFromBase64Scalar<byte>(base64.AsSpan())];
int bytesConsumed = 0;
int bytesWritten = 0;
SimdBase64.AVX2.Base64.DecodeFromBase64AVX2(base64.AsSpan(), dataoutput, out bytesConsumed, out bytesWritten, false);
if (bytesWritten != lengths[i])
{
Console.WriteLine($"Error: {bytesWritten} != {lengths[i]}");
#pragma warning disable CA2201
throw new Exception("Error");
}
}
}

public unsafe void RunAVX2DecodingBenchmarkWithAllocUTF16(string[] data, int[] lengths)
{
for (int i = 0; i < FileContent.Length; i++)
{
string s = FileContent[i];
char[] base64 = input16[i];
byte[] dataoutput = new byte[SimdBase64.Scalar.Base64.MaximalBinaryLengthFromBase64Scalar<char>(base64.AsSpan())];
int bytesConsumed = 0;
int bytesWritten = 0;
SimdBase64.AVX2.Base64.DecodeFromBase64AVX2(base64.AsSpan(), dataoutput, out bytesConsumed, out bytesWritten, false);
if (bytesWritten != lengths[i])
{
Console.WriteLine($"Error: {bytesWritten} != {lengths[i]}");
#pragma warning disable CA2201
throw new Exception("Error");
}
}
}


public unsafe void RunARMDecodingBenchmarkUTF8(string[] data, int[] lengths)
{
Expand Down Expand Up @@ -502,6 +579,23 @@ public unsafe void SSEDecodingRealDataWithAllocUTF8()
RunSSEDecodingBenchmarkWithAllocUTF8(FileContent, DecodedLengths);
}

[Benchmark]
[BenchmarkCategory("AVX")]
public unsafe void AVX2DecodingRealDataUTF8()
{
RunAVX2DecodingBenchmarkUTF8(FileContent, DecodedLengths);
}

[Benchmark]
[BenchmarkCategory("AVX")]
public unsafe void AVX2DecodingRealDataWithAllocUTF8()
{
RunAVX2DecodingBenchmarkWithAllocUTF8(FileContent, DecodedLengths);
}




[Benchmark]
[BenchmarkCategory("arm64")]
public unsafe void ARMDecodingRealDataUTF8()
Expand Down Expand Up @@ -536,6 +630,21 @@ public unsafe void SSEDecodingRealDataWithAllocUTF16()
{
RunSSEDecodingBenchmarkWithAllocUTF16(FileContent, DecodedLengths);
}

[Benchmark]
[BenchmarkCategory("AVX")]
public unsafe void AVX2DecodingRealDataUTF16()
{
RunAVX2DecodingBenchmarkUTF16(FileContent, DecodedLengths);
}

[Benchmark]
[BenchmarkCategory("AVX")]
public unsafe void AVX2DecodingRealDataWithAllocUTF16()
{
RunAVX2DecodingBenchmarkWithAllocUTF16(FileContent, DecodedLengths);
}

}
#pragma warning disable CA1515
public class Program
Expand Down
12 changes: 7 additions & 5 deletions src/Base64.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,20 +1,21 @@
using System;
using System.Buffers;
using System.Runtime.CompilerServices;
using System.Runtime.Intrinsics;
using System.Runtime.Intrinsics.Arm;
using System.Runtime.Intrinsics.X86;


namespace SimdBase64
{
public static class Base64 {
public static class Base64
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static int MaximalBinaryLengthFromBase64<T>(ReadOnlySpan<T> input)
{
return Scalar.Base64.MaximalBinaryLengthFromBase64Scalar(input);
}
public unsafe static OperationStatus DecodeFromBase64(ReadOnlySpan<byte> source, Span<byte> dest, out int bytesConsumed, out int bytesWritten, bool isUrl = false) {
public unsafe static OperationStatus DecodeFromBase64(ReadOnlySpan<byte> source, Span<byte> dest, out int bytesConsumed, out int bytesWritten, bool isUrl = false)
{

if (AdvSimd.Arm64.IsSupported && BitConverter.IsLittleEndian)
{
Expand All @@ -38,12 +39,13 @@ public unsafe static OperationStatus DecodeFromBase64(ReadOnlySpan<byte> source,

}

public unsafe static OperationStatus DecodeFromBase64(ReadOnlySpan<char> source, Span<byte> dest, out int bytesConsumed, out int bytesWritten, bool isUrl = false) {
public unsafe static OperationStatus DecodeFromBase64(ReadOnlySpan<char> source, Span<byte> dest, out int bytesConsumed, out int bytesWritten, bool isUrl = false)
{

if (AdvSimd.Arm64.IsSupported && BitConverter.IsLittleEndian)
{
return Arm.Base64.DecodeFromBase64ARM(source, dest, out bytesConsumed, out bytesWritten, isUrl);
}
}
// To be comleted
//if (Vector512.IsHardwareAccelerated && Avx512Vbmi.IsSupported)
//{
Expand Down
Loading