Skip to content

Commit ad96fbb

Browse files
SockworkOrangeSockworkOrange
authored
support batch insert of binary data in mysql (#229)
* support batch insert of binary data in mysql * uncollapse data types support doc by default
1 parent 9d6c3c5 commit ad96fbb

26 files changed

+353
-134
lines changed

‎CodeGenerator/Generators/UtilsGen.cs

+16-2
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ private UsingDirectiveSyntax[] GetUsingDirectives()
5353

5454
private MemberDeclarationSyntax GetUtilsClass()
5555
{
56+
// TODO move driver specific logic to DB driver interface
5657
var optionalTransformQueryForSqliteBatch = dbDriver.Options.DriverName is DriverName.Sqlite
5758
? """
5859
private static readonly Regex ValuesRegex = new Regex(@"VALUES\s*\((?<params>[^)]*)\)", RegexOptions.IgnoreCase);
@@ -80,7 +81,7 @@ public static string TransformQueryForSqliteBatch(string originalSql, int cntRec
8081
? $$"""
8182
public class NullToStringConverter : DefaultTypeConverter
8283
{
83-
public override {{dbDriver.AddNullableSuffixIfNeeded("string", true)}} ConvertToString(
84+
public override {{dbDriver.AddNullableSuffixIfNeeded("string", false)}} ConvertToString(
8485
{{dbDriver.AddNullableSuffixIfNeeded("object", false)}} value, IWriterRow row, MemberMapData memberMapData)
8586
{
8687
return value == null ? @"\N" : base.ConvertToString(value, row, memberMapData);
@@ -89,7 +90,7 @@ public class NullToStringConverter : DefaultTypeConverter
8990
9091
public class BoolToBitConverter : DefaultTypeConverter
9192
{
92-
public override {{dbDriver.AddNullableSuffixIfNeeded("string", true)}} ConvertToString(
93+
public override {{dbDriver.AddNullableSuffixIfNeeded("string", false)}} ConvertToString(
9394
{{dbDriver.AddNullableSuffixIfNeeded("object", false)}} value, IWriterRow row, MemberMapData memberMapData)
9495
{
9596
switch (value)
@@ -103,6 +104,19 @@ public class BoolToBitConverter : DefaultTypeConverter
103104
}
104105
}
105106
}
107+
108+
public class ByteArrayConverter : DefaultTypeConverter
109+
{
110+
public override {{dbDriver.AddNullableSuffixIfNeeded("string", false)}} ConvertToString(
111+
{{dbDriver.AddNullableSuffixIfNeeded("object", false)}} value, IWriterRow row, MemberMapData memberMapData)
112+
{
113+
if (value == null)
114+
return @"\N";
115+
if (value is byte[] byteArray)
116+
return System.Text.Encoding.UTF8.GetString(byteArray);
117+
return base.ConvertToString(value, row, memberMapData);
118+
}
119+
}
106120
"""
107121
: string.Empty;
108122

‎Drivers/DbDriver.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ public abstract class DbDriver
1515

1616
public Dictionary<string, Table> Tables { get; }
1717

18-
private HashSet<string> NullableTypesInDotnetCore { get; } = ["string", "object"];
18+
private HashSet<string> NullableTypesInDotnetCore { get; } = ["string", "object", "byte[]"]; // TODO add arrays in here in a non hard-coded manner
1919

2020
private HashSet<string> NullableTypes { get; } = ["bool", "byte", "short", "int", "long", "float", "double", "decimal", "DateTime"];
2121

‎Drivers/MySqlConnectorDriver.cs

+13-5
Original file line numberDiff line numberDiff line change
@@ -172,22 +172,29 @@ public string GetCopyFromImpl(Query query, string queryTextConstant)
172172

173173
var csvWriterVar = Variable.CsvWriter.AsVarName();
174174
var loaderVar = Variable.Loader.AsVarName();
175+
var optionsVar = Variable.Options.AsVarName();
175176
var connectionVar = Variable.Connection.AsVarName();
176177
var nullConverterFn = Variable.NullConverterFn.AsVarName();
177178

178179
var loaderColumns = query.Params.Select(p => $"\"{p.Column.Name}\"").JoinByComma();
179180
var (establishConnection, connectionOpen) = EstablishConnection(query);
181+
180182
return $$"""
181183
const string supportedDateTimeFormat = "yyyy-MM-dd H:mm:ss";
182-
var {{Variable.Config.AsVarName()}} = new CsvConfiguration(CultureInfo.CurrentCulture) { Delimiter = "{{csvDelimiter}}" };
184+
var {{Variable.Config.AsVarName()}} = new CsvConfiguration(CultureInfo.CurrentCulture)
185+
{
186+
Delimiter = "{{csvDelimiter}}",
187+
NewLine = "\n"
188+
};
183189
var {{nullConverterFn}} = new Utils.NullToStringConverter();
184190
using (var {{Variable.Writer.AsVarName()}} = new StreamWriter("{{tempCsvFilename}}", false, new UTF8Encoding(false)))
185191
using (var {{csvWriterVar}} = new CsvWriter({{Variable.Writer.AsVarName()}}, {{Variable.Config.AsVarName()}}))
186192
{
187-
var {{Variable.Options}} = new TypeConverterOptions { Formats = new[] { supportedDateTimeFormat } };
188-
{{csvWriterVar}}.Context.TypeConverterOptionsCache.AddOptions<DateTime>({{Variable.Options}});
189-
{{csvWriterVar}}.Context.TypeConverterOptionsCache.AddOptions<DateTime?>({{Variable.Options}});
193+
var {{optionsVar}} = new TypeConverterOptions { Formats = new[] { supportedDateTimeFormat } };
194+
{{csvWriterVar}}.Context.TypeConverterOptionsCache.AddOptions<DateTime>({{optionsVar}});
195+
{{csvWriterVar}}.Context.TypeConverterOptionsCache.AddOptions<DateTime?>({{optionsVar}});
190196
{{csvWriterVar}}.Context.TypeConverterCache.AddConverter<bool?>(new Utils.BoolToBitConverter());
197+
{{csvWriterVar}}.Context.TypeConverterCache.AddConverter<{{AddNullableSuffixIfNeeded("byte[]", false)}}>(new Utils.ByteArrayConverter());
191198
{{csvWriterVar}}.Context.TypeConverterCache.AddConverter<byte?>({{nullConverterFn}});
192199
{{csvWriterVar}}.Context.TypeConverterCache.AddConverter<short?>({{nullConverterFn}});
193200
{{csvWriterVar}}.Context.TypeConverterCache.AddConverter<int?>({{nullConverterFn}});
@@ -212,7 +219,8 @@ public string GetCopyFromImpl(Query query, string queryTextConstant)
212219
FieldTerminator = "{{csvDelimiter}}",
213220
FieldQuotationCharacter = '"',
214221
FieldQuotationOptional = true,
215-
NumberOfLinesToSkip = 1
222+
NumberOfLinesToSkip = 1,
223+
LineTerminator = "\n"
216224
};
217225
{{loaderVar}}.Columns.AddRange(new List<string> { {{loaderColumns}} });
218226
await {{loaderVar}}.LoadAsync();

‎docs/04_Postgres.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ data type that can have a unique constraint.
1212
Implemented via the `COPY FROM` command which can load binary data directly from `stdin`.
1313
</details>
1414

15-
<details>
15+
<details open>
1616
<summary>Supported Data Types</summary>
1717

1818
Since in batch insert the data is not validated by the SQL itself but written in a binary format,

‎docs/05_MySql.md

+7-7
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ Requires us to first save the input batch as a CSV, and then load it via the dri
3434

3535
</details>
3636

37-
<details>
37+
<details open>
3838
<summary>Supported Data Types</summary>
3939

4040
Since in batch insert the data is not validated by the SQL itself but written and read from a CSV,
@@ -63,12 +63,12 @@ we consider support for the different data types separately for batch inserts an
6363
| mediumtext | ✅ | ✅ |
6464
| text | ✅ | ✅ |
6565
| longtext | ✅ | ✅ |
66-
| binary | ✅ | ❌ |
67-
| varbinary | ✅ | ❌ |
68-
| tinyblob | ✅ | ❌ |
69-
| blob | ✅ | ❌ |
70-
| mediumblob | ✅ | ❌ |
71-
| longblob | ✅ | ❌ |
66+
| binary | ✅ | ✅ |
67+
| varbinary | ✅ | ✅ |
68+
| tinyblob | ✅ | ✅ |
69+
| blob | ✅ | ✅ |
70+
| mediumblob | ✅ | ✅ |
71+
| longblob | ✅ | ✅ |
7272
| enum | ❌ | ❌ |
7373
| set | ❌ | ❌ |
7474
| json | ❌ | ❌ |

‎docs/06_Sqlite.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ INSERT INTO tab1 (field1, field2) VALUES
2525

2626
</details>
2727

28-
<details>
28+
<details open>
2929
<summary>Supported Data Types</summary>
3030

3131
| DB Type | Supported? |

‎end2end/EndToEndScaffold/Templates.cs

+32-8
Original file line numberDiff line numberDiff line change
@@ -668,7 +668,7 @@ private static void AssertSingularEquals(QuerySql.GetPostgresTypesAggRow expecte
668668
{
669669
Impl = $$"""
670670
[Test]
671-
[TestCase(false, true, 0x32, 13, 2084, 3124, -54355, 324245, -67865, 9787668656, "&", "\u1857", "\u2649", "Sheena is a Punk Rocker", "Holiday in Cambodia", "London's Calling", "London's Burning", "Police & Thieves", "2000-1-30", "1983-11-3 02:01:22", new byte[] { 0x15, 0x16, 0x17 }, new byte[] { 0x15, 0x22 }, new byte[] { 0x23 }, new byte[] { 0x33, 0x13 }, new byte[] { 0x11, 0x62, 0x10 }, new byte[] { 0x38, 0x45, 0x06 })]
671+
[TestCase(false, true, 0x32, 13, 2084, 3124, -54355, 324245, -67865, 9787668656, "&", "\u1857", "\u2649", "Sheena is a Punk Rocker", "Holiday in Cambodia", "London's Calling", "London's Burning", "Police & Thieves", "2000-1-30", "1983-11-3 02:01:22", new byte[] { 0x15, 0x16, 0x17 }, new byte[] { 0x15, 0x24 }, new byte[] { 0x23 }, new byte[] { 0x33, 0x13 }, new byte[] { 0x11, 0x62, 0x10 }, new byte[] { 0x38, 0x45, 0x06 })]
672672
[TestCase(null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, "1970-1-1 00:00:01", new byte[] { 0x0, 0x0, 0x0 }, new byte[] { }, new byte[] { }, new byte[] { }, new byte[] { }, new byte[] { })]
673673
[TestCase(null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, "1970-1-1 00:00:01", null, null, null, null, null, null)]
674674
public async Task TestMySqlTypes(
@@ -797,9 +797,9 @@ private static void AssertSingularEquals(QuerySql.GetMysqlTypesRow expected, Que
797797
{
798798
Impl = $$"""
799799
[Test]
800-
[TestCase(100, true, false, 0x05, -13, 324, -98760, 987965, 3132423, -7785442, 3.4f, -31.555666, 11.098643, 34.4424, 423.2445, 998.9994542, 21.214312452534, "D", "\u4321", "\u2345", "Parasite", "Clockwork Orange", "Dr. Strangelove", "Interview with a Vampire", "Memento", 1993, "2000-1-30", "1983-11-3 02:01:22", "2010-1-30 08:11:00")]
801-
[TestCase(500, false, true, 0x12, 8, -555, 66979, -423425, -9798642, 3297398, 1.23f, 99.35542, 32.33345, -12.3456, -55.55556, -11.1123334, 33.423542356346, "3", "\u1234", "\u6543", "Splendor in the Grass", "Pulp Fiction", "Chinatown", "Repulsion", "Million Dollar Baby", 2025, "2012-9-20", "2012-1-20 22:12:34", "1984-6-5 20:12:12")]
802-
[TestCase(10, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, "1970-1-1 00:00:01")]
800+
[TestCase(100, true, false, 0x05, -13, 324, -98760, 987965, 3132423, -7785442, 3.4f, -31.555666, 11.098643, 34.4424, 423.2445, 998.9994542, 21.214312452534, "D", "\u4321", "\u2345", "Parasite", "Clockwork Orange", "Dr. Strangelove", "Interview with a Vampire", "Memento", 1993, "2000-1-30", "1983-11-3 02:01:22", "2010-1-30 08:11:00", new byte[] { 0x15, 0x16, 0x17 }, new byte[] { 0x15, 0x20 }, new byte[] { 0x23 }, new byte[] { 0x33, 0x13 }, new byte[] { 0x11, 0x62, 0x10 }, new byte[] { 0x38, 0x45, 0x06, 0x04 })]
801+
[TestCase(500, false, true, 0x12, 8, -555, 66979, -423425, -9798642, 3297398, 1.23f, 99.35542, 32.33345, -12.3456, -55.55556, -11.1123334, 33.423542356346, "3", "\u1234", "\u6543", "Splendor in the Grass", "Pulp Fiction", "Chinatown", "Repulsion", "Million Dollar Baby", 2025, "2012-9-20", "2012-1-20 22:12:34", "1984-6-5 20:12:12", new byte[] { 0x0, 0x0, 0x0 }, new byte[] { }, new byte[] { }, new byte[] { }, new byte[] { }, new byte[] { })]
802+
[TestCase(10, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, "1970-1-1 00:00:01", null, null, null, null, null, null)]
803803
public async Task TestCopyFrom(
804804
int batchSize,
805805
bool? cBool,
@@ -829,7 +829,13 @@ public async Task TestCopyFrom(
829829
short? cYear,
830830
DateTime? cDate,
831831
DateTime? cDatetime,
832-
DateTime? cTimestamp)
832+
DateTime? cTimestamp,
833+
byte[] cBinary,
834+
byte[] cVarbinary,
835+
byte[] cTinyblob,
836+
byte[] cBlob,
837+
byte[] cMediumblob,
838+
byte[] cLongblob)
833839
{
834840
var batchArgs = Enumerable.Range(0, batchSize)
835841
.Select(_ => new QuerySql.InsertMysqlTypesBatchArgs
@@ -853,7 +859,13 @@ public async Task TestCopyFrom(
853859
CYear = cYear,
854860
CDate = cDate,
855861
CDatetime = cDatetime,
856-
CTimestamp = cTimestamp
862+
CTimestamp = cTimestamp,
863+
CBinary = cBinary,
864+
CVarbinary = cVarbinary,
865+
CTinyblob = cTinyblob,
866+
CBlob = cBlob,
867+
CMediumblob = cMediumblob,
868+
CLongblob = cLongblob
857869
})
858870
.ToList();
859871
await QuerySql.InsertMysqlTypesBatch(batchArgs);
@@ -879,7 +891,13 @@ public async Task TestCopyFrom(
879891
CYear = cYear,
880892
CDate = cDate,
881893
CDatetime = cDatetime,
882-
CTimestamp = cTimestamp
894+
CTimestamp = cTimestamp,
895+
CBinary = cBinary,
896+
CVarbinary = cVarbinary,
897+
CTinyblob = cTinyblob,
898+
CBlob = cBlob,
899+
CMediumblob = cMediumblob,
900+
CLongblob = cLongblob
883901
};
884902
var actual = await QuerySql.GetMysqlTypesAgg();
885903
AssertSingularEquals(expected, actual{{UnknownRecordValuePlaceholder}});
@@ -908,14 +926,20 @@ private static void AssertSingularEquals(QuerySql.GetMysqlTypesAggRow expected,
908926
Assert.That(actual.CDate, Is.EqualTo(expected.CDate));
909927
Assert.That(actual.CDatetime, Is.EqualTo(expected.CDatetime));
910928
Assert.That(actual.CTimestamp, Is.EqualTo(expected.CTimestamp));
929+
Assert.That(actual.CBinary, Is.EqualTo(expected.CBinary));
930+
Assert.That(actual.CVarbinary, Is.EqualTo(expected.CVarbinary));
931+
Assert.That(actual.CTinyblob, Is.EqualTo(expected.CTinyblob));
932+
Assert.That(actual.CBlob, Is.EqualTo(expected.CBlob));
933+
Assert.That(actual.CMediumblob, Is.EqualTo(expected.CMediumblob));
934+
Assert.That(actual.CLongblob, Is.EqualTo(expected.CLongblob));
911935
}
912936
"""
913937
},
914938
[KnownTestType.SqliteDataTypes] = new TestImpl
915939
{
916940
Impl = $$"""
917941
[Test]
918-
[TestCase(-54355, 9787.66, "Songs of Love and Hate", new byte[] { 0x15, 0x20, 0x22 })]
942+
[TestCase(-54355, 9787.66, "Songs of Love and Hate", new byte[] { 0x15, 0x20, 0x33 })]
919943
[TestCase(null, null, null, new byte[] { })]
920944
[TestCase(null, null, null, null)]
921945
public async Task TestSqliteTypes(

0 commit comments

Comments
 (0)