Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

support batch insert of binary data in mysql #229

Merged
merged 4 commits into from
Mar 15, 2025
Merged

support batch insert of binary data in mysql #229

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
30d876e
support batch insert of binary data in mysql
SockworkOrange Mar 7, 2025
044ed3b
uncollapse data types support doc by default
SockworkOrange Mar 15, 2025
45c3c95
specify line terminator as unix line endings when loading CSV to mysql
SockworkOrange Mar 15, 2025
968d415
add TODO
SockworkOrange Mar 15, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 16 additions & 2 deletions CodeGenerator/Generators/UtilsGen.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ private UsingDirectiveSyntax[] GetUsingDirectives()

private MemberDeclarationSyntax GetUtilsClass()
{
// TODO move driver specific logic to DB driver interface
var optionalTransformQueryForSqliteBatch = dbDriver.Options.DriverName is DriverName.Sqlite
? """
private static readonly Regex ValuesRegex = new Regex(@"VALUES\s*\((?<params>[^)]*)\)", RegexOptions.IgnoreCase);
Expand Down Expand Up @@ -80,7 +81,7 @@ public static string TransformQueryForSqliteBatch(string originalSql, int cntRec
? $$"""
public class NullToStringConverter : DefaultTypeConverter
{
public override {{dbDriver.AddNullableSuffixIfNeeded("string", true)}} ConvertToString(
public override {{dbDriver.AddNullableSuffixIfNeeded("string", false)}} ConvertToString(
{{dbDriver.AddNullableSuffixIfNeeded("object", false)}} value, IWriterRow row, MemberMapData memberMapData)
{
return value == null ? @"\N" : base.ConvertToString(value, row, memberMapData);
Expand All @@ -89,7 +90,7 @@ public class NullToStringConverter : DefaultTypeConverter

public class BoolToBitConverter : DefaultTypeConverter
{
public override {{dbDriver.AddNullableSuffixIfNeeded("string", true)}} ConvertToString(
public override {{dbDriver.AddNullableSuffixIfNeeded("string", false)}} ConvertToString(
{{dbDriver.AddNullableSuffixIfNeeded("object", false)}} value, IWriterRow row, MemberMapData memberMapData)
{
switch (value)
Expand All @@ -103,6 +104,19 @@ public class BoolToBitConverter : DefaultTypeConverter
}
}
}

public class ByteArrayConverter : DefaultTypeConverter
{
public override {{dbDriver.AddNullableSuffixIfNeeded("string", false)}} ConvertToString(
{{dbDriver.AddNullableSuffixIfNeeded("object", false)}} value, IWriterRow row, MemberMapData memberMapData)
{
if (value == null)
return @"\N";
if (value is byte[] byteArray)
return System.Text.Encoding.UTF8.GetString(byteArray);
return base.ConvertToString(value, row, memberMapData);
}
}
"""
: string.Empty;

Expand Down
2 changes: 1 addition & 1 deletion Drivers/DbDriver.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ public abstract class DbDriver

public Dictionary<string, Table> Tables { get; }

private HashSet<string> NullableTypesInDotnetCore { get; } = ["string", "object"];
private HashSet<string> NullableTypesInDotnetCore { get; } = ["string", "object", "byte[]"]; // TODO add arrays in here in a non hard-coded manner

private HashSet<string> NullableTypes { get; } = ["bool", "byte", "short", "int", "long", "float", "double", "decimal", "DateTime"];

Expand Down
18 changes: 13 additions & 5 deletions Drivers/MySqlConnectorDriver.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -172,22 +172,29 @@ public string GetCopyFromImpl(Query query, string queryTextConstant)

var csvWriterVar = Variable.CsvWriter.AsVarName();
var loaderVar = Variable.Loader.AsVarName();
var optionsVar = Variable.Options.AsVarName();
var connectionVar = Variable.Connection.AsVarName();
var nullConverterFn = Variable.NullConverterFn.AsVarName();

var loaderColumns = query.Params.Select(p => $"\"{p.Column.Name}\"").JoinByComma();
var (establishConnection, connectionOpen) = EstablishConnection(query);

return $$"""
const string supportedDateTimeFormat = "yyyy-MM-dd H:mm:ss";
var {{Variable.Config.AsVarName()}} = new CsvConfiguration(CultureInfo.CurrentCulture) { Delimiter = "{{csvDelimiter}}" };
var {{Variable.Config.AsVarName()}} = new CsvConfiguration(CultureInfo.CurrentCulture)
{
Delimiter = "{{csvDelimiter}}",
NewLine = "\n"
};
var {{nullConverterFn}} = new Utils.NullToStringConverter();
using (var {{Variable.Writer.AsVarName()}} = new StreamWriter("{{tempCsvFilename}}", false, new UTF8Encoding(false)))
using (var {{csvWriterVar}} = new CsvWriter({{Variable.Writer.AsVarName()}}, {{Variable.Config.AsVarName()}}))
{
var {{Variable.Options}} = new TypeConverterOptions { Formats = new[] { supportedDateTimeFormat } };
{{csvWriterVar}}.Context.TypeConverterOptionsCache.AddOptions<DateTime>({{Variable.Options}});
{{csvWriterVar}}.Context.TypeConverterOptionsCache.AddOptions<DateTime?>({{Variable.Options}});
var {{optionsVar}} = new TypeConverterOptions { Formats = new[] { supportedDateTimeFormat } };
{{csvWriterVar}}.Context.TypeConverterOptionsCache.AddOptions<DateTime>({{optionsVar}});
{{csvWriterVar}}.Context.TypeConverterOptionsCache.AddOptions<DateTime?>({{optionsVar}});
{{csvWriterVar}}.Context.TypeConverterCache.AddConverter<bool?>(new Utils.BoolToBitConverter());
{{csvWriterVar}}.Context.TypeConverterCache.AddConverter<{{AddNullableSuffixIfNeeded("byte[]", false)}}>(new Utils.ByteArrayConverter());
{{csvWriterVar}}.Context.TypeConverterCache.AddConverter<byte?>({{nullConverterFn}});
{{csvWriterVar}}.Context.TypeConverterCache.AddConverter<short?>({{nullConverterFn}});
{{csvWriterVar}}.Context.TypeConverterCache.AddConverter<int?>({{nullConverterFn}});
Expand All @@ -212,7 +219,8 @@ public string GetCopyFromImpl(Query query, string queryTextConstant)
FieldTerminator = "{{csvDelimiter}}",
FieldQuotationCharacter = '"',
FieldQuotationOptional = true,
NumberOfLinesToSkip = 1
NumberOfLinesToSkip = 1,
LineTerminator = "\n"
};
{{loaderVar}}.Columns.AddRange(new List<string> { {{loaderColumns}} });
await {{loaderVar}}.LoadAsync();
Expand Down
2 changes: 1 addition & 1 deletion docs/04_Postgres.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ data type that can have a unique constraint.
Implemented via the `COPY FROM` command which can load binary data directly from `stdin`.
</details>

<details>
<details open>
<summary>Supported Data Types</summary>

Since in batch insert the data is not validated by the SQL itself but written in a binary format,
Expand Down
14 changes: 7 additions & 7 deletions docs/05_MySql.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ Requires us to first save the input batch as a CSV, and then load it via the dri

</details>

<details>
<details open>
<summary>Supported Data Types</summary>

Since in batch insert the data is not validated by the SQL itself but written and read from a CSV,
Expand Down Expand Up @@ -63,12 +63,12 @@ we consider support for the different data types separately for batch inserts an
| mediumtext | ✅ | ✅ |
| text | ✅ | ✅ |
| longtext | ✅ | ✅ |
| binary | ✅ | ❌ |
| varbinary | ✅ | ❌ |
| tinyblob | ✅ | ❌ |
| blob | ✅ | ❌ |
| mediumblob | ✅ | ❌ |
| longblob | ✅ | ❌ |
| binary | ✅ | ✅ |
| varbinary | ✅ | ✅ |
| tinyblob | ✅ | ✅ |
| blob | ✅ | ✅ |
| mediumblob | ✅ | ✅ |
| longblob | ✅ | ✅ |
| enum | ❌ | ❌ |
| set | ❌ | ❌ |
| json | ❌ | ❌ |
Expand Down
2 changes: 1 addition & 1 deletion docs/06_Sqlite.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ INSERT INTO tab1 (field1, field2) VALUES

</details>

<details>
<details open>
<summary>Supported Data Types</summary>

| DB Type | Supported? |
Expand Down
40 changes: 32 additions & 8 deletions end2end/EndToEndScaffold/Templates.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -668,7 +668,7 @@ private static void AssertSingularEquals(QuerySql.GetPostgresTypesAggRow expecte
{
Impl = $$"""
[Test]
[TestCase(false, true, 0x32, 13, 2084, 3124, -54355, 324245, -67865, 9787668656, "&", "\u1857", "\u2649", "Sheena is a Punk Rocker", "Holiday in Cambodia", "London's Calling", "London's Burning", "Police & Thieves", "2000-1-30", "1983-11-3 02:01:22", new byte[] { 0x15, 0x16, 0x17 }, new byte[] { 0x15, 0x22 }, new byte[] { 0x23 }, new byte[] { 0x33, 0x13 }, new byte[] { 0x11, 0x62, 0x10 }, new byte[] { 0x38, 0x45, 0x06 })]
[TestCase(false, true, 0x32, 13, 2084, 3124, -54355, 324245, -67865, 9787668656, "&", "\u1857", "\u2649", "Sheena is a Punk Rocker", "Holiday in Cambodia", "London's Calling", "London's Burning", "Police & Thieves", "2000-1-30", "1983-11-3 02:01:22", new byte[] { 0x15, 0x16, 0x17 }, new byte[] { 0x15, 0x24 }, new byte[] { 0x23 }, new byte[] { 0x33, 0x13 }, new byte[] { 0x11, 0x62, 0x10 }, new byte[] { 0x38, 0x45, 0x06 })]
[TestCase(null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, "1970-1-1 00:00:01", new byte[] { 0x0, 0x0, 0x0 }, new byte[] { }, new byte[] { }, new byte[] { }, new byte[] { }, new byte[] { })]
[TestCase(null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, "1970-1-1 00:00:01", null, null, null, null, null, null)]
public async Task TestMySqlTypes(
Expand Down Expand Up @@ -797,9 +797,9 @@ private static void AssertSingularEquals(QuerySql.GetMysqlTypesRow expected, Que
{
Impl = $$"""
[Test]
[TestCase(100, true, false, 0x05, -13, 324, -98760, 987965, 3132423, -7785442, 3.4f, -31.555666, 11.098643, 34.4424, 423.2445, 998.9994542, 21.214312452534, "D", "\u4321", "\u2345", "Parasite", "Clockwork Orange", "Dr. Strangelove", "Interview with a Vampire", "Memento", 1993, "2000-1-30", "1983-11-3 02:01:22", "2010-1-30 08:11:00")]
[TestCase(500, false, true, 0x12, 8, -555, 66979, -423425, -9798642, 3297398, 1.23f, 99.35542, 32.33345, -12.3456, -55.55556, -11.1123334, 33.423542356346, "3", "\u1234", "\u6543", "Splendor in the Grass", "Pulp Fiction", "Chinatown", "Repulsion", "Million Dollar Baby", 2025, "2012-9-20", "2012-1-20 22:12:34", "1984-6-5 20:12:12")]
[TestCase(10, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, "1970-1-1 00:00:01")]
[TestCase(100, true, false, 0x05, -13, 324, -98760, 987965, 3132423, -7785442, 3.4f, -31.555666, 11.098643, 34.4424, 423.2445, 998.9994542, 21.214312452534, "D", "\u4321", "\u2345", "Parasite", "Clockwork Orange", "Dr. Strangelove", "Interview with a Vampire", "Memento", 1993, "2000-1-30", "1983-11-3 02:01:22", "2010-1-30 08:11:00", new byte[] { 0x15, 0x16, 0x17 }, new byte[] { 0x15, 0x20 }, new byte[] { 0x23 }, new byte[] { 0x33, 0x13 }, new byte[] { 0x11, 0x62, 0x10 }, new byte[] { 0x38, 0x45, 0x06, 0x04 })]
[TestCase(500, false, true, 0x12, 8, -555, 66979, -423425, -9798642, 3297398, 1.23f, 99.35542, 32.33345, -12.3456, -55.55556, -11.1123334, 33.423542356346, "3", "\u1234", "\u6543", "Splendor in the Grass", "Pulp Fiction", "Chinatown", "Repulsion", "Million Dollar Baby", 2025, "2012-9-20", "2012-1-20 22:12:34", "1984-6-5 20:12:12", new byte[] { 0x0, 0x0, 0x0 }, new byte[] { }, new byte[] { }, new byte[] { }, new byte[] { }, new byte[] { })]
[TestCase(10, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, "1970-1-1 00:00:01", null, null, null, null, null, null)]
public async Task TestCopyFrom(
int batchSize,
bool? cBool,
Expand Down Expand Up @@ -829,7 +829,13 @@ public async Task TestCopyFrom(
short? cYear,
DateTime? cDate,
DateTime? cDatetime,
DateTime? cTimestamp)
DateTime? cTimestamp,
byte[] cBinary,
byte[] cVarbinary,
byte[] cTinyblob,
byte[] cBlob,
byte[] cMediumblob,
byte[] cLongblob)
{
var batchArgs = Enumerable.Range(0, batchSize)
.Select(_ => new QuerySql.InsertMysqlTypesBatchArgs
Expand All @@ -853,7 +859,13 @@ public async Task TestCopyFrom(
CYear = cYear,
CDate = cDate,
CDatetime = cDatetime,
CTimestamp = cTimestamp
CTimestamp = cTimestamp,
CBinary = cBinary,
CVarbinary = cVarbinary,
CTinyblob = cTinyblob,
CBlob = cBlob,
CMediumblob = cMediumblob,
CLongblob = cLongblob
})
.ToList();
await QuerySql.InsertMysqlTypesBatch(batchArgs);
Expand All @@ -879,7 +891,13 @@ public async Task TestCopyFrom(
CYear = cYear,
CDate = cDate,
CDatetime = cDatetime,
CTimestamp = cTimestamp
CTimestamp = cTimestamp,
CBinary = cBinary,
CVarbinary = cVarbinary,
CTinyblob = cTinyblob,
CBlob = cBlob,
CMediumblob = cMediumblob,
CLongblob = cLongblob
};
var actual = await QuerySql.GetMysqlTypesAgg();
AssertSingularEquals(expected, actual{{UnknownRecordValuePlaceholder}});
Expand Down Expand Up @@ -908,14 +926,20 @@ private static void AssertSingularEquals(QuerySql.GetMysqlTypesAggRow expected,
Assert.That(actual.CDate, Is.EqualTo(expected.CDate));
Assert.That(actual.CDatetime, Is.EqualTo(expected.CDatetime));
Assert.That(actual.CTimestamp, Is.EqualTo(expected.CTimestamp));
Assert.That(actual.CBinary, Is.EqualTo(expected.CBinary));
Assert.That(actual.CVarbinary, Is.EqualTo(expected.CVarbinary));
Assert.That(actual.CTinyblob, Is.EqualTo(expected.CTinyblob));
Assert.That(actual.CBlob, Is.EqualTo(expected.CBlob));
Assert.That(actual.CMediumblob, Is.EqualTo(expected.CMediumblob));
Assert.That(actual.CLongblob, Is.EqualTo(expected.CLongblob));
}
"""
},
[KnownTestType.SqliteDataTypes] = new TestImpl
{
Impl = $$"""
[Test]
[TestCase(-54355, 9787.66, "Songs of Love and Hate", new byte[] { 0x15, 0x20, 0x22 })]
[TestCase(-54355, 9787.66, "Songs of Love and Hate", new byte[] { 0x15, 0x20, 0x33 })]
[TestCase(null, null, null, new byte[] { })]
[TestCase(null, null, null, null)]
public async Task TestSqliteTypes(
Expand Down
Loading
Loading