Skip to content

Commit 68e7887

Browse files
SockworkOrangeSockworkOrange
authored
support :copyfrom annotation in mysql (#189)
1 parent 5258f0c commit 68e7887

File tree

28 files changed

+467
-83
lines changed

28 files changed

+467
-83
lines changed

‎.env

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
SOURCE_WASM_FILE_UBUNTU="WasmRunner/obj/release/net8.0/wasi-wasm/wasm/for-publish/WasmRunner.wasm"
22
SOURCE_WASM_FILE="WasmRunner/bin/Release/net8.0/wasi-wasm/AppBundle/WasmRunner.wasm"
3-
MYSQL_CONNECTION_STRING="server=localhost;database=tests;user=root"
3+
MYSQL_CONNECTION_STRING="server=localhost;database=tests;user=root;AllowLoadLocalInfile=true"
44
POSTGRES_CONNECTION_STRING="host=localhost;database=tests;username=postgres;password=pass"
55
POSTGRES_USER="postgres"
66
POSTGRES_PASSWORD="pass"

‎.github/workflows/legacy-tests.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ jobs:
100100
run: |
101101
$env:Path += ";C:\Program Files\MySQL\MySQL Server 8.0\bin"
102102
[Environment]::SetEnvironmentVariable("Path", $env:Path, "Machine")
103-
mysql -u root -e "CREATE DATABASE $Env:TESTS_DB;"
103+
mysql -u root -e "SET GLOBAL local_infile=1; CREATE DATABASE $Env:TESTS_DB;"
104104
mysql -u root $Env:TESTS_DB --execute="source examples/config/mysql/schema.sql"
105105
106106
- name: Run Tests

‎CodeGenerator/CodeGenerator.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -231,9 +231,9 @@ private MemberDeclarationSyntax AddMethodDeclaration(Query query)
231231
":exec" => DbDriver.ExecDeclare(queryTextConstant, argInterface, query),
232232
":one" => DbDriver.OneDeclare(queryTextConstant, argInterface, returnInterface, query),
233233
":many" => DbDriver.ManyDeclare(queryTextConstant, argInterface, returnInterface, query),
234+
":execrows" => ((IExecRows)DbDriver).ExecRowsDeclare(queryTextConstant, argInterface, query),
234235
":execlastid" => ((IExecLastId)DbDriver).ExecLastIdDeclare(queryTextConstant, argInterface, query),
235236
":copyfrom" => ((ICopyFrom)DbDriver).CopyFromDeclare(queryTextConstant, argInterface, query),
236-
":execrows" => ((IExecRows)DbDriver).ExecRowsDeclare(queryTextConstant, argInterface, query),
237237
_ => throw new NotImplementedException($"{query.Cmd} is not implemented")
238238
};
239239

‎CodeGenerator/Generators/CsprojGen.cs

+11-13
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ internal class CsprojGen(string outputDirectory, string projectName, string name
1111
private const string DefaultNpgsqlVersion = "8.0.6";
1212
private const string DefaultMysqlConnectorVersion = "2.4.0";
1313
private const string DefaultSqliteVersion = "9.0.0";
14+
private const string DefaultCsvHelperVersion = "33.0.1";
1415

1516
public File GenerateFile()
1617
{
@@ -25,7 +26,6 @@ public File GenerateFile()
2526
private string GetFileContents()
2627
{
2728
var optionalNullableProperty = options.DotnetFramework.LatestDotnetSupported() ? Environment.NewLine + " <Nullable>enable</Nullable>" : "";
28-
2929
return $"""
3030
<!--{Consts.AutoGeneratedComment}-->
3131
<!--Run the following to add the project to the solution:
@@ -46,19 +46,17 @@ dotnet sln add {outputDirectory}/{projectName}.csproj
4646

4747
string GetPackageReferences()
4848
{
49-
var driverVersion = GetDriverVersion(options);
50-
if (options.UseDapper)
51-
return $"""
52-
<ItemGroup>
53-
<PackageReference Include="{options.DriverName.ToName()}" Version="{driverVersion}"/>
54-
<PackageReference Include="Dapper" Version="{GetDapperVersion(options)}"/>
55-
</ItemGroup>
56-
""";
49+
var optionalDapperPackageReference = options.UseDapper
50+
? Environment.NewLine + $""" <PackageReference Include="Dapper" Version="{GetDapperVersion(options)}"/>"""
51+
: "";
52+
var optionalCsvHelper = options.DriverName == DriverName.MySqlConnector
53+
? Environment.NewLine + $""" <PackageReference Include="CsvHelper" Version="{DefaultCsvHelperVersion}"/>"""
54+
: "";
5755
return $"""
58-
<ItemGroup>
59-
<PackageReference Include="{options.DriverName.ToName()}" Version="{driverVersion}"/>
60-
</ItemGroup>
61-
""";
56+
<ItemGroup>
57+
<PackageReference Include="{options.DriverName.ToName()}" Version="{GetDriverVersion(options)}"/>{optionalDapperPackageReference}{optionalCsvHelper}
58+
</ItemGroup>
59+
""";
6260
}
6361
}
6462

‎Drivers/DbDriver.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,7 @@ protected string GetConnectionStringField()
108108

109109
public string GetIdColumnType()
110110
{
111-
return options.DriverName switch
111+
return Options.DriverName switch
112112
{
113113
DriverName.Sqlite => "int",
114114
_ => "long"

‎Drivers/Generators/CopyFromDeclareGen.cs

+1-45
Original file line numberDiff line numberDiff line change
@@ -13,52 +13,8 @@ public MemberDeclarationSyntax Generate(string queryTextConstant, string argInte
1313
return ParseMemberDeclaration($$"""
1414
public async Task {{query.Name}}(List<{{argInterface}}> args)
1515
{
16-
{{GetMethodBody(queryTextConstant, query)}}
16+
{{((ICopyFrom)dbDriver).GetCopyFromImpl(query, queryTextConstant)}}
1717
}
1818
""")!;
1919
}
20-
21-
22-
private string GetMethodBody(string queryTextConstant, Query query)
23-
{
24-
var (establishConnection, connectionOpen) = dbDriver.EstablishConnection(query);
25-
var beginBinaryImport = $"{Variable.Connection.AsVarName()}.BeginBinaryImportAsync({queryTextConstant}";
26-
var addRowsToCopyCommand = AddRowsToCopyCommand(query);
27-
return $$"""
28-
using ({{establishConnection}})
29-
{
30-
{{connectionOpen.AppendSemicolonUnlessEmpty()}}
31-
await {{Variable.Connection.AsVarName()}}.OpenAsync();
32-
using (var {{Variable.Writer.AsVarName()}} = await {{beginBinaryImport}}))
33-
{
34-
{{addRowsToCopyCommand}}
35-
await {{Variable.Writer.AsVarName()}}.CompleteAsync();
36-
}
37-
await {{Variable.Connection.AsVarName()}}.CloseAsync();
38-
}
39-
""";
40-
}
41-
42-
private string AddRowsToCopyCommand(Query query)
43-
{
44-
var constructRow = new List<string>()
45-
.Append($"await {Variable.Writer.AsVarName()}.StartRowAsync();")
46-
.Concat(query.Params
47-
.Select(p =>
48-
{
49-
var typeOverride = dbDriver.GetColumnDbTypeOverride(p.Column);
50-
var partialStmt =
51-
$"await {Variable.Writer.AsVarName()}.WriteAsync({Variable.Row.AsVarName()}.{p.Column.Name.ToPascalCase()}";
52-
return typeOverride is null
53-
? $"{partialStmt});"
54-
: $"{partialStmt}, {typeOverride});";
55-
}))
56-
.JoinByNewLine();
57-
return $$"""
58-
foreach (var {{Variable.Row.AsVarName()}} in args)
59-
{
60-
{{constructRow}}
61-
}
62-
""";
63-
}
6420
}

‎Drivers/MySqlConnectorDriver.cs

+40-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,8 @@
99

1010
namespace SqlcGenCsharp.Drivers;
1111

12-
public partial class MySqlConnectorDriver(Options options) : DbDriver(options), IOne, IMany, IExec, IExecRows, IExecLastId
12+
public partial class MySqlConnectorDriver(Options options) : DbDriver(options), IOne, IMany, IExec, IExecRows,
13+
IExecLastId, ICopyFrom
1314
{
1415
protected override List<ColumnMapping> ColumnMappings { get; } =
1516
[
@@ -59,6 +60,10 @@ public override UsingDirectiveSyntax[] GetUsingDirectives()
5960
{
6061
return base.GetUsingDirectives()
6162
.Append(UsingDirective(ParseName("MySqlConnector")))
63+
.Append(UsingDirective(ParseName("System.Globalization")))
64+
.Append(UsingDirective(ParseName("System.IO")))
65+
.Append(UsingDirective(ParseName("CsvHelper")))
66+
.Append(UsingDirective(ParseName("CsvHelper.Configuration")))
6267
.ToArray();
6368
}
6469

@@ -120,4 +125,38 @@ public MemberDeclarationSyntax ExecRowsDeclare(string queryTextConstant, string
120125
{
121126
return new ExecRowsDeclareGen(this).Generate(queryTextConstant, argInterface, query);
122127
}
128+
129+
public MemberDeclarationSyntax CopyFromDeclare(string queryTextConstant, string argInterface, Query query)
130+
{
131+
return new CopyFromDeclareGen(this).Generate(queryTextConstant, argInterface, query);
132+
}
133+
134+
public string GetCopyFromImpl(Query query, string queryTextConstant)
135+
{
136+
const string tempCsvFilename = "input.csv";
137+
const string csvDelimiter = ",";
138+
139+
var (establishConnection, connectionOpen) = EstablishConnection(query);
140+
return $$"""
141+
var {{Variable.Config.AsVarName()}} = new CsvConfiguration(CultureInfo.CurrentCulture) { Delimiter = "{{csvDelimiter}}" };
142+
using (var {{Variable.Writer.AsVarName()}} = new StreamWriter("{{tempCsvFilename}}"))
143+
using (var {{Variable.CsvWriter.AsVarName()}} = new CsvWriter({{Variable.Writer.AsVarName()}}, {{Variable.Config.AsVarName()}}))
144+
await {{Variable.CsvWriter.AsVarName()}}.WriteRecordsAsync({{Variable.Args.AsVarName()}});
145+
146+
using ({{establishConnection}})
147+
{
148+
{{connectionOpen.AppendSemicolonUnlessEmpty()}}
149+
var {{Variable.Loader.AsVarName()}} = new MySqlBulkLoader({{Variable.Connection.AsVarName()}})
150+
{
151+
Local = true,
152+
TableName = "{{query.InsertIntoTable.Name}}",
153+
FieldTerminator = "{{csvDelimiter}}",
154+
FileName = "{{tempCsvFilename}}",
155+
NumberOfLinesToSkip = 1
156+
};
157+
await {{Variable.Loader.AsVarName()}}.LoadAsync();
158+
await {{Variable.Connection.AsVarName()}}.CloseAsync();
159+
}
160+
""";
161+
}
123162
}

‎Drivers/NpgsqlDriver.cs

+49-6
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88

99
namespace SqlcGenCsharp.Drivers;
1010

11-
public class NpgsqlDriver : DbDriver, IOne, IMany, IExec, ICopyFrom, IExecRows, IExecLastId
11+
public class NpgsqlDriver : DbDriver, IOne, IMany, IExec, IExecRows, IExecLastId, ICopyFrom
1212
{
1313
public NpgsqlDriver(Options options) : base(options)
1414
{
@@ -145,11 +145,6 @@ public override MemberDeclarationSyntax ManyDeclare(string queryTextConstant, st
145145
return new ManyDeclareGen(this).Generate(queryTextConstant, argInterface, returnInterface, query);
146146
}
147147

148-
public MemberDeclarationSyntax CopyFromDeclare(string queryTextConstant, string argInterface, Query query)
149-
{
150-
return new CopyFromDeclareGen(this).Generate(queryTextConstant, argInterface, query);
151-
}
152-
153148
public MemberDeclarationSyntax ExecRowsDeclare(string queryTextConstant, string argInterface, Query query)
154149
{
155150
return new ExecRowsDeclareGen(this).Generate(queryTextConstant, argInterface, query);
@@ -159,4 +154,52 @@ public MemberDeclarationSyntax ExecLastIdDeclare(string queryTextConstant, strin
159154
{
160155
return new ExecLastIdDeclareGen(this).Generate(queryTextConstant, argInterface, query);
161156
}
157+
158+
public MemberDeclarationSyntax CopyFromDeclare(string queryTextConstant, string argInterface, Query query)
159+
{
160+
return new CopyFromDeclareGen(this).Generate(queryTextConstant, argInterface, query);
161+
}
162+
163+
public string GetCopyFromImpl(Query query, string queryTextConstant)
164+
{
165+
var (establishConnection, connectionOpen) = EstablishConnection(query);
166+
var beginBinaryImport = $"{Variable.Connection.AsVarName()}.BeginBinaryImportAsync({queryTextConstant}";
167+
var addRowsToCopyCommand = AddRowsToCopyCommand();
168+
return $$"""
169+
using ({{establishConnection}})
170+
{
171+
{{connectionOpen.AppendSemicolonUnlessEmpty()}}
172+
await {{Variable.Connection.AsVarName()}}.OpenAsync();
173+
using (var {{Variable.Writer.AsVarName()}} = await {{beginBinaryImport}}))
174+
{
175+
{{addRowsToCopyCommand}}
176+
await {{Variable.Writer.AsVarName()}}.CompleteAsync();
177+
}
178+
await {{Variable.Connection.AsVarName()}}.CloseAsync();
179+
}
180+
""";
181+
182+
string AddRowsToCopyCommand()
183+
{
184+
var constructRow = new List<string>()
185+
.Append($"await {Variable.Writer.AsVarName()}.StartRowAsync();")
186+
.Concat(query.Params
187+
.Select(p =>
188+
{
189+
var typeOverride = GetColumnDbTypeOverride(p.Column);
190+
var partialStmt =
191+
$"await {Variable.Writer.AsVarName()}.WriteAsync({Variable.Row.AsVarName()}.{p.Column.Name.ToPascalCase()}";
192+
return typeOverride is null
193+
? $"{partialStmt});"
194+
: $"{partialStmt}, {typeOverride});";
195+
}))
196+
.JoinByNewLine();
197+
return $$"""
198+
foreach (var {{Variable.Row.AsVarName()}} in args)
199+
{
200+
{{constructRow}}
201+
}
202+
""";
203+
}
204+
}
162205
}

‎Drivers/QueryAnnotations.cs

+2
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,8 @@ public interface IExecLastId
2626
public interface ICopyFrom
2727
{
2828
MemberDeclarationSyntax CopyFromDeclare(string queryTextConstant, string argInterface, Query query);
29+
30+
string GetCopyFromImpl(Query query, string queryTextConstant);
2931
}
3032

3133
public interface IExecRows

‎Drivers/Variable.cs

+4-1
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,17 @@ namespace SqlcGenCsharp.Drivers;
22

33
public enum Variable
44
{
5+
Config,
56
ConnectionString,
67
Connection,
78
Reader,
89
Row,
910
Writer,
11+
CsvWriter,
1012
Command,
1113
Result,
12-
Args
14+
Args,
15+
Loader
1316
}
1417

1518
public static class VariablesExtensions

‎EndToEndTests/MySqlConnectorTester.cs

+29-1
Original file line numberDiff line numberDiff line change
@@ -2,19 +2,23 @@
22
using NUnit.Framework;
33
using NUnit.Framework.Legacy;
44
using System;
5+
using System.Linq;
56
using System.Threading.Tasks;
67

78
namespace SqlcGenCsharpTests;
89

9-
public class MySqlConnectorTester : IOneTester, IManyTester, IExecTester, IExecRowsTester, IExecLastIdTester
10+
public class MySqlConnectorTester : IOneTester, IManyTester, IExecTester, IExecRowsTester, IExecLastIdTester, ICopyFromTester
1011
{
12+
private static readonly Random Randomizer = new();
13+
1114
private QuerySql QuerySql { get; } = new(
1215
Environment.GetEnvironmentVariable(EndToEndCommon.MySqlConnectionStringEnv)!);
1316

1417
[TearDown]
1518
public async Task EmptyTestsTable()
1619
{
1720
await QuerySql.TruncateAuthors();
21+
await QuerySql.TruncateCopyToTests();
1822
}
1923

2024
[Test]
@@ -137,4 +141,28 @@ public async Task TestExecLastId()
137141
Bio: DataGenerator.GenericQuote1
138142
});
139143
}
144+
145+
[Test]
146+
public async Task TestCopyFrom()
147+
{
148+
const int batchSize = 100;
149+
var createAuthorBatchArgs = Enumerable.Range(0, batchSize)
150+
.Select(_ => GenerateRandom())
151+
.ToList();
152+
await QuerySql.CopyToTests(createAuthorBatchArgs);
153+
var countRows = QuerySql.CountCopyRows().Result!.Value.Cnt;
154+
ClassicAssert.AreEqual(batchSize, countRows);
155+
return;
156+
157+
QuerySql.CopyToTestsArgs GenerateRandom()
158+
{
159+
return new QuerySql.CopyToTestsArgs
160+
{
161+
CVarchar = Randomizer.Next().ToString(),
162+
CInt = Randomizer.Next(),
163+
CDate = DateTime.Now.Subtract(TimeSpan.FromMilliseconds(Randomizer.Next())),
164+
CTimestamp = DateTime.Now.Subtract(TimeSpan.FromMilliseconds(Randomizer.Next()))
165+
};
166+
}
167+
}
140168
}

‎LegacyEndToEndTests/MySqlConnectorTester.cs

+27-1
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,10 @@
88

99
namespace SqlcGenCsharpTests
1010
{
11-
public class MySqlConnectorTester : IOneTester, IManyTester, IExecTester, IExecRowsTester, IExecLastIdTester
11+
public class MySqlConnectorTester : IOneTester, IManyTester, IExecTester, IExecRowsTester, IExecLastIdTester, ICopyFromTester
1212
{
13+
private static readonly Random Randomizer = new Random();
14+
1315
private QuerySql QuerySql { get; } = new QuerySql(
1416
Environment.GetEnvironmentVariable(EndToEndCommon.MySqlConnectionStringEnv));
1517

@@ -131,6 +133,30 @@ public async Task TestExecLastId()
131133
Assert.That(Equals(expected, actual));
132134
}
133135

136+
[Test]
137+
public async Task TestCopyFrom()
138+
{
139+
const int batchSize = 100;
140+
var createAuthorBatchArgs = Enumerable.Range(0, batchSize)
141+
.Select(_ => GenerateRandom())
142+
.ToList();
143+
await QuerySql.CopyToTests(createAuthorBatchArgs);
144+
var countRows = QuerySql.CountCopyRows().Result.Cnt;
145+
ClassicAssert.AreEqual(batchSize, countRows);
146+
return;
147+
148+
QuerySql.CopyToTestsArgs GenerateRandom()
149+
{
150+
return new QuerySql.CopyToTestsArgs
151+
{
152+
CVarchar = Randomizer.Next().ToString(),
153+
CInt = Randomizer.Next(),
154+
CDate = DateTime.Now.Subtract(TimeSpan.FromMilliseconds(Randomizer.Next())),
155+
CTimestamp = DateTime.Now.Subtract(TimeSpan.FromMilliseconds(Randomizer.Next()))
156+
};
157+
}
158+
}
159+
134160
private static bool Equals(QuerySql.GetAuthorRow x, QuerySql.GetAuthorRow y)
135161
{
136162
return x.Name.Equals(y.Name) && x.Bio.Equals(y.Bio);

0 commit comments

Comments
 (0)