[release] 1. fix mysql :copyfrom implementation (#215)

2. fix copyfrom tests for mysql and postgres - testing more types and verifying that the data was loaded correctly
3. fix warnings in generated code by handling nulls correctly and adding required modifier when needed

Author: SockworkOrange

CodeGenerator/Generators/DataClassesGen.cs

@@ -2,6 +2,7 @@
 using Microsoft.CodeAnalysis.CSharp.Syntax;
 using Plugin;
 using SqlcGenCsharp.Drivers;
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using static Microsoft.CodeAnalysis.CSharp.SyntaxFactory;
@@ -18,28 +19,18 @@ public MemberDeclarationSyntax Generate(string name, ClassMember classMember, IL
         return GenerateAsCLass(className, columns);
     }
 
-    private RecordDeclarationSyntax GenerateAsRecord(string className, IList<Column> columns)
+    private MemberDeclarationSyntax GenerateAsRecord(string className, IList<Column> columns)
     {
-        return RecordDeclaration(Token(SyntaxKind.StructKeyword), className)
-            .AddModifiers(
-                Token(SyntaxKind.PublicKeyword),
-                Token(SyntaxKind.ReadOnlyKeyword),
-                Token(SyntaxKind.RecordKeyword)
-            )
-            .WithParameterList(ColumnsToParameterList())
-            .WithSemicolonToken(Token(SyntaxKind.SemicolonToken));
-
-        ParameterListSyntax ColumnsToParameterList()
-        {
-            var seenEmbed = new Dictionary<string, int>();
-            return ParameterList(SeparatedList(columns
-                .Select(column => Parameter(Identifier(GetFieldName(column, seenEmbed)))
-                    .WithType(ParseTypeName(dbDriver.GetCsharpType(column))))));
-        }
+        var seenEmbed = new Dictionary<string, int>();
+        var recordParameters = columns
+            .Select(column => $"{dbDriver.GetCsharpType(column)} {GetFieldName(column, seenEmbed)}")
+            .JoinByComma();
+        return ParseMemberDeclaration($"public readonly record struct {className} ({recordParameters});")!;
     }
 
     private ClassDeclarationSyntax GenerateAsCLass(string className, IList<Column> columns)
     {
+        var modernDotnetSupported = dbDriver.Options.DotnetFramework.LatestDotnetSupported();
         return ClassDeclaration(className)
             .AddModifiers(Token(SyntaxKind.PublicKeyword))
             .AddMembers(ColumnsToProperties())
@@ -48,8 +39,19 @@ private ClassDeclarationSyntax GenerateAsCLass(string className, IList<Column> c
         MemberDeclarationSyntax[] ColumnsToProperties()
         {
             var seenEmbed = new Dictionary<string, int>();
-            return columns.Select(column => ParseMemberDeclaration(
-                    $"public {dbDriver.GetCsharpType(column)} {GetFieldName(column, seenEmbed)} {{ get; set; }}"))
+            return columns.Select(column =>
+                {
+                    var csharpType = dbDriver.GetCsharpType(column);
+                    var requiredModifierNeeded = modernDotnetSupported && // required modifier supported by .Net framework
+                        column.NotNull && // the field is not null, hence required
+                        !dbDriver.IsTypeNullableForAllRuntimes(csharpType); // TODO document
+                    var optionalRequiredModifier = requiredModifierNeeded ? "required" : string.Empty;
+                    var setterMethod = modernDotnetSupported ? "init" : "set";
+                    return ParseMemberDeclaration(
+                        $$"""
+                          public {{optionalRequiredModifier}} {{csharpType}} {{GetFieldName(column, seenEmbed)}} { get; {{setterMethod}}; }
+                          """);
+                })
                 .Cast<MemberDeclarationSyntax>()
                 .ToArray();
         }

CodeGenerator/Generators/QueriesGen.cs

@@ -17,7 +17,8 @@ internal class QueriesGen(DbDriver dbDriver, Options options, string namespaceNa
         "UnusedAutoPropertyAccessor.Global",
         "NotAccessedPositionalProperty.Global",
         "ConvertToUsingDeclaration",
-        "UseAwaitUsing"
+        "UseAwaitUsing",
+        "UseObjectOrCollectionInitializer"
     ];
 
     private RootGen RootGen { get; } = new(options);

Drivers/DbDriver.cs

@@ -15,8 +15,6 @@ public abstract class DbDriver(Options options, Dictionary<string, Table> tables
 
     public Dictionary<string, Table> Tables { get; } = tables;
 
-    private HashSet<string> NullableTypesInAllRuntimes { get; } = ["long", "double", "int", "float", "bool", "DateTime"];
-
     protected abstract List<ColumnMapping> ColumnMappings { get; }
 
     public virtual UsingDirectiveSyntax[] GetUsingDirectives()
@@ -91,6 +89,9 @@ public string GetColumnReader(Column column, int ordinal)
 
     public abstract string CreateSqlCommand(string sqlTextConstant);
 
+    private HashSet<string> NullableTypesInAllRuntimes { get; } = ["long", "double", "int", "float", "bool", "DateTime"];
+
+    // TODO move out from driver + rename
     public bool IsTypeNullableForAllRuntimes(string csharpType)
     {
         return NullableTypesInAllRuntimes.Contains(csharpType.Replace("?", ""));

Drivers/Generators/CommonGen.cs

@@ -13,35 +13,48 @@ public static string GetMethodParameterList(string argInterface, IEnumerable<Par
             : $"{argInterface} {Variable.Args.AsVarName()}")}";
     }
 
-    public string AddParametersToCommand(IEnumerable<Parameter> parameters)
+    public static string AddParametersToCommand(IEnumerable<Parameter> parameters)
     {
         return parameters.Select(p =>
         {
             var commandVar = Variable.Command.AsVarName();
             var param = p.Column.Name.ToPascalCase();
-            var nullCheck = dbDriver.Options.DotnetFramework.LatestDotnetSupported() && !p.Column.NotNull ? "!" : "";
+            var argsVar = Variable.Args.AsVarName();
             return p.Column.IsSqlcSlice
                 ? $$"""
-                    for (int i = 0; i < {{Variable.Args.AsVarName()}}.{{param}}.Length; i++)
-                       {{commandVar}}.Parameters.AddWithValue($"@{{param}}Arg{i}", {{Variable.Args.AsVarName()}}.{{param}}[i]);
+                    for (int i = 0; i < {{argsVar}}.{{param}}.Length; i++)
+                        {{commandVar}}.Parameters.AddWithValue($"@{{param}}Arg{i}", {{argsVar}}.{{param}}[i]);
                     """
-                : $"{commandVar}.Parameters.AddWithValue(\"@{p.Column.Name}\", args.{param}{nullCheck});";
+                : $$"""
+                   if ({{argsVar}}.{{param}} != null) 
+                        {{commandVar}}.Parameters.AddWithValue("@{{p.Column.Name}}", {{argsVar}}.{{param}});
+                   """;
         }).JoinByNewLine();
     }
 
     public static string ConstructDapperParamsDict(IList<Parameter> parameters)
     {
         if (!parameters.Any()) return string.Empty;
         var initParamsDict = $"var {Variable.QueryParams.AsVarName()} = new Dictionary<string, object>();";
+        var argsVar = Variable.Args.AsVarName();
+        var queryParamsVar = Variable.QueryParams.AsVarName();
+
         var dapperParamsCommands = parameters.Select(p =>
         {
             var param = p.Column.Name.ToPascalCase();
-            return p.Column.IsSqlcSlice
-                ? $$"""
-                    for (int i = 0; i < {{Variable.Args.AsVarName()}}.{{param}}.Length; i++)
-                       {{Variable.QueryParams.AsVarName()}}.Add($"@{{param}}Arg{i}", {{Variable.Args.AsVarName()}}.{{param}}[i]);
-                    """
-                : $"{Variable.QueryParams.AsVarName()}.Add(\"{p.Column.Name}\", {Variable.Args.AsVarName()}.{param});";
+            if (p.Column.IsSqlcSlice)
+                return $$"""
+                        for (int i = 0; i < {{argsVar}}.{{param}}.Length; i++)
+                            {{queryParamsVar}}.Add($"@{{param}}Arg{i}", {{argsVar}}.{{param}}[i]);
+                        """;
+
+            var addParamToDict = $"{queryParamsVar}.Add(\"{p.Column.Name}\", {argsVar}.{param});";
+            return p.Column.NotNull
+                ? addParamToDict
+                : $"""
+                    if ({argsVar}.{param} != null) 
+                        {addParamToDict}
+                    """;
         });
 
         return $$"""

Drivers/MySqlConnectorDriver.cs

@@ -38,7 +38,8 @@ public partial class MySqlConnectorDriver(Options options, Dictionary<string, Ta
                 { "time", null },
                 { "tinytext", null },
                 { "varchar", null },
-                { "var_string", null }
+                { "var_string", null },
+                { "json", null }
             }, ordinal => $"reader.GetString({ordinal})"),
         new("DateTime",
             new Dictionary<string, string?> { { "date", null }, { "datetime", null }, { "timestamp", null } }, ordinal => $"reader.GetDateTime({ordinal})"),
@@ -54,25 +55,30 @@ public partial class MySqlConnectorDriver(Options options, Dictionary<string, Ta
         new("double",
             new Dictionary<string, string?> { { "double", null }, { "float", null } }, ordinal => $"reader.GetDouble({ordinal})"),
         new("object",
-            new Dictionary<string, string?> { { "json", null } }, ordinal => $"reader.GetString({ordinal})")
+            new Dictionary<string, string?> { }, ordinal => $"reader.GetValue({ordinal})")
     ];
 
     public override UsingDirectiveSyntax[] GetUsingDirectives()
     {
         return base.GetUsingDirectives()
-            .Append(UsingDirective(ParseName("MySqlConnector")))
-            .Append(UsingDirective(ParseName("System.Globalization")))
-            .Append(UsingDirective(ParseName("System.IO")))
-            .Append(UsingDirective(ParseName("CsvHelper")))
-            .Append(UsingDirective(ParseName("CsvHelper.Configuration")))
+            .Concat(
+            [
+                UsingDirective(ParseName("MySqlConnector")),
+                UsingDirective(ParseName("System.Globalization")),
+                UsingDirective(ParseName("System.IO")),
+                UsingDirective(ParseName("CsvHelper")),
+                UsingDirective(ParseName("CsvHelper.Configuration")),
+                UsingDirective(ParseName("CsvHelper.TypeConversion")),
+                UsingDirective(ParseName("System.Text"))
+            ])
             .ToArray();
     }
 
     public override ConnectionGenCommands EstablishConnection(Query query)
     {
         return new ConnectionGenCommands(
             $"var {Variable.Connection.AsVarName()} = new MySqlConnection({GetConnectionStringField()})",
-            $"{Variable.Connection.AsVarName()}.Open()"
+            $"await {Variable.Connection.AsVarName()}.OpenAsync()"
         );
     }
 
@@ -137,26 +143,39 @@ public string GetCopyFromImpl(Query query, string queryTextConstant)
         const string tempCsvFilename = "input.csv";
         const string csvDelimiter = ",";
 
+        var csvWriterVar = Variable.CsvWriter.AsVarName();
+        var loaderVar = Variable.Loader.AsVarName();
+        var connectionVar = Variable.Connection.AsVarName();
+
+        var loaderColumns = query.Params.Select(p => $"\"{p.Column.Name}\"").JoinByComma();
         var (establishConnection, connectionOpen) = EstablishConnection(query);
         return $$"""  
+                 const string supportedDateTimeFormat = "yyyy-MM-dd H:mm:ss";
                  var {{Variable.Config.AsVarName()}} = new CsvConfiguration(CultureInfo.CurrentCulture) { Delimiter = "{{csvDelimiter}}" };
-                 using (var {{Variable.Writer.AsVarName()}} = new StreamWriter("{{tempCsvFilename}}"))
-                 using (var {{Variable.CsvWriter.AsVarName()}} = new CsvWriter({{Variable.Writer.AsVarName()}}, {{Variable.Config.AsVarName()}}))
-                    await {{Variable.CsvWriter.AsVarName()}}.WriteRecordsAsync({{Variable.Args.AsVarName()}});
+                 using (var {{Variable.Writer.AsVarName()}} = new StreamWriter("{{tempCsvFilename}}", false, new UTF8Encoding(false)))
+                 using (var {{csvWriterVar}} = new CsvWriter({{Variable.Writer.AsVarName()}}, {{Variable.Config.AsVarName()}}))
+                 {
+                    var options = new TypeConverterOptions { Formats = new[] { supportedDateTimeFormat } };
+                    {{csvWriterVar}}.Context.TypeConverterOptionsCache.AddOptions<DateTime>(options);
+                    {{csvWriterVar}}.Context.TypeConverterOptionsCache.AddOptions<DateTime?>(options);
+                    await {{csvWriterVar}}.WriteRecordsAsync({{Variable.Args.AsVarName()}});
+                 }
                  
                  using ({{establishConnection}})
                  {
                      {{connectionOpen.AppendSemicolonUnlessEmpty()}}
-                     var {{Variable.Loader.AsVarName()}} = new MySqlBulkLoader({{Variable.Connection.AsVarName()}})
+                     var {{loaderVar}} = new MySqlBulkLoader({{connectionVar}})
                      {
                          Local = true, 
                          TableName = "{{query.InsertIntoTable.Name}}", 
                          FieldTerminator = "{{csvDelimiter}}",
                          FileName = "{{tempCsvFilename}}",
+                         FieldQuotationCharacter = '"',
                          NumberOfLinesToSkip = 1
                      };
-                     await {{Variable.Loader.AsVarName()}}.LoadAsync();
-                     await {{Variable.Connection.AsVarName()}}.CloseAsync();
+                     {{loaderVar}}.Columns.AddRange(new List<string> { {{loaderColumns}} });
+                     await {{loaderVar}}.LoadAsync();
+                     await {{connectionVar}}.CloseAsync();
                  }
                  """;
     }

Drivers/NpgsqlDriver.cs

@@ -25,7 +25,12 @@ public NpgsqlDriver(Options options, Dictionary<string, Table> tables) : base(op
     protected sealed override List<ColumnMapping> ColumnMappings { get; } =
     [
         new("long",
-            new Dictionary<string, string?> { { "int8", null }, { "bigint", null }, { "bigserial", null } }
+            new Dictionary<string, string?>
+            {
+                { "int8", "NpgsqlDbType.Bigint" },
+                { "bigint", "NpgsqlDbType.Bigint" },
+                { "bigserial", "NpgsqlDbType.Bigint" }
+            }
             , ordinal => $"reader.GetInt64({ordinal})"),
         new("byte[]",
             new Dictionary<string, string?>
@@ -63,16 +68,23 @@ public NpgsqlDriver(Options options, Dictionary<string, Table> tables) : base(op
             {
                 { "integer", "NpgsqlDbType.Integer" },
                 { "int", "NpgsqlDbType.Integer" },
-                { "int2", null },
+                { "int2", "NpgsqlDbType.Smallint" },
                 { "int4", "NpgsqlDbType.Integer" },
-                { "serial", null }
+                { "serial", "NpgsqlDbType.Integer" }
             }, ordinal => $"reader.GetInt32({ordinal})",
                ordinal => $"reader.GetFieldValue<int[]>({ordinal})"),
         new("float",
-            new Dictionary<string, string?> { { "numeric", null }, { "float4", null }, { "float8", null } }
-            , ordinal => $"reader.GetFloat({ordinal})"),
+            new Dictionary<string, string?>
+            {
+                { "numeric", "NpgsqlDbType.Numeric" },
+                { "float4", "NpgsqlDbType.Real" },
+                { "float8", "NpgsqlDbType.Real" }
+            }, ordinal => $"reader.GetFloat({ordinal})"),
         new("decimal",
-            new Dictionary<string, string?> { { "decimal", null } }, ordinal => $"reader.GetDecimal({ordinal})"),
+            new Dictionary<string, string?>
+            {
+                { "decimal", "NpgsqlDbType.Real" }
+            }, ordinal => $"reader.GetDecimal({ordinal})"),
         new("bool",
             new Dictionary<string, string?> { { "bool", null }, { "boolean", null } }, ordinal => $"reader.GetBoolean({ordinal})")
     ];