Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Automatically detect encoding in csv_load script function #511

Merged
merged 1 commit into from
Nov 7, 2023
Merged

Automatically detect encoding in csv_load script function #511

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 24 additions & 11 deletions projects/GKCore/GKCore/GKUtils.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@
namespace GKCore
{
/// <summary>
///
///
/// </summary>
public static class GKUtils
{
Expand Down Expand Up @@ -369,9 +369,9 @@ public static TaskGoalRet GetTaskGoal(GDMTree tree, GDMTaskRecord taskRec)
public static string GetTaskGoalStr(GDMTree tree, GDMTaskRecord taskRec)
{
if (tree == null || taskRec == null) return string.Empty;

string result = "";

var goal = GetTaskGoal(tree, taskRec);

switch (goal.GoalType) {
Expand Down Expand Up @@ -482,7 +482,13 @@ public static CharsetResult DetectCharset(Stream stream, int bufferSize = 32768)
return result;
}

public static StreamReader GetDetectedStreamReader(Stream stream)
public static Encoding DetectEncoding(string fileName)
{
using (var file = File.OpenRead(fileName))
return DetectEncoding(file);
}

public static Encoding DetectEncoding(Stream stream)
{
Encoding defaultEncoding;
try {
Expand All @@ -492,6 +498,13 @@ public static StreamReader GetDetectedStreamReader(Stream stream)
defaultEncoding = Encoding.UTF8;
}

return defaultEncoding;
}

public static StreamReader GetDetectedStreamReader(Stream stream)
{
var defaultEncoding = DetectEncoding(stream);

StreamReader reader = new StreamReader(stream, defaultEncoding);
return reader;
}
Expand Down Expand Up @@ -821,8 +834,8 @@ public static string GetShortDatePattern()
}

/// <summary>
/// The result of the function is a "normalized date", delimited by '.' and fixed order of parts: "dd.mm.yyyy".
/// The pattern and regional date contain the delimiter '/'.
/// The result of the function is a "normalized date", delimited by '.' and fixed order of parts: "dd.mm.yyyy".
/// The pattern and regional date contain the delimiter '/'.
/// The pattern defines the position of the parts in a regional date format.
/// </summary>
/// <param name="regionalDate">date similar "01/20/1970"</param>
Expand Down Expand Up @@ -863,8 +876,8 @@ public static string GetNormalizeDate(string regionalDate, string pattern)
}

/// <summary>
/// The result of the function is a "regional date", delimited by '/' and regional order of parts: "mm/dd/yyyy"
/// or any other. The pattern and regional date contain the delimiter '/'.
/// The result of the function is a "regional date", delimited by '/' and regional order of parts: "mm/dd/yyyy"
/// or any other. The pattern and regional date contain the delimiter '/'.
/// The pattern defines the position of the parts in a regional date format.
/// </summary>
/// <param name="normalizeDate">date with format "dd.mm.yyyy"</param>
Expand Down Expand Up @@ -1235,7 +1248,7 @@ public static int GetDaysForBirth(GDMIndividualRecord iRec)
double curN = curY + curM / 12.0 + curD / 12.0 / 31.0;
bdY = (bdN < curN) ? (curY + 1) : curY;

// There are valid birthdays on February 29th in leap years.
// There are valid birthdays on February 29th in leap years.
// For other years, we need a correction for an acceptable day.
if (bdD == 29 && bdM == 2 && !DateTime.IsLeapYear(bdY)) {
bdD -= 1;
Expand Down Expand Up @@ -1287,7 +1300,7 @@ public static int GetDaysForBirthAnniversary(GDMIndividualRecord iRec, out int y
if (anniversary) {
bdY = (bdN < curN) ? (curY + 1) : curY;

// There are valid birthdays on February 29th in leap years.
// There are valid birthdays on February 29th in leap years.
// For other years, we need a correction for an acceptable day.
if (bdD == 29 && bdM == 2 && !DateTime.IsLeapYear(bdY)) {
bdD -= 1;
Expand Down Expand Up @@ -2080,7 +2093,7 @@ private static void ShowPersonNamesakes(GDMTree tree, GDMIndividualRecord iRec,
int num4 = namesakes.Count;
for (int i = 0; i < num4; i++) {
GDMIndividualRecord relPerson = (GDMIndividualRecord)namesakes.GetObject(i);

summary.Add(" " + HyperLink(relPerson.XRef, namesakes[i], 0));
}
}
Expand Down
6 changes: 4 additions & 2 deletions projects/GKCore/GKCore/ScriptEngine.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -714,9 +714,11 @@ public bool csv_load(string fileName, bool hasHeader)
if (!File.Exists(fileName)) return result;

try {
fCSVData = CSVReader.ReadCSVFile(fileName, Encoding.Unicode, hasHeader);
fCSVData = CSVReader.ReadCSVFile(fileName, GKUtils.DetectEncoding(fileName), hasHeader);
result = true;
} catch {
} catch (Exception e) {
Logger.WriteError("ScriptEngine.csv_load(" + fileName + "): fail", e);

result = false;
}

Expand Down