Skip to content

Fix problems with DataFrame WriteCsv when quotes are present in data #6340

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 21 commits into from
Oct 4, 2022
Merged

Fix problems with DataFrame WriteCsv when quotes are present in data #6340

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
f74d1ab
Add DataFrame.IO tests with separators in data
dakersnar Aug 23, 2022
d72df26
Add test where comma is in header
dakersnar Aug 23, 2022
e9e4254
Add two versions of test cases, likely going to use the helper version
dakersnar Aug 24, 2022
e0ff9a9
Fix separators in data
dakersnar Aug 24, 2022
6a568da
Fix separators in header
dakersnar Aug 24, 2022
9c02ab1
Clean up tests
dakersnar Aug 26, 2022
b9bec7a
Fix issue with not wrapping output with newlines in quotations
dakersnar Aug 26, 2022
6f80297
Accidental commit
dakersnar Aug 26, 2022
d5aaff9
Clean up
dakersnar Sep 9, 2022
be4fcb0
Clean up mini test framework a bit
dakersnar Sep 9, 2022
de5abe8
Fix WriteCsv bug when quotations are present in data.
dakersnar Sep 12, 2022
493a750
Accidental include
dakersnar Sep 12, 2022
3e363b1
Manually merge with main
dakersnar Sep 27, 2022
06e018e
Delete extra line
dakersnar Sep 27, 2022
3ac9e99
Merge branch 'main' of https://github.com/dotnet/machinelearning into…
dakersnar Sep 27, 2022
54815a6
Accidental includes
dakersnar Sep 27, 2022
a2de77d
Use new raw string literal syntax
dakersnar Sep 29, 2022
546c5ff
Add NeedsQuotes helper
dakersnar Sep 29, 2022
cc016d3
IndexOfAny refactor
dakersnar Sep 29, 2022
c9ddd21
Remove accidental include
dakersnar Sep 30, 2022
d7dac25
Add preview language tag to csproj
dakersnar Oct 4, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 11 additions & 8 deletions src/Microsoft.Data.Analysis/DataFrame.IO.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
// See the LICENSE file in the project root for more information.

using System;
using System.Collections;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
Expand Down Expand Up @@ -499,11 +498,11 @@ public static void WriteCsv(DataFrame dataFrame, Stream csvStream,

if (t == typeof(string))
{
bool needsQuotes = ((string)cell).IndexOf(separator) != -1 || ((string)cell).IndexOf('\n') != -1;
if (needsQuotes)
string stringCell = (string)cell;
if (NeedsQuotes(stringCell, separator))
{
record.Append('\"');
record.Append(cell);
record.Append(stringCell.Replace("\"", "\"\"")); // Quotations in CSV data must be escaped with another quotation
record.Append('\"');
continue;
}
Expand All @@ -519,6 +518,7 @@ public static void WriteCsv(DataFrame dataFrame, Stream csvStream,
}
}
}

private static void WriteHeader(StreamWriter csvFile, IReadOnlyList<string> columnNames, char separator)
{
bool firstColumn = true;
Expand All @@ -533,20 +533,23 @@ private static void WriteHeader(StreamWriter csvFile, IReadOnlyList<string> colu
firstColumn = false;
}

bool needsQuotes = name.IndexOf(separator) != -1 || name.IndexOf('\n') != -1;
if (needsQuotes)
if (NeedsQuotes(name, separator))
{
csvFile.Write('\"');
csvFile.Write(name);
csvFile.Write(name.Replace("\"", "\"\"")); // Quotations in CSV data must be escaped with another quotation
csvFile.Write('\"');
}
else
{
csvFile.Write(name);
}
}

csvFile.WriteLine();
}

private static bool NeedsQuotes(string csvCell, char separator)
{
return csvCell.AsSpan().IndexOfAny(separator, '\n', '\"') != -1;
}
}
}
122 changes: 96 additions & 26 deletions test/Microsoft.Data.Analysis.Tests/DataFrame.IOTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1064,10 +1064,12 @@ public static IEnumerable<object[]> CsvWithTextQualifiers_TestData()
{
yield return new object[] // Comma Separators in Data
{
@"Name,Age,Description
Paul,34,""Paul lives in Vermont, VA.""
Victor,29,""Victor: Funny guy""
Maria,31,",
"""
Name,Age,Description
Paul,34,"Paul lives in Vermont, VA."
Victor,29,"Victor: Funny guy"
Maria,31,
""",
',',
new Type[] { typeof(string), typeof(int), typeof(string) },
new LoadCsvVerifyingHelper(
Expand All @@ -1085,10 +1087,12 @@ public static IEnumerable<object[]> CsvWithTextQualifiers_TestData()
};
yield return new object[] // Colon Separators in Data
{
@"Name:Age:Description
Paul:34:""Paul lives in Vermont, VA.""
Victor:29:""Victor: Funny guy""
Maria:31:",
"""
Name:Age:Description
Paul:34:"Paul lives in Vermont, VA."
Victor:29:"Victor: Funny guy"
Maria:31:
""",
':',
new Type[] { typeof(string), typeof(int), typeof(string) },
new LoadCsvVerifyingHelper(
Expand All @@ -1106,10 +1110,12 @@ public static IEnumerable<object[]> CsvWithTextQualifiers_TestData()
};
yield return new object[] // Comma Separators in Header
{
@"""Na,me"",Age,Description
Paul,34,""Paul lives in Vermont, VA.""
Victor,29,""Victor: Funny guy""
Maria,31,",
"""
"Na,me",Age,Description
Paul,34,"Paul lives in Vermont, VA."
Victor,29,"Victor: Funny guy"
Maria,31,
""",
',',
new Type[] { typeof(string), typeof(int), typeof(string) },
new LoadCsvVerifyingHelper(
Expand All @@ -1127,11 +1133,13 @@ public static IEnumerable<object[]> CsvWithTextQualifiers_TestData()
};
yield return new object[] // Newlines In Data
{
@"Name,Age,Description
Paul,34,""Paul lives in Vermont
VA.""
Victor,29,""Victor: Funny guy""
Maria,31,",
"""
Name,Age,Description
Paul,34,"Paul lives in Vermont
VA."
Victor,29,"Victor: Funny guy"
Maria,31,
""",
',',
new Type[] { typeof(string), typeof(int), typeof(string) },
new LoadCsvVerifyingHelper(
Expand All @@ -1141,27 +1149,89 @@ public static IEnumerable<object[]> CsvWithTextQualifiers_TestData()
new Type[] { typeof(string), typeof(int), typeof(string) },
new object[][]
{
new object[] { "Paul", 34, @"Paul lives in Vermont
VA." },
new object[]
{
"Paul",
34,
"""
Paul lives in Vermont
VA.
"""
},
new object[] { "Victor", 29, "Victor: Funny guy" },
new object[] { "Maria", 31, "" }
}
)
};
yield return new object[] // Newlines In Header
{
@"""Na
me"":Age:Description
Paul:34:""Paul lives in Vermont, VA.""
Victor:29:""Victor: Funny guy""
Maria:31:",
"""
"Na
me":Age:Description
Paul:34:"Paul lives in Vermont, VA."
Victor:29:"Victor: Funny guy"
Maria:31:
""",
':',
new Type[] { typeof(string), typeof(int), typeof(string) },
new LoadCsvVerifyingHelper(
3,
3,
new string[] { @"Na
me", "Age", "Description" },
new string[]
{
"""
Na
me
""",
"Age",
"Description"
},
new Type[] { typeof(string), typeof(int), typeof(string) },
new object[][]
{
new object[] { "Paul", 34, "Paul lives in Vermont, VA." },
new object[] { "Victor", 29, "Victor: Funny guy" },
new object[] { "Maria", 31, "" }
}
)
};
yield return new object[] // Quotations in Data
{
"""
Name,Age,Description
Paul,34,"Paul lives in ""Vermont VA""."
Victor,29,"Victor: Funny guy"
Maria,31,
""",
',',
new Type[] { typeof(string), typeof(int), typeof(string) },
new LoadCsvVerifyingHelper(
3,
3,
new string[] { "Name", "Age", "Description" },
new Type[] { typeof(string), typeof(int), typeof(string) },
new object[][]
{
new object[] { "Paul", 34, """Paul lives in "Vermont VA".""" },
new object[] { "Victor", 29, "Victor: Funny guy" },
new object[] { "Maria", 31, "" }
}
)
};
yield return new object[] // Quotations in Header
{
"""
Name,Age,"De""script""ion"
Paul,34,"Paul lives in Vermont, VA."
Victor,29,"Victor: Funny guy"
Maria,31,
""",
',',
new Type[] { typeof(string), typeof(int), typeof(string) },
new LoadCsvVerifyingHelper(
3,
3,
new string[] { "Name", "Age", """De"script"ion""" },
new Type[] { typeof(string), typeof(int), typeof(string) },
new object[][]
{
Expand Down
1 change: 1 addition & 0 deletions test/Microsoft.Data.Analysis.Tests/Microsoft.Data.Analysis.Tests.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<NoWarn>$(NoWarn);MSML_ParameterLocalVarName;MSML_PrivateFieldName;MSML_ExtendBaseTestClass;MSML_GeneralName</NoWarn>
<LangVersion>preview</LangVersion>
</PropertyGroup>

<ItemGroup>
Expand Down