Allow to define CultureInfo for parsing values on reading DataFrame from csv (#6782)

asmirnov82 · web-flow · commit d9dbf99d97aa · 2023-08-31T21:36:46.000-06:00
* Use CultureInfo for parsing values in csv file

* Fix merge issues
diff --git a/src/Microsoft.Data.Analysis/DataFrame.IO.cs b/src/Microsoft.Data.Analysis/DataFrame.IO.cs
@@ -98,18 +98,21 @@ private static Type MaxKind(Type a, Type b)
         /// <param name="guessRows">number of rows used to guess types</param>
         /// <param name="addIndexColumn">add one column with the row index</param>
         /// <param name="encoding">The character encoding. Defaults to UTF8 if not specified</param>
+        /// <param name="renameDuplicatedColumns">If set to true, columns with repeated names are auto-renamed.</param>
+        /// <param name="cultureInfo">culture info for formatting values</param>
         /// <returns>DataFrame</returns>
         public static DataFrame LoadCsv(string filename,
                                 char separator = ',', bool header = true,
                                 string[] columnNames = null, Type[] dataTypes = null,
                                 int numRows = -1, int guessRows = 10,
-                                bool addIndexColumn = false, Encoding encoding = null)
+                                bool addIndexColumn = false, Encoding encoding = null,
+                                bool renameDuplicatedColumns = false, CultureInfo cultureInfo = null)
         {
             using (Stream fileStream = new FileStream(filename, FileMode.Open))
             {
                 return LoadCsv(fileStream,
                                   separator: separator, header: header, columnNames: columnNames, dataTypes: dataTypes, numberOfRowsToRead: numRows,
-                                  guessRows: guessRows, addIndexColumn: addIndexColumn, encoding: encoding);
+                                  guessRows: guessRows, addIndexColumn: addIndexColumn, encoding: encoding, renameDuplicatedColumns: renameDuplicatedColumns, cultureInfo: cultureInfo);
             }
         }
 
@@ -351,8 +354,14 @@ private static DataFrame ReadCsvLinesIntoDataFrame(WrappedStreamReaderOrStringRe
                                 char separator = ',', bool header = true,
                                 string[] columnNames = null, Type[] dataTypes = null,
                                 long numberOfRowsToRead = -1, int guessRows = 10, bool addIndexColumn = false,
-                                bool renameDuplicatedColumns = false)
+                                bool renameDuplicatedColumns = false,
+                                CultureInfo cultureInfo = null)
         {
+            if (cultureInfo == null)
+            {
+                cultureInfo = CultureInfo.CurrentCulture;
+            }
+
             if (dataTypes == null && guessRows <= 0)
             {
                 throw new ArgumentException(string.Format(Strings.ExpectedEitherGuessRowsOrDataTypes, nameof(guessRows), nameof(dataTypes)));
@@ -452,7 +461,7 @@ private static DataFrame ReadCsvLinesIntoDataFrame(WrappedStreamReaderOrStringRe
                     }
                     else
                     {
-                        ret.Append(fields, inPlace: true);
+                        ret.Append(fields, inPlace: true, cultureInfo: cultureInfo);
                     }
                     ++rowline;
                 }
@@ -508,7 +517,6 @@ public TextReader GetTextReader()
                 }
 
             }
-
         }
 
         /// <summary>
@@ -522,14 +530,18 @@ public TextReader GetTextReader()
         /// <param name="numberOfRowsToRead">number of rows to read not including the header(if present)</param>
         /// <param name="guessRows">number of rows used to guess types</param>
         /// <param name="addIndexColumn">add one column with the row index</param>
+        /// <param name="renameDuplicatedColumns">If set to true, columns with repeated names are auto-renamed.</param>
+        /// <param name="cultureInfo">culture info for formatting values</param>
         /// <returns><see cref="DataFrame"/></returns>
         public static DataFrame LoadCsvFromString(string csvString,
                                 char separator = ',', bool header = true,
                                 string[] columnNames = null, Type[] dataTypes = null,
-                                long numberOfRowsToRead = -1, int guessRows = 10, bool addIndexColumn = false)
+                                long numberOfRowsToRead = -1, int guessRows = 10, bool addIndexColumn = false,
+                                bool renameDuplicatedColumns = false,
+                                CultureInfo cultureInfo = null)
         {
             WrappedStreamReaderOrStringReader wrappedStreamReaderOrStringReader = new WrappedStreamReaderOrStringReader(csvString);
-            return ReadCsvLinesIntoDataFrame(wrappedStreamReaderOrStringReader, separator, header, columnNames, dataTypes, numberOfRowsToRead, guessRows, addIndexColumn);
+            return ReadCsvLinesIntoDataFrame(wrappedStreamReaderOrStringReader, separator, header, columnNames, dataTypes, numberOfRowsToRead, guessRows, addIndexColumn, renameDuplicatedColumns, cultureInfo);
         }
 
         /// <summary>
@@ -545,12 +557,13 @@ public static DataFrame LoadCsvFromString(string csvString,
         /// <param name="addIndexColumn">add one column with the row index</param>
         /// <param name="encoding">The character encoding. Defaults to UTF8 if not specified</param>
         /// <param name="renameDuplicatedColumns">If set to true, columns with repeated names are auto-renamed.</param>
+        /// <param name="cultureInfo">culture info for formatting values</param>
         /// <returns><see cref="DataFrame"/></returns>
         public static DataFrame LoadCsv(Stream csvStream,
                                 char separator = ',', bool header = true,
                                 string[] columnNames = null, Type[] dataTypes = null,
                                 long numberOfRowsToRead = -1, int guessRows = 10, bool addIndexColumn = false,
-                                Encoding encoding = null, bool renameDuplicatedColumns = false)
+                                Encoding encoding = null, bool renameDuplicatedColumns = false, CultureInfo cultureInfo = null)
         {
             if (!csvStream.CanSeek)
             {
@@ -563,7 +576,7 @@ public static DataFrame LoadCsv(Stream csvStream,
             }
 
             WrappedStreamReaderOrStringReader wrappedStreamReaderOrStringReader = new WrappedStreamReaderOrStringReader(csvStream, encoding ?? Encoding.UTF8);
-            return ReadCsvLinesIntoDataFrame(wrappedStreamReaderOrStringReader, separator, header, columnNames, dataTypes, numberOfRowsToRead, guessRows, addIndexColumn, renameDuplicatedColumns);
+            return ReadCsvLinesIntoDataFrame(wrappedStreamReaderOrStringReader, separator, header, columnNames, dataTypes, numberOfRowsToRead, guessRows, addIndexColumn, renameDuplicatedColumns, cultureInfo);
         }
 
         /// <summary>
diff --git a/src/Microsoft.Data.Analysis/DataFrame.cs b/src/Microsoft.Data.Analysis/DataFrame.cs
@@ -5,6 +5,7 @@
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
+using System.Globalization;
 using System.Linq;
 using System.Text;
 
@@ -484,12 +485,13 @@ private void ResizeByOneAndAppend(DataFrameColumn column, object value)
         /// <remarks>If a <seealso cref="DataFrameRow"/> in <paramref name="rows"/> is null, a null value is appended to each column</remarks>
         /// <param name="rows">The rows to be appended to this DataFrame </param> 
         /// <param name="inPlace">If set, appends <paramref name="rows"/> in place. Otherwise, a new DataFrame is returned with the <paramref name="rows"/> appended</param>
-        public DataFrame Append(IEnumerable<DataFrameRow> rows, bool inPlace = false)
+        /// <param name="cultureInfo">culture info for formatting values</param>
+        public DataFrame Append(IEnumerable<DataFrameRow> rows, bool inPlace = false, CultureInfo cultureInfo = null)
         {
             DataFrame ret = inPlace ? this : Clone();
             foreach (DataFrameRow row in rows)
             {
-                ret.Append(row, inPlace: true);
+                ret.Append(row, inPlace: true, cultureInfo: cultureInfo);
             }
             return ret;
         }
@@ -501,8 +503,14 @@ public DataFrame Append(IEnumerable<DataFrameRow> rows, bool inPlace = false)
         /// <remarks>If <paramref name="row"/> is null, a null value is appended to each column</remarks>
         /// <param name="row"></param> 
         /// <param name="inPlace">If set, appends a <paramref name="row"/> in place. Otherwise, a new DataFrame is returned with an appended <paramref name="row"/> </param>
-        public DataFrame Append(IEnumerable<object> row = null, bool inPlace = false)
+        /// <param name="cultureInfo">culture info for formatting values</param>
+        public DataFrame Append(IEnumerable<object> row = null, bool inPlace = false, CultureInfo cultureInfo = null)
         {
+            if (cultureInfo == null)
+            {
+                cultureInfo = CultureInfo.CurrentCulture;
+            }
+
             DataFrame ret = inPlace ? this : Clone();
             IEnumerator<DataFrameColumn> columnEnumerator = ret.Columns.GetEnumerator();
             bool columnMoveNext = columnEnumerator.MoveNext();
@@ -530,7 +538,7 @@ public DataFrame Append(IEnumerable<object> row = null, bool inPlace = false)
                     }
                     if (value != null)
                     {
-                        value = Convert.ChangeType(value, column.DataType);
+                        value = Convert.ChangeType(value, column.DataType, cultureInfo);
 
                         if (value is null)
                         {
diff --git a/test/Microsoft.Data.Analysis.Tests/DataFrame.IOTests.cs b/test/Microsoft.Data.Analysis.Tests/DataFrame.IOTests.cs
@@ -14,6 +14,7 @@
 using System.Data.SQLite.EF6;
 using Xunit;
 using Microsoft.ML.TestFramework.Attributes;
+using System.Threading;
 
 namespace Microsoft.Data.Analysis.Tests
 {
@@ -154,6 +155,42 @@ void ReducedRowsTest(DataFrame reducedRows)
             ReducedRowsTest(csvDf);
         }
 
+        [Fact]
+        public void TestReadCsvWithHeaderCultureInfoAndSeparator()
+        {
+            string data = @$"vendor_id;rate_code;passenger_count;trip_time_in_secs;trip_distance;payment_type;fare_amount
+CMT;1;1;1271;3,8;CRD;17,5
+CMT;1;1;474;1,5;CRD;8
+CMT;1;1;637;1,4;CRD;8,5
+CMT;1;1;181;0,6;CSH;4,5";
+
+            void RegularTest(DataFrame df)
+            {
+                Assert.Equal(4, df.Rows.Count);
+                Assert.Equal(7, df.Columns.Count);
+
+                Assert.Equal(3.8f, (float)df["trip_distance"][0]);
+                Assert.Equal(17.5f, (float)df["fare_amount"][0]);
+
+                Assert.Equal(1.5f, (float)df["trip_distance"][1]);
+                Assert.Equal(8f, (float)df["fare_amount"][1]);
+
+                Assert.Equal(1.4f, (float)df["trip_distance"][2]);
+                Assert.Equal(8.5f, (float)df["fare_amount"][2]);
+
+                VerifyColumnTypes(df);
+            }
+
+            // de-DE has ',' as decimal separator
+            var cultureInfo = new CultureInfo("de-DE");
+            DataFrame df = DataFrame.LoadCsv(GetStream(data), separator: ';', cultureInfo: cultureInfo);
+
+            RegularTest(df);
+
+            DataFrame csvDf = DataFrame.LoadCsvFromString(data, separator: ';', cultureInfo: cultureInfo);
+            RegularTest(csvDf);
+        }
+
         [Fact]
         public void TestReadCsvWithHeaderAndDuplicatedColumns_WithoutRenaming()
         {

Original file line number	Diff line number	Diff line change
`@@ -98,18 +98,21 @@ private static Type MaxKind(Type a, Type b)`
`98`	`98`	`/// <param name="guessRows">number of rows used to guess types</param>`
`99`	`99`	`/// <param name="addIndexColumn">add one column with the row index</param>`
`100`	`100`	`/// <param name="encoding">The character encoding. Defaults to UTF8 if not specified</param>`
	`101`	`+ /// <param name="renameDuplicatedColumns">If set to true, columns with repeated names are auto-renamed.</param>`
	`102`	`+ /// <param name="cultureInfo">culture info for formatting values</param>`
`101`	`103`	`/// <returns>DataFrame</returns>`
`102`	`104`	`public static DataFrame LoadCsv(string filename,`
`103`	`105`	`char separator = ',', bool header = true,`
`104`	`106`	`string[] columnNames = null, Type[] dataTypes = null,`
`105`	`107`	`int numRows = -1, int guessRows = 10,`
`106`		`- bool addIndexColumn = false, Encoding encoding = null)`
	`108`	`+ bool addIndexColumn = false, Encoding encoding = null,`
	`109`	`+ bool renameDuplicatedColumns = false, CultureInfo cultureInfo = null)`
`107`	`110`	`{`
`108`	`111`	`using (Stream fileStream = new FileStream(filename, FileMode.Open))`
`109`	`112`	`{`
`110`	`113`	`return LoadCsv(fileStream,`
`111`	`114`	`separator: separator, header: header, columnNames: columnNames, dataTypes: dataTypes, numberOfRowsToRead: numRows,`
`112`		`- guessRows: guessRows, addIndexColumn: addIndexColumn, encoding: encoding);`
	`115`	`+ guessRows: guessRows, addIndexColumn: addIndexColumn, encoding: encoding, renameDuplicatedColumns: renameDuplicatedColumns, cultureInfo: cultureInfo);`
`113`	`116`	`}`
`114`	`117`	`}`
`115`	`118`
`@@ -351,8 +354,14 @@ private static DataFrame ReadCsvLinesIntoDataFrame(WrappedStreamReaderOrStringRe`
`351`	`354`	`char separator = ',', bool header = true,`
`352`	`355`	`string[] columnNames = null, Type[] dataTypes = null,`
`353`	`356`	`long numberOfRowsToRead = -1, int guessRows = 10, bool addIndexColumn = false,`
`354`		`- bool renameDuplicatedColumns = false)`
	`357`	`+ bool renameDuplicatedColumns = false,`
	`358`	`+ CultureInfo cultureInfo = null)`
`355`	`359`	`{`
	`360`	`+ if (cultureInfo == null)`
	`361`	`+ {`
	`362`	`+ cultureInfo = CultureInfo.CurrentCulture;`
	`363`	`+ }`
	`364`	`+`
`356`	`365`	`if (dataTypes == null && guessRows <= 0)`
`357`	`366`	`{`
`358`	`367`	`throw new ArgumentException(string.Format(Strings.ExpectedEitherGuessRowsOrDataTypes, nameof(guessRows), nameof(dataTypes)));`
`@@ -452,7 +461,7 @@ private static DataFrame ReadCsvLinesIntoDataFrame(WrappedStreamReaderOrStringRe`
`452`	`461`	`}`
`453`	`462`	`else`
`454`	`463`	`{`
`455`		`- ret.Append(fields, inPlace: true);`
	`464`	`+ ret.Append(fields, inPlace: true, cultureInfo: cultureInfo);`
`456`	`465`	`}`
`457`	`466`	`++rowline;`
`458`	`467`	`}`
`@@ -508,7 +517,6 @@ public TextReader GetTextReader()`
`508`	`517`	`}`
`509`	`518`
`510`	`519`	`}`
`511`		`-`
`512`	`520`	`}`
`513`	`521`
`514`	`522`	`/// <summary>`
`@@ -522,14 +530,18 @@ public TextReader GetTextReader()`
`522`	`530`	`/// <param name="numberOfRowsToRead">number of rows to read not including the header(if present)</param>`
`523`	`531`	`/// <param name="guessRows">number of rows used to guess types</param>`
`524`	`532`	`/// <param name="addIndexColumn">add one column with the row index</param>`
	`533`	`+ /// <param name="renameDuplicatedColumns">If set to true, columns with repeated names are auto-renamed.</param>`
	`534`	`+ /// <param name="cultureInfo">culture info for formatting values</param>`
`525`	`535`	`/// <returns><see cref="DataFrame"/></returns>`
`526`	`536`	`public static DataFrame LoadCsvFromString(string csvString,`
`527`	`537`	`char separator = ',', bool header = true,`
`528`	`538`	`string[] columnNames = null, Type[] dataTypes = null,`
`529`		`- long numberOfRowsToRead = -1, int guessRows = 10, bool addIndexColumn = false)`
	`539`	`+ long numberOfRowsToRead = -1, int guessRows = 10, bool addIndexColumn = false,`
	`540`	`+ bool renameDuplicatedColumns = false,`
	`541`	`+ CultureInfo cultureInfo = null)`
`530`	`542`	`{`
`531`	`543`	`WrappedStreamReaderOrStringReader wrappedStreamReaderOrStringReader = new WrappedStreamReaderOrStringReader(csvString);`
`532`		`- return ReadCsvLinesIntoDataFrame(wrappedStreamReaderOrStringReader, separator, header, columnNames, dataTypes, numberOfRowsToRead, guessRows, addIndexColumn);`
	`544`	`+ return ReadCsvLinesIntoDataFrame(wrappedStreamReaderOrStringReader, separator, header, columnNames, dataTypes, numberOfRowsToRead, guessRows, addIndexColumn, renameDuplicatedColumns, cultureInfo);`
`533`	`545`	`}`
`534`	`546`
`535`	`547`	`/// <summary>`
`@@ -545,12 +557,13 @@ public static DataFrame LoadCsvFromString(string csvString,`
`545`	`557`	`/// <param name="addIndexColumn">add one column with the row index</param>`
`546`	`558`	`/// <param name="encoding">The character encoding. Defaults to UTF8 if not specified</param>`
`547`	`559`	`/// <param name="renameDuplicatedColumns">If set to true, columns with repeated names are auto-renamed.</param>`
	`560`	`+ /// <param name="cultureInfo">culture info for formatting values</param>`
`548`	`561`	`/// <returns><see cref="DataFrame"/></returns>`
`549`	`562`	`public static DataFrame LoadCsv(Stream csvStream,`
`550`	`563`	`char separator = ',', bool header = true,`
`551`	`564`	`string[] columnNames = null, Type[] dataTypes = null,`
`552`	`565`	`long numberOfRowsToRead = -1, int guessRows = 10, bool addIndexColumn = false,`
`553`		`- Encoding encoding = null, bool renameDuplicatedColumns = false)`
	`566`	`+ Encoding encoding = null, bool renameDuplicatedColumns = false, CultureInfo cultureInfo = null)`
`554`	`567`	`{`
`555`	`568`	`if (!csvStream.CanSeek)`
`556`	`569`	`{`
`@@ -563,7 +576,7 @@ public static DataFrame LoadCsv(Stream csvStream,`
`563`	`576`	`}`
`564`	`577`
`565`	`578`	`WrappedStreamReaderOrStringReader wrappedStreamReaderOrStringReader = new WrappedStreamReaderOrStringReader(csvStream, encoding ?? Encoding.UTF8);`
`566`		`- return ReadCsvLinesIntoDataFrame(wrappedStreamReaderOrStringReader, separator, header, columnNames, dataTypes, numberOfRowsToRead, guessRows, addIndexColumn, renameDuplicatedColumns);`
	`579`	`+ return ReadCsvLinesIntoDataFrame(wrappedStreamReaderOrStringReader, separator, header, columnNames, dataTypes, numberOfRowsToRead, guessRows, addIndexColumn, renameDuplicatedColumns, cultureInfo);`
`567`	`580`	`}`
`568`	`581`
`569`	`582`	`/// <summary>`
Original file line number	Diff line number	Diff line change
`@@ -5,6 +5,7 @@`
`5`	`5`	`using System;`
`6`	`6`	`using System.Collections.Generic;`
`7`	`7`	`using System.Diagnostics;`
	`8`	`+using System.Globalization;`
`8`	`9`	`using System.Linq;`
`9`	`10`	`using System.Text;`
`10`	`11`
`@@ -484,12 +485,13 @@ private void ResizeByOneAndAppend(DataFrameColumn column, object value)`
`484`	`485`	`/// <remarks>If a <seealso cref="DataFrameRow"/> in <paramref name="rows"/> is null, a null value is appended to each column</remarks>`
`485`	`486`	`/// <param name="rows">The rows to be appended to this DataFrame </param>`
`486`	`487`	`/// <param name="inPlace">If set, appends <paramref name="rows"/> in place. Otherwise, a new DataFrame is returned with the <paramref name="rows"/> appended</param>`
`487`		`- public DataFrame Append(IEnumerable<DataFrameRow> rows, bool inPlace = false)`
	`488`	`+ /// <param name="cultureInfo">culture info for formatting values</param>`
	`489`	`+ public DataFrame Append(IEnumerable<DataFrameRow> rows, bool inPlace = false, CultureInfo cultureInfo = null)`
`488`	`490`	`{`
`489`	`491`	`DataFrame ret = inPlace ? this : Clone();`
`490`	`492`	`foreach (DataFrameRow row in rows)`
`491`	`493`	`{`
`492`		`- ret.Append(row, inPlace: true);`
	`494`	`+ ret.Append(row, inPlace: true, cultureInfo: cultureInfo);`
`493`	`495`	`}`
`494`	`496`	`return ret;`
`495`	`497`	`}`
`@@ -501,8 +503,14 @@ public DataFrame Append(IEnumerable<DataFrameRow> rows, bool inPlace = false)`
`501`	`503`	`/// <remarks>If <paramref name="row"/> is null, a null value is appended to each column</remarks>`
`502`	`504`	`/// <param name="row"></param>`
`503`	`505`	`/// <param name="inPlace">If set, appends a <paramref name="row"/> in place. Otherwise, a new DataFrame is returned with an appended <paramref name="row"/> </param>`
`504`		`- public DataFrame Append(IEnumerable<object> row = null, bool inPlace = false)`
	`506`	`+ /// <param name="cultureInfo">culture info for formatting values</param>`
	`507`	`+ public DataFrame Append(IEnumerable<object> row = null, bool inPlace = false, CultureInfo cultureInfo = null)`
`505`	`508`	`{`
	`509`	`+ if (cultureInfo == null)`
	`510`	`+ {`
	`511`	`+ cultureInfo = CultureInfo.CurrentCulture;`
	`512`	`+ }`
	`513`	`+`
`506`	`514`	`DataFrame ret = inPlace ? this : Clone();`
`507`	`515`	`IEnumerator<DataFrameColumn> columnEnumerator = ret.Columns.GetEnumerator();`
`508`	`516`	`bool columnMoveNext = columnEnumerator.MoveNext();`
`@@ -530,7 +538,7 @@ public DataFrame Append(IEnumerable<object> row = null, bool inPlace = false)`
`530`	`538`	`}`
`531`	`539`	`if (value != null)`
`532`	`540`	`{`
`533`		`- value = Convert.ChangeType(value, column.DataType);`
	`541`	`+ value = Convert.ChangeType(value, column.DataType, cultureInfo);`
`534`	`542`
`535`	`543`	`if (value is null)`
`536`	`544`	`{`