From 4f3ae7fba9570f992b345db268802df4641800b9 Mon Sep 17 00:00:00 2001 From: Andrei Faber Date: Tue, 12 Oct 2021 12:38:00 -0500 Subject: [PATCH 01/14] refactoring - removed copy/paste in DataFrame.CreateColumn() --- src/Microsoft.Data.Analysis/DataFrame.IO.cs | 37 ++++++++++++--------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/src/Microsoft.Data.Analysis/DataFrame.IO.cs b/src/Microsoft.Data.Analysis/DataFrame.IO.cs index 1253ed6149..4567f0d7e7 100644 --- a/src/Microsoft.Data.Analysis/DataFrame.IO.cs +++ b/src/Microsoft.Data.Analysis/DataFrame.IO.cs @@ -115,68 +115,68 @@ private static string GetColumnName(string[] columnNames, int columnIndex) return columnNames == null ? "Column" + columnIndex.ToString() : columnNames[columnIndex]; } - private static DataFrameColumn CreateColumn(Type kind, string[] columnNames, int columnIndex) + private static DataFrameColumn CreateColumn(Type kind, string columnName) { DataFrameColumn ret; if (kind == typeof(bool)) { - ret = new BooleanDataFrameColumn(GetColumnName(columnNames, columnIndex)); + ret = new BooleanDataFrameColumn(columnName); } else if (kind == typeof(int)) { - ret = new Int32DataFrameColumn(GetColumnName(columnNames, columnIndex)); + ret = new Int32DataFrameColumn(columnName); } else if (kind == typeof(float)) { - ret = new SingleDataFrameColumn(GetColumnName(columnNames, columnIndex)); + ret = new SingleDataFrameColumn(columnName); } else if (kind == typeof(string)) { - ret = new StringDataFrameColumn(GetColumnName(columnNames, columnIndex), 0); + ret = new StringDataFrameColumn(columnName, 0); } else if (kind == typeof(long)) { - ret = new Int64DataFrameColumn(GetColumnName(columnNames, columnIndex)); + ret = new Int64DataFrameColumn(columnName); } else if (kind == typeof(decimal)) { - ret = new DecimalDataFrameColumn(GetColumnName(columnNames, columnIndex)); + ret = new DecimalDataFrameColumn(columnName); } else if (kind == typeof(byte)) { - ret = new ByteDataFrameColumn(GetColumnName(columnNames, columnIndex)); + ret = new ByteDataFrameColumn(columnName); } else if (kind == typeof(char)) { - ret = new CharDataFrameColumn(GetColumnName(columnNames, columnIndex)); + ret = new CharDataFrameColumn(columnName); } else if (kind == typeof(double)) { - ret = new DoubleDataFrameColumn(GetColumnName(columnNames, columnIndex)); + ret = new DoubleDataFrameColumn(columnName); } else if (kind == typeof(sbyte)) { - ret = new SByteDataFrameColumn(GetColumnName(columnNames, columnIndex)); + ret = new SByteDataFrameColumn(columnName); } else if (kind == typeof(short)) { - ret = new Int16DataFrameColumn(GetColumnName(columnNames, columnIndex)); + ret = new Int16DataFrameColumn(columnName); } else if (kind == typeof(uint)) { - ret = new UInt32DataFrameColumn(GetColumnName(columnNames, columnIndex)); + ret = new UInt32DataFrameColumn(columnName); } else if (kind == typeof(ulong)) { - ret = new UInt64DataFrameColumn(GetColumnName(columnNames, columnIndex)); + ret = new UInt64DataFrameColumn(columnName); } else if (kind == typeof(ushort)) { - ret = new UInt16DataFrameColumn(GetColumnName(columnNames, columnIndex)); + ret = new UInt16DataFrameColumn(columnName); } else if (kind == typeof(DateTime)) { - ret = new PrimitiveDataFrameColumn(GetColumnName(columnNames, columnIndex)); + ret = new PrimitiveDataFrameColumn(columnName); } else { @@ -185,6 +185,11 @@ private static DataFrameColumn CreateColumn(Type kind, string[] columnNames, int return ret; } + private static DataFrameColumn CreateColumn(Type kind, string[] columnNames, int columnIndex) + { + return CreateColumn(kind, GetColumnName(columnNames, columnIndex)); + } + private static DataFrame ReadCsvLinesIntoDataFrame(WrappedStreamReaderOrStringReader wrappedReader, char separator = ',', bool header = true, string[] columnNames = null, Type[] dataTypes = null, From b4809fc700385b7bb8ff8f70bd2f5adbe765ada2 Mon Sep 17 00:00:00 2001 From: Andrei Faber Date: Tue, 12 Oct 2021 12:51:31 -0500 Subject: [PATCH 02/14] added a universal loading method and export to DataTable --- src/Microsoft.Data.Analysis/DataFrame.IO.cs | 61 +++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/src/Microsoft.Data.Analysis/DataFrame.IO.cs b/src/Microsoft.Data.Analysis/DataFrame.IO.cs index 4567f0d7e7..aae7ed4d0c 100644 --- a/src/Microsoft.Data.Analysis/DataFrame.IO.cs +++ b/src/Microsoft.Data.Analysis/DataFrame.IO.cs @@ -5,6 +5,7 @@ using System; using System.Collections; using System.Collections.Generic; +using System.Data; using System.Globalization; using System.IO; using System.Text; @@ -110,6 +111,66 @@ public static DataFrame LoadCsv(string filename, } } + public static DataFrame LoadFrom(IEnumerable> vals, IList<(string, Type)> columnInfos) + { + var columnsCount = columnInfos.Count; + var columns = new List(columnsCount); + + foreach (var (name, type) in columnInfos) + { + var column = CreateColumn(type, name); + columns.Add(column); + } + + var res = new DataFrame(columns); + + foreach (var items in vals) + { + for (var c = 0; c < items.Count; c++) + { + items[c] = items[c]; + } + res.Append(items, inPlace: true); + } + + return res; + } + + public void SaveTo(DataTable table) + { + var columnsCount = Columns.Count; + + if (table.Columns.Count == 0) + { + foreach (var column in Columns) + { + table.Columns.Add(column.Name, column.DataType); + } + } + else + { + if (table.Columns.Count != columnsCount) + throw new ArgumentException(); + } + + var items = new object[columnsCount]; + foreach (var row in Rows) + { + for (var c = 0; c < columnsCount; c++) + { + items[c] = row[c]; + } + table.Rows.Add(items); + } + } + + public DataTable ToTable() + { + var res = new DataTable(); + SaveTo(res); + return res; + } + private static string GetColumnName(string[] columnNames, int columnIndex) { return columnNames == null ? "Column" + columnIndex.ToString() : columnNames[columnIndex]; From 6a9f89f943d34d3cc18c3ab156c046a48807f55c Mon Sep 17 00:00:00 2001 From: Andrei Faber Date: Tue, 12 Oct 2021 16:59:15 -0500 Subject: [PATCH 03/14] added tests for new loading/saving methods in DataFrame --- .../DataFrame.IOTests.cs | 52 +++++++++++++++++++ .../Microsoft.Data.Analysis.Tests.csproj | 1 + 2 files changed, 53 insertions(+) diff --git a/test/Microsoft.Data.Analysis.Tests/DataFrame.IOTests.cs b/test/Microsoft.Data.Analysis.Tests/DataFrame.IOTests.cs index b35d15b207..163cf8df1e 100644 --- a/test/Microsoft.Data.Analysis.Tests/DataFrame.IOTests.cs +++ b/test/Microsoft.Data.Analysis.Tests/DataFrame.IOTests.cs @@ -3,6 +3,8 @@ // See the LICENSE file in the project root for more information. using System; +using System.Collections.Generic; +using System.Data; using System.Globalization; using System.IO; using System.Linq; @@ -975,5 +977,55 @@ public void TestMixedDataTypesInCsv() Assert.Equal("", emptyColumn[i]); } } + + [Fact] + public void TestLoadFromEnumerable() + { + var (columns, vals) = GetTestData(); + var dataFrame = DataFrame.LoadFrom(vals, columns); + + var resColumns = dataFrame.Columns.Select(column => (column.Name, column.DataType)).ToArray(); + Assert.Equal(columns, resColumns); + + var resVals = dataFrame.Rows.Select(row => row.ToArray()).ToArray(); + Assert.Equal(vals, resVals); + } + + [Fact] + public void TestSaveToDataTable() + { + var (columns, vals) = GetTestData(); + var dataFrame = DataFrame.LoadFrom(vals, columns); + + var table = dataFrame.ToTable(); + + var resColumns = table.Columns.Cast().Select(column => (column.ColumnName, column.DataType)).ToArray(); + Assert.Equal(columns, resColumns); + + var resVals = table.Rows.Cast().Select(row => row.ItemArray).ToArray(); + Assert.Equal(vals, resVals); + } + + static ((string name, Type type)[] columns, object[][] vals) GetTestData() + { + const int RowsCount = 10_000; + + var columns = new[] + { + ("ID", typeof(int)), + ("Text", typeof(string)) + }; + + var vals = new object[RowsCount][]; + for (var i = 0; i < RowsCount; i++) + { + var row = new object[columns.Length]; + row[0] = i; + row[1] = $"test {i}"; + vals[i] = row; + } + + return (columns, vals); + } } } diff --git a/test/Microsoft.Data.Analysis.Tests/Microsoft.Data.Analysis.Tests.csproj b/test/Microsoft.Data.Analysis.Tests/Microsoft.Data.Analysis.Tests.csproj index 8032812924..667d50404e 100644 --- a/test/Microsoft.Data.Analysis.Tests/Microsoft.Data.Analysis.Tests.csproj +++ b/test/Microsoft.Data.Analysis.Tests/Microsoft.Data.Analysis.Tests.csproj @@ -1,6 +1,7 @@  $(NoWarn);MSML_ParameterLocalVarName;MSML_PrivateFieldName;MSML_ExtendBaseTestClass;MSML_GeneralName + netcoreapp3.1 From d1ee9e5a68b926317268dda946ef802052dd1b38 Mon Sep 17 00:00:00 2001 From: Andrei Faber Date: Wed, 13 Oct 2021 12:04:08 -0500 Subject: [PATCH 04/14] improved error handling - DataFrame.LoadFrom() --- src/Microsoft.Data.Analysis/DataFrame.IO.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Microsoft.Data.Analysis/DataFrame.IO.cs b/src/Microsoft.Data.Analysis/DataFrame.IO.cs index aae7ed4d0c..577d5e434e 100644 --- a/src/Microsoft.Data.Analysis/DataFrame.IO.cs +++ b/src/Microsoft.Data.Analysis/DataFrame.IO.cs @@ -151,6 +151,11 @@ public void SaveTo(DataTable table) { if (table.Columns.Count != columnsCount) throw new ArgumentException(); + for (var c = 0; c < columnsCount; c++) + { + if (table.Columns[c].DataType != Columns[c].DataType) + throw new ArgumentException(); + } } var items = new object[columnsCount]; From 1e18c352d5e557555379ad3ae1f684bfa87af1c0 Mon Sep 17 00:00:00 2001 From: Andrei Faber Date: Wed, 13 Oct 2021 16:28:39 -0500 Subject: [PATCH 05/14] DataFrame - importing and exporting data using ADO.NET providers --- src/Microsoft.Data.Analysis/DataFrame.IO.cs | 84 ++++++++++++++++++- src/Microsoft.Data.Analysis/Extensions.cs | 37 ++++++++ .../DataFrame.IOTests.cs | 82 ++++++++++++++++-- .../Microsoft.Data.Analysis.Tests.csproj | 4 + 4 files changed, 198 insertions(+), 9 deletions(-) create mode 100644 src/Microsoft.Data.Analysis/Extensions.cs diff --git a/src/Microsoft.Data.Analysis/DataFrame.IO.cs b/src/Microsoft.Data.Analysis/DataFrame.IO.cs index 577d5e434e..4c8e181018 100644 --- a/src/Microsoft.Data.Analysis/DataFrame.IO.cs +++ b/src/Microsoft.Data.Analysis/DataFrame.IO.cs @@ -6,9 +6,11 @@ using System.Collections; using System.Collections.Generic; using System.Data; +using System.Data.Common; using System.Globalization; using System.IO; using System.Text; +using System.Threading.Tasks; namespace Microsoft.Data.Analysis { @@ -163,7 +165,7 @@ public void SaveTo(DataTable table) { for (var c = 0; c < columnsCount; c++) { - items[c] = row[c]; + items[c] = row[c] ?? DBNull.Value; } table.Rows.Add(items); } @@ -176,6 +178,86 @@ public DataTable ToTable() return res; } + public static DataFrame FromSchema(DbDataReader reader) + { + var columnsCount = reader.FieldCount; + var columns = new DataFrameColumn[columnsCount]; + + for (var c = 0; c < columnsCount; c++) + { + var type = reader.GetFieldType(c); + var name = reader.GetName(c); + var column = CreateColumn(type, name); + columns[c] = column; + } + + var res = new DataFrame(columns); + return res; + } + + public static async Task LoadFrom(DbDataReader reader) + { + var res = FromSchema(reader); + var columnsCount = reader.FieldCount; + + var items = new object[columnsCount]; + while (await reader.ReadAsync()) + { + for (var c = 0; c < columnsCount; c++) + { + items[c] = reader.IsDBNull(c) + ? null + : reader[c]; + } + res.Append(items, inPlace: true); + } + + reader.Close(); + + return res; + } + + public static async Task LoadFrom(DbDataAdapter adapter) + { + using var reader = adapter.SelectCommand.ExecuteReader(); + return await LoadFrom(reader); + } + + public void SaveTo(DbDataAdapter dataAdapter, DbProviderFactory factory) + { + using var commandBuilder = factory.CreateCommandBuilder(); + commandBuilder.DataAdapter = dataAdapter; + dataAdapter.InsertCommand = commandBuilder.GetInsertCommand(); + dataAdapter.UpdateCommand = commandBuilder.GetUpdateCommand(); + dataAdapter.DeleteCommand = commandBuilder.GetDeleteCommand(); + + using var table = ToTable(); + + var connection = dataAdapter.SelectCommand.Connection; + var needClose = connection.TryOpen(); + + try + { + using var transaction = connection.BeginTransaction(); + try + { + dataAdapter.Update(table); + } + catch + { + transaction.Rollback(); + transaction.Dispose(); + throw; + } + transaction.Commit(); + } + finally + { + if (needClose) + connection.Close(); + } + } + private static string GetColumnName(string[] columnNames, int columnIndex) { return columnNames == null ? "Column" + columnIndex.ToString() : columnNames[columnIndex]; diff --git a/src/Microsoft.Data.Analysis/Extensions.cs b/src/Microsoft.Data.Analysis/Extensions.cs new file mode 100644 index 0000000000..3e3d20b4a4 --- /dev/null +++ b/src/Microsoft.Data.Analysis/Extensions.cs @@ -0,0 +1,37 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Data; +using System.Data.Common; +using System.Text; + +namespace Microsoft.Data.Analysis +{ + public static class Extensions + { + public static DbDataAdapter CreateDataAdapter(this DbProviderFactory factory, DbConnection connection, string tableName) + { + var query = connection.CreateCommand(); + query.CommandText = $"SELECT * FROM {tableName}"; + var res = factory.CreateDataAdapter(); + res.SelectCommand = query; + return res; + } + + public static bool TryOpen(this DbConnection connection) + { + if (connection.State == ConnectionState.Closed) + { + connection.Open(); + return true; + } + else + { + return false; + } + } + } +} diff --git a/test/Microsoft.Data.Analysis.Tests/DataFrame.IOTests.cs b/test/Microsoft.Data.Analysis.Tests/DataFrame.IOTests.cs index 163cf8df1e..c53d2dea47 100644 --- a/test/Microsoft.Data.Analysis.Tests/DataFrame.IOTests.cs +++ b/test/Microsoft.Data.Analysis.Tests/DataFrame.IOTests.cs @@ -5,10 +5,13 @@ using System; using System.Collections.Generic; using System.Data; +using System.Data.Common; using System.Globalization; using System.IO; using System.Linq; using System.Text; +using System.Data.SQLite; +using System.Data.SQLite.EF6; using Xunit; namespace Microsoft.Data.Analysis.Tests @@ -983,12 +986,7 @@ public void TestLoadFromEnumerable() { var (columns, vals) = GetTestData(); var dataFrame = DataFrame.LoadFrom(vals, columns); - - var resColumns = dataFrame.Columns.Select(column => (column.Name, column.DataType)).ToArray(); - Assert.Equal(columns, resColumns); - - var resVals = dataFrame.Rows.Select(row => row.ToArray()).ToArray(); - Assert.Equal(vals, resVals); + AssertEqual(dataFrame, columns, vals); } [Fact] @@ -1006,18 +1004,54 @@ public void TestSaveToDataTable() Assert.Equal(vals, resVals); } + [Fact] + public async void TestSQLite() + { + var (columns, vals) = GetTestData(); + var dataFrame = DataFrame.LoadFrom(vals, columns); + + try + { + var (factory, connection) = InitSQLiteDb(); + using (factory) + { + using (connection) + { + using var dataAdapter = factory.CreateDataAdapter(connection, TableName); + dataFrame.SaveTo(dataAdapter, factory); + + var resDataFrame = await DataFrame.LoadFrom(dataAdapter); + + AssertEqual(resDataFrame, columns, vals); + } + } + } + finally + { + CleanupSQLiteDb(); + } + } + + static void AssertEqual(DataFrame dataFrame, (string name, Type type)[] columns, object[][] vals) + { + var resColumns = dataFrame.Columns.Select(column => (column.Name, column.DataType)).ToArray(); + Assert.Equal(columns, resColumns); + var resVals = dataFrame.Rows.Select(row => row.ToArray()).ToArray(); + Assert.Equal(vals, resVals); + } + static ((string name, Type type)[] columns, object[][] vals) GetTestData() { const int RowsCount = 10_000; var columns = new[] { - ("ID", typeof(int)), + ("ID", typeof(long)), ("Text", typeof(string)) }; var vals = new object[RowsCount][]; - for (var i = 0; i < RowsCount; i++) + for (var i = 0L; i < RowsCount; i++) { var row = new object[columns.Length]; row[0] = i; @@ -1027,5 +1061,37 @@ public void TestSaveToDataTable() return (columns, vals); } + + static (SQLiteProviderFactory factory, DbConnection connection) InitSQLiteDb() + { + var connectionString = $"DataSource={SQLitePath};Version=3;New=True;Compress=True;"; + + SQLiteConnection.CreateFile(SQLitePath); + var factory = new SQLiteProviderFactory(); + + var connection = factory.CreateConnection(); + connection.ConnectionString = connectionString; + connection.Open(); + + using var command = connection.CreateCommand(); + command.CommandText = $"CREATE TABLE {TableName} (ID INTEGER NOT NULL PRIMARY KEY ASC, Text VARCHAR(25))"; + command.ExecuteNonQuery(); + + return (factory, connection); + } + + static void CleanupSQLiteDb() + { + if (File.Exists(SQLitePath)) + File.Delete(SQLitePath); + } + + static readonly string BasePath = + Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location); + + const string DbName = "TestDb"; + const string TableName = "TestTable"; + + static readonly string SQLitePath = $@"{BasePath}\{DbName}.sqlite"; } } diff --git a/test/Microsoft.Data.Analysis.Tests/Microsoft.Data.Analysis.Tests.csproj b/test/Microsoft.Data.Analysis.Tests/Microsoft.Data.Analysis.Tests.csproj index 667d50404e..ceff4f7c04 100644 --- a/test/Microsoft.Data.Analysis.Tests/Microsoft.Data.Analysis.Tests.csproj +++ b/test/Microsoft.Data.Analysis.Tests/Microsoft.Data.Analysis.Tests.csproj @@ -35,4 +35,8 @@ + + + + From fa4262d1f3f3bde45f862448abc7165c72d46987 Mon Sep 17 00:00:00 2001 From: Andrei Faber Date: Thu, 14 Oct 2021 10:40:03 -0500 Subject: [PATCH 06/14] DataFrame.LoadFrom() - use async --- src/Microsoft.Data.Analysis/DataFrame.IO.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.Data.Analysis/DataFrame.IO.cs b/src/Microsoft.Data.Analysis/DataFrame.IO.cs index 4c8e181018..d1afbceb2c 100644 --- a/src/Microsoft.Data.Analysis/DataFrame.IO.cs +++ b/src/Microsoft.Data.Analysis/DataFrame.IO.cs @@ -219,7 +219,7 @@ public static async Task LoadFrom(DbDataReader reader) public static async Task LoadFrom(DbDataAdapter adapter) { - using var reader = adapter.SelectCommand.ExecuteReader(); + using var reader = await adapter.SelectCommand.ExecuteReaderAsync(); return await LoadFrom(reader); } From f7fc6758fa0e324f8e703aa36eae9224a2df5550 Mon Sep 17 00:00:00 2001 From: Andrei Faber Date: Thu, 14 Oct 2021 10:48:08 -0500 Subject: [PATCH 07/14] DataFrame.LoadFrom() - minor refactorings --- test/Microsoft.Data.Analysis.Tests/DataFrame.IOTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/Microsoft.Data.Analysis.Tests/DataFrame.IOTests.cs b/test/Microsoft.Data.Analysis.Tests/DataFrame.IOTests.cs index c53d2dea47..2d4b83b816 100644 --- a/test/Microsoft.Data.Analysis.Tests/DataFrame.IOTests.cs +++ b/test/Microsoft.Data.Analysis.Tests/DataFrame.IOTests.cs @@ -995,7 +995,7 @@ public void TestSaveToDataTable() var (columns, vals) = GetTestData(); var dataFrame = DataFrame.LoadFrom(vals, columns); - var table = dataFrame.ToTable(); + using var table = dataFrame.ToTable(); var resColumns = table.Columns.Cast().Select(column => (column.ColumnName, column.DataType)).ToArray(); Assert.Equal(columns, resColumns); @@ -1087,7 +1087,7 @@ static void CleanupSQLiteDb() } static readonly string BasePath = - Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location); + Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location) + "\\"; const string DbName = "TestDb"; const string TableName = "TestTable"; From 3324d4437f1c61e47382c1b5fbb9aaacbc415d94 Mon Sep 17 00:00:00 2001 From: Michael Sharp <51342856+michaelgsharp@users.noreply.github.com> Date: Thu, 21 Oct 2021 10:32:02 -0700 Subject: [PATCH 08/14] Update Microsoft.Data.Analysis.Tests.csproj Changed version of System.Data.SQLite --- .../Microsoft.Data.Analysis.Tests.csproj | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/Microsoft.Data.Analysis.Tests/Microsoft.Data.Analysis.Tests.csproj b/test/Microsoft.Data.Analysis.Tests/Microsoft.Data.Analysis.Tests.csproj index ceff4f7c04..7f6c32c9e1 100644 --- a/test/Microsoft.Data.Analysis.Tests/Microsoft.Data.Analysis.Tests.csproj +++ b/test/Microsoft.Data.Analysis.Tests/Microsoft.Data.Analysis.Tests.csproj @@ -1,7 +1,6 @@  $(NoWarn);MSML_ParameterLocalVarName;MSML_PrivateFieldName;MSML_ExtendBaseTestClass;MSML_GeneralName - netcoreapp3.1 @@ -36,7 +35,7 @@ - + From fb0e49d02f0e26a96bace1ee6124e1a9d8721556 Mon Sep 17 00:00:00 2001 From: Michael Sharp <51342856+michaelgsharp@users.noreply.github.com> Date: Thu, 21 Oct 2021 10:58:36 -0700 Subject: [PATCH 09/14] Update Microsoft.Data.Analysis.Tests.csproj --- .../Microsoft.Data.Analysis.Tests.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Microsoft.Data.Analysis.Tests/Microsoft.Data.Analysis.Tests.csproj b/test/Microsoft.Data.Analysis.Tests/Microsoft.Data.Analysis.Tests.csproj index 7f6c32c9e1..da6de3c188 100644 --- a/test/Microsoft.Data.Analysis.Tests/Microsoft.Data.Analysis.Tests.csproj +++ b/test/Microsoft.Data.Analysis.Tests/Microsoft.Data.Analysis.Tests.csproj @@ -35,7 +35,7 @@ - + From a1154146a4ef8bc69ec4ea5e374983be43504104 Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Sat, 6 May 2023 14:35:44 -0600 Subject: [PATCH 10/14] fixed chown command --- eng/helix.proj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/helix.proj b/eng/helix.proj index b7d3b60086..ef55768a73 100644 --- a/eng/helix.proj +++ b/eng/helix.proj @@ -96,7 +96,7 @@ - $(HelixPreCommands);export ML_TEST_DATADIR=$HELIX_CORRELATION_PAYLOAD;export MICROSOFTML_RESOURCE_PATH=$HELIX_WORKITEM_ROOT;sudo chmod -R 777 $HELIX_WORKITEM_ROOT;sudo chown -R $(whoami) $HELIX_WORKITEM_ROOT + $(HelixPreCommands);export ML_TEST_DATADIR=$HELIX_CORRELATION_PAYLOAD;export MICROSOFTML_RESOURCE_PATH=$HELIX_WORKITEM_ROOT;sudo chmod -R 777 $HELIX_WORKITEM_ROOT;sudo chown -R $USER $HELIX_WORKITEM_ROOT $(HelixPreCommands);set ML_TEST_DATADIR=%HELIX_CORRELATION_PAYLOAD%;set MICROSOFTML_RESOURCE_PATH=%HELIX_WORKITEM_ROOT% $(HelixPreCommands);install_name_tool -change "/usr/local/opt/libomp/lib/libomp.dylib" "@loader_path/libomp.dylib" libSymSgdNative.dylib From ec1333273d2bbfb21db0f39e1951f3c77d093e85 Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Mon, 8 May 2023 11:41:16 -0600 Subject: [PATCH 11/14] sql db test path change --- test/Microsoft.Data.Analysis.Tests/DataFrame.IOTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Microsoft.Data.Analysis.Tests/DataFrame.IOTests.cs b/test/Microsoft.Data.Analysis.Tests/DataFrame.IOTests.cs index 33178a00e4..3138218ee5 100644 --- a/test/Microsoft.Data.Analysis.Tests/DataFrame.IOTests.cs +++ b/test/Microsoft.Data.Analysis.Tests/DataFrame.IOTests.cs @@ -1131,7 +1131,7 @@ static void CleanupSQLiteDb() } static readonly string BasePath = - Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location) + "\\"; + Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location) + "/"; const string DbName = "TestDb"; const string TableName = "TestTable"; From d0b02d866131ee8fca2de3b4b7466689d23de08d Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Mon, 8 May 2023 12:15:29 -0600 Subject: [PATCH 12/14] sql db test path change --- eng/Versions.props | 2 +- test/Microsoft.Data.Analysis.Tests/DataFrame.IOTests.cs | 2 +- .../Microsoft.Data.Analysis.Tests.csproj | 4 +++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/eng/Versions.props b/eng/Versions.props index 052539cb30..0c6252b6f5 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -87,7 +87,7 @@ 0.0.6-test 0.0.7-test 4.6.1 - 1.0.112.2 + 1.0.113 1.2.7 2.4.2 diff --git a/test/Microsoft.Data.Analysis.Tests/DataFrame.IOTests.cs b/test/Microsoft.Data.Analysis.Tests/DataFrame.IOTests.cs index 3138218ee5..db8acf7795 100644 --- a/test/Microsoft.Data.Analysis.Tests/DataFrame.IOTests.cs +++ b/test/Microsoft.Data.Analysis.Tests/DataFrame.IOTests.cs @@ -1136,7 +1136,7 @@ static void CleanupSQLiteDb() const string DbName = "TestDb"; const string TableName = "TestTable"; - static readonly string SQLitePath = $@"{BasePath}\{DbName}.sqlite"; + static readonly string SQLitePath = $@"{BasePath}/{DbName}.sqlite"; public readonly struct LoadCsvVerifyingHelper { diff --git a/test/Microsoft.Data.Analysis.Tests/Microsoft.Data.Analysis.Tests.csproj b/test/Microsoft.Data.Analysis.Tests/Microsoft.Data.Analysis.Tests.csproj index 6f9592a391..8358bc7126 100644 --- a/test/Microsoft.Data.Analysis.Tests/Microsoft.Data.Analysis.Tests.csproj +++ b/test/Microsoft.Data.Analysis.Tests/Microsoft.Data.Analysis.Tests.csproj @@ -36,7 +36,9 @@ - + + + From 9351a464e835b32c544ad009e71c5e07cd1dacff Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Mon, 8 May 2023 13:49:46 -0600 Subject: [PATCH 13/14] sql db test fix --- test/Microsoft.Data.Analysis.Tests/DataFrame.IOTests.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/Microsoft.Data.Analysis.Tests/DataFrame.IOTests.cs b/test/Microsoft.Data.Analysis.Tests/DataFrame.IOTests.cs index db8acf7795..398e849907 100644 --- a/test/Microsoft.Data.Analysis.Tests/DataFrame.IOTests.cs +++ b/test/Microsoft.Data.Analysis.Tests/DataFrame.IOTests.cs @@ -13,6 +13,7 @@ using System.Data.SQLite; using System.Data.SQLite.EF6; using Xunit; +using Microsoft.ML.TestFramework.Attributes; namespace Microsoft.Data.Analysis.Tests { @@ -1048,7 +1049,7 @@ public void TestSaveToDataTable() Assert.Equal(vals, resVals); } - [Fact] + [X86X64FactAttribute("The SQLite un-managed code, SQLite.interop, only supports x86/x64 architectures.")] public async void TestSQLite() { var (columns, vals) = GetTestData(); From 9bbeed2b034d56af80bdda3c9f1283af938a1f8d Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Mon, 8 May 2023 14:20:44 -0600 Subject: [PATCH 14/14] sql db test fix --- .../Microsoft.Data.Analysis.Tests.csproj | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/Microsoft.Data.Analysis.Tests/Microsoft.Data.Analysis.Tests.csproj b/test/Microsoft.Data.Analysis.Tests/Microsoft.Data.Analysis.Tests.csproj index 8358bc7126..2717befa32 100644 --- a/test/Microsoft.Data.Analysis.Tests/Microsoft.Data.Analysis.Tests.csproj +++ b/test/Microsoft.Data.Analysis.Tests/Microsoft.Data.Analysis.Tests.csproj @@ -8,6 +8,8 @@ + + @@ -32,12 +34,12 @@ - + - +