Merge remote-tracking branch 'origin/main' into 6110_unable_to_filter_by_null

Author: asmirnov82

.vsts-dotnet-ci.yml

@@ -2,6 +2,20 @@
 # ML.NET's PR validation build
 ################################################################################
 
+pr:
+  branches:
+    include:
+    - main
+    - feature/*
+    - release/*
+
+trigger:
+  branches:
+    include:
+    - main
+    - feature/*
+    - release/*
+
 resources:
   containers:
   - container: CentosContainer

build/.night-build.yml

@@ -15,9 +15,8 @@ schedules:
   branches:
     include:
     - main
-    - releases/1.6.0
-    - features/automl
-    - features/integrationPackage
+    - feature/*
+    - release/*
   always: true
 
 resources:

build/.outer-loop-build.yml

@@ -15,9 +15,8 @@ schedules:
   branches:
     include:
     - main
-    - releases/1.6.0
-    - features/automl
-    - features/integrationPackage
+    - feature/*
+    - release/*
   always: true
 
 

build/codecoverage-ci.yml

@@ -2,6 +2,20 @@
 # ML.NET's Code Coverage validation build
 ################################################################################
 
+pr:
+  branches:
+    include:
+    - main
+    - feature/*
+    - release/*
+
+trigger:
+  branches:
+    include:
+    - main
+    - feature/*
+    - release/*
+
 jobs:
 - template: /build/ci/job-template.yml
   parameters:

eng/Versions.props

@@ -30,7 +30,7 @@
     <SystemTextJsonVersion>6.0.1</SystemTextJsonVersion>
     <SystemThreadingChannelsVersion>4.7.1</SystemThreadingChannelsVersion>
     <!-- Other product dependencies -->
-    <ApacheArrowVersion>2.0.0</ApacheArrowVersion>
+    <ApacheArrowVersion>11.0.0</ApacheArrowVersion>
     <GoogleProtobufVersion>3.19.6</GoogleProtobufVersion>
     <LightGBMVersion>2.3.1</LightGBMVersion>
     <MicrosoftCodeAnalysisAnalyzersVersion>3.3.0</MicrosoftCodeAnalysisAnalyzersVersion>

src/Microsoft.Data.Analysis/DataFrame.Arrow.cs

@@ -101,10 +101,18 @@ private static void AppendDataFrameColumnFromArrowArray(Field field, IArrowArray
                         AppendDataFrameColumnFromArrowArray(fieldsEnumerator.Current, structArrayEnumerator.Current, ret, field.Name + "_");
                     }
                     break;
-                case ArrowTypeId.Decimal:
+                case ArrowTypeId.Date64:
+                    Date64Array arrowDate64Array = (Date64Array)arrowArray;
+                    dataFrameColumn = new DateTimeDataFrameColumn(fieldName, arrowDate64Array.Data.Length);
+                    for (int i = 0; i < arrowDate64Array.Data.Length; i++)
+                    {
+                        dataFrameColumn[i] = arrowDate64Array.GetDateTime(i);
+                    }
+                    break;
+                case ArrowTypeId.Decimal128:
+                case ArrowTypeId.Decimal256:
                 case ArrowTypeId.Binary:
                 case ArrowTypeId.Date32:
-                case ArrowTypeId.Date64:
                 case ArrowTypeId.Dictionary:
                 case ArrowTypeId.FixedSizedBinary:
                 case ArrowTypeId.HalfFloat:
@@ -114,6 +122,7 @@ private static void AppendDataFrameColumnFromArrowArray(Field field, IArrowArray
                 case ArrowTypeId.Null:
                 case ArrowTypeId.Time32:
                 case ArrowTypeId.Time64:
+                case ArrowTypeId.Timestamp:
                 default:
                     throw new NotImplementedException($"{fieldType.Name}");
             }
@@ -145,7 +154,7 @@ public static DataFrame FromArrowRecordBatch(RecordBatch recordBatch)
         }
 
         /// <summary>
-        /// Returns an <see cref="IEnumerable{RecordBatch}"/> without copying data
+        /// Returns an <see cref="IEnumerable{RecordBatch}"/> mostly without copying data
         /// </summary>
         public IEnumerable<RecordBatch> ToArrowRecordBatches()
         {

src/Microsoft.Data.Analysis/DataFrame.IO.cs

@@ -336,7 +336,7 @@ private static DataFrameColumn CreateColumn(Type kind, string columnName)
             }
             else if (kind == typeof(DateTime))
             {
-                ret = new PrimitiveDataFrameColumn<DateTime>(columnName);
+                ret = new DateTimeDataFrameColumn(columnName);
             }
             else
             {

src/Microsoft.Data.Analysis/DataFrame.Join.cs

@@ -30,7 +30,7 @@ private void SetSuffixForDuplicatedColumnNames(DataFrame dataFrame, DataFrameCol
             {
                 // Pre-existing column. Change name
                 DataFrameColumn existingColumn = dataFrame.Columns[index];
-                dataFrame._columnCollection.SetColumnName(existingColumn, existingColumn.Name + leftSuffix);
+                existingColumn.SetName(existingColumn.Name + leftSuffix);
                 column.SetName(column.Name + rightSuffix);
                 index = dataFrame._columnCollection.IndexOf(column.Name);
             }

src/Microsoft.Data.Analysis/DataFrame.cs

@@ -301,7 +301,7 @@ public DataFrame AddPrefix(string prefix, bool inPlace = false)
             for (int i = 0; i < df.Columns.Count; i++)
             {
                 DataFrameColumn column = df.Columns[i];
-                df._columnCollection.SetColumnName(column, prefix + column.Name);
+                column.SetName(prefix + column.Name);
                 df.OnColumnsChanged();
             }
             return df;
@@ -316,7 +316,7 @@ public DataFrame AddSuffix(string suffix, bool inPlace = false)
             for (int i = 0; i < df.Columns.Count; i++)
             {
                 DataFrameColumn column = df.Columns[i];
-                df._columnCollection.SetColumnName(column, column.Name + suffix);
+                column.SetName(column.Name + suffix);
                 df.OnColumnsChanged();
             }
             return df;

src/Microsoft.Data.Analysis/DataFrameBuffer.cs

@@ -66,7 +66,10 @@ public void EnsureCapacity(int numberOfValues)
 
             if (newLength > Capacity)
             {
-                var newCapacity = Math.Max(newLength * Size, ReadOnlyBuffer.Length * 2);
+                //Double buffer size, but not higher than MaxByteCapacity
+                var doubledSize = (int)Math.Min((long)ReadOnlyBuffer.Length * 2, MaxCapacityInBytes);
+                var newCapacity = Math.Max(newLength * Size, doubledSize);
+
                 var memory = new Memory<byte>(new byte[newCapacity]);
                 _memory.CopyTo(memory);
                 _memory = memory;