diff --git a/src/Microsoft.Data.Analysis/PrimitiveColumnContainerHelpers.cs b/src/Microsoft.Data.Analysis/PrimitiveColumnContainerHelpers.cs index 22ef32d979..eb6edc0aff 100644 --- a/src/Microsoft.Data.Analysis/PrimitiveColumnContainerHelpers.cs +++ b/src/Microsoft.Data.Analysis/PrimitiveColumnContainerHelpers.cs @@ -11,16 +11,14 @@ internal static class PrimitiveColumnContainerHelpers internal static DataFrameBuffer GetOrCreateMutable(this IList> bufferList, int index) where T : unmanaged { - ReadOnlyDataFrameBuffer sourceBuffer = bufferList[index]; - DataFrameBuffer mutableBuffer = sourceBuffer as DataFrameBuffer; + var sourceBuffer = bufferList[index]; - if (mutableBuffer == null) + if (sourceBuffer is not DataFrameBuffer mutableBuffer) { mutableBuffer = DataFrameBuffer.GetMutableBuffer(sourceBuffer); bufferList[index] = mutableBuffer; } - return mutableBuffer; } } diff --git a/src/Microsoft.Data.Analysis/PrimitiveDataFrameColumnArithmetic.cs b/src/Microsoft.Data.Analysis/PrimitiveDataFrameColumnArithmetic.cs index 89143a8316..13903a2a90 100644 --- a/src/Microsoft.Data.Analysis/PrimitiveDataFrameColumnArithmetic.cs +++ b/src/Microsoft.Data.Analysis/PrimitiveDataFrameColumnArithmetic.cs @@ -203,6 +203,7 @@ public void Modulo(bool scalar, PrimitiveColumnContainer column) public void And(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -210,7 +211,16 @@ public void And(PrimitiveColumnContainer left, PrimitiveColumnContainer column) public void Or(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -250,7 +261,16 @@ public void Or(PrimitiveColumnContainer left, PrimitiveColumnContainer column) public void Xor(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -290,7 +311,16 @@ public void Xor(PrimitiveColumnContainer left, PrimitiveColumnContainer public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -445,7 +476,16 @@ public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer column) public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -485,7 +526,16 @@ public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnContain var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (byte)(span[i] - otherSpan[i]); + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (byte)(span[i] - otherSpan[i]); + } + else + { + left[index] = null; + } + + index++; } } } @@ -518,6 +568,7 @@ public void Subtract(byte scalar, PrimitiveColumnContainer column) public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -525,7 +576,16 @@ public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnContain var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (byte)(span[i] * otherSpan[i]); + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (byte)(span[i] * otherSpan[i]); + } + else + { + left[index] = null; + } + + index++; } } } @@ -558,6 +618,7 @@ public void Multiply(byte scalar, PrimitiveColumnContainer column) public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -565,7 +626,16 @@ public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContainer var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (byte)(span[i] / otherSpan[i]); + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (byte)(span[i] / otherSpan[i]); + } + else + { + left[index] = null; + } + + index++; } } } @@ -598,6 +668,7 @@ public void Divide(byte scalar, PrimitiveColumnContainer column) public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -605,7 +676,16 @@ public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContainer var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (byte)(span[i] % otherSpan[i]); + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (byte)(span[i] % otherSpan[i]); + } + else + { + left[index] = null; + } + + index++; } } } @@ -638,6 +718,7 @@ public void Modulo(byte scalar, PrimitiveColumnContainer column) public void And(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -645,7 +726,16 @@ public void And(PrimitiveColumnContainer left, PrimitiveColumnContainer column) public void Or(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -685,7 +776,16 @@ public void Or(PrimitiveColumnContainer left, PrimitiveColumnContainer column) public void Xor(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -725,7 +826,16 @@ public void Xor(PrimitiveColumnContainer left, PrimitiveColumnContainer public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -980,7 +1091,16 @@ public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer column) public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -1020,7 +1141,16 @@ public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnContain var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (char)(span[i] - otherSpan[i]); + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (char)(span[i] - otherSpan[i]); + } + else + { + left[index] = null; + } + + index++; } } } @@ -1053,6 +1183,7 @@ public void Subtract(char scalar, PrimitiveColumnContainer column) public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -1060,7 +1191,16 @@ public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnContain var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (char)(span[i] * otherSpan[i]); + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (char)(span[i] * otherSpan[i]); + } + else + { + left[index] = null; + } + + index++; } } } @@ -1093,6 +1233,7 @@ public void Multiply(char scalar, PrimitiveColumnContainer column) public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -1100,7 +1241,16 @@ public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContainer var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (char)(span[i] / otherSpan[i]); + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (char)(span[i] / otherSpan[i]); + } + else + { + left[index] = null; + } + + index++; } } } @@ -1133,6 +1283,7 @@ public void Divide(char scalar, PrimitiveColumnContainer column) public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -1140,7 +1291,16 @@ public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContainer var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (char)(span[i] % otherSpan[i]); + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (char)(span[i] % otherSpan[i]); + } + else + { + left[index] = null; + } + + index++; } } } @@ -1173,6 +1333,7 @@ public void Modulo(char scalar, PrimitiveColumnContainer column) public void And(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -1180,7 +1341,16 @@ public void And(PrimitiveColumnContainer left, PrimitiveColumnContainer column) public void Or(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -1220,7 +1391,16 @@ public void Or(PrimitiveColumnContainer left, PrimitiveColumnContainer column) public void Xor(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -1260,7 +1441,16 @@ public void Xor(PrimitiveColumnContainer left, PrimitiveColumnContainer public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -1515,7 +1706,16 @@ public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (decimal)(span[i] + otherSpan[i]); + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (decimal)(span[i] + otherSpan[i]); + } + else + { + left[index] = null; + } + + index++; } } } @@ -1548,6 +1748,7 @@ public void Add(decimal scalar, PrimitiveColumnContainer column) public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -1555,7 +1756,16 @@ public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnCont var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (decimal)(span[i] - otherSpan[i]); + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (decimal)(span[i] - otherSpan[i]); + } + else + { + left[index] = null; + } + + index++; } } } @@ -1588,6 +1798,7 @@ public void Subtract(decimal scalar, PrimitiveColumnContainer column) public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -1595,7 +1806,16 @@ public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnCont var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (decimal)(span[i] * otherSpan[i]); + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (decimal)(span[i] * otherSpan[i]); + } + else + { + left[index] = null; + } + + index++; } } } @@ -1628,6 +1848,7 @@ public void Multiply(decimal scalar, PrimitiveColumnContainer column) public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -1635,7 +1856,16 @@ public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContai var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (decimal)(span[i] / otherSpan[i]); + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (decimal)(span[i] / otherSpan[i]); + } + else + { + left[index] = null; + } + + index++; } } } @@ -1668,6 +1898,7 @@ public void Divide(decimal scalar, PrimitiveColumnContainer column) public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -1675,7 +1906,16 @@ public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContai var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (decimal)(span[i] % otherSpan[i]); + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (decimal)(span[i] % otherSpan[i]); + } + else + { + left[index] = null; + } + + index++; } } } @@ -1952,6 +2192,7 @@ internal class DoubleArithmetic : IPrimitiveDataFrameColumnArithmetic public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -1959,7 +2200,16 @@ public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer< var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (double)(span[i] + otherSpan[i]); + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (double)(span[i] + otherSpan[i]); + } + else + { + left[index] = null; + } + + index++; } } } @@ -1992,6 +2242,7 @@ public void Add(double scalar, PrimitiveColumnContainer column) public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -1999,7 +2250,16 @@ public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnConta var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (double)(span[i] - otherSpan[i]); + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (double)(span[i] - otherSpan[i]); + } + else + { + left[index] = null; + } + + index++; } } } @@ -2032,6 +2292,7 @@ public void Subtract(double scalar, PrimitiveColumnContainer column) public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -2039,7 +2300,16 @@ public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnConta var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (double)(span[i] * otherSpan[i]); + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (double)(span[i] * otherSpan[i]); + } + else + { + left[index] = null; + } + + index++; } } } @@ -2072,6 +2342,7 @@ public void Multiply(double scalar, PrimitiveColumnContainer column) public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -2079,7 +2350,16 @@ public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContain var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (double)(span[i] / otherSpan[i]); + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (double)(span[i] / otherSpan[i]); + } + else + { + left[index] = null; + } + + index++; } } } @@ -2112,6 +2392,7 @@ public void Divide(double scalar, PrimitiveColumnContainer column) public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -2119,7 +2400,16 @@ public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContain var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (double)(span[i] % otherSpan[i]); + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (double)(span[i] % otherSpan[i]); + } + else + { + left[index] = null; + } + + index++; } } } @@ -2396,6 +2686,7 @@ internal class FloatArithmetic : IPrimitiveDataFrameColumnArithmetic public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -2403,7 +2694,16 @@ public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer column) public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -2443,7 +2744,16 @@ public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnContai var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (float)(span[i] - otherSpan[i]); + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (float)(span[i] - otherSpan[i]); + } + else + { + left[index] = null; + } + + index++; } } } @@ -2476,6 +2786,7 @@ public void Subtract(float scalar, PrimitiveColumnContainer column) public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -2483,7 +2794,16 @@ public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnContai var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (float)(span[i] * otherSpan[i]); + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (float)(span[i] * otherSpan[i]); + } + else + { + left[index] = null; + } + + index++; } } } @@ -2516,6 +2836,7 @@ public void Multiply(float scalar, PrimitiveColumnContainer column) public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -2523,7 +2844,16 @@ public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContaine var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (float)(span[i] / otherSpan[i]); + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (float)(span[i] / otherSpan[i]); + } + else + { + left[index] = null; + } + + index++; } } } @@ -2556,6 +2886,7 @@ public void Divide(float scalar, PrimitiveColumnContainer column) public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -2563,7 +2894,16 @@ public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContaine var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (float)(span[i] % otherSpan[i]); + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (float)(span[i] % otherSpan[i]); + } + else + { + left[index] = null; + } + + index++; } } } @@ -2840,6 +3180,7 @@ internal class IntArithmetic : IPrimitiveDataFrameColumnArithmetic public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -2847,7 +3188,16 @@ public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer column) public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -2887,7 +3238,16 @@ public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnContaine var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (int)(span[i] - otherSpan[i]); + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (int)(span[i] - otherSpan[i]); + } + else + { + left[index] = null; + } + + index++; } } } @@ -2920,6 +3280,7 @@ public void Subtract(int scalar, PrimitiveColumnContainer column) public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -2927,7 +3288,16 @@ public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnContaine var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (int)(span[i] * otherSpan[i]); + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (int)(span[i] * otherSpan[i]); + } + else + { + left[index] = null; + } + + index++; } } } @@ -2960,6 +3330,7 @@ public void Multiply(int scalar, PrimitiveColumnContainer column) public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -2967,7 +3338,16 @@ public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContainer< var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (int)(span[i] / otherSpan[i]); + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (int)(span[i] / otherSpan[i]); + } + else + { + left[index] = null; + } + + index++; } } } @@ -3000,6 +3380,7 @@ public void Divide(int scalar, PrimitiveColumnContainer column) public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -3007,7 +3388,16 @@ public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContainer< var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (int)(span[i] % otherSpan[i]); + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (int)(span[i] % otherSpan[i]); + } + else + { + left[index] = null; + } + + index++; } } } @@ -3040,6 +3430,7 @@ public void Modulo(int scalar, PrimitiveColumnContainer column) public void And(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -3047,7 +3438,16 @@ public void And(PrimitiveColumnContainer left, PrimitiveColumnContainer column) public void Or(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -3087,7 +3488,16 @@ public void Or(PrimitiveColumnContainer left, PrimitiveColumnContainer var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (int)(span[i] | otherSpan[i]); + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (int)(span[i] | otherSpan[i]); + } + else + { + left[index] = null; + } + + index++; } } } @@ -3120,6 +3530,7 @@ public void Or(int scalar, PrimitiveColumnContainer column) public void Xor(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -3127,7 +3538,16 @@ public void Xor(PrimitiveColumnContainer left, PrimitiveColumnContainer public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -3382,7 +3803,16 @@ public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer column) public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -3422,7 +3853,16 @@ public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnContain var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (long)(span[i] - otherSpan[i]); + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (long)(span[i] - otherSpan[i]); + } + else + { + left[index] = null; + } + + index++; } } } @@ -3455,6 +3895,7 @@ public void Subtract(long scalar, PrimitiveColumnContainer column) public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -3462,7 +3903,16 @@ public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnContain var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (long)(span[i] * otherSpan[i]); + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (long)(span[i] * otherSpan[i]); + } + else + { + left[index] = null; + } + + index++; } } } @@ -3495,6 +3945,7 @@ public void Multiply(long scalar, PrimitiveColumnContainer column) public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -3502,7 +3953,16 @@ public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContainer var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (long)(span[i] / otherSpan[i]); + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (long)(span[i] / otherSpan[i]); + } + else + { + left[index] = null; + } + + index++; } } } @@ -3535,6 +3995,7 @@ public void Divide(long scalar, PrimitiveColumnContainer column) public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -3542,7 +4003,16 @@ public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContainer var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (long)(span[i] % otherSpan[i]); + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (long)(span[i] % otherSpan[i]); + } + else + { + left[index] = null; + } + + index++; } } } @@ -3575,6 +4045,7 @@ public void Modulo(long scalar, PrimitiveColumnContainer column) public void And(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -3582,7 +4053,16 @@ public void And(PrimitiveColumnContainer left, PrimitiveColumnContainer column) public void Or(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -3622,7 +4103,16 @@ public void Or(PrimitiveColumnContainer left, PrimitiveColumnContainer column) public void Xor(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -3662,7 +4153,16 @@ public void Xor(PrimitiveColumnContainer left, PrimitiveColumnContainer public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -3917,7 +4418,16 @@ public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer column) public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -3957,7 +4468,16 @@ public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnContai var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (sbyte)(span[i] - otherSpan[i]); + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (sbyte)(span[i] - otherSpan[i]); + } + else + { + left[index] = null; + } + + index++; } } } @@ -3990,6 +4510,7 @@ public void Subtract(sbyte scalar, PrimitiveColumnContainer column) public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -3997,7 +4518,16 @@ public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnContai var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (sbyte)(span[i] * otherSpan[i]); + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (sbyte)(span[i] * otherSpan[i]); + } + else + { + left[index] = null; + } + + index++; } } } @@ -4030,6 +4560,7 @@ public void Multiply(sbyte scalar, PrimitiveColumnContainer column) public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -4037,7 +4568,16 @@ public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContaine var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (sbyte)(span[i] / otherSpan[i]); + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (sbyte)(span[i] / otherSpan[i]); + } + else + { + left[index] = null; + } + + index++; } } } @@ -4070,6 +4610,7 @@ public void Divide(sbyte scalar, PrimitiveColumnContainer column) public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -4077,7 +4618,16 @@ public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContaine var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (sbyte)(span[i] % otherSpan[i]); + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (sbyte)(span[i] % otherSpan[i]); + } + else + { + left[index] = null; + } + + index++; } } } @@ -4110,6 +4660,7 @@ public void Modulo(sbyte scalar, PrimitiveColumnContainer column) public void And(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -4117,7 +4668,16 @@ public void And(PrimitiveColumnContainer left, PrimitiveColumnContainer column) public void Or(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -4157,7 +4718,16 @@ public void Or(PrimitiveColumnContainer left, PrimitiveColumnContainer column) public void Xor(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -4197,7 +4768,16 @@ public void Xor(PrimitiveColumnContainer left, PrimitiveColumnContainer public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -4452,7 +5033,16 @@ public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer column) public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -4492,7 +5083,16 @@ public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnContai var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (short)(span[i] - otherSpan[i]); + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (short)(span[i] - otherSpan[i]); + } + else + { + left[index] = null; + } + + index++; } } } @@ -4525,6 +5125,7 @@ public void Subtract(short scalar, PrimitiveColumnContainer column) public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -4532,7 +5133,16 @@ public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnContai var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (short)(span[i] * otherSpan[i]); + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (short)(span[i] * otherSpan[i]); + } + else + { + left[index] = null; + } + + index++; } } } @@ -4565,6 +5175,7 @@ public void Multiply(short scalar, PrimitiveColumnContainer column) public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -4572,7 +5183,16 @@ public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContaine var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (short)(span[i] / otherSpan[i]); + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (short)(span[i] / otherSpan[i]); + } + else + { + left[index] = null; + } + + index++; } } } @@ -4605,6 +5225,7 @@ public void Divide(short scalar, PrimitiveColumnContainer column) public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -4612,7 +5233,16 @@ public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContaine var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (short)(span[i] % otherSpan[i]); + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (short)(span[i] % otherSpan[i]); + } + else + { + left[index] = null; + } + + index++; } } } @@ -4645,6 +5275,7 @@ public void Modulo(short scalar, PrimitiveColumnContainer column) public void And(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -4652,7 +5283,16 @@ public void And(PrimitiveColumnContainer left, PrimitiveColumnContainer column) public void Or(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -4692,7 +5333,16 @@ public void Or(PrimitiveColumnContainer left, PrimitiveColumnContainer column) public void Xor(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -4732,7 +5383,16 @@ public void Xor(PrimitiveColumnContainer left, PrimitiveColumnContainer public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -4987,7 +5648,16 @@ public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer column) public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -5027,7 +5698,16 @@ public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnContain var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (uint)(span[i] - otherSpan[i]); + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (uint)(span[i] - otherSpan[i]); + } + else + { + left[index] = null; + } + + index++; } } } @@ -5060,6 +5740,7 @@ public void Subtract(uint scalar, PrimitiveColumnContainer column) public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -5067,7 +5748,16 @@ public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnContain var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (uint)(span[i] * otherSpan[i]); + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (uint)(span[i] * otherSpan[i]); + } + else + { + left[index] = null; + } + + index++; } } } @@ -5100,6 +5790,7 @@ public void Multiply(uint scalar, PrimitiveColumnContainer column) public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -5107,7 +5798,16 @@ public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContainer var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (uint)(span[i] / otherSpan[i]); + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (uint)(span[i] / otherSpan[i]); + } + else + { + left[index] = null; + } + + index++; } } } @@ -5140,6 +5840,7 @@ public void Divide(uint scalar, PrimitiveColumnContainer column) public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -5147,7 +5848,16 @@ public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContainer var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (uint)(span[i] % otherSpan[i]); + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (uint)(span[i] % otherSpan[i]); + } + else + { + left[index] = null; + } + + index++; } } } @@ -5180,6 +5890,7 @@ public void Modulo(uint scalar, PrimitiveColumnContainer column) public void And(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -5187,7 +5898,16 @@ public void And(PrimitiveColumnContainer left, PrimitiveColumnContainer column) public void Or(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -5227,7 +5948,16 @@ public void Or(PrimitiveColumnContainer left, PrimitiveColumnContainer column) public void Xor(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -5267,7 +5998,16 @@ public void Xor(PrimitiveColumnContainer left, PrimitiveColumnContainer public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -5522,7 +6263,16 @@ public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer column) public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -5562,7 +6313,16 @@ public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnContai var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (ulong)(span[i] - otherSpan[i]); + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (ulong)(span[i] - otherSpan[i]); + } + else + { + left[index] = null; + } + + index++; } } } @@ -5595,6 +6355,7 @@ public void Subtract(ulong scalar, PrimitiveColumnContainer column) public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -5602,7 +6363,16 @@ public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnContai var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (ulong)(span[i] * otherSpan[i]); + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (ulong)(span[i] * otherSpan[i]); + } + else + { + left[index] = null; + } + + index++; } } } @@ -5635,6 +6405,7 @@ public void Multiply(ulong scalar, PrimitiveColumnContainer column) public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -5642,7 +6413,16 @@ public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContaine var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (ulong)(span[i] / otherSpan[i]); + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (ulong)(span[i] / otherSpan[i]); + } + else + { + left[index] = null; + } + + index++; } } } @@ -5675,6 +6455,7 @@ public void Divide(ulong scalar, PrimitiveColumnContainer column) public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -5682,7 +6463,16 @@ public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContaine var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (ulong)(span[i] % otherSpan[i]); + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (ulong)(span[i] % otherSpan[i]); + } + else + { + left[index] = null; + } + + index++; } } } @@ -5715,6 +6505,7 @@ public void Modulo(ulong scalar, PrimitiveColumnContainer column) public void And(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -5722,7 +6513,16 @@ public void And(PrimitiveColumnContainer left, PrimitiveColumnContainer column) public void Or(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -5762,7 +6563,16 @@ public void Or(PrimitiveColumnContainer left, PrimitiveColumnContainer
    column) public void Xor(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -5802,7 +6613,16 @@ public void Xor(PrimitiveColumnContainer left, PrimitiveColumnContainer public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -6057,7 +6878,16 @@ public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer< var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (ushort)(span[i] + otherSpan[i]); + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (ushort)(span[i] + otherSpan[i]); + } + else + { + left[index] = null; + } + + index++; } } } @@ -6090,6 +6920,7 @@ public void Add(ushort scalar, PrimitiveColumnContainer column) public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -6097,7 +6928,16 @@ public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnConta var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (ushort)(span[i] - otherSpan[i]); + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (ushort)(span[i] - otherSpan[i]); + } + else + { + left[index] = null; + } + + index++; } } } @@ -6130,6 +6970,7 @@ public void Subtract(ushort scalar, PrimitiveColumnContainer column) public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -6137,7 +6978,16 @@ public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnConta var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (ushort)(span[i] * otherSpan[i]); + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (ushort)(span[i] * otherSpan[i]); + } + else + { + left[index] = null; + } + + index++; } } } @@ -6170,6 +7020,7 @@ public void Multiply(ushort scalar, PrimitiveColumnContainer column) public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -6177,7 +7028,16 @@ public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContain var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (ushort)(span[i] / otherSpan[i]); + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (ushort)(span[i] / otherSpan[i]); + } + else + { + left[index] = null; + } + + index++; } } } @@ -6210,6 +7070,7 @@ public void Divide(ushort scalar, PrimitiveColumnContainer column) public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -6217,7 +7078,16 @@ public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContain var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (ushort)(span[i] % otherSpan[i]); + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (ushort)(span[i] % otherSpan[i]); + } + else + { + left[index] = null; + } + + index++; } } } @@ -6250,6 +7120,7 @@ public void Modulo(ushort scalar, PrimitiveColumnContainer column) public void And(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -6257,7 +7128,16 @@ public void And(PrimitiveColumnContainer left, PrimitiveColumnContainer< var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (ushort)(span[i] & otherSpan[i]); + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (ushort)(span[i] & otherSpan[i]); + } + else + { + left[index] = null; + } + + index++; } } } @@ -6290,6 +7170,7 @@ public void And(ushort scalar, PrimitiveColumnContainer column) public void Or(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -6297,7 +7178,16 @@ public void Or(PrimitiveColumnContainer left, PrimitiveColumnContainer column) public void Xor(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -6337,7 +7228,16 @@ public void Xor(PrimitiveColumnContainer left, PrimitiveColumnContainer< var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (ushort)(span[i] ^ otherSpan[i]); + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (ushort)(span[i] ^ otherSpan[i]); + } + else + { + left[index] = null; + } + + index++; } } } diff --git a/src/Microsoft.Data.Analysis/PrimitiveDataFrameColumnArithmetic.tt b/src/Microsoft.Data.Analysis/PrimitiveDataFrameColumnArithmetic.tt index 9ad6e313ee..57942b6a6c 100644 --- a/src/Microsoft.Data.Analysis/PrimitiveDataFrameColumnArithmetic.tt +++ b/src/Microsoft.Data.Analysis/PrimitiveDataFrameColumnArithmetic.tt @@ -68,6 +68,9 @@ namespace Microsoft.Data.Analysis <# if ((method.IsNumeric && !type.SupportsNumeric) || (method.IsBitwise && !type.SupportsBitwise) || (type.UnsupportedMethods.Contains(method.MethodName))) { #> throw new NotSupportedException(); <# } else if (method.Operator != null) { #> +<# if (method.MethodType == MethodType.Binary) { #> + long index = 0; +<# } #> <# if (method.MethodType == MethodType.Comparison) { #> PrimitiveColumnContainer ret = new PrimitiveColumnContainer(left.Length); long index = 0; @@ -95,7 +98,16 @@ namespace Microsoft.Data.Analysis <# if (method.MethodType == MethodType.BinaryScalar || method.MethodType == MethodType.BinaryInt) { #> span[i] = (<#=type.TypeName#>)(span[i] <#= method.Operator #> <#= method.Op2Name #>); <# } else if (method.MethodType == MethodType.Binary) { #> - span[i] = (<#=type.TypeName#>)(span[i] <#= method.Operator #> otherSpan[i]); + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (<#=type.TypeName#>)(span[i] <#= method.Operator #> otherSpan[i]); + } + else + { + left[index] = null; + } + + index++; <# } else if (method.MethodType == MethodType.Comparison) { #> ret[index++] = (span[i] <#= method.Operator #> otherSpan[i]); <# } else if (method.MethodType == MethodType.ComparisonScalar) { #> diff --git a/test/Microsoft.Data.Analysis.Tests/DataFrameTests.cs b/test/Microsoft.Data.Analysis.Tests/DataFrameTests.cs index 35bf334d6a..423a6ef1b1 100644 --- a/test/Microsoft.Data.Analysis.Tests/DataFrameTests.cs +++ b/test/Microsoft.Data.Analysis.Tests/DataFrameTests.cs @@ -3620,5 +3620,78 @@ public void Test_StringColumnEqualsNull() Assert.Equal(2, filteredNullDf.Columns["index"][0]); Assert.Equal(5, filteredNullDf.Columns["index"][1]); } + + [Fact] + public void Test_ArithmeticsSumWithNull() + { + // Arrange + var left_column = new PrimitiveDataFrameColumn("Left", new int?[] { 1, 1, null, null }); + var right_column = new PrimitiveDataFrameColumn("Right", new int?[] { 1, null, 1, null }); + + // Act + var sum = left_column + right_column; + + // Assert + Assert.Equal(3, sum.NullCount); + + Assert.Equal(2, sum[0]); // 1 + 1 + Assert.Null(sum[1]); // null + 1 + Assert.Null(sum[2]); // 1 + null + Assert.Null(sum[3]); // null + null + } + + [Fact] + public void Test_ArithmeticsDiffWithNull() + { + // Arrange + var left_column = new PrimitiveDataFrameColumn("Left", new int?[] { 1, 1, null, null }); + var right_column = new PrimitiveDataFrameColumn("Right", new int?[] { 1, null, 1, null }); + + // Act + var diff = left_column - right_column; + + // Assert + Assert.Equal(3, diff.NullCount); + Assert.Equal(0, diff[0]); // 1 - 1 + Assert.Null(diff[1]); // null - 1 + Assert.Null(diff[2]); // 1 - null + Assert.Null(diff[3]); // null - null + } + + [Fact] + public void Test_ArithmeticsMultWithNull() + { + // Arrange + var left_column = new PrimitiveDataFrameColumn("Left", new int?[] { 1, 1, null, null }); + var right_column = new PrimitiveDataFrameColumn("Right", new int?[] { 1, null, 1, null }); + + // Act + var mult = left_column * right_column; + + // Assert + Assert.Equal(3, mult.NullCount); + Assert.Equal(1, mult[0]); // 1 * 1 + Assert.Null(mult[1]); // null * 1 + Assert.Null(mult[2]); // 1 * null + Assert.Null(mult[3]); // null * null + } + + [Fact] + public void Test_ArithmeticsDivWithNull() + { + // Arrange + var left_column = new PrimitiveDataFrameColumn("Left", new int?[] { 1, 1, null, null }); + var right_column = new PrimitiveDataFrameColumn("Right", new int?[] { 1, null, 1, null }); + + // Act + var mult = left_column / right_column; + + // Assert + Assert.Equal(3, mult.NullCount); + Assert.Equal(1, mult[0]); // 1 / 1 + Assert.Null(mult[1]); // null / 1 + Assert.Null(mult[2]); // 1 / null + Assert.Null(mult[3]); // null / null + } } }