From a71e510107a0e0680daddfd92b36a04c0cf1fd8f Mon Sep 17 00:00:00 2001 From: Aleksei Smirnov Date: Thu, 20 Jul 2023 13:15:11 +0300 Subject: [PATCH 1/3] Fix inconsistent null handling in DataFrame Arithmetics --- .../PrimitiveColumnContainerHelpers.cs | 1 - .../PrimitiveDataFrameColumnArithmetic.cs | 450 ++++++++++++++++++ .../PrimitiveDataFrameColumnArithmetic.tt | 7 + .../DataFrameTests.cs | 51 ++ 4 files changed, 508 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.Data.Analysis/PrimitiveColumnContainerHelpers.cs b/src/Microsoft.Data.Analysis/PrimitiveColumnContainerHelpers.cs index 22ef32d979..b4da9a311a 100644 --- a/src/Microsoft.Data.Analysis/PrimitiveColumnContainerHelpers.cs +++ b/src/Microsoft.Data.Analysis/PrimitiveColumnContainerHelpers.cs @@ -20,7 +20,6 @@ internal static DataFrameBuffer GetOrCreateMutable(this IList left, PrimitiveColumnContainer column, bool scalar) @@ -234,6 +239,11 @@ public void Or(PrimitiveColumnContainer left, PrimitiveColumnContainer column, bool scalar) @@ -271,6 +281,11 @@ public void Xor(PrimitiveColumnContainer left, PrimitiveColumnContainer column, bool scalar) @@ -401,6 +416,11 @@ public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer column, byte scalar) @@ -438,6 +458,11 @@ public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnContain { span[i] = (byte)(span[i] - otherSpan[i]); } + + var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); + var bitmapSpan = mutableBitMapBuffer.Span; + for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) + bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; } } public void Subtract(PrimitiveColumnContainer column, byte scalar) @@ -475,6 +500,11 @@ public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnContain { span[i] = (byte)(span[i] * otherSpan[i]); } + + var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); + var bitmapSpan = mutableBitMapBuffer.Span; + for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) + bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; } } public void Multiply(PrimitiveColumnContainer column, byte scalar) @@ -512,6 +542,11 @@ public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContainer { span[i] = (byte)(span[i] / otherSpan[i]); } + + var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); + var bitmapSpan = mutableBitMapBuffer.Span; + for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) + bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; } } public void Divide(PrimitiveColumnContainer column, byte scalar) @@ -549,6 +584,11 @@ public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContainer { span[i] = (byte)(span[i] % otherSpan[i]); } + + var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); + var bitmapSpan = mutableBitMapBuffer.Span; + for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) + bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; } } public void Modulo(PrimitiveColumnContainer column, byte scalar) @@ -586,6 +626,11 @@ public void And(PrimitiveColumnContainer left, PrimitiveColumnContainer column, byte scalar) @@ -623,6 +668,11 @@ public void Or(PrimitiveColumnContainer left, PrimitiveColumnContainer column, byte scalar) @@ -660,6 +710,11 @@ public void Xor(PrimitiveColumnContainer left, PrimitiveColumnContainer column, byte scalar) @@ -874,6 +929,11 @@ public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer column, char scalar) @@ -911,6 +971,11 @@ public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnContain { span[i] = (char)(span[i] - otherSpan[i]); } + + var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); + var bitmapSpan = mutableBitMapBuffer.Span; + for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) + bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; } } public void Subtract(PrimitiveColumnContainer column, char scalar) @@ -948,6 +1013,11 @@ public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnContain { span[i] = (char)(span[i] * otherSpan[i]); } + + var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); + var bitmapSpan = mutableBitMapBuffer.Span; + for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) + bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; } } public void Multiply(PrimitiveColumnContainer column, char scalar) @@ -985,6 +1055,11 @@ public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContainer { span[i] = (char)(span[i] / otherSpan[i]); } + + var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); + var bitmapSpan = mutableBitMapBuffer.Span; + for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) + bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; } } public void Divide(PrimitiveColumnContainer column, char scalar) @@ -1022,6 +1097,11 @@ public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContainer { span[i] = (char)(span[i] % otherSpan[i]); } + + var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); + var bitmapSpan = mutableBitMapBuffer.Span; + for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) + bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; } } public void Modulo(PrimitiveColumnContainer column, char scalar) @@ -1059,6 +1139,11 @@ public void And(PrimitiveColumnContainer left, PrimitiveColumnContainer column, char scalar) @@ -1096,6 +1181,11 @@ public void Or(PrimitiveColumnContainer left, PrimitiveColumnContainer column, char scalar) @@ -1133,6 +1223,11 @@ public void Xor(PrimitiveColumnContainer left, PrimitiveColumnContainer column, char scalar) @@ -1347,6 +1442,11 @@ public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer { span[i] = (decimal)(span[i] + otherSpan[i]); } + + var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); + var bitmapSpan = mutableBitMapBuffer.Span; + for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) + bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; } } public void Add(PrimitiveColumnContainer column, decimal scalar) @@ -1384,6 +1484,11 @@ public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnCont { span[i] = (decimal)(span[i] - otherSpan[i]); } + + var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); + var bitmapSpan = mutableBitMapBuffer.Span; + for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) + bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; } } public void Subtract(PrimitiveColumnContainer column, decimal scalar) @@ -1421,6 +1526,11 @@ public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnCont { span[i] = (decimal)(span[i] * otherSpan[i]); } + + var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); + var bitmapSpan = mutableBitMapBuffer.Span; + for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) + bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; } } public void Multiply(PrimitiveColumnContainer column, decimal scalar) @@ -1458,6 +1568,11 @@ public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContai { span[i] = (decimal)(span[i] / otherSpan[i]); } + + var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); + var bitmapSpan = mutableBitMapBuffer.Span; + for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) + bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; } } public void Divide(PrimitiveColumnContainer column, decimal scalar) @@ -1495,6 +1610,11 @@ public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContai { span[i] = (decimal)(span[i] % otherSpan[i]); } + + var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); + var bitmapSpan = mutableBitMapBuffer.Span; + for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) + bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; } } public void Modulo(PrimitiveColumnContainer column, decimal scalar) @@ -1729,6 +1849,11 @@ public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer< { span[i] = (double)(span[i] + otherSpan[i]); } + + var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); + var bitmapSpan = mutableBitMapBuffer.Span; + for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) + bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; } } public void Add(PrimitiveColumnContainer column, double scalar) @@ -1766,6 +1891,11 @@ public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnConta { span[i] = (double)(span[i] - otherSpan[i]); } + + var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); + var bitmapSpan = mutableBitMapBuffer.Span; + for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) + bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; } } public void Subtract(PrimitiveColumnContainer column, double scalar) @@ -1803,6 +1933,11 @@ public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnConta { span[i] = (double)(span[i] * otherSpan[i]); } + + var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); + var bitmapSpan = mutableBitMapBuffer.Span; + for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) + bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; } } public void Multiply(PrimitiveColumnContainer column, double scalar) @@ -1840,6 +1975,11 @@ public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContain { span[i] = (double)(span[i] / otherSpan[i]); } + + var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); + var bitmapSpan = mutableBitMapBuffer.Span; + for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) + bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; } } public void Divide(PrimitiveColumnContainer column, double scalar) @@ -1877,6 +2017,11 @@ public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContain { span[i] = (double)(span[i] % otherSpan[i]); } + + var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); + var bitmapSpan = mutableBitMapBuffer.Span; + for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) + bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; } } public void Modulo(PrimitiveColumnContainer column, double scalar) @@ -2111,6 +2256,11 @@ public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer column, float scalar) @@ -2148,6 +2298,11 @@ public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnContai { span[i] = (float)(span[i] - otherSpan[i]); } + + var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); + var bitmapSpan = mutableBitMapBuffer.Span; + for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) + bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; } } public void Subtract(PrimitiveColumnContainer column, float scalar) @@ -2185,6 +2340,11 @@ public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnContai { span[i] = (float)(span[i] * otherSpan[i]); } + + var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); + var bitmapSpan = mutableBitMapBuffer.Span; + for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) + bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; } } public void Multiply(PrimitiveColumnContainer column, float scalar) @@ -2222,6 +2382,11 @@ public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContaine { span[i] = (float)(span[i] / otherSpan[i]); } + + var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); + var bitmapSpan = mutableBitMapBuffer.Span; + for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) + bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; } } public void Divide(PrimitiveColumnContainer column, float scalar) @@ -2259,6 +2424,11 @@ public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContaine { span[i] = (float)(span[i] % otherSpan[i]); } + + var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); + var bitmapSpan = mutableBitMapBuffer.Span; + for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) + bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; } } public void Modulo(PrimitiveColumnContainer column, float scalar) @@ -2493,6 +2663,11 @@ public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer column, int scalar) @@ -2530,6 +2705,11 @@ public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnContaine { span[i] = (int)(span[i] - otherSpan[i]); } + + var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); + var bitmapSpan = mutableBitMapBuffer.Span; + for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) + bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; } } public void Subtract(PrimitiveColumnContainer column, int scalar) @@ -2567,6 +2747,11 @@ public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnContaine { span[i] = (int)(span[i] * otherSpan[i]); } + + var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); + var bitmapSpan = mutableBitMapBuffer.Span; + for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) + bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; } } public void Multiply(PrimitiveColumnContainer column, int scalar) @@ -2604,6 +2789,11 @@ public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContainer< { span[i] = (int)(span[i] / otherSpan[i]); } + + var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); + var bitmapSpan = mutableBitMapBuffer.Span; + for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) + bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; } } public void Divide(PrimitiveColumnContainer column, int scalar) @@ -2641,6 +2831,11 @@ public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContainer< { span[i] = (int)(span[i] % otherSpan[i]); } + + var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); + var bitmapSpan = mutableBitMapBuffer.Span; + for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) + bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; } } public void Modulo(PrimitiveColumnContainer column, int scalar) @@ -2678,6 +2873,11 @@ public void And(PrimitiveColumnContainer left, PrimitiveColumnContainer column, int scalar) @@ -2715,6 +2915,11 @@ public void Or(PrimitiveColumnContainer left, PrimitiveColumnContainer { span[i] = (int)(span[i] | otherSpan[i]); } + + var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); + var bitmapSpan = mutableBitMapBuffer.Span; + for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) + bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; } } public void Or(PrimitiveColumnContainer column, int scalar) @@ -2752,6 +2957,11 @@ public void Xor(PrimitiveColumnContainer left, PrimitiveColumnContainer column, int scalar) @@ -2966,6 +3176,11 @@ public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer column, long scalar) @@ -3003,6 +3218,11 @@ public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnContain { span[i] = (long)(span[i] - otherSpan[i]); } + + var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); + var bitmapSpan = mutableBitMapBuffer.Span; + for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) + bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; } } public void Subtract(PrimitiveColumnContainer column, long scalar) @@ -3040,6 +3260,11 @@ public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnContain { span[i] = (long)(span[i] * otherSpan[i]); } + + var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); + var bitmapSpan = mutableBitMapBuffer.Span; + for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) + bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; } } public void Multiply(PrimitiveColumnContainer column, long scalar) @@ -3077,6 +3302,11 @@ public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContainer { span[i] = (long)(span[i] / otherSpan[i]); } + + var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); + var bitmapSpan = mutableBitMapBuffer.Span; + for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) + bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; } } public void Divide(PrimitiveColumnContainer column, long scalar) @@ -3114,6 +3344,11 @@ public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContainer { span[i] = (long)(span[i] % otherSpan[i]); } + + var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); + var bitmapSpan = mutableBitMapBuffer.Span; + for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) + bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; } } public void Modulo(PrimitiveColumnContainer column, long scalar) @@ -3151,6 +3386,11 @@ public void And(PrimitiveColumnContainer left, PrimitiveColumnContainer column, long scalar) @@ -3188,6 +3428,11 @@ public void Or(PrimitiveColumnContainer left, PrimitiveColumnContainer column, long scalar) @@ -3225,6 +3470,11 @@ public void Xor(PrimitiveColumnContainer left, PrimitiveColumnContainer column, long scalar) @@ -3439,6 +3689,11 @@ public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer column, sbyte scalar) @@ -3476,6 +3731,11 @@ public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnContai { span[i] = (sbyte)(span[i] - otherSpan[i]); } + + var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); + var bitmapSpan = mutableBitMapBuffer.Span; + for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) + bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; } } public void Subtract(PrimitiveColumnContainer column, sbyte scalar) @@ -3513,6 +3773,11 @@ public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnContai { span[i] = (sbyte)(span[i] * otherSpan[i]); } + + var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); + var bitmapSpan = mutableBitMapBuffer.Span; + for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) + bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; } } public void Multiply(PrimitiveColumnContainer column, sbyte scalar) @@ -3550,6 +3815,11 @@ public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContaine { span[i] = (sbyte)(span[i] / otherSpan[i]); } + + var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); + var bitmapSpan = mutableBitMapBuffer.Span; + for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) + bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; } } public void Divide(PrimitiveColumnContainer column, sbyte scalar) @@ -3587,6 +3857,11 @@ public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContaine { span[i] = (sbyte)(span[i] % otherSpan[i]); } + + var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); + var bitmapSpan = mutableBitMapBuffer.Span; + for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) + bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; } } public void Modulo(PrimitiveColumnContainer column, sbyte scalar) @@ -3624,6 +3899,11 @@ public void And(PrimitiveColumnContainer left, PrimitiveColumnContainer column, sbyte scalar) @@ -3661,6 +3941,11 @@ public void Or(PrimitiveColumnContainer left, PrimitiveColumnContainer column, sbyte scalar) @@ -3698,6 +3983,11 @@ public void Xor(PrimitiveColumnContainer left, PrimitiveColumnContainer column, sbyte scalar) @@ -3912,6 +4202,11 @@ public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer column, short scalar) @@ -3949,6 +4244,11 @@ public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnContai { span[i] = (short)(span[i] - otherSpan[i]); } + + var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); + var bitmapSpan = mutableBitMapBuffer.Span; + for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) + bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; } } public void Subtract(PrimitiveColumnContainer column, short scalar) @@ -3986,6 +4286,11 @@ public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnContai { span[i] = (short)(span[i] * otherSpan[i]); } + + var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); + var bitmapSpan = mutableBitMapBuffer.Span; + for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) + bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; } } public void Multiply(PrimitiveColumnContainer column, short scalar) @@ -4023,6 +4328,11 @@ public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContaine { span[i] = (short)(span[i] / otherSpan[i]); } + + var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); + var bitmapSpan = mutableBitMapBuffer.Span; + for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) + bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; } } public void Divide(PrimitiveColumnContainer column, short scalar) @@ -4060,6 +4370,11 @@ public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContaine { span[i] = (short)(span[i] % otherSpan[i]); } + + var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); + var bitmapSpan = mutableBitMapBuffer.Span; + for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) + bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; } } public void Modulo(PrimitiveColumnContainer column, short scalar) @@ -4097,6 +4412,11 @@ public void And(PrimitiveColumnContainer left, PrimitiveColumnContainer column, short scalar) @@ -4134,6 +4454,11 @@ public void Or(PrimitiveColumnContainer left, PrimitiveColumnContainer column, short scalar) @@ -4171,6 +4496,11 @@ public void Xor(PrimitiveColumnContainer left, PrimitiveColumnContainer column, short scalar) @@ -4385,6 +4715,11 @@ public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer column, uint scalar) @@ -4422,6 +4757,11 @@ public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnContain { span[i] = (uint)(span[i] - otherSpan[i]); } + + var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); + var bitmapSpan = mutableBitMapBuffer.Span; + for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) + bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; } } public void Subtract(PrimitiveColumnContainer column, uint scalar) @@ -4459,6 +4799,11 @@ public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnContain { span[i] = (uint)(span[i] * otherSpan[i]); } + + var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); + var bitmapSpan = mutableBitMapBuffer.Span; + for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) + bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; } } public void Multiply(PrimitiveColumnContainer column, uint scalar) @@ -4496,6 +4841,11 @@ public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContainer { span[i] = (uint)(span[i] / otherSpan[i]); } + + var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); + var bitmapSpan = mutableBitMapBuffer.Span; + for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) + bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; } } public void Divide(PrimitiveColumnContainer column, uint scalar) @@ -4533,6 +4883,11 @@ public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContainer { span[i] = (uint)(span[i] % otherSpan[i]); } + + var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); + var bitmapSpan = mutableBitMapBuffer.Span; + for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) + bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; } } public void Modulo(PrimitiveColumnContainer column, uint scalar) @@ -4570,6 +4925,11 @@ public void And(PrimitiveColumnContainer left, PrimitiveColumnContainer column, uint scalar) @@ -4607,6 +4967,11 @@ public void Or(PrimitiveColumnContainer left, PrimitiveColumnContainer column, uint scalar) @@ -4644,6 +5009,11 @@ public void Xor(PrimitiveColumnContainer left, PrimitiveColumnContainer column, uint scalar) @@ -4858,6 +5228,11 @@ public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer column, ulong scalar) @@ -4895,6 +5270,11 @@ public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnContai { span[i] = (ulong)(span[i] - otherSpan[i]); } + + var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); + var bitmapSpan = mutableBitMapBuffer.Span; + for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) + bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; } } public void Subtract(PrimitiveColumnContainer column, ulong scalar) @@ -4932,6 +5312,11 @@ public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnContai { span[i] = (ulong)(span[i] * otherSpan[i]); } + + var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); + var bitmapSpan = mutableBitMapBuffer.Span; + for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) + bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; } } public void Multiply(PrimitiveColumnContainer column, ulong scalar) @@ -4969,6 +5354,11 @@ public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContaine { span[i] = (ulong)(span[i] / otherSpan[i]); } + + var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); + var bitmapSpan = mutableBitMapBuffer.Span; + for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) + bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; } } public void Divide(PrimitiveColumnContainer column, ulong scalar) @@ -5006,6 +5396,11 @@ public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContaine { span[i] = (ulong)(span[i] % otherSpan[i]); } + + var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); + var bitmapSpan = mutableBitMapBuffer.Span; + for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) + bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; } } public void Modulo(PrimitiveColumnContainer column, ulong scalar) @@ -5043,6 +5438,11 @@ public void And(PrimitiveColumnContainer left, PrimitiveColumnContainer column, ulong scalar) @@ -5080,6 +5480,11 @@ public void Or(PrimitiveColumnContainer left, PrimitiveColumnContainer
    column, ulong scalar) @@ -5117,6 +5522,11 @@ public void Xor(PrimitiveColumnContainer left, PrimitiveColumnContainer column, ulong scalar) @@ -5331,6 +5741,11 @@ public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer< { span[i] = (ushort)(span[i] + otherSpan[i]); } + + var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); + var bitmapSpan = mutableBitMapBuffer.Span; + for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) + bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; } } public void Add(PrimitiveColumnContainer column, ushort scalar) @@ -5368,6 +5783,11 @@ public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnConta { span[i] = (ushort)(span[i] - otherSpan[i]); } + + var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); + var bitmapSpan = mutableBitMapBuffer.Span; + for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) + bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; } } public void Subtract(PrimitiveColumnContainer column, ushort scalar) @@ -5405,6 +5825,11 @@ public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnConta { span[i] = (ushort)(span[i] * otherSpan[i]); } + + var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); + var bitmapSpan = mutableBitMapBuffer.Span; + for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) + bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; } } public void Multiply(PrimitiveColumnContainer column, ushort scalar) @@ -5442,6 +5867,11 @@ public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContain { span[i] = (ushort)(span[i] / otherSpan[i]); } + + var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); + var bitmapSpan = mutableBitMapBuffer.Span; + for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) + bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; } } public void Divide(PrimitiveColumnContainer column, ushort scalar) @@ -5479,6 +5909,11 @@ public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContain { span[i] = (ushort)(span[i] % otherSpan[i]); } + + var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); + var bitmapSpan = mutableBitMapBuffer.Span; + for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) + bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; } } public void Modulo(PrimitiveColumnContainer column, ushort scalar) @@ -5516,6 +5951,11 @@ public void And(PrimitiveColumnContainer left, PrimitiveColumnContainer< { span[i] = (ushort)(span[i] & otherSpan[i]); } + + var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); + var bitmapSpan = mutableBitMapBuffer.Span; + for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) + bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; } } public void And(PrimitiveColumnContainer column, ushort scalar) @@ -5553,6 +5993,11 @@ public void Or(PrimitiveColumnContainer left, PrimitiveColumnContainer column, ushort scalar) @@ -5590,6 +6035,11 @@ public void Xor(PrimitiveColumnContainer left, PrimitiveColumnContainer< { span[i] = (ushort)(span[i] ^ otherSpan[i]); } + + var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); + var bitmapSpan = mutableBitMapBuffer.Span; + for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) + bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; } } public void Xor(PrimitiveColumnContainer column, ushort scalar) diff --git a/src/Microsoft.Data.Analysis/PrimitiveDataFrameColumnArithmetic.tt b/src/Microsoft.Data.Analysis/PrimitiveDataFrameColumnArithmetic.tt index a1543efd2d..ae5e306edd 100644 --- a/src/Microsoft.Data.Analysis/PrimitiveDataFrameColumnArithmetic.tt +++ b/src/Microsoft.Data.Analysis/PrimitiveDataFrameColumnArithmetic.tt @@ -89,6 +89,13 @@ namespace Microsoft.Data.Analysis throw new NotImplementedException(); <# } #> } +<# if (method.MethodType == MethodType.Binary) { #> + + var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); + var bitmapSpan = mutableBitMapBuffer.Span; + for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) + bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; +<# } #> } <# } #> } diff --git a/test/Microsoft.Data.Analysis.Tests/DataFrameTests.cs b/test/Microsoft.Data.Analysis.Tests/DataFrameTests.cs index 35bf334d6a..426df6e550 100644 --- a/test/Microsoft.Data.Analysis.Tests/DataFrameTests.cs +++ b/test/Microsoft.Data.Analysis.Tests/DataFrameTests.cs @@ -3620,5 +3620,56 @@ public void Test_StringColumnEqualsNull() Assert.Equal(2, filteredNullDf.Columns["index"][0]); Assert.Equal(5, filteredNullDf.Columns["index"][1]); } + + [Fact] + public void Test_ArithmeticsSumWithNull() + { + // Arrange + var left_column = new PrimitiveDataFrameColumn("Left", new int?[] { 1, 1, null, null }); + var right_column = new PrimitiveDataFrameColumn("Right", new int?[] { 1, null, 1, null }); + + // Act + var sum = left_column + right_column; + + // Assert + Assert.Equal(2, sum[0]); // 1 + 1 + Assert.Null(sum[1]); // null + 1 + Assert.Null(sum[2]); // 1 + null + Assert.Null(sum[3]); // null + null + } + + [Fact] + public void Test_ArithmeticsDiffWithNull() + { + // Arrange + var left_column = new PrimitiveDataFrameColumn("Left", new int?[] { 1, 1, null, null }); + var right_column = new PrimitiveDataFrameColumn("Right", new int?[] { 1, null, 1, null }); + + // Act + var diff = left_column - right_column; + + // Assert + Assert.Equal(0, diff[0]); // 1 - 1 + Assert.Null(diff[1]); // null - 1 + Assert.Null(diff[2]); // 1 - null + Assert.Null(diff[3]); // null - null + } + + [Fact] + public void Test_ArithmeticsMultWithNull() + { + // Arrange + var left_column = new PrimitiveDataFrameColumn("Left", new int?[] { 1, 1, null, null }); + var right_column = new PrimitiveDataFrameColumn("Right", new int?[] { 1, null, 1, null }); + + // Act + var mult = left_column * right_column; + + // Assert + Assert.Equal(1, mult[0]); // 1 * 1 + Assert.Null(mult[1]); // null * 1 + Assert.Null(mult[2]); // 1 * null + Assert.Null(mult[3]); // null * null + } } } From 0d09a5c282fe0d48ba863e87e51cd9b1da956bd6 Mon Sep 17 00:00:00 2001 From: Aleksei Smirnov Date: Thu, 20 Jul 2023 14:26:24 +0300 Subject: [PATCH 2/3] Fix Null Count and division by zero issues --- .../PrimitiveDataFrameColumnArithmetic.cs | 1530 +++++++++++------ .../PrimitiveDataFrameColumnArithmetic.tt | 20 +- .../DataFrameTests.cs | 22 + 3 files changed, 1023 insertions(+), 549 deletions(-) diff --git a/src/Microsoft.Data.Analysis/PrimitiveDataFrameColumnArithmetic.cs b/src/Microsoft.Data.Analysis/PrimitiveDataFrameColumnArithmetic.cs index 2fe8fd7f85..d9ba286a93 100644 --- a/src/Microsoft.Data.Analysis/PrimitiveDataFrameColumnArithmetic.cs +++ b/src/Microsoft.Data.Analysis/PrimitiveDataFrameColumnArithmetic.cs @@ -188,6 +188,7 @@ public void Modulo(bool scalar, PrimitiveColumnContainer column) } public void And(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -195,13 +196,17 @@ public void And(PrimitiveColumnContainer left, PrimitiveColumnContainer column, bool scalar) @@ -230,6 +235,7 @@ public void And(bool scalar, PrimitiveColumnContainer column) } public void Or(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -237,13 +243,17 @@ public void Or(PrimitiveColumnContainer left, PrimitiveColumnContainer column, bool scalar) @@ -272,6 +282,7 @@ public void Or(bool scalar, PrimitiveColumnContainer column) } public void Xor(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -279,13 +290,17 @@ public void Xor(PrimitiveColumnContainer left, PrimitiveColumnContainer column, bool scalar) @@ -407,6 +422,7 @@ internal class ByteArithmetic : IPrimitiveDataFrameColumnArithmetic { public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -414,13 +430,17 @@ public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer column, byte scalar) @@ -449,6 +469,7 @@ public void Add(byte scalar, PrimitiveColumnContainer column) } public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -456,13 +477,17 @@ public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnContain var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (byte)(span[i] - otherSpan[i]); - } + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (byte)(span[i] - otherSpan[i]); + } + else + { + left[index] = null; + } - var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); - var bitmapSpan = mutableBitMapBuffer.Span; - for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) - bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; + index++; + } } } public void Subtract(PrimitiveColumnContainer column, byte scalar) @@ -491,6 +516,7 @@ public void Subtract(byte scalar, PrimitiveColumnContainer column) } public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -498,13 +524,17 @@ public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnContain var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (byte)(span[i] * otherSpan[i]); - } + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (byte)(span[i] * otherSpan[i]); + } + else + { + left[index] = null; + } - var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); - var bitmapSpan = mutableBitMapBuffer.Span; - for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) - bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; + index++; + } } } public void Multiply(PrimitiveColumnContainer column, byte scalar) @@ -533,6 +563,7 @@ public void Multiply(byte scalar, PrimitiveColumnContainer column) } public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -540,13 +571,17 @@ public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContainer var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (byte)(span[i] / otherSpan[i]); - } + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (byte)(span[i] / otherSpan[i]); + } + else + { + left[index] = null; + } - var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); - var bitmapSpan = mutableBitMapBuffer.Span; - for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) - bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; + index++; + } } } public void Divide(PrimitiveColumnContainer column, byte scalar) @@ -575,6 +610,7 @@ public void Divide(byte scalar, PrimitiveColumnContainer column) } public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -582,13 +618,17 @@ public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContainer var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (byte)(span[i] % otherSpan[i]); - } + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (byte)(span[i] % otherSpan[i]); + } + else + { + left[index] = null; + } - var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); - var bitmapSpan = mutableBitMapBuffer.Span; - for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) - bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; + index++; + } } } public void Modulo(PrimitiveColumnContainer column, byte scalar) @@ -617,6 +657,7 @@ public void Modulo(byte scalar, PrimitiveColumnContainer column) } public void And(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -624,13 +665,17 @@ public void And(PrimitiveColumnContainer left, PrimitiveColumnContainer column, byte scalar) @@ -659,6 +704,7 @@ public void And(byte scalar, PrimitiveColumnContainer column) } public void Or(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -666,13 +712,17 @@ public void Or(PrimitiveColumnContainer left, PrimitiveColumnContainer column, byte scalar) @@ -701,6 +751,7 @@ public void Or(byte scalar, PrimitiveColumnContainer column) } public void Xor(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -708,13 +759,17 @@ public void Xor(PrimitiveColumnContainer left, PrimitiveColumnContainer column, byte scalar) @@ -920,6 +975,7 @@ internal class CharArithmetic : IPrimitiveDataFrameColumnArithmetic { public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -927,13 +983,17 @@ public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer column, char scalar) @@ -962,6 +1022,7 @@ public void Add(char scalar, PrimitiveColumnContainer column) } public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -969,13 +1030,17 @@ public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnContain var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (char)(span[i] - otherSpan[i]); - } + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (char)(span[i] - otherSpan[i]); + } + else + { + left[index] = null; + } - var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); - var bitmapSpan = mutableBitMapBuffer.Span; - for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) - bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; + index++; + } } } public void Subtract(PrimitiveColumnContainer column, char scalar) @@ -1004,6 +1069,7 @@ public void Subtract(char scalar, PrimitiveColumnContainer column) } public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -1011,13 +1077,17 @@ public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnContain var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (char)(span[i] * otherSpan[i]); - } + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (char)(span[i] * otherSpan[i]); + } + else + { + left[index] = null; + } - var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); - var bitmapSpan = mutableBitMapBuffer.Span; - for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) - bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; + index++; + } } } public void Multiply(PrimitiveColumnContainer column, char scalar) @@ -1046,6 +1116,7 @@ public void Multiply(char scalar, PrimitiveColumnContainer column) } public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -1053,13 +1124,17 @@ public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContainer var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (char)(span[i] / otherSpan[i]); - } + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (char)(span[i] / otherSpan[i]); + } + else + { + left[index] = null; + } - var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); - var bitmapSpan = mutableBitMapBuffer.Span; - for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) - bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; + index++; + } } } public void Divide(PrimitiveColumnContainer column, char scalar) @@ -1088,6 +1163,7 @@ public void Divide(char scalar, PrimitiveColumnContainer column) } public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -1095,13 +1171,17 @@ public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContainer var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (char)(span[i] % otherSpan[i]); - } + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (char)(span[i] % otherSpan[i]); + } + else + { + left[index] = null; + } - var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); - var bitmapSpan = mutableBitMapBuffer.Span; - for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) - bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; + index++; + } } } public void Modulo(PrimitiveColumnContainer column, char scalar) @@ -1130,6 +1210,7 @@ public void Modulo(char scalar, PrimitiveColumnContainer column) } public void And(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -1137,13 +1218,17 @@ public void And(PrimitiveColumnContainer left, PrimitiveColumnContainer column, char scalar) @@ -1172,6 +1257,7 @@ public void And(char scalar, PrimitiveColumnContainer column) } public void Or(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -1179,13 +1265,17 @@ public void Or(PrimitiveColumnContainer left, PrimitiveColumnContainer column, char scalar) @@ -1214,6 +1304,7 @@ public void Or(char scalar, PrimitiveColumnContainer column) } public void Xor(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -1221,13 +1312,17 @@ public void Xor(PrimitiveColumnContainer left, PrimitiveColumnContainer column, char scalar) @@ -1433,6 +1528,7 @@ internal class DecimalArithmetic : IPrimitiveDataFrameColumnArithmetic { public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -1440,13 +1536,17 @@ public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (decimal)(span[i] + otherSpan[i]); - } + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (decimal)(span[i] + otherSpan[i]); + } + else + { + left[index] = null; + } - var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); - var bitmapSpan = mutableBitMapBuffer.Span; - for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) - bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; + index++; + } } } public void Add(PrimitiveColumnContainer column, decimal scalar) @@ -1475,6 +1575,7 @@ public void Add(decimal scalar, PrimitiveColumnContainer column) } public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -1482,13 +1583,17 @@ public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnCont var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (decimal)(span[i] - otherSpan[i]); - } + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (decimal)(span[i] - otherSpan[i]); + } + else + { + left[index] = null; + } - var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); - var bitmapSpan = mutableBitMapBuffer.Span; - for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) - bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; + index++; + } } } public void Subtract(PrimitiveColumnContainer column, decimal scalar) @@ -1517,6 +1622,7 @@ public void Subtract(decimal scalar, PrimitiveColumnContainer column) } public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -1524,13 +1630,17 @@ public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnCont var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (decimal)(span[i] * otherSpan[i]); - } + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (decimal)(span[i] * otherSpan[i]); + } + else + { + left[index] = null; + } - var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); - var bitmapSpan = mutableBitMapBuffer.Span; - for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) - bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; + index++; + } } } public void Multiply(PrimitiveColumnContainer column, decimal scalar) @@ -1559,6 +1669,7 @@ public void Multiply(decimal scalar, PrimitiveColumnContainer column) } public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -1566,13 +1677,17 @@ public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContai var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (decimal)(span[i] / otherSpan[i]); - } + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (decimal)(span[i] / otherSpan[i]); + } + else + { + left[index] = null; + } - var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); - var bitmapSpan = mutableBitMapBuffer.Span; - for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) - bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; + index++; + } } } public void Divide(PrimitiveColumnContainer column, decimal scalar) @@ -1601,6 +1716,7 @@ public void Divide(decimal scalar, PrimitiveColumnContainer column) } public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -1608,13 +1724,17 @@ public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContai var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (decimal)(span[i] % otherSpan[i]); - } + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (decimal)(span[i] % otherSpan[i]); + } + else + { + left[index] = null; + } - var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); - var bitmapSpan = mutableBitMapBuffer.Span; - for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) - bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; + index++; + } } } public void Modulo(PrimitiveColumnContainer column, decimal scalar) @@ -1840,6 +1960,7 @@ internal class DoubleArithmetic : IPrimitiveDataFrameColumnArithmetic { public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -1847,13 +1968,17 @@ public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer< var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (double)(span[i] + otherSpan[i]); - } + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (double)(span[i] + otherSpan[i]); + } + else + { + left[index] = null; + } - var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); - var bitmapSpan = mutableBitMapBuffer.Span; - for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) - bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; + index++; + } } } public void Add(PrimitiveColumnContainer column, double scalar) @@ -1882,6 +2007,7 @@ public void Add(double scalar, PrimitiveColumnContainer column) } public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -1889,13 +2015,17 @@ public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnConta var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (double)(span[i] - otherSpan[i]); - } + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (double)(span[i] - otherSpan[i]); + } + else + { + left[index] = null; + } - var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); - var bitmapSpan = mutableBitMapBuffer.Span; - for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) - bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; + index++; + } } } public void Subtract(PrimitiveColumnContainer column, double scalar) @@ -1924,6 +2054,7 @@ public void Subtract(double scalar, PrimitiveColumnContainer column) } public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -1931,13 +2062,17 @@ public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnConta var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (double)(span[i] * otherSpan[i]); - } + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (double)(span[i] * otherSpan[i]); + } + else + { + left[index] = null; + } - var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); - var bitmapSpan = mutableBitMapBuffer.Span; - for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) - bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; + index++; + } } } public void Multiply(PrimitiveColumnContainer column, double scalar) @@ -1966,6 +2101,7 @@ public void Multiply(double scalar, PrimitiveColumnContainer column) } public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -1973,13 +2109,17 @@ public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContain var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (double)(span[i] / otherSpan[i]); - } + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (double)(span[i] / otherSpan[i]); + } + else + { + left[index] = null; + } - var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); - var bitmapSpan = mutableBitMapBuffer.Span; - for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) - bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; + index++; + } } } public void Divide(PrimitiveColumnContainer column, double scalar) @@ -2008,6 +2148,7 @@ public void Divide(double scalar, PrimitiveColumnContainer column) } public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -2015,13 +2156,17 @@ public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContain var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (double)(span[i] % otherSpan[i]); - } + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (double)(span[i] % otherSpan[i]); + } + else + { + left[index] = null; + } - var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); - var bitmapSpan = mutableBitMapBuffer.Span; - for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) - bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; + index++; + } } } public void Modulo(PrimitiveColumnContainer column, double scalar) @@ -2247,6 +2392,7 @@ internal class FloatArithmetic : IPrimitiveDataFrameColumnArithmetic { public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -2254,13 +2400,17 @@ public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer column, float scalar) @@ -2289,6 +2439,7 @@ public void Add(float scalar, PrimitiveColumnContainer column) } public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -2296,13 +2447,17 @@ public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnContai var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (float)(span[i] - otherSpan[i]); - } + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (float)(span[i] - otherSpan[i]); + } + else + { + left[index] = null; + } - var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); - var bitmapSpan = mutableBitMapBuffer.Span; - for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) - bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; + index++; + } } } public void Subtract(PrimitiveColumnContainer column, float scalar) @@ -2331,6 +2486,7 @@ public void Subtract(float scalar, PrimitiveColumnContainer column) } public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -2338,13 +2494,17 @@ public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnContai var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (float)(span[i] * otherSpan[i]); - } + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (float)(span[i] * otherSpan[i]); + } + else + { + left[index] = null; + } - var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); - var bitmapSpan = mutableBitMapBuffer.Span; - for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) - bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; + index++; + } } } public void Multiply(PrimitiveColumnContainer column, float scalar) @@ -2373,6 +2533,7 @@ public void Multiply(float scalar, PrimitiveColumnContainer column) } public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -2380,13 +2541,17 @@ public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContaine var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (float)(span[i] / otherSpan[i]); - } + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (float)(span[i] / otherSpan[i]); + } + else + { + left[index] = null; + } - var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); - var bitmapSpan = mutableBitMapBuffer.Span; - for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) - bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; + index++; + } } } public void Divide(PrimitiveColumnContainer column, float scalar) @@ -2415,6 +2580,7 @@ public void Divide(float scalar, PrimitiveColumnContainer column) } public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -2422,13 +2588,17 @@ public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContaine var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (float)(span[i] % otherSpan[i]); - } + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (float)(span[i] % otherSpan[i]); + } + else + { + left[index] = null; + } - var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); - var bitmapSpan = mutableBitMapBuffer.Span; - for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) - bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; + index++; + } } } public void Modulo(PrimitiveColumnContainer column, float scalar) @@ -2654,6 +2824,7 @@ internal class IntArithmetic : IPrimitiveDataFrameColumnArithmetic { public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -2661,13 +2832,17 @@ public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer column, int scalar) @@ -2696,6 +2871,7 @@ public void Add(int scalar, PrimitiveColumnContainer column) } public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -2703,13 +2879,17 @@ public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnContaine var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (int)(span[i] - otherSpan[i]); - } + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (int)(span[i] - otherSpan[i]); + } + else + { + left[index] = null; + } - var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); - var bitmapSpan = mutableBitMapBuffer.Span; - for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) - bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; + index++; + } } } public void Subtract(PrimitiveColumnContainer column, int scalar) @@ -2738,6 +2918,7 @@ public void Subtract(int scalar, PrimitiveColumnContainer column) } public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -2745,13 +2926,17 @@ public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnContaine var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (int)(span[i] * otherSpan[i]); - } + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (int)(span[i] * otherSpan[i]); + } + else + { + left[index] = null; + } - var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); - var bitmapSpan = mutableBitMapBuffer.Span; - for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) - bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; + index++; + } } } public void Multiply(PrimitiveColumnContainer column, int scalar) @@ -2780,6 +2965,7 @@ public void Multiply(int scalar, PrimitiveColumnContainer column) } public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -2787,13 +2973,17 @@ public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContainer< var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (int)(span[i] / otherSpan[i]); - } + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (int)(span[i] / otherSpan[i]); + } + else + { + left[index] = null; + } - var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); - var bitmapSpan = mutableBitMapBuffer.Span; - for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) - bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; + index++; + } } } public void Divide(PrimitiveColumnContainer column, int scalar) @@ -2822,6 +3012,7 @@ public void Divide(int scalar, PrimitiveColumnContainer column) } public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -2829,13 +3020,17 @@ public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContainer< var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (int)(span[i] % otherSpan[i]); - } + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (int)(span[i] % otherSpan[i]); + } + else + { + left[index] = null; + } - var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); - var bitmapSpan = mutableBitMapBuffer.Span; - for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) - bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; + index++; + } } } public void Modulo(PrimitiveColumnContainer column, int scalar) @@ -2864,6 +3059,7 @@ public void Modulo(int scalar, PrimitiveColumnContainer column) } public void And(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -2871,13 +3067,17 @@ public void And(PrimitiveColumnContainer left, PrimitiveColumnContainer column, int scalar) @@ -2906,6 +3106,7 @@ public void And(int scalar, PrimitiveColumnContainer column) } public void Or(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -2913,13 +3114,17 @@ public void Or(PrimitiveColumnContainer left, PrimitiveColumnContainer var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (int)(span[i] | otherSpan[i]); - } + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (int)(span[i] | otherSpan[i]); + } + else + { + left[index] = null; + } - var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); - var bitmapSpan = mutableBitMapBuffer.Span; - for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) - bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; + index++; + } } } public void Or(PrimitiveColumnContainer column, int scalar) @@ -2948,6 +3153,7 @@ public void Or(int scalar, PrimitiveColumnContainer column) } public void Xor(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -2955,13 +3161,17 @@ public void Xor(PrimitiveColumnContainer left, PrimitiveColumnContainer column, int scalar) @@ -3167,6 +3377,7 @@ internal class LongArithmetic : IPrimitiveDataFrameColumnArithmetic { public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -3174,13 +3385,17 @@ public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer column, long scalar) @@ -3209,6 +3424,7 @@ public void Add(long scalar, PrimitiveColumnContainer column) } public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -3216,13 +3432,17 @@ public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnContain var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (long)(span[i] - otherSpan[i]); - } + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (long)(span[i] - otherSpan[i]); + } + else + { + left[index] = null; + } - var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); - var bitmapSpan = mutableBitMapBuffer.Span; - for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) - bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; + index++; + } } } public void Subtract(PrimitiveColumnContainer column, long scalar) @@ -3251,6 +3471,7 @@ public void Subtract(long scalar, PrimitiveColumnContainer column) } public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -3258,13 +3479,17 @@ public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnContain var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (long)(span[i] * otherSpan[i]); - } + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (long)(span[i] * otherSpan[i]); + } + else + { + left[index] = null; + } - var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); - var bitmapSpan = mutableBitMapBuffer.Span; - for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) - bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; + index++; + } } } public void Multiply(PrimitiveColumnContainer column, long scalar) @@ -3293,6 +3518,7 @@ public void Multiply(long scalar, PrimitiveColumnContainer column) } public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -3300,13 +3526,17 @@ public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContainer var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (long)(span[i] / otherSpan[i]); - } + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (long)(span[i] / otherSpan[i]); + } + else + { + left[index] = null; + } - var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); - var bitmapSpan = mutableBitMapBuffer.Span; - for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) - bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; + index++; + } } } public void Divide(PrimitiveColumnContainer column, long scalar) @@ -3335,6 +3565,7 @@ public void Divide(long scalar, PrimitiveColumnContainer column) } public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -3342,13 +3573,17 @@ public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContainer var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (long)(span[i] % otherSpan[i]); - } + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (long)(span[i] % otherSpan[i]); + } + else + { + left[index] = null; + } - var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); - var bitmapSpan = mutableBitMapBuffer.Span; - for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) - bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; + index++; + } } } public void Modulo(PrimitiveColumnContainer column, long scalar) @@ -3377,6 +3612,7 @@ public void Modulo(long scalar, PrimitiveColumnContainer column) } public void And(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -3384,13 +3620,17 @@ public void And(PrimitiveColumnContainer left, PrimitiveColumnContainer column, long scalar) @@ -3419,6 +3659,7 @@ public void And(long scalar, PrimitiveColumnContainer column) } public void Or(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -3426,13 +3667,17 @@ public void Or(PrimitiveColumnContainer left, PrimitiveColumnContainer column, long scalar) @@ -3461,6 +3706,7 @@ public void Or(long scalar, PrimitiveColumnContainer column) } public void Xor(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -3468,13 +3714,17 @@ public void Xor(PrimitiveColumnContainer left, PrimitiveColumnContainer column, long scalar) @@ -3680,6 +3930,7 @@ internal class SByteArithmetic : IPrimitiveDataFrameColumnArithmetic { public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -3687,13 +3938,17 @@ public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer column, sbyte scalar) @@ -3722,6 +3977,7 @@ public void Add(sbyte scalar, PrimitiveColumnContainer column) } public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -3729,13 +3985,17 @@ public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnContai var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (sbyte)(span[i] - otherSpan[i]); - } + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (sbyte)(span[i] - otherSpan[i]); + } + else + { + left[index] = null; + } - var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); - var bitmapSpan = mutableBitMapBuffer.Span; - for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) - bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; + index++; + } } } public void Subtract(PrimitiveColumnContainer column, sbyte scalar) @@ -3764,6 +4024,7 @@ public void Subtract(sbyte scalar, PrimitiveColumnContainer column) } public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -3771,13 +4032,17 @@ public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnContai var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (sbyte)(span[i] * otherSpan[i]); - } + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (sbyte)(span[i] * otherSpan[i]); + } + else + { + left[index] = null; + } - var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); - var bitmapSpan = mutableBitMapBuffer.Span; - for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) - bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; + index++; + } } } public void Multiply(PrimitiveColumnContainer column, sbyte scalar) @@ -3806,6 +4071,7 @@ public void Multiply(sbyte scalar, PrimitiveColumnContainer column) } public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -3813,13 +4079,17 @@ public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContaine var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (sbyte)(span[i] / otherSpan[i]); - } + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (sbyte)(span[i] / otherSpan[i]); + } + else + { + left[index] = null; + } - var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); - var bitmapSpan = mutableBitMapBuffer.Span; - for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) - bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; + index++; + } } } public void Divide(PrimitiveColumnContainer column, sbyte scalar) @@ -3848,6 +4118,7 @@ public void Divide(sbyte scalar, PrimitiveColumnContainer column) } public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -3855,13 +4126,17 @@ public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContaine var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (sbyte)(span[i] % otherSpan[i]); - } + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (sbyte)(span[i] % otherSpan[i]); + } + else + { + left[index] = null; + } - var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); - var bitmapSpan = mutableBitMapBuffer.Span; - for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) - bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; + index++; + } } } public void Modulo(PrimitiveColumnContainer column, sbyte scalar) @@ -3890,6 +4165,7 @@ public void Modulo(sbyte scalar, PrimitiveColumnContainer column) } public void And(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -3897,13 +4173,17 @@ public void And(PrimitiveColumnContainer left, PrimitiveColumnContainer column, sbyte scalar) @@ -3932,6 +4212,7 @@ public void And(sbyte scalar, PrimitiveColumnContainer column) } public void Or(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -3939,13 +4220,17 @@ public void Or(PrimitiveColumnContainer left, PrimitiveColumnContainer column, sbyte scalar) @@ -3974,6 +4259,7 @@ public void Or(sbyte scalar, PrimitiveColumnContainer column) } public void Xor(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -3981,13 +4267,17 @@ public void Xor(PrimitiveColumnContainer left, PrimitiveColumnContainer column, sbyte scalar) @@ -4193,6 +4483,7 @@ internal class ShortArithmetic : IPrimitiveDataFrameColumnArithmetic { public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -4200,13 +4491,17 @@ public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer column, short scalar) @@ -4235,6 +4530,7 @@ public void Add(short scalar, PrimitiveColumnContainer column) } public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -4242,13 +4538,17 @@ public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnContai var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (short)(span[i] - otherSpan[i]); - } + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (short)(span[i] - otherSpan[i]); + } + else + { + left[index] = null; + } - var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); - var bitmapSpan = mutableBitMapBuffer.Span; - for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) - bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; + index++; + } } } public void Subtract(PrimitiveColumnContainer column, short scalar) @@ -4277,6 +4577,7 @@ public void Subtract(short scalar, PrimitiveColumnContainer column) } public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -4284,13 +4585,17 @@ public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnContai var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (short)(span[i] * otherSpan[i]); - } + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (short)(span[i] * otherSpan[i]); + } + else + { + left[index] = null; + } - var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); - var bitmapSpan = mutableBitMapBuffer.Span; - for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) - bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; + index++; + } } } public void Multiply(PrimitiveColumnContainer column, short scalar) @@ -4319,6 +4624,7 @@ public void Multiply(short scalar, PrimitiveColumnContainer column) } public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -4326,13 +4632,17 @@ public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContaine var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (short)(span[i] / otherSpan[i]); - } + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (short)(span[i] / otherSpan[i]); + } + else + { + left[index] = null; + } - var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); - var bitmapSpan = mutableBitMapBuffer.Span; - for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) - bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; + index++; + } } } public void Divide(PrimitiveColumnContainer column, short scalar) @@ -4361,6 +4671,7 @@ public void Divide(short scalar, PrimitiveColumnContainer column) } public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -4368,13 +4679,17 @@ public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContaine var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (short)(span[i] % otherSpan[i]); - } + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (short)(span[i] % otherSpan[i]); + } + else + { + left[index] = null; + } - var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); - var bitmapSpan = mutableBitMapBuffer.Span; - for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) - bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; + index++; + } } } public void Modulo(PrimitiveColumnContainer column, short scalar) @@ -4403,6 +4718,7 @@ public void Modulo(short scalar, PrimitiveColumnContainer column) } public void And(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -4410,13 +4726,17 @@ public void And(PrimitiveColumnContainer left, PrimitiveColumnContainer column, short scalar) @@ -4445,6 +4765,7 @@ public void And(short scalar, PrimitiveColumnContainer column) } public void Or(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -4452,13 +4773,17 @@ public void Or(PrimitiveColumnContainer left, PrimitiveColumnContainer column, short scalar) @@ -4487,6 +4812,7 @@ public void Or(short scalar, PrimitiveColumnContainer column) } public void Xor(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -4494,13 +4820,17 @@ public void Xor(PrimitiveColumnContainer left, PrimitiveColumnContainer column, short scalar) @@ -4706,6 +5036,7 @@ internal class UIntArithmetic : IPrimitiveDataFrameColumnArithmetic { public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -4713,13 +5044,17 @@ public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer column, uint scalar) @@ -4748,6 +5083,7 @@ public void Add(uint scalar, PrimitiveColumnContainer column) } public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -4755,13 +5091,17 @@ public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnContain var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (uint)(span[i] - otherSpan[i]); - } + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (uint)(span[i] - otherSpan[i]); + } + else + { + left[index] = null; + } - var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); - var bitmapSpan = mutableBitMapBuffer.Span; - for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) - bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; + index++; + } } } public void Subtract(PrimitiveColumnContainer column, uint scalar) @@ -4790,6 +5130,7 @@ public void Subtract(uint scalar, PrimitiveColumnContainer column) } public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -4797,13 +5138,17 @@ public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnContain var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (uint)(span[i] * otherSpan[i]); - } + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (uint)(span[i] * otherSpan[i]); + } + else + { + left[index] = null; + } - var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); - var bitmapSpan = mutableBitMapBuffer.Span; - for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) - bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; + index++; + } } } public void Multiply(PrimitiveColumnContainer column, uint scalar) @@ -4832,6 +5177,7 @@ public void Multiply(uint scalar, PrimitiveColumnContainer column) } public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -4839,13 +5185,17 @@ public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContainer var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (uint)(span[i] / otherSpan[i]); - } + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (uint)(span[i] / otherSpan[i]); + } + else + { + left[index] = null; + } - var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); - var bitmapSpan = mutableBitMapBuffer.Span; - for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) - bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; + index++; + } } } public void Divide(PrimitiveColumnContainer column, uint scalar) @@ -4874,6 +5224,7 @@ public void Divide(uint scalar, PrimitiveColumnContainer column) } public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -4881,13 +5232,17 @@ public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContainer var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (uint)(span[i] % otherSpan[i]); - } + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (uint)(span[i] % otherSpan[i]); + } + else + { + left[index] = null; + } - var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); - var bitmapSpan = mutableBitMapBuffer.Span; - for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) - bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; + index++; + } } } public void Modulo(PrimitiveColumnContainer column, uint scalar) @@ -4916,6 +5271,7 @@ public void Modulo(uint scalar, PrimitiveColumnContainer column) } public void And(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -4923,13 +5279,17 @@ public void And(PrimitiveColumnContainer left, PrimitiveColumnContainer column, uint scalar) @@ -4958,6 +5318,7 @@ public void And(uint scalar, PrimitiveColumnContainer column) } public void Or(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -4965,13 +5326,17 @@ public void Or(PrimitiveColumnContainer left, PrimitiveColumnContainer column, uint scalar) @@ -5000,6 +5365,7 @@ public void Or(uint scalar, PrimitiveColumnContainer column) } public void Xor(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -5007,13 +5373,17 @@ public void Xor(PrimitiveColumnContainer left, PrimitiveColumnContainer column, uint scalar) @@ -5219,6 +5589,7 @@ internal class ULongArithmetic : IPrimitiveDataFrameColumnArithmetic { public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -5226,13 +5597,17 @@ public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer column, ulong scalar) @@ -5261,6 +5636,7 @@ public void Add(ulong scalar, PrimitiveColumnContainer column) } public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -5268,13 +5644,17 @@ public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnContai var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (ulong)(span[i] - otherSpan[i]); - } + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (ulong)(span[i] - otherSpan[i]); + } + else + { + left[index] = null; + } - var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); - var bitmapSpan = mutableBitMapBuffer.Span; - for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) - bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; + index++; + } } } public void Subtract(PrimitiveColumnContainer column, ulong scalar) @@ -5303,6 +5683,7 @@ public void Subtract(ulong scalar, PrimitiveColumnContainer column) } public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -5310,13 +5691,17 @@ public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnContai var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (ulong)(span[i] * otherSpan[i]); - } + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (ulong)(span[i] * otherSpan[i]); + } + else + { + left[index] = null; + } - var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); - var bitmapSpan = mutableBitMapBuffer.Span; - for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) - bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; + index++; + } } } public void Multiply(PrimitiveColumnContainer column, ulong scalar) @@ -5345,6 +5730,7 @@ public void Multiply(ulong scalar, PrimitiveColumnContainer column) } public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -5352,13 +5738,17 @@ public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContaine var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (ulong)(span[i] / otherSpan[i]); - } + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (ulong)(span[i] / otherSpan[i]); + } + else + { + left[index] = null; + } - var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); - var bitmapSpan = mutableBitMapBuffer.Span; - for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) - bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; + index++; + } } } public void Divide(PrimitiveColumnContainer column, ulong scalar) @@ -5387,6 +5777,7 @@ public void Divide(ulong scalar, PrimitiveColumnContainer column) } public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -5394,13 +5785,17 @@ public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContaine var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (ulong)(span[i] % otherSpan[i]); - } + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (ulong)(span[i] % otherSpan[i]); + } + else + { + left[index] = null; + } - var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); - var bitmapSpan = mutableBitMapBuffer.Span; - for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) - bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; + index++; + } } } public void Modulo(PrimitiveColumnContainer column, ulong scalar) @@ -5429,6 +5824,7 @@ public void Modulo(ulong scalar, PrimitiveColumnContainer column) } public void And(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -5436,13 +5832,17 @@ public void And(PrimitiveColumnContainer left, PrimitiveColumnContainer column, ulong scalar) @@ -5471,6 +5871,7 @@ public void And(ulong scalar, PrimitiveColumnContainer column) } public void Or(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -5478,13 +5879,17 @@ public void Or(PrimitiveColumnContainer left, PrimitiveColumnContainer
      column, ulong scalar) @@ -5513,6 +5918,7 @@ public void Or(ulong scalar, PrimitiveColumnContainer column) } public void Xor(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -5520,13 +5926,17 @@ public void Xor(PrimitiveColumnContainer left, PrimitiveColumnContainer column, ulong scalar) @@ -5732,6 +6142,7 @@ internal class UShortArithmetic : IPrimitiveDataFrameColumnArithmetic { public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -5739,13 +6150,17 @@ public void Add(PrimitiveColumnContainer left, PrimitiveColumnContainer< var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (ushort)(span[i] + otherSpan[i]); - } + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (ushort)(span[i] + otherSpan[i]); + } + else + { + left[index] = null; + } - var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); - var bitmapSpan = mutableBitMapBuffer.Span; - for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) - bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; + index++; + } } } public void Add(PrimitiveColumnContainer column, ushort scalar) @@ -5774,6 +6189,7 @@ public void Add(ushort scalar, PrimitiveColumnContainer column) } public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -5781,13 +6197,17 @@ public void Subtract(PrimitiveColumnContainer left, PrimitiveColumnConta var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (ushort)(span[i] - otherSpan[i]); - } + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (ushort)(span[i] - otherSpan[i]); + } + else + { + left[index] = null; + } - var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); - var bitmapSpan = mutableBitMapBuffer.Span; - for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) - bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; + index++; + } } } public void Subtract(PrimitiveColumnContainer column, ushort scalar) @@ -5816,6 +6236,7 @@ public void Subtract(ushort scalar, PrimitiveColumnContainer column) } public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -5823,13 +6244,17 @@ public void Multiply(PrimitiveColumnContainer left, PrimitiveColumnConta var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (ushort)(span[i] * otherSpan[i]); - } + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (ushort)(span[i] * otherSpan[i]); + } + else + { + left[index] = null; + } - var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); - var bitmapSpan = mutableBitMapBuffer.Span; - for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) - bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; + index++; + } } } public void Multiply(PrimitiveColumnContainer column, ushort scalar) @@ -5858,6 +6283,7 @@ public void Multiply(ushort scalar, PrimitiveColumnContainer column) } public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -5865,13 +6291,17 @@ public void Divide(PrimitiveColumnContainer left, PrimitiveColumnContain var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (ushort)(span[i] / otherSpan[i]); - } + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (ushort)(span[i] / otherSpan[i]); + } + else + { + left[index] = null; + } - var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); - var bitmapSpan = mutableBitMapBuffer.Span; - for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) - bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; + index++; + } } } public void Divide(PrimitiveColumnContainer column, ushort scalar) @@ -5900,6 +6330,7 @@ public void Divide(ushort scalar, PrimitiveColumnContainer column) } public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -5907,13 +6338,17 @@ public void Modulo(PrimitiveColumnContainer left, PrimitiveColumnContain var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (ushort)(span[i] % otherSpan[i]); - } + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (ushort)(span[i] % otherSpan[i]); + } + else + { + left[index] = null; + } - var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); - var bitmapSpan = mutableBitMapBuffer.Span; - for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) - bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; + index++; + } } } public void Modulo(PrimitiveColumnContainer column, ushort scalar) @@ -5942,6 +6377,7 @@ public void Modulo(ushort scalar, PrimitiveColumnContainer column) } public void And(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -5949,13 +6385,17 @@ public void And(PrimitiveColumnContainer left, PrimitiveColumnContainer< var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (ushort)(span[i] & otherSpan[i]); - } + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (ushort)(span[i] & otherSpan[i]); + } + else + { + left[index] = null; + } - var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); - var bitmapSpan = mutableBitMapBuffer.Span; - for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) - bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; + index++; + } } } public void And(PrimitiveColumnContainer column, ushort scalar) @@ -5984,6 +6424,7 @@ public void And(ushort scalar, PrimitiveColumnContainer column) } public void Or(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -5991,13 +6432,17 @@ public void Or(PrimitiveColumnContainer left, PrimitiveColumnContainer column, ushort scalar) @@ -6026,6 +6471,7 @@ public void Or(ushort scalar, PrimitiveColumnContainer column) } public void Xor(PrimitiveColumnContainer left, PrimitiveColumnContainer right) { + long index = 0; for (int b = 0; b < left.Buffers.Count; b++) { var mutableBuffer = left.Buffers.GetOrCreateMutable(b); @@ -6033,13 +6479,17 @@ public void Xor(PrimitiveColumnContainer left, PrimitiveColumnContainer< var otherSpan = right.Buffers[b].ReadOnlySpan; for (int i = 0; i < span.Length; i++) { - span[i] = (ushort)(span[i] ^ otherSpan[i]); - } + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (ushort)(span[i] ^ otherSpan[i]); + } + else + { + left[index] = null; + } - var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); - var bitmapSpan = mutableBitMapBuffer.Span; - for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) - bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; + index++; + } } } public void Xor(PrimitiveColumnContainer column, ushort scalar) diff --git a/src/Microsoft.Data.Analysis/PrimitiveDataFrameColumnArithmetic.tt b/src/Microsoft.Data.Analysis/PrimitiveDataFrameColumnArithmetic.tt index ae5e306edd..dc95566e60 100644 --- a/src/Microsoft.Data.Analysis/PrimitiveDataFrameColumnArithmetic.tt +++ b/src/Microsoft.Data.Analysis/PrimitiveDataFrameColumnArithmetic.tt @@ -58,7 +58,7 @@ namespace Microsoft.Data.Analysis <# if ((method.IsNumeric && !type.SupportsNumeric) || (method.IsBitwise && !type.SupportsBitwise) || (type.UnsupportedMethods.Contains(method.MethodName))) { #> throw new NotSupportedException(); <# } else if (method.Operator != null) { #> -<# if (method.MethodType == MethodType.Comparison || method.MethodType == MethodType.ComparisonScalar) { #> +<# if (method.MethodType == MethodType.Comparison || method.MethodType == MethodType.ComparisonScalar || method.MethodType == MethodType.Binary) { #> long index = 0; <# } #> for (int b = 0; b < <#= method.Op1Name #>.Buffers.Count; b++) @@ -80,7 +80,16 @@ namespace Microsoft.Data.Analysis <# if (method.MethodType == MethodType.BinaryScalar || method.MethodType == MethodType.BinaryInt) { #> span[i] = (<#=type.TypeName#>)(span[i] <#= method.Operator #> <#= method.Op2Name #>); <# } else if (method.MethodType == MethodType.Binary) { #> - span[i] = (<#=type.TypeName#>)(span[i] <#= method.Operator #> otherSpan[i]); + if (BitmapHelper.IsValid(right.NullBitMapBuffers[b].ReadOnlySpan, i)) + { + span[i] = (<#=type.TypeName#>)(span[i] <#= method.Operator #> otherSpan[i]); + } + else + { + left[index] = null; + } + + index++; <# } else if (method.MethodType == MethodType.Comparison) { #> ret[index++] = (span[i] <#= method.Operator #> otherSpan[i]); <# } else if (method.MethodType == MethodType.ComparisonScalar) { #> @@ -89,13 +98,6 @@ namespace Microsoft.Data.Analysis throw new NotImplementedException(); <# } #> } -<# if (method.MethodType == MethodType.Binary) { #> - - var mutableBitMapBuffer = left.NullBitMapBuffers.GetOrCreateMutable(b); - var bitmapSpan = mutableBitMapBuffer.Span; - for (int i = 0; i < left.NullBitMapBuffers[b].Length; i++) - bitmapSpan[i] &= right.NullBitMapBuffers[b][i]; -<# } #> } <# } #> } diff --git a/test/Microsoft.Data.Analysis.Tests/DataFrameTests.cs b/test/Microsoft.Data.Analysis.Tests/DataFrameTests.cs index 426df6e550..423a6ef1b1 100644 --- a/test/Microsoft.Data.Analysis.Tests/DataFrameTests.cs +++ b/test/Microsoft.Data.Analysis.Tests/DataFrameTests.cs @@ -3632,6 +3632,8 @@ public void Test_ArithmeticsSumWithNull() var sum = left_column + right_column; // Assert + Assert.Equal(3, sum.NullCount); + Assert.Equal(2, sum[0]); // 1 + 1 Assert.Null(sum[1]); // null + 1 Assert.Null(sum[2]); // 1 + null @@ -3649,6 +3651,7 @@ public void Test_ArithmeticsDiffWithNull() var diff = left_column - right_column; // Assert + Assert.Equal(3, diff.NullCount); Assert.Equal(0, diff[0]); // 1 - 1 Assert.Null(diff[1]); // null - 1 Assert.Null(diff[2]); // 1 - null @@ -3666,10 +3669,29 @@ public void Test_ArithmeticsMultWithNull() var mult = left_column * right_column; // Assert + Assert.Equal(3, mult.NullCount); Assert.Equal(1, mult[0]); // 1 * 1 Assert.Null(mult[1]); // null * 1 Assert.Null(mult[2]); // 1 * null Assert.Null(mult[3]); // null * null } + + [Fact] + public void Test_ArithmeticsDivWithNull() + { + // Arrange + var left_column = new PrimitiveDataFrameColumn("Left", new int?[] { 1, 1, null, null }); + var right_column = new PrimitiveDataFrameColumn("Right", new int?[] { 1, null, 1, null }); + + // Act + var mult = left_column / right_column; + + // Assert + Assert.Equal(3, mult.NullCount); + Assert.Equal(1, mult[0]); // 1 / 1 + Assert.Null(mult[1]); // null / 1 + Assert.Null(mult[2]); // 1 / null + Assert.Null(mult[3]); // null / null + } } } From 99bbe5d73ec2caf253cd9bc9b8f880ead6f54574 Mon Sep 17 00:00:00 2001 From: Aleksei Smirnov Date: Wed, 30 Aug 2023 21:30:29 +0300 Subject: [PATCH 3/3] Minor changes to restart build and rerun flaky tests --- .../PrimitiveColumnContainerHelpers.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Microsoft.Data.Analysis/PrimitiveColumnContainerHelpers.cs b/src/Microsoft.Data.Analysis/PrimitiveColumnContainerHelpers.cs index b4da9a311a..eb6edc0aff 100644 --- a/src/Microsoft.Data.Analysis/PrimitiveColumnContainerHelpers.cs +++ b/src/Microsoft.Data.Analysis/PrimitiveColumnContainerHelpers.cs @@ -11,10 +11,9 @@ internal static class PrimitiveColumnContainerHelpers internal static DataFrameBuffer GetOrCreateMutable(this IList> bufferList, int index) where T : unmanaged { - ReadOnlyDataFrameBuffer sourceBuffer = bufferList[index]; - DataFrameBuffer mutableBuffer = sourceBuffer as DataFrameBuffer; + var sourceBuffer = bufferList[index]; - if (mutableBuffer == null) + if (sourceBuffer is not DataFrameBuffer mutableBuffer) { mutableBuffer = DataFrameBuffer.GetMutableBuffer(sourceBuffer); bufferList[index] = mutableBuffer;