fix isin with nans and large arrays

Author: Hanspagh

pandas/core/algorithms.py

@@ -440,7 +440,12 @@ def isin(comps: AnyArrayLike, values: AnyArrayLike) -> np.ndarray:
     # GH16012
     # Ensure np.in1d doesn't get object types or it *may* throw an exception
     if len(comps) > 1_000_000 and not is_object_dtype(comps):
-        f = np.in1d
+        # If the the values include nan we need to check for nan explicitly
+        # since np.nan it not equal to np.nan
+        if any(np.isnan(values)):
+            f = lambda c, v: np.logical_or(np.in1d(c, v), np.isnan(c))
+        else:
+            f = np.in1d
     elif is_integer_dtype(comps):
         try:
             values = values.astype("int64", copy=False)

pandas/tests/test_algos.py

@@ -801,7 +801,6 @@ def test_i8(self):
         tm.assert_numpy_array_equal(result, expected)
 
     def test_large(self):
-
         s = pd.date_range("20000101", periods=2000000, freq="s").values
         result = algos.isin(s, s[0:2])
         expected = np.zeros(len(s), dtype=bool)
@@ -841,6 +840,13 @@ def test_same_nan_is_in(self):
         result = algos.isin(comps, values)
         tm.assert_numpy_array_equal(expected, result)
 
+    def test_same_nan_is_in_large(self):
+        s = np.tile(1.0, 1_000_001)
+        s[0] = np.nan
+        result = algos.isin(s, [np.nan, 1])
+        expected = np.ones(len(s), dtype=bool)
+        tm.assert_numpy_array_equal(result, expected)
+
     def test_same_object_is_in(self):
         # GH 22160
         # there could be special treatment for nans