Merge pull request #142 from pandas-dev/master

Sync Fork from Upstream Repo

Author: sthagen

pandas/_libs/parsers.pyx

@@ -337,7 +337,7 @@ cdef class TextReader:
         object skiprows
         object dtype
         object usecols
-        list dtype_cast_order
+        list dtype_cast_order  # list[np.dtype]
         set unnamed_cols
         set noconvert
 

pandas/_testing/asserters.py

@@ -976,8 +976,8 @@ def assert_series_equal(
         left_values = left._values
         right_values = right._values
         # Only check exact if dtype is numeric
-        if is_extension_array_dtype(left_values) and is_extension_array_dtype(
-            right_values
+        if isinstance(left_values, ExtensionArray) and isinstance(
+            right_values, ExtensionArray
         ):
             assert_extension_array_equal(
                 left_values,

pandas/core/algorithms.py

@@ -235,41 +235,26 @@ def _reconstruct_data(
         # Catch DatetimeArray/TimedeltaArray
         return values
 
-    if is_extension_array_dtype(dtype):
-        # error: Item "dtype[Any]" of "Union[dtype[Any], ExtensionDtype]" has no
-        # attribute "construct_array_type"
-        cls = dtype.construct_array_type()  # type: ignore[union-attr]
+    if not isinstance(dtype, np.dtype):
+        # i.e. ExtensionDtype
+        cls = dtype.construct_array_type()
         if isinstance(values, cls) and values.dtype == dtype:
             return values
 
         values = cls._from_sequence(values)
     elif is_bool_dtype(dtype):
-        # error: Argument 1 to "astype" of "_ArrayOrScalarCommon" has
-        # incompatible type "Union[dtype, ExtensionDtype]"; expected
-        # "Union[dtype, None, type, _SupportsDtype, str, Tuple[Any, int],
-        # Tuple[Any, Union[int, Sequence[int]]], List[Any], _DtypeDict,
-        # Tuple[Any, Any]]"
-        values = values.astype(dtype, copy=False)  # type: ignore[arg-type]
+        values = values.astype(dtype, copy=False)
 
         # we only support object dtypes bool Index
         if isinstance(original, ABCIndex):
             values = values.astype(object, copy=False)
     elif dtype is not None:
         if is_datetime64_dtype(dtype):
-            # error: Incompatible types in assignment (expression has type
-            # "str", variable has type "Union[dtype, ExtensionDtype]")
-            dtype = "datetime64[ns]"  # type: ignore[assignment]
+            dtype = np.dtype("datetime64[ns]")
         elif is_timedelta64_dtype(dtype):
-            # error: Incompatible types in assignment (expression has type
-            # "str", variable has type "Union[dtype, ExtensionDtype]")
-            dtype = "timedelta64[ns]"  # type: ignore[assignment]
+            dtype = np.dtype("timedelta64[ns]")
 
-        # error: Argument 1 to "astype" of "_ArrayOrScalarCommon" has
-        # incompatible type "Union[dtype, ExtensionDtype]"; expected
-        # "Union[dtype, None, type, _SupportsDtype, str, Tuple[Any, int],
-        # Tuple[Any, Union[int, Sequence[int]]], List[Any], _DtypeDict,
-        # Tuple[Any, Any]]"
-        values = values.astype(dtype, copy=False)  # type: ignore[arg-type]
+        values = values.astype(dtype, copy=False)
 
     return values
 
@@ -772,7 +757,8 @@ def factorize(
             uniques = Index(uniques)
         return codes, uniques
 
-    if is_extension_array_dtype(values.dtype):
+    if not isinstance(values.dtype, np.dtype):
+        # i.e. ExtensionDtype
         codes, uniques = values.factorize(na_sentinel=na_sentinel)
         dtype = original.dtype
     else:
@@ -1662,7 +1648,8 @@ def diff(arr, n: int, axis: int = 0, stacklevel=3):
         arr = arr.to_numpy()
         dtype = arr.dtype
 
-    if is_extension_array_dtype(dtype):
+    if not isinstance(dtype, np.dtype):
+        # i.e ExtensionDtype
         if hasattr(arr, f"__{op.__name__}__"):
             if axis != 0:
                 raise ValueError(f"cannot diff {type(arr).__name__} on axis={axis}")

pandas/core/arrays/categorical.py

@@ -66,7 +66,10 @@
     needs_i8_conversion,
     pandas_dtype,
 )
-from pandas.core.dtypes.dtypes import CategoricalDtype
+from pandas.core.dtypes.dtypes import (
+    CategoricalDtype,
+    ExtensionDtype,
+)
 from pandas.core.dtypes.generic import (
     ABCIndex,
     ABCSeries,
@@ -504,7 +507,7 @@ def astype(self, dtype: Dtype, copy: bool = True) -> ArrayLike:
             result = self._set_dtype(dtype)
 
         # TODO: consolidate with ndarray case?
-        elif is_extension_array_dtype(dtype):
+        elif isinstance(dtype, ExtensionDtype):
             result = pd_array(self, dtype=dtype, copy=copy)
 
         elif is_integer_dtype(dtype) and self.isna().any():
@@ -515,28 +518,15 @@ def astype(self, dtype: Dtype, copy: bool = True) -> ArrayLike:
             # variable has type "Categorical")
             result = np.array(  # type: ignore[assignment]
                 self,
-                # error: Argument "dtype" to "array" has incompatible type
-                # "Union[ExtensionDtype, str, dtype[Any], Type[str], Type[float],
-                # Type[int], Type[complex], Type[bool], Type[object]]"; expected
-                # "Union[dtype[Any], None, type, _SupportsDType, str, Union[Tuple[Any,
-                # int], Tuple[Any, Union[int, Sequence[int]]], List[Any], _DTypeDict,
-                # Tuple[Any, Any]]]"
-                dtype=dtype,  # type: ignore[arg-type]
+                dtype=dtype,
                 copy=copy,
             )
 
         else:
             # GH8628 (PERF): astype category codes instead of astyping array
             try:
                 new_cats = np.asarray(self.categories)
-                # error: Argument "dtype" to "astype" of "_ArrayOrScalarCommon" has
-                # incompatible type "Union[ExtensionDtype, dtype[Any]]"; expected
-                # "Union[dtype[Any], None, type, _SupportsDType, str, Union[Tuple[Any,
-                # int], Tuple[Any, Union[int, Sequence[int]]], List[Any], _DTypeDict,
-                # Tuple[Any, Any]]]"
-                new_cats = new_cats.astype(
-                    dtype=dtype, copy=copy  # type: ignore[arg-type]
-                )
+                new_cats = new_cats.astype(dtype=dtype, copy=copy)
             except (
                 TypeError,  # downstream error msg for CategoricalIndex is misleading
                 ValueError,

pandas/core/arrays/datetimelike.py

@@ -123,6 +123,8 @@
 from pandas.tseries import frequencies
 
 if TYPE_CHECKING:
+    from typing import Literal
+
     from pandas.core.arrays import (
         DatetimeArray,
         TimedeltaArray,
@@ -458,6 +460,14 @@ def astype(self, dtype, copy=True):
     def view(self: DatetimeLikeArrayT) -> DatetimeLikeArrayT:
         ...
 
+    @overload
+    def view(self, dtype: Literal["M8[ns]"]) -> DatetimeArray:
+        ...
+
+    @overload
+    def view(self, dtype: Literal["m8[ns]"]) -> TimedeltaArray:
+        ...
+
     @overload
     def view(self, dtype: Optional[Dtype] = ...) -> ArrayLike:
         ...
@@ -878,12 +888,11 @@ def _isnan(self) -> np.ndarray:
         return self.asi8 == iNaT
 
     @property  # NB: override with cache_readonly in immutable subclasses
-    def _hasnans(self) -> np.ndarray:
+    def _hasnans(self) -> bool:
         """
         return if I have any nans; enables various perf speedups
         """
-        # error: Incompatible return value type (got "bool", expected "ndarray")
-        return bool(self._isnan.any())  # type: ignore[return-value]
+        return bool(self._isnan.any())
 
     def _maybe_mask_results(
         self, result: np.ndarray, fill_value=iNaT, convert=None

pandas/core/arrays/period.py

@@ -643,11 +643,7 @@ def fillna(self, value=None, method=None, limit=None) -> PeriodArray:
         if method is not None:
             # view as dt64 so we get treated as timelike in core.missing
             dta = self.view("M8[ns]")
-            # error: Item "ndarray" of "Union[ExtensionArray, ndarray]" has no attribute
-            # "fillna"
-            result = dta.fillna(  # type: ignore[union-attr]
-                value=value, method=method, limit=limit
-            )
+            result = dta.fillna(value=value, method=method, limit=limit)
             return result.view(self.dtype)
         return super().fillna(value=value, method=method, limit=limit)
 

pandas/core/arrays/sparse/dtype.py

@@ -27,7 +27,6 @@
 from pandas.core.dtypes.cast import astype_nansafe
 from pandas.core.dtypes.common import (
     is_bool_dtype,
-    is_extension_array_dtype,
     is_object_dtype,
     is_scalar,
     is_string_dtype,
@@ -339,14 +338,10 @@ def update_dtype(self, dtype):
         dtype = pandas_dtype(dtype)
 
         if not isinstance(dtype, cls):
-            if is_extension_array_dtype(dtype):
+            if not isinstance(dtype, np.dtype):
                 raise TypeError("sparse arrays of extension dtypes not supported")
 
-            # error: Item "ExtensionArray" of "Union[ExtensionArray, ndarray]" has no
-            # attribute "item"
-            fill_value = astype_nansafe(  # type: ignore[union-attr]
-                np.array(self.fill_value), dtype
-            ).item()
+            fill_value = astype_nansafe(np.array(self.fill_value), dtype).item()
             dtype = cls(dtype, fill_value=fill_value)
 
         return dtype

pandas/core/dtypes/cast.py

@@ -25,6 +25,7 @@
     Type,
     Union,
     cast,
+    overload,
 )
 import warnings
 
@@ -107,6 +108,8 @@
 )
 
 if TYPE_CHECKING:
+    from typing import Literal
+
     from pandas import Series
     from pandas.core.arrays import (
         DatetimeArray,
@@ -1164,6 +1167,20 @@ def astype_td64_unit_conversion(
     return result
 
 
+@overload
+def astype_nansafe(
+    arr: np.ndarray, dtype: np.dtype, copy: bool = ..., skipna: bool = ...
+) -> np.ndarray:
+    ...
+
+
+@overload
+def astype_nansafe(
+    arr: np.ndarray, dtype: ExtensionDtype, copy: bool = ..., skipna: bool = ...
+) -> ExtensionArray:
+    ...
+
+
 def astype_nansafe(
     arr: np.ndarray, dtype: DtypeObj, copy: bool = True, skipna: bool = False
 ) -> ArrayLike:
@@ -1190,14 +1207,10 @@ def astype_nansafe(
         flags = arr.flags
         flat = arr.ravel("K")
         result = astype_nansafe(flat, dtype, copy=copy, skipna=skipna)
-        order = "F" if flags.f_contiguous else "C"
+        order: Literal["C", "F"] = "F" if flags.f_contiguous else "C"
         # error: Item "ExtensionArray" of "Union[ExtensionArray, ndarray]" has no
         # attribute "reshape"
-        # error: No overload variant of "reshape" of "_ArrayOrScalarCommon" matches
-        # argument types "Tuple[int, ...]", "str"
-        return result.reshape(  # type: ignore[union-attr,call-overload]
-            arr.shape, order=order
-        )
+        return result.reshape(arr.shape, order=order)  # type: ignore[union-attr]
 
     # We get here with 0-dim from sparse
     arr = np.atleast_1d(arr)