Merge branch 'main' into kvikio-2

Author: dcherian

.pre-commit-config.yaml

@@ -18,21 +18,21 @@ repos:
         files: ^xarray/
   - repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: 'v0.0.287'
+    rev: 'v0.0.292'
     hooks:
       - id: ruff
         args: ["--fix"]
   # https://github.com/python/black#version-control-integration
   - repo: https://github.com/psf/black
-    rev: 23.7.0
+    rev: 23.9.1
     hooks:
       - id: black-jupyter
   - repo: https://github.com/keewis/blackdoc
     rev: v0.3.8
     hooks:
       - id: blackdoc
         exclude: "generate_aggregations.py"
-        additional_dependencies: ["black==23.7.0"]
+        additional_dependencies: ["black==23.9.1"]
       - id: blackdoc-autoupdate-black
   - repo: https://github.com/pre-commit/mirrors-mypy
     rev: v1.5.1

asv_bench/benchmarks/rolling.py

@@ -5,10 +5,10 @@
 
 from . import parameterized, randn, requires_dask
 
-nx = 300
+nx = 3000
 long_nx = 30000
 ny = 200
-nt = 100
+nt = 1000
 window = 20
 
 randn_xy = randn((nx, ny), frac_nan=0.1)
@@ -115,6 +115,11 @@ def peakmem_1drolling_reduce(self, func, use_bottleneck):
             roll = self.ds.var3.rolling(t=100)
             getattr(roll, func)()
 
+    @parameterized(["stride"], ([None, 5, 50]))
+    def peakmem_1drolling_construct(self, stride):
+        self.ds.var2.rolling(t=100).construct("w", stride=stride)
+        self.ds.var3.rolling(t=100).construct("w", stride=stride)
+
 
 class DatasetRollingMemory(RollingMemory):
     @parameterized(["func", "use_bottleneck"], (["sum", "max", "mean"], [True, False]))
@@ -128,3 +133,7 @@ def peakmem_1drolling_reduce(self, func, use_bottleneck):
         with xr.set_options(use_bottleneck=use_bottleneck):
             roll = self.ds.rolling(t=100)
             getattr(roll, func)()
+
+    @parameterized(["stride"], ([None, 5, 50]))
+    def peakmem_1drolling_construct(self, stride):
+        self.ds.rolling(t=100).construct("w", stride=stride)

doc/whats-new.rst

@@ -14,25 +14,66 @@ What's New
 
     np.random.seed(123456)
 
-.. _whats-new.2023.08.1:
+.. _whats-new.2023.09.1:
 
-v2023.08.1 (unreleased)
+v2023.09.1 (unreleased)
 -----------------------
 
 New Features
 ~~~~~~~~~~~~
 
+- :py:meth:`DataArray.where` & :py:meth:`Dataset.where` accept a callable for
+  the ``other`` parameter, passing the object as the first argument. Previously,
+  this was only valid for the ``cond`` parameter. (:issue:`8255`)
+  By `Maximilian Roos <https://github.com/max-sixty>`_.
+
+Breaking changes
+~~~~~~~~~~~~~~~~
+
+
+Deprecations
+~~~~~~~~~~~~
+
+
+Bug fixes
+~~~~~~~~~
+
+
+Documentation
+~~~~~~~~~~~~~
+
+
+Internal Changes
+~~~~~~~~~~~~~~~~
+
+
+.. _whats-new.2023.09.0:
+
+v2023.09.0 (Sep 26, 2023)
+-------------------------
+
+This release continues work on the new :py:class:`xarray.Coordinates` object, allows to provide `preferred_chunks` when
+reading from netcdf files, enables :py:func:`xarray.apply_ufunc` to handle missing core dimensions and fixes several bugs.
+
+Thanks to the 24 contributors to this release: Alexander Fischer, Amrest Chinkamol, Benoit Bovy, Darsh Ranjan, Deepak Cherian,
+Gianfranco Costamagna, Gregorio L. Trevisan, Illviljan, Joe Hamman, JR, Justus Magin, Kai Mühlbauer, Kian-Meng Ang, Kyle Sunden,
+Martin Raspaud, Mathias Hauser, Mattia Almansi, Maximilian Roos, András Gunyhó, Michael Niklas, Richard Kleijn, Riulinchen,
+Tom Nicholas and Wiktor Kraśnicki.
+
+We welcome the following new contributors to Xarray!: Alexander Fischer, Amrest Chinkamol, Darsh Ranjan, Gianfranco Costamagna, Gregorio L. Trevisan,
+Kian-Meng Ang, Riulinchen and Wiktor Kraśnicki.
+
+New Features
+~~~~~~~~~~~~
+
 - Added the :py:meth:`Coordinates.assign` method that can be used to combine
   different collections of coordinates prior to assign them to a Dataset or
   DataArray (:pull:`8102`) at once.
   By `Benoît Bovy <https://github.com/benbovy>`_.
 - Provide `preferred_chunks` for data read from netcdf files (:issue:`1440`, :pull:`7948`).
   By `Martin Raspaud <https://github.com/mraspaud>`_.
-- Improved static typing of reduction methods (:pull:`6746`).
-  By `Richard Kleijn <https://github.com/rhkleijn>`_.
 - Added `on_missing_core_dims` to :py:meth:`apply_ufunc` to allow for copying or
-  dropping a :py:class:`Dataset`'s variables with missing core dimensions.
-  (:pull:`8138`)
+  dropping a :py:class:`Dataset`'s variables with missing core dimensions (:pull:`8138`).
   By `Maximilian Roos <https://github.com/max-sixty>`_.
 
 Breaking changes
@@ -61,6 +102,8 @@ Deprecations
 Bug fixes
 ~~~~~~~~~
 
+- Improved static typing of reduction methods (:pull:`6746`).
+  By `Richard Kleijn <https://github.com/rhkleijn>`_.
 - Fix bug where empty attrs would generate inconsistent tokens (:issue:`6970`, :pull:`8101`).
   By `Mattia Almansi <https://github.com/malmans2>`_.
 - Improved handling of multi-coordinate indexes when updating coordinates, including bug fixes
@@ -71,26 +114,39 @@ Bug fixes
   :pull:`8104`).
   By `Benoît Bovy <https://github.com/benbovy>`_.
 - Fix bug where :py:class:`DataArray` instances on the right-hand side
-  of :py:meth:`DataArray.__setitem__` lose dimension names.
-  (:issue:`7030`, :pull:`8067`) By `Darsh Ranjan <https://github.com/dranjan>`_.
+  of :py:meth:`DataArray.__setitem__` lose dimension names (:issue:`7030`, :pull:`8067`).
+  By `Darsh Ranjan <https://github.com/dranjan>`_.
 - Return ``float64`` in presence of ``NaT`` in :py:class:`~core.accessor_dt.DatetimeAccessor` and
-  special case ``NaT`` handling in :py:meth:`~core.accessor_dt.DatetimeAccessor.isocalendar()`
+  special case ``NaT`` handling in :py:meth:`~core.accessor_dt.DatetimeAccessor.isocalendar`
   (:issue:`7928`, :pull:`8084`).
   By `Kai Mühlbauer <https://github.com/kmuehlbauer>`_.
+- Fix :py:meth:`~core.rolling.DatasetRolling.construct` with stride on Datasets without indexes.
+  (:issue:`7021`, :pull:`7578`).
+  By `Amrest Chinkamol <https://github.com/p4perf4ce>`_ and `Michael Niklas <https://github.com/headtr1ck>`_.
 - Calling plot with kwargs ``col``, ``row`` or ``hue`` no longer squeezes dimensions passed via these arguments
   (:issue:`7552`, :pull:`8174`).
   By `Wiktor Kraśnicki <https://github.com/wkrasnicki>`_.
-- Fixed a bug where casting from ``float`` to ``int64`` (undefined for ``NaN``) led to varying
-  issues (:issue:`7817`, :issue:`7942`, :issue:`7790`, :issue:`6191`, :issue:`7096`,
+- Fixed a bug where casting from ``float`` to ``int64`` (undefined for ``NaN``) led to varying issues (:issue:`7817`, :issue:`7942`, :issue:`7790`, :issue:`6191`, :issue:`7096`,
   :issue:`1064`, :pull:`7827`).
   By `Kai Mühlbauer <https://github.com/kmuehlbauer>`_.
+- Fixed a bug where inaccurate ``coordinates`` silently failed to decode variable (:issue:`1809`, :pull:`8195`).
+  By `Kai Mühlbauer <https://github.com/kmuehlbauer>`_
 - ``.rolling_exp`` functions no longer mistakenly lose non-dimensioned coords
-  (:issue:`6528`, :pull:`8114`)
+  (:issue:`6528`, :pull:`8114`).
   By `Maximilian Roos <https://github.com/max-sixty>`_.
+- In the event that user-provided datetime64/timedelta64 units and integer dtype encoding parameters conflict with each other, override the units to preserve an integer dtype for most faithful serialization to disk (:issue:`1064`, :pull:`8201`).
+  By `Kai Mühlbauer <https://github.com/kmuehlbauer>`_.
+- Static typing of dunder ops methods (like :py:meth:`DataArray.__eq__`) has been fixed.
+  Remaining issues are upstream problems (:issue:`7780`, :pull:`8204`).
+  By `Michael Niklas <https://github.com/headtr1ck>`_.
+- Fix type annotation for ``center`` argument of plotting methods (like :py:meth:`xarray.plot.dataarray_plot.pcolormesh`) (:pull:`8261`).
+  By `Pieter Eendebak <https://github.com/eendebakpt>`_.
 
 Documentation
 ~~~~~~~~~~~~~
 
+- Make documentation of :py:meth:`DataArray.where` clearer (:issue:`7767`, :pull:`7955`).
+  By `Riulinchen <https://github.com/Riulinchen>`_.
 
 Internal Changes
 ~~~~~~~~~~~~~~~~
@@ -103,6 +159,8 @@ Internal Changes
   than `.reduce`, as the start of a broader effort to move non-reducing
   functions away from ```.reduce``, (:pull:`8114`).
   By `Maximilian Roos <https://github.com/max-sixty>`_.
+- Test range of fill_value's in test_interpolate_pd_compat (:issue:`8146`, :pull:`8189`).
+  By `Kai Mühlbauer <https://github.com/kmuehlbauer>`_.
 
 .. _whats-new.2023.08.0:
 

pyproject.toml

@@ -72,14 +72,15 @@ source = ["xarray"]
 exclude_lines = ["pragma: no cover", "if TYPE_CHECKING"]
 
 [tool.mypy]
+enable_error_code = "redundant-self"
 exclude = 'xarray/util/generate_.*\.py'
 files = "xarray"
 show_error_codes = true
 show_error_context = true
 warn_redundant_casts = true
 warn_unused_ignores = true
 
-# Most of the numerical computing stack doesn't have type annotations yet.
+# Much of the numerical computing stack doesn't have type annotations yet.
 [[tool.mypy.overrides]]
 ignore_missing_imports = true
 module = [
@@ -95,7 +96,6 @@ module = [
   "fsspec.*",
   "h5netcdf.*",
   "h5py.*",
-  "importlib_metadata.*",
   "iris.*",
   "matplotlib.*",
   "mpl_toolkits.*",
@@ -118,6 +118,50 @@ module = [
   "numpy.exceptions.*", # remove once support for `numpy<2.0` has been dropped
 ]
 
+# Gradually we want to add more modules to this list, ratcheting up our total
+# coverage. Once a module is here, functions are checked by mypy regardless of
+# whether they have type annotations. It would be especially useful to have test
+# files listed here, because without them being checked, we don't have a great
+# way of testing our annotations.
+[[tool.mypy.overrides]]
+check_untyped_defs = true
+module = [
+  "xarray.core.accessor_dt",
+  "xarray.core.accessor_str",
+  "xarray.core.alignment",
+  "xarray.core.computation",
+  "xarray.core.rolling_exp",
+  "xarray.indexes.*",
+  "xarray.tests.*",
+]
+# This then excludes some modules from the above list. (So ideally we remove
+# from here in time...)
+[[tool.mypy.overrides]]
+check_untyped_defs = false
+module = [
+  "xarray.tests.test_coarsen",
+  "xarray.tests.test_coding_times",
+  "xarray.tests.test_combine",
+  "xarray.tests.test_computation",
+  "xarray.tests.test_concat",
+  "xarray.tests.test_coordinates",
+  "xarray.tests.test_dask",
+  "xarray.tests.test_dataarray",
+  "xarray.tests.test_duck_array_ops",
+  "xarray.tests.test_groupby",
+  "xarray.tests.test_indexing",
+  "xarray.tests.test_merge",
+  "xarray.tests.test_missing",
+  "xarray.tests.test_parallelcompat",
+  "xarray.tests.test_plot",
+  "xarray.tests.test_sparse",
+  "xarray.tests.test_ufuncs",
+  "xarray.tests.test_units",
+  "xarray.tests.test_utils",
+  "xarray.tests.test_variable",
+  "xarray.tests.test_weighted",
+]
+
 [tool.ruff]
 builtins = ["ellipsis"]
 exclude = [
@@ -146,7 +190,9 @@ select = [
 known-first-party = ["xarray"]
 
 [tool.pytest.ini_options]
-addopts = '--strict-markers'
+addopts = ["--strict-config", "--strict-markers"]
+log_cli_level = "INFO"
+minversion = "7"
 filterwarnings = [
   "ignore:Using a non-tuple sequence for multidimensional indexing is deprecated:FutureWarning",
 ]

xarray/__init__.py

@@ -1,3 +1,5 @@
+from importlib.metadata import version as _version
+
 from xarray import testing, tutorial
 from xarray.backends.api import (
     load_dataarray,
@@ -41,12 +43,6 @@
 from xarray.core.variable import IndexVariable, Variable, as_variable
 from xarray.util.print_versions import show_versions
 
-try:
-    from importlib.metadata import version as _version
-except ImportError:
-    # if the fallback library is missing, we are doomed.
-    from importlib_metadata import version as _version
-
 try:
     __version__ = _version("xarray")
 except Exception:

xarray/backends/api.py

@@ -488,6 +488,9 @@ def open_dataset(
           as coordinate variables.
         - "all": Set variables referred to in  ``'grid_mapping'``, ``'bounds'`` and
           other attributes as coordinate variables.
+
+        Only existing variables can be set as coordinates. Missing variables
+        will be silently ignored.
     drop_variables: str or iterable of str, optional
         A variable or list of variables to exclude from being parsed from the
         dataset. This may be useful to drop variables with problems or
@@ -691,6 +694,9 @@ def open_dataarray(
           as coordinate variables.
         - "all": Set variables referred to in  ``'grid_mapping'``, ``'bounds'`` and
           other attributes as coordinate variables.
+
+        Only existing variables can be set as coordinates. Missing variables
+        will be silently ignored.
     drop_variables: str or iterable of str, optional
         A variable or list of variables to exclude from being parsed from the
         dataset. This may be useful to drop variables with problems or