Skip to content

Commit c8dac58

Browse files
crusaderkyshoyer
crusaderky
authored and
shoyer
committed
MAGA (Make Azure Green Again) (#3436)
* Support for patch version in min_deps_check.py * Fix requirements * Fix requirements * cftime is 1.0.4 incompatible * Use cftime PR#127 * trivial * trivial
1 parent 72be873 commit c8dac58

File tree

7 files changed

+55
-40
lines changed

7 files changed

+55
-40
lines changed

‎ci/azure/install.yml‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,17 +15,18 @@ steps:
1515
--no-deps \
1616
--pre \
1717
--upgrade \
18-
numpy \
1918
matplotlib \
2019
pandas \
2120
scipy
21+
# numpy \ # FIXME https://github.com/pydata/xarray/issues/3409
2222
pip install \
2323
--no-deps \
2424
--upgrade \
2525
git+https://github.com/dask/dask \
2626
git+https://github.com/dask/distributed \
2727
git+https://github.com/zarr-developers/zarr \
28-
git+https://github.com/Unidata/cftime
28+
git+https://github.com/Unidata/cftime.git@refs/pull/127/merge
29+
# git+https://github.com/Unidata/cftime # FIXME PR 127 not merged yet
2930
condition: eq(variables['UPSTREAM_DEV'], 'true')
3031
displayName: Install upstream dev dependencies
3132

‎ci/min_deps_check.py‎

Lines changed: 43 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
import sys
77
from concurrent.futures import ThreadPoolExecutor
88
from datetime import datetime, timedelta
9-
from typing import Dict, Iterator, Tuple
9+
from typing import Dict, Iterator, Optional, Tuple
1010

1111
import yaml
1212

@@ -34,10 +34,14 @@ def error(msg: str) -> None:
3434
print("ERROR:", msg)
3535

3636

37-
def parse_requirements(fname) -> Iterator[Tuple[str, int, int]]:
37+
def warning(msg: str) -> None:
38+
print("WARNING:", msg)
39+
40+
41+
def parse_requirements(fname) -> Iterator[Tuple[str, int, int, Optional[int]]]:
3842
"""Load requirements/py36-min-all-deps.yml
3943
40-
Yield (package name, major version, minor version)
44+
Yield (package name, major version, minor version, [patch version])
4145
"""
4246
global has_errors
4347

@@ -52,15 +56,18 @@ def parse_requirements(fname) -> Iterator[Tuple[str, int, int]]:
5256
if pkg.endswith("<") or pkg.endswith(">") or eq != "=":
5357
error("package should be pinned with exact version: " + row)
5458
continue
59+
5560
try:
56-
major, minor = version.split(".")
57-
except ValueError:
58-
error("expected major.minor (without patch): " + row)
59-
continue
60-
try:
61-
yield pkg, int(major), int(minor)
61+
version_tup = tuple(int(x) for x in version.split("."))
6262
except ValueError:
63-
error("failed to parse version: " + row)
63+
raise ValueError("non-numerical version: " + row)
64+
65+
if len(version_tup) == 2:
66+
yield (pkg, *version_tup, None) # type: ignore
67+
elif len(version_tup) == 3:
68+
yield (pkg, *version_tup) # type: ignore
69+
else:
70+
raise ValueError("expected major.minor or major.minor.patch: " + row)
6471

6572

6673
def query_conda(pkg: str) -> Dict[Tuple[int, int], datetime]:
@@ -80,9 +87,9 @@ def query_conda(pkg: str) -> Dict[Tuple[int, int], datetime]:
8087
label = label.strip()
8188
if label == "file name":
8289
value = value.strip()[len(pkg) :]
83-
major, minor = value.split("-")[1].split(".")[:2]
84-
major = int(major)
85-
minor = int(minor)
90+
smajor, sminor = value.split("-")[1].split(".")[:2]
91+
major = int(smajor)
92+
minor = int(sminor)
8693
if label == "timestamp":
8794
assert major is not None
8895
assert minor is not None
@@ -109,17 +116,15 @@ def query_conda(pkg: str) -> Dict[Tuple[int, int], datetime]:
109116

110117

111118
def process_pkg(
112-
pkg: str, req_major: int, req_minor: int
113-
) -> Tuple[str, int, int, str, int, int, str, str]:
119+
pkg: str, req_major: int, req_minor: int, req_patch: Optional[int]
120+
) -> Tuple[str, str, str, str, str, str]:
114121
"""Compare package version from requirements file to available versions in conda.
115122
Return row to build pandas dataframe:
116123
117124
- package name
118-
- major version in requirements file
119-
- minor version in requirements file
125+
- major.minor.[patch] version in requirements file
120126
- publication date of version in requirements file (YYYY-MM-DD)
121-
- major version suggested by policy
122-
- minor version suggested by policy
127+
- major.minor version suggested by policy
123128
- publication date of version suggested by policy (YYYY-MM-DD)
124129
- status ("<", "=", "> (!)")
125130
"""
@@ -130,7 +135,7 @@ def process_pkg(
130135
req_published = versions[req_major, req_minor]
131136
except KeyError:
132137
error("not found in conda: " + pkg)
133-
return pkg, req_major, req_minor, "-", 0, 0, "-", "(!)"
138+
return pkg, fmt_version(req_major, req_minor, req_patch), "-", "-", "-", "(!)"
134139

135140
policy_months = POLICY_MONTHS.get(pkg, POLICY_MONTHS_DEFAULT)
136141
policy_published = datetime.now() - timedelta(days=policy_months * 30)
@@ -153,30 +158,39 @@ def process_pkg(
153158
else:
154159
status = "="
155160

161+
if req_patch is not None:
162+
warning("patch version should not appear in requirements file: " + pkg)
163+
status += " (w)"
164+
156165
return (
157166
pkg,
158-
req_major,
159-
req_minor,
167+
fmt_version(req_major, req_minor, req_patch),
160168
req_published.strftime("%Y-%m-%d"),
161-
policy_major,
162-
policy_minor,
169+
fmt_version(policy_major, policy_minor),
163170
policy_published_actual.strftime("%Y-%m-%d"),
164171
status,
165172
)
166173

167174

175+
def fmt_version(major: int, minor: int, patch: int = None) -> str:
176+
if patch is None:
177+
return f"{major}.{minor}"
178+
else:
179+
return f"{major}.{minor}.{patch}"
180+
181+
168182
def main() -> None:
169183
fname = sys.argv[1]
170184
with ThreadPoolExecutor(8) as ex:
171185
futures = [
172-
ex.submit(process_pkg, pkg, major, minor)
173-
for pkg, major, minor in parse_requirements(fname)
186+
ex.submit(process_pkg, pkg, major, minor, patch)
187+
for pkg, major, minor, patch in parse_requirements(fname)
174188
]
175189
rows = [f.result() for f in futures]
176190

177-
print("Package Required Policy Status")
178-
print("------------- ----------------- ----------------- ------")
179-
fmt = "{:13} {:>1d}.{:<2d} ({:10}) {:>1d}.{:<2d} ({:10}) {}"
191+
print("Package Required Policy Status")
192+
print("------------- -------------------- -------------------- ------")
193+
fmt = "{:13} {:7} ({:10}) {:7} ({:10}) {}"
180194
for row in rows:
181195
print(fmt.format(*row))
182196

‎ci/requirements/py36-min-all-deps.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ dependencies:
1313
- cartopy=0.17
1414
- cdms2=3.1
1515
- cfgrib=0.9
16-
- cftime=1.0.3
16+
- cftime=1.0.3 # FIXME need 1.0.5 (not released yet); 1.0.4 is broken
1717
- coveralls
1818
- dask=1.2
1919
- distributed=1.27

‎ci/requirements/py36.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ dependencies:
99
- cartopy
1010
- cdms2
1111
- cfgrib
12-
- cftime=1.0.3.4
12+
- cftime<1.0.4 # FIXME need 1.0.5 (not released yet); 1.0.4 is broken
1313
- coveralls
1414
- dask
1515
- distributed
@@ -25,7 +25,7 @@ dependencies:
2525
- nc-time-axis
2626
- netcdf4
2727
- numba
28-
- numpy
28+
- numpy<1.18 # FIXME https://github.com/pydata/xarray/issues/3409
2929
- pandas
3030
- pint
3131
- pip

‎ci/requirements/py37-windows.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,8 @@ dependencies:
88
- bottleneck
99
- cartopy
1010
# - cdms2 # Not available on Windows
11-
# - cfgrib>=0.9.2 # Causes Python interpreter crash on Windows
12-
- cftime=1.0.3.4
11+
# - cfgrib # Causes Python interpreter crash on Windows
12+
- cftime<1.0.4 # FIXME need 1.0.5 (not released yet); 1.0.4 is broken
1313
- coveralls
1414
- dask
1515
- distributed
@@ -25,7 +25,7 @@ dependencies:
2525
- nc-time-axis
2626
- netcdf4
2727
- numba
28-
- numpy
28+
- numpy<1.18 # FIXME https://github.com/pydata/xarray/issues/3409
2929
- pandas
3030
- pint
3131
- pip

‎ci/requirements/py37.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ dependencies:
99
- cartopy
1010
- cdms2
1111
- cfgrib
12-
- cftime=1.0.3.4
12+
- cftime<1.0.4 # FIXME need 1.0.5 (not released yet); 1.0.4 is broken
1313
- coveralls
1414
- dask
1515
- distributed
@@ -25,7 +25,7 @@ dependencies:
2525
- nc-time-axis
2626
- netcdf4
2727
- numba
28-
- numpy
28+
- numpy<1.18 # FIXME https://github.com/pydata/xarray/issues/3409
2929
- pandas
3030
- pint
3131
- pip

‎doc/whats-new.rst‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ Bug fixes
4242
but cloudpickle isn't (:issue:`3401`) by `Rhys Doyle <https://github.com/rdoyle45>`_
4343

4444
- Sync with cftime by removing `dayofwk=-1` for cftime>=1.0.4.
45-
By `Anderson Banihirwe <https://github.com/andersy005>`_.
45+
By `Anderson Banihirwe <https://github.com/andersy005>`_.
4646

4747

4848
Documentation

0 commit comments

Comments
 (0)