From 5b20f43294af8dc964558dccf0e8d55b3215896c Mon Sep 17 00:00:00 2001 From: Nathan McDougall Date: Tue, 23 Jul 2024 10:17:31 +1200 Subject: [PATCH 01/13] Add rdata as a dependency --- pyproject.toml | 1 + requirements/dev.txt | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index f1f7af39..f981ae3f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,6 +60,7 @@ test = [ "pytest-dotenv", "pytest-parallel", "s3fs", + "rdata", ] [build-system] diff --git a/requirements/dev.txt b/requirements/dev.txt index 2b2a43a0..ac7e8c51 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -237,6 +237,8 @@ numpy==2.0.0 # fastparquet # pandas # pyarrow + # rdata + # xarray oauthlib==3.2.2 # via requests-oauthlib packaging==24.1 @@ -246,10 +248,13 @@ packaging==24.1 # ipykernel # pytest # pytest-cases + # xarray pandas==2.2.2 # via # fastparquet # pins (setup.cfg) + # rdata + # xarray parso==0.8.4 # via jedi pexpect==4.9.0 @@ -346,6 +351,8 @@ pyzmq==26.0.3 # jupyter-client quartodoc==0.7.5 # via pins (setup.cfg) +rdata==0.11.2 + # via pins (setup.cfg) referencing==0.35.1 # via # jsonschema @@ -413,6 +420,7 @@ typing-extensions==4.12.2 # pydantic # pydantic-core # quartodoc + # rdata tzdata==2024.1 # via pandas urllib3==2.2.2 @@ -429,6 +437,8 @@ wheel==0.43.0 # via pip-tools wrapt==1.16.0 # via aiobotocore +xarray==2024.6.0 + # via rdata xxhash==3.4.1 # via pins (setup.cfg) yarl==1.9.4 From 6acd1309d6a783bfa2d36de9933e73a72e71530d Mon Sep 17 00:00:00 2001 From: Nathan McDougall Date: Tue, 23 Jul 2024 10:18:16 +1200 Subject: [PATCH 02/13] Don't explicitly install rdata an extra time. Don't reference [test] in `pip install -e .` --- .github/workflows/ci.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9575fda2..43363f04 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,7 +52,7 @@ jobs: pip install -r $REQUIREMENTS fi - python -m pip install -e .[test] + python -m pip install -e . - name: Set up Cloud SDK uses: google-github-actions/setup-gcloud@v0 @@ -91,8 +91,7 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install -r requirements/dev.txt - python -m pip install rdata - python -m pip install -e .[test] + python -m pip install -e . - name: run Posit Connect run: | @@ -121,7 +120,7 @@ jobs: run: | python -m pip install --upgrade pip - python -m pip install -e .[test] + python -m pip install -e . - name: Run tests run: | # TODO: better way to disable all cloud backend tests? From bb884aff659bb67e441f71f30ff5e2b93de98940 Mon Sep 17 00:00:00 2001 From: Nathan McDougall Date: Tue, 23 Jul 2024 10:51:34 +1200 Subject: [PATCH 03/13] Revert removal of `[test]` in cases where it is still needed. --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 43363f04..9acf1121 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,7 +52,7 @@ jobs: pip install -r $REQUIREMENTS fi - python -m pip install -e . + python -m pip install -e .[test] - name: Set up Cloud SDK uses: google-github-actions/setup-gcloud@v0 @@ -120,7 +120,7 @@ jobs: run: | python -m pip install --upgrade pip - python -m pip install -e . + python -m pip install -e .[test] - name: Run tests run: | # TODO: better way to disable all cloud backend tests? From 473a3d15993f174893d155a501acc279529baec3 Mon Sep 17 00:00:00 2001 From: Nathan McDougall Date: Wed, 24 Jul 2024 11:21:46 +1200 Subject: [PATCH 04/13] Add read_rds workaround for rdata on Python 3.8. --- pins/drivers.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pins/drivers.py b/pins/drivers.py index 5aa3e186..633f74a4 100644 --- a/pins/drivers.py +++ b/pins/drivers.py @@ -124,6 +124,14 @@ def load_data( try: import rdata + # Can be removed once support for Python 3.8 is dropped. + # The issue is that the last `rdata` available on Python 3.8 was v0.9, + # which doesn'thave the same `rdata.read_rds()` function. + # See https://github.com/rstudio/pins-python/pull/265 + if not hasattr(rdata, "read_rds"): + parsed = rdata.parser.parse_file(f) + rdata.conversion.convert(parsed) + return rdata.read_rds(f) except ModuleNotFoundError: raise ModuleNotFoundError( From 45d1ce5b7d9cdd555692abed4c2bce643e1b4871 Mon Sep 17 00:00:00 2001 From: Nathan McDougall Date: Thu, 25 Jul 2024 10:01:40 +1200 Subject: [PATCH 05/13] Remove `hasattr` check --- pins/drivers.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/pins/drivers.py b/pins/drivers.py index 633f74a4..b78a7886 100644 --- a/pins/drivers.py +++ b/pins/drivers.py @@ -124,15 +124,10 @@ def load_data( try: import rdata - # Can be removed once support for Python 3.8 is dropped. - # The issue is that the last `rdata` available on Python 3.8 was v0.9, - # which doesn'thave the same `rdata.read_rds()` function. + # Equivalent to `rdata.read_rds(f)` but compatible with Python 3.8. # See https://github.com/rstudio/pins-python/pull/265 - if not hasattr(rdata, "read_rds"): - parsed = rdata.parser.parse_file(f) - rdata.conversion.convert(parsed) - - return rdata.read_rds(f) + parsed = rdata.parser.parse_file(f) + return rdata.conversion.convert(parsed) except ModuleNotFoundError: raise ModuleNotFoundError( "Install the 'rdata' package to attempt to convert 'rds' files into Python objects." From 0c503aa1fbccbe30fee33ce4549f3e8b9ba1496c Mon Sep 17 00:00:00 2001 From: Nathan McDougall Date: Wed, 21 Aug 2024 10:09:15 +1200 Subject: [PATCH 06/13] Add type conversion for 3.8 support of rds load --- pins/drivers.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pins/drivers.py b/pins/drivers.py index b78a7886..1059f7d9 100644 --- a/pins/drivers.py +++ b/pins/drivers.py @@ -1,6 +1,8 @@ from pathlib import Path from typing import Sequence +import fsspec.implementations.local + from .config import PINS_ENV_INSECURE_READ, get_allow_pickle_read from .errors import PinsInsecureReadError from .meta import Meta @@ -126,7 +128,12 @@ def load_data( # Equivalent to `rdata.read_rds(f)` but compatible with Python 3.8. # See https://github.com/rstudio/pins-python/pull/265 - parsed = rdata.parser.parse_file(f) + if isinstance(f, fsspec.implementations.local.LocalFileOpener): + # rdata requires f to be a BinaryIO object. + io_f = f.f + else: + io_f = f + parsed = rdata.parser.parse_file(io_f) return rdata.conversion.convert(parsed) except ModuleNotFoundError: raise ModuleNotFoundError( From 1439fec940da8bf20e70115fa6ceff9065193e0c Mon Sep 17 00:00:00 2001 From: Nathan McDougall Date: Wed, 21 Aug 2024 15:46:39 +1200 Subject: [PATCH 07/13] Use f.name for 3.8 rdata call --- pins/drivers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pins/drivers.py b/pins/drivers.py index 1059f7d9..82e3b959 100644 --- a/pins/drivers.py +++ b/pins/drivers.py @@ -130,10 +130,10 @@ def load_data( # See https://github.com/rstudio/pins-python/pull/265 if isinstance(f, fsspec.implementations.local.LocalFileOpener): # rdata requires f to be a BinaryIO object. - io_f = f.f + fp = f.name else: - io_f = f - parsed = rdata.parser.parse_file(io_f) + fp = f + parsed = rdata.parser.parse_file(fp) return rdata.conversion.convert(parsed) except ModuleNotFoundError: raise ModuleNotFoundError( From 4899331d5f5202a577a09ebb75f5648fe238c5a1 Mon Sep 17 00:00:00 2001 From: Nathan McDougall Date: Wed, 21 Aug 2024 15:48:42 +1200 Subject: [PATCH 08/13] Use f.name rather than type conversion. --- pins/drivers.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/pins/drivers.py b/pins/drivers.py index 82e3b959..cfc8ebe2 100644 --- a/pins/drivers.py +++ b/pins/drivers.py @@ -1,8 +1,6 @@ from pathlib import Path from typing import Sequence -import fsspec.implementations.local - from .config import PINS_ENV_INSECURE_READ, get_allow_pickle_read from .errors import PinsInsecureReadError from .meta import Meta @@ -128,12 +126,7 @@ def load_data( # Equivalent to `rdata.read_rds(f)` but compatible with Python 3.8. # See https://github.com/rstudio/pins-python/pull/265 - if isinstance(f, fsspec.implementations.local.LocalFileOpener): - # rdata requires f to be a BinaryIO object. - fp = f.name - else: - fp = f - parsed = rdata.parser.parse_file(fp) + parsed = rdata.parser.parse_file(f.name) return rdata.conversion.convert(parsed) except ModuleNotFoundError: raise ModuleNotFoundError( From 534a1ea9670b73e7387829f8c623c71d869f4a4b Mon Sep 17 00:00:00 2001 From: Nathan McDougall Date: Wed, 21 Aug 2024 15:55:26 +1200 Subject: [PATCH 09/13] Enable `reportMissingImports` in pyright --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f981ae3f..fded9014 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -107,7 +107,6 @@ reportGeneralTypeIssues = false reportIncompatibleMethodOverride = false reportIncompatibleVariableOverride = false reportIndexIssue = false -reportMissingImports = false reportMissingTypeStubs = false reportOptionalIterable = false reportOptionalMemberAccess = false From c0218716570ce10e7f989db6d0de14c241a8bb86 Mon Sep 17 00:00:00 2001 From: Nathan McDougall Date: Wed, 21 Aug 2024 16:06:45 +1200 Subject: [PATCH 10/13] Isolate protected import for pyright --- pins/drivers.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/pins/drivers.py b/pins/drivers.py index cfc8ebe2..aa29a8d5 100644 --- a/pins/drivers.py +++ b/pins/drivers.py @@ -123,15 +123,14 @@ def load_data( elif meta.type == "rds": try: import rdata - - # Equivalent to `rdata.read_rds(f)` but compatible with Python 3.8. - # See https://github.com/rstudio/pins-python/pull/265 - parsed = rdata.parser.parse_file(f.name) - return rdata.conversion.convert(parsed) except ModuleNotFoundError: raise ModuleNotFoundError( "Install the 'rdata' package to attempt to convert 'rds' files into Python objects." ) + # Equivalent to `rdata.read_rds(f)` but compatible with Python 3.8. + # See https://github.com/rstudio/pins-python/pull/265 + parsed = rdata.parser.parse_file(f.name) + return rdata.conversion.convert(parsed) raise NotImplementedError(f"No driver for type {meta.type}") From 4dd2274e22af1b8e68c67ef3067910d567d24294 Mon Sep 17 00:00:00 2001 From: Nathan McDougall Date: Wed, 21 Aug 2024 16:25:25 +1200 Subject: [PATCH 11/13] Disable pyright for protected optional import --- pins/drivers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pins/drivers.py b/pins/drivers.py index aa29a8d5..9fafb334 100644 --- a/pins/drivers.py +++ b/pins/drivers.py @@ -122,11 +122,12 @@ def load_data( elif meta.type == "rds": try: - import rdata + import rdata # pyright: ignore[reportMissingImport] except ModuleNotFoundError: raise ModuleNotFoundError( "Install the 'rdata' package to attempt to convert 'rds' files into Python objects." ) + # Equivalent to `rdata.read_rds(f)` but compatible with Python 3.8. # See https://github.com/rstudio/pins-python/pull/265 parsed = rdata.parser.parse_file(f.name) From 159ec96ce377434b3dbf1c778bb84feb8f7751c1 Mon Sep 17 00:00:00 2001 From: Nathan McDougall Date: Wed, 21 Aug 2024 16:28:30 +1200 Subject: [PATCH 12/13] =?UTF-8?q?Fix=20typo=20=F0=9F=A4=A6=E2=80=8D?= =?UTF-8?q?=E2=99=82=EF=B8=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pins/drivers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pins/drivers.py b/pins/drivers.py index 9fafb334..50cc28b3 100644 --- a/pins/drivers.py +++ b/pins/drivers.py @@ -122,7 +122,7 @@ def load_data( elif meta.type == "rds": try: - import rdata # pyright: ignore[reportMissingImport] + import rdata # pyright: ignore[reportMissingImports] except ModuleNotFoundError: raise ModuleNotFoundError( "Install the 'rdata' package to attempt to convert 'rds' files into Python objects." From 13ada20446d25d7619663b7339353cffdae0a8cd Mon Sep 17 00:00:00 2001 From: isabelizimm Date: Mon, 16 Dec 2024 16:34:00 -0500 Subject: [PATCH 13/13] dropped 3.8, no longer need workaround --- pins/drivers.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/pins/drivers.py b/pins/drivers.py index 51c41cdd..73c5d3ce 100644 --- a/pins/drivers.py +++ b/pins/drivers.py @@ -115,16 +115,13 @@ def load_data( elif meta.type == "rds": try: import rdata # pyright: ignore[reportMissingImports] + + return rdata.read_rds(f) except ModuleNotFoundError: raise ModuleNotFoundError( "Install the 'rdata' package to attempt to convert 'rds' files into Python objects." ) - # Equivalent to `rdata.read_rds(f)` but compatible with Python 3.8. - # See https://github.com/rstudio/pins-python/pull/265 - parsed = rdata.parser.parse_file(f.name) - return rdata.conversion.convert(parsed) - raise NotImplementedError(f"No driver for type {meta.type}")