From 9784bc928f759b5cff41b35f27294db867266963 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Fri, 22 Mar 2024 11:16:19 +0000 Subject: [PATCH 01/11] Update smoke_tests --- test/smoke_tests/smoke_tests.py | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/test/smoke_tests/smoke_tests.py b/test/smoke_tests/smoke_tests.py index 2fbaeec5ec..58d579716a 100644 --- a/test/smoke_tests/smoke_tests.py +++ b/test/smoke_tests/smoke_tests.py @@ -1,28 +1,6 @@ """Run smoke tests""" -import os -import re - -import torchdata import torchtext -import torchtext.version # noqa: F401 - -NIGHTLY_ALLOWED_DELTA = 3 -channel = os.getenv("MATRIX_CHANNEL") - - -def validateTorchdataVersion(): - from datetime import datetime - - date_t_str = re.findall(r"dev\d+", torchdata.__version__)[0] - date_t_delta = datetime.now() - datetime.strptime(date_t_str[3:], "%Y%m%d") - - if date_t_delta.days >= NIGHTLY_ALLOWED_DELTA: - raise RuntimeError(f"torchdata binary {torchdata.__version__} is more than {NIGHTLY_ALLOWED_DELTA} days old!") - -if channel == "nightly": - validateTorchdataVersion() print("torchtext version is ", torchtext.__version__) -print("torchdata version is ", torchdata.__version__) From 864ac61ca4056e4caa0ccf7426876370ea3eaa76 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Fri, 22 Mar 2024 11:17:34 +0000 Subject: [PATCH 02/11] Update setup.py --- setup.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/setup.py b/setup.py index d008cb9c90..a3fb2707c3 100644 --- a/setup.py +++ b/setup.py @@ -63,14 +63,10 @@ def _init_submodule(): print("-- Building version " + VERSION) pytorch_package_version = os.getenv("PYTORCH_VERSION") -torchdata_package_version = os.getenv("TORCHDATA_VERSION") pytorch_package_dep = "torch" if pytorch_package_version is not None: pytorch_package_dep += "==" + pytorch_package_version -torchdata_package_dep = "torchdata" -if torchdata_package_version is not None: - torchdata_package_dep += "==" + torchdata_package_version class clean(distutils.command.clean.clean): @@ -104,7 +100,7 @@ def run(self): description="Text utilities, models, transforms, and datasets for PyTorch.", long_description=read("README.rst"), license="BSD", - install_requires=["tqdm", "requests", pytorch_package_dep, "numpy", torchdata_package_dep], + install_requires=["tqdm", "requests", pytorch_package_dep, "numpy"], python_requires=">=3.8", classifiers=[ "Programming Language :: Python :: 3.8", From fa181243fa87f098f7c8233e7a51ad356046db1d Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Fri, 22 Mar 2024 11:18:33 +0000 Subject: [PATCH 03/11] Update _download_hooks --- torchtext/_download_hooks.py | 1 - 1 file changed, 1 deletion(-) diff --git a/torchtext/_download_hooks.py b/torchtext/_download_hooks.py index 89baafafa5..f7a236482b 100644 --- a/torchtext/_download_hooks.py +++ b/torchtext/_download_hooks.py @@ -4,7 +4,6 @@ # This is to allow monkey-patching in fbcode from torch.hub import load_state_dict_from_url # noqa -from torchdata.datapipes.iter import HttpReader, GDriveReader # noqa F401 from tqdm import tqdm From 43ce25b92859d69f83b6d5ef7572df60003de108 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Fri, 22 Mar 2024 11:19:26 +0000 Subject: [PATCH 04/11] udpate requirements.txt --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index cbc13eefbf..079025ca62 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,7 +19,6 @@ Sphinx pytest expecttest parameterized -torchdata>0.5 # Lets pytest find our code by automatically modifying PYTHONPATH pytest-pythonpath From 6b6def693b358f33987e44b35dc3d05c0f130023 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Fri, 22 Mar 2024 11:26:43 +0000 Subject: [PATCH 05/11] Ignore all dataset tests --- pytest.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/pytest.ini b/pytest.ini index c7ba710bd7..b9bb2d26ca 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,4 +1,5 @@ [pytest] +addopts = --ignore-glob=test/torchtext_unittest/datasets/* testpaths = test/ python_paths = ./ markers = From f2a63dfd9fc86c8499fabeaa2bf21a2c13a2e902 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Fri, 22 Mar 2024 11:30:30 +0000 Subject: [PATCH 06/11] Update packaging/* --- packaging/pkg_helpers.bash | 12 ------------ packaging/torchtext/meta.yaml | 1 - 2 files changed, 13 deletions(-) diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash index 5c45e8937a..221e1d639a 100644 --- a/packaging/pkg_helpers.bash +++ b/packaging/pkg_helpers.bash @@ -190,14 +190,6 @@ setup_pip_pytorch_version() { -f https://download.pytorch.org/whl/torch_stable.html \ -f "https://download.pytorch.org/whl/${UPLOAD_CHANNEL}/torch_${UPLOAD_CHANNEL}.html" fi - if [[ -z "$TORCHDATA_VERSION" ]]; then - pip_install --pre torchdata -f "https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html" - export TORCHDATA_VERSION="$(pip show torchdata | grep ^Version: | sed 's/Version: *//' | sed 's/+.\+//')" - else - pip_install "torchdata==$TORCHDATA_VERSION" \ - -f https://download.pytorch.org/whl/torch_stable.html \ - -f "https://download.pytorch.org/whl/${UPLOAD_CHANNEL}/torch_${UPLOAD_CHANNEL}.html" - fi } # Fill PYTORCH_VERSION with the latest conda nightly version, and @@ -232,10 +224,6 @@ setup_conda_pytorch_constraint() { export CONDA_EXTRA_BUILD_CONSTRAINT="- mkl<=2021.2.0" fi fi - if [[ -z "$TORCHDATA_VERSION" ]]; then - export TORCHDATA_VERSION="$(conda search --json 'torchdata[channel=pytorch-nightly]' | ${PYTHON} -c "import sys, json, re; print(re.sub(r'\\+.*$', '', json.load(sys.stdin)['torchdata'][-1]['version']))")" - fi - export CONDA_TORCHDATA_CONSTRAINT="- torchdata==$TORCHDATA_VERSION" } # Translate CUDA_VERSION into CUDA_CUDATOOLKIT_CONSTRAINT diff --git a/packaging/torchtext/meta.yaml b/packaging/torchtext/meta.yaml index 03221505e5..9d7502200d 100644 --- a/packaging/torchtext/meta.yaml +++ b/packaging/torchtext/meta.yaml @@ -24,7 +24,6 @@ requirements: - requests - tqdm {{ environ.get('CONDA_PYTORCH_CONSTRAINT') }} - {{ environ.get('CONDA_TORCHDATA_CONSTRAINT') }} build: string: py{{py}} From 3bc86ce37cfc82286362591672fdc5f991f411d9 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Fri, 22 Mar 2024 11:32:37 +0000 Subject: [PATCH 07/11] Update .circleci/* --- .circleci/unittest/linux/scripts/install.sh | 5 ----- .circleci/unittest/windows/scripts/install.sh | 5 ----- 2 files changed, 10 deletions(-) diff --git a/.circleci/unittest/linux/scripts/install.sh b/.circleci/unittest/linux/scripts/install.sh index b5043d6065..fa56e74f7d 100755 --- a/.circleci/unittest/linux/scripts/install.sh +++ b/.circleci/unittest/linux/scripts/install.sh @@ -1,7 +1,6 @@ #!/usr/bin/env bash unset PYTORCH_VERSION -unset TORCHDATA_VERSION # For unittest, nightly PyTorch is used as the following section, # so no need to set PYTORCH_VERSION. # In fact, keeping PYTORCH_VERSION forces us to hardcode PyTorch version in config. @@ -30,10 +29,6 @@ printf "* Installing PyTorch\n" ) -printf "Installing torchdata nightly with portalocker\n" -pip install "portalocker>=2.0.0" -pip install --pre torchdata --index-url https://download.pytorch.org/whl/nightly/cpu - printf "* Installing torchtext\n" python setup.py develop diff --git a/.circleci/unittest/windows/scripts/install.sh b/.circleci/unittest/windows/scripts/install.sh index 9ce0558fcd..7eb4810408 100644 --- a/.circleci/unittest/windows/scripts/install.sh +++ b/.circleci/unittest/windows/scripts/install.sh @@ -1,7 +1,6 @@ #!/usr/bin/env bash unset PYTORCH_VERSION -unset TORCHDATA_VERSION # For unittest, nightly PyTorch is used as the following section, # so no need to set PYTORCH_VERSION. # In fact, keeping PYTORCH_VERSION forces us to hardcode PyTorch version in config. @@ -19,10 +18,6 @@ conda activate ./env printf "* Installing PyTorch\n" conda install -y -c "pytorch-${UPLOAD_CHANNEL}" ${CONDA_CHANNEL_FLAGS} pytorch cpuonly -printf "* Installing torchdata nightly with portalocker\n" -pip install "portalocker>=2.0.0" -pip install --pre torchdata --index-url https://download.pytorch.org/whl/nightly/cpu - printf "* Installing pywin32_postinstall script\n" curl --output pywin32_postinstall.py https://raw.githubusercontent.com/mhammond/pywin32/main/pywin32_postinstall.py python pywin32_postinstall.py -install From 72bfe35d24e2125aac772d6d564b853835937836 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Fri, 22 Mar 2024 11:36:02 +0000 Subject: [PATCH 08/11] Update build workflows --- .github/workflows/build-conda-linux.yml | 2 +- .github/workflows/build-conda-m1.yml | 2 +- .github/workflows/build-conda-windows.yml | 2 +- .github/workflows/build-wheels-linux.yml | 2 +- .github/workflows/build-wheels-m1.yml | 2 +- .github/workflows/build-wheels-windows.yml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-conda-linux.yml b/.github/workflows/build-conda-linux.yml index 87dc12a010..6a2a9a775c 100644 --- a/.github/workflows/build-conda-linux.yml +++ b/.github/workflows/build-conda-linux.yml @@ -29,7 +29,7 @@ jobs: matrix: include: - repository: pytorch/text - pre-script: packaging/install_torchdata.sh + pre-script: "" post-script: "" conda-package-directory: packaging/torchtext smoke-test-script: test/smoke_tests/smoke_tests.py diff --git a/.github/workflows/build-conda-m1.yml b/.github/workflows/build-conda-m1.yml index b59b27c7fa..c0e9b561cc 100644 --- a/.github/workflows/build-conda-m1.yml +++ b/.github/workflows/build-conda-m1.yml @@ -28,7 +28,7 @@ jobs: matrix: include: - repository: pytorch/text - pre-script: packaging/install_torchdata.sh + pre-script: "" post-script: "" conda-package-directory: packaging/torchtext smoke-test-script: test/smoke_tests/smoke_tests.py diff --git a/.github/workflows/build-conda-windows.yml b/.github/workflows/build-conda-windows.yml index db1037574a..7f7af58a07 100644 --- a/.github/workflows/build-conda-windows.yml +++ b/.github/workflows/build-conda-windows.yml @@ -29,7 +29,7 @@ jobs: matrix: include: - repository: pytorch/text - pre-script: packaging/install_torchdata.sh + pre-script: "" post-script: "" conda-package-directory: packaging/torchtext smoke-test-script: test/smoke_tests/smoke_tests.py diff --git a/.github/workflows/build-wheels-linux.yml b/.github/workflows/build-wheels-linux.yml index 2daa0be5de..2f49308fc7 100644 --- a/.github/workflows/build-wheels-linux.yml +++ b/.github/workflows/build-wheels-linux.yml @@ -34,7 +34,7 @@ jobs: matrix: include: - repository: pytorch/text - pre-script: packaging/install_torchdata.sh + pre-script: "" post-script: "" smoke-test-script: test/smoke_tests/smoke_tests.py package-name: torchtext diff --git a/.github/workflows/build-wheels-m1.yml b/.github/workflows/build-wheels-m1.yml index 49fbdd3a94..8e9ba24c95 100644 --- a/.github/workflows/build-wheels-m1.yml +++ b/.github/workflows/build-wheels-m1.yml @@ -32,7 +32,7 @@ jobs: matrix: include: - repository: pytorch/text - pre-script: packaging/install_torchdata.sh + pre-script: "" post-script: "" package-name: torchtext smoke-test-script: test/smoke_tests/smoke_tests.py diff --git a/.github/workflows/build-wheels-windows.yml b/.github/workflows/build-wheels-windows.yml index 9dce4179d6..fe2327a3c2 100644 --- a/.github/workflows/build-wheels-windows.yml +++ b/.github/workflows/build-wheels-windows.yml @@ -33,7 +33,7 @@ jobs: matrix: include: - repository: pytorch/text - pre-script: packaging/install_torchdata.sh + pre-script: "" env-script: packaging/vc_env_helper.bat post-script: "" smoke-test-script: test/smoke_tests/smoke_tests.py From 6b4a32babd2ef6429fc4c063ea8751fe7942416c Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Fri, 22 Mar 2024 11:41:23 +0000 Subject: [PATCH 09/11] Update rest of workflows --- .github/workflows/codeql.yml | 1 - .github/workflows/integration-test.yml | 4 +--- .github/workflows/test-linux-cpu.yml | 5 +---- .github/workflows/test-linux-gpu.yml | 5 +---- .github/workflows/test-macos-cpu.yml | 5 +---- .github/workflows/test-windows-cpu.yml | 5 +---- 6 files changed, 5 insertions(+), 20 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index b6af768134..8e6163288c 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -31,7 +31,6 @@ jobs: - name: Install Torch run: | python -m pip install cmake - python -m pip install --quiet --pre torch torchdata -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html sudo ln -s /usr/bin/ninja /usr/bin/ninja-build - name: Build TorchText diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 98824c2d74..e1bfabecbf 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -39,15 +39,13 @@ jobs: python -m spacy download en_core_web_sm printf "* Downloading SpaCy German models\n" python -m spacy download de_core_news_sm - # Install PyTorch, Torchvision, and TorchData + # Install PyTorch, Torchvision set -ex conda install \ --yes \ -c "pytorch-${CHANNEL}" \ -c nvidia "pytorch-${CHANNEL}"::pytorch[build="*${VERSION}*"] \ "${CUDATOOLKIT}" - printf "Installing torchdata nightly\n" - python3 -m pip install --pre torchdata --index-url https://download.pytorch.org/whl/nightly/cpu python3 setup.py develop # Install integration test dependencies python3 -m pip --quiet install parameterized diff --git a/.github/workflows/test-linux-cpu.yml b/.github/workflows/test-linux-cpu.yml index 518f5f4383..6b3dbf0f5b 100644 --- a/.github/workflows/test-linux-cpu.yml +++ b/.github/workflows/test-linux-cpu.yml @@ -50,16 +50,13 @@ jobs: printf "* Downloading SpaCy German models\n" python -m spacy download de_core_news_sm - # Install PyTorch, Torchvision, and TorchData + # Install PyTorch, Torchvision set -ex conda install \ --yes \ -c "pytorch-${CHANNEL}" \ -c nvidia "pytorch-${CHANNEL}"::pytorch[build="*${VERSION}*"] \ "${CUDATOOLKIT}" - printf "Installing torchdata nightly\n" - python3 -m pip install "portalocker>=2.0.0" - python3 -m pip install --pre torchdata --index-url https://download.pytorch.org/whl/nightly/cpu python3 setup.py develop python3 -m pip install parameterized diff --git a/.github/workflows/test-linux-gpu.yml b/.github/workflows/test-linux-gpu.yml index 1bc124c47e..b216085976 100644 --- a/.github/workflows/test-linux-gpu.yml +++ b/.github/workflows/test-linux-gpu.yml @@ -54,7 +54,7 @@ jobs: printf "* Downloading SpaCy German models\n" python -m spacy download de_core_news_sm - # Install PyTorch and TorchData + # Install PyTorch set -ex conda install \ --yes \ @@ -62,9 +62,6 @@ jobs: -c "pytorch-${CHANNEL}" \ -c nvidia "pytorch-${CHANNEL}"::pytorch[build="*${VERSION}*"] \ "${CUDATOOLKIT}" - printf "Installing torchdata nightly\n" - python3 -m pip install "portalocker>=2.0.0" - python3 -m pip install --pre torchdata --index-url https://download.pytorch.org/whl/nightly/cpu --quiet python3 setup.py develop python3 -m pip install parameterized --quiet diff --git a/.github/workflows/test-macos-cpu.yml b/.github/workflows/test-macos-cpu.yml index 774595bef9..4595627b50 100644 --- a/.github/workflows/test-macos-cpu.yml +++ b/.github/workflows/test-macos-cpu.yml @@ -55,7 +55,7 @@ jobs: printf "* Downloading SpaCy German models\n" python -m spacy download de_core_news_sm - # Install PyTorch, Torchvision, and TorchData + # Install PyTorch, Torchvision set -ex conda install \ --yes \ @@ -64,9 +64,6 @@ jobs: "${MKL_CONSTRAINT}" \ pytorch \ "${CUDATOOLKIT}" - printf "Installing torchdata nightly\n" - python3 -m pip install "portalocker>=2.0.0" - python3 -m pip install --pre torchdata --index-url https://download.pytorch.org/whl/nightly/cpu python3 setup.py develop python3 -m pip install parameterized diff --git a/.github/workflows/test-windows-cpu.yml b/.github/workflows/test-windows-cpu.yml index 1915ddd91c..0b6c9aa666 100644 --- a/.github/workflows/test-windows-cpu.yml +++ b/.github/workflows/test-windows-cpu.yml @@ -51,15 +51,12 @@ jobs: printf "* Downloading SpaCy German models\n" python -m spacy download de_core_news_sm - # Install PyTorch, Torchvision, and TorchData + # Install PyTorch, Torchvision conda install \ --yes \ -c "pytorch-${CHANNEL}" \ pytorch \ cpuonly - printf "Installing torchdata nightly\n" - python -m pip install "portalocker>=2.0.0" - python -m pip install --pre torchdata --index-url https://download.pytorch.org/whl/nightly/cpu printf "* Installing pywin32_postinstall script\n" curl --output pywin32_postinstall.py https://raw.githubusercontent.com/mhammond/pywin32/main/pywin32_postinstall.py From c848f7ae7a1a49d3d680c746616f2ee9b4e41eb3 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Fri, 22 Mar 2024 11:41:50 +0000 Subject: [PATCH 10/11] Remove install_torchdata.sh --- packaging/install_torchdata.sh | 40 ---------------------------------- 1 file changed, 40 deletions(-) delete mode 100755 packaging/install_torchdata.sh diff --git a/packaging/install_torchdata.sh b/packaging/install_torchdata.sh deleted file mode 100755 index 7db52358a3..0000000000 --- a/packaging/install_torchdata.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash -package_type="$PACKAGE_TYPE" -channel="$CHANNEL" -if [ -z "$package_type" ]; then - package_type="wheel" -fi -if [ -z "$channel" ]; then - channel="nightly" -fi - -# Wrong values -if [ "$package_type" != "wheel" ] && [ "$package_type" != "conda" ]; then - exit 1 -fi -if [ "$channel" != "nightly" ] && [ "$channel" != "test" ]; then - exit 1 -fi - - -if [ "$package_type" = "wheel" ]; then - install_cmd="pip install" - if [ "$channel" = "nightly" ]; then - install_cmd="${install_cmd} --pre" - fi - install_channel="--index-url https://download.pytorch.org/whl/${channel}/cpu" -else - install_cmd="conda install" - install_channel="-c pytorch-${channel}" -fi - -$install_cmd torchdata $install_channel - -if [ "$package_type" = "wheel" ]; then - TORCHDATA_VERSION="$(pip show torchdata | grep ^Version: | sed 's/Version: *//' | sed 's/+.\+//')" -else - TORCHDATA_VERSION="$(conda list -fe torchdata | grep torchdata | sed -e 's/torchdata=\(.*\)=py.*/\1/')" - echo "export CONDA_TORCHDATA_CONSTRAINT='- torchdata==${TORCHDATA_VERSION}'" >> "${BUILD_ENV_FILE}" -fi - -echo "export TORCHDATA_VERSION=${TORCHDATA_VERSION}" >> "${BUILD_ENV_FILE}" From 9e1377423452b704d695064f554fda480fbf5e67 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Fri, 22 Mar 2024 11:43:29 +0000 Subject: [PATCH 11/11] linter --- .github/workflows/test-linux-gpu.yml | 2 +- .github/workflows/validate-binaries.yml | 2 +- README.rst | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-linux-gpu.yml b/.github/workflows/test-linux-gpu.yml index b216085976..f51afd4fb9 100644 --- a/.github/workflows/test-linux-gpu.yml +++ b/.github/workflows/test-linux-gpu.yml @@ -54,7 +54,7 @@ jobs: printf "* Downloading SpaCy German models\n" python -m spacy download de_core_news_sm - # Install PyTorch + # Install PyTorch set -ex conda install \ --yes \ diff --git a/.github/workflows/validate-binaries.yml b/.github/workflows/validate-binaries.yml index fe55276dd8..6ba6debc92 100644 --- a/.github/workflows/validate-binaries.yml +++ b/.github/workflows/validate-binaries.yml @@ -44,7 +44,7 @@ on: required: false type: string pytorch_version: - description: 'PyTorch version to validate (ie. 2.0, 2.2.2, etc.) - optional' + description: "PyTorch version to validate (ie. 2.0, 2.2.2, etc.) - optional" default: "" required: false type: string diff --git a/README.rst b/README.rst index c99be4af4b..a31853f769 100644 --- a/README.rst +++ b/README.rst @@ -12,7 +12,7 @@ torchtext +++++++++ -CAUTION: As of September 2023 we have paused active development of TorchText because our focus has shifted away from building out this library offering. +CAUTION: As of September 2023 we have paused active development of TorchText because our focus has shifted away from building out this library offering. We will continue to release new versions but do not anticipate any new feature development as we figure out future investments in this space. This repository consists of: