From ab03859fa5f8283c05fb9da525a3e73bdc3683f8 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Fri, 22 Mar 2024 15:07:09 +0000 Subject: [PATCH 1/3] Remove stuff --- torchtext/datasets/ag_news.py | 2 +- torchtext/datasets/amazonreviewfull.py | 2 +- torchtext/datasets/amazonreviewpolarity.py | 2 +- torchtext/datasets/cc100.py | 2 +- torchtext/datasets/cola.py | 2 +- torchtext/datasets/conll2000chunking.py | 2 +- torchtext/datasets/dbpedia.py | 2 +- torchtext/datasets/enwik9.py | 2 +- torchtext/datasets/imdb.py | 2 +- torchtext/datasets/iwslt2016.py | 2 +- torchtext/datasets/iwslt2017.py | 2 +- torchtext/datasets/mnli.py | 2 +- torchtext/datasets/multi30k.py | 2 +- torchtext/datasets/penntreebank.py | 2 +- torchtext/datasets/qnli.py | 2 +- torchtext/datasets/qqp.py | 2 +- torchtext/datasets/rte.py | 2 +- torchtext/datasets/sogounews.py | 2 +- torchtext/datasets/squad1.py | 2 +- torchtext/datasets/squad2.py | 2 +- torchtext/datasets/sst2.py | 2 +- torchtext/datasets/stsb.py | 2 +- torchtext/datasets/udpos.py | 2 +- torchtext/datasets/wikitext103.py | 2 +- torchtext/datasets/wikitext2.py | 2 +- torchtext/datasets/wnli.py | 2 +- torchtext/datasets/yahooanswers.py | 2 +- torchtext/datasets/yelpreviewfull.py | 2 +- torchtext/datasets/yelpreviewpolarity.py | 2 +- 29 files changed, 29 insertions(+), 29 deletions(-) diff --git a/torchtext/datasets/ag_news.py b/torchtext/datasets/ag_news.py index 5f1c7741f6..df927925bd 100644 --- a/torchtext/datasets/ag_news.py +++ b/torchtext/datasets/ag_news.py @@ -2,7 +2,7 @@ from functools import partial from typing import Union, Tuple -from torchdata.datapipes.iter import FileOpener, IterableWrapper + from torchtext._download_hooks import HttpReader from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( diff --git a/torchtext/datasets/amazonreviewfull.py b/torchtext/datasets/amazonreviewfull.py index 06e688279a..25fa77374e 100644 --- a/torchtext/datasets/amazonreviewfull.py +++ b/torchtext/datasets/amazonreviewfull.py @@ -2,7 +2,7 @@ from functools import partial from typing import Union, Tuple -from torchdata.datapipes.iter import FileOpener, IterableWrapper + from torchtext._download_hooks import GDriveReader from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( diff --git a/torchtext/datasets/amazonreviewpolarity.py b/torchtext/datasets/amazonreviewpolarity.py index 9616dc1d9e..6970a61fa4 100644 --- a/torchtext/datasets/amazonreviewpolarity.py +++ b/torchtext/datasets/amazonreviewpolarity.py @@ -2,7 +2,7 @@ from functools import partial from typing import Union, Tuple -from torchdata.datapipes.iter import FileOpener, IterableWrapper + from torchtext._download_hooks import GDriveReader from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( diff --git a/torchtext/datasets/cc100.py b/torchtext/datasets/cc100.py index 4ce2e92dd8..d3b5b01f62 100644 --- a/torchtext/datasets/cc100.py +++ b/torchtext/datasets/cc100.py @@ -1,7 +1,7 @@ import os.path from functools import partial -from torchdata.datapipes.iter import FileOpener, IterableWrapper + from torchtext._download_hooks import HttpReader from torchtext.data.datasets_utils import ( _create_dataset_directory, diff --git a/torchtext/datasets/cola.py b/torchtext/datasets/cola.py index 214c435d03..9fab5ced22 100644 --- a/torchtext/datasets/cola.py +++ b/torchtext/datasets/cola.py @@ -3,7 +3,7 @@ from functools import partial from typing import Union, Tuple -from torchdata.datapipes.iter import FileOpener, IterableWrapper + from torchtext._download_hooks import HttpReader from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import _create_dataset_directory, _wrap_split_argument diff --git a/torchtext/datasets/conll2000chunking.py b/torchtext/datasets/conll2000chunking.py index acbd9cbd0c..c4818e17e6 100644 --- a/torchtext/datasets/conll2000chunking.py +++ b/torchtext/datasets/conll2000chunking.py @@ -2,7 +2,7 @@ from functools import partial from typing import Union, Tuple -from torchdata.datapipes.iter import FileOpener, IterableWrapper + from torchtext._download_hooks import HttpReader from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( diff --git a/torchtext/datasets/dbpedia.py b/torchtext/datasets/dbpedia.py index be86f1a98c..ceec2758c8 100644 --- a/torchtext/datasets/dbpedia.py +++ b/torchtext/datasets/dbpedia.py @@ -2,7 +2,7 @@ from functools import partial from typing import Union, Tuple -from torchdata.datapipes.iter import FileOpener, IterableWrapper + from torchtext._download_hooks import GDriveReader from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( diff --git a/torchtext/datasets/enwik9.py b/torchtext/datasets/enwik9.py index cbd5e647a7..5f487455f7 100644 --- a/torchtext/datasets/enwik9.py +++ b/torchtext/datasets/enwik9.py @@ -1,7 +1,7 @@ import os from functools import partial -from torchdata.datapipes.iter import FileOpener, IterableWrapper + from torchtext._download_hooks import HttpReader from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import _create_dataset_directory diff --git a/torchtext/datasets/imdb.py b/torchtext/datasets/imdb.py index 09fba57b04..a136aa159d 100644 --- a/torchtext/datasets/imdb.py +++ b/torchtext/datasets/imdb.py @@ -3,7 +3,7 @@ from pathlib import Path from typing import Tuple, Union -from torchdata.datapipes.iter import FileOpener, IterableWrapper + from torchtext._download_hooks import HttpReader from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import _create_dataset_directory diff --git a/torchtext/datasets/iwslt2016.py b/torchtext/datasets/iwslt2016.py index dd4b806e8c..3ce748ecad 100644 --- a/torchtext/datasets/iwslt2016.py +++ b/torchtext/datasets/iwslt2016.py @@ -1,7 +1,7 @@ import os from functools import partial -from torchdata.datapipes.iter import FileOpener, IterableWrapper + from torchtext._download_hooks import GDriveReader from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( diff --git a/torchtext/datasets/iwslt2017.py b/torchtext/datasets/iwslt2017.py index 3707986d54..be06dc9087 100644 --- a/torchtext/datasets/iwslt2017.py +++ b/torchtext/datasets/iwslt2017.py @@ -1,7 +1,7 @@ import os from functools import partial -from torchdata.datapipes.iter import FileOpener, IterableWrapper + from torchtext._download_hooks import GDriveReader from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( diff --git a/torchtext/datasets/mnli.py b/torchtext/datasets/mnli.py index f4335c5ccf..2b7031b352 100644 --- a/torchtext/datasets/mnli.py +++ b/torchtext/datasets/mnli.py @@ -3,7 +3,7 @@ import os from functools import partial -from torchdata.datapipes.iter import FileOpener, IterableWrapper + # we import HttpReader from _download_hooks so we can swap out public URLs # with interal URLs when the dataset is used within Facebook diff --git a/torchtext/datasets/multi30k.py b/torchtext/datasets/multi30k.py index ea1c2015ae..3abb046e11 100644 --- a/torchtext/datasets/multi30k.py +++ b/torchtext/datasets/multi30k.py @@ -2,7 +2,7 @@ from functools import partial from typing import Union, Tuple -from torchdata.datapipes.iter import FileOpener, IterableWrapper + from torchtext._download_hooks import GDriveReader # noqa from torchtext._download_hooks import HttpReader from torchtext._internal.module_utils import is_module_available diff --git a/torchtext/datasets/penntreebank.py b/torchtext/datasets/penntreebank.py index 1e0d9f295f..b7365f2abe 100644 --- a/torchtext/datasets/penntreebank.py +++ b/torchtext/datasets/penntreebank.py @@ -2,7 +2,7 @@ from functools import partial from typing import Tuple, Union -from torchdata.datapipes.iter import FileOpener, IterableWrapper + from torchtext._download_hooks import GDriveReader # noqa from torchtext._download_hooks import HttpReader from torchtext._internal.module_utils import is_module_available diff --git a/torchtext/datasets/qnli.py b/torchtext/datasets/qnli.py index aa71eeb208..9fe8581d57 100644 --- a/torchtext/datasets/qnli.py +++ b/torchtext/datasets/qnli.py @@ -3,7 +3,7 @@ import os from functools import partial -from torchdata.datapipes.iter import FileOpener, IterableWrapper + # we import HttpReader from _download_hooks so we can swap out public URLs # with interal URLs when the dataset is used within Facebook diff --git a/torchtext/datasets/qqp.py b/torchtext/datasets/qqp.py index 013a6a82a8..bf93c2b06c 100644 --- a/torchtext/datasets/qqp.py +++ b/torchtext/datasets/qqp.py @@ -1,7 +1,7 @@ import os from functools import partial -from torchdata.datapipes.iter import FileOpener, IterableWrapper + from torchtext._download_hooks import HttpReader from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import _create_dataset_directory diff --git a/torchtext/datasets/rte.py b/torchtext/datasets/rte.py index 06355468ae..e0dc444314 100644 --- a/torchtext/datasets/rte.py +++ b/torchtext/datasets/rte.py @@ -3,7 +3,7 @@ import os from functools import partial -from torchdata.datapipes.iter import FileOpener, IterableWrapper + # we import HttpReader from _download_hooks so we can swap out public URLs # with interal URLs when the dataset is used within Facebook diff --git a/torchtext/datasets/sogounews.py b/torchtext/datasets/sogounews.py index 80c7c9af9a..2d06b679df 100644 --- a/torchtext/datasets/sogounews.py +++ b/torchtext/datasets/sogounews.py @@ -2,7 +2,7 @@ from functools import partial from typing import Union, Tuple -from torchdata.datapipes.iter import FileOpener, IterableWrapper + from torchtext._download_hooks import GDriveReader from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( diff --git a/torchtext/datasets/squad1.py b/torchtext/datasets/squad1.py index 5c83bcdec2..061cc60356 100644 --- a/torchtext/datasets/squad1.py +++ b/torchtext/datasets/squad1.py @@ -2,7 +2,7 @@ from functools import partial from typing import Union, Tuple -from torchdata.datapipes.iter import FileOpener, IterableWrapper + from torchtext._download_hooks import HttpReader from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( diff --git a/torchtext/datasets/squad2.py b/torchtext/datasets/squad2.py index 48ef86556c..533518dc49 100644 --- a/torchtext/datasets/squad2.py +++ b/torchtext/datasets/squad2.py @@ -2,7 +2,7 @@ from functools import partial from typing import Union, Tuple -from torchdata.datapipes.iter import FileOpener, IterableWrapper + from torchtext._download_hooks import HttpReader from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( diff --git a/torchtext/datasets/sst2.py b/torchtext/datasets/sst2.py index 132b22d68d..80df5e4d6a 100644 --- a/torchtext/datasets/sst2.py +++ b/torchtext/datasets/sst2.py @@ -2,7 +2,7 @@ import os from functools import partial -from torchdata.datapipes.iter import FileOpener, IterableWrapper + # we import HttpReader from _download_hooks so we can swap out public URLs # with interal URLs when the dataset is used within Facebook diff --git a/torchtext/datasets/stsb.py b/torchtext/datasets/stsb.py index 324ed77245..41d44eaab5 100644 --- a/torchtext/datasets/stsb.py +++ b/torchtext/datasets/stsb.py @@ -2,7 +2,7 @@ import os from functools import partial -from torchdata.datapipes.iter import FileOpener, IterableWrapper + # we import HttpReader from _download_hooks so we can swap out public URLs # with interal URLs when the dataset is used within Facebook diff --git a/torchtext/datasets/udpos.py b/torchtext/datasets/udpos.py index 3c7b76b124..92e9a980b5 100644 --- a/torchtext/datasets/udpos.py +++ b/torchtext/datasets/udpos.py @@ -2,7 +2,7 @@ from functools import partial from typing import Union, Tuple -from torchdata.datapipes.iter import FileOpener, IterableWrapper + from torchtext._download_hooks import HttpReader from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( diff --git a/torchtext/datasets/wikitext103.py b/torchtext/datasets/wikitext103.py index 0914d708e9..67f9a07667 100644 --- a/torchtext/datasets/wikitext103.py +++ b/torchtext/datasets/wikitext103.py @@ -2,7 +2,7 @@ from functools import partial from typing import Union, Tuple -from torchdata.datapipes.iter import FileOpener, IterableWrapper + from torchtext._download_hooks import HttpReader from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( diff --git a/torchtext/datasets/wikitext2.py b/torchtext/datasets/wikitext2.py index ec686b94cd..8b101ba12c 100644 --- a/torchtext/datasets/wikitext2.py +++ b/torchtext/datasets/wikitext2.py @@ -2,7 +2,7 @@ from functools import partial from typing import Union, Tuple -from torchdata.datapipes.iter import FileOpener, IterableWrapper + from torchtext._download_hooks import HttpReader from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( diff --git a/torchtext/datasets/wnli.py b/torchtext/datasets/wnli.py index c864275899..cf24155ef4 100644 --- a/torchtext/datasets/wnli.py +++ b/torchtext/datasets/wnli.py @@ -2,7 +2,7 @@ import os from functools import partial -from torchdata.datapipes.iter import FileOpener, IterableWrapper + # we import HttpReader from _download_hooks so we can swap out public URLs # with interal URLs when the dataset is used within Facebook diff --git a/torchtext/datasets/yahooanswers.py b/torchtext/datasets/yahooanswers.py index 9fad10ff1d..908e95dcc6 100644 --- a/torchtext/datasets/yahooanswers.py +++ b/torchtext/datasets/yahooanswers.py @@ -2,7 +2,7 @@ from functools import partial from typing import Union, Tuple -from torchdata.datapipes.iter import FileOpener, IterableWrapper + from torchtext._download_hooks import GDriveReader from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( diff --git a/torchtext/datasets/yelpreviewfull.py b/torchtext/datasets/yelpreviewfull.py index 1272dae45c..7750141c56 100644 --- a/torchtext/datasets/yelpreviewfull.py +++ b/torchtext/datasets/yelpreviewfull.py @@ -2,7 +2,7 @@ from functools import partial from typing import Union, Tuple -from torchdata.datapipes.iter import FileOpener, IterableWrapper + from torchtext._download_hooks import GDriveReader from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( diff --git a/torchtext/datasets/yelpreviewpolarity.py b/torchtext/datasets/yelpreviewpolarity.py index 90e1e31e59..755b3b924d 100644 --- a/torchtext/datasets/yelpreviewpolarity.py +++ b/torchtext/datasets/yelpreviewpolarity.py @@ -2,7 +2,7 @@ from functools import partial from typing import Union, Tuple -from torchdata.datapipes.iter import FileOpener, IterableWrapper + from torchtext._download_hooks import GDriveReader from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( From ab8b8e2bcfe295082c33fdb936415b9f7d577075 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Fri, 22 Mar 2024 16:53:25 +0000 Subject: [PATCH 2/3] stuff --- torchtext/datasets/ag_news.py | 3 ++- torchtext/datasets/amazonreviewfull.py | 3 ++- torchtext/datasets/amazonreviewpolarity.py | 3 ++- torchtext/datasets/cc100.py | 8 +++++++- torchtext/datasets/cnndm.py | 12 ++++++------ torchtext/datasets/cola.py | 3 ++- torchtext/datasets/conll2000chunking.py | 3 ++- torchtext/datasets/dbpedia.py | 3 ++- torchtext/datasets/enwik9.py | 3 ++- torchtext/datasets/imdb.py | 3 ++- torchtext/datasets/iwslt2016.py | 3 ++- torchtext/datasets/iwslt2017.py | 3 ++- torchtext/datasets/mnli.py | 3 ++- torchtext/datasets/mrpc.py | 2 +- torchtext/datasets/multi30k.py | 5 +++-- torchtext/datasets/penntreebank.py | 5 +++-- torchtext/datasets/qnli.py | 3 ++- torchtext/datasets/qqp.py | 3 ++- torchtext/datasets/rte.py | 3 ++- torchtext/datasets/sogounews.py | 3 ++- torchtext/datasets/squad1.py | 4 +++- torchtext/datasets/squad2.py | 3 ++- torchtext/datasets/sst2.py | 3 ++- torchtext/datasets/stsb.py | 4 +++- torchtext/datasets/udpos.py | 3 ++- torchtext/datasets/wikitext103.py | 4 +++- torchtext/datasets/wikitext2.py | 3 ++- torchtext/datasets/wnli.py | 3 ++- torchtext/datasets/yahooanswers.py | 3 ++- torchtext/datasets/yelpreviewfull.py | 3 ++- torchtext/datasets/yelpreviewpolarity.py | 3 ++- 31 files changed, 75 insertions(+), 38 deletions(-) diff --git a/torchtext/datasets/ag_news.py b/torchtext/datasets/ag_news.py index df927925bd..1e1bf61f02 100644 --- a/torchtext/datasets/ag_news.py +++ b/torchtext/datasets/ag_news.py @@ -3,7 +3,7 @@ from typing import Union, Tuple -from torchtext._download_hooks import HttpReader + from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( _wrap_split_argument, @@ -65,6 +65,7 @@ def AG_NEWS(root: str, split: Union[Tuple[str], str]): raise ModuleNotFoundError( "Package `torchdata` not found. Please install following instructions at https://github.com/pytorch/data" ) + from torchdata.datapipes.iter import FileOpener, GDriveReader, HttpReader, IterableWrapper # noqa url_dp = IterableWrapper([URL[split]]) cache_dp = url_dp.on_disk_cache( diff --git a/torchtext/datasets/amazonreviewfull.py b/torchtext/datasets/amazonreviewfull.py index 25fa77374e..475b8dcff0 100644 --- a/torchtext/datasets/amazonreviewfull.py +++ b/torchtext/datasets/amazonreviewfull.py @@ -3,7 +3,7 @@ from typing import Union, Tuple -from torchtext._download_hooks import GDriveReader + from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( _wrap_split_argument, @@ -79,6 +79,7 @@ def AmazonReviewFull(root: str, split: Union[Tuple[str], str]): raise ModuleNotFoundError( "Package `torchdata` not found. Please install following instructions at https://github.com/pytorch/data" ) + from torchdata.datapipes.iter import FileOpener, GDriveReader, HttpReader, IterableWrapper # noqa url_dp = IterableWrapper([URL]) cache_compressed_dp = url_dp.on_disk_cache( diff --git a/torchtext/datasets/amazonreviewpolarity.py b/torchtext/datasets/amazonreviewpolarity.py index 6970a61fa4..3df6b019ea 100644 --- a/torchtext/datasets/amazonreviewpolarity.py +++ b/torchtext/datasets/amazonreviewpolarity.py @@ -3,7 +3,7 @@ from typing import Union, Tuple -from torchtext._download_hooks import GDriveReader + from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( _wrap_split_argument, @@ -76,6 +76,7 @@ def AmazonReviewPolarity(root: str, split: Union[Tuple[str], str]): raise ModuleNotFoundError( "Package `torchdata` not found. Please install following instructions at https://github.com/pytorch/data" ) + from torchdata.datapipes.iter import FileOpener, GDriveReader, HttpReader, IterableWrapper # noqa url_dp = IterableWrapper([URL]) cache_compressed_dp = url_dp.on_disk_cache( diff --git a/torchtext/datasets/cc100.py b/torchtext/datasets/cc100.py index d3b5b01f62..bfb4b96a67 100644 --- a/torchtext/datasets/cc100.py +++ b/torchtext/datasets/cc100.py @@ -2,7 +2,8 @@ from functools import partial -from torchtext._download_hooks import HttpReader + +from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( _create_dataset_directory, ) @@ -167,6 +168,11 @@ def CC100(root: str, language_code: str = "en"): """ if language_code not in VALID_CODES: raise ValueError(f"Invalid language code {language_code}") + if not is_module_available("torchdata"): + raise ModuleNotFoundError( + "Package `torchdata` not found. Please install following instructions at https://github.com/pytorch/data" + ) + from torchdata.datapipes.iter import FileOpener, GDriveReader, HttpReader, IterableWrapper # noqa url = URL % language_code url_dp = IterableWrapper([url]) diff --git a/torchtext/datasets/cnndm.py b/torchtext/datasets/cnndm.py index 2adba04fd1..33a70f52f8 100644 --- a/torchtext/datasets/cnndm.py +++ b/torchtext/datasets/cnndm.py @@ -3,12 +3,6 @@ from functools import partial from typing import Union, Set, Tuple -from torchdata.datapipes.iter import ( - FileOpener, - IterableWrapper, - OnlineReader, - GDriveReader, -) from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( _wrap_split_argument, @@ -141,6 +135,12 @@ def CNNDM(root: str, split: Union[Tuple[str], str]): raise ModuleNotFoundError( "Package `torchdata` not found. Please install following instructions at https://github.com/pytorch/data" ) + from torchdata.datapipes.iter import ( + FileOpener, + IterableWrapper, + OnlineReader, + GDriveReader, + ) cnn_dp = _load_stories(root, "cnn", split) dailymail_dp = _load_stories(root, "dailymail", split) diff --git a/torchtext/datasets/cola.py b/torchtext/datasets/cola.py index 9fab5ced22..43494ae1ce 100644 --- a/torchtext/datasets/cola.py +++ b/torchtext/datasets/cola.py @@ -4,7 +4,7 @@ from typing import Union, Tuple -from torchtext._download_hooks import HttpReader + from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import _create_dataset_directory, _wrap_split_argument @@ -76,6 +76,7 @@ def CoLA(root: str, split: Union[Tuple[str], str]): raise ModuleNotFoundError( "Package `torchdata` not found. Please install following instructions at https://github.com/pytorch/data" ) + from torchdata.datapipes.iter import FileOpener, GDriveReader, HttpReader, IterableWrapper # noqa url_dp = IterableWrapper([URL]) cache_compressed_dp = url_dp.on_disk_cache( diff --git a/torchtext/datasets/conll2000chunking.py b/torchtext/datasets/conll2000chunking.py index c4818e17e6..2f9de527a8 100644 --- a/torchtext/datasets/conll2000chunking.py +++ b/torchtext/datasets/conll2000chunking.py @@ -3,7 +3,7 @@ from typing import Union, Tuple -from torchtext._download_hooks import HttpReader + from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( _wrap_split_argument, @@ -68,6 +68,7 @@ def CoNLL2000Chunking(root: str, split: Union[Tuple[str], str]): raise ModuleNotFoundError( "Package `torchdata` not found. Please install following instructions at https://github.com/pytorch/data" ) + from torchdata.datapipes.iter import FileOpener, GDriveReader, HttpReader, IterableWrapper # noqa url_dp = IterableWrapper([URL[split]]) diff --git a/torchtext/datasets/dbpedia.py b/torchtext/datasets/dbpedia.py index ceec2758c8..53803ba662 100644 --- a/torchtext/datasets/dbpedia.py +++ b/torchtext/datasets/dbpedia.py @@ -3,7 +3,7 @@ from typing import Union, Tuple -from torchtext._download_hooks import GDriveReader + from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( _wrap_split_argument, @@ -75,6 +75,7 @@ def DBpedia(root: str, split: Union[Tuple[str], str]): raise ModuleNotFoundError( "Package `torchdata` not found. Please install following instructions at https://github.com/pytorch/data" ) + from torchdata.datapipes.iter import FileOpener, GDriveReader, HttpReader, IterableWrapper # noqa url_dp = IterableWrapper([URL]) cache_compressed_dp = url_dp.on_disk_cache( diff --git a/torchtext/datasets/enwik9.py b/torchtext/datasets/enwik9.py index 5f487455f7..f966bada50 100644 --- a/torchtext/datasets/enwik9.py +++ b/torchtext/datasets/enwik9.py @@ -2,7 +2,7 @@ from functools import partial -from torchtext._download_hooks import HttpReader + from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import _create_dataset_directory @@ -50,6 +50,7 @@ def EnWik9(root: str): raise ModuleNotFoundError( "Package `torchdata` not found. Please install following instructions at https://github.com/pytorch/data" ) + from torchdata.datapipes.iter import FileOpener, GDriveReader, HttpReader, IterableWrapper # noqa url_dp = IterableWrapper([URL]) cache_compressed_dp = url_dp.on_disk_cache( diff --git a/torchtext/datasets/imdb.py b/torchtext/datasets/imdb.py index a136aa159d..380e97c5e9 100644 --- a/torchtext/datasets/imdb.py +++ b/torchtext/datasets/imdb.py @@ -4,7 +4,7 @@ from typing import Tuple, Union -from torchtext._download_hooks import HttpReader + from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import _create_dataset_directory from torchtext.data.datasets_utils import _wrap_split_argument @@ -89,6 +89,7 @@ def IMDB(root: str, split: Union[Tuple[str], str]): raise ModuleNotFoundError( "Package `torchdata` not found. Please install following instructions at https://github.com/pytorch/data" ) + from torchdata.datapipes.iter import FileOpener, GDriveReader, HttpReader, IterableWrapper # noqa url_dp = IterableWrapper([URL]) diff --git a/torchtext/datasets/iwslt2016.py b/torchtext/datasets/iwslt2016.py index 3ce748ecad..746b40bb62 100644 --- a/torchtext/datasets/iwslt2016.py +++ b/torchtext/datasets/iwslt2016.py @@ -2,7 +2,7 @@ from functools import partial -from torchtext._download_hooks import GDriveReader + from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( _clean_files, @@ -219,6 +219,7 @@ def IWSLT2016( raise ModuleNotFoundError( "Package `torchdata` not found. Please install following instructions at https://github.com/pytorch/data" ) + from torchdata.datapipes.iter import FileOpener, GDriveReader, HttpReader, IterableWrapper # noqa if not isinstance(language_pair, list) and not isinstance(language_pair, tuple): raise ValueError("language_pair must be list or tuple but got {} instead".format(type(language_pair))) diff --git a/torchtext/datasets/iwslt2017.py b/torchtext/datasets/iwslt2017.py index be06dc9087..36e233832b 100644 --- a/torchtext/datasets/iwslt2017.py +++ b/torchtext/datasets/iwslt2017.py @@ -2,7 +2,7 @@ from functools import partial -from torchtext._download_hooks import GDriveReader + from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( _clean_files, @@ -184,6 +184,7 @@ def IWSLT2017(root=".data", split=("train", "valid", "test"), language_pair=("de raise ModuleNotFoundError( "Package `torchdata` not found. Please install following instructions at https://github.com/pytorch/data" ) + from torchdata.datapipes.iter import FileOpener, GDriveReader, HttpReader, IterableWrapper # noqa valid_set = "dev2010" test_set = "tst2010" diff --git a/torchtext/datasets/mnli.py b/torchtext/datasets/mnli.py index 2b7031b352..aebfd08a39 100644 --- a/torchtext/datasets/mnli.py +++ b/torchtext/datasets/mnli.py @@ -7,7 +7,7 @@ # we import HttpReader from _download_hooks so we can swap out public URLs # with interal URLs when the dataset is used within Facebook -from torchtext._download_hooks import HttpReader + from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( _create_dataset_directory, @@ -89,6 +89,7 @@ def MNLI(root, split): raise ModuleNotFoundError( "Package `torchdata` not found. Please install following instructions at https://github.com/pytorch/data" ) + from torchdata.datapipes.iter import FileOpener, GDriveReader, HttpReader, IterableWrapper # noqa url_dp = IterableWrapper([URL]) cache_compressed_dp = url_dp.on_disk_cache( diff --git a/torchtext/datasets/mrpc.py b/torchtext/datasets/mrpc.py index e9abea1721..c3e6f72a91 100644 --- a/torchtext/datasets/mrpc.py +++ b/torchtext/datasets/mrpc.py @@ -3,7 +3,6 @@ from functools import partial from typing import Union, Tuple -from torchdata.datapipes.iter import FileOpener, HttpReader, IterableWrapper from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( _wrap_split_argument, @@ -67,6 +66,7 @@ def MRPC(root: str, split: Union[Tuple[str], str]): raise ModuleNotFoundError( "Package `torchdata` not found. Please install following instructions at https://github.com/pytorch/data" ) + from torchdata.datapipes.iter import FileOpener, GDriveReader, HttpReader, IterableWrapper # noqa url_dp = IterableWrapper([URL[split]]) # cache data on-disk with sanity check diff --git a/torchtext/datasets/multi30k.py b/torchtext/datasets/multi30k.py index 3abb046e11..ed4deb82bd 100644 --- a/torchtext/datasets/multi30k.py +++ b/torchtext/datasets/multi30k.py @@ -3,8 +3,8 @@ from typing import Union, Tuple -from torchtext._download_hooks import GDriveReader # noqa -from torchtext._download_hooks import HttpReader + # noqa + from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( _wrap_split_argument, @@ -89,6 +89,7 @@ def Multi30k(root: str, split: Union[Tuple[str], str], language_pair: Tuple[str] raise ModuleNotFoundError( "Package `torchdata` not found. Please install following instructions at https://github.com/pytorch/data" ) + from torchdata.datapipes.iter import FileOpener, GDriveReader, HttpReader, IterableWrapper # noqa url_dp = IterableWrapper([URL[split]]) diff --git a/torchtext/datasets/penntreebank.py b/torchtext/datasets/penntreebank.py index b7365f2abe..3720a14c48 100644 --- a/torchtext/datasets/penntreebank.py +++ b/torchtext/datasets/penntreebank.py @@ -3,8 +3,8 @@ from typing import Tuple, Union -from torchtext._download_hooks import GDriveReader # noqa -from torchtext._download_hooks import HttpReader + # noqa + from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( _wrap_split_argument, @@ -70,6 +70,7 @@ def PennTreebank(root, split: Union[Tuple[str], str]): raise ModuleNotFoundError( "Package `torchdata` not found. Please install following instructions at https://github.com/pytorch/data" ) + from torchdata.datapipes.iter import FileOpener, GDriveReader, HttpReader, IterableWrapper # noqa url_dp = IterableWrapper([URL[split]]) cache_dp = url_dp.on_disk_cache( diff --git a/torchtext/datasets/qnli.py b/torchtext/datasets/qnli.py index 9fe8581d57..6422ee93e7 100644 --- a/torchtext/datasets/qnli.py +++ b/torchtext/datasets/qnli.py @@ -7,7 +7,7 @@ # we import HttpReader from _download_hooks so we can swap out public URLs # with interal URLs when the dataset is used within Facebook -from torchtext._download_hooks import HttpReader + from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( _create_dataset_directory, @@ -81,6 +81,7 @@ def QNLI(root, split): raise ModuleNotFoundError( "Package `torchdata` not found. Please install following instructions at `https://github.com/pytorch/data`" ) + from torchdata.datapipes.iter import FileOpener, GDriveReader, HttpReader, IterableWrapper # noqa url_dp = IterableWrapper([URL]) cache_compressed_dp = url_dp.on_disk_cache( diff --git a/torchtext/datasets/qqp.py b/torchtext/datasets/qqp.py index bf93c2b06c..f0686e6df2 100644 --- a/torchtext/datasets/qqp.py +++ b/torchtext/datasets/qqp.py @@ -2,7 +2,7 @@ from functools import partial -from torchtext._download_hooks import HttpReader + from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import _create_dataset_directory @@ -48,6 +48,7 @@ def QQP(root: str): raise ModuleNotFoundError( "Package `torchdata` not found. Please install following instructions at https://github.com/pytorch/data" ) + from torchdata.datapipes.iter import FileOpener, GDriveReader, HttpReader, IterableWrapper # noqa url_dp = IterableWrapper([URL]) cache_dp = url_dp.on_disk_cache( diff --git a/torchtext/datasets/rte.py b/torchtext/datasets/rte.py index e0dc444314..3ab15f6e13 100644 --- a/torchtext/datasets/rte.py +++ b/torchtext/datasets/rte.py @@ -7,7 +7,7 @@ # we import HttpReader from _download_hooks so we can swap out public URLs # with interal URLs when the dataset is used within Facebook -from torchtext._download_hooks import HttpReader + from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( _create_dataset_directory, @@ -81,6 +81,7 @@ def RTE(root, split): raise ModuleNotFoundError( "Package `torchdata` not found. Please install following instructions at `https://github.com/pytorch/data`" ) + from torchdata.datapipes.iter import FileOpener, GDriveReader, HttpReader, IterableWrapper # noqa url_dp = IterableWrapper([URL]) cache_compressed_dp = url_dp.on_disk_cache( diff --git a/torchtext/datasets/sogounews.py b/torchtext/datasets/sogounews.py index 2d06b679df..3dfb9db1a2 100644 --- a/torchtext/datasets/sogounews.py +++ b/torchtext/datasets/sogounews.py @@ -3,7 +3,7 @@ from typing import Union, Tuple -from torchtext._download_hooks import GDriveReader + from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( _wrap_split_argument, @@ -79,6 +79,7 @@ def SogouNews(root: str, split: Union[Tuple[str], str]): raise ModuleNotFoundError( "Package `torchdata` not found. Please install following instructions at https://github.com/pytorch/data" ) + from torchdata.datapipes.iter import FileOpener, GDriveReader, HttpReader, IterableWrapper # noqa url_dp = IterableWrapper([URL]) cache_compressed_dp = url_dp.on_disk_cache( diff --git a/torchtext/datasets/squad1.py b/torchtext/datasets/squad1.py index 061cc60356..a2cfa1532c 100644 --- a/torchtext/datasets/squad1.py +++ b/torchtext/datasets/squad1.py @@ -3,7 +3,7 @@ from typing import Union, Tuple -from torchtext._download_hooks import HttpReader + from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( _wrap_split_argument, @@ -62,6 +62,8 @@ def SQuAD1(root: str, split: Union[Tuple[str], str]): raise ModuleNotFoundError( "Package `torchdata` not found. Please install following instructions at https://github.com/pytorch/data" ) + from torchdata.datapipes.iter import FileOpener, GDriveReader, HttpReader, IterableWrapper # noqa + url_dp = IterableWrapper([URL[split]]) # cache data on-disk with sanity check diff --git a/torchtext/datasets/squad2.py b/torchtext/datasets/squad2.py index 533518dc49..2359a91e21 100644 --- a/torchtext/datasets/squad2.py +++ b/torchtext/datasets/squad2.py @@ -3,7 +3,7 @@ from typing import Union, Tuple -from torchtext._download_hooks import HttpReader + from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( _wrap_split_argument, @@ -63,6 +63,7 @@ def SQuAD2(root: str, split: Union[Tuple[str], str]): raise ModuleNotFoundError( "Package `torchdata` not found. Please install following instructions at https://github.com/pytorch/data" ) + from torchdata.datapipes.iter import FileOpener, GDriveReader, HttpReader, IterableWrapper # noqa url_dp = IterableWrapper([URL[split]]) # cache data on-disk with sanity check diff --git a/torchtext/datasets/sst2.py b/torchtext/datasets/sst2.py index 80df5e4d6a..9a350197ee 100644 --- a/torchtext/datasets/sst2.py +++ b/torchtext/datasets/sst2.py @@ -6,7 +6,7 @@ # we import HttpReader from _download_hooks so we can swap out public URLs # with interal URLs when the dataset is used within Facebook -from torchtext._download_hooks import HttpReader + from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( _create_dataset_directory, @@ -86,6 +86,7 @@ def SST2(root, split): raise ModuleNotFoundError( "Package `torchdata` not found. Please install following instructions at https://github.com/pytorch/data" ) + from torchdata.datapipes.iter import FileOpener, GDriveReader, HttpReader, IterableWrapper # noqa url_dp = IterableWrapper([URL]) cache_compressed_dp = url_dp.on_disk_cache( diff --git a/torchtext/datasets/stsb.py b/torchtext/datasets/stsb.py index 41d44eaab5..892ecc7288 100644 --- a/torchtext/datasets/stsb.py +++ b/torchtext/datasets/stsb.py @@ -6,7 +6,7 @@ # we import HttpReader from _download_hooks so we can swap out public URLs # with interal URLs when the dataset is used within Facebook -from torchtext._download_hooks import HttpReader + from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( _create_dataset_directory, @@ -82,6 +82,8 @@ def STSB(root, split): raise ModuleNotFoundError( "Package `torchdata` not found. Please install following instructions at https://github.com/pytorch/data" ) + from torchdata.datapipes.iter import FileOpener, GDriveReader, HttpReader, IterableWrapper # noqa + url_dp = IterableWrapper([URL]) cache_compressed_dp = url_dp.on_disk_cache( diff --git a/torchtext/datasets/udpos.py b/torchtext/datasets/udpos.py index 92e9a980b5..d93d0627cf 100644 --- a/torchtext/datasets/udpos.py +++ b/torchtext/datasets/udpos.py @@ -3,7 +3,7 @@ from typing import Union, Tuple -from torchtext._download_hooks import HttpReader + from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( _wrap_split_argument, @@ -66,6 +66,7 @@ def UDPOS(root: str, split: Union[Tuple[str], str]): raise ModuleNotFoundError( "Package `torchdata` not found. Please install following instructions at https://github.com/pytorch/data" ) + from torchdata.datapipes.iter import FileOpener, GDriveReader, HttpReader, IterableWrapper # noqa url_dp = IterableWrapper([URL]) cache_compressed_dp = url_dp.on_disk_cache( diff --git a/torchtext/datasets/wikitext103.py b/torchtext/datasets/wikitext103.py index 67f9a07667..1b394c67dc 100644 --- a/torchtext/datasets/wikitext103.py +++ b/torchtext/datasets/wikitext103.py @@ -3,7 +3,7 @@ from typing import Union, Tuple -from torchtext._download_hooks import HttpReader + from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( _wrap_split_argument, @@ -71,6 +71,8 @@ def WikiText103(root: str, split: Union[Tuple[str], str]): raise ModuleNotFoundError( "Package `torchdata` not found. Please install following instructions at https://github.com/pytorch/data" ) + from torchdata.datapipes.iter import FileOpener, GDriveReader, HttpReader, IterableWrapper # noqa + url_dp = IterableWrapper([URL]) # cache data on-disk diff --git a/torchtext/datasets/wikitext2.py b/torchtext/datasets/wikitext2.py index 8b101ba12c..96cc7c12e5 100644 --- a/torchtext/datasets/wikitext2.py +++ b/torchtext/datasets/wikitext2.py @@ -3,7 +3,7 @@ from typing import Union, Tuple -from torchtext._download_hooks import HttpReader + from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( _wrap_split_argument, @@ -71,6 +71,7 @@ def WikiText2(root: str, split: Union[Tuple[str], str]): raise ModuleNotFoundError( "Package `torchdata` not found. Please install following instructions at https://github.com/pytorch/data" ) + from torchdata.datapipes.iter import FileOpener, GDriveReader, HttpReader, IterableWrapper # noqa url_dp = IterableWrapper([URL]) # cache data on-disk diff --git a/torchtext/datasets/wnli.py b/torchtext/datasets/wnli.py index cf24155ef4..a36093590d 100644 --- a/torchtext/datasets/wnli.py +++ b/torchtext/datasets/wnli.py @@ -6,7 +6,7 @@ # we import HttpReader from _download_hooks so we can swap out public URLs # with interal URLs when the dataset is used within Facebook -from torchtext._download_hooks import HttpReader + from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( _create_dataset_directory, @@ -78,6 +78,7 @@ def WNLI(root, split): raise ModuleNotFoundError( "Package `torchdata` not found. Please install following instructions at `https://github.com/pytorch/data`" ) + from torchdata.datapipes.iter import FileOpener, GDriveReader, HttpReader, IterableWrapper # noqa url_dp = IterableWrapper([URL]) cache_compressed_dp = url_dp.on_disk_cache( diff --git a/torchtext/datasets/yahooanswers.py b/torchtext/datasets/yahooanswers.py index 908e95dcc6..d5073781a1 100644 --- a/torchtext/datasets/yahooanswers.py +++ b/torchtext/datasets/yahooanswers.py @@ -3,7 +3,7 @@ from typing import Union, Tuple -from torchtext._download_hooks import GDriveReader + from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( _wrap_split_argument, @@ -75,6 +75,7 @@ def YahooAnswers(root: str, split: Union[Tuple[str], str]): raise ModuleNotFoundError( "Package `torchdata` not found. Please install following instructions at https://github.com/pytorch/data" ) + from torchdata.datapipes.iter import FileOpener, GDriveReader, HttpReader, IterableWrapper # noqa url_dp = IterableWrapper([URL]) diff --git a/torchtext/datasets/yelpreviewfull.py b/torchtext/datasets/yelpreviewfull.py index 7750141c56..6fab3ab8aa 100644 --- a/torchtext/datasets/yelpreviewfull.py +++ b/torchtext/datasets/yelpreviewfull.py @@ -3,7 +3,7 @@ from typing import Union, Tuple -from torchtext._download_hooks import GDriveReader + from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( _wrap_split_argument, @@ -74,6 +74,7 @@ def YelpReviewFull(root: str, split: Union[Tuple[str], str]): raise ModuleNotFoundError( "Package `torchdata` not found. Please install following instructions at https://github.com/pytorch/data" ) + from torchdata.datapipes.iter import FileOpener, GDriveReader, HttpReader, IterableWrapper # noqa url_dp = IterableWrapper([URL]) diff --git a/torchtext/datasets/yelpreviewpolarity.py b/torchtext/datasets/yelpreviewpolarity.py index 755b3b924d..b80d589e09 100644 --- a/torchtext/datasets/yelpreviewpolarity.py +++ b/torchtext/datasets/yelpreviewpolarity.py @@ -3,7 +3,7 @@ from typing import Union, Tuple -from torchtext._download_hooks import GDriveReader + from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( _wrap_split_argument, @@ -74,6 +74,7 @@ def YelpReviewPolarity(root: str, split: Union[Tuple[str], str]): raise ModuleNotFoundError( "Package `torchdata` not found. Please install following instructions at https://github.com/pytorch/data" ) + from torchdata.datapipes.iter import FileOpener, GDriveReader, HttpReader, IterableWrapper # noqa url_dp = IterableWrapper([URL]) From 7d494b1508740cc00e32d4957cc555e4ad57418a Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Fri, 22 Mar 2024 16:54:52 +0000 Subject: [PATCH 3/3] lint --- torchtext/datasets/ag_news.py | 2 -- torchtext/datasets/amazonreviewfull.py | 2 -- torchtext/datasets/amazonreviewpolarity.py | 2 -- torchtext/datasets/cc100.py | 2 -- torchtext/datasets/cnndm.py | 2 +- torchtext/datasets/cola.py | 2 -- torchtext/datasets/conll2000chunking.py | 2 -- torchtext/datasets/dbpedia.py | 2 -- torchtext/datasets/enwik9.py | 2 -- torchtext/datasets/imdb.py | 2 -- torchtext/datasets/iwslt2016.py | 2 -- torchtext/datasets/iwslt2017.py | 2 -- torchtext/datasets/mnli.py | 2 -- torchtext/datasets/multi30k.py | 3 +-- torchtext/datasets/penntreebank.py | 3 +-- torchtext/datasets/qnli.py | 2 -- torchtext/datasets/qqp.py | 2 -- torchtext/datasets/rte.py | 2 -- torchtext/datasets/sogounews.py | 2 -- torchtext/datasets/squad1.py | 3 --- torchtext/datasets/squad2.py | 2 -- torchtext/datasets/sst2.py | 2 -- torchtext/datasets/stsb.py | 3 --- torchtext/datasets/udpos.py | 2 -- torchtext/datasets/wikitext103.py | 3 --- torchtext/datasets/wikitext2.py | 2 -- torchtext/datasets/wnli.py | 2 -- torchtext/datasets/yahooanswers.py | 2 -- torchtext/datasets/yelpreviewfull.py | 2 -- torchtext/datasets/yelpreviewpolarity.py | 2 -- 30 files changed, 3 insertions(+), 62 deletions(-) diff --git a/torchtext/datasets/ag_news.py b/torchtext/datasets/ag_news.py index 1e1bf61f02..93f398329c 100644 --- a/torchtext/datasets/ag_news.py +++ b/torchtext/datasets/ag_news.py @@ -2,8 +2,6 @@ from functools import partial from typing import Union, Tuple - - from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( _wrap_split_argument, diff --git a/torchtext/datasets/amazonreviewfull.py b/torchtext/datasets/amazonreviewfull.py index 475b8dcff0..c916d2e034 100644 --- a/torchtext/datasets/amazonreviewfull.py +++ b/torchtext/datasets/amazonreviewfull.py @@ -2,8 +2,6 @@ from functools import partial from typing import Union, Tuple - - from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( _wrap_split_argument, diff --git a/torchtext/datasets/amazonreviewpolarity.py b/torchtext/datasets/amazonreviewpolarity.py index 3df6b019ea..a0ed0c6c40 100644 --- a/torchtext/datasets/amazonreviewpolarity.py +++ b/torchtext/datasets/amazonreviewpolarity.py @@ -2,8 +2,6 @@ from functools import partial from typing import Union, Tuple - - from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( _wrap_split_argument, diff --git a/torchtext/datasets/cc100.py b/torchtext/datasets/cc100.py index bfb4b96a67..0f7cf2920f 100644 --- a/torchtext/datasets/cc100.py +++ b/torchtext/datasets/cc100.py @@ -1,8 +1,6 @@ import os.path from functools import partial - - from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( _create_dataset_directory, diff --git a/torchtext/datasets/cnndm.py b/torchtext/datasets/cnndm.py index 33a70f52f8..92b2da8ce1 100644 --- a/torchtext/datasets/cnndm.py +++ b/torchtext/datasets/cnndm.py @@ -135,7 +135,7 @@ def CNNDM(root: str, split: Union[Tuple[str], str]): raise ModuleNotFoundError( "Package `torchdata` not found. Please install following instructions at https://github.com/pytorch/data" ) - from torchdata.datapipes.iter import ( + from torchdata.datapipes.iter import ( # noqa FileOpener, IterableWrapper, OnlineReader, diff --git a/torchtext/datasets/cola.py b/torchtext/datasets/cola.py index 43494ae1ce..6ec6cd8b29 100644 --- a/torchtext/datasets/cola.py +++ b/torchtext/datasets/cola.py @@ -3,8 +3,6 @@ from functools import partial from typing import Union, Tuple - - from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import _create_dataset_directory, _wrap_split_argument diff --git a/torchtext/datasets/conll2000chunking.py b/torchtext/datasets/conll2000chunking.py index 2f9de527a8..983059faf1 100644 --- a/torchtext/datasets/conll2000chunking.py +++ b/torchtext/datasets/conll2000chunking.py @@ -2,8 +2,6 @@ from functools import partial from typing import Union, Tuple - - from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( _wrap_split_argument, diff --git a/torchtext/datasets/dbpedia.py b/torchtext/datasets/dbpedia.py index 53803ba662..d563f965cb 100644 --- a/torchtext/datasets/dbpedia.py +++ b/torchtext/datasets/dbpedia.py @@ -2,8 +2,6 @@ from functools import partial from typing import Union, Tuple - - from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( _wrap_split_argument, diff --git a/torchtext/datasets/enwik9.py b/torchtext/datasets/enwik9.py index f966bada50..8b30cc4da8 100644 --- a/torchtext/datasets/enwik9.py +++ b/torchtext/datasets/enwik9.py @@ -1,8 +1,6 @@ import os from functools import partial - - from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import _create_dataset_directory diff --git a/torchtext/datasets/imdb.py b/torchtext/datasets/imdb.py index 380e97c5e9..cefedc4bf0 100644 --- a/torchtext/datasets/imdb.py +++ b/torchtext/datasets/imdb.py @@ -3,8 +3,6 @@ from pathlib import Path from typing import Tuple, Union - - from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import _create_dataset_directory from torchtext.data.datasets_utils import _wrap_split_argument diff --git a/torchtext/datasets/iwslt2016.py b/torchtext/datasets/iwslt2016.py index 746b40bb62..f1a05dcaea 100644 --- a/torchtext/datasets/iwslt2016.py +++ b/torchtext/datasets/iwslt2016.py @@ -1,8 +1,6 @@ import os from functools import partial - - from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( _clean_files, diff --git a/torchtext/datasets/iwslt2017.py b/torchtext/datasets/iwslt2017.py index 36e233832b..2095647fe4 100644 --- a/torchtext/datasets/iwslt2017.py +++ b/torchtext/datasets/iwslt2017.py @@ -1,8 +1,6 @@ import os from functools import partial - - from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( _clean_files, diff --git a/torchtext/datasets/mnli.py b/torchtext/datasets/mnli.py index aebfd08a39..def9354b53 100644 --- a/torchtext/datasets/mnli.py +++ b/torchtext/datasets/mnli.py @@ -3,8 +3,6 @@ import os from functools import partial - - # we import HttpReader from _download_hooks so we can swap out public URLs # with interal URLs when the dataset is used within Facebook diff --git a/torchtext/datasets/multi30k.py b/torchtext/datasets/multi30k.py index ed4deb82bd..db666bfda9 100644 --- a/torchtext/datasets/multi30k.py +++ b/torchtext/datasets/multi30k.py @@ -2,8 +2,7 @@ from functools import partial from typing import Union, Tuple - - # noqa +# noqa from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( diff --git a/torchtext/datasets/penntreebank.py b/torchtext/datasets/penntreebank.py index 3720a14c48..a7f504b9a4 100644 --- a/torchtext/datasets/penntreebank.py +++ b/torchtext/datasets/penntreebank.py @@ -2,8 +2,7 @@ from functools import partial from typing import Tuple, Union - - # noqa +# noqa from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( diff --git a/torchtext/datasets/qnli.py b/torchtext/datasets/qnli.py index 6422ee93e7..cbdca8fbc4 100644 --- a/torchtext/datasets/qnli.py +++ b/torchtext/datasets/qnli.py @@ -3,8 +3,6 @@ import os from functools import partial - - # we import HttpReader from _download_hooks so we can swap out public URLs # with interal URLs when the dataset is used within Facebook diff --git a/torchtext/datasets/qqp.py b/torchtext/datasets/qqp.py index f0686e6df2..887675cfde 100644 --- a/torchtext/datasets/qqp.py +++ b/torchtext/datasets/qqp.py @@ -1,8 +1,6 @@ import os from functools import partial - - from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import _create_dataset_directory diff --git a/torchtext/datasets/rte.py b/torchtext/datasets/rte.py index 3ab15f6e13..61915a1790 100644 --- a/torchtext/datasets/rte.py +++ b/torchtext/datasets/rte.py @@ -3,8 +3,6 @@ import os from functools import partial - - # we import HttpReader from _download_hooks so we can swap out public URLs # with interal URLs when the dataset is used within Facebook diff --git a/torchtext/datasets/sogounews.py b/torchtext/datasets/sogounews.py index 3dfb9db1a2..440e811ce4 100644 --- a/torchtext/datasets/sogounews.py +++ b/torchtext/datasets/sogounews.py @@ -2,8 +2,6 @@ from functools import partial from typing import Union, Tuple - - from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( _wrap_split_argument, diff --git a/torchtext/datasets/squad1.py b/torchtext/datasets/squad1.py index a2cfa1532c..0949eb103c 100644 --- a/torchtext/datasets/squad1.py +++ b/torchtext/datasets/squad1.py @@ -2,8 +2,6 @@ from functools import partial from typing import Union, Tuple - - from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( _wrap_split_argument, @@ -64,7 +62,6 @@ def SQuAD1(root: str, split: Union[Tuple[str], str]): ) from torchdata.datapipes.iter import FileOpener, GDriveReader, HttpReader, IterableWrapper # noqa - url_dp = IterableWrapper([URL[split]]) # cache data on-disk with sanity check cache_dp = url_dp.on_disk_cache( diff --git a/torchtext/datasets/squad2.py b/torchtext/datasets/squad2.py index 2359a91e21..0ad1e25ac1 100644 --- a/torchtext/datasets/squad2.py +++ b/torchtext/datasets/squad2.py @@ -2,8 +2,6 @@ from functools import partial from typing import Union, Tuple - - from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( _wrap_split_argument, diff --git a/torchtext/datasets/sst2.py b/torchtext/datasets/sst2.py index 9a350197ee..a14cf45709 100644 --- a/torchtext/datasets/sst2.py +++ b/torchtext/datasets/sst2.py @@ -2,8 +2,6 @@ import os from functools import partial - - # we import HttpReader from _download_hooks so we can swap out public URLs # with interal URLs when the dataset is used within Facebook diff --git a/torchtext/datasets/stsb.py b/torchtext/datasets/stsb.py index 892ecc7288..1f66bf5279 100644 --- a/torchtext/datasets/stsb.py +++ b/torchtext/datasets/stsb.py @@ -2,8 +2,6 @@ import os from functools import partial - - # we import HttpReader from _download_hooks so we can swap out public URLs # with interal URLs when the dataset is used within Facebook @@ -84,7 +82,6 @@ def STSB(root, split): ) from torchdata.datapipes.iter import FileOpener, GDriveReader, HttpReader, IterableWrapper # noqa - url_dp = IterableWrapper([URL]) cache_compressed_dp = url_dp.on_disk_cache( filepath_fn=partial(_filepath_fn, root), diff --git a/torchtext/datasets/udpos.py b/torchtext/datasets/udpos.py index d93d0627cf..c6ee494dae 100644 --- a/torchtext/datasets/udpos.py +++ b/torchtext/datasets/udpos.py @@ -2,8 +2,6 @@ from functools import partial from typing import Union, Tuple - - from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( _wrap_split_argument, diff --git a/torchtext/datasets/wikitext103.py b/torchtext/datasets/wikitext103.py index 1b394c67dc..6baff13ad6 100644 --- a/torchtext/datasets/wikitext103.py +++ b/torchtext/datasets/wikitext103.py @@ -2,8 +2,6 @@ from functools import partial from typing import Union, Tuple - - from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( _wrap_split_argument, @@ -73,7 +71,6 @@ def WikiText103(root: str, split: Union[Tuple[str], str]): ) from torchdata.datapipes.iter import FileOpener, GDriveReader, HttpReader, IterableWrapper # noqa - url_dp = IterableWrapper([URL]) # cache data on-disk cache_compressed_dp = url_dp.on_disk_cache( diff --git a/torchtext/datasets/wikitext2.py b/torchtext/datasets/wikitext2.py index 96cc7c12e5..94e90f2031 100644 --- a/torchtext/datasets/wikitext2.py +++ b/torchtext/datasets/wikitext2.py @@ -2,8 +2,6 @@ from functools import partial from typing import Union, Tuple - - from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( _wrap_split_argument, diff --git a/torchtext/datasets/wnli.py b/torchtext/datasets/wnli.py index a36093590d..f4574d5e4e 100644 --- a/torchtext/datasets/wnli.py +++ b/torchtext/datasets/wnli.py @@ -2,8 +2,6 @@ import os from functools import partial - - # we import HttpReader from _download_hooks so we can swap out public URLs # with interal URLs when the dataset is used within Facebook diff --git a/torchtext/datasets/yahooanswers.py b/torchtext/datasets/yahooanswers.py index d5073781a1..da357977cb 100644 --- a/torchtext/datasets/yahooanswers.py +++ b/torchtext/datasets/yahooanswers.py @@ -2,8 +2,6 @@ from functools import partial from typing import Union, Tuple - - from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( _wrap_split_argument, diff --git a/torchtext/datasets/yelpreviewfull.py b/torchtext/datasets/yelpreviewfull.py index 6fab3ab8aa..7bea8f1211 100644 --- a/torchtext/datasets/yelpreviewfull.py +++ b/torchtext/datasets/yelpreviewfull.py @@ -2,8 +2,6 @@ from functools import partial from typing import Union, Tuple - - from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( _wrap_split_argument, diff --git a/torchtext/datasets/yelpreviewpolarity.py b/torchtext/datasets/yelpreviewpolarity.py index b80d589e09..08559f0c68 100644 --- a/torchtext/datasets/yelpreviewpolarity.py +++ b/torchtext/datasets/yelpreviewpolarity.py @@ -2,8 +2,6 @@ from functools import partial from typing import Union, Tuple - - from torchtext._internal.module_utils import is_module_available from torchtext.data.datasets_utils import ( _wrap_split_argument,