From 63d9f8a34e9bf254572e37e3f86a4a572b5fbfc3 Mon Sep 17 00:00:00 2001 From: Elijah Rippeth Date: Wed, 19 Jan 2022 08:21:22 -0500 Subject: [PATCH 1/4] add double caching for yahoo to speed up extracted reading. --- torchtext/datasets/yahooanswers.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/torchtext/datasets/yahooanswers.py b/torchtext/datasets/yahooanswers.py index 7e95e2c6ca..afaa0c406b 100644 --- a/torchtext/datasets/yahooanswers.py +++ b/torchtext/datasets/yahooanswers.py @@ -33,7 +33,7 @@ @_add_docstring_header(num_lines=NUM_LINES, num_classes=10) @_create_dataset_directory(dataset_name=DATASET_NAME) -@_wrap_split_argument(('train', 'test')) +@_wrap_split_argument(("train", "test")) def YahooAnswers(root: str, split: Union[Tuple[str], str]): if not is_module_available("torchdata"): raise ModuleNotFoundError("Package `torchdata` not found. Please install following instructions at `https://github.com/pytorch/data`") @@ -42,13 +42,27 @@ def YahooAnswers(root: str, split: Union[Tuple[str], str]): cache_dp = url_dp.on_disk_cache( filepath_fn=lambda x: os.path.join(root, _PATH), - hash_dict={os.path.join(root, _PATH): MD5}, hash_type="md5" + hash_dict={os.path.join(root, _PATH): MD5}, + hash_type="md5" ) cache_dp = GDriveReader(cache_dp).end_caching(mode="wb", same_filepath_fn=True) cache_dp = FileOpener(cache_dp, mode="b") - extracted_files = cache_dp.read_from_tar() + def extracted_filepath_fn(_): + file_path = os.path.join(root, _EXTRACTED_FILES[split]) + dir_path = os.path.dirname(file_path) + if not os.path.exists(dir_path): + os.makedirs(dir_path) + return file_path - filter_extracted_files = extracted_files.filter(lambda x: _EXTRACTED_FILES[split] in x[0]) + cache_dp = cache_dp.on_disk_cache( + filepath_fn=extracted_filepath_fn + ) + cache_dp = cache_dp.read_from_tar() + + cache_dp = cache_dp.filter(lambda x: _EXTRACTED_FILES[split] in x[0]) + cache_dp = cache_dp.end_caching(mode="wb", same_filepath_fn=True) + + cache_dp = FileOpener(cache_dp, mode="b") - return filter_extracted_files.parse_csv().map(fn=lambda t: (int(t[0]), " ".join(t[1:]))) + return cache_dp.parse_csv().map(fn=lambda t: (int(t[0]), " ".join(t[1:]))) From 9e79a983b737d2e7ab86304bda2d3d6a67346fc0 Mon Sep 17 00:00:00 2001 From: Elijah Rippeth Date: Wed, 19 Jan 2022 14:44:27 -0500 Subject: [PATCH 2/4] simplify filepath_fn --- torchtext/datasets/yahooanswers.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/torchtext/datasets/yahooanswers.py b/torchtext/datasets/yahooanswers.py index afaa0c406b..c2d2a856c1 100644 --- a/torchtext/datasets/yahooanswers.py +++ b/torchtext/datasets/yahooanswers.py @@ -48,18 +48,8 @@ def YahooAnswers(root: str, split: Union[Tuple[str], str]): cache_dp = GDriveReader(cache_dp).end_caching(mode="wb", same_filepath_fn=True) cache_dp = FileOpener(cache_dp, mode="b") - def extracted_filepath_fn(_): - file_path = os.path.join(root, _EXTRACTED_FILES[split]) - dir_path = os.path.dirname(file_path) - if not os.path.exists(dir_path): - os.makedirs(dir_path) - return file_path - - cache_dp = cache_dp.on_disk_cache( - filepath_fn=extracted_filepath_fn - ) + cache_dp = cache_dp.on_disk_cache(filepath_fn=lambda x: os.path.join(root, _EXTRACTED_FILES[split])) cache_dp = cache_dp.read_from_tar() - cache_dp = cache_dp.filter(lambda x: _EXTRACTED_FILES[split] in x[0]) cache_dp = cache_dp.end_caching(mode="wb", same_filepath_fn=True) From aa0ba5ab4f06af49f20af87cf0c32a084ec0a8d3 Mon Sep 17 00:00:00 2001 From: Elijah Rippeth Date: Wed, 19 Jan 2022 14:45:22 -0500 Subject: [PATCH 3/4] rename dps for consistency. --- torchtext/datasets/yahooanswers.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/torchtext/datasets/yahooanswers.py b/torchtext/datasets/yahooanswers.py index c2d2a856c1..e011e2c04f 100644 --- a/torchtext/datasets/yahooanswers.py +++ b/torchtext/datasets/yahooanswers.py @@ -40,19 +40,21 @@ def YahooAnswers(root: str, split: Union[Tuple[str], str]): url_dp = IterableWrapper([URL]) - cache_dp = url_dp.on_disk_cache( + cache_compressed_dp = url_dp.on_disk_cache( filepath_fn=lambda x: os.path.join(root, _PATH), hash_dict={os.path.join(root, _PATH): MD5}, hash_type="md5" ) - cache_dp = GDriveReader(cache_dp).end_caching(mode="wb", same_filepath_fn=True) - cache_dp = FileOpener(cache_dp, mode="b") + cache_compressed_dp = GDriveReader(cache_compressed_dp).end_caching(mode="wb", same_filepath_fn=True) + cache_compressed_dp = FileOpener(cache_compressed_dp, mode="b") - cache_dp = cache_dp.on_disk_cache(filepath_fn=lambda x: os.path.join(root, _EXTRACTED_FILES[split])) - cache_dp = cache_dp.read_from_tar() - cache_dp = cache_dp.filter(lambda x: _EXTRACTED_FILES[split] in x[0]) - cache_dp = cache_dp.end_caching(mode="wb", same_filepath_fn=True) + cache_decompressed_dp = cache_compressed_dp.on_disk_cache( + filepath_fn=lambda x: os.path.join(root, _EXTRACTED_FILES[split]) + ) + cache_decompressed_dp = cache_decompressed_dp.read_from_tar() + cache_decompressed_dp = cache_decompressed_dp.filter(lambda x: _EXTRACTED_FILES[split] in x[0]) + cache_decompressed_dp = cache_decompressed_dp.end_caching(mode="wb", same_filepath_fn=True) - cache_dp = FileOpener(cache_dp, mode="b") + data_dp = FileOpener(cache_decompressed_dp, mode="b") - return cache_dp.parse_csv().map(fn=lambda t: (int(t[0]), " ".join(t[1:]))) + return data_dp.parse_csv().map(fn=lambda t: (int(t[0]), " ".join(t[1:]))) From 4d38900a6838f647fce34b3bc28dd5c0131c116b Mon Sep 17 00:00:00 2001 From: Elijah Rippeth Date: Thu, 20 Jan 2022 12:06:31 -0500 Subject: [PATCH 4/4] add FileOpener within caching block for more consistency. --- torchtext/datasets/yahooanswers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/torchtext/datasets/yahooanswers.py b/torchtext/datasets/yahooanswers.py index e011e2c04f..70c3d8b807 100644 --- a/torchtext/datasets/yahooanswers.py +++ b/torchtext/datasets/yahooanswers.py @@ -51,6 +51,7 @@ def YahooAnswers(root: str, split: Union[Tuple[str], str]): cache_decompressed_dp = cache_compressed_dp.on_disk_cache( filepath_fn=lambda x: os.path.join(root, _EXTRACTED_FILES[split]) ) + cache_decompressed_dp = FileOpener(cache_decompressed_dp, mode="b") cache_decompressed_dp = cache_decompressed_dp.read_from_tar() cache_decompressed_dp = cache_decompressed_dp.filter(lambda x: _EXTRACTED_FILES[split] in x[0]) cache_decompressed_dp = cache_decompressed_dp.end_caching(mode="wb", same_filepath_fn=True)