From e11217ed731ee8b8e60c5aee2d37b1eefdb10328 Mon Sep 17 00:00:00 2001 From: Parmeet Singh Bhatia Date: Mon, 17 Jan 2022 02:42:28 -0500 Subject: [PATCH 1/5] caching extracted files --- torchtext/datasets/amazonreviewpolarity.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/torchtext/datasets/amazonreviewpolarity.py b/torchtext/datasets/amazonreviewpolarity.py index de1f975a7b..c63039ef85 100644 --- a/torchtext/datasets/amazonreviewpolarity.py +++ b/torchtext/datasets/amazonreviewpolarity.py @@ -23,8 +23,8 @@ _PATH = 'amazon_review_polarity_csv.tar.gz' _EXTRACTED_FILES = { - 'train': os.path.join('amazon_review_polarity_csv', 'train.csv'), - 'test': os.path.join('amazon_review_polarity_csv', 'test.csv'), + 'train': os.path.join(_PATH, 'amazon_review_polarity_csv', 'train.csv'), + 'test': os.path.join(_PATH, 'amazon_review_polarity_csv', 'test.csv'), } @@ -40,11 +40,13 @@ def AmazonReviewPolarity(root: str, split: Union[Tuple[str], str]): raise ModuleNotFoundError("Package `torchdata` not found. Please install following instructions at `https://github.com/pytorch/data`") url_dp = IterableWrapper([URL]) - cache_dp = url_dp.on_disk_cache( + cache_compressed_dp = url_dp.on_disk_cache( filepath_fn=lambda x: os.path.join(root, _PATH), hash_dict={os.path.join(root, _PATH): MD5}, hash_type="md5" ) - cache_dp = GDriveReader(cache_dp).end_caching(mode="wb", same_filepath_fn=True) - cache_dp = FileOpener(cache_dp, mode="b") - extracted_files = cache_dp.read_from_tar() - filter_extracted_files = extracted_files.filter(lambda x: _EXTRACTED_FILES[split] in x[0]) - return filter_extracted_files.parse_csv().map(fn=lambda t: (int(t[0]), ' '.join(t[1:]))) + cache_compressed_dp = GDriveReader(cache_compressed_dp).end_caching(mode="wb", same_filepath_fn=True) + cache_decompressed_dp = cache_compressed_dp.on_disk_cache( + filepath_fn=lambda x: os.path.join(root, os.path.dirname(_EXTRACTED_FILES[split]), os.path.basename(x))) + cache_decompressed_dp = FileOpener(cache_decompressed_dp, mode="b").read_from_tar() + cache_compressed_dp = cache_decompressed_dp.end_caching(mode="wb", same_filepath_fn=True) + data_dp = FileOpener(cache_decompressed_dp.filter(lambda x: _EXTRACTED_FILES[split] in x[0]).map(lambda x: x[0]), mode='b') + return data_dp.parse_csv().map(fn=lambda t: (int(t[0]), ' '.join(t[1:]))) From c09eb57e6d497224e5174e93a0bdb8ce7ff8ae7f Mon Sep 17 00:00:00 2001 From: Parmeet Singh Bhatia Date: Tue, 18 Jan 2022 18:31:39 -0500 Subject: [PATCH 2/5] fix caching --- torchtext/datasets/amazonreviewpolarity.py | 28 +++++++++++++++++----- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/torchtext/datasets/amazonreviewpolarity.py b/torchtext/datasets/amazonreviewpolarity.py index c63039ef85..41daa193de 100644 --- a/torchtext/datasets/amazonreviewpolarity.py +++ b/torchtext/datasets/amazonreviewpolarity.py @@ -23,8 +23,8 @@ _PATH = 'amazon_review_polarity_csv.tar.gz' _EXTRACTED_FILES = { - 'train': os.path.join(_PATH, 'amazon_review_polarity_csv', 'train.csv'), - 'test': os.path.join(_PATH, 'amazon_review_polarity_csv', 'test.csv'), + 'train': os.path.join('amazon_review_polarity_csv', 'train.csv'), + 'test': os.path.join('amazon_review_polarity_csv', 'test.csv'), } @@ -44,9 +44,25 @@ def AmazonReviewPolarity(root: str, split: Union[Tuple[str], str]): filepath_fn=lambda x: os.path.join(root, _PATH), hash_dict={os.path.join(root, _PATH): MD5}, hash_type="md5" ) cache_compressed_dp = GDriveReader(cache_compressed_dp).end_caching(mode="wb", same_filepath_fn=True) + + def extracted_filepath_fn(x): + file_path = os.path.join(root, _EXTRACTED_FILES[split]) + dir_path = os.path.dirname(file_path) + if not os.path.exists(dir_path): + os.makedirs(dir_path) + return file_path + cache_decompressed_dp = cache_compressed_dp.on_disk_cache( - filepath_fn=lambda x: os.path.join(root, os.path.dirname(_EXTRACTED_FILES[split]), os.path.basename(x))) - cache_decompressed_dp = FileOpener(cache_decompressed_dp, mode="b").read_from_tar() - cache_compressed_dp = cache_decompressed_dp.end_caching(mode="wb", same_filepath_fn=True) - data_dp = FileOpener(cache_decompressed_dp.filter(lambda x: _EXTRACTED_FILES[split] in x[0]).map(lambda x: x[0]), mode='b') + filepath_fn=extracted_filepath_fn) + cache_decompressed_dp = FileOpener(cache_decompressed_dp, mode="b").\ + read_from_tar().\ + filter(lambda x: _EXTRACTED_FILES[split] in x[0]).\ + map(lambda x: (x[0].replace('_PATH' + '/', ''), x[1])) + cache_decompressed_dp = cache_decompressed_dp.end_caching(mode="wb", same_filepath_fn=True) + data_dp = FileOpener(cache_decompressed_dp, mode='b') + + # data_dp = FileOpener(cache_compressed_dp, mode='b') + # data_dp = data_dp.read_from_tar() + # data_dp = data_dp.filter(lambda x: _EXTRACTED_FILES[split] in x[0]) + return data_dp.parse_csv().map(fn=lambda t: (int(t[0]), ' '.join(t[1:]))) From 27e94a79700e3a551969c46c2aa0df66a6283503 Mon Sep 17 00:00:00 2001 From: Parmeet Singh Bhatia Date: Tue, 18 Jan 2022 18:43:04 -0500 Subject: [PATCH 3/5] remove comparison code --- torchtext/datasets/amazonreviewpolarity.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/torchtext/datasets/amazonreviewpolarity.py b/torchtext/datasets/amazonreviewpolarity.py index 41daa193de..6229b73d49 100644 --- a/torchtext/datasets/amazonreviewpolarity.py +++ b/torchtext/datasets/amazonreviewpolarity.py @@ -61,8 +61,4 @@ def extracted_filepath_fn(x): cache_decompressed_dp = cache_decompressed_dp.end_caching(mode="wb", same_filepath_fn=True) data_dp = FileOpener(cache_decompressed_dp, mode='b') - # data_dp = FileOpener(cache_compressed_dp, mode='b') - # data_dp = data_dp.read_from_tar() - # data_dp = data_dp.filter(lambda x: _EXTRACTED_FILES[split] in x[0]) - return data_dp.parse_csv().map(fn=lambda t: (int(t[0]), ' '.join(t[1:]))) From 4b5bcd99b6e31ead5ee009d9d0b88dfcc768f0a3 Mon Sep 17 00:00:00 2001 From: Parmeet Singh Bhatia Date: Tue, 18 Jan 2022 22:57:49 -0500 Subject: [PATCH 4/5] remove mapping of path --- torchtext/datasets/amazonreviewpolarity.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/torchtext/datasets/amazonreviewpolarity.py b/torchtext/datasets/amazonreviewpolarity.py index 6229b73d49..aaac03421a 100644 --- a/torchtext/datasets/amazonreviewpolarity.py +++ b/torchtext/datasets/amazonreviewpolarity.py @@ -54,11 +54,8 @@ def extracted_filepath_fn(x): cache_decompressed_dp = cache_compressed_dp.on_disk_cache( filepath_fn=extracted_filepath_fn) - cache_decompressed_dp = FileOpener(cache_decompressed_dp, mode="b").\ - read_from_tar().\ - filter(lambda x: _EXTRACTED_FILES[split] in x[0]).\ - map(lambda x: (x[0].replace('_PATH' + '/', ''), x[1])) + cache_decompressed_dp = FileOpener(cache_decompressed_dp, mode="b").read_from_tar().filter(lambda x: _EXTRACTED_FILES[split] in x[0]) cache_decompressed_dp = cache_decompressed_dp.end_caching(mode="wb", same_filepath_fn=True) - data_dp = FileOpener(cache_decompressed_dp, mode='b') + data_dp = FileOpener(cache_decompressed_dp, mode="b") return data_dp.parse_csv().map(fn=lambda t: (int(t[0]), ' '.join(t[1:]))) From 36d5370938f7b5a67cba85172497a759513f43a8 Mon Sep 17 00:00:00 2001 From: Parmeet Singh Bhatia Date: Wed, 19 Jan 2022 13:21:00 -0500 Subject: [PATCH 5/5] fix directory creation --- torchtext/datasets/amazonreviewpolarity.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/torchtext/datasets/amazonreviewpolarity.py b/torchtext/datasets/amazonreviewpolarity.py index aaac03421a..e585922c19 100644 --- a/torchtext/datasets/amazonreviewpolarity.py +++ b/torchtext/datasets/amazonreviewpolarity.py @@ -45,17 +45,9 @@ def AmazonReviewPolarity(root: str, split: Union[Tuple[str], str]): ) cache_compressed_dp = GDriveReader(cache_compressed_dp).end_caching(mode="wb", same_filepath_fn=True) - def extracted_filepath_fn(x): - file_path = os.path.join(root, _EXTRACTED_FILES[split]) - dir_path = os.path.dirname(file_path) - if not os.path.exists(dir_path): - os.makedirs(dir_path) - return file_path - - cache_decompressed_dp = cache_compressed_dp.on_disk_cache( - filepath_fn=extracted_filepath_fn) + cache_decompressed_dp = cache_compressed_dp.on_disk_cache(filepath_fn=lambda x: os.path.join(root, _EXTRACTED_FILES[split])) cache_decompressed_dp = FileOpener(cache_decompressed_dp, mode="b").read_from_tar().filter(lambda x: _EXTRACTED_FILES[split] in x[0]) cache_decompressed_dp = cache_decompressed_dp.end_caching(mode="wb", same_filepath_fn=True) - data_dp = FileOpener(cache_decompressed_dp, mode="b") + data_dp = FileOpener(cache_decompressed_dp, mode='b') return data_dp.parse_csv().map(fn=lambda t: (int(t[0]), ' '.join(t[1:])))