|
5 | 5 | from torchdata.datapipes.iter import FileOpener, GDriveReader, IterableWrapper, FileLister |
6 | 6 |
|
7 | 7 | import os |
8 | | -from torchtext.utils import (download_from_url, extract_archive) |
9 | 8 | from torchtext.data.datasets_utils import ( |
10 | | - _RawTextIterableDataset, |
11 | 9 | _wrap_split_argument, |
12 | 10 | _clean_xml_file, |
13 | 11 | _clean_tags_file, |
14 | | - _read_text_iterator, |
15 | 12 | ) |
16 | 13 | from torchtext.data.datasets_utils import _create_dataset_directory |
17 | 14 |
|
@@ -160,7 +157,7 @@ def _construct_filepaths(paths, src_filename, tgt_filename): |
160 | 157 |
|
161 | 158 | @_create_dataset_directory(dataset_name=DATASET_NAME) |
162 | 159 | @_wrap_split_argument(("train", "valid", "test")) |
163 | | -def IWSLT2016(root = '.data', split=('train', 'valid', 'test'), language_pair=('de', 'en'), valid_set='tst2013', test_set='tst2014'): |
| 160 | +def IWSLT2016(root='.data', split=('train', 'valid', 'test'), language_pair=('de', 'en'), valid_set='tst2013', test_set='tst2014'): |
164 | 161 | """IWSLT2016 dataset |
165 | 162 |
|
166 | 163 | The available datasets include following: |
@@ -198,12 +195,6 @@ def IWSLT2016(root = '.data', split=('train', 'valid', 'test'), language_pair=(' |
198 | 195 | >>> src_sentence, tgt_sentence = next(train_iter) |
199 | 196 |
|
200 | 197 | """ |
201 | | - num_lines_set_identifier = { |
202 | | - 'train': 'train', |
203 | | - 'valid': valid_set, |
204 | | - 'test': test_set |
205 | | - } |
206 | | - |
207 | 198 | if not is_module_available("torchdata"): |
208 | 199 | raise ModuleNotFoundError("Package `torchdata` not found. Please install following instructions at `https://github.com/pytorch/data`") |
209 | 200 |
|
|
0 commit comments