From b5b274440aceb91ffcdc84514eb972d75eaa77d5 Mon Sep 17 00:00:00 2001 From: Parmeet Singh Bhatia Date: Fri, 27 May 2022 12:03:55 -0400 Subject: [PATCH] Fix Mock tests due to change in datasets directory --- test/datasets/test_agnews.py | 2 +- test/datasets/test_amazonreviews.py | 4 +++- test/datasets/test_cc100.py | 2 +- test/datasets/test_cola.py | 2 +- test/datasets/test_conll2000chunking.py | 2 +- test/datasets/test_dbpedia.py | 2 +- test/datasets/test_enwik9.py | 2 +- test/datasets/test_imdb.py | 2 +- test/datasets/test_iwslt2016.py | 6 ++++-- test/datasets/test_iwslt2017.py | 8 ++++++-- test/datasets/test_mnli.py | 2 +- test/datasets/test_mrpc.py | 2 +- test/datasets/test_multi30k.py | 2 +- test/datasets/test_penntreebank.py | 2 +- test/datasets/test_qqp.py | 2 +- test/datasets/test_sogounews.py | 2 +- test/datasets/test_squads.py | 2 +- test/datasets/test_sst2.py | 2 +- test/datasets/test_stsb.py | 2 +- test/datasets/test_udpos.py | 2 +- test/datasets/test_wikitexts.py | 4 +++- test/datasets/test_yahooanswers.py | 2 +- test/datasets/test_yelpreviews.py | 4 +++- 23 files changed, 37 insertions(+), 25 deletions(-) diff --git a/test/datasets/test_agnews.py b/test/datasets/test_agnews.py index 0f0abd2acf..44abb5403c 100644 --- a/test/datasets/test_agnews.py +++ b/test/datasets/test_agnews.py @@ -42,7 +42,7 @@ class TestAGNews(TempDirMixin, TorchtextTestCase): def setUpClass(cls): super().setUpClass() cls.root_dir = cls.get_base_temp_dir() - cls.samples = _get_mock_dataset(cls.root_dir) + cls.samples = _get_mock_dataset(os.path.join(cls.root_dir, "datasets")) cls.patcher = patch("torchdata.datapipes.iter.util.cacheholder._hash_check", return_value=True) cls.patcher.start() diff --git a/test/datasets/test_amazonreviews.py b/test/datasets/test_amazonreviews.py index 3300ee2cd5..d8203a69b2 100644 --- a/test/datasets/test_amazonreviews.py +++ b/test/datasets/test_amazonreviews.py @@ -69,7 +69,9 @@ def tearDownClass(cls): @nested_params([AmazonReviewFull, AmazonReviewPolarity], ["train", "test"]) def test_amazon_reviews(self, amazon_review_dataset, split): - expected_samples = _get_mock_dataset(self.root_dir, amazon_review_dataset.__name__)[split] + expected_samples = _get_mock_dataset(os.path.join(self.root_dir, "datasets"), amazon_review_dataset.__name__)[ + split + ] dataset = amazon_review_dataset(root=self.root_dir, split=split) samples = list(dataset) diff --git a/test/datasets/test_cc100.py b/test/datasets/test_cc100.py index f7e47684aa..80a0659f98 100644 --- a/test/datasets/test_cc100.py +++ b/test/datasets/test_cc100.py @@ -40,7 +40,7 @@ class TestCC100(TempDirMixin, TorchtextTestCase): def setUpClass(cls): super().setUpClass() cls.root_dir = cls.get_base_temp_dir() - cls.samples = _get_mock_dataset(cls.root_dir) + cls.samples = _get_mock_dataset(os.path.join(cls.root_dir, "datasets")) cls.patcher = patch("torchdata.datapipes.iter.util.cacheholder._hash_check", return_value=True) cls.patcher.start() diff --git a/test/datasets/test_cola.py b/test/datasets/test_cola.py index d55eb31fbf..603fcb98c1 100644 --- a/test/datasets/test_cola.py +++ b/test/datasets/test_cola.py @@ -51,7 +51,7 @@ class TestCoLA(TempDirMixin, TorchtextTestCase): def setUpClass(cls): super().setUpClass() cls.root_dir = cls.get_base_temp_dir() - cls.samples = _get_mock_dataset(cls.root_dir) + cls.samples = _get_mock_dataset(os.path.join(cls.root_dir, "datasets")) cls.patcher = patch("torchdata.datapipes.iter.util.cacheholder._hash_check", return_value=True) cls.patcher.start() diff --git a/test/datasets/test_conll2000chunking.py b/test/datasets/test_conll2000chunking.py index 056cbae6ed..5f6e85180f 100644 --- a/test/datasets/test_conll2000chunking.py +++ b/test/datasets/test_conll2000chunking.py @@ -53,7 +53,7 @@ class TestCoNLL2000Chunking(TempDirMixin, TorchtextTestCase): def setUpClass(cls): super().setUpClass() cls.root_dir = cls.get_base_temp_dir() - cls.samples = _get_mock_dataset(cls.root_dir) + cls.samples = _get_mock_dataset(os.path.join(cls.root_dir, "datasets")) cls.patcher = patch("torchdata.datapipes.iter.util.cacheholder._hash_check", return_value=True) cls.patcher.start() diff --git a/test/datasets/test_dbpedia.py b/test/datasets/test_dbpedia.py index adee45d0d5..4a62ae2400 100644 --- a/test/datasets/test_dbpedia.py +++ b/test/datasets/test_dbpedia.py @@ -50,7 +50,7 @@ class TestDBpedia(TempDirMixin, TorchtextTestCase): def setUpClass(cls): super().setUpClass() cls.root_dir = cls.get_base_temp_dir() - cls.samples = _get_mock_dataset(cls.root_dir) + cls.samples = _get_mock_dataset(os.path.join(cls.root_dir, "datasets")) cls.patcher = patch("torchdata.datapipes.iter.util.cacheholder._hash_check", return_value=True) cls.patcher.start() diff --git a/test/datasets/test_enwik9.py b/test/datasets/test_enwik9.py index 9c33a11785..5bc2893606 100644 --- a/test/datasets/test_enwik9.py +++ b/test/datasets/test_enwik9.py @@ -47,7 +47,7 @@ class TestEnWik9(TempDirMixin, TorchtextTestCase): def setUpClass(cls): super().setUpClass() cls.root_dir = cls.get_base_temp_dir() - cls.samples = _get_mock_dataset(cls.root_dir) + cls.samples = _get_mock_dataset(os.path.join(cls.root_dir, "datasets")) cls.patcher = patch("torchdata.datapipes.iter.util.cacheholder._hash_check", return_value=True) cls.patcher.start() diff --git a/test/datasets/test_imdb.py b/test/datasets/test_imdb.py index a906b9103e..cb9ab3b62d 100644 --- a/test/datasets/test_imdb.py +++ b/test/datasets/test_imdb.py @@ -56,7 +56,7 @@ class TestIMDB(TempDirMixin, TorchtextTestCase): def setUpClass(cls): super().setUpClass() cls.root_dir = cls.get_base_temp_dir() - cls.samples = _get_mock_dataset(cls.root_dir) + cls.samples = _get_mock_dataset(os.path.join(cls.root_dir, "datasets")) cls.patcher = patch("torchdata.datapipes.iter.util.cacheholder._hash_check", return_value=True) cls.patcher.start() diff --git a/test/datasets/test_iwslt2016.py b/test/datasets/test_iwslt2016.py index 9631a12a3a..5dc54aa116 100644 --- a/test/datasets/test_iwslt2016.py +++ b/test/datasets/test_iwslt2016.py @@ -164,7 +164,7 @@ def tearDownClass(cls): def test_iwslt2016(self, split, src, tgt, dev_set, test_set): with tempfile.TemporaryDirectory() as root_dir: - expected_samples = _get_mock_dataset(root_dir, split, src, tgt, dev_set, test_set) + expected_samples = _get_mock_dataset(os.path.join(root_dir, "datasets"), split, src, tgt, dev_set, test_set) dataset = IWSLT2016( root=root_dir, @@ -185,7 +185,9 @@ def test_iwslt2016_split_argument(self, split): language_pair = ("de", "en") valid_set = "tst2013" test_set = "tst2014" - _ = _get_mock_dataset(root_dir, split, language_pair[0], language_pair[1], valid_set, test_set) + _ = _get_mock_dataset( + os.path.join(root_dir, "datasets"), split, language_pair[0], language_pair[1], valid_set, test_set + ) dataset1 = IWSLT2016( root=root_dir, split=split, diff --git a/test/datasets/test_iwslt2017.py b/test/datasets/test_iwslt2017.py index 650801bee6..8b61eefeaa 100644 --- a/test/datasets/test_iwslt2017.py +++ b/test/datasets/test_iwslt2017.py @@ -155,7 +155,9 @@ def tearDownClass(cls): def test_iwslt2017(self, split, src, tgt): with tempfile.TemporaryDirectory() as root_dir: - expected_samples = _get_mock_dataset(root_dir, split, src, tgt, "dev2010", "tst2010") + expected_samples = _get_mock_dataset( + os.path.join(root_dir, "datasets"), split, src, tgt, "dev2010", "tst2010" + ) dataset = IWSLT2017(root=root_dir, split=split, language_pair=(src, tgt)) @@ -170,7 +172,9 @@ def test_iwslt2017_split_argument(self, split): language_pair = ("de", "en") valid_set = "dev2010" test_set = "tst2010" - _ = _get_mock_dataset(root_dir, split, language_pair[0], language_pair[1], valid_set, test_set) + _ = _get_mock_dataset( + os.path.join(root_dir, "datasets"), split, language_pair[0], language_pair[1], valid_set, test_set + ) dataset1 = IWSLT2017(root=root_dir, split=split, language_pair=language_pair) (dataset2,) = IWSLT2017(root=root_dir, split=(split,), language_pair=language_pair) diff --git a/test/datasets/test_mnli.py b/test/datasets/test_mnli.py index fed6fca241..80549735e1 100644 --- a/test/datasets/test_mnli.py +++ b/test/datasets/test_mnli.py @@ -56,7 +56,7 @@ class TestMNLI(TempDirMixin, TorchtextTestCase): def setUpClass(cls): super().setUpClass() cls.root_dir = cls.get_base_temp_dir() - cls.samples = _get_mock_dataset(cls.root_dir) + cls.samples = _get_mock_dataset(os.path.join(cls.root_dir, "datasets")) cls.patcher = patch("torchdata.datapipes.iter.util.cacheholder._hash_check", return_value=True) cls.patcher.start() diff --git a/test/datasets/test_mrpc.py b/test/datasets/test_mrpc.py index a5ed910a4e..8a942d9fac 100644 --- a/test/datasets/test_mrpc.py +++ b/test/datasets/test_mrpc.py @@ -44,7 +44,7 @@ class TestMRPC(TempDirMixin, TorchtextTestCase): def setUpClass(cls): super().setUpClass() cls.root_dir = cls.get_base_temp_dir() - cls.samples = _get_mock_dataset(cls.root_dir) + cls.samples = _get_mock_dataset(os.path.join(cls.root_dir, "datasets")) cls.patcher = patch("torchdata.datapipes.iter.util.cacheholder._hash_check", return_value=True) cls.patcher.start() diff --git a/test/datasets/test_multi30k.py b/test/datasets/test_multi30k.py index 9613aca37a..e79c8a81c8 100644 --- a/test/datasets/test_multi30k.py +++ b/test/datasets/test_multi30k.py @@ -47,7 +47,7 @@ class TestMulti30k(TempDirMixin, TorchtextTestCase): def setUpClass(cls): super().setUpClass() cls.root_dir = cls.get_base_temp_dir() - cls.samples = _get_mock_dataset(cls.root_dir) + cls.samples = _get_mock_dataset(os.path.join(cls.root_dir, "datasets")) cls.patcher = patch("torchdata.datapipes.iter.util.cacheholder._hash_check", return_value=True) cls.patcher.start() diff --git a/test/datasets/test_penntreebank.py b/test/datasets/test_penntreebank.py index 6655459e84..eabfe3a108 100644 --- a/test/datasets/test_penntreebank.py +++ b/test/datasets/test_penntreebank.py @@ -41,7 +41,7 @@ class TestPennTreebank(TempDirMixin, TorchtextTestCase): def setUpClass(cls): super().setUpClass() cls.root_dir = cls.get_base_temp_dir() - cls.samples = _get_mock_dataset(cls.root_dir) + cls.samples = _get_mock_dataset(os.path.join(cls.root_dir, "datasets")) cls.patcher = patch("torchdata.datapipes.iter.util.cacheholder._hash_check", return_value=True) cls.patcher.start() diff --git a/test/datasets/test_qqp.py b/test/datasets/test_qqp.py index 1b11c8f767..58e06b019f 100644 --- a/test/datasets/test_qqp.py +++ b/test/datasets/test_qqp.py @@ -41,7 +41,7 @@ class TestQQP(TempDirMixin, TorchtextTestCase): def setUpClass(cls): super().setUpClass() cls.root_dir = cls.get_base_temp_dir() - cls.samples = _get_mock_dataset(cls.root_dir) + cls.samples = _get_mock_dataset(os.path.join(cls.root_dir, "datasets")) cls.patcher = patch("torchdata.datapipes.iter.util.cacheholder._hash_check", return_value=True) cls.patcher.start() diff --git a/test/datasets/test_sogounews.py b/test/datasets/test_sogounews.py index c60728fbc8..cd4d639b3e 100644 --- a/test/datasets/test_sogounews.py +++ b/test/datasets/test_sogounews.py @@ -48,7 +48,7 @@ class TestSogouNews(TempDirMixin, TorchtextTestCase): def setUpClass(cls): super().setUpClass() cls.root_dir = cls.get_base_temp_dir() - cls.samples = _get_mock_dataset(cls.root_dir) + cls.samples = _get_mock_dataset(os.path.join(cls.root_dir, "datasets")) cls.patcher = patch("torchdata.datapipes.iter.util.cacheholder._hash_check", return_value=True) cls.patcher.start() diff --git a/test/datasets/test_squads.py b/test/datasets/test_squads.py index 0519633a32..1738aa3e52 100644 --- a/test/datasets/test_squads.py +++ b/test/datasets/test_squads.py @@ -87,7 +87,7 @@ def tearDownClass(cls): @nested_params([SQuAD1, SQuAD2], ["train", "dev"]) def test_squads(self, squad_dataset, split): - expected_samples = _get_mock_dataset(self.root_dir, squad_dataset.__name__)[split] + expected_samples = _get_mock_dataset(os.path.join(self.root_dir, "datasets"), squad_dataset.__name__)[split] dataset = squad_dataset(root=self.root_dir, split=split) samples = list(dataset) diff --git a/test/datasets/test_sst2.py b/test/datasets/test_sst2.py index bb3c93a73a..996cee6d99 100644 --- a/test/datasets/test_sst2.py +++ b/test/datasets/test_sst2.py @@ -59,7 +59,7 @@ class TestSST2(TempDirMixin, TorchtextTestCase): def setUpClass(cls): super().setUpClass() cls.root_dir = cls.get_base_temp_dir() - cls.samples = _get_mock_dataset(cls.root_dir) + cls.samples = _get_mock_dataset(os.path.join(cls.root_dir, "datasets")) cls.patcher = patch("torchdata.datapipes.iter.util.cacheholder._hash_check", return_value=True) cls.patcher.start() diff --git a/test/datasets/test_stsb.py b/test/datasets/test_stsb.py index f74cd2b6d2..f2511b3ab8 100644 --- a/test/datasets/test_stsb.py +++ b/test/datasets/test_stsb.py @@ -62,7 +62,7 @@ class TestSTSB(TempDirMixin, TorchtextTestCase): def setUpClass(cls): super().setUpClass() cls.root_dir = cls.get_base_temp_dir() - cls.samples = _get_mock_dataset(cls.root_dir) + cls.samples = _get_mock_dataset(os.path.join(cls.root_dir, "datasets")) cls.patcher = patch("torchdata.datapipes.iter.util.cacheholder._hash_check", return_value=True) cls.patcher.start() diff --git a/test/datasets/test_udpos.py b/test/datasets/test_udpos.py index ae02e914a4..f2e7260f36 100644 --- a/test/datasets/test_udpos.py +++ b/test/datasets/test_udpos.py @@ -56,7 +56,7 @@ class TestUDPOS(TempDirMixin, TorchtextTestCase): def setUpClass(cls): super().setUpClass() cls.root_dir = cls.get_base_temp_dir() - cls.samples = _get_mock_dataset(cls.root_dir) + cls.samples = _get_mock_dataset(os.path.join(cls.root_dir, "datasets")) cls.patcher = patch("torchdata.datapipes.iter.util.cacheholder._hash_check", return_value=True) cls.patcher.start() diff --git a/test/datasets/test_wikitexts.py b/test/datasets/test_wikitexts.py index 4706a53093..ae70995fc0 100644 --- a/test/datasets/test_wikitexts.py +++ b/test/datasets/test_wikitexts.py @@ -71,7 +71,9 @@ def tearDownClass(cls): @nested_params([WikiText103, WikiText2], ["train", "valid", "test"]) def test_wikitexts(self, wikitext_dataset, split): - expected_samples = _get_mock_dataset(self.root_dir, base_dir_name=wikitext_dataset.__name__)[split] + expected_samples = _get_mock_dataset( + os.path.join(self.root_dir, "datasets"), base_dir_name=wikitext_dataset.__name__ + )[split] dataset = wikitext_dataset(root=self.root_dir, split=split) samples = list(dataset) diff --git a/test/datasets/test_yahooanswers.py b/test/datasets/test_yahooanswers.py index d3a37bb580..15e9bbc046 100644 --- a/test/datasets/test_yahooanswers.py +++ b/test/datasets/test_yahooanswers.py @@ -48,7 +48,7 @@ class TestYahooAnswers(TempDirMixin, TorchtextTestCase): def setUpClass(cls): super().setUpClass() cls.root_dir = cls.get_base_temp_dir() - cls.samples = _get_mock_dataset(cls.root_dir) + cls.samples = _get_mock_dataset(os.path.join(cls.root_dir, "datasets")) cls.patcher = patch("torchdata.datapipes.iter.util.cacheholder._hash_check", return_value=True) cls.patcher.start() diff --git a/test/datasets/test_yelpreviews.py b/test/datasets/test_yelpreviews.py index 93c335c4a7..f9f299a80b 100644 --- a/test/datasets/test_yelpreviews.py +++ b/test/datasets/test_yelpreviews.py @@ -70,7 +70,9 @@ def tearDownClass(cls): @nested_params([YelpReviewPolarity, YelpReviewFull], ["train", "test"]) def test_yelpreviews(self, yelp_dataset, split): - expected_samples = _get_mock_dataset(self.root_dir, base_dir_name=yelp_dataset.__name__)[split] + expected_samples = _get_mock_dataset( + os.path.join(self.root_dir, "datasets"), base_dir_name=yelp_dataset.__name__ + )[split] dataset = yelp_dataset(root=self.root_dir, split=split) samples = list(dataset)