From 8f8825c05307b83cf273442e3c6bea5b1d51d20a Mon Sep 17 00:00:00 2001 From: Virgile Mison Date: Fri, 28 Jan 2022 20:18:22 -0500 Subject: [PATCH 1/4] add test_conll200chunking.py to mock CoNLL data --- test/datasets/test_conll2000chunking.py | 88 +++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 test/datasets/test_conll2000chunking.py diff --git a/test/datasets/test_conll2000chunking.py b/test/datasets/test_conll2000chunking.py new file mode 100644 index 0000000000..aa62ccf8d3 --- /dev/null +++ b/test/datasets/test_conll2000chunking.py @@ -0,0 +1,88 @@ +import os +import random +import string +import gzip +from collections import defaultdict +from unittest.mock import patch + +from parameterized import parameterized +from torchtext.datasets.conll2000chunking import CoNLL2000Chunking + +from ..common.case_utils import TempDirMixin, zip_equal +from ..common.torchtext_test_case import TorchtextTestCase + +def _get_mock_dataset(root_dir): + """ + root_dir: directory to the mocked dataset + """ + base_dir = os.path.join(root_dir, "CoNLL2000Chunking") + temp_dataset_dir = os.path.join(base_dir, "conll2000chunking") + os.makedirs(temp_dataset_dir, exist_ok=True) + + seed = 1 + mocked_data = defaultdict(list) + for file_name in ["train.txt", "test.txt"]: + txt_file = os.path.join(temp_dataset_dir, file_name) + mocked_lines = mocked_data[os.path.splitext(file_name)[0]] + with open(txt_file, "w") as f: + for i in range(5): + label = seed % 2 + rand_strings = [random.choice(string.ascii_letters) for i in range(seed)] + rand_label_1 = [random.choice(string.ascii_letters) for i in range(seed)] + rand_label_2 = [random.choice(string.ascii_letters) for i in range(seed)] + # one token per line (each sample ends with an extra \n) + for rand_string, label_1, label_2 in zip(rand_strings, rand_label_1, rand_label_2): + f.write(f"{rand_string} {label_1} {label_2}\n") + f.write("\n") + dataset_line = (rand_strings, rand_label_1, rand_label_2) + # append line to correct dataset split + mocked_lines.append(dataset_line) + seed += 1 + + + # create gz file from dataset folder + for file_name in ("train.txt", "test.txt"): + compressed_dataset_path = os.path.join(temp_dataset_dir, f"{file_name}.gz") + with gzip.open(compressed_dataset_path, "wb") as gz_file: + txt_file = os.path.join(temp_dataset_dir, file_name) + with open(txt_file, "rb") as file_in: + gz_file.writelines(file_in) + + return mocked_data + + +class TestCoNLL2000Chunking(TempDirMixin, TorchtextTestCase): + root_dir = None + samples = [] + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.root_dir = cls.get_base_temp_dir() + cls.samples = _get_mock_dataset(cls.root_dir) + cls.patcher = patch( + "torchdata.datapipes.iter.util.cacheholder._hash_check", return_value=True + ) + cls.patcher.start() + print(cls.root_dir) + + @classmethod + def tearDownClass(cls): + cls.patcher.stop() + super().tearDownClass() + + @parameterized.expand(["train", "test"]) + def test_conll2000chunking(self, split): + dataset = CoNLL2000Chunking(root=self.root_dir, split=split) + samples = list(dataset) + expected_samples = self.samples[split] + for sample, expected_sample in zip_equal(samples, expected_samples): + self.assertEqual(sample, expected_sample) + + @parameterized.expand(["train", "test"]) + def test_conll2000chunking_split_argument(self, split): + dataset1 = CoNLL2000Chunking(root=self.root_dir, split=split) + (dataset2,) = CoNLL2000Chunking(root=self.root_dir, split=(split,)) + + for d1, d2 in zip_equal(dataset1, dataset2): + self.assertEqual(d1, d2) From b160e06d4501c761449e6dd2939aba0e054c6d2e Mon Sep 17 00:00:00 2001 From: VirgileHlav Date: Fri, 28 Jan 2022 21:37:06 -0500 Subject: [PATCH 2/4] formatting remove print + add second line --- test/datasets/test_conll2000chunking.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/datasets/test_conll2000chunking.py b/test/datasets/test_conll2000chunking.py index aa62ccf8d3..fd9ab2d4fa 100644 --- a/test/datasets/test_conll2000chunking.py +++ b/test/datasets/test_conll2000chunking.py @@ -11,6 +11,7 @@ from ..common.case_utils import TempDirMixin, zip_equal from ..common.torchtext_test_case import TorchtextTestCase + def _get_mock_dataset(root_dir): """ root_dir: directory to the mocked dataset @@ -64,7 +65,6 @@ def setUpClass(cls): "torchdata.datapipes.iter.util.cacheholder._hash_check", return_value=True ) cls.patcher.start() - print(cls.root_dir) @classmethod def tearDownClass(cls): From 5f57cdac85bc9ff293f89a11ba0bf88cbbf2ea11 Mon Sep 17 00:00:00 2001 From: Virgile Mison Date: Mon, 7 Feb 2022 10:36:34 -0500 Subject: [PATCH 3/4] PR comments change --- test/datasets/test_conll2000chunking.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/test/datasets/test_conll2000chunking.py b/test/datasets/test_conll2000chunking.py index fd9ab2d4fa..0382519b5f 100644 --- a/test/datasets/test_conll2000chunking.py +++ b/test/datasets/test_conll2000chunking.py @@ -22,7 +22,7 @@ def _get_mock_dataset(root_dir): seed = 1 mocked_data = defaultdict(list) - for file_name in ["train.txt", "test.txt"]: + for file_name in ("train.txt", "test.txt"): txt_file = os.path.join(temp_dataset_dir, file_name) mocked_lines = mocked_data[os.path.splitext(file_name)[0]] with open(txt_file, "w") as f: @@ -41,13 +41,10 @@ def _get_mock_dataset(root_dir): seed += 1 - # create gz file from dataset folder - for file_name in ("train.txt", "test.txt"): + # create gz file from dataset folder compressed_dataset_path = os.path.join(temp_dataset_dir, f"{file_name}.gz") - with gzip.open(compressed_dataset_path, "wb") as gz_file: - txt_file = os.path.join(temp_dataset_dir, file_name) - with open(txt_file, "rb") as file_in: - gz_file.writelines(file_in) + with gzip.open(compressed_dataset_path, "wb") as gz_file, open(txt_file, "rb") as file_in: + gz_file.writelines(file_in) return mocked_data From 83d98aa61a9f848fe18a28e0ca2bfa0c56244127 Mon Sep 17 00:00:00 2001 From: Virgile Mison Date: Tue, 8 Feb 2022 13:57:26 -0500 Subject: [PATCH 4/4] update directory to temp --- test/datasets/test_conll2000chunking.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/datasets/test_conll2000chunking.py b/test/datasets/test_conll2000chunking.py index 0382519b5f..ce7de36079 100644 --- a/test/datasets/test_conll2000chunking.py +++ b/test/datasets/test_conll2000chunking.py @@ -17,7 +17,7 @@ def _get_mock_dataset(root_dir): root_dir: directory to the mocked dataset """ base_dir = os.path.join(root_dir, "CoNLL2000Chunking") - temp_dataset_dir = os.path.join(base_dir, "conll2000chunking") + temp_dataset_dir = os.path.join(base_dir, "temp_dataset_dir") os.makedirs(temp_dataset_dir, exist_ok=True) seed = 1