From d569d354c0783be66d66db3c9504735296281a11 Mon Sep 17 00:00:00 2001 From: Arnav Mehta <65492948+arnavmehta7@users.noreply.github.com> Date: Thu, 15 Sep 2022 19:17:12 +0530 Subject: [PATCH 01/14] todo-decorator-remove-solved --- test/prototype/test_functional.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/prototype/test_functional.py b/test/prototype/test_functional.py index ef4f096277..64a0f2db81 100644 --- a/test/prototype/test_functional.py +++ b/test/prototype/test_functional.py @@ -10,8 +10,6 @@ class TestFunctional(TorchtextTestCase): - # TODO(Nayef211): remove decorator once https://github.com/pytorch/pytorch/issues/38207 is closed - @unittest.skipIf(platform.system() == "Windows", "Test is known to fail on Windows.") def test_BasicEnglishNormalize(self) -> None: test_sample = "'\".
,()!?;: Basic English Normalization for a Line of Text '\".
,()!?;:" ref_results = [ From d287d92db35b1436e27760caf7275b2c99e7545e Mon Sep 17 00:00:00 2001 From: Arnav Mehta <65492948+arnavmehta7@users.noreply.github.com> Date: Thu, 15 Sep 2022 19:21:47 +0530 Subject: [PATCH 02/14] todo-update-toOriginalServer-resolved --- torchtext/datasets/multi30k.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/torchtext/datasets/multi30k.py b/torchtext/datasets/multi30k.py index 47f5fb3a33..07bea245ae 100644 --- a/torchtext/datasets/multi30k.py +++ b/torchtext/datasets/multi30k.py @@ -12,11 +12,10 @@ from torchdata.datapipes.iter import FileOpener, IterableWrapper from torchtext._download_hooks import HttpReader -# TODO: Update URL to original once the server is back up (see https://github.com/pytorch/text/issues/1756) URL = { - "train": r"https://raw.githubusercontent.com/neychev/small_DL_repo/master/datasets/Multi30k/training.tar.gz", - "valid": r"https://raw.githubusercontent.com/neychev/small_DL_repo/master/datasets/Multi30k/validation.tar.gz", - "test": r"https://raw.githubusercontent.com/neychev/small_DL_repo/master/datasets/Multi30k/mmt16_task1_test.tar.gz", + "train": r"http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/training.tar.gz", + "valid": r"http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/validation.tar.gz", + "test": r"http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/mmt16_task1_test.tar.gz", } MD5 = { From 6c1708f1e0a8d153ff9b420e459313fa3038b81f Mon Sep 17 00:00:00 2001 From: Nayef Ahmed <22487263+Nayef211@users.noreply.github.com> Date: Thu, 15 Sep 2022 17:41:36 -0400 Subject: [PATCH 03/14] Turn off mask checking for torchtext which is known to have a legal mask (#1896) (#1906) Summary: Pull Request resolved: https://github.com/pytorch/text/pull/1896 Turn off mask checking for torchtext which is known to have a legal mask Reviewed By: zrphercule Differential Revision: D39445703 fbshipit-source-id: 3f0cacfd39ea11a16c7a06f339872554333b5e97 Co-authored-by: Michael Gschwind --- torchtext/models/roberta/modules.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/torchtext/models/roberta/modules.py b/torchtext/models/roberta/modules.py index 590502e77b..cd76b97909 100644 --- a/torchtext/models/roberta/modules.py +++ b/torchtext/models/roberta/modules.py @@ -120,7 +120,12 @@ def __init__( batch_first=True, norm_first=normalize_before, ) - self.layers = torch.nn.TransformerEncoder(encoder_layer=layer, num_layers=num_encoder_layers) + self.layers = torch.nn.TransformerEncoder( + encoder_layer=layer, + num_layers=num_encoder_layers, + enable_nested_tensor=True, + mask_check=False, + ) self.positional_embedding = PositionalEmbedding(max_seq_len, embedding_dim, padding_idx) self.embedding_layer_norm = nn.LayerNorm(embedding_dim) self.dropout = nn.Dropout(dropout) From 94c53cb4b0134f45dcd7b55f78147469ad004998 Mon Sep 17 00:00:00 2001 From: Nayef Ahmed <22487263+Nayef211@users.noreply.github.com> Date: Thu, 15 Sep 2022 20:15:59 -0400 Subject: [PATCH 04/14] Update doc theme to the latest (#1899) * Delete experimental_datasets_raw.rst - Deleting outdated docs on experimental datasets * Update sphinx theme to latest --- docs/requirements.txt | 2 +- docs/source/experimental_datasets_raw.rst | 46 ----------------------- 2 files changed, 1 insertion(+), 47 deletions(-) delete mode 100644 docs/source/experimental_datasets_raw.rst diff --git a/docs/requirements.txt b/docs/requirements.txt index 82de9e49f4..3f57086887 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,5 +1,5 @@ Jinja2<3.1.0 sphinx==3.5.4 --e git+https://github.com/pytorch/pytorch_sphinx_theme.git@b4d0005#egg=pytorch_sphinx_theme +-e git+https://github.com/pytorch/pytorch_sphinx_theme.git@cece053#egg=pytorch_sphinx_theme matplotlib sphinx_gallery diff --git a/docs/source/experimental_datasets_raw.rst b/docs/source/experimental_datasets_raw.rst deleted file mode 100644 index 1b93a1746f..0000000000 --- a/docs/source/experimental_datasets_raw.rst +++ /dev/null @@ -1,46 +0,0 @@ -torchtext.experimental.datasets.raw -=================================== - -.. currentmodule:: torchtext.experimental.datasets.raw - -General use cases are as follows: :: - - - # import datasets - from torchtext.experimental.datasets.raw import Multi30k - - train_iter = Multi30k(split='train') - - def tokenize(label, line): - return line.split() - - tokens_src = [] - tokens_tgt = [] - - for line in train_iter: - src, tgt = line - tokens_src += tokenize(src) - tokens_tgt += tokenize(tgt) - -The following datasets are available: - -.. contents:: Datasets - :local: - - -Machine Translation -^^^^^^^^^^^^^^^^^^^ - -WMT14 -~~~~~ - -.. autofunction:: WMT14 - - -Language Modeling -^^^^^^^^^^^^^^^^^ - -WMTNewsCrawl -~~~~~~~~~~~~ - -.. autofunction:: WMTNewsCrawl From 67d26928d2bba65a22666ae6b2e3bd3fdaee8c7d Mon Sep 17 00:00:00 2001 From: Nayef Ahmed <22487263+Nayef211@users.noreply.github.com> Date: Thu, 15 Sep 2022 20:19:09 -0400 Subject: [PATCH 05/14] Fix test execution in torchtext (#1889) * Delete checkpoints after integration test * Use pytest fixtures to autodelete model assets in integration tests * Created new unittest directory and new workflow for integration tests * Fix imports * Readded ignored assets * Remove changes from download_hooks * change to install instead of develop * Using setup.py develop on unittest and integration tests * set cxx abi flag to 1 * remove symbol prints * set cxx_abi flag * Add expecttest dep * Finalize integration test workflow * Remove cd to root dir * Add helper function to pass correct D_GLIBCXX_USE_CXX11_ABI value from pytorch * Added cxx_abi flag to cmake_cxx_flags * Resolving PR comments * Fix call to compiled_with_cxx11_abi fn * Added new cache variable to store D_GLIBCXX_USE_CXX11_ABI flag value --- .circleci/unittest/linux/scripts/run_test.sh | 3 +- .../unittest/windows/scripts/run_test.sh | 3 +- .github/workflows/integration-test.yml | 31 ++++++++++++++++++ CMakeLists.txt | 4 +-- test/.gitignore | 0 test/integration_tests/conftest.py | 4 +-- .../prototype}/test_models.py | 6 ++-- test/integration_tests/test_models.py | 10 ++++-- test/prototype/models/__init__.py | 0 test/{ => torchtext_unittest}/__init__.py | 0 .../asset/SST2/SST-2.zip | Bin .../asset/bert_base_cased_vocab.txt | 0 .../asset/bert_base_uncased_vocab.txt | 0 .../asset/clip_encoder.json | 0 .../asset/clip_vocab.bpe | 0 .../asset/glove.6B.zip | Bin .../asset/glove.840B.300d.zip | Bin .../asset/gpt2_bpe_encoder.json | 0 .../asset/gpt2_bpe_vocab.bpe | 0 .../asset/label_names.txt | 0 .../asset/raw_datasets.jsonl | 0 .../asset/roberta.base.output.pt | Bin .../asset/roberta.large.output.pt | Bin .../asset/spm_example.model | Bin .../asset/t5.base.encoder.output.pt | Bin .../asset/t5.base.generation.output.pt | Bin .../asset/t5.base.model.output.pt | Bin .../asset/t5.large.encoder.output.pt | Bin .../asset/t5.large.generation.output.pt | Bin .../asset/t5.large.model.output.pt | Bin .../asset/t5.small.encoder.output.pt | Bin .../asset/t5.small.generation.output.pt | Bin .../asset/t5.small.model.output.pt | Bin .../asset/t5_tokenizer_base.model | Bin ...ext_normalization_ag_news_ref_results.test | 0 .../asset/text_normalization_ag_news_test.csv | 0 .../asset/vectors_test.csv | 0 .../asset/vocab_raw_text_test.txt | 0 .../asset/vocab_test.txt | 0 .../asset/vocab_test2.txt | 0 .../asset/wiki.en.vec | 0 .../asset/xlmr.base.output.pt | Bin .../asset/xlmr.large.output.pt | Bin .../common/__init__.py | 0 .../{ => torchtext_unittest}/common/assets.py | 0 .../common/case_utils.py | 0 .../common/parameterized_utils.py | 0 .../common/torchtext_test_case.py | 0 .../{ => torchtext_unittest}/csrc/__init__.py | 0 .../csrc/test_gpt2_bpe_tokenizer.py | 0 .../{ => torchtext_unittest}/data/__init__.py | 0 .../data/test_dataset_utils.py | 0 .../data/test_functional.py | 0 .../{ => torchtext_unittest}/data/test_jit.py | 0 .../data/test_metrics.py | 0 .../data/test_modules.py | 0 .../data/test_utils.py | 0 .../datasets/__init__.py | 0 .../datasets/common.py | 0 .../datasets/test_agnews.py | 0 .../datasets/test_amazonreviews.py | 0 .../datasets/test_cc100.py | 0 .../datasets/test_cnndm.py | 0 .../datasets/test_cola.py | 0 .../datasets/test_conll2000chunking.py | 0 .../datasets/test_dbpedia.py | 0 .../datasets/test_enwik9.py | 0 .../datasets/test_imdb.py | 0 .../datasets/test_iwslt2016.py | 0 .../datasets/test_iwslt2017.py | 0 .../datasets/test_mnli.py | 0 .../datasets/test_mrpc.py | 0 .../datasets/test_multi30k.py | 0 .../datasets/test_penntreebank.py | 0 .../datasets/test_qnli.py | 0 .../datasets/test_qqp.py | 0 .../datasets/test_rte.py | 0 .../datasets/test_sogounews.py | 0 .../datasets/test_squads.py | 0 .../datasets/test_sst2.py | 0 .../datasets/test_stsb.py | 0 .../datasets/test_udpos.py | 0 .../datasets/test_wikitexts.py | 0 .../datasets/test_wnli.py | 0 .../datasets/test_yahooanswers.py | 0 .../datasets/test_yelpreviews.py | 0 .../models/__init__.py | 0 .../models/test_models.py | 0 .../models/test_transformers.py | 0 .../prototype/__init__.py | 0 .../prototype/models}/__init__.py | 0 .../prototype/models/test_models.py | 2 +- .../prototype/models/test_transforms.py | 4 +-- .../prototype/test_functional.py | 0 .../prototype/test_transforms.py | 4 +-- .../prototype/test_vectors.py | 2 +- .../prototype/test_with_asset.py | 2 +- test/{ => torchtext_unittest}/test_build.py | 0 .../test_functional.py | 0 .../test_transforms.py | 0 test/{ => torchtext_unittest}/test_utils.py | 2 +- test/{ => torchtext_unittest}/test_vocab.py | 2 +- tools/setup_helpers/extension.py | 5 +++ 103 files changed, 64 insertions(+), 20 deletions(-) create mode 100644 .github/workflows/integration-test.yml delete mode 100644 test/.gitignore rename test/{prototype/integration_tests => integration_tests/prototype}/test_models.py (95%) delete mode 100644 test/prototype/models/__init__.py rename test/{ => torchtext_unittest}/__init__.py (100%) rename test/{ => torchtext_unittest}/asset/SST2/SST-2.zip (100%) rename test/{ => torchtext_unittest}/asset/bert_base_cased_vocab.txt (100%) rename test/{ => torchtext_unittest}/asset/bert_base_uncased_vocab.txt (100%) rename test/{ => torchtext_unittest}/asset/clip_encoder.json (100%) rename test/{ => torchtext_unittest}/asset/clip_vocab.bpe (100%) rename test/{ => torchtext_unittest}/asset/glove.6B.zip (100%) rename test/{ => torchtext_unittest}/asset/glove.840B.300d.zip (100%) rename test/{ => torchtext_unittest}/asset/gpt2_bpe_encoder.json (100%) rename test/{ => torchtext_unittest}/asset/gpt2_bpe_vocab.bpe (100%) rename test/{ => torchtext_unittest}/asset/label_names.txt (100%) rename test/{ => torchtext_unittest}/asset/raw_datasets.jsonl (100%) rename test/{ => torchtext_unittest}/asset/roberta.base.output.pt (100%) rename test/{ => torchtext_unittest}/asset/roberta.large.output.pt (100%) rename test/{ => torchtext_unittest}/asset/spm_example.model (100%) rename test/{ => torchtext_unittest}/asset/t5.base.encoder.output.pt (100%) rename test/{ => torchtext_unittest}/asset/t5.base.generation.output.pt (100%) rename test/{ => torchtext_unittest}/asset/t5.base.model.output.pt (100%) rename test/{ => torchtext_unittest}/asset/t5.large.encoder.output.pt (100%) rename test/{ => torchtext_unittest}/asset/t5.large.generation.output.pt (100%) rename test/{ => torchtext_unittest}/asset/t5.large.model.output.pt (100%) rename test/{ => torchtext_unittest}/asset/t5.small.encoder.output.pt (100%) rename test/{ => torchtext_unittest}/asset/t5.small.generation.output.pt (100%) rename test/{ => torchtext_unittest}/asset/t5.small.model.output.pt (100%) rename test/{ => torchtext_unittest}/asset/t5_tokenizer_base.model (100%) rename test/{ => torchtext_unittest}/asset/text_normalization_ag_news_ref_results.test (100%) rename test/{ => torchtext_unittest}/asset/text_normalization_ag_news_test.csv (100%) rename test/{ => torchtext_unittest}/asset/vectors_test.csv (100%) rename test/{ => torchtext_unittest}/asset/vocab_raw_text_test.txt (100%) rename test/{ => torchtext_unittest}/asset/vocab_test.txt (100%) rename test/{ => torchtext_unittest}/asset/vocab_test2.txt (100%) rename test/{ => torchtext_unittest}/asset/wiki.en.vec (100%) rename test/{ => torchtext_unittest}/asset/xlmr.base.output.pt (100%) rename test/{ => torchtext_unittest}/asset/xlmr.large.output.pt (100%) rename test/{ => torchtext_unittest}/common/__init__.py (100%) rename test/{ => torchtext_unittest}/common/assets.py (100%) rename test/{ => torchtext_unittest}/common/case_utils.py (100%) rename test/{ => torchtext_unittest}/common/parameterized_utils.py (100%) rename test/{ => torchtext_unittest}/common/torchtext_test_case.py (100%) rename test/{ => torchtext_unittest}/csrc/__init__.py (100%) rename test/{ => torchtext_unittest}/csrc/test_gpt2_bpe_tokenizer.py (100%) rename test/{ => torchtext_unittest}/data/__init__.py (100%) rename test/{ => torchtext_unittest}/data/test_dataset_utils.py (100%) rename test/{ => torchtext_unittest}/data/test_functional.py (100%) rename test/{ => torchtext_unittest}/data/test_jit.py (100%) rename test/{ => torchtext_unittest}/data/test_metrics.py (100%) rename test/{ => torchtext_unittest}/data/test_modules.py (100%) rename test/{ => torchtext_unittest}/data/test_utils.py (100%) rename test/{ => torchtext_unittest}/datasets/__init__.py (100%) rename test/{ => torchtext_unittest}/datasets/common.py (100%) rename test/{ => torchtext_unittest}/datasets/test_agnews.py (100%) rename test/{ => torchtext_unittest}/datasets/test_amazonreviews.py (100%) rename test/{ => torchtext_unittest}/datasets/test_cc100.py (100%) rename test/{ => torchtext_unittest}/datasets/test_cnndm.py (100%) rename test/{ => torchtext_unittest}/datasets/test_cola.py (100%) rename test/{ => torchtext_unittest}/datasets/test_conll2000chunking.py (100%) rename test/{ => torchtext_unittest}/datasets/test_dbpedia.py (100%) rename test/{ => torchtext_unittest}/datasets/test_enwik9.py (100%) rename test/{ => torchtext_unittest}/datasets/test_imdb.py (100%) rename test/{ => torchtext_unittest}/datasets/test_iwslt2016.py (100%) rename test/{ => torchtext_unittest}/datasets/test_iwslt2017.py (100%) rename test/{ => torchtext_unittest}/datasets/test_mnli.py (100%) rename test/{ => torchtext_unittest}/datasets/test_mrpc.py (100%) rename test/{ => torchtext_unittest}/datasets/test_multi30k.py (100%) rename test/{ => torchtext_unittest}/datasets/test_penntreebank.py (100%) rename test/{ => torchtext_unittest}/datasets/test_qnli.py (100%) rename test/{ => torchtext_unittest}/datasets/test_qqp.py (100%) rename test/{ => torchtext_unittest}/datasets/test_rte.py (100%) rename test/{ => torchtext_unittest}/datasets/test_sogounews.py (100%) rename test/{ => torchtext_unittest}/datasets/test_squads.py (100%) rename test/{ => torchtext_unittest}/datasets/test_sst2.py (100%) rename test/{ => torchtext_unittest}/datasets/test_stsb.py (100%) rename test/{ => torchtext_unittest}/datasets/test_udpos.py (100%) rename test/{ => torchtext_unittest}/datasets/test_wikitexts.py (100%) rename test/{ => torchtext_unittest}/datasets/test_wnli.py (100%) rename test/{ => torchtext_unittest}/datasets/test_yahooanswers.py (100%) rename test/{ => torchtext_unittest}/datasets/test_yelpreviews.py (100%) rename test/{ => torchtext_unittest}/models/__init__.py (100%) rename test/{ => torchtext_unittest}/models/test_models.py (100%) rename test/{ => torchtext_unittest}/models/test_transformers.py (100%) rename test/{ => torchtext_unittest}/prototype/__init__.py (100%) rename test/{prototype/integration_tests => torchtext_unittest/prototype/models}/__init__.py (100%) rename test/{ => torchtext_unittest}/prototype/models/test_models.py (98%) rename test/{ => torchtext_unittest}/prototype/models/test_transforms.py (93%) rename test/{ => torchtext_unittest}/prototype/test_functional.py (100%) rename test/{ => torchtext_unittest}/prototype/test_transforms.py (97%) rename test/{ => torchtext_unittest}/prototype/test_vectors.py (98%) rename test/{ => torchtext_unittest}/prototype/test_with_asset.py (99%) rename test/{ => torchtext_unittest}/test_build.py (100%) rename test/{ => torchtext_unittest}/test_functional.py (100%) rename test/{ => torchtext_unittest}/test_transforms.py (100%) rename test/{ => torchtext_unittest}/test_utils.py (98%) rename test/{ => torchtext_unittest}/test_vocab.py (99%) diff --git a/.circleci/unittest/linux/scripts/run_test.sh b/.circleci/unittest/linux/scripts/run_test.sh index c8322ea5f9..3b44c3af62 100755 --- a/.circleci/unittest/linux/scripts/run_test.sh +++ b/.circleci/unittest/linux/scripts/run_test.sh @@ -6,4 +6,5 @@ eval "$(./conda/bin/conda shell.bash hook)" conda activate ./env python -m torch.utils.collect_env -pytest --cov=torchtext --junitxml=test-results/junit.xml -v --durations 20 test +cd test +pytest --cov=torchtext --junitxml=test-results/junit.xml -v --durations 20 torchtext_unittest diff --git a/.circleci/unittest/windows/scripts/run_test.sh b/.circleci/unittest/windows/scripts/run_test.sh index 909177e2d4..b8a62f2c56 100644 --- a/.circleci/unittest/windows/scripts/run_test.sh +++ b/.circleci/unittest/windows/scripts/run_test.sh @@ -6,4 +6,5 @@ eval "$(./conda/Scripts/conda.exe 'shell.bash' 'hook')" conda activate ./env python -m torch.utils.collect_env -pytest --cov=torchtext --junitxml=test-results/junit.xml -v --durations 20 test +cd test +pytest --cov=torchtext --junitxml=test-results/junit.xml -v --durations 20 torchtext_unittest diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml new file mode 100644 index 0000000000..0c005a12a1 --- /dev/null +++ b/.github/workflows/integration-test.yml @@ -0,0 +1,31 @@ +name: Integration Test + +on: + pull_request: + branches: [main] + + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-18.04 + strategy: + fail-fast: false + matrix: + python-version: [3.8] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install packages + run: | + python -m pip install --quiet --upgrade pip + python -m pip install --quiet --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html + python -m pip install --quiet pytest requests cmake ninja sentencepiece parameterized tqdm expecttest + python setup.py install + - name: Run integration test + run: | + cd test && pytest integration_tests -v --use-tmp-hub-dir diff --git a/CMakeLists.txt b/CMakeLists.txt index 1ead15d46f..ed39e644a3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -51,6 +51,7 @@ option(BUILD_TORCHTEXT_PYTHON_EXTENSION "Build Python extension" OFF) set(CMAKE_MODULE_PATH "${CMAKE_MODULE_PATH};${CMAKE_CURRENT_SOURCE_DIR}/cmake") set(TORCH_INSTALL_PREFIX "${CMAKE_PREFIX_PATH}/../.." CACHE STRING "Install path for torch") +set(TORCH_COMPILED_WITH_CXX_ABI "-D_GLIBCXX_USE_CXX11_ABI=0" CACHE STRING "Compile torchtext with cxx11_abi") find_library(TORCH_C10_LIBRARY c10 PATHS "${TORCH_INSTALL_PREFIX}/lib") find_library(TORCH_LIBRARY torch PATHS "${TORCH_INSTALL_PREFIX}/lib") @@ -60,8 +61,7 @@ if(MSVC) set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>") endif() -# TORCH_CXX_FLAGS contains the same -D_GLIBCXX_USE_CXX11_ABI value as PyTorch -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall ${TORCH_CXX_FLAGS}") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${TORCH_COMPILED_WITH_CXX_ABI} -Wall ${TORCH_CXX_FLAGS}") add_subdirectory(third_party) add_subdirectory(torchtext/csrc) diff --git a/test/.gitignore b/test/.gitignore deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/integration_tests/conftest.py b/test/integration_tests/conftest.py index 6d051420ee..eff4bc0599 100644 --- a/test/integration_tests/conftest.py +++ b/test/integration_tests/conftest.py @@ -15,9 +15,9 @@ def pytest_addoption(parser): ) -@pytest.fixture(scope="class") +@pytest.fixture(autouse=True, scope="class") def temp_hub_dir(tmp_path_factory, pytestconfig): - if not pytestconfig.getoption("--use-tmp-hub-dir"): + if not pytestconfig.getoption("use_tmp_hub_dir"): yield else: tmp_dir = tmp_path_factory.mktemp("hub", numbered=True).resolve() diff --git a/test/prototype/integration_tests/test_models.py b/test/integration_tests/prototype/test_models.py similarity index 95% rename from test/prototype/integration_tests/test_models.py rename to test/integration_tests/prototype/test_models.py index 4130d67aa4..7743807031 100644 --- a/test/prototype/integration_tests/test_models.py +++ b/test/integration_tests/prototype/test_models.py @@ -1,9 +1,6 @@ import pytest # noqa: F401 import torch from parameterized import parameterized, parameterized_class -from test.common.assets import get_asset_path -from test.common.parameterized_utils import nested_params -from test.common.torchtext_test_case import TorchtextTestCase from torchtext.prototype.models import ( T5_BASE_ENCODER, T5_BASE, @@ -18,6 +15,9 @@ T5Transform, ) from torchtext.prototype.models.t5.wrapper import T5Wrapper +from torchtext_unittest.common.assets import get_asset_path +from torchtext_unittest.common.parameterized_utils import nested_params +from torchtext_unittest.common.torchtext_test_case import TorchtextTestCase BUNDLERS = { diff --git a/test/integration_tests/test_models.py b/test/integration_tests/test_models.py index d140c8190a..03422ea691 100644 --- a/test/integration_tests/test_models.py +++ b/test/integration_tests/test_models.py @@ -7,9 +7,15 @@ XLMR_BASE_ENCODER, XLMR_LARGE_ENCODER, ) +from torchtext_unittest.common.assets import get_asset_path +from torchtext_unittest.common.torchtext_test_case import TorchtextTestCase -from ..common.assets import get_asset_path -from ..common.torchtext_test_case import TorchtextTestCase +BUNDLERS = { + "xlmr_base": XLMR_BASE_ENCODER, + "xlmr_large": XLMR_LARGE_ENCODER, + "roberta_base": ROBERTA_BASE_ENCODER, + "roberta_large": ROBERTA_LARGE_ENCODER, +} BUNDLERS = { "xlmr_base": XLMR_BASE_ENCODER, diff --git a/test/prototype/models/__init__.py b/test/prototype/models/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/__init__.py b/test/torchtext_unittest/__init__.py similarity index 100% rename from test/__init__.py rename to test/torchtext_unittest/__init__.py diff --git a/test/asset/SST2/SST-2.zip b/test/torchtext_unittest/asset/SST2/SST-2.zip similarity index 100% rename from test/asset/SST2/SST-2.zip rename to test/torchtext_unittest/asset/SST2/SST-2.zip diff --git a/test/asset/bert_base_cased_vocab.txt b/test/torchtext_unittest/asset/bert_base_cased_vocab.txt similarity index 100% rename from test/asset/bert_base_cased_vocab.txt rename to test/torchtext_unittest/asset/bert_base_cased_vocab.txt diff --git a/test/asset/bert_base_uncased_vocab.txt b/test/torchtext_unittest/asset/bert_base_uncased_vocab.txt similarity index 100% rename from test/asset/bert_base_uncased_vocab.txt rename to test/torchtext_unittest/asset/bert_base_uncased_vocab.txt diff --git a/test/asset/clip_encoder.json b/test/torchtext_unittest/asset/clip_encoder.json similarity index 100% rename from test/asset/clip_encoder.json rename to test/torchtext_unittest/asset/clip_encoder.json diff --git a/test/asset/clip_vocab.bpe b/test/torchtext_unittest/asset/clip_vocab.bpe similarity index 100% rename from test/asset/clip_vocab.bpe rename to test/torchtext_unittest/asset/clip_vocab.bpe diff --git a/test/asset/glove.6B.zip b/test/torchtext_unittest/asset/glove.6B.zip similarity index 100% rename from test/asset/glove.6B.zip rename to test/torchtext_unittest/asset/glove.6B.zip diff --git a/test/asset/glove.840B.300d.zip b/test/torchtext_unittest/asset/glove.840B.300d.zip similarity index 100% rename from test/asset/glove.840B.300d.zip rename to test/torchtext_unittest/asset/glove.840B.300d.zip diff --git a/test/asset/gpt2_bpe_encoder.json b/test/torchtext_unittest/asset/gpt2_bpe_encoder.json similarity index 100% rename from test/asset/gpt2_bpe_encoder.json rename to test/torchtext_unittest/asset/gpt2_bpe_encoder.json diff --git a/test/asset/gpt2_bpe_vocab.bpe b/test/torchtext_unittest/asset/gpt2_bpe_vocab.bpe similarity index 100% rename from test/asset/gpt2_bpe_vocab.bpe rename to test/torchtext_unittest/asset/gpt2_bpe_vocab.bpe diff --git a/test/asset/label_names.txt b/test/torchtext_unittest/asset/label_names.txt similarity index 100% rename from test/asset/label_names.txt rename to test/torchtext_unittest/asset/label_names.txt diff --git a/test/asset/raw_datasets.jsonl b/test/torchtext_unittest/asset/raw_datasets.jsonl similarity index 100% rename from test/asset/raw_datasets.jsonl rename to test/torchtext_unittest/asset/raw_datasets.jsonl diff --git a/test/asset/roberta.base.output.pt b/test/torchtext_unittest/asset/roberta.base.output.pt similarity index 100% rename from test/asset/roberta.base.output.pt rename to test/torchtext_unittest/asset/roberta.base.output.pt diff --git a/test/asset/roberta.large.output.pt b/test/torchtext_unittest/asset/roberta.large.output.pt similarity index 100% rename from test/asset/roberta.large.output.pt rename to test/torchtext_unittest/asset/roberta.large.output.pt diff --git a/test/asset/spm_example.model b/test/torchtext_unittest/asset/spm_example.model similarity index 100% rename from test/asset/spm_example.model rename to test/torchtext_unittest/asset/spm_example.model diff --git a/test/asset/t5.base.encoder.output.pt b/test/torchtext_unittest/asset/t5.base.encoder.output.pt similarity index 100% rename from test/asset/t5.base.encoder.output.pt rename to test/torchtext_unittest/asset/t5.base.encoder.output.pt diff --git a/test/asset/t5.base.generation.output.pt b/test/torchtext_unittest/asset/t5.base.generation.output.pt similarity index 100% rename from test/asset/t5.base.generation.output.pt rename to test/torchtext_unittest/asset/t5.base.generation.output.pt diff --git a/test/asset/t5.base.model.output.pt b/test/torchtext_unittest/asset/t5.base.model.output.pt similarity index 100% rename from test/asset/t5.base.model.output.pt rename to test/torchtext_unittest/asset/t5.base.model.output.pt diff --git a/test/asset/t5.large.encoder.output.pt b/test/torchtext_unittest/asset/t5.large.encoder.output.pt similarity index 100% rename from test/asset/t5.large.encoder.output.pt rename to test/torchtext_unittest/asset/t5.large.encoder.output.pt diff --git a/test/asset/t5.large.generation.output.pt b/test/torchtext_unittest/asset/t5.large.generation.output.pt similarity index 100% rename from test/asset/t5.large.generation.output.pt rename to test/torchtext_unittest/asset/t5.large.generation.output.pt diff --git a/test/asset/t5.large.model.output.pt b/test/torchtext_unittest/asset/t5.large.model.output.pt similarity index 100% rename from test/asset/t5.large.model.output.pt rename to test/torchtext_unittest/asset/t5.large.model.output.pt diff --git a/test/asset/t5.small.encoder.output.pt b/test/torchtext_unittest/asset/t5.small.encoder.output.pt similarity index 100% rename from test/asset/t5.small.encoder.output.pt rename to test/torchtext_unittest/asset/t5.small.encoder.output.pt diff --git a/test/asset/t5.small.generation.output.pt b/test/torchtext_unittest/asset/t5.small.generation.output.pt similarity index 100% rename from test/asset/t5.small.generation.output.pt rename to test/torchtext_unittest/asset/t5.small.generation.output.pt diff --git a/test/asset/t5.small.model.output.pt b/test/torchtext_unittest/asset/t5.small.model.output.pt similarity index 100% rename from test/asset/t5.small.model.output.pt rename to test/torchtext_unittest/asset/t5.small.model.output.pt diff --git a/test/asset/t5_tokenizer_base.model b/test/torchtext_unittest/asset/t5_tokenizer_base.model similarity index 100% rename from test/asset/t5_tokenizer_base.model rename to test/torchtext_unittest/asset/t5_tokenizer_base.model diff --git a/test/asset/text_normalization_ag_news_ref_results.test b/test/torchtext_unittest/asset/text_normalization_ag_news_ref_results.test similarity index 100% rename from test/asset/text_normalization_ag_news_ref_results.test rename to test/torchtext_unittest/asset/text_normalization_ag_news_ref_results.test diff --git a/test/asset/text_normalization_ag_news_test.csv b/test/torchtext_unittest/asset/text_normalization_ag_news_test.csv similarity index 100% rename from test/asset/text_normalization_ag_news_test.csv rename to test/torchtext_unittest/asset/text_normalization_ag_news_test.csv diff --git a/test/asset/vectors_test.csv b/test/torchtext_unittest/asset/vectors_test.csv similarity index 100% rename from test/asset/vectors_test.csv rename to test/torchtext_unittest/asset/vectors_test.csv diff --git a/test/asset/vocab_raw_text_test.txt b/test/torchtext_unittest/asset/vocab_raw_text_test.txt similarity index 100% rename from test/asset/vocab_raw_text_test.txt rename to test/torchtext_unittest/asset/vocab_raw_text_test.txt diff --git a/test/asset/vocab_test.txt b/test/torchtext_unittest/asset/vocab_test.txt similarity index 100% rename from test/asset/vocab_test.txt rename to test/torchtext_unittest/asset/vocab_test.txt diff --git a/test/asset/vocab_test2.txt b/test/torchtext_unittest/asset/vocab_test2.txt similarity index 100% rename from test/asset/vocab_test2.txt rename to test/torchtext_unittest/asset/vocab_test2.txt diff --git a/test/asset/wiki.en.vec b/test/torchtext_unittest/asset/wiki.en.vec similarity index 100% rename from test/asset/wiki.en.vec rename to test/torchtext_unittest/asset/wiki.en.vec diff --git a/test/asset/xlmr.base.output.pt b/test/torchtext_unittest/asset/xlmr.base.output.pt similarity index 100% rename from test/asset/xlmr.base.output.pt rename to test/torchtext_unittest/asset/xlmr.base.output.pt diff --git a/test/asset/xlmr.large.output.pt b/test/torchtext_unittest/asset/xlmr.large.output.pt similarity index 100% rename from test/asset/xlmr.large.output.pt rename to test/torchtext_unittest/asset/xlmr.large.output.pt diff --git a/test/common/__init__.py b/test/torchtext_unittest/common/__init__.py similarity index 100% rename from test/common/__init__.py rename to test/torchtext_unittest/common/__init__.py diff --git a/test/common/assets.py b/test/torchtext_unittest/common/assets.py similarity index 100% rename from test/common/assets.py rename to test/torchtext_unittest/common/assets.py diff --git a/test/common/case_utils.py b/test/torchtext_unittest/common/case_utils.py similarity index 100% rename from test/common/case_utils.py rename to test/torchtext_unittest/common/case_utils.py diff --git a/test/common/parameterized_utils.py b/test/torchtext_unittest/common/parameterized_utils.py similarity index 100% rename from test/common/parameterized_utils.py rename to test/torchtext_unittest/common/parameterized_utils.py diff --git a/test/common/torchtext_test_case.py b/test/torchtext_unittest/common/torchtext_test_case.py similarity index 100% rename from test/common/torchtext_test_case.py rename to test/torchtext_unittest/common/torchtext_test_case.py diff --git a/test/csrc/__init__.py b/test/torchtext_unittest/csrc/__init__.py similarity index 100% rename from test/csrc/__init__.py rename to test/torchtext_unittest/csrc/__init__.py diff --git a/test/csrc/test_gpt2_bpe_tokenizer.py b/test/torchtext_unittest/csrc/test_gpt2_bpe_tokenizer.py similarity index 100% rename from test/csrc/test_gpt2_bpe_tokenizer.py rename to test/torchtext_unittest/csrc/test_gpt2_bpe_tokenizer.py diff --git a/test/data/__init__.py b/test/torchtext_unittest/data/__init__.py similarity index 100% rename from test/data/__init__.py rename to test/torchtext_unittest/data/__init__.py diff --git a/test/data/test_dataset_utils.py b/test/torchtext_unittest/data/test_dataset_utils.py similarity index 100% rename from test/data/test_dataset_utils.py rename to test/torchtext_unittest/data/test_dataset_utils.py diff --git a/test/data/test_functional.py b/test/torchtext_unittest/data/test_functional.py similarity index 100% rename from test/data/test_functional.py rename to test/torchtext_unittest/data/test_functional.py diff --git a/test/data/test_jit.py b/test/torchtext_unittest/data/test_jit.py similarity index 100% rename from test/data/test_jit.py rename to test/torchtext_unittest/data/test_jit.py diff --git a/test/data/test_metrics.py b/test/torchtext_unittest/data/test_metrics.py similarity index 100% rename from test/data/test_metrics.py rename to test/torchtext_unittest/data/test_metrics.py diff --git a/test/data/test_modules.py b/test/torchtext_unittest/data/test_modules.py similarity index 100% rename from test/data/test_modules.py rename to test/torchtext_unittest/data/test_modules.py diff --git a/test/data/test_utils.py b/test/torchtext_unittest/data/test_utils.py similarity index 100% rename from test/data/test_utils.py rename to test/torchtext_unittest/data/test_utils.py diff --git a/test/datasets/__init__.py b/test/torchtext_unittest/datasets/__init__.py similarity index 100% rename from test/datasets/__init__.py rename to test/torchtext_unittest/datasets/__init__.py diff --git a/test/datasets/common.py b/test/torchtext_unittest/datasets/common.py similarity index 100% rename from test/datasets/common.py rename to test/torchtext_unittest/datasets/common.py diff --git a/test/datasets/test_agnews.py b/test/torchtext_unittest/datasets/test_agnews.py similarity index 100% rename from test/datasets/test_agnews.py rename to test/torchtext_unittest/datasets/test_agnews.py diff --git a/test/datasets/test_amazonreviews.py b/test/torchtext_unittest/datasets/test_amazonreviews.py similarity index 100% rename from test/datasets/test_amazonreviews.py rename to test/torchtext_unittest/datasets/test_amazonreviews.py diff --git a/test/datasets/test_cc100.py b/test/torchtext_unittest/datasets/test_cc100.py similarity index 100% rename from test/datasets/test_cc100.py rename to test/torchtext_unittest/datasets/test_cc100.py diff --git a/test/datasets/test_cnndm.py b/test/torchtext_unittest/datasets/test_cnndm.py similarity index 100% rename from test/datasets/test_cnndm.py rename to test/torchtext_unittest/datasets/test_cnndm.py diff --git a/test/datasets/test_cola.py b/test/torchtext_unittest/datasets/test_cola.py similarity index 100% rename from test/datasets/test_cola.py rename to test/torchtext_unittest/datasets/test_cola.py diff --git a/test/datasets/test_conll2000chunking.py b/test/torchtext_unittest/datasets/test_conll2000chunking.py similarity index 100% rename from test/datasets/test_conll2000chunking.py rename to test/torchtext_unittest/datasets/test_conll2000chunking.py diff --git a/test/datasets/test_dbpedia.py b/test/torchtext_unittest/datasets/test_dbpedia.py similarity index 100% rename from test/datasets/test_dbpedia.py rename to test/torchtext_unittest/datasets/test_dbpedia.py diff --git a/test/datasets/test_enwik9.py b/test/torchtext_unittest/datasets/test_enwik9.py similarity index 100% rename from test/datasets/test_enwik9.py rename to test/torchtext_unittest/datasets/test_enwik9.py diff --git a/test/datasets/test_imdb.py b/test/torchtext_unittest/datasets/test_imdb.py similarity index 100% rename from test/datasets/test_imdb.py rename to test/torchtext_unittest/datasets/test_imdb.py diff --git a/test/datasets/test_iwslt2016.py b/test/torchtext_unittest/datasets/test_iwslt2016.py similarity index 100% rename from test/datasets/test_iwslt2016.py rename to test/torchtext_unittest/datasets/test_iwslt2016.py diff --git a/test/datasets/test_iwslt2017.py b/test/torchtext_unittest/datasets/test_iwslt2017.py similarity index 100% rename from test/datasets/test_iwslt2017.py rename to test/torchtext_unittest/datasets/test_iwslt2017.py diff --git a/test/datasets/test_mnli.py b/test/torchtext_unittest/datasets/test_mnli.py similarity index 100% rename from test/datasets/test_mnli.py rename to test/torchtext_unittest/datasets/test_mnli.py diff --git a/test/datasets/test_mrpc.py b/test/torchtext_unittest/datasets/test_mrpc.py similarity index 100% rename from test/datasets/test_mrpc.py rename to test/torchtext_unittest/datasets/test_mrpc.py diff --git a/test/datasets/test_multi30k.py b/test/torchtext_unittest/datasets/test_multi30k.py similarity index 100% rename from test/datasets/test_multi30k.py rename to test/torchtext_unittest/datasets/test_multi30k.py diff --git a/test/datasets/test_penntreebank.py b/test/torchtext_unittest/datasets/test_penntreebank.py similarity index 100% rename from test/datasets/test_penntreebank.py rename to test/torchtext_unittest/datasets/test_penntreebank.py diff --git a/test/datasets/test_qnli.py b/test/torchtext_unittest/datasets/test_qnli.py similarity index 100% rename from test/datasets/test_qnli.py rename to test/torchtext_unittest/datasets/test_qnli.py diff --git a/test/datasets/test_qqp.py b/test/torchtext_unittest/datasets/test_qqp.py similarity index 100% rename from test/datasets/test_qqp.py rename to test/torchtext_unittest/datasets/test_qqp.py diff --git a/test/datasets/test_rte.py b/test/torchtext_unittest/datasets/test_rte.py similarity index 100% rename from test/datasets/test_rte.py rename to test/torchtext_unittest/datasets/test_rte.py diff --git a/test/datasets/test_sogounews.py b/test/torchtext_unittest/datasets/test_sogounews.py similarity index 100% rename from test/datasets/test_sogounews.py rename to test/torchtext_unittest/datasets/test_sogounews.py diff --git a/test/datasets/test_squads.py b/test/torchtext_unittest/datasets/test_squads.py similarity index 100% rename from test/datasets/test_squads.py rename to test/torchtext_unittest/datasets/test_squads.py diff --git a/test/datasets/test_sst2.py b/test/torchtext_unittest/datasets/test_sst2.py similarity index 100% rename from test/datasets/test_sst2.py rename to test/torchtext_unittest/datasets/test_sst2.py diff --git a/test/datasets/test_stsb.py b/test/torchtext_unittest/datasets/test_stsb.py similarity index 100% rename from test/datasets/test_stsb.py rename to test/torchtext_unittest/datasets/test_stsb.py diff --git a/test/datasets/test_udpos.py b/test/torchtext_unittest/datasets/test_udpos.py similarity index 100% rename from test/datasets/test_udpos.py rename to test/torchtext_unittest/datasets/test_udpos.py diff --git a/test/datasets/test_wikitexts.py b/test/torchtext_unittest/datasets/test_wikitexts.py similarity index 100% rename from test/datasets/test_wikitexts.py rename to test/torchtext_unittest/datasets/test_wikitexts.py diff --git a/test/datasets/test_wnli.py b/test/torchtext_unittest/datasets/test_wnli.py similarity index 100% rename from test/datasets/test_wnli.py rename to test/torchtext_unittest/datasets/test_wnli.py diff --git a/test/datasets/test_yahooanswers.py b/test/torchtext_unittest/datasets/test_yahooanswers.py similarity index 100% rename from test/datasets/test_yahooanswers.py rename to test/torchtext_unittest/datasets/test_yahooanswers.py diff --git a/test/datasets/test_yelpreviews.py b/test/torchtext_unittest/datasets/test_yelpreviews.py similarity index 100% rename from test/datasets/test_yelpreviews.py rename to test/torchtext_unittest/datasets/test_yelpreviews.py diff --git a/test/models/__init__.py b/test/torchtext_unittest/models/__init__.py similarity index 100% rename from test/models/__init__.py rename to test/torchtext_unittest/models/__init__.py diff --git a/test/models/test_models.py b/test/torchtext_unittest/models/test_models.py similarity index 100% rename from test/models/test_models.py rename to test/torchtext_unittest/models/test_models.py diff --git a/test/models/test_transformers.py b/test/torchtext_unittest/models/test_transformers.py similarity index 100% rename from test/models/test_transformers.py rename to test/torchtext_unittest/models/test_transformers.py diff --git a/test/prototype/__init__.py b/test/torchtext_unittest/prototype/__init__.py similarity index 100% rename from test/prototype/__init__.py rename to test/torchtext_unittest/prototype/__init__.py diff --git a/test/prototype/integration_tests/__init__.py b/test/torchtext_unittest/prototype/models/__init__.py similarity index 100% rename from test/prototype/integration_tests/__init__.py rename to test/torchtext_unittest/prototype/models/__init__.py diff --git a/test/prototype/models/test_models.py b/test/torchtext_unittest/prototype/models/test_models.py similarity index 98% rename from test/prototype/models/test_models.py rename to test/torchtext_unittest/prototype/models/test_models.py index 6bdc4986c4..7d7fc9da66 100644 --- a/test/prototype/models/test_models.py +++ b/test/torchtext_unittest/prototype/models/test_models.py @@ -2,8 +2,8 @@ from unittest.mock import patch import torch -from test.common.torchtext_test_case import TorchtextTestCase from torch.nn import functional as F +from torchtext_unittest.common.torchtext_test_case import TorchtextTestCase class TestModels(TorchtextTestCase): diff --git a/test/prototype/models/test_transforms.py b/test/torchtext_unittest/prototype/models/test_transforms.py similarity index 93% rename from test/prototype/models/test_transforms.py rename to test/torchtext_unittest/prototype/models/test_transforms.py index e86f354fd0..82d70a4719 100644 --- a/test/prototype/models/test_transforms.py +++ b/test/torchtext_unittest/prototype/models/test_transforms.py @@ -1,7 +1,7 @@ import torch -from test.common.assets import get_asset_path -from test.common.torchtext_test_case import TorchtextTestCase from torchtext.prototype.models import T5Transform +from torchtext_unittest.common.assets import get_asset_path +from torchtext_unittest.common.torchtext_test_case import TorchtextTestCase class TestTransforms(TorchtextTestCase): diff --git a/test/prototype/test_functional.py b/test/torchtext_unittest/prototype/test_functional.py similarity index 100% rename from test/prototype/test_functional.py rename to test/torchtext_unittest/prototype/test_functional.py diff --git a/test/prototype/test_transforms.py b/test/torchtext_unittest/prototype/test_transforms.py similarity index 97% rename from test/prototype/test_transforms.py rename to test/torchtext_unittest/prototype/test_transforms.py index 71e9c02f74..3b28b07864 100644 --- a/test/prototype/test_transforms.py +++ b/test/torchtext_unittest/prototype/test_transforms.py @@ -3,14 +3,14 @@ import tempfile import torch -from test.common.assets import get_asset_path -from test.common.torchtext_test_case import TorchtextTestCase from torchtext.prototype.transforms import ( sentencepiece_processor, sentencepiece_tokenizer, VectorTransform, ) from torchtext.prototype.vectors import FastText +from torchtext_unittest.common.assets import get_asset_path +from torchtext_unittest.common.torchtext_test_case import TorchtextTestCase class TestTransforms(TorchtextTestCase): diff --git a/test/prototype/test_vectors.py b/test/torchtext_unittest/prototype/test_vectors.py similarity index 98% rename from test/prototype/test_vectors.py rename to test/torchtext_unittest/prototype/test_vectors.py index 088fb343cb..2c001cc265 100644 --- a/test/prototype/test_vectors.py +++ b/test/torchtext_unittest/prototype/test_vectors.py @@ -4,8 +4,8 @@ import unittest import torch -from test.common.torchtext_test_case import TorchtextTestCase from torchtext.prototype.vectors import build_vectors +from torchtext_unittest.common.torchtext_test_case import TorchtextTestCase class TestVectors(TorchtextTestCase): diff --git a/test/prototype/test_with_asset.py b/test/torchtext_unittest/prototype/test_with_asset.py similarity index 99% rename from test/prototype/test_with_asset.py rename to test/torchtext_unittest/prototype/test_with_asset.py index c5a85086f4..ff3f732c7c 100644 --- a/test/prototype/test_with_asset.py +++ b/test/torchtext_unittest/prototype/test_with_asset.py @@ -6,7 +6,6 @@ from functools import partial import torch -from test.common.torchtext_test_case import TorchtextTestCase from torch.utils.data import DataLoader from torchtext.data.functional import custom_replace from torchtext.prototype.transforms import ( @@ -19,6 +18,7 @@ from torchtext.prototype.vectors import build_vectors, FastText, GloVe, load_vectors_from_file_path from torchtext.prototype.vocab_factory import build_vocab_from_text_file, load_vocab_from_file from torchtext.utils import download_from_url +from torchtext_unittest.common.torchtext_test_case import TorchtextTestCase from ..common.assets import get_asset_path diff --git a/test/test_build.py b/test/torchtext_unittest/test_build.py similarity index 100% rename from test/test_build.py rename to test/torchtext_unittest/test_build.py diff --git a/test/test_functional.py b/test/torchtext_unittest/test_functional.py similarity index 100% rename from test/test_functional.py rename to test/torchtext_unittest/test_functional.py diff --git a/test/test_transforms.py b/test/torchtext_unittest/test_transforms.py similarity index 100% rename from test/test_transforms.py rename to test/torchtext_unittest/test_transforms.py diff --git a/test/test_utils.py b/test/torchtext_unittest/test_utils.py similarity index 98% rename from test/test_utils.py rename to test/torchtext_unittest/test_utils.py index c28299dc82..b647603bf0 100644 --- a/test/test_utils.py +++ b/test/torchtext_unittest/test_utils.py @@ -5,9 +5,9 @@ import unittest from urllib.parse import urljoin -from test.common.assets import conditional_remove, get_asset_path from torchtext import _TEXT_BUCKET from torchtext import utils +from torchtext_unittest.common.assets import conditional_remove, get_asset_path from .common.torchtext_test_case import TorchtextTestCase diff --git a/test/test_vocab.py b/test/torchtext_unittest/test_vocab.py similarity index 99% rename from test/test_vocab.py rename to test/torchtext_unittest/test_vocab.py index ff46450ae3..ca310e1a68 100644 --- a/test/test_vocab.py +++ b/test/torchtext_unittest/test_vocab.py @@ -4,8 +4,8 @@ import pytest import torch -from test.common.torchtext_test_case import TorchtextTestCase from torchtext.vocab import build_vocab_from_iterator, vocab +from torchtext_unittest.common.torchtext_test_case import TorchtextTestCase class TestVocab(TorchtextTestCase): diff --git a/tools/setup_helpers/extension.py b/tools/setup_helpers/extension.py index 1f7236e4c2..760b3bb798 100644 --- a/tools/setup_helpers/extension.py +++ b/tools/setup_helpers/extension.py @@ -21,6 +21,10 @@ _ROOT_DIR = _THIS_DIR.parent.parent.resolve() +def _get_cxx11_abi(): + return "-D_GLIBCXX_USE_CXX11_ABI=" + str(int(torch.compiled_with_cxx11_abi())) + + def get_ext_modules(): modules = [ Extension(name=_LIBTORCHTEXT_NAME, sources=[]), @@ -72,6 +76,7 @@ def build_extension(self, ext): "-DBUILD_SHARED_LIBS=OFF", "-DCMAKE_POLICY_DEFAULT_CMP0063=NEW", "-DSPM_ENABLE_SHARED=OFF", + f"-DTORCH_COMPILED_WITH_CXX_ABI={_get_cxx11_abi()}", ] build_args = ["--target", "install"] From 147bae7b42181fb4f4249db2f7e9ae3e4127a15b Mon Sep 17 00:00:00 2001 From: Arnav Mehta <65492948+arnavmehta7@users.noreply.github.com> Date: Fri, 16 Sep 2022 08:23:51 +0530 Subject: [PATCH 06/14] corrected Linting --- test/prototype/test_functional.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/prototype/test_functional.py b/test/prototype/test_functional.py index 64a0f2db81..5f60f1a0e7 100644 --- a/test/prototype/test_functional.py +++ b/test/prototype/test_functional.py @@ -1,11 +1,7 @@ import os -import platform -import unittest - import torch import torchtext.data as data from torchtext.prototype.transforms import basic_english_normalize - from ..common.torchtext_test_case import TorchtextTestCase From 22b731d4c733de8cf31a93fa78517892bc44a0d1 Mon Sep 17 00:00:00 2001 From: Arnav Mehta <65492948+arnavmehta7@users.noreply.github.com> Date: Fri, 16 Sep 2022 08:39:11 +0530 Subject: [PATCH 07/14] LintingCorrectedAgain --- test/prototype/test_functional.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/prototype/test_functional.py b/test/prototype/test_functional.py index 5f60f1a0e7..f6da11e809 100644 --- a/test/prototype/test_functional.py +++ b/test/prototype/test_functional.py @@ -1,7 +1,9 @@ import os + import torch import torchtext.data as data from torchtext.prototype.transforms import basic_english_normalize + from ..common.torchtext_test_case import TorchtextTestCase From dc948337580f3cde985f74a0466f0d177a438438 Mon Sep 17 00:00:00 2001 From: Arnav Mehta <65492948+arnavmehta7@users.noreply.github.com> Date: Thu, 15 Sep 2022 19:17:12 +0530 Subject: [PATCH 08/14] todo-decorator-remove-solved --- test/torchtext_unittest/prototype/test_functional.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/torchtext_unittest/prototype/test_functional.py b/test/torchtext_unittest/prototype/test_functional.py index ef4f096277..64a0f2db81 100644 --- a/test/torchtext_unittest/prototype/test_functional.py +++ b/test/torchtext_unittest/prototype/test_functional.py @@ -10,8 +10,6 @@ class TestFunctional(TorchtextTestCase): - # TODO(Nayef211): remove decorator once https://github.com/pytorch/pytorch/issues/38207 is closed - @unittest.skipIf(platform.system() == "Windows", "Test is known to fail on Windows.") def test_BasicEnglishNormalize(self) -> None: test_sample = "'\".
,()!?;: Basic English Normalization for a Line of Text '\".
,()!?;:" ref_results = [ From 4bee7bf9f853a57492f74341fccbf199bd6d2bb2 Mon Sep 17 00:00:00 2001 From: Arnav Mehta <65492948+arnavmehta7@users.noreply.github.com> Date: Thu, 15 Sep 2022 19:21:47 +0530 Subject: [PATCH 09/14] todo-update-toOriginalServer-resolved --- torchtext/datasets/multi30k.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/torchtext/datasets/multi30k.py b/torchtext/datasets/multi30k.py index 47f5fb3a33..07bea245ae 100644 --- a/torchtext/datasets/multi30k.py +++ b/torchtext/datasets/multi30k.py @@ -12,11 +12,10 @@ from torchdata.datapipes.iter import FileOpener, IterableWrapper from torchtext._download_hooks import HttpReader -# TODO: Update URL to original once the server is back up (see https://github.com/pytorch/text/issues/1756) URL = { - "train": r"https://raw.githubusercontent.com/neychev/small_DL_repo/master/datasets/Multi30k/training.tar.gz", - "valid": r"https://raw.githubusercontent.com/neychev/small_DL_repo/master/datasets/Multi30k/validation.tar.gz", - "test": r"https://raw.githubusercontent.com/neychev/small_DL_repo/master/datasets/Multi30k/mmt16_task1_test.tar.gz", + "train": r"http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/training.tar.gz", + "valid": r"http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/validation.tar.gz", + "test": r"http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/mmt16_task1_test.tar.gz", } MD5 = { From 2e631d05d1e6cc9a116ea44557df44eb5fd582b9 Mon Sep 17 00:00:00 2001 From: Arnav Mehta <65492948+arnavmehta7@users.noreply.github.com> Date: Fri, 16 Sep 2022 08:23:51 +0530 Subject: [PATCH 10/14] corrected Linting --- test/torchtext_unittest/prototype/test_functional.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/torchtext_unittest/prototype/test_functional.py b/test/torchtext_unittest/prototype/test_functional.py index 64a0f2db81..5f60f1a0e7 100644 --- a/test/torchtext_unittest/prototype/test_functional.py +++ b/test/torchtext_unittest/prototype/test_functional.py @@ -1,11 +1,7 @@ import os -import platform -import unittest - import torch import torchtext.data as data from torchtext.prototype.transforms import basic_english_normalize - from ..common.torchtext_test_case import TorchtextTestCase From 83edf9795bba21c988dfdfab598a1f368966b5cd Mon Sep 17 00:00:00 2001 From: Arnav Mehta <65492948+arnavmehta7@users.noreply.github.com> Date: Fri, 16 Sep 2022 08:39:11 +0530 Subject: [PATCH 11/14] LintingCorrectedAgain --- test/torchtext_unittest/prototype/test_functional.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/torchtext_unittest/prototype/test_functional.py b/test/torchtext_unittest/prototype/test_functional.py index 5f60f1a0e7..f6da11e809 100644 --- a/test/torchtext_unittest/prototype/test_functional.py +++ b/test/torchtext_unittest/prototype/test_functional.py @@ -1,7 +1,9 @@ import os + import torch import torchtext.data as data from torchtext.prototype.transforms import basic_english_normalize + from ..common.torchtext_test_case import TorchtextTestCase From 26933cd96d65ea8f22dab98a0b5b4e844ac2876e Mon Sep 17 00:00:00 2001 From: Arnav Mehta <65492948+arnavmehta7@users.noreply.github.com> Date: Fri, 16 Sep 2022 08:23:51 +0530 Subject: [PATCH 12/14] corrected Linting --- test/torchtext_unittest/prototype/test_functional.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/torchtext_unittest/prototype/test_functional.py b/test/torchtext_unittest/prototype/test_functional.py index f6da11e809..5f60f1a0e7 100644 --- a/test/torchtext_unittest/prototype/test_functional.py +++ b/test/torchtext_unittest/prototype/test_functional.py @@ -1,9 +1,7 @@ import os - import torch import torchtext.data as data from torchtext.prototype.transforms import basic_english_normalize - from ..common.torchtext_test_case import TorchtextTestCase From 9f49c88e33335aab48e1da5381a1a9d9e8409486 Mon Sep 17 00:00:00 2001 From: Arnav Mehta <65492948+arnavmehta7@users.noreply.github.com> Date: Fri, 16 Sep 2022 08:39:11 +0530 Subject: [PATCH 13/14] LintingCorrectedAgain --- test/torchtext_unittest/prototype/test_functional.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/torchtext_unittest/prototype/test_functional.py b/test/torchtext_unittest/prototype/test_functional.py index 5f60f1a0e7..f6da11e809 100644 --- a/test/torchtext_unittest/prototype/test_functional.py +++ b/test/torchtext_unittest/prototype/test_functional.py @@ -1,7 +1,9 @@ import os + import torch import torchtext.data as data from torchtext.prototype.transforms import basic_english_normalize + from ..common.torchtext_test_case import TorchtextTestCase From 30cfb497abe9cad4146f3cf30f2d6632c1b5ae0c Mon Sep 17 00:00:00 2001 From: Arnav Mehta <65492948+arnavmehta7@users.noreply.github.com> Date: Sun, 25 Sep 2022 00:29:35 +0530 Subject: [PATCH 14/14] final