From 54e0962783bda2455e1b3d24e49b16bf63928d90 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Tue, 21 Feb 2023 11:39:05 -0800 Subject: [PATCH 01/35] Update build.sh --- .jenkins/build.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index d09b0a8782a..37febed72ba 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -29,6 +29,12 @@ pip install -r $DIR/../requirements.txt # pip uninstall -y torch torchvision torchaudio torchtext # pip install --pre --upgrade -f https://download.pytorch.org/whl/test/cu116/torch_test.html torch torchdata torchvision torchaudio torchtext +# Install 2.0 nightly + +pip uninstall -y torch torchvision torchaudio torchtext +pip install --pre --upgrade -f https://download.pytorch.org/whl/test/cu118/torch_test.html torch torchdata torchvision torchaudio torchtext + + # Install two language tokenizers for Translation with TorchText tutorial python -m spacy download en_core_web_sm python -m spacy download de_core_news_sm From c23f3c0f2fc8414746251add839e32e3a8d82712 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Wed, 22 Feb 2023 08:41:09 -0800 Subject: [PATCH 02/35] Update build.sh --- .jenkins/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index 37febed72ba..a5a33e94433 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -32,8 +32,8 @@ pip install -r $DIR/../requirements.txt # Install 2.0 nightly pip uninstall -y torch torchvision torchaudio torchtext -pip install --pre --upgrade -f https://download.pytorch.org/whl/test/cu118/torch_test.html torch torchdata torchvision torchaudio torchtext - +#pip install --pre --upgrade -f https://download.pytorch.org/whl/test/cu118/torch_test.html torch torchdata torchvision torchaudio torchtext +pip3 install torch torchdata torchvision torchaudio torchtext --index-url https://download.pytorch.org/whl/test/cpu/torch_test.html # Install two language tokenizers for Translation with TorchText tutorial python -m spacy download en_core_web_sm From 878b6d0d1a4218e43489833d7725421e5c5cce66 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Wed, 22 Feb 2023 09:09:18 -0800 Subject: [PATCH 03/35] Update build.sh --- .jenkins/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index a5a33e94433..819967adf61 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -33,7 +33,7 @@ pip install -r $DIR/../requirements.txt pip uninstall -y torch torchvision torchaudio torchtext #pip install --pre --upgrade -f https://download.pytorch.org/whl/test/cu118/torch_test.html torch torchdata torchvision torchaudio torchtext -pip3 install torch torchdata torchvision torchaudio torchtext --index-url https://download.pytorch.org/whl/test/cpu/torch_test.html +pip3 install torch torchdata torchvision torchaudio --index-url https://download.pytorch.org/whl/test/cpu/torch_test.html # Install two language tokenizers for Translation with TorchText tutorial python -m spacy download en_core_web_sm From c8ede95a8e45aa0940c58de6db3a4edb26e23f76 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Wed, 22 Feb 2023 13:03:25 -0800 Subject: [PATCH 04/35] Update build.sh --- .jenkins/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index 819967adf61..0848116fb53 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -33,7 +33,7 @@ pip install -r $DIR/../requirements.txt pip uninstall -y torch torchvision torchaudio torchtext #pip install --pre --upgrade -f https://download.pytorch.org/whl/test/cu118/torch_test.html torch torchdata torchvision torchaudio torchtext -pip3 install torch torchdata torchvision torchaudio --index-url https://download.pytorch.org/whl/test/cpu/torch_test.html +pip3 install torch torchdata torchvision torchaudio torchtext --index-url https://download.pytorch.org/whl/test/cpu # Install two language tokenizers for Translation with TorchText tutorial python -m spacy download en_core_web_sm From a219afbf82e03453ae1aa16eb311168f664d23e0 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Thu, 23 Feb 2023 14:40:50 -0800 Subject: [PATCH 05/35] Update build.sh --- .jenkins/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index 0848116fb53..79724e69b07 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -33,7 +33,7 @@ pip install -r $DIR/../requirements.txt pip uninstall -y torch torchvision torchaudio torchtext #pip install --pre --upgrade -f https://download.pytorch.org/whl/test/cu118/torch_test.html torch torchdata torchvision torchaudio torchtext -pip3 install torch torchdata torchvision torchaudio torchtext --index-url https://download.pytorch.org/whl/test/cpu +pip3 install torch torchdata torchvision torchaudio --index-url https://download.pytorch.org/whl/test/cpu # Install two language tokenizers for Translation with TorchText tutorial python -m spacy download en_core_web_sm From a661b711e320992c45a3f6ed4df274b1b4d292bc Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Thu, 23 Feb 2023 15:23:16 -0800 Subject: [PATCH 06/35] Update build.sh --- .jenkins/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index 79724e69b07..0848116fb53 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -33,7 +33,7 @@ pip install -r $DIR/../requirements.txt pip uninstall -y torch torchvision torchaudio torchtext #pip install --pre --upgrade -f https://download.pytorch.org/whl/test/cu118/torch_test.html torch torchdata torchvision torchaudio torchtext -pip3 install torch torchdata torchvision torchaudio --index-url https://download.pytorch.org/whl/test/cpu +pip3 install torch torchdata torchvision torchaudio torchtext --index-url https://download.pytorch.org/whl/test/cpu # Install two language tokenizers for Translation with TorchText tutorial python -m spacy download en_core_web_sm From d75acc849241250a28811388b3d492336420233e Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Thu, 23 Feb 2023 15:50:27 -0800 Subject: [PATCH 07/35] Update build.sh --- .jenkins/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index 0848116fb53..63d72c42391 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -33,7 +33,7 @@ pip install -r $DIR/../requirements.txt pip uninstall -y torch torchvision torchaudio torchtext #pip install --pre --upgrade -f https://download.pytorch.org/whl/test/cu118/torch_test.html torch torchdata torchvision torchaudio torchtext -pip3 install torch torchdata torchvision torchaudio torchtext --index-url https://download.pytorch.org/whl/test/cpu +pip install torch torchdata torchvision torchaudio torchtext --index-url https://download.pytorch.org/whl/test/cpu # Install two language tokenizers for Translation with TorchText tutorial python -m spacy download en_core_web_sm From 4aaf33ce71583d66e460360192ccb8ac8346d9d5 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Fri, 24 Feb 2023 08:07:25 -0800 Subject: [PATCH 08/35] Update build.sh --- .jenkins/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index 63d72c42391..89a609d98d3 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -31,7 +31,7 @@ pip install -r $DIR/../requirements.txt # Install 2.0 nightly -pip uninstall -y torch torchvision torchaudio torchtext +pip uninstall -y torch torchvision torchaudio torchtext torchdata #pip install --pre --upgrade -f https://download.pytorch.org/whl/test/cu118/torch_test.html torch torchdata torchvision torchaudio torchtext pip install torch torchdata torchvision torchaudio torchtext --index-url https://download.pytorch.org/whl/test/cpu From 3a7102500687ac102dfe018fbc950822a04e8167 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Fri, 24 Feb 2023 08:42:23 -0800 Subject: [PATCH 09/35] Update build.sh --- .jenkins/build.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index 89a609d98d3..f59c97b8ea9 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -33,7 +33,8 @@ pip install -r $DIR/../requirements.txt pip uninstall -y torch torchvision torchaudio torchtext torchdata #pip install --pre --upgrade -f https://download.pytorch.org/whl/test/cu118/torch_test.html torch torchdata torchvision torchaudio torchtext -pip install torch torchdata torchvision torchaudio torchtext --index-url https://download.pytorch.org/whl/test/cpu +pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/test/cu117 +pip3 install torchdata torchtext --index-url https://download.pytorch.org/whl/test/cpu # Install two language tokenizers for Translation with TorchText tutorial python -m spacy download en_core_web_sm From 092f7be613afe3a769fbb9e07682c1ea77d9d318 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Fri, 24 Feb 2023 11:48:36 -0800 Subject: [PATCH 10/35] Fix #2219 and disabling failing tutorials --- .jenkins/validate_tutorials_built.py | 2 ++ beginner_source/introyt/autogradyt_tutorial.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.jenkins/validate_tutorials_built.py b/.jenkins/validate_tutorials_built.py index 5d627b70398..12583e3c126 100644 --- a/.jenkins/validate_tutorials_built.py +++ b/.jenkins/validate_tutorials_built.py @@ -51,6 +51,8 @@ "recipes/Captum_Recipe", "flask_rest_api_tutorial", "text_to_speech_with_torchaudio", + "forward_ad_usage", # remove after #2218 is resolved. + "text_sentiment_ngrams_tutorial" # reenable after 2.0 release. ] diff --git a/beginner_source/introyt/autogradyt_tutorial.py b/beginner_source/introyt/autogradyt_tutorial.py index 19c25c5e81d..1ce49f3cd03 100644 --- a/beginner_source/introyt/autogradyt_tutorial.py +++ b/beginner_source/introyt/autogradyt_tutorial.py @@ -334,7 +334,7 @@ def forward(self, x): print(model.layer2.weight.grad[0][0:10]) -optimizer.zero_grad() +optimizer.zero_grad(set_to_none=False) print(model.layer2.weight.grad[0][0:10]) From 2f9504a23709b40ab8537a2d595599ac02d73d19 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Thu, 2 Mar 2023 10:43:25 -0800 Subject: [PATCH 11/35] Update validate_tutorials_built.py --- .jenkins/validate_tutorials_built.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.jenkins/validate_tutorials_built.py b/.jenkins/validate_tutorials_built.py index 12583e3c126..b3ab59f234a 100644 --- a/.jenkins/validate_tutorials_built.py +++ b/.jenkins/validate_tutorials_built.py @@ -52,7 +52,8 @@ "flask_rest_api_tutorial", "text_to_speech_with_torchaudio", "forward_ad_usage", # remove after #2218 is resolved. - "text_sentiment_ngrams_tutorial" # reenable after 2.0 release. + "text_sentiment_ngrams_tutorial", # reenable after 2.0 release. + "tensorboard_profiler_tutorial", # reenable after 2.0 release. ] From c0a283df3276a73b6ddf7c6b42f7eaf2622fbc74 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Thu, 2 Mar 2023 10:43:43 -0800 Subject: [PATCH 12/35] Update validate_tutorials_built.py --- .jenkins/validate_tutorials_built.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.jenkins/validate_tutorials_built.py b/.jenkins/validate_tutorials_built.py index b3ab59f234a..4a2f379c3a4 100644 --- a/.jenkins/validate_tutorials_built.py +++ b/.jenkins/validate_tutorials_built.py @@ -53,7 +53,7 @@ "text_to_speech_with_torchaudio", "forward_ad_usage", # remove after #2218 is resolved. "text_sentiment_ngrams_tutorial", # reenable after 2.0 release. - "tensorboard_profiler_tutorial", # reenable after 2.0 release. + "tensorboard_profiler_tutorial" # reenable after 2.0 release. ] From 58542d0e2f8c58495ac57cfc92ffdf9cf17d3ef5 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Mon, 6 Mar 2023 12:30:53 -0800 Subject: [PATCH 13/35] Update validate_tutorials_built.py --- .jenkins/validate_tutorials_built.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.jenkins/validate_tutorials_built.py b/.jenkins/validate_tutorials_built.py index 68fdabac1c4..b7b70560383 100644 --- a/.jenkins/validate_tutorials_built.py +++ b/.jenkins/validate_tutorials_built.py @@ -52,7 +52,10 @@ "text_to_speech_with_torchaudio", "forward_ad_usage", # remove after #2218 is resolved. "text_sentiment_ngrams_tutorial", # reenable after 2.0 release. - "tensorboard_profiler_tutorial" # reenable after 2.0 release. + "tensorboard_profiler_tutorial", # reenable after 2.0 release. + "nvfuser_intro_tutorial.py", # reenable after 2.0 release. + "memory_format_tutorial.py" # reenable after 2.0 release. + ] From ca273bae93b70b3d54ddd1d89f08204d6a1399c8 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Mon, 6 Mar 2023 13:25:22 -0800 Subject: [PATCH 14/35] Update validate_tutorials_built.py --- .jenkins/validate_tutorials_built.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.jenkins/validate_tutorials_built.py b/.jenkins/validate_tutorials_built.py index b7b70560383..bedb92252e9 100644 --- a/.jenkins/validate_tutorials_built.py +++ b/.jenkins/validate_tutorials_built.py @@ -53,8 +53,8 @@ "forward_ad_usage", # remove after #2218 is resolved. "text_sentiment_ngrams_tutorial", # reenable after 2.0 release. "tensorboard_profiler_tutorial", # reenable after 2.0 release. - "nvfuser_intro_tutorial.py", # reenable after 2.0 release. - "memory_format_tutorial.py" # reenable after 2.0 release. + "nvfuser_intro_tutorial", # reenable after 2.0 release. + "memory_format_tutorial" # reenable after 2.0 release. ] From 66a6e4f9fef8ab769fdb0040ab1d4a8f0817c112 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Mon, 6 Mar 2023 13:41:15 -0800 Subject: [PATCH 15/35] Update build.sh --- .jenkins/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index f59c97b8ea9..6176e4bf4a8 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -33,7 +33,7 @@ pip install -r $DIR/../requirements.txt pip uninstall -y torch torchvision torchaudio torchtext torchdata #pip install --pre --upgrade -f https://download.pytorch.org/whl/test/cu118/torch_test.html torch torchdata torchvision torchaudio torchtext -pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/test/cu117 +pip3 install torch torchvision torchaudio --no-cache-dir --index-url https://download.pytorch.org/whl/test/cu117 pip3 install torchdata torchtext --index-url https://download.pytorch.org/whl/test/cpu # Install two language tokenizers for Translation with TorchText tutorial From 1164e035661f94a70b487d94e4f4445f7b917654 Mon Sep 17 00:00:00 2001 From: Joe Cummings Date: Mon, 27 Feb 2023 17:04:10 -0500 Subject: [PATCH 16/35] Add T5 tutorial --- beginner_source/t5_tutorial.py | 470 +++++++++++++++++++++++++++++++++ 1 file changed, 470 insertions(+) create mode 100644 beginner_source/t5_tutorial.py diff --git a/beginner_source/t5_tutorial.py b/beginner_source/t5_tutorial.py new file mode 100644 index 00000000000..d6165a20096 --- /dev/null +++ b/beginner_source/t5_tutorial.py @@ -0,0 +1,470 @@ +""" +T5-Base Model for Summarization, Sentiment Classification, and Translation +========================================================================== + +**Author**: `Pendo Abbo `__ +**Author**: `Joe Cummings `__ + +""" + +###################################################################### +# Overview +# -------- +# +# This tutorial demonstrates how to use a pre-trained T5 Model for summarization, sentiment classification, and +# translation tasks. We will demonstrate how to use the torchtext library to: +# +# 1. Build a text pre-processing pipeline for a T5 model +# 2. Instantiate a pre-trained T5 model with base configuration +# 3. Read in the CNNDM, IMDB, and Multi30k datasets and pre-process their texts in preparation for the model +# 4. Perform text summarization, sentiment classification, and translation +# +# + +###################################################################### +# Common imports +# -------------- +import torch + +DEVICE = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu") + + +####################################################################### +# Data Transformation +# ------------------- +# +# The T5 model does not work with raw text. Instead, it requires the text to be transformed into numerical form +# in order to perform training and inference. The following transformations are required for the T5 model: +# +# 1. Tokenize text +# 2. Convert tokens into (integer) IDs +# 3. Truncate the sequences to a specified maximum length +# 4. Add end-of-sequence (EOS) and padding token IDs +# +# T5 uses a SentencePiece model for text tokenization. Below, we use a pre-trained SentencePiece model to build +# the text pre-processing pipeline using torchtext's T5Transform. Note that the transform supports both +# batched and non-batched text input (i.e. one can either pass a single sentence or a list of sentences), however +# the T5 model expects the input to be batched. +# + +from torchtext.models import T5Transform + +padding_idx = 0 +eos_idx = 1 +max_seq_len = 512 +t5_sp_model_path = "https://download.pytorch.org/models/text/t5_tokenizer_base.model" + +transform = T5Transform( + sp_model_path=t5_sp_model_path, + max_seq_len=max_seq_len, + eos_idx=eos_idx, + padding_idx=padding_idx, +) + +####################################################################### +# Alternatively, we can also use the transform shipped with the pre-trained models that does all of the above out-of-the-box +# +# :: +# +# from torchtext.models import T5_BASE_GENERATION +# transform = T5_BASE_GENERATION.transform() +# + + +###################################################################### +# Model Preparation +# ----------------- +# +# torchtext provides SOTA pre-trained models that can be used directly for NLP tasks or fine-tuned on downstream tasks. Below +# we use the pre-trained T5 model with standard base configuration to perform text summarization, sentiment classification, and +# translation. For additional details on available pre-trained models, please refer to documentation at +# https://pytorch.org/text/main/models.html +# +# +from torchtext.models import T5_BASE_GENERATION + + +t5_base = T5_BASE_GENERATION +transform = t5_base.transform() +model = t5_base.get_model() +model.eval() +model.to(DEVICE) + + +####################################################################### +# GenerationUtils +# ------------------ +# +# We can use torchtext's `GenerationUtils` to produce an output sequence based on the input sequence provided. This calls on the +# model's encoder and decoder, and iteratively expands the decoded sequences until the end-of-sequence token is generated +# for all sequences in the batch. The `generate` method shown below uses greedy search to generate the sequences. Beam search and +# other decoding strategies are also supported. +# +# +from torchtext.prototype.generate import GenerationUtils + +sequence_generator = GenerationUtils(model) + + +####################################################################### +# Datasets +# -------- +# torchtext provides several standard NLP datasets. For a complete list, refer to the documentation +# at https://pytorch.org/text/stable/datasets.html. These datasets are built using composable torchdata +# datapipes and hence support standard flow-control and mapping/transformation using user defined +# functions and transforms. +# +# Below, we demonstrate how to pre-process the CNNDM dataset to include the prefix necessary for the +# model to indentify the task it is performing. The CNNDM dataset has a train, validation, and test +# split. Below we demo on the test split. +# +# The T5 model uses the prefix "summarize" for text summarization. For more information on task +# prefixes, please visit Appendix D of the T5 Paper at https://arxiv.org/pdf/1910.10683.pdf +# +# .. note:: +# Using datapipes is still currently subject to a few caveats. If you wish +# to extend this example to include shuffling, multi-processing, or +# distributed learning, please see :ref:`this note ` +# for further instructions. + +from functools import partial + +from torch.utils.data import DataLoader +from torchtext.datasets import CNNDM + +cnndm_batch_size = 5 +cnndm_datapipe = CNNDM(split="test") +task = "summarize" + + +def apply_prefix(task, x): + return f"{task}: " + x[0], x[1] + + +cnndm_datapipe = cnndm_datapipe.map(partial(apply_prefix, task)) +cnndm_datapipe = cnndm_datapipe.batch(cnndm_batch_size) +cnndm_datapipe = cnndm_datapipe.rows2columnar(["article", "abstract"]) +cnndm_dataloader = DataLoader(cnndm_datapipe, batch_size=None) + +####################################################################### +# Alternately we can also use batched API (i.e apply the prefix on the whole batch) +# +# :: +# +# def batch_prefix(task, x): +# return { +# "article": [f'{task}: ' + y for y in x["article"]], +# "abstract": x["abstract"] +# } +# +# cnndm_batch_size = 5 +# cnndm_datapipe = CNNDM(split="test") +# task = 'summarize' +# +# cnndm_datapipe = cnndm_datapipe.batch(cnndm_batch_size).rows2columnar(["article", "abstract"]) +# cnndm_datapipe = cnndm_datapipe.map(partial(batch_prefix, task)) +# cnndm_dataloader = DataLoader(cnndm_datapipe, batch_size=None) +# + +####################################################################### +# We can also load the IMDB dataset, which will be used to demonstrate sentiment classification using the T5 model. +# This dataset has a train and test split. Below we demo on the test split. +# +# The T5 model was trained on the SST2 dataset (also available in torchtext) for sentiment classification using the +# prefix "sst2 sentence". Therefore, we will use this prefix to perform sentiment classification on the IMDB dataset. +# + +from torchtext.datasets import IMDB + +imdb_batch_size = 3 +imdb_datapipe = IMDB(split="test") +task = "sst2 sentence" +labels = {"neg": "negative", "pos": "positive"} + + +def process_labels(labels, x): + return x[1], labels[x[0]] + + +imdb_datapipe = imdb_datapipe.map(partial(process_labels, labels)) +imdb_datapipe = imdb_datapipe.map(partial(apply_prefix, task)) +imdb_datapipe = imdb_datapipe.batch(imdb_batch_size) +imdb_datapipe = imdb_datapipe.rows2columnar(["text", "label"]) +imdb_dataloader = DataLoader(imdb_datapipe, batch_size=None) + +####################################################################### +# Finally, we can also load the Multi30k dataset to demonstrate English to German translation using the T5 model. +# This dataset has a train, validation, and test split. Below we demo on the test split. +# +# The T5 model uses the prefix "translate English to German" for this task. + +from torchtext.datasets import Multi30k + +multi_batch_size = 5 +language_pair = ("en", "de") +multi_datapipe = Multi30k(split="test", language_pair=language_pair) +task = "translate English to German" + +multi_datapipe = multi_datapipe.map(partial(apply_prefix, task)) +multi_datapipe = multi_datapipe.batch(multi_batch_size) +multi_datapipe = multi_datapipe.rows2columnar(["english", "german"]) +multi_dataloader = DataLoader(multi_datapipe, batch_size=None) + +####################################################################### +# Generate Summaries +# ------------------ +# +# We can put all of the components together to generate summaries on the first batch of articles in the CNNDM test set +# using a beam size of 1. +# + +batch = next(iter(cnndm_dataloader)) +input_text = batch["article"] +target = batch["abstract"] +beam_size = 1 + +model_input = transform(input_text) +model_output = sequence_generator.generate(model_input, eos_idx=eos_idx, beam_size=beam_size) +output_text = transform.decode(model_output.tolist()) + +for i in range(cnndm_batch_size): + print(f"Example {i+1}:\n") + print(f"prediction: {output_text[i]}\n") + print(f"target: {target[i]}\n\n") + + +####################################################################### +# Summarization Output +# -------------------- +# +# :: +# +# Example 1: +# +# prediction: the Palestinians become the 123rd member of the international criminal +# court . the accession was marked by a ceremony at the Hague, where the court is based . +# the ICC opened a preliminary examination into the situation in the occupied +# Palestinian territory . +# +# target: Membership gives the ICC jurisdiction over alleged crimes committed in +# Palestinian territories since last June . Israel and the United States opposed the +# move, which could open the door to war crimes investigations against Israelis . +# +# +# Example 2: +# +# prediction: a stray pooch has used up at least three of her own after being hit by a +# car and buried in a field . the dog managed to stagger to a nearby farm, dirt-covered +# and emaciated, where she was found . she suffered a dislocated jaw, leg injuries and a +# caved-in sinus cavity -- and still requires surgery to help her breathe . +# +# target: Theia, a bully breed mix, was apparently hit by a car, whacked with a hammer +# and buried in a field . "She's a true miracle dog and she deserves a good life," says +# Sara Mellado, who is looking for a home for Theia . +# +# +# Example 3: +# +# prediction: mohammad Javad Zarif arrived in Iran on a sunny friday morning . he has gone +# a long way to bring Iran in from the cold and allow it to rejoin the international +# community . but there are some facts about him that are less well-known . +# +# target: Mohammad Javad Zarif has spent more time with John Kerry than any other +# foreign minister . He once participated in a takeover of the Iranian Consulate in San +# Francisco . The Iranian foreign minister tweets in English . +# +# +# Example 4: +# +# prediction: five americans were monitored for three weeks after being exposed to Ebola in +# west africa . one of the five had a heart-related issue and has been discharged but hasn't +# left the area . they are clinicians for Partners in Health, a Boston-based aid group . +# +# target: 17 Americans were exposed to the Ebola virus while in Sierra Leone in March . +# Another person was diagnosed with the disease and taken to hospital in Maryland . +# National Institutes of Health says the patient is in fair condition after weeks of +# treatment . +# +# +# Example 5: +# +# prediction: the student was identified during an investigation by campus police and +# the office of student affairs . he admitted to placing the noose on the tree early +# Wednesday morning . the incident is one of several recent racist events to affect +# college students . +# +# target: Student is no longer on Duke University campus and will face disciplinary +# review . School officials identified student during investigation and the person +# admitted to hanging the noose, Duke says . The noose, made of rope, was discovered on +# campus about 2 a.m. +# + + +####################################################################### +# Generate Sentiment Classifications +# ---------------------------------- +# +# Similarly, we can use the model to generate sentiment classifications on the first batch of reviews from the IMDB test set +# using a beam size of 1. +# + +batch = next(iter(imdb_dataloader)) +input_text = batch["text"] +target = batch["label"] +beam_size = 1 + +model_input = transform(input_text) +model_output = sequence_generator.generate(model_input, eos_idx=eos_idx, beam_size=beam_size) +output_text = transform.decode(model_output.tolist()) + +for i in range(imdb_batch_size): + print(f"Example {i+1}:\n") + print(f"input_text: {input_text[i]}\n") + print(f"prediction: {output_text[i]}\n") + print(f"target: {target[i]}\n\n") + + +####################################################################### +# Sentiment Output +# ---------------- +# +# :: +# +# Example 1: +# +# input_text: sst2 sentence: I love sci-fi and am willing to put up with a lot. Sci-fi +# movies/TV are usually underfunded, under-appreciated and misunderstood. I tried to like +# this, I really did, but it is to good TV sci-fi as Babylon 5 is to Star Trek (the original). +# Silly prosthetics, cheap cardboard sets, stilted dialogues, CG that doesn't match the +# background, and painfully one-dimensional characters cannot be overcome with a 'sci-fi' +# setting. (I'm sure there are those of you out there who think Babylon 5 is good sci-fi TV. +# It's not. It's clichéd and uninspiring.) While US viewers might like emotion and character +# development, sci-fi is a genre that does not take itself seriously (cf. Star Trek). It may +# treat important issues, yet not as a serious philosophy. It's really difficult to care about +# the characters here as they are not simply foolish, just missing a spark of life. Their +# actions and reactions are wooden and predictable, often painful to watch. The makers of Earth +# KNOW it's rubbish as they have to always say "Gene Roddenberry's Earth..." otherwise people +# would not continue watching. Roddenberry's ashes must be turning in their orbit as this dull, +# cheap, poorly edited (watching it without advert breaks really brings this home) trudging +# Trabant of a show lumbers into space. Spoiler. So, kill off a main character. And then bring +# him back as another actor. Jeeez. Dallas all over again. +# +# prediction: negative +# +# target: negative +# +# +# Example 2: +# +# input_text: sst2 sentence: Worth the entertainment value of a rental, especially if you like +# action movies. This one features the usual car chases, fights with the great Van Damme kick +# style, shooting battles with the 40 shell load shotgun, and even terrorist style bombs. All +# of this is entertaining and competently handled but there is nothing that really blows you +# away if you've seen your share before.

The plot is made interesting by the +# inclusion of a rabbit, which is clever but hardly profound. Many of the characters are +# heavily stereotyped -- the angry veterans, the terrified illegal aliens, the crooked cops, +# the indifferent feds, the bitchy tough lady station head, the crooked politician, the fat +# federale who looks like he was typecast as the Mexican in a Hollywood movie from the 1940s. +# All passably acted but again nothing special.

I thought the main villains were +# pretty well done and fairly well acted. By the end of the movie you certainly knew who the +# good guys were and weren't. There was an emotional lift as the really bad ones got their just +# deserts. Very simplistic, but then you weren't expecting Hamlet, right? The only thing I found +# really annoying was the constant cuts to VDs daughter during the last fight scene.

+# Not bad. Not good. Passable 4. +# +# prediction: negative +# +# target: negative +# +# +# Example 3: +# +# input_text: sst2 sentence: its a totally average film with a few semi-alright action sequences +# that make the plot seem a little better and remind the viewer of the classic van dam films. +# parts of the plot don't make sense and seem to be added in to use up time. the end plot is that +# of a very basic type that doesn't leave the viewer guessing and any twists are obvious from the +# beginning. the end scene with the flask backs don't make sense as they are added in and seem to +# have little relevance to the history of van dam's character. not really worth watching again, +# bit disappointed in the end production, even though it is apparent it was shot on a low budget +# certain shots and sections in the film are of poor directed quality. +# +# prediction: negative +# +# target: negative +# + + +####################################################################### +# Generate Translations +# --------------------- +# +# Finally, we can also use the model to generate English to German translations on the first batch of examples from the Multi30k +# test set using a beam size of 4. +# + +batch = next(iter(multi_dataloader)) +input_text = batch["english"] +target = batch["german"] +beam_size = 4 + +model_input = transform(input_text) +model_output = sequence_generator.generate(model_input, eos_idx=eos_idx, beam_size=beam_size) +output_text = transform.decode(model_output.tolist()) + +for i in range(multi_batch_size): + print(f"Example {i+1}:\n") + print(f"input_text: {input_text[i]}\n") + print(f"prediction: {output_text[i]}\n") + print(f"target: {target[i]}\n\n") + + +####################################################################### +# Translation Output +# ------------------ +# +# :: +# +# Example 1: +# +# input_text: translate English to German: A man in an orange hat starring at something. +# +# prediction: Ein Mann in einem orangen Hut, der an etwas schaut. +# +# target: Ein Mann mit einem orangefarbenen Hut, der etwas anstarrt. +# +# +# Example 2: +# +# input_text: translate English to German: A Boston Terrier is running on lush green grass in front of a white fence. +# +# prediction: Ein Boston Terrier läuft auf üppigem grünem Gras vor einem weißen Zaun. +# +# target: Ein Boston Terrier läuft über saftig-grünes Gras vor einem weißen Zaun. +# +# +# Example 3: +# +# input_text: translate English to German: A girl in karate uniform breaking a stick with a front kick. +# +# prediction: Ein Mädchen in Karate-Uniform bricht einen Stöck mit einem Frontkick. +# +# target: Ein Mädchen in einem Karateanzug bricht ein Brett mit einem Tritt. +# +# +# Example 4: +# +# input_text: translate English to German: Five people wearing winter jackets and helmets stand in the snow, with snowmobiles in the background. +# +# prediction: Fünf Menschen mit Winterjacken und Helmen stehen im Schnee, mit Schneemobilen im Hintergrund. +# +# target: Fünf Leute in Winterjacken und mit Helmen stehen im Schnee mit Schneemobilen im Hintergrund. +# +# +# Example 5: +# +# input_text: translate English to German: People are fixing the roof of a house. +# +# prediction: Die Leute fixieren das Dach eines Hauses. +# +# target: Leute Reparieren das Dach eines Hauses. +# From c5c36403aca20018b97119510b76a7cced754a29 Mon Sep 17 00:00:00 2001 From: Joe Cummings Date: Mon, 27 Feb 2023 17:13:47 -0500 Subject: [PATCH 17/35] Remove weird author formatting --- beginner_source/t5_tutorial.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/beginner_source/t5_tutorial.py b/beginner_source/t5_tutorial.py index d6165a20096..828d09dffd3 100644 --- a/beginner_source/t5_tutorial.py +++ b/beginner_source/t5_tutorial.py @@ -2,8 +2,7 @@ T5-Base Model for Summarization, Sentiment Classification, and Translation ========================================================================== -**Author**: `Pendo Abbo `__ -**Author**: `Joe Cummings `__ +**Author**: `Pendo Abbo `__ , `Joe Cummings `__ """ From 892da0588012ca8a8e59e8b4775cafbd2d3f1682 Mon Sep 17 00:00:00 2001 From: Joe Cummings Date: Thu, 2 Mar 2023 10:51:22 -0500 Subject: [PATCH 18/35] Update beginner_source/t5_tutorial.py Co-authored-by: Svetlana Karslioglu --- beginner_source/t5_tutorial.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beginner_source/t5_tutorial.py b/beginner_source/t5_tutorial.py index 828d09dffd3..6223313f672 100644 --- a/beginner_source/t5_tutorial.py +++ b/beginner_source/t5_tutorial.py @@ -63,7 +63,7 @@ ####################################################################### # Alternatively, we can also use the transform shipped with the pre-trained models that does all of the above out-of-the-box # -# :: +# .. code-block:: # # from torchtext.models import T5_BASE_GENERATION # transform = T5_BASE_GENERATION.transform() From 0cbdb8c9d8881bc6645c832e6cb44fc6bb80e095 Mon Sep 17 00:00:00 2001 From: Joe Cummings Date: Thu, 2 Mar 2023 10:51:28 -0500 Subject: [PATCH 19/35] Update beginner_source/t5_tutorial.py Co-authored-by: Svetlana Karslioglu --- beginner_source/t5_tutorial.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beginner_source/t5_tutorial.py b/beginner_source/t5_tutorial.py index 6223313f672..6502e195cf1 100644 --- a/beginner_source/t5_tutorial.py +++ b/beginner_source/t5_tutorial.py @@ -42,7 +42,7 @@ # # T5 uses a SentencePiece model for text tokenization. Below, we use a pre-trained SentencePiece model to build # the text pre-processing pipeline using torchtext's T5Transform. Note that the transform supports both -# batched and non-batched text input (i.e. one can either pass a single sentence or a list of sentences), however +# batched and non-batched text input (for example, one can either pass a single sentence or a list of sentences), however # the T5 model expects the input to be batched. # From df5625f16aa85cac5303a57a7a018d3237bb4c00 Mon Sep 17 00:00:00 2001 From: Joe Cummings Date: Thu, 2 Mar 2023 10:51:37 -0500 Subject: [PATCH 20/35] Update beginner_source/t5_tutorial.py Co-authored-by: Svetlana Karslioglu --- beginner_source/t5_tutorial.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beginner_source/t5_tutorial.py b/beginner_source/t5_tutorial.py index 6502e195cf1..2e134dce94b 100644 --- a/beginner_source/t5_tutorial.py +++ b/beginner_source/t5_tutorial.py @@ -76,7 +76,7 @@ # # torchtext provides SOTA pre-trained models that can be used directly for NLP tasks or fine-tuned on downstream tasks. Below # we use the pre-trained T5 model with standard base configuration to perform text summarization, sentiment classification, and -# translation. For additional details on available pre-trained models, please refer to documentation at +# translation. For additional details on available pre-trained models, see `the torchtext documentation `__ # https://pytorch.org/text/main/models.html # # From 6aa02c50e6d62b82f79eaea418ef1448128867d1 Mon Sep 17 00:00:00 2001 From: Joe Cummings Date: Thu, 2 Mar 2023 10:51:45 -0500 Subject: [PATCH 21/35] Update beginner_source/t5_tutorial.py Co-authored-by: Svetlana Karslioglu --- beginner_source/t5_tutorial.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beginner_source/t5_tutorial.py b/beginner_source/t5_tutorial.py index 2e134dce94b..3e4f67d1027 100644 --- a/beginner_source/t5_tutorial.py +++ b/beginner_source/t5_tutorial.py @@ -118,7 +118,7 @@ # split. Below we demo on the test split. # # The T5 model uses the prefix "summarize" for text summarization. For more information on task -# prefixes, please visit Appendix D of the T5 Paper at https://arxiv.org/pdf/1910.10683.pdf +# prefixes, please visit Appendix D of the `T5 Paper `__ # # .. note:: # Using datapipes is still currently subject to a few caveats. If you wish From 88f0ad9fef717aa6d309f385f4ca1c29c791aa3b Mon Sep 17 00:00:00 2001 From: Joe Cummings Date: Thu, 2 Mar 2023 10:51:51 -0500 Subject: [PATCH 22/35] Update beginner_source/t5_tutorial.py Co-authored-by: Svetlana Karslioglu --- beginner_source/t5_tutorial.py | 1 - 1 file changed, 1 deletion(-) diff --git a/beginner_source/t5_tutorial.py b/beginner_source/t5_tutorial.py index 3e4f67d1027..d6b7c08a34c 100644 --- a/beginner_source/t5_tutorial.py +++ b/beginner_source/t5_tutorial.py @@ -77,7 +77,6 @@ # torchtext provides SOTA pre-trained models that can be used directly for NLP tasks or fine-tuned on downstream tasks. Below # we use the pre-trained T5 model with standard base configuration to perform text summarization, sentiment classification, and # translation. For additional details on available pre-trained models, see `the torchtext documentation `__ -# https://pytorch.org/text/main/models.html # # from torchtext.models import T5_BASE_GENERATION From 6d0bb1c1a77aad48c771c6ac7a6b1c1a26692d23 Mon Sep 17 00:00:00 2001 From: Joe Cummings Date: Thu, 2 Mar 2023 10:51:59 -0500 Subject: [PATCH 23/35] Update beginner_source/t5_tutorial.py Co-authored-by: Svetlana Karslioglu --- beginner_source/t5_tutorial.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beginner_source/t5_tutorial.py b/beginner_source/t5_tutorial.py index d6b7c08a34c..3190f7cd0e7 100644 --- a/beginner_source/t5_tutorial.py +++ b/beginner_source/t5_tutorial.py @@ -93,7 +93,7 @@ # GenerationUtils # ------------------ # -# We can use torchtext's `GenerationUtils` to produce an output sequence based on the input sequence provided. This calls on the +# We can use torchtext's ``GenerationUtils`` to produce an output sequence based on the input sequence provided. This calls on the # model's encoder and decoder, and iteratively expands the decoded sequences until the end-of-sequence token is generated # for all sequences in the batch. The `generate` method shown below uses greedy search to generate the sequences. Beam search and # other decoding strategies are also supported. From 726df9e7ab8e50c526c2781cc474afe98c65ee29 Mon Sep 17 00:00:00 2001 From: Joe Cummings Date: Thu, 2 Mar 2023 10:52:09 -0500 Subject: [PATCH 24/35] Update beginner_source/t5_tutorial.py Co-authored-by: Svetlana Karslioglu --- beginner_source/t5_tutorial.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beginner_source/t5_tutorial.py b/beginner_source/t5_tutorial.py index 3190f7cd0e7..42e87fe998e 100644 --- a/beginner_source/t5_tutorial.py +++ b/beginner_source/t5_tutorial.py @@ -95,7 +95,7 @@ # # We can use torchtext's ``GenerationUtils`` to produce an output sequence based on the input sequence provided. This calls on the # model's encoder and decoder, and iteratively expands the decoded sequences until the end-of-sequence token is generated -# for all sequences in the batch. The `generate` method shown below uses greedy search to generate the sequences. Beam search and +# for all sequences in the batch. The ``generate`` method shown below uses greedy search to generate the sequences. Beam search and # other decoding strategies are also supported. # # From 4d8510b108ea93f4ad20e3f3d71a141b61075e7b Mon Sep 17 00:00:00 2001 From: Joe Cummings Date: Thu, 2 Mar 2023 10:52:15 -0500 Subject: [PATCH 25/35] Update beginner_source/t5_tutorial.py Co-authored-by: Svetlana Karslioglu --- beginner_source/t5_tutorial.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beginner_source/t5_tutorial.py b/beginner_source/t5_tutorial.py index 42e87fe998e..d5501778b07 100644 --- a/beginner_source/t5_tutorial.py +++ b/beginner_source/t5_tutorial.py @@ -145,7 +145,7 @@ def apply_prefix(task, x): cnndm_dataloader = DataLoader(cnndm_datapipe, batch_size=None) ####################################################################### -# Alternately we can also use batched API (i.e apply the prefix on the whole batch) +# Alternately, we can also use batched API, for example, apply the prefix on the whole batch: # # :: # From bc2be9c25ee0e153f97933c5a6b36dc44bf77759 Mon Sep 17 00:00:00 2001 From: Joe Cummings Date: Thu, 2 Mar 2023 10:52:23 -0500 Subject: [PATCH 26/35] Update beginner_source/t5_tutorial.py Co-authored-by: Svetlana Karslioglu --- beginner_source/t5_tutorial.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beginner_source/t5_tutorial.py b/beginner_source/t5_tutorial.py index d5501778b07..4ec7739b9de 100644 --- a/beginner_source/t5_tutorial.py +++ b/beginner_source/t5_tutorial.py @@ -112,7 +112,7 @@ # datapipes and hence support standard flow-control and mapping/transformation using user defined # functions and transforms. # -# Below, we demonstrate how to pre-process the CNNDM dataset to include the prefix necessary for the +# Below we demonstrate how to pre-process the CNNDM dataset to include the prefix necessary for the # model to indentify the task it is performing. The CNNDM dataset has a train, validation, and test # split. Below we demo on the test split. # From fd2d47c76961f4ffa890620b0c9b857f91670cbb Mon Sep 17 00:00:00 2001 From: Joe Cummings Date: Thu, 2 Mar 2023 10:52:31 -0500 Subject: [PATCH 27/35] Update beginner_source/t5_tutorial.py Co-authored-by: Svetlana Karslioglu --- beginner_source/t5_tutorial.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beginner_source/t5_tutorial.py b/beginner_source/t5_tutorial.py index 4ec7739b9de..13cc3ff98f1 100644 --- a/beginner_source/t5_tutorial.py +++ b/beginner_source/t5_tutorial.py @@ -235,7 +235,7 @@ def process_labels(labels, x): # Summarization Output # -------------------- # -# :: +# .. code-block:: # # Example 1: # From 373776b2d4303c959493fff7c04d634d506d5631 Mon Sep 17 00:00:00 2001 From: Joe Cummings Date: Thu, 2 Mar 2023 10:52:38 -0500 Subject: [PATCH 28/35] Update beginner_source/t5_tutorial.py Co-authored-by: Svetlana Karslioglu --- beginner_source/t5_tutorial.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beginner_source/t5_tutorial.py b/beginner_source/t5_tutorial.py index 13cc3ff98f1..e8bc10c05a7 100644 --- a/beginner_source/t5_tutorial.py +++ b/beginner_source/t5_tutorial.py @@ -147,7 +147,7 @@ def apply_prefix(task, x): ####################################################################### # Alternately, we can also use batched API, for example, apply the prefix on the whole batch: # -# :: +# .. code-block:: # # def batch_prefix(task, x): # return { From 35de8851cb5e3e017e4c7c59d3bd15b574db1b4e Mon Sep 17 00:00:00 2001 From: Joe Cummings Date: Thu, 2 Mar 2023 10:53:55 -0500 Subject: [PATCH 29/35] Update beam_size name --- beginner_source/t5_tutorial.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beginner_source/t5_tutorial.py b/beginner_source/t5_tutorial.py index e8bc10c05a7..6e18b640f92 100644 --- a/beginner_source/t5_tutorial.py +++ b/beginner_source/t5_tutorial.py @@ -406,7 +406,7 @@ def process_labels(labels, x): beam_size = 4 model_input = transform(input_text) -model_output = sequence_generator.generate(model_input, eos_idx=eos_idx, beam_size=beam_size) +model_output = sequence_generator.generate(model_input, eos_idx=eos_idx, num_beams=beam_size) output_text = transform.decode(model_output.tolist()) for i in range(multi_batch_size): From cdd2af305e8ddcd239089631a60a7bff186c01c8 Mon Sep 17 00:00:00 2001 From: Joe Cummings Date: Thu, 2 Mar 2023 11:04:53 -0500 Subject: [PATCH 30/35] Remove space before comma --- beginner_source/t5_tutorial.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beginner_source/t5_tutorial.py b/beginner_source/t5_tutorial.py index 6e18b640f92..060e29cba8c 100644 --- a/beginner_source/t5_tutorial.py +++ b/beginner_source/t5_tutorial.py @@ -2,7 +2,7 @@ T5-Base Model for Summarization, Sentiment Classification, and Translation ========================================================================== -**Author**: `Pendo Abbo `__ , `Joe Cummings `__ +**Author**: `Pendo Abbo `__, `Joe Cummings `__ """ From a36650aec36e6351c3e34d52d90f6367220781bd Mon Sep 17 00:00:00 2001 From: Joe Cummings Date: Thu, 2 Mar 2023 12:21:02 -0500 Subject: [PATCH 31/35] Fix last two beam_size kw errors --- beginner_source/t5_tutorial.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beginner_source/t5_tutorial.py b/beginner_source/t5_tutorial.py index 060e29cba8c..1bada7736e1 100644 --- a/beginner_source/t5_tutorial.py +++ b/beginner_source/t5_tutorial.py @@ -222,7 +222,7 @@ def process_labels(labels, x): beam_size = 1 model_input = transform(input_text) -model_output = sequence_generator.generate(model_input, eos_idx=eos_idx, beam_size=beam_size) +model_output = sequence_generator.generate(model_input, eos_idx=eos_idx, num_beams=beam_size) output_text = transform.decode(model_output.tolist()) for i in range(cnndm_batch_size): @@ -312,7 +312,7 @@ def process_labels(labels, x): beam_size = 1 model_input = transform(input_text) -model_output = sequence_generator.generate(model_input, eos_idx=eos_idx, beam_size=beam_size) +model_output = sequence_generator.generate(model_input, eos_idx=eos_idx, num_beams=beam_size) output_text = transform.decode(model_output.tolist()) for i in range(imdb_batch_size): From 36d872cb8021d6ab332e3021d31ed17141c46ce5 Mon Sep 17 00:00:00 2001 From: Joe Cummings Date: Mon, 6 Mar 2023 14:25:58 -0500 Subject: [PATCH 32/35] Remove device settings --- beginner_source/t5_tutorial.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/beginner_source/t5_tutorial.py b/beginner_source/t5_tutorial.py index 1bada7736e1..76923382b05 100644 --- a/beginner_source/t5_tutorial.py +++ b/beginner_source/t5_tutorial.py @@ -20,14 +20,6 @@ # # -###################################################################### -# Common imports -# -------------- -import torch - -DEVICE = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu") - - ####################################################################### # Data Transformation # ------------------- @@ -86,7 +78,6 @@ transform = t5_base.transform() model = t5_base.get_model() model.eval() -model.to(DEVICE) ####################################################################### From b937af607130bb6177a751170d60cfa079b5766b Mon Sep 17 00:00:00 2001 From: Joe Cummings Date: Mon, 6 Mar 2023 15:27:56 -0500 Subject: [PATCH 33/35] Update IMDB dataloader mapping --- beginner_source/t5_tutorial.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/beginner_source/t5_tutorial.py b/beginner_source/t5_tutorial.py index 76923382b05..d2ea7148081 100644 --- a/beginner_source/t5_tutorial.py +++ b/beginner_source/t5_tutorial.py @@ -168,11 +168,11 @@ def apply_prefix(task, x): imdb_batch_size = 3 imdb_datapipe = IMDB(split="test") task = "sst2 sentence" -labels = {"neg": "negative", "pos": "positive"} +labels = {"1": "negative", "2": "positive"} def process_labels(labels, x): - return x[1], labels[x[0]] + return x[1], labels[str(x[0])] imdb_datapipe = imdb_datapipe.map(partial(process_labels, labels)) @@ -361,7 +361,7 @@ def process_labels(labels, x): # really annoying was the constant cuts to VDs daughter during the last fight scene.

# Not bad. Not good. Passable 4. # -# prediction: negative +# prediction: positive # # target: negative # @@ -388,13 +388,12 @@ def process_labels(labels, x): # --------------------- # # Finally, we can also use the model to generate English to German translations on the first batch of examples from the Multi30k -# test set using a beam size of 4. +# test set. # batch = next(iter(multi_dataloader)) input_text = batch["english"] target = batch["german"] -beam_size = 4 model_input = transform(input_text) model_output = sequence_generator.generate(model_input, eos_idx=eos_idx, num_beams=beam_size) From 1db14e49909e6dfb3bd0dc6956ff707870ae2c15 Mon Sep 17 00:00:00 2001 From: Joe Cummings Date: Tue, 7 Mar 2023 09:56:26 -0500 Subject: [PATCH 34/35] Shuffle CNNDM dataloader --- beginner_source/t5_tutorial.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/beginner_source/t5_tutorial.py b/beginner_source/t5_tutorial.py index d2ea7148081..3741454edf0 100644 --- a/beginner_source/t5_tutorial.py +++ b/beginner_source/t5_tutorial.py @@ -133,7 +133,7 @@ def apply_prefix(task, x): cnndm_datapipe = cnndm_datapipe.map(partial(apply_prefix, task)) cnndm_datapipe = cnndm_datapipe.batch(cnndm_batch_size) cnndm_datapipe = cnndm_datapipe.rows2columnar(["article", "abstract"]) -cnndm_dataloader = DataLoader(cnndm_datapipe, batch_size=None) +cnndm_dataloader = DataLoader(cnndm_datapipe, shuffle=True, batch_size=None) ####################################################################### # Alternately, we can also use batched API, for example, apply the prefix on the whole batch: @@ -223,21 +223,19 @@ def process_labels(labels, x): ####################################################################### -# Summarization Output +# Summarization Output (Might vary since we shuffle the dataloader) # -------------------- # # .. code-block:: # # Example 1: # -# prediction: the Palestinians become the 123rd member of the international criminal -# court . the accession was marked by a ceremony at the Hague, where the court is based . -# the ICC opened a preliminary examination into the situation in the occupied -# Palestinian territory . +# prediction: the 24-year-old has been tattooed for over a decade . he has landed in australia +# to start work on a new campaign . he says he is 'taking it in your stride' to be honest . # -# target: Membership gives the ICC jurisdiction over alleged crimes committed in -# Palestinian territories since last June . Israel and the United States opposed the -# move, which could open the door to war crimes investigations against Israelis . +# target: London-based model Stephen James Hendry famed for his full body tattoo . The supermodel +# is in Sydney for a new modelling campaign . Australian fans understood to have already located +# him at his hotel . The 24-year-old heartthrob is recently single . # # # Example 2: From d273aaa645ff3f16ff0cee41db6d957f4dac86fb Mon Sep 17 00:00:00 2001 From: Nikita Shulga Date: Thu, 16 Mar 2023 16:19:00 -0700 Subject: [PATCH 35/35] Apply suggestions from code review --- .jenkins/build.sh | 7 ------- .jenkins/validate_tutorials_built.py | 6 ------ beginner_source/introyt/autogradyt_tutorial.py | 2 +- 3 files changed, 1 insertion(+), 14 deletions(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index 6176e4bf4a8..d09b0a8782a 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -29,13 +29,6 @@ pip install -r $DIR/../requirements.txt # pip uninstall -y torch torchvision torchaudio torchtext # pip install --pre --upgrade -f https://download.pytorch.org/whl/test/cu116/torch_test.html torch torchdata torchvision torchaudio torchtext -# Install 2.0 nightly - -pip uninstall -y torch torchvision torchaudio torchtext torchdata -#pip install --pre --upgrade -f https://download.pytorch.org/whl/test/cu118/torch_test.html torch torchdata torchvision torchaudio torchtext -pip3 install torch torchvision torchaudio --no-cache-dir --index-url https://download.pytorch.org/whl/test/cu117 -pip3 install torchdata torchtext --index-url https://download.pytorch.org/whl/test/cpu - # Install two language tokenizers for Translation with TorchText tutorial python -m spacy download en_core_web_sm python -m spacy download de_core_news_sm diff --git a/.jenkins/validate_tutorials_built.py b/.jenkins/validate_tutorials_built.py index bedb92252e9..7811fa0509f 100644 --- a/.jenkins/validate_tutorials_built.py +++ b/.jenkins/validate_tutorials_built.py @@ -50,12 +50,6 @@ "recipes/Captum_Recipe", "flask_rest_api_tutorial", "text_to_speech_with_torchaudio", - "forward_ad_usage", # remove after #2218 is resolved. - "text_sentiment_ngrams_tutorial", # reenable after 2.0 release. - "tensorboard_profiler_tutorial", # reenable after 2.0 release. - "nvfuser_intro_tutorial", # reenable after 2.0 release. - "memory_format_tutorial" # reenable after 2.0 release. - ] diff --git a/beginner_source/introyt/autogradyt_tutorial.py b/beginner_source/introyt/autogradyt_tutorial.py index 1ce49f3cd03..19c25c5e81d 100644 --- a/beginner_source/introyt/autogradyt_tutorial.py +++ b/beginner_source/introyt/autogradyt_tutorial.py @@ -334,7 +334,7 @@ def forward(self, x): print(model.layer2.weight.grad[0][0:10]) -optimizer.zero_grad(set_to_none=False) +optimizer.zero_grad() print(model.layer2.weight.grad[0][0:10])