From 2ecea0729248a22b3a2c452f68f8ff2e360bfe80 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Fri, 24 Feb 2023 11:19:51 +0100 Subject: [PATCH 1/7] add gallery example for datapoints --- gallery/plot_datapoints.py | 122 +++++++++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 gallery/plot_datapoints.py diff --git a/gallery/plot_datapoints.py b/gallery/plot_datapoints.py new file mode 100644 index 00000000000..e74638a1b6c --- /dev/null +++ b/gallery/plot_datapoints.py @@ -0,0 +1,122 @@ +""" +============== +Datapoints FAQ +============== + +The :mod:`torchvision.datapoints` namespace was introduced to ``torchvision.transforms.v2``. This example showcases what +these datapoints are and how they behave. +""" + +import PIL.Image + +import torch +import torchvision + +# We are using BETA APIs, so we deactivate the associated warning, thereby acknowledging that +# some APIs may slightly change in the future +torchvision.disable_beta_transforms_warning() + +from torchvision import datapoints + + +######################################################################################################################## +# What are datapoints? +# -------------------- +# +# Datapoints are zero-copy tensor subclasses: + +tensor = torch.rand(3, 256, 256) +image = datapoints.Image(tensor) + +assert isinstance(image, torch.Tensor) +assert image.data_ptr() == tensor.data_ptr() + + +######################################################################################################################## +# They are needed in :mod:`torchvision.transforms.v2` for the automatic dispatch to the correct kernel. +# +# What datapoints are supported? +# ------------------------------ +# +# So far :mod:`torchvision.datapoints` supports four different datapoints: +# +# * :class:`~torchvision.datapoints.Image` +# * :class:`~torchvision.datapoints.Video` +# * :class:`~torchvision.datapoints.BoundingBox` +# * :class:`~torchvision.datapoints.Mask` +# +# How do I construct a datapoint? +# ------------------------------- +# +# Each datapoint class takes any tensor-like data that can be turned into a :class:`~torch.Tensor` + +image = datapoints.Image([[0, 1], [1, 0]]) +print(image) + + +######################################################################################################################## +# Similar to other PyTorch creations ops, the constructor also takes the ``dtype``, ``device``, and ``requires_grad`` +# parameters. + +float_image = datapoints.Image([[0, 1], [1, 0]], dtype=torch.float32, requires_grad=True) +print(float_image) + + +######################################################################################################################## +# In addition, :class:`~torchvision.datapoints.Image` and :class:`~torchvision.datapoints.Mask` also take a +# :class:`PIL.Image.Image` directly: + +image = datapoints.Image(PIL.Image.open("assets/astronaut.jpg")) +print(image.shape, image.dtype) + +######################################################################################################################## +# In general, the datapoints can also store additional metadata that complements the underlying tensor. For example, +# :class:`~torchvision.datapoints.BoundingBox` stores the format as well as the spatial size of the corresponding image +# alongside the actual values: + +bounding_box = datapoints.BoundingBox( + [17, 16, 344, 495], format=datapoints.BoundingBoxFormat.XYXY, spatial_size=image.shape[-2:] +) +print(bounding_box) + + +######################################################################################################################## +# Do I have to wrap the output of the datasets myself? +# ---------------------------------------------------- +# +# Only if you are using custom datasets. For the builtin ones, you can use +# :func:`torchvision.datasets.wrap_dataset_for_transforms_v2`. Note that the function also supports subclasses of the +# builtin datasets. Meaning, if your custom dataset subclasses from a builtin one and the output type is the same, you +# also don't have to wrap manually. +# +# How do the datapoints behave inside a computation? +# -------------------------------------------------- +# +# Datapoints look and feel just like regular tensors. Everything that is supported on plain :class:`torch.Tensor`'s +# also works on datapoints. +# Since for most operations involving datapoints, it cannot be safely inferred whether the result should retain the +# datapoint type, the result will be a plain tensor: + +assert type(image) is datapoints.Image + +new_image = image + 0 + +assert type(new_image) is torch.Tensor + +######################################################################################################################## +# There are two exceptions to this rule: +# +# 1. The operations :meth:`~torch.Tensor.clone`, :meth:`~torch.Tensor.to`, and :meth:`~torch.Tensor.requires_grad_` +# retain the datapoint type. +# 2. Inplace operations on datapoints cannot change the type of the datapoint they are called on. However, if you use +# the flow style, the returned value will be unwrapped: + +image = datapoints.Image([[0, 1], [1, 0]]) + +new_image = image.add_(1) + +assert type(image) is datapoints.Image +print(image) + +assert type(new_image) is torch.Tensor +assert (new_image == image).all() From 20b824ee3f28325da3df44a75bf4156d95459313 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Fri, 24 Feb 2023 13:31:08 +0100 Subject: [PATCH 2/7] Apply suggestions from code review Co-authored-by: vfdev --- gallery/plot_datapoints.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/gallery/plot_datapoints.py b/gallery/plot_datapoints.py index e74638a1b6c..890a2a3f6b8 100644 --- a/gallery/plot_datapoints.py +++ b/gallery/plot_datapoints.py @@ -38,7 +38,7 @@ # What datapoints are supported? # ------------------------------ # -# So far :mod:`torchvision.datapoints` supports four different datapoints: +# So far :mod:`torchvision.datapoints` supports four types of datapoints: # # * :class:`~torchvision.datapoints.Image` # * :class:`~torchvision.datapoints.Video` @@ -71,7 +71,7 @@ ######################################################################################################################## # In general, the datapoints can also store additional metadata that complements the underlying tensor. For example, -# :class:`~torchvision.datapoints.BoundingBox` stores the format as well as the spatial size of the corresponding image +# :class:`~torchvision.datapoints.BoundingBox` stores the coordinates format as well as the spatial size of the corresponding image # alongside the actual values: bounding_box = datapoints.BoundingBox( @@ -84,9 +84,9 @@ # Do I have to wrap the output of the datasets myself? # ---------------------------------------------------- # -# Only if you are using custom datasets. For the builtin ones, you can use +# Only if you are using custom datasets. For the built-in ones, you can use # :func:`torchvision.datasets.wrap_dataset_for_transforms_v2`. Note that the function also supports subclasses of the -# builtin datasets. Meaning, if your custom dataset subclasses from a builtin one and the output type is the same, you +# built-in datasets. Meaning, if your custom dataset subclasses from a built-in one and the output type is the same, you # also don't have to wrap manually. # # How do the datapoints behave inside a computation? @@ -95,7 +95,7 @@ # Datapoints look and feel just like regular tensors. Everything that is supported on plain :class:`torch.Tensor`'s # also works on datapoints. # Since for most operations involving datapoints, it cannot be safely inferred whether the result should retain the -# datapoint type, the result will be a plain tensor: +# datapoint type, we choose to return a plain tensor instead of a datapoint (this might change, see note below): assert type(image) is datapoints.Image From 0d59fdf71b0a1cd9ab181e335df442bb32589da2 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Fri, 24 Feb 2023 13:33:43 +0100 Subject: [PATCH 3/7] Update gallery/plot_datapoints.py Co-authored-by: Nicolas Hug --- gallery/plot_datapoints.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gallery/plot_datapoints.py b/gallery/plot_datapoints.py index 890a2a3f6b8..e709ce80d98 100644 --- a/gallery/plot_datapoints.py +++ b/gallery/plot_datapoints.py @@ -101,7 +101,8 @@ new_image = image + 0 -assert type(new_image) is torch.Tensor +assert isinstance(new_image, torch.Tensor) +assert not isinstance(new_image, datapoints.Image) ######################################################################################################################## # There are two exceptions to this rule: From 6f40d91b6d4b2e85ed79b4f9cf11c6e080fc9746 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Fri, 24 Feb 2023 13:44:26 +0100 Subject: [PATCH 4/7] address other comments --- gallery/plot_datapoints.py | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/gallery/plot_datapoints.py b/gallery/plot_datapoints.py index e709ce80d98..81b183b75cd 100644 --- a/gallery/plot_datapoints.py +++ b/gallery/plot_datapoints.py @@ -4,7 +4,10 @@ ============== The :mod:`torchvision.datapoints` namespace was introduced to ``torchvision.transforms.v2``. This example showcases what -these datapoints are and how they behave. +these datapoints are and how they behave. This is a fairly low-level topic that most users will not need to worry about: +you do not need to understand the internals of datapoints to efficiently rely on ``torchvision.transforms.v2``. It may +however be useful for advanced users trying to implement their own datasets, transforms, or work directly with the +datapoints. """ import PIL.Image @@ -50,7 +53,7 @@ # # Each datapoint class takes any tensor-like data that can be turned into a :class:`~torch.Tensor` -image = datapoints.Image([[0, 1], [1, 0]]) +image = datapoints.Image([[[[0, 1], [1, 0]]]]) print(image) @@ -58,7 +61,7 @@ # Similar to other PyTorch creations ops, the constructor also takes the ``dtype``, ``device``, and ``requires_grad`` # parameters. -float_image = datapoints.Image([[0, 1], [1, 0]], dtype=torch.float32, requires_grad=True) +float_image = datapoints.Image([[[0, 1], [1, 0]]], dtype=torch.float32, requires_grad=True) print(float_image) @@ -71,8 +74,8 @@ ######################################################################################################################## # In general, the datapoints can also store additional metadata that complements the underlying tensor. For example, -# :class:`~torchvision.datapoints.BoundingBox` stores the coordinates format as well as the spatial size of the corresponding image -# alongside the actual values: +# :class:`~torchvision.datapoints.BoundingBox` stores the coordinate format as well as the spatial size of the +# corresponding image alongside the actual values: bounding_box = datapoints.BoundingBox( [17, 16, 344, 495], format=datapoints.BoundingBoxFormat.XYXY, spatial_size=image.shape[-2:] @@ -92,17 +95,16 @@ # How do the datapoints behave inside a computation? # -------------------------------------------------- # -# Datapoints look and feel just like regular tensors. Everything that is supported on plain :class:`torch.Tensor`'s +# Datapoints look and feel just like regular tensors. Everything that is supported on a plain :class:`torch.Tensor` # also works on datapoints. # Since for most operations involving datapoints, it cannot be safely inferred whether the result should retain the # datapoint type, we choose to return a plain tensor instead of a datapoint (this might change, see note below): -assert type(image) is datapoints.Image +assert isinstance(image, datapoints.Image) new_image = image + 0 -assert isinstance(new_image, torch.Tensor) -assert not isinstance(new_image, datapoints.Image) +assert isinstance(new_image, torch.Tensor) and not isinstance(new_image, datapoints.Image) ######################################################################################################################## # There are two exceptions to this rule: @@ -112,12 +114,19 @@ # 2. Inplace operations on datapoints cannot change the type of the datapoint they are called on. However, if you use # the flow style, the returned value will be unwrapped: -image = datapoints.Image([[0, 1], [1, 0]]) +image = datapoints.Image([[[0, 1], [1, 0]]]) new_image = image.add_(1) -assert type(image) is datapoints.Image +assert isinstance(image, torch.Tensor) print(image) -assert type(new_image) is torch.Tensor +assert isinstance(new_image, torch.Tensor) and not isinstance(new_image, datapoints.Image) assert (new_image == image).all() + +######################################################################################################################## +# .. note:: +# +# This "unwrapping" behaviour is something we're actively seeking feedback on. If you find this surprising or if you +# have any suggestions on how to better support your use-cases, please reach out to us via this issue: +# https://github.com/pytorch/vision/issues/7319 From 5ce6912bf161a99f67ef1f4615b91abcb436e39b Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Fri, 24 Feb 2023 14:06:36 +0100 Subject: [PATCH 5/7] clarify flow style --- gallery/plot_datapoints.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gallery/plot_datapoints.py b/gallery/plot_datapoints.py index 81b183b75cd..660db3a5bc8 100644 --- a/gallery/plot_datapoints.py +++ b/gallery/plot_datapoints.py @@ -116,7 +116,7 @@ image = datapoints.Image([[[0, 1], [1, 0]]]) -new_image = image.add_(1) +new_image = image.add_(1).mul_(2) assert isinstance(image, torch.Tensor) print(image) From 5d480a4d2b678de9e42f0c22f6a8c1cbba0bf714 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Fri, 24 Feb 2023 14:09:12 +0100 Subject: [PATCH 6/7] nit --- gallery/plot_datapoints.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gallery/plot_datapoints.py b/gallery/plot_datapoints.py index 660db3a5bc8..bae86628d4f 100644 --- a/gallery/plot_datapoints.py +++ b/gallery/plot_datapoints.py @@ -36,7 +36,8 @@ ######################################################################################################################## -# They are needed in :mod:`torchvision.transforms.v2` for the automatic dispatch to the correct kernel. +# Under the hood, they are needed in :mod:`torchvision.transforms.v2` to correctly dispatch to the appropriate low-level +# kernel. # # What datapoints are supported? # ------------------------------ From c41ccbc6f68a169976b24a5fc62cb441137da42c Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Fri, 24 Feb 2023 15:00:52 +0100 Subject: [PATCH 7/7] address more comments --- gallery/plot_datapoints.py | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/gallery/plot_datapoints.py b/gallery/plot_datapoints.py index bae86628d4f..83ca6793598 100644 --- a/gallery/plot_datapoints.py +++ b/gallery/plot_datapoints.py @@ -3,11 +3,11 @@ Datapoints FAQ ============== -The :mod:`torchvision.datapoints` namespace was introduced to ``torchvision.transforms.v2``. This example showcases what -these datapoints are and how they behave. This is a fairly low-level topic that most users will not need to worry about: -you do not need to understand the internals of datapoints to efficiently rely on ``torchvision.transforms.v2``. It may -however be useful for advanced users trying to implement their own datasets, transforms, or work directly with the -datapoints. +The :mod:`torchvision.datapoints` namespace was introduced together with ``torchvision.transforms.v2``. This example +showcases what these datapoints are and how they behave. This is a fairly low-level topic that most users will not need +to worry about: you do not need to understand the internals of datapoints to efficiently rely on +``torchvision.transforms.v2``. It may however be useful for advanced users trying to implement their own datasets, +transforms, or work directly with the datapoints. """ import PIL.Image @@ -36,8 +36,8 @@ ######################################################################################################################## -# Under the hood, they are needed in :mod:`torchvision.transforms.v2` to correctly dispatch to the appropriate low-level -# kernel. +# Under the hood, they are needed in :mod:`torchvision.transforms.v2` to correctly dispatch to the appropriate function +# for the input data. # # What datapoints are supported? # ------------------------------ @@ -108,6 +108,12 @@ assert isinstance(new_image, torch.Tensor) and not isinstance(new_image, datapoints.Image) ######################################################################################################################## +# .. note:: +# +# This "unwrapping" behaviour is something we're actively seeking feedback on. If you find this surprising or if you +# have any suggestions on how to better support your use-cases, please reach out to us via this issue: +# https://github.com/pytorch/vision/issues/7319 +# # There are two exceptions to this rule: # # 1. The operations :meth:`~torch.Tensor.clone`, :meth:`~torch.Tensor.to`, and :meth:`~torch.Tensor.requires_grad_` @@ -124,10 +130,3 @@ assert isinstance(new_image, torch.Tensor) and not isinstance(new_image, datapoints.Image) assert (new_image == image).all() - -######################################################################################################################## -# .. note:: -# -# This "unwrapping" behaviour is something we're actively seeking feedback on. If you find this surprising or if you -# have any suggestions on how to better support your use-cases, please reach out to us via this issue: -# https://github.com/pytorch/vision/issues/7319