From e870ad74084097cb7239da99d230b0c3f1028869 Mon Sep 17 00:00:00 2001 From: fsx950223 Date: Thu, 19 Mar 2020 11:56:28 +0000 Subject: [PATCH 01/14] add cutout op --- tensorflow_addons/image/BUILD | 13 ++ tensorflow_addons/image/cutout_ops.py | 161 +++++++++++++++++++++ tensorflow_addons/image/cutout_ops_test.py | 69 +++++++++ 3 files changed, 243 insertions(+) create mode 100644 tensorflow_addons/image/cutout_ops.py create mode 100644 tensorflow_addons/image/cutout_ops_test.py diff --git a/tensorflow_addons/image/BUILD b/tensorflow_addons/image/BUILD index a1a09d66bf..c8358b372a 100644 --- a/tensorflow_addons/image/BUILD +++ b/tensorflow_addons/image/BUILD @@ -18,6 +18,7 @@ py_library( "connected_components.py", "resampler_ops.py", "compose_ops.py", + "cutout_ops.py", ]), data = [ ":sparse_image_warp_test_data", @@ -118,6 +119,18 @@ py_test( ], ) +py_test( + name = "cutout_ops_test", + size = "small", + srcs = [ + "cutout_ops_test.py", + ], + main = "cutout_ops_test.py", + deps = [ + ":image", + ], +) + py_test( name = "sparse_image_warp_test", size = "medium", diff --git a/tensorflow_addons/image/cutout_ops.py b/tensorflow_addons/image/cutout_ops.py new file mode 100644 index 0000000000..96d48e2411 --- /dev/null +++ b/tensorflow_addons/image/cutout_ops.py @@ -0,0 +1,161 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Cutout op""" + +import tensorflow as tf +from tensorflow_addons.utils.types import TensorLike, Number +from tensorflow_addons.image.utils import from_4D_image, to_4D_image +from tensorflow.python.keras.utils import conv_utils +import numpy as np + + +def _get_image_wh(images, data_format): + if tf.equal(tf.rank(images), 4): + if data_format == "channels_last": + image_height, image_width = tf.shape(images)[1], tf.shape(images)[2] + else: + image_height, image_width = tf.shape(images)[2], tf.shape(images)[3] + else: + image_height, image_width = tf.shape(images)[0], tf.shape(images)[1] + + return image_height, image_width + + +def random_cutout( + images: TensorLike, + mask_size: TensorLike, + constant_values: Number = 0, + seed: Number = None, + data_format: str = "channels_last", +) -> tf.Tensor: + """Apply cutout (https://arxiv.org/abs/1708.04552) to images. + + This operation applies a (2*pad_size[0] x 2*pad_size[1]) mask of zeros to + a random location within `img`. The pixel values filled in will be of the + value `replace`. The located where the mask will be applied is randomly + chosen uniformly over the whole images. + + Args: + images: A tensor of shape + (num_images, num_rows, num_columns, num_channels) + (NHWC), (num_images, num_channels, num_rows, num_columns)(NCHW), (num_rows, num_columns, num_channels) (HWC), or + (num_rows, num_columns) (HW). + mask_size: Specifies how big the zero mask that will be generated is that + is applied to the images. The mask will be of size + (2*pad_size[0] x 2*pad_size[1]). + constant_values: What pixel value to fill in the images in the area that has + the cutout mask applied to it. + seed: A Python integer. Used in combination with `tf.random.set_seed` to + create a reproducible sequence of tensors across multiple calls. + data_format: A string, one of `channels_last` (default) or `channels_first`. + The ordering of the dimensions in the inputs. + `channels_last` corresponds to inputs with shape + `(batch_size, ..., channels)` while `channels_first` corresponds to + inputs with shape `(batch_size, channels, ...)`. + Returns: + An image Tensor. + """ + if np.isscalar(mask_size): + mask_size = [mask_size, mask_size] + data_format = conv_utils.normalize_data_format(data_format) + # Sample the center location in the images where the zero mask will be applied. + image_height, image_width = _get_image_wh(images, data_format) + cutout_center_height = tf.random.uniform( + shape=[], minval=0, maxval=image_height, dtype=tf.int32, seed=seed + ) + cutout_center_width = tf.random.uniform( + shape=[], minval=0, maxval=image_width, dtype=tf.int32, seed=seed + ) + return cutout( + images, + mask_size, + [cutout_center_height, cutout_center_width], + constant_values, + data_format, + ) + + +def cutout( + images: TensorLike, + mask_size: TensorLike, + offset: TensorLike, + constant_values: Number = 0, + data_format: str = "channels_last", +) -> tf.Tensor: + """Apply cutout (https://arxiv.org/abs/1708.04552) to images. + + This operation applies a (2*pad_size[0] x 2*pad_size[1]) mask of zeros to + a random location within `img`. The pixel values filled in will be of the + value `replace`. The located where the mask will be applied is randomly + chosen uniformly over the whole images. + + Args: + images: A tensor of shape + (num_images, num_rows, num_columns, num_channels) + (NHWC), (num_images, num_channels, num_rows, num_columns)(NCHW), (num_rows, num_columns, num_channels) (HWC), or + (num_rows, num_columns) (HW). + mask_size: Specifies how big the zero mask that will be generated is that + is applied to the images. The mask will be of size + (2*pad_size[0] x 2*pad_size[1]). + offset: A tuple of (height, width) + constant_values: What pixel value to fill in the images in the area that has + the cutout mask applied to it. + data_format: A string, one of `channels_last` (default) or `channels_first`. + The ordering of the dimensions in the inputs. + `channels_last` corresponds to inputs with shape + `(batch_size, ..., channels)` while `channels_first` corresponds to + inputs with shape `(batch_size, channels, ...)`. + Returns: + An image Tensor. + """ + with tf.name_scope("cutout"): + if np.isscalar(mask_size): + mask_size = [mask_size, mask_size] + data_format = conv_utils.normalize_data_format(data_format) + image_height, image_width = _get_image_wh(images, data_format) + cutout_center_height = offset[0] + cutout_center_width = offset[1] + + lower_pad = tf.maximum(0, cutout_center_height - mask_size[0]) + upper_pad = tf.maximum(0, image_height - cutout_center_height - mask_size[0]) + left_pad = tf.maximum(0, cutout_center_width - mask_size[1]) + right_pad = tf.maximum(0, image_width - cutout_center_width - mask_size[1]) + + cutout_shape = [ + image_height - (lower_pad + upper_pad), + image_width - (left_pad + right_pad), + ] + padding_dims = [[lower_pad, upper_pad], [left_pad, right_pad]] + mask = tf.pad( + tf.zeros(cutout_shape, dtype=images.dtype), padding_dims, constant_values=1 + ) + mask_4d = to_4D_image(mask) + if tf.equal(tf.rank(images), 3): + mask = tf.tile(from_4D_image(mask_4d, 3), [1, 1, tf.shape(images)[-1]]) + elif tf.equal(tf.rank(images), 4): + if data_format == "channels_last": + mask = tf.tile( + mask_4d, [tf.shape(images)[0], 1, 1, tf.shape(images)[-1]] + ) + else: + mask = tf.tile( + mask_4d, [tf.shape(images)[0], tf.shape(images)[1], 1, 1] + ) + images = tf.where( + tf.equal(mask, 0), + tf.ones_like(images, dtype=images.dtype) * constant_values, + images, + ) + return images diff --git a/tensorflow_addons/image/cutout_ops_test.py b/tensorflow_addons/image/cutout_ops_test.py new file mode 100644 index 0000000000..6c3b7c767f --- /dev/null +++ b/tensorflow_addons/image/cutout_ops_test.py @@ -0,0 +1,69 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for cutout.""" + +import sys + +import pytest +import tensorflow as tf +import numpy as np +from absl.testing import parameterized +from tensorflow_addons.image.cutout_ops import cutout, random_cutout +from tensorflow_addons.utils import test_utils +from tensorflow_addons.image.utils import from_4D_image, to_4D_image + + +@parameterized.named_parameters( + ("float16", np.float16), ("float32", np.float32), ("uint8", np.uint8) +) +@pytest.mark.usefixtures("maybe_run_functions_eagerly") +def test_different_dtypes(dtype): + test_image = tf.ones([1, 40, 40, 1], dtype=dtype) + result_image = cutout(test_image, 2, [2, 2]) + cutout_area = tf.zeros([4, 4], dtype=dtype) + cutout_area = tf.pad(cutout_area, ((0, 36), (0, 36)), constant_values=1) + expect_image = to_4D_image(cutout_area) + np.testing.assert_allclose(result_image, expect_image) + + +@pytest.mark.usefixtures("maybe_run_functions_eagerly") +def test_different_channels(): + test_image = tf.ones([1, 40, 40, 1], dtype=np.uint8) + cutout_area = tf.zeros([4, 4], dtype=np.uint8) + cutout_area = tf.pad(cutout_area, ((0, 36), (0, 36)), constant_values=1) + expect_image = to_4D_image(cutout_area) + for channel in [0, 1, 3, 4]: + result_image = random_cutout(test_image, 20, seed=1234) + np.testing.assert_allclose(tf.shape(result_image), tf.shape(expect_image)) + + +@pytest.mark.usefixtures("maybe_run_functions_eagerly") +def test_different_ranks(): + test_image_4d = tf.ones([1, 40, 40, 1], dtype=np.uint8) + cutout_area = tf.zeros([4, 4], dtype=np.uint8) + cutout_area = tf.pad(cutout_area, ((0, 36), (0, 36)), constant_values=1) + expect_image_4d = to_4D_image(cutout_area) + + test_image_2d = from_4D_image(test_image_4d, 2) + expect_image_2d = from_4D_image(expect_image_4d, 2) + result_image_2d = random_cutout(test_image_2d, 20, seed=1234) + np.testing.assert_allclose(tf.shape(result_image_2d), tf.shape(expect_image_2d)) + + result_image_4d = random_cutout(test_image_4d, 20, seed=1234) + np.testing.assert_allclose(tf.shape(result_image_4d), tf.shape(expect_image_4d)) + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__])) From 5203f3e8a7d679b52972e8a95b86b0af56aa7c1e Mon Sep 17 00:00:00 2001 From: fsx950223 Date: Thu, 19 Mar 2020 11:59:09 +0000 Subject: [PATCH 02/14] export module --- .github/CODEOWNERS | 1 + tensorflow_addons/image/__init__.py | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 6ce517bc57..dfccefe294 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -37,6 +37,7 @@ /tensorflow_addons/callbacks/tqdm_progress_bar*.py @shun-lin /tensorflow_addons/image/connected_components*.py @sayoojbk +/tensorflow_addons/image/cutout_ops*.py @fsx950223 /tensorflow_addons/image/dense_image_warp*.py @windQAQ /tensorflow_addons/image/distance_transform*.py @mels630 /tensorflow_addons/image/distort_image_ops*.py @windqaq diff --git a/tensorflow_addons/image/__init__.py b/tensorflow_addons/image/__init__.py index fbd5cda029..891d9d05f0 100644 --- a/tensorflow_addons/image/__init__.py +++ b/tensorflow_addons/image/__init__.py @@ -29,3 +29,4 @@ from tensorflow_addons.image.transform_ops import transform from tensorflow_addons.image.translate_ops import translate from tensorflow_addons.image.compose_ops import blend +from tensorflow_addons.image.cutout_ops import random_cutout, cutout From 43c8e2d6a2c58cec66e722a2e0f9bf0ee10b1209 Mon Sep 17 00:00:00 2001 From: fsx950223 Date: Thu, 19 Mar 2020 12:05:57 +0000 Subject: [PATCH 03/14] remove test_utils --- tensorflow_addons/image/cutout_ops_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tensorflow_addons/image/cutout_ops_test.py b/tensorflow_addons/image/cutout_ops_test.py index 6c3b7c767f..cca26a1683 100644 --- a/tensorflow_addons/image/cutout_ops_test.py +++ b/tensorflow_addons/image/cutout_ops_test.py @@ -21,7 +21,6 @@ import numpy as np from absl.testing import parameterized from tensorflow_addons.image.cutout_ops import cutout, random_cutout -from tensorflow_addons.utils import test_utils from tensorflow_addons.image.utils import from_4D_image, to_4D_image From edaa30ce1b0cc8e15cb43ee3ab9e83e6d8653fde Mon Sep 17 00:00:00 2001 From: fsx950223 Date: Thu, 19 Mar 2020 23:45:12 +0800 Subject: [PATCH 04/14] use tf.rank --- tensorflow_addons/image/cutout_ops.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tensorflow_addons/image/cutout_ops.py b/tensorflow_addons/image/cutout_ops.py index 96d48e2411..0a70bc2ba2 100644 --- a/tensorflow_addons/image/cutout_ops.py +++ b/tensorflow_addons/image/cutout_ops.py @@ -18,7 +18,6 @@ from tensorflow_addons.utils.types import TensorLike, Number from tensorflow_addons.image.utils import from_4D_image, to_4D_image from tensorflow.python.keras.utils import conv_utils -import numpy as np def _get_image_wh(images, data_format): @@ -67,7 +66,7 @@ def random_cutout( Returns: An image Tensor. """ - if np.isscalar(mask_size): + if tf.equal(tf.rank(mask_size), 0): mask_size = [mask_size, mask_size] data_format = conv_utils.normalize_data_format(data_format) # Sample the center location in the images where the zero mask will be applied. @@ -121,7 +120,7 @@ def cutout( An image Tensor. """ with tf.name_scope("cutout"): - if np.isscalar(mask_size): + if tf.equal(tf.rank(mask_size), 0): mask_size = [mask_size, mask_size] data_format = conv_utils.normalize_data_format(data_format) image_height, image_width = _get_image_wh(images, data_format) From 8be97a0453a18287d76b3e213fabb29c12593ed7 Mon Sep 17 00:00:00 2001 From: fsx950223 Date: Fri, 20 Mar 2020 01:47:55 +0000 Subject: [PATCH 05/14] remove decorator --- tensorflow_addons/image/cutout_ops_test.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tensorflow_addons/image/cutout_ops_test.py b/tensorflow_addons/image/cutout_ops_test.py index cca26a1683..f0a73e81a7 100644 --- a/tensorflow_addons/image/cutout_ops_test.py +++ b/tensorflow_addons/image/cutout_ops_test.py @@ -27,7 +27,6 @@ @parameterized.named_parameters( ("float16", np.float16), ("float32", np.float32), ("uint8", np.uint8) ) -@pytest.mark.usefixtures("maybe_run_functions_eagerly") def test_different_dtypes(dtype): test_image = tf.ones([1, 40, 40, 1], dtype=dtype) result_image = cutout(test_image, 2, [2, 2]) @@ -37,7 +36,6 @@ def test_different_dtypes(dtype): np.testing.assert_allclose(result_image, expect_image) -@pytest.mark.usefixtures("maybe_run_functions_eagerly") def test_different_channels(): test_image = tf.ones([1, 40, 40, 1], dtype=np.uint8) cutout_area = tf.zeros([4, 4], dtype=np.uint8) @@ -48,7 +46,6 @@ def test_different_channels(): np.testing.assert_allclose(tf.shape(result_image), tf.shape(expect_image)) -@pytest.mark.usefixtures("maybe_run_functions_eagerly") def test_different_ranks(): test_image_4d = tf.ones([1, 40, 40, 1], dtype=np.uint8) cutout_area = tf.zeros([4, 4], dtype=np.uint8) From f057afff50f9239dad7c421cdaf892270bf2cc7f Mon Sep 17 00:00:00 2001 From: fsx950223 Date: Fri, 20 Mar 2020 03:53:46 +0000 Subject: [PATCH 06/14] add tf function test --- tensorflow_addons/image/cutout_ops.py | 6 ++++-- tensorflow_addons/image/cutout_ops_test.py | 9 +++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/tensorflow_addons/image/cutout_ops.py b/tensorflow_addons/image/cutout_ops.py index 0a70bc2ba2..8aa28acef4 100644 --- a/tensorflow_addons/image/cutout_ops.py +++ b/tensorflow_addons/image/cutout_ops.py @@ -66,8 +66,9 @@ def random_cutout( Returns: An image Tensor. """ + mask_size = tf.convert_to_tensor(mask_size) if tf.equal(tf.rank(mask_size), 0): - mask_size = [mask_size, mask_size] + mask_size = tf.stack([mask_size, mask_size]) data_format = conv_utils.normalize_data_format(data_format) # Sample the center location in the images where the zero mask will be applied. image_height, image_width = _get_image_wh(images, data_format) @@ -120,8 +121,9 @@ def cutout( An image Tensor. """ with tf.name_scope("cutout"): + mask_size = tf.convert_to_tensor(mask_size) if tf.equal(tf.rank(mask_size), 0): - mask_size = [mask_size, mask_size] + mask_size = tf.stack([mask_size, mask_size]) data_format = conv_utils.normalize_data_format(data_format) image_height, image_width = _get_image_wh(images, data_format) cutout_center_height = offset[0] diff --git a/tensorflow_addons/image/cutout_ops_test.py b/tensorflow_addons/image/cutout_ops_test.py index f0a73e81a7..fa6b2df472 100644 --- a/tensorflow_addons/image/cutout_ops_test.py +++ b/tensorflow_addons/image/cutout_ops_test.py @@ -61,5 +61,14 @@ def test_different_ranks(): np.testing.assert_allclose(tf.shape(result_image_4d), tf.shape(expect_image_4d)) +def test_with_tf_function(): + test_image = tf.ones([1, 40, 40, 1], dtype=tf.uint8) + result_image = tf.function(cutout)(test_image, 2, [2, 2]) + cutout_area = tf.zeros([4, 4], dtype=tf.uint8) + cutout_area = tf.pad(cutout_area, ((0, 36), (0, 36)), constant_values=1) + expect_image = to_4D_image(cutout_area) + np.testing.assert_allclose(result_image, expect_image) + + if __name__ == "__main__": sys.exit(pytest.main([__file__])) From 7723f22e202dc1ee5bf930cc028dfea565a66a0e Mon Sep 17 00:00:00 2001 From: fsx950223 Date: Fri, 20 Mar 2020 22:15:17 +0800 Subject: [PATCH 07/14] fix cutout channels test --- tensorflow_addons/image/cutout_ops_test.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tensorflow_addons/image/cutout_ops_test.py b/tensorflow_addons/image/cutout_ops_test.py index fa6b2df472..6d9ce999d7 100644 --- a/tensorflow_addons/image/cutout_ops_test.py +++ b/tensorflow_addons/image/cutout_ops_test.py @@ -37,11 +37,12 @@ def test_different_dtypes(dtype): def test_different_channels(): - test_image = tf.ones([1, 40, 40, 1], dtype=np.uint8) - cutout_area = tf.zeros([4, 4], dtype=np.uint8) - cutout_area = tf.pad(cutout_area, ((0, 36), (0, 36)), constant_values=1) - expect_image = to_4D_image(cutout_area) for channel in [0, 1, 3, 4]: + test_image = tf.ones([1, 40, 40, channel], dtype=np.uint8) + cutout_area = tf.zeros([4, 4], dtype=np.uint8) + cutout_area = tf.pad(cutout_area, ((0, 36), (0, 36)), constant_values=1) + expect_image = to_4D_image(cutout_area) + expect_image = tf.tile(expect_image, [1, 1, 1, channel]) result_image = random_cutout(test_image, 20, seed=1234) np.testing.assert_allclose(tf.shape(result_image), tf.shape(expect_image)) From 40e31d3065a7a8c125a0b26a062ed2608edd16f5 Mon Sep 17 00:00:00 2001 From: fsx950223 Date: Mon, 23 Mar 2020 11:18:51 +0000 Subject: [PATCH 08/14] add norm param --- tensorflow_addons/image/cutout_ops.py | 30 ++++++++++++++++----------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/tensorflow_addons/image/cutout_ops.py b/tensorflow_addons/image/cutout_ops.py index 8aa28acef4..e94bc8b281 100644 --- a/tensorflow_addons/image/cutout_ops.py +++ b/tensorflow_addons/image/cutout_ops.py @@ -32,6 +32,15 @@ def _get_image_wh(images, data_format): return image_height, image_width +def _norm_params(images, mask_size, data_format): + mask_size = tf.convert_to_tensor(mask_size) + if tf.equal(tf.rank(mask_size), 0): + mask_size = tf.stack([mask_size, mask_size]) + data_format = conv_utils.normalize_data_format(data_format) + image_height, image_width = _get_image_wh(images, data_format) + return mask_size, data_format, image_height, image_width + + def random_cutout( images: TensorLike, mask_size: TensorLike, @@ -66,12 +75,10 @@ def random_cutout( Returns: An image Tensor. """ - mask_size = tf.convert_to_tensor(mask_size) - if tf.equal(tf.rank(mask_size), 0): - mask_size = tf.stack([mask_size, mask_size]) - data_format = conv_utils.normalize_data_format(data_format) - # Sample the center location in the images where the zero mask will be applied. - image_height, image_width = _get_image_wh(images, data_format) + mask_size, data_format, image_height, image_width = _norm_params( + images, mask_size, data_format + ) + cutout_center_height = tf.random.uniform( shape=[], minval=0, maxval=image_height, dtype=tf.int32, seed=seed ) @@ -90,7 +97,7 @@ def random_cutout( def cutout( images: TensorLike, mask_size: TensorLike, - offset: TensorLike, + offset: TensorLike = (0, 0), constant_values: Number = 0, data_format: str = "channels_last", ) -> tf.Tensor: @@ -121,11 +128,10 @@ def cutout( An image Tensor. """ with tf.name_scope("cutout"): - mask_size = tf.convert_to_tensor(mask_size) - if tf.equal(tf.rank(mask_size), 0): - mask_size = tf.stack([mask_size, mask_size]) - data_format = conv_utils.normalize_data_format(data_format) - image_height, image_width = _get_image_wh(images, data_format) + mask_size, data_format, image_height, image_width = _norm_params( + images, mask_size, data_format + ) + cutout_center_height = offset[0] cutout_center_width = offset[1] From 049b65a994ea943296873628de48f595a028d881 Mon Sep 17 00:00:00 2001 From: fsx950223 Date: Tue, 24 Mar 2020 03:21:58 +0000 Subject: [PATCH 09/14] change batch random strategy --- tensorflow_addons/image/cutout_ops.py | 112 ++++++++++----------- tensorflow_addons/image/cutout_ops_test.py | 31 ++++-- 2 files changed, 75 insertions(+), 68 deletions(-) diff --git a/tensorflow_addons/image/cutout_ops.py b/tensorflow_addons/image/cutout_ops.py index e94bc8b281..a7093a3530 100644 --- a/tensorflow_addons/image/cutout_ops.py +++ b/tensorflow_addons/image/cutout_ops.py @@ -16,25 +16,21 @@ import tensorflow as tf from tensorflow_addons.utils.types import TensorLike, Number -from tensorflow_addons.image.utils import from_4D_image, to_4D_image from tensorflow.python.keras.utils import conv_utils def _get_image_wh(images, data_format): - if tf.equal(tf.rank(images), 4): - if data_format == "channels_last": - image_height, image_width = tf.shape(images)[1], tf.shape(images)[2] - else: - image_height, image_width = tf.shape(images)[2], tf.shape(images)[3] + if data_format == "channels_last": + image_height, image_width = tf.shape(images)[1], tf.shape(images)[2] else: - image_height, image_width = tf.shape(images)[0], tf.shape(images)[1] + image_height, image_width = tf.shape(images)[2], tf.shape(images)[3] return image_height, image_width def _norm_params(images, mask_size, data_format): mask_size = tf.convert_to_tensor(mask_size) - if tf.equal(tf.rank(mask_size), 0): + if tf.rank(mask_size) == 0: mask_size = tf.stack([mask_size, mask_size]) data_format = conv_utils.normalize_data_format(data_format) image_height, image_width = _get_image_wh(images, data_format) @@ -50,19 +46,18 @@ def random_cutout( ) -> tf.Tensor: """Apply cutout (https://arxiv.org/abs/1708.04552) to images. - This operation applies a (2*pad_size[0] x 2*pad_size[1]) mask of zeros to + This operation applies a (2 * mask_height x 2 * mask_width) mask of zeros to a random location within `img`. The pixel values filled in will be of the value `replace`. The located where the mask will be applied is randomly chosen uniformly over the whole images. Args: images: A tensor of shape - (num_images, num_rows, num_columns, num_channels) - (NHWC), (num_images, num_channels, num_rows, num_columns)(NCHW), (num_rows, num_columns, num_channels) (HWC), or - (num_rows, num_columns) (HW). + (batch_size, height, width, channels) + (NHWC), (batch_size, channels, height, width)(NCHW). mask_size: Specifies how big the zero mask that will be generated is that is applied to the images. The mask will be of size - (2*pad_size[0] x 2*pad_size[1]). + (2 * mask_height x 2 * mask_width). constant_values: What pixel value to fill in the images in the area that has the cutout mask applied to it. seed: A Python integer. Used in combination with `tf.random.set_seed` to @@ -75,24 +70,21 @@ def random_cutout( Returns: An image Tensor. """ + batch_size = tf.shape(images)[0] mask_size, data_format, image_height, image_width = _norm_params( images, mask_size, data_format ) cutout_center_height = tf.random.uniform( - shape=[], minval=0, maxval=image_height, dtype=tf.int32, seed=seed + shape=[batch_size], minval=0, maxval=image_height, dtype=tf.int32, seed=seed ) cutout_center_width = tf.random.uniform( - shape=[], minval=0, maxval=image_width, dtype=tf.int32, seed=seed - ) - return cutout( - images, - mask_size, - [cutout_center_height, cutout_center_width], - constant_values, - data_format, + shape=[batch_size], minval=0, maxval=image_width, dtype=tf.int32, seed=seed ) + offset = tf.transpose([cutout_center_height, cutout_center_width], [1, 0]) + return cutout(images, mask_size, offset, constant_values, data_format,) + def cutout( images: TensorLike, @@ -103,19 +95,17 @@ def cutout( ) -> tf.Tensor: """Apply cutout (https://arxiv.org/abs/1708.04552) to images. - This operation applies a (2*pad_size[0] x 2*pad_size[1]) mask of zeros to + This operation applies a (2 * mask_height x 2 * mask_width) mask of zeros to a random location within `img`. The pixel values filled in will be of the value `replace`. The located where the mask will be applied is randomly chosen uniformly over the whole images. Args: - images: A tensor of shape - (num_images, num_rows, num_columns, num_channels) - (NHWC), (num_images, num_channels, num_rows, num_columns)(NCHW), (num_rows, num_columns, num_channels) (HWC), or - (num_rows, num_columns) (HW). + images: A tensor of shape (batch_size, height, width, channels) + (NHWC), (batch_size, channels, height, width)(NCHW). mask_size: Specifies how big the zero mask that will be generated is that is applied to the images. The mask will be of size - (2*pad_size[0] x 2*pad_size[1]). + (2 * mask_height x 2 * mask_width). offset: A tuple of (height, width) constant_values: What pixel value to fill in the images in the area that has the cutout mask applied to it. @@ -128,40 +118,48 @@ def cutout( An image Tensor. """ with tf.name_scope("cutout"): + offset = tf.convert_to_tensor(offset) mask_size, data_format, image_height, image_width = _norm_params( images, mask_size, data_format ) - - cutout_center_height = offset[0] - cutout_center_width = offset[1] - - lower_pad = tf.maximum(0, cutout_center_height - mask_size[0]) - upper_pad = tf.maximum(0, image_height - cutout_center_height - mask_size[0]) - left_pad = tf.maximum(0, cutout_center_width - mask_size[1]) - right_pad = tf.maximum(0, image_width - cutout_center_width - mask_size[1]) - - cutout_shape = [ - image_height - (lower_pad + upper_pad), - image_width - (left_pad + right_pad), - ] - padding_dims = [[lower_pad, upper_pad], [left_pad, right_pad]] - mask = tf.pad( - tf.zeros(cutout_shape, dtype=images.dtype), padding_dims, constant_values=1 + if tf.rank(offset) == 1: + offset = tf.expand_dims(offset, 0) + cutout_center_heights = offset[:, 0] + cutout_center_widths = offset[:, 1] + + lower_pads = tf.maximum(0, cutout_center_heights - mask_size[0]) + upper_pads = tf.maximum(0, image_height - cutout_center_heights - mask_size[0]) + left_pads = tf.maximum(0, cutout_center_widths - mask_size[1]) + right_pads = tf.maximum(0, image_width - cutout_center_widths - mask_size[1]) + + cutout_shape = tf.transpose( + [ + image_height - (lower_pads + upper_pads), + image_width - (left_pads + right_pads), + ], + [1, 0], ) - mask_4d = to_4D_image(mask) - if tf.equal(tf.rank(images), 3): - mask = tf.tile(from_4D_image(mask_4d, 3), [1, 1, tf.shape(images)[-1]]) - elif tf.equal(tf.rank(images), 4): - if data_format == "channels_last": - mask = tf.tile( - mask_4d, [tf.shape(images)[0], 1, 1, tf.shape(images)[-1]] - ) - else: - mask = tf.tile( - mask_4d, [tf.shape(images)[0], tf.shape(images)[1], 1, 1] - ) + masks = tf.TensorArray(images.dtype, 0, dynamic_size=True) + for i in tf.range(tf.shape(cutout_shape)[0]): + padding_dims = [ + [lower_pads[i], upper_pads[i]], + [left_pads[i], right_pads[i]], + ] + mask = tf.pad( + tf.zeros(cutout_shape[i], dtype=images.dtype), + padding_dims, + constant_values=1, + ) + masks = masks.write(i, mask) + + if data_format == "channels_last": + mask_4d = tf.expand_dims(masks.stack(), -1) + mask = tf.tile(mask_4d, [1, 1, 1, tf.shape(images)[-1]]) + else: + mask_4d = tf.expand_dims(masks.stack(), 1) + mask = tf.tile(mask_4d, [1, tf.shape(images)[1], 1, 1]) images = tf.where( - tf.equal(mask, 0), + mask == 0, tf.ones_like(images, dtype=images.dtype) * constant_values, images, ) diff --git a/tensorflow_addons/image/cutout_ops_test.py b/tensorflow_addons/image/cutout_ops_test.py index 6d9ce999d7..f9c2ccff67 100644 --- a/tensorflow_addons/image/cutout_ops_test.py +++ b/tensorflow_addons/image/cutout_ops_test.py @@ -47,28 +47,37 @@ def test_different_channels(): np.testing.assert_allclose(tf.shape(result_image), tf.shape(expect_image)) -def test_different_ranks(): - test_image_4d = tf.ones([1, 40, 40, 1], dtype=np.uint8) +def test_batch_size(): + test_image = tf.ones([10, 40, 40, 1], dtype=np.uint8) cutout_area = tf.zeros([4, 4], dtype=np.uint8) cutout_area = tf.pad(cutout_area, ((0, 36), (0, 36)), constant_values=1) - expect_image_4d = to_4D_image(cutout_area) + expect_image = to_4D_image(cutout_area) + expect_image = tf.tile(expect_image, [10, 1, 1, 1]) + result_image = random_cutout(test_image, 20, seed=1234) + np.testing.assert_allclose(tf.shape(result_image), tf.shape(expect_image)) - test_image_2d = from_4D_image(test_image_4d, 2) - expect_image_2d = from_4D_image(expect_image_4d, 2) - result_image_2d = random_cutout(test_image_2d, 20, seed=1234) - np.testing.assert_allclose(tf.shape(result_image_2d), tf.shape(expect_image_2d)) - result_image_4d = random_cutout(test_image_4d, 20, seed=1234) - np.testing.assert_allclose(tf.shape(result_image_4d), tf.shape(expect_image_4d)) +def test_channel_first(): + test_image = tf.ones([10, 1, 40, 40], dtype=np.uint8) + cutout_area = tf.zeros([4, 4], dtype=np.uint8) + cutout_area = tf.pad(cutout_area, ((0, 36), (0, 36)), constant_values=1) + expect_image = tf.expand_dims(cutout_area, 0) + expect_image = tf.expand_dims(expect_image, 0) + expect_image = tf.tile(expect_image, [10, 1, 1, 1]) + result_image = random_cutout( + test_image, 20, seed=1234, data_format="channels_first" + ) + np.testing.assert_allclose(tf.shape(result_image), tf.shape(expect_image)) +@pytest.mark.usefixtures("maybe_run_functions_eagerly") def test_with_tf_function(): test_image = tf.ones([1, 40, 40, 1], dtype=tf.uint8) - result_image = tf.function(cutout)(test_image, 2, [2, 2]) + result_image = tf.function(random_cutout)(test_image, 2) cutout_area = tf.zeros([4, 4], dtype=tf.uint8) cutout_area = tf.pad(cutout_area, ((0, 36), (0, 36)), constant_values=1) expect_image = to_4D_image(cutout_area) - np.testing.assert_allclose(result_image, expect_image) + np.testing.assert_allclose(tf.shape(result_image), tf.shape(expect_image)) if __name__ == "__main__": From 0e9b39d88fa4d115a2038ea3ad7130efe2453a27 Mon Sep 17 00:00:00 2001 From: fsx950223 Date: Tue, 24 Mar 2020 03:39:18 +0000 Subject: [PATCH 10/14] fix flake8 --- tensorflow_addons/image/cutout_ops_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow_addons/image/cutout_ops_test.py b/tensorflow_addons/image/cutout_ops_test.py index f9c2ccff67..c152812efc 100644 --- a/tensorflow_addons/image/cutout_ops_test.py +++ b/tensorflow_addons/image/cutout_ops_test.py @@ -21,7 +21,7 @@ import numpy as np from absl.testing import parameterized from tensorflow_addons.image.cutout_ops import cutout, random_cutout -from tensorflow_addons.image.utils import from_4D_image, to_4D_image +from tensorflow_addons.image.utils import to_4D_image @parameterized.named_parameters( From 7a4ba5b29cc7a89b2892cbe4d47433d2f0bfcbdb Mon Sep 17 00:00:00 2001 From: fsx950223 Date: Thu, 26 Mar 2020 09:45:02 +0000 Subject: [PATCH 11/14] add more checks --- tensorflow_addons/image/cutout_ops.py | 33 ++++++++++++++-------- tensorflow_addons/image/cutout_ops_test.py | 26 +++++++++-------- 2 files changed, 36 insertions(+), 23 deletions(-) diff --git a/tensorflow_addons/image/cutout_ops.py b/tensorflow_addons/image/cutout_ops.py index a7093a3530..0ae171b435 100644 --- a/tensorflow_addons/image/cutout_ops.py +++ b/tensorflow_addons/image/cutout_ops.py @@ -30,11 +30,20 @@ def _get_image_wh(images, data_format): def _norm_params(images, mask_size, data_format): mask_size = tf.convert_to_tensor(mask_size) - if tf.rank(mask_size) == 0: - mask_size = tf.stack([mask_size, mask_size]) - data_format = conv_utils.normalize_data_format(data_format) - image_height, image_width = _get_image_wh(images, data_format) - return mask_size, data_format, image_height, image_width + with tf.control_dependencies( + [ + tf.assert_equal( + tf.reduce_any(mask_size % 2 != 0), + False, + "mask_size should be divisible by 2", + ) + ] + ): + if tf.rank(mask_size) == 0: + mask_size = tf.stack([mask_size, mask_size]) + data_format = conv_utils.normalize_data_format(data_format) + image_height, image_width = _get_image_wh(images, data_format) + return mask_size, data_format, image_height, image_width def random_cutout( @@ -46,7 +55,7 @@ def random_cutout( ) -> tf.Tensor: """Apply cutout (https://arxiv.org/abs/1708.04552) to images. - This operation applies a (2 * mask_height x 2 * mask_width) mask of zeros to + This operation applies a (mask_height x mask_width) mask of zeros to a random location within `img`. The pixel values filled in will be of the value `replace`. The located where the mask will be applied is randomly chosen uniformly over the whole images. @@ -57,7 +66,7 @@ def random_cutout( (NHWC), (batch_size, channels, height, width)(NCHW). mask_size: Specifies how big the zero mask that will be generated is that is applied to the images. The mask will be of size - (2 * mask_height x 2 * mask_width). + (mask_height x mask_width). Note: mask_size should be divisible by 2. constant_values: What pixel value to fill in the images in the area that has the cutout mask applied to it. seed: A Python integer. Used in combination with `tf.random.set_seed` to @@ -95,8 +104,8 @@ def cutout( ) -> tf.Tensor: """Apply cutout (https://arxiv.org/abs/1708.04552) to images. - This operation applies a (2 * mask_height x 2 * mask_width) mask of zeros to - a random location within `img`. The pixel values filled in will be of the + This operation applies a (mask_height x mask_width) mask of zeros to + a location within `img` specified by the offset. The pixel values filled in will be of the value `replace`. The located where the mask will be applied is randomly chosen uniformly over the whole images. @@ -105,8 +114,8 @@ def cutout( (NHWC), (batch_size, channels, height, width)(NCHW). mask_size: Specifies how big the zero mask that will be generated is that is applied to the images. The mask will be of size - (2 * mask_height x 2 * mask_width). - offset: A tuple of (height, width) + (mask_height x mask_width). + offset: A tuple of (height, width) or (batch_size, 2) constant_values: What pixel value to fill in the images in the area that has the cutout mask applied to it. data_format: A string, one of `channels_last` (default) or `channels_first`. @@ -122,6 +131,8 @@ def cutout( mask_size, data_format, image_height, image_width = _norm_params( images, mask_size, data_format ) + mask_size = mask_size // 2 + if tf.rank(offset) == 1: offset = tf.expand_dims(offset, 0) cutout_center_heights = offset[:, 0] diff --git a/tensorflow_addons/image/cutout_ops_test.py b/tensorflow_addons/image/cutout_ops_test.py index c152812efc..f3dc74a963 100644 --- a/tensorflow_addons/image/cutout_ops_test.py +++ b/tensorflow_addons/image/cutout_ops_test.py @@ -19,21 +19,19 @@ import pytest import tensorflow as tf import numpy as np -from absl.testing import parameterized from tensorflow_addons.image.cutout_ops import cutout, random_cutout from tensorflow_addons.image.utils import to_4D_image -@parameterized.named_parameters( - ("float16", np.float16), ("float32", np.float32), ("uint8", np.uint8) -) +@pytest.mark.parametrize("dtype", [np.float16, np.float32, np.uint8]) def test_different_dtypes(dtype): test_image = tf.ones([1, 40, 40, 1], dtype=dtype) - result_image = cutout(test_image, 2, [2, 2]) + result_image = cutout(test_image, 4, [2, 2]) cutout_area = tf.zeros([4, 4], dtype=dtype) cutout_area = tf.pad(cutout_area, ((0, 36), (0, 36)), constant_values=1) expect_image = to_4D_image(cutout_area) np.testing.assert_allclose(result_image, expect_image) + assert result_image.dtype == dtype def test_different_channels(): @@ -48,13 +46,11 @@ def test_different_channels(): def test_batch_size(): - test_image = tf.ones([10, 40, 40, 1], dtype=np.uint8) - cutout_area = tf.zeros([4, 4], dtype=np.uint8) - cutout_area = tf.pad(cutout_area, ((0, 36), (0, 36)), constant_values=1) - expect_image = to_4D_image(cutout_area) - expect_image = tf.tile(expect_image, [10, 1, 1, 1]) + test_image = tf.random.uniform([10, 40, 40, 1], dtype=np.float32) result_image = random_cutout(test_image, 20, seed=1234) - np.testing.assert_allclose(tf.shape(result_image), tf.shape(expect_image)) + np.testing.assert_allclose(tf.shape(result_image), [10, 40, 40, 1]) + means = np.mean(result_image, axis=(1, 2, 3)) + np.testing.assert_allclose(len(set(means)), 10) def test_channel_first(): @@ -73,7 +69,13 @@ def test_channel_first(): @pytest.mark.usefixtures("maybe_run_functions_eagerly") def test_with_tf_function(): test_image = tf.ones([1, 40, 40, 1], dtype=tf.uint8) - result_image = tf.function(random_cutout)(test_image, 2) + result_image = tf.function( + random_cutout, + input_signature=[ + tf.TensorSpec(shape=[None, 40, 40, 1], dtype=tf.uint8), + tf.TensorSpec(shape=[], dtype=tf.int32), + ], + )(test_image, 2) cutout_area = tf.zeros([4, 4], dtype=tf.uint8) cutout_area = tf.pad(cutout_area, ((0, 36), (0, 36)), constant_values=1) expect_image = to_4D_image(cutout_area) From bebe07f50e11838793d608082dcaa1aacb2d2339 Mon Sep 17 00:00:00 2001 From: fsx950223 Date: Thu, 26 Mar 2020 10:14:25 +0000 Subject: [PATCH 12/14] add missing comment --- tensorflow_addons/image/cutout_ops.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tensorflow_addons/image/cutout_ops.py b/tensorflow_addons/image/cutout_ops.py index 0ae171b435..d87e22d029 100644 --- a/tensorflow_addons/image/cutout_ops.py +++ b/tensorflow_addons/image/cutout_ops.py @@ -78,6 +78,8 @@ def random_cutout( inputs with shape `(batch_size, channels, ...)`. Returns: An image Tensor. + Raises: + InvalidArgumentError: if mask_size can't be divisible by 2. """ batch_size = tf.shape(images)[0] mask_size, data_format, image_height, image_width = _norm_params( @@ -114,7 +116,7 @@ def cutout( (NHWC), (batch_size, channels, height, width)(NCHW). mask_size: Specifies how big the zero mask that will be generated is that is applied to the images. The mask will be of size - (mask_height x mask_width). + (mask_height x mask_width). Note: mask_size should be divisible by 2. offset: A tuple of (height, width) or (batch_size, 2) constant_values: What pixel value to fill in the images in the area that has the cutout mask applied to it. @@ -125,6 +127,8 @@ def cutout( inputs with shape `(batch_size, channels, ...)`. Returns: An image Tensor. + Raises: + InvalidArgumentError: if mask_size can't be divisible by 2. """ with tf.name_scope("cutout"): offset = tf.convert_to_tensor(offset) From 595d63df4b6b47771a2d5a04a3263b9b01801fd4 Mon Sep 17 00:00:00 2001 From: fsx950223 Date: Thu, 26 Mar 2020 10:26:54 +0000 Subject: [PATCH 13/14] add seed --- tensorflow_addons/image/cutout_ops_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow_addons/image/cutout_ops_test.py b/tensorflow_addons/image/cutout_ops_test.py index f3dc74a963..6858c3d143 100644 --- a/tensorflow_addons/image/cutout_ops_test.py +++ b/tensorflow_addons/image/cutout_ops_test.py @@ -46,7 +46,7 @@ def test_different_channels(): def test_batch_size(): - test_image = tf.random.uniform([10, 40, 40, 1], dtype=np.float32) + test_image = tf.random.uniform([10, 40, 40, 1], dtype=np.float32, seed=1234) result_image = random_cutout(test_image, 20, seed=1234) np.testing.assert_allclose(tf.shape(result_image), [10, 40, 40, 1]) means = np.mean(result_image, axis=(1, 2, 3)) From df94c2df8d544f97a20d5845efb8f52f4ffe5e8e Mon Sep 17 00:00:00 2001 From: fsx950223 Date: Thu, 26 Mar 2020 10:40:58 +0000 Subject: [PATCH 14/14] remove control dependencies --- tensorflow_addons/image/cutout_ops.py | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/tensorflow_addons/image/cutout_ops.py b/tensorflow_addons/image/cutout_ops.py index d87e22d029..0c4d494be9 100644 --- a/tensorflow_addons/image/cutout_ops.py +++ b/tensorflow_addons/image/cutout_ops.py @@ -30,20 +30,17 @@ def _get_image_wh(images, data_format): def _norm_params(images, mask_size, data_format): mask_size = tf.convert_to_tensor(mask_size) - with tf.control_dependencies( - [ - tf.assert_equal( - tf.reduce_any(mask_size % 2 != 0), - False, - "mask_size should be divisible by 2", - ) - ] - ): - if tf.rank(mask_size) == 0: - mask_size = tf.stack([mask_size, mask_size]) - data_format = conv_utils.normalize_data_format(data_format) - image_height, image_width = _get_image_wh(images, data_format) - return mask_size, data_format, image_height, image_width + if tf.executing_eagerly(): + tf.assert_equal( + tf.reduce_any(mask_size % 2 != 0), + False, + "mask_size should be divisible by 2", + ) + if tf.rank(mask_size) == 0: + mask_size = tf.stack([mask_size, mask_size]) + data_format = conv_utils.normalize_data_format(data_format) + image_height, image_width = _get_image_wh(images, data_format) + return mask_size, data_format, image_height, image_width def random_cutout(