From f18b2aa9e89a6f7513b1285360dd5ffe04253bb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?= Date: Thu, 17 Jan 2019 11:55:31 +0800 Subject: [PATCH 1/5] ENH: implement Maxout layer --- tensorflow_addons/layers/BUILD | 17 +++- tensorflow_addons/layers/__init__.py | 2 +- tensorflow_addons/layers/python/maxout.py | 94 +++++++++++++++++++ .../layers/python/maxout_test.py | 79 ++++++++++++++++ 4 files changed, 190 insertions(+), 2 deletions(-) create mode 100644 tensorflow_addons/layers/python/maxout.py create mode 100644 tensorflow_addons/layers/python/maxout_test.py diff --git a/tensorflow_addons/layers/BUILD b/tensorflow_addons/layers/BUILD index 1d5c07d687..0258816c7e 100644 --- a/tensorflow_addons/layers/BUILD +++ b/tensorflow_addons/layers/BUILD @@ -7,6 +7,7 @@ py_library( srcs = ([ "__init__.py", "python/__init__.py", + "python/maxout.py", "python/wrappers.py", ]), srcs_version = "PY2AND3", @@ -22,4 +23,18 @@ py_test( ":layers_py", ], srcs_version = "PY2AND3", -) \ No newline at end of file +) + +py_test( + name = "maxout_py_test", + size = "small", + srcs = [ + "python/maxout_test.py", + ], + main = "python/maxout_test.py", + deps = [ + ":layers_py", + ], + srcs_version = "PY2AND3", +) + diff --git a/tensorflow_addons/layers/__init__.py b/tensorflow_addons/layers/__init__.py index 0141f32668..09a236c8c9 100644 --- a/tensorflow_addons/layers/__init__.py +++ b/tensorflow_addons/layers/__init__.py @@ -19,5 +19,5 @@ from __future__ import division from __future__ import print_function -# Weight Normalization Wrapper +from tensorflow_addons.layers.python.maxout import Maxout from tensorflow_addons.layers.python.wrappers import WeightNormalization diff --git a/tensorflow_addons/layers/python/maxout.py b/tensorflow_addons/layers/python/maxout.py new file mode 100644 index 0000000000..159f6e7979 --- /dev/null +++ b/tensorflow_addons/layers/python/maxout.py @@ -0,0 +1,94 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Implementing Maxout layer.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.python.framework import ops +from tensorflow.python.framework import tensor_shape +from tensorflow.python.keras.engine.base_layer import Layer +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import math_ops + + +class Maxout(Layer): + """Applies Maxout to the input. + + "Maxout Networks" Ian J. Goodfellow, David Warde-Farley, Mehdi Mirza, Aaron + Courville, Yoshua Bengio. https://arxiv.org/abs/1302.4389 + + Usually the operation is performed in the filter/channel dimension. This can + also be used after Dense layers to reduce number of features. + + Arguments: + num_units: Specifies how many features will remain after maxout + in the `axis` dimension (usually channel). + This must be a factor of number of features. + axis: The dimension where max pooling will be performed. Default is the + last dimension. + + Input shape: + nD tensor with shape: `(batch_size, ..., axis_dim, ...)`. + + Output shape: + nD tensor with shape: `(batch_size, ..., num_units, ...)`. + """ + + def __init__(self, num_units, axis=-1, **kwargs): + super(Maxout, self).__init__(**kwargs) + self.num_units = num_units + self.axis = axis + + def call(self, inputs): + inputs = ops.convert_to_tensor(inputs) + shape = inputs.get_shape().as_list() + # Dealing with batches with arbitrary sizes + for i in range(len(shape)): + if shape[i] is None: + shape[i] = array_ops.shape(inputs)[i] + + num_channels = shape[self.axis] + if (not isinstance(num_channels, ops.Tensor) + and num_channels % self.num_units): + raise ValueError('number of features({}) is not ' + 'a multiple of num_units({})'.format( + num_channels, self.num_units)) + + if self.axis < 0: + axis = self.axis + len(shape) + else: + axis = self.axis + assert axis >= 0, 'Find invalid axis: {}'.format(self.axis) + + expand_shape = shape[:] + expand_shape[axis] = self.num_units + k = num_channels // self.num_units + expand_shape.insert(axis, k) + + outputs = math_ops.reduce_max( + array_ops.reshape(inputs, expand_shape), axis, keepdims=False) + return outputs + + def compute_output_shape(self, input_shape): + input_shape = tensor_shape.TensorShape(input_shape).as_list() + input_shape[self.axis] = self.num_units + return tensor_shape.TensorShape(input_shape) + + def get_config(self): + config = {'num_units': self.num_units, 'axis': self.axis} + base_config = super(Maxout, self).get_config() + return dict(list(base_config.items()) + list(config.items())) diff --git a/tensorflow_addons/layers/python/maxout_test.py b/tensorflow_addons/layers/python/maxout_test.py new file mode 100644 index 0000000000..ac45a23c45 --- /dev/null +++ b/tensorflow_addons/layers/python/maxout_test.py @@ -0,0 +1,79 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for Maxout layer.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.python.keras import testing_utils +from tensorflow.python.keras.utils import generic_utils +from tensorflow.python.platform import test +from tensorflow_addons.layers.python.maxout import Maxout + + +class MaxOutTest(test.TestCase): + def test_simple(self): + # TODO: more simple way to deserialize the layers in addons. + with generic_utils.custom_object_scope({'Maxout': Maxout}): + testing_utils.layer_test( + Maxout, kwargs={'num_units': 3}, input_shape=(5, 4, 2, 18)) + + def test_nchw(self): + with generic_utils.custom_object_scope({'Maxout': Maxout}): + testing_utils.layer_test( + Maxout, + kwargs={ + 'num_units': 4, + 'axis': 1 + }, + input_shape=(2, 20, 3, 6)) + + with generic_utils.custom_object_scope({'Maxout': Maxout}): + testing_utils.layer_test( + Maxout, + kwargs={ + 'num_units': 4, + 'axis': -3 + }, + input_shape=(2, 20, 3, 6)) + + def test_unknown(self): + inputs = np.random.random((5, 4, 2, 18)).astype('float32') + with generic_utils.custom_object_scope({'Maxout': Maxout}): + testing_utils.layer_test( + Maxout, + kwargs={'num_units': 3}, + input_shape=(5, 4, 2, None), + input_data=inputs) + + with generic_utils.custom_object_scope({'Maxout': Maxout}): + testing_utils.layer_test( + Maxout, + kwargs={'num_units': 3}, + input_shape=(None, None, None, None), + input_data=inputs) + + def test_invalid_shape(self): + with self.assertRaisesRegexp(ValueError, r'number of features'): + with generic_utils.custom_object_scope({'Maxout': Maxout}): + testing_utils.layer_test( + Maxout, kwargs={'num_units': 3}, input_shape=(5, 4, 2, 7)) + + +if __name__ == '__main__': + test.main() From 2fe098edf7f02178a9d7b857460c3dc15de6529d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?= Date: Thu, 17 Jan 2019 13:16:38 +0800 Subject: [PATCH 2/5] CLN: fix style error in layers/BUILD file --- tensorflow_addons/layers/BUILD | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tensorflow_addons/layers/BUILD b/tensorflow_addons/layers/BUILD index 0258816c7e..49b97f6854 100644 --- a/tensorflow_addons/layers/BUILD +++ b/tensorflow_addons/layers/BUILD @@ -4,12 +4,12 @@ package(default_visibility = ["//visibility:public"]) py_library( name = "layers_py", - srcs = ([ + srcs = [ "__init__.py", "python/__init__.py", "python/maxout.py", "python/wrappers.py", - ]), + ], srcs_version = "PY2AND3", ) @@ -20,8 +20,8 @@ py_test( ], main = "python/wrappers_test.py", deps = [ - ":layers_py", - ], + ":layers_py", + ], srcs_version = "PY2AND3", ) @@ -33,8 +33,8 @@ py_test( ], main = "python/maxout_test.py", deps = [ - ":layers_py", - ], + ":layers_py", + ], srcs_version = "PY2AND3", ) From 53d5d9eabd1c39ff2b2166a9e248a1030e0668d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?= Date: Thu, 17 Jan 2019 14:57:46 +0800 Subject: [PATCH 3/5] CLN: minor fix --- tensorflow_addons/layers/python/maxout.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tensorflow_addons/layers/python/maxout.py b/tensorflow_addons/layers/python/maxout.py index 159f6e7979..2695b53200 100644 --- a/tensorflow_addons/layers/python/maxout.py +++ b/tensorflow_addons/layers/python/maxout.py @@ -35,11 +35,11 @@ class Maxout(Layer): also be used after Dense layers to reduce number of features. Arguments: - num_units: Specifies how many features will remain after maxout - in the `axis` dimension (usually channel). - This must be a factor of number of features. - axis: The dimension where max pooling will be performed. Default is the - last dimension. + num_units: Specifies how many features will remain after maxout + in the `axis` dimension (usually channel). + This must be a factor of number of features. + axis: The dimension where max pooling will be performed. Default is the + last dimension. Input shape: nD tensor with shape: `(batch_size, ..., axis_dim, ...)`. From 26cc9183de44abf2b19765ddefbb4ef5baca3e25 Mon Sep 17 00:00:00 2001 From: Sean Morgan Date: Fri, 25 Jan 2019 15:49:37 -0500 Subject: [PATCH 4/5] Register custom layer --- tensorflow_addons/layers/python/maxout.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tensorflow_addons/layers/python/maxout.py b/tensorflow_addons/layers/python/maxout.py index 2695b53200..0beb2ae24d 100644 --- a/tensorflow_addons/layers/python/maxout.py +++ b/tensorflow_addons/layers/python/maxout.py @@ -20,6 +20,7 @@ from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape +from tensorflow.python.keras.utils import generic_utils from tensorflow.python.keras.engine.base_layer import Layer from tensorflow.python.ops import array_ops from tensorflow.python.ops import math_ops @@ -92,3 +93,6 @@ def get_config(self): config = {'num_units': self.num_units, 'axis': self.axis} base_config = super(Maxout, self).get_config() return dict(list(base_config.items()) + list(config.items())) + + +generic_utils._GLOBAL_CUSTOM_OBJECTS['Maxout'] = Maxout From f8349a9cf3f6c608933c1f232e9ea8b138c54fee Mon Sep 17 00:00:00 2001 From: Sean Morgan Date: Fri, 25 Jan 2019 15:50:34 -0500 Subject: [PATCH 5/5] Modify tests for pre-registered layer --- .../layers/python/maxout_test.py | 64 ++++++++----------- 1 file changed, 28 insertions(+), 36 deletions(-) diff --git a/tensorflow_addons/layers/python/maxout_test.py b/tensorflow_addons/layers/python/maxout_test.py index ac45a23c45..22e381f8c2 100644 --- a/tensorflow_addons/layers/python/maxout_test.py +++ b/tensorflow_addons/layers/python/maxout_test.py @@ -21,58 +21,50 @@ import numpy as np from tensorflow.python.keras import testing_utils -from tensorflow.python.keras.utils import generic_utils from tensorflow.python.platform import test from tensorflow_addons.layers.python.maxout import Maxout class MaxOutTest(test.TestCase): def test_simple(self): - # TODO: more simple way to deserialize the layers in addons. - with generic_utils.custom_object_scope({'Maxout': Maxout}): - testing_utils.layer_test( - Maxout, kwargs={'num_units': 3}, input_shape=(5, 4, 2, 18)) + testing_utils.layer_test( + Maxout, kwargs={'num_units': 3}, input_shape=(5, 4, 2, 18)) def test_nchw(self): - with generic_utils.custom_object_scope({'Maxout': Maxout}): - testing_utils.layer_test( - Maxout, - kwargs={ - 'num_units': 4, - 'axis': 1 - }, - input_shape=(2, 20, 3, 6)) + testing_utils.layer_test( + Maxout, + kwargs={ + 'num_units': 4, + 'axis': 1 + }, + input_shape=(2, 20, 3, 6)) - with generic_utils.custom_object_scope({'Maxout': Maxout}): - testing_utils.layer_test( - Maxout, - kwargs={ - 'num_units': 4, - 'axis': -3 - }, - input_shape=(2, 20, 3, 6)) + testing_utils.layer_test( + Maxout, + kwargs={ + 'num_units': 4, + 'axis': -3 + }, + input_shape=(2, 20, 3, 6)) def test_unknown(self): inputs = np.random.random((5, 4, 2, 18)).astype('float32') - with generic_utils.custom_object_scope({'Maxout': Maxout}): - testing_utils.layer_test( - Maxout, - kwargs={'num_units': 3}, - input_shape=(5, 4, 2, None), - input_data=inputs) + testing_utils.layer_test( + Maxout, + kwargs={'num_units': 3}, + input_shape=(5, 4, 2, None), + input_data=inputs) - with generic_utils.custom_object_scope({'Maxout': Maxout}): - testing_utils.layer_test( - Maxout, - kwargs={'num_units': 3}, - input_shape=(None, None, None, None), - input_data=inputs) + testing_utils.layer_test( + Maxout, + kwargs={'num_units': 3}, + input_shape=(None, None, None, None), + input_data=inputs) def test_invalid_shape(self): with self.assertRaisesRegexp(ValueError, r'number of features'): - with generic_utils.custom_object_scope({'Maxout': Maxout}): - testing_utils.layer_test( - Maxout, kwargs={'num_units': 3}, input_shape=(5, 4, 2, 7)) + testing_utils.layer_test( + Maxout, kwargs={'num_units': 3}, input_shape=(5, 4, 2, 7)) if __name__ == '__main__':