From e0034ed10864ce7c185ee05e3ec9fdd77385cc06 Mon Sep 17 00:00:00 2001 From: gabrieldemarmiesse Date: Thu, 27 Feb 2020 19:55:58 +0000 Subject: [PATCH 1/2] Better precision when comparing implementations. --- tensorflow_addons/activations/gelu_test.py | 4 ++-- tensorflow_addons/activations/hardshrink_test.py | 4 ++-- tensorflow_addons/activations/lisht_test.py | 4 ++-- tensorflow_addons/activations/mish_test.py | 4 ++-- tensorflow_addons/activations/softshrink_test.py | 4 ++-- tensorflow_addons/activations/tanhshrink_test.py | 4 ++-- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/tensorflow_addons/activations/gelu_test.py b/tensorflow_addons/activations/gelu_test.py index da44fb4315..82e7240dc4 100644 --- a/tensorflow_addons/activations/gelu_test.py +++ b/tensorflow_addons/activations/gelu_test.py @@ -66,10 +66,10 @@ def verify_funcs_are_equivalent(self, dtype): t.watch(x) y_native = gelu(x, approximate=approximate) y_py = _gelu_py(x, approximate=approximate) - self.assertAllCloseAccordingToType(y_native, y_py, atol=1e-4) + self.assertAllCloseAccordingToType(y_native, y_py) grad_native = t.gradient(y_native, x) grad_py = t.gradient(y_py, x) - self.assertAllCloseAccordingToType(grad_native, grad_py, atol=1e-4) + self.assertAllCloseAccordingToType(grad_native, grad_py) if __name__ == "__main__": diff --git a/tensorflow_addons/activations/hardshrink_test.py b/tensorflow_addons/activations/hardshrink_test.py index 63eff43ec5..ff6e4afc6e 100644 --- a/tensorflow_addons/activations/hardshrink_test.py +++ b/tensorflow_addons/activations/hardshrink_test.py @@ -71,12 +71,12 @@ def verify_funcs_are_equivalent(self, dtype): y_native = hardshrink(x, lower, upper) y_py = _hardshrink_py(x, lower, upper) - self.assertAllCloseAccordingToType(y_native, y_py, atol=1e-4) + self.assertAllCloseAccordingToType(y_native, y_py) grad_native = t.gradient(y_native, x) grad_py = t.gradient(y_py, x) - self.assertAllCloseAccordingToType(grad_native, grad_py, atol=1e-4) + self.assertAllCloseAccordingToType(grad_native, grad_py) if __name__ == "__main__": diff --git a/tensorflow_addons/activations/lisht_test.py b/tensorflow_addons/activations/lisht_test.py index d98ba5a117..9a64b7de7b 100644 --- a/tensorflow_addons/activations/lisht_test.py +++ b/tensorflow_addons/activations/lisht_test.py @@ -60,12 +60,12 @@ def verify_funcs_are_equivalent(self, dtype): y_native = lisht(x) y_py = _lisht_py(x) - self.assertAllCloseAccordingToType(y_native, y_py, atol=1e-4) + self.assertAllCloseAccordingToType(y_native, y_py) grad_native = t.gradient(y_native, x) grad_py = t.gradient(y_py, x) - self.assertAllCloseAccordingToType(grad_native, grad_py, atol=1e-4) + self.assertAllCloseAccordingToType(grad_native, grad_py) if __name__ == "__main__": diff --git a/tensorflow_addons/activations/mish_test.py b/tensorflow_addons/activations/mish_test.py index 6c2a8f7ab4..e03e0373a0 100644 --- a/tensorflow_addons/activations/mish_test.py +++ b/tensorflow_addons/activations/mish_test.py @@ -58,12 +58,12 @@ def verify_funcs_are_equivalent(self, dtype): y_native = mish(x) y_py = _mish_py(x) - self.assertAllCloseAccordingToType(y_native, y_py, atol=1e-4) + self.assertAllCloseAccordingToType(y_native, y_py) grad_native = t.gradient(y_native, x) grad_py = t.gradient(y_py, x) - self.assertAllCloseAccordingToType(grad_native, grad_py, atol=1e-4) + self.assertAllCloseAccordingToType(grad_native, grad_py) if __name__ == "__main__": diff --git a/tensorflow_addons/activations/softshrink_test.py b/tensorflow_addons/activations/softshrink_test.py index 5338726026..522bc0dc11 100644 --- a/tensorflow_addons/activations/softshrink_test.py +++ b/tensorflow_addons/activations/softshrink_test.py @@ -71,12 +71,12 @@ def verify_funcs_are_equivalent(self, dtype): y_native = softshrink(x, lower, upper) y_py = _softshrink_py(x, lower, upper) - self.assertAllCloseAccordingToType(y_native, y_py, atol=1e-4) + self.assertAllCloseAccordingToType(y_native, y_py) grad_native = t.gradient(y_native, x) grad_py = t.gradient(y_py, x) - self.assertAllCloseAccordingToType(grad_native, grad_py, atol=1e-4) + self.assertAllCloseAccordingToType(grad_native, grad_py) if __name__ == "__main__": diff --git a/tensorflow_addons/activations/tanhshrink_test.py b/tensorflow_addons/activations/tanhshrink_test.py index 37658fe1cf..01362a0aa0 100644 --- a/tensorflow_addons/activations/tanhshrink_test.py +++ b/tensorflow_addons/activations/tanhshrink_test.py @@ -39,10 +39,10 @@ def verify_funcs_are_equivalent(self, dtype): t.watch(x) y_native = tanhshrink(x) y_py = _tanhshrink_py(x) - self.assertAllCloseAccordingToType(y_native, y_py, atol=1e-4) + self.assertAllCloseAccordingToType(y_native, y_py) grad_native = t.gradient(y_native, x) grad_py = t.gradient(y_py, x) - self.assertAllCloseAccordingToType(grad_native, grad_py, atol=1e-4) + self.assertAllCloseAccordingToType(grad_native, grad_py) if __name__ == "__main__": From af52fe08e5eaa8b95a0451e0fd835ec9e51ff0ca Mon Sep 17 00:00:00 2001 From: gabrieldemarmiesse Date: Thu, 27 Feb 2020 20:17:16 +0000 Subject: [PATCH 2/2] Gelu with float32 has not the same gradient as the C++ version. --- tensorflow_addons/activations/gelu_test.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tensorflow_addons/activations/gelu_test.py b/tensorflow_addons/activations/gelu_test.py index 82e7240dc4..92b1183e55 100644 --- a/tensorflow_addons/activations/gelu_test.py +++ b/tensorflow_addons/activations/gelu_test.py @@ -69,7 +69,10 @@ def verify_funcs_are_equivalent(self, dtype): self.assertAllCloseAccordingToType(y_native, y_py) grad_native = t.gradient(y_native, x) grad_py = t.gradient(y_py, x) - self.assertAllCloseAccordingToType(grad_native, grad_py) + # TODO: lower atol to 1e-6 + # currently it doesn't work. + # It necessitates changing the Python or C++ implementation. + self.assertAllCloseAccordingToType(grad_native, grad_py, atol=1e-5) if __name__ == "__main__":