From d1063cdba4e699397ef1f1e449b595ac4ebbd5a5 Mon Sep 17 00:00:00 2001
From: ashutosh1919 <ashutoshhathidara98@gmail.com>
Date: Sat, 25 Apr 2020 12:17:00 +0500
Subject: [PATCH 1/3] Added serializability tests in all optimizers and fixed
 serializability bug in yogi

---
 .../optimizers/tests/conditional_gradient_test.py    | 12 ++++++++++++
 tensorflow_addons/optimizers/tests/lamb_test.py      |  7 +++++++
 tensorflow_addons/optimizers/tests/lazy_adam_test.py |  7 +++++++
 tensorflow_addons/optimizers/tests/lookahead_test.py |  7 +++++++
 .../optimizers/tests/moving_average_test.py          | 10 ++++++++++
 tensorflow_addons/optimizers/tests/novograd_test.py  |  7 +++++++
 .../tests/stochastic_weight_averaging_test.py        | 10 ++++++++++
 .../optimizers/tests/weight_decay_optimizers_test.py |  7 +++++++
 tensorflow_addons/optimizers/tests/yogi_test.py      |  7 +++++++
 tensorflow_addons/optimizers/yogi.py                 |  8 ++++++--
 10 files changed, 80 insertions(+), 2 deletions(-)

diff --git a/tensorflow_addons/optimizers/tests/conditional_gradient_test.py b/tensorflow_addons/optimizers/tests/conditional_gradient_test.py
index 694fab8164..0e84234fb9 100644
--- a/tensorflow_addons/optimizers/tests/conditional_gradient_test.py
+++ b/tensorflow_addons/optimizers/tests/conditional_gradient_test.py
@@ -1457,3 +1457,15 @@ def test_sparse_nuclear():
             ),
             var1[2].numpy(),
         )
+
+
+def test_serialization():
+    learning_rate = 0.1
+    lambda_ = 0.1
+    ord = "nuclear"
+    optimizer = cg_lib.ConditionalGradient(
+        learning_rate=learning_rate, lambda_=lambda_, ord=ord
+    )
+    config = tf.keras.optimizers.serialize(optimizer)
+    new_optimizer = tf.keras.optimizers.deserialize(config)
+    assert new_optimizer.get_config() == optimizer.get_config()
diff --git a/tensorflow_addons/optimizers/tests/lamb_test.py b/tensorflow_addons/optimizers/tests/lamb_test.py
index 253bedab92..06113d7ad3 100644
--- a/tensorflow_addons/optimizers/tests/lamb_test.py
+++ b/tensorflow_addons/optimizers/tests/lamb_test.py
@@ -345,3 +345,10 @@ def test_exclude_layer_adaptation():
     assert opt._do_layer_adaptation("var0")
     assert not opt._do_layer_adaptation("var1")
     assert not opt._do_layer_adaptation("var1_weight")
+
+
+def test_serialization():
+    optimizer = lamb.LAMB(1e-4)
+    config = tf.keras.optimizers.serialize(optimizer)
+    new_optimizer = tf.keras.optimizers.deserialize(config)
+    assert new_optimizer.get_config() == optimizer.get_config()
diff --git a/tensorflow_addons/optimizers/tests/lazy_adam_test.py b/tensorflow_addons/optimizers/tests/lazy_adam_test.py
index 7560254262..16a5614219 100644
--- a/tensorflow_addons/optimizers/tests/lazy_adam_test.py
+++ b/tensorflow_addons/optimizers/tests/lazy_adam_test.py
@@ -253,3 +253,10 @@ def test_slots_unique_eager():
     # There should be iteration, and two unique slot variables for v1 and v2.
     assert 5 == len(opt.variables())
     assert opt.variables()[0] == opt.iterations
+
+
+def test_serialization():
+    optimizer = lazy_adam.LazyAdam()
+    config = tf.keras.optimizers.serialize(optimizer)
+    new_optimizer = tf.keras.optimizers.deserialize(config)
+    assert new_optimizer.get_config() == optimizer.get_config()
diff --git a/tensorflow_addons/optimizers/tests/lookahead_test.py b/tensorflow_addons/optimizers/tests/lookahead_test.py
index b27f0cfe71..257f32b63e 100644
--- a/tensorflow_addons/optimizers/tests/lookahead_test.py
+++ b/tensorflow_addons/optimizers/tests/lookahead_test.py
@@ -149,3 +149,10 @@ def test_get_config():
     config = opt.get_config()
     assert config["sync_period"] == 10
     assert config["slow_step_size"] == 0.4
+
+
+def test_serialization():
+    optimizer = Lookahead("adam", sync_period=10, slow_step_size=0.4)
+    config = tf.keras.optimizers.serialize(optimizer)
+    new_optimizer = tf.keras.optimizers.deserialize(config)
+    assert new_optimizer.get_config() == optimizer.get_config()
diff --git a/tensorflow_addons/optimizers/tests/moving_average_test.py b/tensorflow_addons/optimizers/tests/moving_average_test.py
index 25f7d752c9..f89d652a02 100644
--- a/tensorflow_addons/optimizers/tests/moving_average_test.py
+++ b/tensorflow_addons/optimizers/tests/moving_average_test.py
@@ -169,3 +169,13 @@ def test_fit_simple_linear_model():
 
     max_abs_diff = np.max(np.abs(predicted - y))
     assert max_abs_diff < 5e-3
+
+
+def test_serialization():
+    sgd_opt = tf.keras.optimizers.SGD(lr=2.0, nesterov=True, momentum=0.3, decay=0.1)
+    optimizer = MovingAverage(
+        sgd_opt, average_decay=0.5, num_updates=None, sequential_update=False
+    )
+    config = tf.keras.optimizers.serialize(optimizer)
+    new_optimizer = tf.keras.optimizers.deserialize(config)
+    assert new_optimizer.get_config() == optimizer.get_config()
diff --git a/tensorflow_addons/optimizers/tests/novograd_test.py b/tensorflow_addons/optimizers/tests/novograd_test.py
index 45d8ecb4f8..f822761954 100644
--- a/tensorflow_addons/optimizers/tests/novograd_test.py
+++ b/tensorflow_addons/optimizers/tests/novograd_test.py
@@ -139,3 +139,10 @@ def test_get_config():
     assert config["learning_rate"] == 1e-4
     assert config["weight_decay"] == 0.0
     assert config["grad_averaging"] is False
+
+
+def test_serialization():
+    optimizer = NovoGrad(lr=1e-4, weight_decay=0.0, grad_averaging=False)
+    config = tf.keras.optimizers.serialize(optimizer)
+    new_optimizer = tf.keras.optimizers.deserialize(config)
+    assert new_optimizer.get_config() == optimizer.get_config()
diff --git a/tensorflow_addons/optimizers/tests/stochastic_weight_averaging_test.py b/tensorflow_addons/optimizers/tests/stochastic_weight_averaging_test.py
index 820c827d60..c4d09a7c6c 100644
--- a/tensorflow_addons/optimizers/tests/stochastic_weight_averaging_test.py
+++ b/tensorflow_addons/optimizers/tests/stochastic_weight_averaging_test.py
@@ -113,3 +113,13 @@ def test_fit_simple_linear_model():
 
     max_abs_diff = np.max(np.abs(predicted - y))
     assert max_abs_diff < 1e-3
+
+
+def test_serialization():
+    start_averaging = 0
+    average_period = 1
+    sgd = tf.keras.optimizers.SGD(lr=1.0)
+    optimizer = SWA(sgd, start_averaging, average_period)
+    config = tf.keras.optimizers.serialize(optimizer)
+    new_optimizer = tf.keras.optimizers.deserialize(config)
+    assert new_optimizer.get_config() == optimizer.get_config()
diff --git a/tensorflow_addons/optimizers/tests/weight_decay_optimizers_test.py b/tensorflow_addons/optimizers/tests/weight_decay_optimizers_test.py
index c4b004e159..98f1379b2a 100644
--- a/tensorflow_addons/optimizers/tests/weight_decay_optimizers_test.py
+++ b/tensorflow_addons/optimizers/tests/weight_decay_optimizers_test.py
@@ -348,3 +348,10 @@ def test_optimizer_sparse(dtype, optimizer):
     do_test_sparse_repeated_indices(
         dtype, optimizer, learning_rate=0.001, momentum=0.9, weight_decay=WEIGHT_DECAY,
     )
+
+
+def test_serialization():
+    optimizer = weight_decay_optimizers.AdamW(learning_rate=1e-4, weight_decay=1e-4)
+    config = tf.keras.optimizers.serialize(optimizer)
+    new_optimizer = tf.keras.optimizers.deserialize(config)
+    assert new_optimizer.get_config() == optimizer.get_config()
diff --git a/tensorflow_addons/optimizers/tests/yogi_test.py b/tensorflow_addons/optimizers/tests/yogi_test.py
index daa8d52675..1cc0498eff 100644
--- a/tensorflow_addons/optimizers/tests/yogi_test.py
+++ b/tensorflow_addons/optimizers/tests/yogi_test.py
@@ -337,3 +337,10 @@ def test_get_config():
     opt = yogi.Yogi(1e-4)
     config = opt.get_config()
     assert config["learning_rate"] == 1e-4
+
+
+def test_serialization():
+    optimizer = yogi.Yogi(1e-4)
+    config = tf.keras.optimizers.serialize(optimizer)
+    new_optimizer = tf.keras.optimizers.deserialize(config)
+    assert new_optimizer.get_config() == optimizer.get_config()
diff --git a/tensorflow_addons/optimizers/yogi.py b/tensorflow_addons/optimizers/yogi.py
index 09db073d1d..bb4d264b02 100644
--- a/tensorflow_addons/optimizers/yogi.py
+++ b/tensorflow_addons/optimizers/yogi.py
@@ -320,8 +320,12 @@ def get_config(self):
                 "beta1": self._serialize_hyperparameter("beta_1"),
                 "beta2": self._serialize_hyperparameter("beta_2"),
                 "epsilon": self._serialize_hyperparameter("epsilon"),
-                "l1_t": self._serialize_hyperparameter("l1_regularization_strength"),
-                "l2_t": self._serialize_hyperparameter("l2_regularization_strength"),
+                "l1_regularization_strength": self._serialize_hyperparameter(
+                    "l1_regularization_strength"
+                ),
+                "l2_regularization_strength": self._serialize_hyperparameter(
+                    "l2_regularization_strength"
+                ),
                 "activation": self._activation,
                 "initial_accumulator_value": self._initial_accumulator_value,
             }

From 176c46e78632117fff906e91c5b0cac98938494b Mon Sep 17 00:00:00 2001
From: ashutosh1919 <ashutoshhathidara98@gmail.com>
Date: Sun, 26 Apr 2020 10:07:29 +0500
Subject: [PATCH 2/3] Check for review

---
 tensorflow_addons/optimizers/conditional_gradient.py           | 1 +
 .../optimizers/tests/conditional_gradient_test.py              | 3 ++-
 tensorflow_addons/testing/serialization.py                     | 2 +-
 3 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/tensorflow_addons/optimizers/conditional_gradient.py b/tensorflow_addons/optimizers/conditional_gradient.py
index 22fbc275f1..e62c8f9e36 100644
--- a/tensorflow_addons/optimizers/conditional_gradient.py
+++ b/tensorflow_addons/optimizers/conditional_gradient.py
@@ -86,6 +86,7 @@ def __init__(
                 compatibility, recommended to use `learning_rate` instead.
         """
         super().__init__(name=name, **kwargs)
+        self.__name__ = name
         self._set_hyper("learning_rate", kwargs.get("lr", learning_rate))
         self._set_hyper("lambda_", lambda_)
         self.epsilon = epsilon or tf.keras.backend.epsilon()
diff --git a/tensorflow_addons/optimizers/tests/conditional_gradient_test.py b/tensorflow_addons/optimizers/tests/conditional_gradient_test.py
index 0e84234fb9..5b729397c9 100644
--- a/tensorflow_addons/optimizers/tests/conditional_gradient_test.py
+++ b/tensorflow_addons/optimizers/tests/conditional_gradient_test.py
@@ -21,6 +21,7 @@
 import tensorflow as tf
 from tensorflow_addons.utils import test_utils
 from tensorflow_addons.optimizers import conditional_gradient as cg_lib
+from tensorflow_addons.testing import serialization
 
 
 def _dtypes_to_test(use_gpu):
@@ -1468,4 +1469,4 @@ def test_serialization():
     )
     config = tf.keras.optimizers.serialize(optimizer)
     new_optimizer = tf.keras.optimizers.deserialize(config)
-    assert new_optimizer.get_config() == optimizer.get_config()
+    serialization.check_config(new_optimizer.get_config(), optimizer, False)
diff --git a/tensorflow_addons/testing/serialization.py b/tensorflow_addons/testing/serialization.py
index 16aeeea481..988615dc77 100644
--- a/tensorflow_addons/testing/serialization.py
+++ b/tensorflow_addons/testing/serialization.py
@@ -76,7 +76,7 @@ def check_config(config, class_, strict):
                     class_.__name__
                 )
             )
-        if parameter_name not in config:
+        if parameter_name != "kwargs" and parameter_name not in config:
             raise KeyError(
                 "The constructor parameter {} is not present in the config dict "
                 "obtained with `.get_config()` of {}. All parameters should be set to "

From 70c9b2766a9c1e7e020cf64a035c43439b2a8e5f Mon Sep 17 00:00:00 2001
From: ashutosh1919 <ashutoshhathidara98@gmail.com>
Date: Mon, 27 Apr 2020 15:12:45 +0500
Subject: [PATCH 3/3] Reverted changes

---
 tensorflow_addons/optimizers/conditional_gradient.py           | 1 -
 .../optimizers/tests/conditional_gradient_test.py              | 3 +--
 tensorflow_addons/testing/serialization.py                     | 2 +-
 3 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/tensorflow_addons/optimizers/conditional_gradient.py b/tensorflow_addons/optimizers/conditional_gradient.py
index e62c8f9e36..22fbc275f1 100644
--- a/tensorflow_addons/optimizers/conditional_gradient.py
+++ b/tensorflow_addons/optimizers/conditional_gradient.py
@@ -86,7 +86,6 @@ def __init__(
                 compatibility, recommended to use `learning_rate` instead.
         """
         super().__init__(name=name, **kwargs)
-        self.__name__ = name
         self._set_hyper("learning_rate", kwargs.get("lr", learning_rate))
         self._set_hyper("lambda_", lambda_)
         self.epsilon = epsilon or tf.keras.backend.epsilon()
diff --git a/tensorflow_addons/optimizers/tests/conditional_gradient_test.py b/tensorflow_addons/optimizers/tests/conditional_gradient_test.py
index 5b729397c9..d153cc0534 100644
--- a/tensorflow_addons/optimizers/tests/conditional_gradient_test.py
+++ b/tensorflow_addons/optimizers/tests/conditional_gradient_test.py
@@ -21,7 +21,6 @@
 import tensorflow as tf
 from tensorflow_addons.utils import test_utils
 from tensorflow_addons.optimizers import conditional_gradient as cg_lib
-from tensorflow_addons.testing import serialization
 
 
 def _dtypes_to_test(use_gpu):
@@ -1469,4 +1468,4 @@ def test_serialization():
     )
     config = tf.keras.optimizers.serialize(optimizer)
     new_optimizer = tf.keras.optimizers.deserialize(config)
-    serialization.check_config(new_optimizer.get_config(), optimizer, False)
+    assert optimizer.get_config() == new_optimizer.get_config()
diff --git a/tensorflow_addons/testing/serialization.py b/tensorflow_addons/testing/serialization.py
index 988615dc77..16aeeea481 100644
--- a/tensorflow_addons/testing/serialization.py
+++ b/tensorflow_addons/testing/serialization.py
@@ -76,7 +76,7 @@ def check_config(config, class_, strict):
                     class_.__name__
                 )
             )
-        if parameter_name != "kwargs" and parameter_name not in config:
+        if parameter_name not in config:
             raise KeyError(
                 "The constructor parameter {} is not present in the config dict "
                 "obtained with `.get_config()` of {}. All parameters should be set to "