diff --git a/tensorflow_addons/optimizers/lazy_adam.py b/tensorflow_addons/optimizers/lazy_adam.py
index 698b3c2a72..4efe4416b3 100644
--- a/tensorflow_addons/optimizers/lazy_adam.py
+++ b/tensorflow_addons/optimizers/lazy_adam.py
@@ -55,7 +55,7 @@ def _resource_apply_sparse(self, grad, var, indices):
         local_step = tf.cast(self.iterations + 1, var_dtype)
         beta_1_power = tf.math.pow(beta_1_t, local_step)
         beta_2_power = tf.math.pow(beta_2_t, local_step)
-        epsilon_t = self._get_hyper('epsilon', var_dtype)
+        epsilon_t = tf.convert_to_tensor(self.epsilon, var_dtype)
         lr = (lr_t * tf.math.sqrt(1 - beta_2_power) / (1 - beta_1_power))
 
         # \\(m := beta1 * m + (1 - beta1) * g_t\\)
diff --git a/tensorflow_addons/optimizers/weight_decay_optimizers_test.py b/tensorflow_addons/optimizers/weight_decay_optimizers_test.py
index e265eecb3c..a4c203975c 100644
--- a/tensorflow_addons/optimizers/weight_decay_optimizers_test.py
+++ b/tensorflow_addons/optimizers/weight_decay_optimizers_test.py
@@ -221,7 +221,7 @@ def testBasicCallableParams(self):
             learning_rate=lambda: 0.001,
             beta_1=lambda: 0.9,
             beta_2=lambda: 0.999,
-            epsilon=lambda: 1e-8,
+            epsilon=1e-8,
             weight_decay=lambda: WEIGHT_DECAY)