diff --git a/tensorflow_addons/optimizers/moving_average.py b/tensorflow_addons/optimizers/moving_average.py
index 89367c8668..2af1072ff1 100644
--- a/tensorflow_addons/optimizers/moving_average.py
+++ b/tensorflow_addons/optimizers/moving_average.py
@@ -78,9 +78,6 @@ def __init__(self,
             raise TypeError(
                 "optimizer is not an object of tf.keras.optimizers.Optimizer")
 
-        if num_updates is not None and not isinstance(num_updates, int):
-            raise TypeError("num_updates must be None or of integer type")
-
         if not isinstance(sequential_update, bool):
             raise TypeError("sequential_update must be of bool type")