Add testing module, with an function check_metric_serialization (#1295)

* Added testing module

* Added an example in cohens_kappa_test.py

* Should work with bazel now.

* Changed import order.

* Forgot the if __name__ == "__main__"

* Use the new way of testing.

Author: gabrieldemarmiesse

tensorflow_addons/BUILD

@@ -26,6 +26,7 @@ py_library(
         "//tensorflow_addons/optimizers",
         "//tensorflow_addons/rnn",
         "//tensorflow_addons/seq2seq",
+        "//tensorflow_addons/testing",
         "//tensorflow_addons/text",
         "//tensorflow_addons/utils",
     ],

tensorflow_addons/metrics/BUILD

@@ -15,6 +15,7 @@ py_library(
         "utils.py",
     ],
     deps = [
+        "//tensorflow_addons/testing",
         "//tensorflow_addons/utils",
     ],
 )

tensorflow_addons/metrics/cohens_kappa.py

@@ -71,7 +71,6 @@ def __init__(
         sparse_labels: bool = False,
         regression: bool = False,
         dtype: AcceptableDTypes = None,
-        **kwargs
     ):
         """Creates a `CohenKappa` instance.
 

tensorflow_addons/metrics/cohens_kappa_test.py

@@ -19,6 +19,7 @@
 import tensorflow as tf
 from tensorflow_addons.metrics import CohenKappa
 from tensorflow_addons.utils import test_utils
+from tensorflow_addons.testing.serialization import check_metric_serialization
 
 
 @test_utils.run_all_in_graph_and_eager_modes
@@ -232,3 +233,12 @@ def test_with_ohe_labels():
 
     obj.update_state(y_true, y_pred)
     np.testing.assert_allclose(0.19999999, obj.result().numpy())
+
+
+def test_cohen_kappa_serialization():
+    actuals = np.array([4, 4, 3, 3, 2, 2, 1, 1], dtype=np.int32)
+    preds = np.array([1, 2, 4, 1, 3, 3, 4, 4], dtype=np.int32)
+    weights = np.array([1, 1, 2, 5, 10, 2, 3, 3], dtype=np.int32)
+
+    ck = CohenKappa(num_classes=5, sparse_labels=True, weightage="quadratic")
+    check_metric_serialization(ck, actuals, preds, weights)

tensorflow_addons/testing/BUILD

@@ -0,0 +1,21 @@
+licenses(["notice"])  # Apache 2.0
+
+package(default_visibility = ["//visibility:public"])
+
+py_library(
+    name = "testing",
+    srcs = [
+        "__init__.py",
+        "serialization.py",
+    ],
+)
+
+py_test(
+    name = "serialization_test",
+    size = "small",
+    srcs = glob(["*_test.py"]),
+    main = "run_all_test.py",
+    deps = [
+        ":testing",
+    ],
+)

tensorflow_addons/testing/__init__.py

@@ -0,0 +1,8 @@
+from pathlib import Path
+import sys
+
+import pytest
+
+if __name__ == "__main__":
+    dirname = Path(__file__).absolute().parent
+    sys.exit(pytest.main([str(dirname)]))

tensorflow_addons/testing/run_all_test.py

@@ -0,0 +1,93 @@
+from typing import Union
+import inspect
+
+import numpy as np
+from tensorflow.keras.metrics import Metric
+import typeguard
+
+
+@typeguard.typechecked
+def check_metric_serialization(
+    metric: Metric,
+    y_true: Union[tuple, np.ndarray],
+    y_pred: Union[tuple, np.ndarray],
+    sample_weight: Union[tuple, np.ndarray, None] = None,
+    strict: bool = True,
+):
+    config = metric.get_config()
+    class_ = metric.__class__
+
+    check_config(config, class_, strict)
+
+    metric_copy = class_(**config)
+    metric_copy.set_weights(metric.get_weights())
+
+    if isinstance(y_true, tuple):
+        y_true = get_random_array(y_true)
+    if isinstance(y_pred, tuple):
+        y_pred = get_random_array(y_pred)
+    if isinstance(sample_weight, tuple) and sample_weight is not None:
+        sample_weight = get_random_array(sample_weight)
+
+    # the behavior should be the same for the original and the copy
+    if sample_weight is None:
+        metric.update_state(y_true, y_pred)
+        metric_copy.update_state(y_true, y_pred)
+    else:
+        metric.update_state(y_true, y_pred, sample_weight)
+        metric_copy.update_state(y_true, y_pred, sample_weight)
+
+    assert_all_arrays_close(metric.get_weights(), metric_copy.get_weights())
+    metric_result = metric.result().numpy()
+    metric_copy_result = metric_copy.result().numpy()
+    if metric_result != metric_copy_result:
+        raise ValueError(
+            "The original gave a result of {} after an "
+            "`.update_states()` call, but the copy gave "
+            "a result of {} after the same "
+            "call.".format(metric_result, metric_copy_result)
+        )
+
+
+def check_config(config, class_, strict):
+    init_signature = inspect.signature(class_.__init__)
+
+    for parameter_name in init_signature.parameters:
+        if parameter_name == "self":
+            continue
+        elif parameter_name == "args" and strict:
+            raise KeyError(
+                "Please do not use args in the class constructor of {}, "
+                "as it hides the real signature "
+                "and degrades the user experience. "
+                "If you have no alternative to *args, "
+                "use `strict=False` in check_metric_serialization.".format(
+                    class_.__name__
+                )
+            )
+        elif parameter_name == "kwargs" and strict:
+            raise KeyError(
+                "Please do not use kwargs in the class constructor of {}, "
+                "as it hides the real signature "
+                "and degrades the user experience. "
+                "If you have no alternative to **kwargs, "
+                "use `strict=False` in check_metric_serialization.".format(
+                    class_.__name__
+                )
+            )
+        if parameter_name not in config:
+            raise KeyError(
+                "The constructor parameter {} is not present in the config dict "
+                "obtained with `.get_config()` of {}. All parameters should be set to "
+                "ensure a perfect copy of the keras object can be obtained when "
+                "serialized.".format(parameter_name, class_.__name__)
+            )
+
+
+def assert_all_arrays_close(list1, list2):
+    for array1, array2 in zip(list1, list2):
+        np.testing.assert_allclose(array1, array2)
+
+
+def get_random_array(shape):
+    return np.random.uniform(size=shape).astype(np.float32)

tensorflow_addons/testing/serialization.py

@@ -0,0 +1,70 @@
+import numpy as np
+import pytest
+import tensorflow as tf
+
+from tensorflow.keras.metrics import MeanAbsoluteError, TrueNegatives, Metric
+from tensorflow_addons.testing.serialization import check_metric_serialization
+
+
+def test_check_metric_serialization_mae():
+    check_metric_serialization(MeanAbsoluteError(), (2, 2), (2, 2))
+    check_metric_serialization(MeanAbsoluteError(name="hello"), (2, 2), (2, 2))
+    check_metric_serialization(MeanAbsoluteError(), (2, 2, 2), (2, 2, 2))
+    check_metric_serialization(MeanAbsoluteError(), (2, 2, 2), (2, 2, 2), (2, 2, 1))
+
+
+def get_random_booleans():
+    return np.random.uniform(0, 2, size=(2, 2))
+
+
+def test_check_metric_serialization_true_negative():
+    check_metric_serialization(
+        TrueNegatives(0.8),
+        np.random.uniform(0, 2, size=(2, 2)).astype(np.bool),
+        np.random.uniform(0, 1, size=(2, 2)).astype(np.float32),
+    )
+
+
+class MyDummyMetric(Metric):
+    def __init__(self, stuff, name):
+        super().__init__(name)
+        self.stuff = stuff
+
+    def update_state(self, y_true, y_pred, sample_weights):
+        pass
+
+    def get_config(self):
+        return super().get_config()
+
+    def result(self):
+        return 3
+
+
+def test_missing_arg():
+    with pytest.raises(KeyError) as exception_info:
+        check_metric_serialization(MyDummyMetric("dodo", "dada"), (2,), (2,))
+
+    assert "stuff" in str(exception_info.value)
+
+
+class MyOtherDummyMetric(Metric):
+    def __init__(self, to_add, name=None, dtype=None):
+        super().__init__(name, dtype)
+        self.to_add = to_add
+        self.sum_of_y_pred = self.add_weight(name="my_sum", initializer="zeros")
+
+    def update_state(self, y_true, y_pred, sample_weights=None):
+        self.sum_of_y_pred.assign_add(tf.math.reduce_sum(y_pred) + self.to_add)
+
+    def get_config(self):
+        config = {"to_add": self.to_add + 1}
+        config.update(super().get_config())
+        return config
+
+    def result(self):
+        return self.sum_of_y_pred
+
+
+def test_wrong_serialization():
+    with pytest.raises(AssertionError):
+        check_metric_serialization(MyOtherDummyMetric(5), (2,), (2,))