Add function r2_score_comparable

Author: sdpython

_unittests/ut_metrics/test_scoring_metrics.py

@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+"""
+@brief      test log(time=12s)
+"""
+import unittest
+import pandas
+import numpy
+from sklearn import datasets
+from sklearn.linear_model import LinearRegression
+from sklearn.metrics import r2_score
+from pyquickhelper.pycode import ExtTestCase
+from mlinsights.metrics import r2_score_comparable
+
+
+class TestScoringMetrics(ExtTestCase):
+
+    def test_r2_score_comparable(self):
+        iris = datasets.load_iris()
+        X = iris.data[:, :4]
+        y = iris.target + 1
+        df = pandas.DataFrame(X)
+        df.columns = ["X1", "X2", "X3", "X4"]
+        model1 = LinearRegression().fit(X, y)
+        model2 = LinearRegression().fit(X, numpy.log(y))
+        r2a = r2_score(y, model1.predict(X))
+        r2b = r2_score(numpy.log(y), model2.predict(X))
+        r2c = r2_score_comparable(y, model2.predict(X), tr='log')
+        r2d = r2_score_comparable(y, model2.predict(X), inv_tr='exp')
+        self.assertEqual(r2b, r2c)
+        self.assertGreater(r2c, r2a)
+        self.assertLesser(r2a, r2d)
+        r2e = r2_score_comparable(y, model2.predict(X), inv_tr='exp', tr='exp')
+        self.assertLesser(r2e, 0)
+
+    def test_r2_score_comparable_exception(self):
+        iris = datasets.load_iris()
+        y = iris.target + 1
+        self.assertRaise(lambda: r2_score_comparable(y, y), ValueError)
+        self.assertRaise(
+            lambda: r2_score_comparable(y, y, tr="log2"),
+            TypeError)
+        self.assertRaise(
+            lambda: r2_score_comparable(y, y, inv_tr="log2"),
+            TypeError)
+
+
+if __name__ == "__main__":
+    unittest.main()

build_script.bat

@@ -5,8 +5,8 @@ set pythonexe="%1"
 goto custom_python:
 
 :default_value_python:
-set pythonexe="c:\Python391_x64\python.exe"
-if not exist %pythonexe% set pythonexe="c:\Python372_x64\python.exe"
+set pythonexe="c:\Python395_x64\python.exe"
+if not exist %pythonexe% set pythonexe="c:\Python391_x64\python.exe"
 :custom_python:
 @echo [python] %pythonexe%
 %pythonexe% -u setup.py build_script

mlinsights/metrics/__init__.py

@@ -4,3 +4,4 @@
 """
 
 from .correlations import non_linear_correlations
+from .scoring_metrics import r2_score_comparable

mlinsights/metrics/scoring_metrics.py

@@ -0,0 +1,93 @@
+"""
+@file
+@brief Metrics to compare machine learning.
+"""
+import numpy
+from sklearn.metrics import r2_score
+
+_known_functions = {
+    'exp': numpy.exp,
+    'log': numpy.log
+}
+
+
+def comparable_metric(metric_function, y_true, y_pred,
+                      tr="log", inv_tr='exp', **kwargs):
+    """
+    Applies function on either the true target or/and the predictions
+    before computing r2 score.
+
+    :param metric_function: metric to compute
+    :param y_true: expected targets
+    :param y_pred: predictions
+    :param sample_weight: weights
+    :param multioutput: see :epkg:`sklearn:metrics:r2_score`
+    :param tr: transformation applied on the target
+    :param inv_tr: transformation applied on the predictions
+    :return: results
+    """
+    tr = _known_functions.get(tr, tr)
+    inv_tr = _known_functions.get(inv_tr, inv_tr)
+    if tr is not None and not callable(tr):
+        raise TypeError("Argument tr must be callable.")
+    if inv_tr is not None and not callable(inv_tr):
+        raise TypeError("Argument inv_tr must be callable.")
+    if tr is None and inv_tr is None:
+        raise ValueError(
+            "tr and inv_tr cannot be both None at the same time.")
+    if tr is None:
+        return metric_function(y_true, inv_tr(y_pred), **kwargs)
+    if inv_tr is None:
+        return metric_function(tr(y_true), y_pred, **kwargs)
+    return metric_function(tr(y_true), inv_tr(y_pred), **kwargs)
+
+
+def r2_score_comparable(y_true, y_pred, *, sample_weight=None,
+                        multioutput='uniform_average',
+                        tr=None, inv_tr=None):
+    """
+    Applies function on either the true target or/and the predictions
+    before computing r2 score.
+
+    :param y_true: expected targets
+    :param y_pred: predictions
+    :param sample_weight: weights
+    :param multioutput: see :epkg:`sklearn:metrics:r2_score`
+    :param tr: transformation applied on the target
+    :param inv_tr: transformation applied on the predictions
+    :return: results
+
+    Example:
+
+    .. runpython::
+        :showcode:
+
+        import numpy
+        from sklearn import datasets
+        from sklearn.model_selection import train_test_split
+        from sklearn.linear_model import LinearRegression
+        from sklearn.metrics import r2_score
+        from mlinsights.metrics import r2_score_comparable
+
+        iris = datasets.load_iris()
+        X = iris.data[:, :4]
+        y = iris.target + 1
+
+        X_train, X_test, y_train, y_test = train_test_split(X, y)
+
+        model1 = LinearRegression().fit(X_train, y_train)
+        print('r2', r2_score(y_test, model1.predict(X_test)))
+        print('r2 log', r2_score(numpy.log(y_test), numpy.log(model1.predict(X_test))))
+        print('r2 log comparable', r2_score_comparable(
+            y_test, model1.predict(X_test), tr="log", inv_tr="log"))
+
+        model2 = LinearRegression().fit(X_train, numpy.log(y_train))
+        print('r2', r2_score(numpy.log(y_test), model2.predict(X_test)))
+        print('r2 log', r2_score(y_test, numpy.exp(model2.predict(X_test))))
+        print('r2 log comparable', r2_score_comparable(
+            y_test, model2.predict(X_test), inv_tr="exp"))
+    """
+    return comparable_metric(r2_score, y_true, y_pred,
+                             sample_weight=sample_weight,
+                             multioutput=multioutput,
+                             tr=tr, inv_tr=inv_tr)