From f71f00996f94cbb64466f535a4eba13c351f8090 Mon Sep 17 00:00:00 2001 From: "Cheng, Zixuan" Date: Fri, 28 Apr 2023 11:38:07 +0800 Subject: [PATCH 1/2] fix metric in mixed precision Signed-off-by: Cheng, Zixuan --- neural_compressor/mix_precision.py | 40 +++++++++++++++++++----------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/neural_compressor/mix_precision.py b/neural_compressor/mix_precision.py index 1f5bc0445ba..4c9551e9c36 100644 --- a/neural_compressor/mix_precision.py +++ b/neural_compressor/mix_precision.py @@ -252,35 +252,47 @@ def metric(self): def metric(self, user_metric): """Set metric class or a dict of built-in metric configures. - 1. neural_compressor have many built-in metrics, user can pass a metric configure dict to tell neural - compressor what metric will be use. - You can set multi-metrics to evaluate the performance of a specific model. + 1. neural_compressor have many built-in metrics, + user can pass a metric configure dict to tell neural compressor what metric will be use. + You also can set multi-metrics to evaluate the performance of a specific model. Single metric: {topk: 1} - Multi-metrics: {topk: 1, MSE: {compare_label: False}, weight: [0.5, 0.5], higher_is_better: [True, False] } - Refer to this [file](../docs/source/metric.md#supported-built-in-metric-matrix) for built-in metric list - 2. User also can set specific metric through this api. The metric class should take the outputs of the model or - postprocess(if have) as inputs, neural_compressor built-in metric always take(predictions, labels) as inputs - for update, and user_metric.metric_cls should be sub_class of neural_compressor.metric.BaseMetric. + For the built-in metrics, please refer to below link: + https://github.com/intel/neural-compressor/blob/master/docs/source/metric.md#supported-built-in-metric-matrix. + + 2. User also can get the built-in metrics by neural_compressor.Metric: + Metric(name="topk", k=1) + 3. User also can set specific metric through this api. The metric class should take the outputs of the model or + postprocess(if have) as inputs, neural_compressor built-in metric always take(predictions, labels) + as inputs for update, and user_metric.metric_cls should be sub_class of neural_compressor.metric.BaseMetric. Args: - user_metric(neural_compressor.metric.Metric or a dict of built-in metric configures): + user_metric(neural_compressor.metric.Metric or a dict of built-in metric configurations): The object of Metric or a dict of built-in metric configurations. + """ - from .metric import Metric as NCMetric, METRICS + from .metric import Metric as NCMetric + from .metric import METRICS if isinstance(user_metric, dict): metric_cfg = user_metric else: if isinstance(user_metric, NCMetric): - name = user_metric.name - metric_cls = user_metric.metric_cls - metric_cfg = {name: {**user_metric.kwargs}} + if user_metric.metric_cls is None: + name = user_metric.name + metric_cls = METRICS(self.conf.quantization.framework).metrics[name] + metric_cfg = {name: {**user_metric.kwargs}} + self._metric = metric_cfg + return + else: + name = user_metric.name + metric_cls = user_metric.metric_cls + metric_cfg = {name: {**user_metric.kwargs}} else: for i in ['reset', 'update', 'result']: assert hasattr(user_metric, i), 'Please realise {} function' \ @@ -288,7 +300,7 @@ def metric(self, user_metric): metric_cls = type(user_metric).__name__ name = 'user_' + metric_cls metric_cfg = {name: id(user_metric)} - metrics = METRICS(self.conf.mixed_precision.framework) + metrics = METRICS(self.conf.quantization.framework) metrics.register(name, metric_cls) self._metric = metric_cfg From 07fb32e442d80351670a2a782bd6757326149680 Mon Sep 17 00:00:00 2001 From: "Cheng, Zixuan" Date: Fri, 28 Apr 2023 13:53:30 +0800 Subject: [PATCH 2/2] minor fix Signed-off-by: Cheng, Zixuan --- neural_compressor/mix_precision.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/neural_compressor/mix_precision.py b/neural_compressor/mix_precision.py index 4c9551e9c36..f997e9f5471 100644 --- a/neural_compressor/mix_precision.py +++ b/neural_compressor/mix_precision.py @@ -285,7 +285,7 @@ def metric(self, user_metric): if isinstance(user_metric, NCMetric): if user_metric.metric_cls is None: name = user_metric.name - metric_cls = METRICS(self.conf.quantization.framework).metrics[name] + metric_cls = METRICS(self.conf.mixed_precision.framework).metrics[name] metric_cfg = {name: {**user_metric.kwargs}} self._metric = metric_cfg return @@ -300,7 +300,7 @@ def metric(self, user_metric): metric_cls = type(user_metric).__name__ name = 'user_' + metric_cls metric_cfg = {name: id(user_metric)} - metrics = METRICS(self.conf.quantization.framework) + metrics = METRICS(self.conf.mixed_precision.framework) metrics.register(name, metric_cls) self._metric = metric_cfg