INC Bench update (#1366)

Author: bmyrcha

conda_meta/full/meta.yaml

@@ -41,6 +41,7 @@ requirements:
     - sqlalchemy ==1.4.27
     - alembic ==1.7.7
     - cython
+    - pywin32  # [win]
 test:
   imports:
     - neural_compressor

neural_compressor/ux/components/configuration_wizard/configuration_parser.py

@@ -15,10 +15,12 @@
 """Configuration type parser."""
 import json
 from collections.abc import Iterable
+from copy import deepcopy
 from typing import Any, Dict, List, Type, Union
 
 from neural_compressor.ux.utils.exceptions import ClientErrorException
 from neural_compressor.ux.utils.hw_info import HWInfo
+from neural_compressor.ux.utils.logger import log
 from neural_compressor.ux.utils.utils import parse_bool_value
 
 
@@ -92,8 +94,9 @@ def __init__(self) -> None:
             "bool": bool,
         }
 
-    def parse(self, data: dict) -> dict:
+    def parse(self, input_data: dict) -> dict:
         """Parse configuration."""
+        data = deepcopy(input_data)
         transforms_data = data.get("transform", None)
         if transforms_data is not None:
             data.update({"transform": self.parse_transforms(transforms_data)})
@@ -110,7 +113,9 @@ def parse(self, data: dict) -> dict:
 
         metric_params = data.get("metric_param", None)
         if metric_params and isinstance(metric_params, dict):
-            data["metric_param"] = self.parse_metric(metric_params)
+            parsed_metric_params = self.parse_metric(metric_params)
+
+            data.update({"metric_param": parsed_metric_params})
 
         if "tuning" in data.keys():
             data["tuning"] = parse_bool_value(data["tuning"])
@@ -227,13 +232,15 @@ def parse_metric(self, metric_data: dict) -> dict:
         for param_name, param_value in metric_data.items():
             if isinstance(param_value, dict):
                 parsed_data.update({param_name: self.parse_metric(param_value)})
-            elif isinstance(param_value, str):
+            elif isinstance(param_value, str) or isinstance(param_value, int):
                 if param_value == "":
                     continue
                 param_type = self.get_param_type("metric", param_name)
                 if param_type is None:
+                    log.debug("Could not find param type.")
                     continue
-                parsed_data.update({param_name: self.parse_value(param_value, param_type)})
+                parsed_value = self.parse_value(param_value, param_type)
+                parsed_data.update({param_name: parsed_value})
         return parsed_data
 
     def get_param_type(
@@ -313,6 +320,8 @@ def normalize_string_list(string_list: str, required_type: Union[Type, List[Type
     if not isinstance(string_list, str):
         return string_list
     if isinstance(required_type, list):
+        string_list = string_list.replace("(", "[")
+        string_list = string_list.replace(")", "]")
         while not string_list.startswith("[["):
             string_list = "[" + string_list
         while not string_list.endswith("]]"):

neural_compressor/ux/components/configuration_wizard/get_boundary_nodes.py

@@ -48,6 +48,7 @@ def get_boundary_nodes(data: Dict[str, Any]) -> None:
         try:
             model = model_repository.get_model(model_path)
         except NotFoundException:
+            log.debug(f"Could not get model instance for {model_path}")
             supported_frameworks = model_repository.get_frameworks()
             raise ClientErrorException(
                 f"Framework for specified model is not yet supported. "

neural_compressor/ux/components/db_manager/db_models/model.py

@@ -14,7 +14,7 @@
 # limitations under the License.
 """The Model class."""
 import json
-from typing import Any, List
+from typing import Any, List, Optional
 
 from sqlalchemy import Boolean, Column, DateTime, Float, ForeignKey, Integer, String
 from sqlalchemy.orm import relationship, session
@@ -54,6 +54,17 @@ class Model(Base):
         "Optimization",
         back_populates="optimized_model",
         primaryjoin="Optimization.optimized_model_id == Model.id",
+    )
+
+    benchmarks: Any = relationship(
+        "Benchmark",
+        back_populates="model",
+        cascade="all, delete",
+    )
+
+    profilings: Any = relationship(
+        "Profiling",
+        back_populates="model",
         cascade="all, delete",
     )
 
@@ -147,6 +158,26 @@ def list(db_session: session.Session, project_id: int) -> dict:
             )
         return {"models": models}
 
+    @staticmethod
+    def delete_model(
+        db_session: session.Session,
+        model_id: int,
+        model_name: str,
+    ) -> Optional[int]:
+        """Remove model from database."""
+        model = (
+            db_session.query(Model)
+            .filter(Model.id == model_id)
+            .filter(Model.name == model_name)
+            .one_or_none()
+        )
+        if model is None:
+            return None
+        db_session.delete(model)
+        db_session.flush()
+
+        return int(model.id)
+
     @staticmethod
     def build_info(model: Any) -> dict:
         """Get model info."""

neural_compressor/ux/components/db_manager/db_models/optimization.py

@@ -15,9 +15,10 @@
 # pylint: disable=no-member
 """The Optimization class."""
 import json
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Union
 
 from sqlalchemy import DDL, Column, DateTime, ForeignKey, Integer, String, event
+from sqlalchemy.engine import Connection
 from sqlalchemy.orm import relationship, session
 from sqlalchemy.sql import func
 
@@ -554,6 +555,21 @@ def list(db_session: session.Session, project_id: int) -> dict:
             optimizations.append(optimization_info)
         return {"optimizations": optimizations}
 
+    @staticmethod
+    def unpin_benchmark(
+        db_connection: Union[session.Session, Connection],
+        benchmark_id: int,
+    ) -> None:
+        """Unpin benchmark from optimization."""
+        update_queries = [
+            f"UPDATE optimization SET performance_benchmark_id=null "
+            f"WHERE performance_benchmark_id={benchmark_id}",
+            f"UPDATE optimization SET accuracy_benchmark_id=null "
+            f"WHERE accuracy_benchmark_id={benchmark_id}",
+        ]
+        for update_query in update_queries:
+            db_connection.execute(update_query)
+
     @staticmethod
     def build_info(
         optimization: Any,

neural_compressor/ux/components/db_manager/db_operations/benchmark_api_interface.py

@@ -16,9 +16,11 @@
 """INC Bench Benchmark API interface."""
 import os
 import shutil
+from sqlite3 import Connection
 from typing import List, Optional, Union
 
-from sqlalchemy.orm import sessionmaker
+from sqlalchemy import event
+from sqlalchemy.orm import Mapper, sessionmaker
 
 from neural_compressor.ux.components.benchmark import Benchmarks
 from neural_compressor.ux.components.configuration_wizard.configuration_parser import (
@@ -27,6 +29,7 @@
 from neural_compressor.ux.components.db_manager.db_manager import DBManager
 from neural_compressor.ux.components.db_manager.db_models.benchmark import Benchmark
 from neural_compressor.ux.components.db_manager.db_models.benchmark_result import BenchmarkResult
+from neural_compressor.ux.components.db_manager.db_models.optimization import Optimization
 from neural_compressor.ux.components.db_manager.db_operations.project_api_interface import (
     ProjectAPIInterface,
 )
@@ -63,6 +66,10 @@ def delete_benchmark(data: dict) -> dict:
             benchmark_details = Benchmark.details(db_session, benchmark_id)
             project_id = benchmark_details["project_id"]
             project_details = ProjectAPIInterface.get_project_details({"id": project_id})
+            Optimization.unpin_benchmark(
+                db_connection=db_session,
+                benchmark_id=benchmark_id,
+            )
             removed_benchmark_id = Benchmark.delete_benchmark(
                 db_session=db_session,
                 benchmark_id=benchmark_id,
@@ -455,3 +462,16 @@ def clean_status(status_to_clean: ExecutionStatus) -> dict:
                 status_to_clean=status_to_clean,
             )
         return response
+
+
+@event.listens_for(Benchmark, "before_delete")
+def before_delete_benchmark_entry(
+    mapper: Mapper,
+    connection: Connection,
+    benchmark: Benchmark,
+) -> None:
+    """Clean up benchmark data before remove."""
+    Optimization.unpin_benchmark(
+        db_connection=connection,
+        benchmark_id=benchmark.id,
+    )

neural_compressor/ux/components/db_manager/db_operations/model_api_interface.py

@@ -96,6 +96,25 @@ def list_models(data: dict) -> dict:
 
         return models_list
 
+    @staticmethod
+    def delete_model(data: dict) -> dict:
+        """Delete model from database."""
+        try:
+            model_id: int = int(data.get("id", None))
+            model_name: str = str(data.get("name", None))
+        except ValueError:
+            raise ClientErrorException("Could not parse value.")
+        except TypeError:
+            raise ClientErrorException("Missing model id or model name.")
+        with Session.begin() as db_session:
+            removed_model_id = Model.delete_model(
+                db_session=db_session,
+                model_id=model_id,
+                model_name=model_name,
+            )
+
+        return {"id": removed_model_id}
+
     @staticmethod
     def parse_model_data(data: dict) -> ModelAddParamsInterface:
         """Parse input data for model."""

neural_compressor/ux/components/model/onnxrt/model.py

@@ -217,7 +217,7 @@ def guard_requirements_installed(self) -> None:
         """Ensure all requirements are installed."""
         check_module("onnx")
         check_module("onnxruntime")
-        if sys.version_info < (3,10): # pragma: no cover
+        if sys.version_info < (3, 10):  # pragma: no cover
             check_module("onnxruntime_extensions")
 
     @property

neural_compressor/ux/components/model/repository.py

@@ -24,6 +24,7 @@
 from neural_compressor.ux.components.model.tensorflow.meta_graph import MetaGraphModel
 from neural_compressor.ux.components.model.tensorflow.saved_model import SavedModelModel
 from neural_compressor.ux.utils.exceptions import NotFoundException
+from neural_compressor.ux.utils.logger import log
 
 
 class ModelRepository:
@@ -43,6 +44,8 @@ def __init__(self) -> None:
     def get_model(self, path: str) -> Model:
         """Get Model for given path."""
         for model_type in self.model_types:
+            supports_path = model_type.supports_path(path)
+            log.debug(f"{model_type.__name__}: {supports_path}")
             if model_type.supports_path(path):
                 return model_type(path)
 

neural_compressor/ux/components/optimization/execute_optimization.py

@@ -233,6 +233,20 @@ def execute_optimization(data: Dict[str, Any]) -> dict:
             if is_pytorch_script:
                 optimized_model_data["model_path"] = logs[0]
 
+            optimization_data = OptimizationAPIInterface.get_optimization_details(
+                {
+                    "id": optimization_id,
+                },
+            )
+            if optimization_data["optimized_model"] is not None:
+                existing_model_id = optimization_data["optimized_model"]["id"]
+                existing_model_name = optimization_data["optimized_model"]["name"]
+                ModelAPIInterface.delete_model(
+                    {
+                        "id": existing_model_id,
+                        "name": existing_model_name,
+                    },
+                )
             optimized_model_id = ModelAPIInterface.add_model(optimized_model_data)
             OptimizationAPIInterface.update_optimized_model(
                 {