From 61b36ef3a9e5414d040deb435a95392f514482f9 Mon Sep 17 00:00:00 2001 From: jatinwadhwa921 Date: Tue, 4 Mar 2025 21:07:38 -0800 Subject: [PATCH] [OVEP] Fix for precision accuracy --- .../openvino/backends/basic_backend.cc | 4 +- .../openvino/openvino_parser_utils.cc | 86 +++++++++++++++++++ .../openvino/openvino_parser_utils.h | 22 +++++ .../openvino/openvino_provider_factory.cc | 60 +------------ 4 files changed, 113 insertions(+), 59 deletions(-) create mode 100644 onnxruntime/core/providers/openvino/openvino_parser_utils.cc create mode 100644 onnxruntime/core/providers/openvino/openvino_parser_utils.h diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.cc b/onnxruntime/core/providers/openvino/backends/basic_backend.cc index d026ce386e5c3..2e808333fd61d 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.cc +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.cc @@ -158,10 +158,8 @@ void BasicBackend::PopulateConfigValue(ov::AnyMap& device_config) { if (session_context_.precision.find("FP32") != std::string::npos) { device_config.emplace(ov::hint::inference_precision("f32")); } - if (session_context_.precision.find("ACCURACY") != std::string::npos && - session_context_.device_type.find("GPU") != std::string::npos) { + if (session_context_.precision.find("ACCURACY") != std::string::npos) { if (session_context_.OpenVINO_Version.at(0) >= 2024) { - device_config.emplace(ov::hint::inference_precision(ov::element::dynamic)); device_config.emplace(ov::hint::execution_mode(ov::hint::ExecutionMode::ACCURACY)); } else { if (!subgraph_context_.model_precision.empty()) diff --git a/onnxruntime/core/providers/openvino/openvino_parser_utils.cc b/onnxruntime/core/providers/openvino/openvino_parser_utils.cc new file mode 100644 index 0000000000000..a7e17d1b8e498 --- /dev/null +++ b/onnxruntime/core/providers/openvino/openvino_parser_utils.cc @@ -0,0 +1,86 @@ +#include +#include "core/providers/openvino/openvino_parser_utils.h" +#include "core/providers/shared_library/provider_api.h" + +namespace onnxruntime { +namespace openvino_ep { + +std::string OpenVINOParserUtils::ParsePrecision(const ProviderOptions& provider_options, + std::string& device_type, + const std::string& option_name) { + using DeviceName = std::string; + using DefaultValue = std::string; + using ValidValues = std::list; + using foo = std::pair; + using ParserHelper = std::map; + + ParserHelper helper = { + {"GPU", {"FP16", {"FP16", "FP32", "ACCURACY"}}}, + {"NPU", {"FP16", {"FP16", "ACCURACY"}}}, + {"CPU", {"FP32", {"FP32", "ACCURACY"}}}, + }; + + std::set deprecated_device_types = { + "CPU_FP32", "GPU_FP32", "GPU.0_FP32", "GPU.1_FP32", "GPU_FP16", + "GPU.0_FP16", "GPU.1_FP16"}; + + bool is_composite = device_type.find(':') != std::string::npos; // FOR devices AUTO:,HETERO:,MULTI: + + if (provider_options.contains(option_name)) { + const auto& precision = provider_options.at(option_name); + + if (is_composite) { + std::set allowed_precisions = {"FP16", "FP32", "ACCURACY"}; + if (allowed_precisions.contains(precision)) { + return precision; + } else { + ORT_THROW("[ERROR] [OpenVINO] Unsupported inference precision is selected. ", + precision, ".\n"); + } + } else { + if (helper.contains(device_type)) { + auto const& valid_values = helper[device_type].second; + + if (precision == "ACCURACY") { + return valid_values.back(); // Return highest supported precision + } else { + if (std::find(valid_values.begin(), valid_values.end(), precision) != valid_values.end()) { + return precision; // Return precision selected if valid + } else { + auto value_iter = valid_values.begin(); + std::string valid_values_joined = *value_iter; + // Append 2nd and up, if only one then ++value_iter is same as end() + for (++value_iter; value_iter != valid_values.end(); ++value_iter) { + valid_values_joined += ", " + *value_iter; + } + + ORT_THROW("[ERROR] [OpenVINO] Unsupported inference precision is selected. ", + device_type, " only supports", valid_values_joined, ".\n"); + } + } + } else if (deprecated_device_types.contains(device_type)) { + LOGS_DEFAULT(WARNING) + << "[OpenVINO] Selected 'device_type' " + device_type + " is deprecated. \n" + << "Update the 'device_type' to specified types 'CPU', 'GPU', 'GPU.0', " + << "'GPU.1', 'NPU' or from HETERO/MULTI/AUTO options and set 'precision' separately. \n"; + auto delimit = device_type.find("_"); + device_type = device_type.substr(0, delimit); + return device_type.substr(delimit + 1); + } else { + ORT_THROW("[ERROR] [OpenVINO] Unsupported device type provided: ", + device_type, "\n"); + } + } + } else { + if (device_type.find("NPU") != std::string::npos || device_type.find("GPU") != std::string::npos) { + return "FP16"; + } else if (device_type.find("CPU") != std::string::npos) { + return "FP32"; + } else { + ORT_THROW("[ERROR] [OpenVINO] Unsupported device is selected", device_type, "\n"); + } + } +} + +} // namespace openvino_ep +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/openvino/openvino_parser_utils.h b/onnxruntime/core/providers/openvino/openvino_parser_utils.h new file mode 100644 index 0000000000000..3e23c9e788463 --- /dev/null +++ b/onnxruntime/core/providers/openvino/openvino_parser_utils.h @@ -0,0 +1,22 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "core/framework/provider_options.h" + +namespace onnxruntime { +namespace openvino_ep { + +class OpenVINOParserUtils { + public: + static std::string ParsePrecision(const ProviderOptions& provider_options, + std::string& device_type, + const std::string& option_name); +}; + +} // namespace openvino_ep +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc index c4fe16e035241..a880c24760707 100644 --- a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc +++ b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc @@ -11,11 +11,12 @@ #include "core/providers/openvino/backend_utils.h" #include "core/session/onnxruntime_session_options_config_keys.h" #include "nlohmann/json.hpp" +#include "core/providers/openvino/openvino_parser_utils.h" namespace onnxruntime { namespace openvino_ep { void ParseConfigOptions(ProviderInfo& pi) { - if(pi.config_options==NULL) + if (pi.config_options == NULL) return; pi.so_disable_cpu_ep_fallback = pi.config_options->GetConfigOrDefault(kOrtSessionOptionsDisableCPUEPFallback, "0") == "1"; @@ -29,7 +30,6 @@ void ParseConfigOptions(ProviderInfo& pi) { map["NPU_COMPILATION_MODE_PARAMS"] = "enable-wd-blockarg-input=true compute-layers-with-higher-precision=Sqrt,Power,ReduceSum"; pi.load_config["NPU"] = std::move(map); } - } void* ParseUint64(const ProviderOptions& provider_options, std::string option_name) { @@ -115,58 +115,6 @@ std::string ParseDeviceType(std::shared_ptr ov_core, const ProviderOptio } } -// Depends on ProviderOptions. -std::string ParsePrecision(const ProviderOptions& provider_options, std::string& device_type, const std::string& option_name) { - using DeviceName = std::string; - using DefaultValue = std::string; - using ValidValues = std::list; - using foo = std::pair; - using ParserHelper = std::map; - ParserHelper helper = { - {"GPU", {"FP16", {"FP16", "FP32"}}}, - {"NPU", {"FP16", {"FP16"}}}, - {"CPU", {"FP32", {"FP32"}}}, - }; - - std::set deprecated_device_types = {"CPU_FP32", "GPU_FP32", - "GPU.0_FP32", "GPU.1_FP32", "GPU_FP16", - "GPU.0_FP16", "GPU.1_FP16"}; - - if (provider_options.contains(option_name)) { - // Start by checking if the device_type is a normal valid one - if (helper.contains(device_type)) { - auto const& valid_values = helper[device_type].second; - const auto& precision = provider_options.at(option_name); - if (precision == "ACCURACY") { - return valid_values.back(); // Return highest supported precision - } else { - if (std::find(valid_values.begin(), valid_values.end(), precision) != valid_values.end()) { - return precision; // Return precision selected if valid - } else { - auto value_iter = valid_values.begin(); - std::string valid_values_joined = *value_iter; - // Append 2nd and up, if only one then ++value_iter is same as end() - for (++value_iter; value_iter != valid_values.end(); ++value_iter) { - valid_values_joined += ", " + *value_iter; - } - - ORT_THROW("[ERROR] [OpenVINO] Unsupported inference precision is selected. ", device_type, " only supports", valid_values_joined, ".\n"); - } - } - } else if (deprecated_device_types.contains(device_type)) { - LOGS_DEFAULT(WARNING) << "[OpenVINO] Selected 'device_type' " + device_type + " is deprecated. \n" - << "Update the 'device_type' to specified types 'CPU', 'GPU', 'GPU.0', " - << "'GPU.1', 'NPU' or from" - << " HETERO/MULTI/AUTO options and set 'precision' separately. \n"; - auto delimit = device_type.find("_"); - device_type = device_type.substr(0, delimit); - return device_type.substr(delimit + 1); - } - } - // Return default - return helper[device_type].first; -} - void ParseProviderOptions([[maybe_unused]] ProviderInfo& result, [[maybe_unused]] const ProviderOptions& config_options) {} struct OpenVINOProviderFactory : IExecutionProviderFactory { @@ -204,7 +152,7 @@ struct OpenVINO_Provider : Provider { const ProviderOptions* provider_options_ptr = reinterpret_cast(pointers_array[0]); const ConfigOptions* config_options = reinterpret_cast(pointers_array[1]); - if(provider_options_ptr == NULL) { + if (provider_options_ptr == NULL) { LOGS_DEFAULT(ERROR) << "[OpenVINO EP] Passed NULL ProviderOptions to CreateExecutionProviderFactory()"; return nullptr; } @@ -234,7 +182,7 @@ struct OpenVINO_Provider : Provider { pi.cache_dir = provider_options.at("cache_dir"); } - pi.precision = ParsePrecision(provider_options, pi.device_type, "precision"); + pi.precision = OpenVINOParserUtils::ParsePrecision(provider_options, pi.device_type, "precision"); if (provider_options.contains("load_config")) { auto parse_config = [&](const std::string& config_str) -> std::map {