Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Bug fix 1.18 #369

Draft
wants to merge 4 commits into
base: master
Choose a base branch
from
Draft

Bug fix 1.18 #369

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
e57c455
Bug fix for OV caching
preetha-intel Apr 22, 2024
c0d65e6
Handle exporting of dynamic shaped models when dynamic backend is req…
sspintel Apr 22, 2024
6c8073d
Support precision for AUTO device
preetha-intel Apr 22, 2024
7900f86
Fix lint issues
preetha-intel Apr 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions onnxruntime/core/providers/openvino/backend_manager.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,14 @@ BackendManager::BackendManager(const GlobalContext& global_context,
// the EPContext node.
Status BackendManager::ExportCompiledBlobAsEPCtxNode(const onnxruntime::GraphViewer& graph_body_viewer,
const logging::Logger& logger) {
if (GetGlobalContext().disable_dynamic_shapes && subgraph_context_.has_dynamic_input_shape) {
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@sspintel Is it feasible to add an assert statement at the time of session options parsing in openvino_provider_options.cc rather at the export stage.

std::string exception_str =
"Exporting dynamically compiled models at runtime is not supported. "
"Cannot export blobs of dynamic models that request static shape inference. "
"To export this model, set disable_dynamic_shapes to False";
ORT_THROW(exception_str);
}

std::string model_blob_str;
auto compiled_model = concrete_backend_->GetOVCompiledModel();
auto graph_name = global_context_.onnx_model_path_name;
Expand Down
2 changes: 2 additions & 0 deletions onnxruntime/core/providers/openvino/backends/basic_backend.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,8 +93,10 @@
} else if (!subgraph_context_.has_dynamic_input_shape &&
global_context_.onnx_model_path_name.find(".onnx") != std::string ::npos) {
// Inputs with static dimenstions
std::string prec_str = (global_context_.precision_str != "ACCURACY") ? global_context_.precision_str : global_context_.model_precision;

Check warning on line 96 in onnxruntime/core/providers/openvino/backends/basic_backend.cc

View workflow job for this annotation

GitHub Actions / cpplint

[cpplint] onnxruntime/core/providers/openvino/backends/basic_backend.cc#L96 

 Lines should be <= 120 characters long  [whitespace/line_length] [2]
Raw output 
onnxruntime/core/providers/openvino/backends/basic_backend.cc:96:  Lines should be <= 120 characters long  [whitespace/line_length] [2]
exe_network_ = global_context_.ie_core.CompileModel(global_context_.onnx_model_path_name,
hw_target,
prec_str,
global_context_.cache_dir,
device_config,
subgraph_context_.subgraph_name);
Expand Down
3 changes: 2 additions & 1 deletion onnxruntime/core/providers/openvino/openvino_execution_provider.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,8 @@ OpenVINOExecutionProvider::GetCapability(const GraphViewer& graph_viewer,
return "";
} else {
auto input_type = graph_viewer.GetInputs()[0]->TypeAsProto()->tensor_type().elem_type();
if (global_context_->precision_str == "ACCURACY" && global_context_->device_type == "GPU") {
if (global_context_->precision_str == "ACCURACY" &&
global_context_->device_type.find("GPU") != std::string::npos) {
if (input_type == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT) {
return "FP32";
} else if (input_type == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT16) {
Expand Down
10 changes: 1 addition & 9 deletions onnxruntime/core/providers/openvino/openvino_execution_provider.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,16 +120,8 @@ struct OpenVINOExecutionProviderInfo {
#endif
} else if (ov_supported_device_types.find(dev_type) != ov_supported_device_types.end()) {
device_type_ = dev_type;
} else if (dev_type.find("HETERO") == 0 || dev_type.find("MULTI") == 0) {
} else if (dev_type.find("HETERO") == 0 || dev_type.find("MULTI") == 0 || dev_type.find("AUTO") == 0) {
std::vector<std::string> devices = parseDevices(dev_type);
precision_ = "FP16";
if (devices[0] == "CPU") {
precision_ = "FP32";
}
device_type_ = dev_type;
} else if (dev_type.find("AUTO") == 0) {
std::vector<std::string> devices = parseDevices(dev_type);
precision_ = "FP32";
device_type_ = dev_type;
} else {
ORT_THROW("Invalid device string: " + dev_type);
Expand Down
22 changes: 11 additions & 11 deletions onnxruntime/core/providers/openvino/openvino_provider_factory.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,23 +136,23 @@ struct OpenVINO_Provider : Provider {
if (provider_options_map.find("precision") != provider_options_map.end()) {
precision = provider_options_map.at("precision").c_str();
}
if (device_type == "CPU") {
if (precision == "" || precision == "ACCURACY" || precision == "FP32") {
precision = "FP32";
} else {
ORT_THROW("[ERROR] [OpenVINO] Unsupported inference precision is selected. CPU only supports FP32 . \n");
if (device_type.find("GPU") != std::string::npos) {
if (precision == "") {
precision = "FP16";
} else if (precision != "ACCURACY" && precision != "FP16" && precision != "FP32") {
ORT_THROW("[ERROR] [OpenVINO] Unsupported inference precision is selected. GPU only supports FP32 / FP16. \n");
}
} else if (device_type == "NPU") {
} else if (device_type.find("NPU") != std::string::npos) {
if (precision == "" || precision == "ACCURACY" || precision == "FP16") {
precision = "FP16";
} else {
ORT_THROW("[ERROR] [OpenVINO] Unsupported inference precision is selected. NPU only supported FP16. \n");
}
} else if (device_type == "GPU") {
if (precision == "") {
precision = "FP16";
} else if (precision != "ACCURACY" && precision != "FP16" && precision != "FP32") {
ORT_THROW("[ERROR] [OpenVINO] Unsupported inference precision is selected. GPU only supports FP32 / FP16. \n");
} else if (device_type.find("CPU") != std::string::npos) {
if (precision == "" || precision == "ACCURACY" || precision == "FP32") {
precision = "FP32";
} else {
ORT_THROW("[ERROR] [OpenVINO] Unsupported inference precision is selected. CPU only supports FP32 . \n");
}
}

Expand Down
6 changes: 4 additions & 2 deletions onnxruntime/core/providers/openvino/ov_interface.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@ OVExeNetwork OVCore::CompileModel(std::shared_ptr<const OVNetwork>& ie_cnn_netwo

OVExeNetwork OVCore::CompileModel(const std::string onnx_model_path,
std::string& hw_target,
std::string precision,
std::string cache_dir,
ov::AnyMap& device_config,
std::string name) {
Expand All @@ -99,7 +100,8 @@ OVExeNetwork OVCore::CompileModel(const std::string onnx_model_path,
obj = oe.compile_model(onnx_model_path,
"AUTO",
ov::device::priorities("GPU", "CPU"),
ov::device::properties("GPU", ov::cache_dir(cache_dir)));
ov::device::properties("GPU", {ov::cache_dir(cache_dir),
ov::hint::inference_precision(precision)}));
} else {
obj = oe.compile_model(onnx_model_path, hw_target, device_config);
}
Expand Down Expand Up @@ -134,7 +136,7 @@ OVExeNetwork OVCore::ImportModel(std::shared_ptr<std::istringstream> model_strea
}

void OVCore::SetCache(std::string cache_dir_path, std::string device_type) {
if (device_type == "AUTO:GPU,CPU") {
if (device_type != "AUTO:GPU,CPU") {
oe.set_property(ov::cache_dir(cache_dir_path));
}
}
Expand Down
1 change: 1 addition & 0 deletions onnxruntime/core/providers/openvino/ov_interface.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ class OVCore {
std::string name);
OVExeNetwork CompileModel(const std::string onnx_model_path,
std::string& hw_target,
std::string precision,
std::string cache_dir,
ov::AnyMap& device_config,
std::string name);
Expand Down
26 changes: 14 additions & 12 deletions onnxruntime/test/perftest/ort_test_session.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -274,29 +274,31 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device
}
} else if (key == "precision") {
auto device_type = ov_options["device_type"];
if (device_type == "CPU") {
if (value == "" || value == "ACCURACY" || value == "FP32") {
ov_options[key] = "FP32";
if (device_type.find("GPU") != std::string::npos) {
if (value == "") {
ov_options[key] = "FP16";
continue;
} else if (value == "ACCURACY" || value == "FP16" || value == "FP32") {
ov_options[key] = value;
continue;
} else {
ORT_THROW("[ERROR] [OpenVINO] Unsupported inference precision is selected. CPU only supports FP32 . \n");
ORT_THROW(
"[ERROR] [OpenVINO] Unsupported inference precision is selected. "
"GPU only supported FP32 / FP16. \n");
}
} else if (device_type == "NPU") {
} else if (device_type.find("NPU") != std::string::npos) {
if (value == "" || value == "ACCURACY" || value == "FP16") {
ov_options[key] = "FP16";
continue;
} else {
ORT_THROW("[ERROR] [OpenVINO] Unsupported inference precision is selected. NPU only supported FP16. \n");
}
} else if (device_type == "GPU") {
if (value == "") {
ov_options[key] = "FP16";
continue;
} else if (value == "ACCURACY" || value == "FP16" || value == "FP32") {
ov_options[key] = value;
} else if (device_type.find("CPU") != std::string::npos) {
if (value == "" || value == "ACCURACY" || value == "FP32") {
ov_options[key] = "FP32";
continue;
} else {
ORT_THROW("[ERROR] [OpenVINO] Unsupported inference precision is selected. GPU only supported FP32 / FP16. \n");
ORT_THROW("[ERROR] [OpenVINO] Unsupported inference precision is selected. CPU only supports FP32 . \n");
}
}
} else if (key == "enable_npu_fast_compile") {
Expand Down
Loading