Skip to content

Commit 86026c4

Browse files
adrianlizarragaashrit-ms
authored andcommitted
[QNN EP]: Clean up QNN logging resources if an error occurs during initialization (#23435)
### Description Re-implementation of #23320 (which was reverted). - Cleans up QNN logging resources if an error occurs during initialization. - Updates `QnnLogging()`, which is a logging callback called by QNN libs, to handle situations in which ORT logging is unavailable, thus avoiding a segmentation fault. - Updates `QnnBackendManager::CreateHtpPowerCfgId()` and `QnnBackendManager::SetHtpPowerConfig()` to check that backend setup is complete. These functions get called in QNN EP's `OnRunStart()` even if QNN backend setup failed and the model is assigned to a different EP. This prevents a segmentation fault. Our Android tests ran into this issue because the QNN backend setup failed, the model was then assigned to CPU EP, and the QNN EP's `OnRunStart()` was still called with an invalid backend. ### Motivation and Context If QNN initialization fails at any point, we have to properly clean up the logging resources so that QNN does not call our `QnnLogging()` callback after the EP has been destroyed.
1 parent 65a6907 commit 86026c4

File tree

1 file changed

+27
-10
lines changed

1 file changed

+27
-10
lines changed

onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc

Lines changed: 27 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -251,6 +251,12 @@ void QnnLogging(const char* format,
251251
ORT_UNUSED_PARAMETER(level);
252252
ORT_UNUSED_PARAMETER(timestamp);
253253

254+
if (!::onnxruntime::logging::LoggingManager::HasDefaultLogger()) {
255+
// QNN may call this logging callback at any point, which means that we need to explicitly check
256+
// that the default logger has been initialized before trying to use it (otherwise get segfault).
257+
return;
258+
}
259+
254260
const auto& logger = ::onnxruntime::logging::LoggingManager::DefaultLogger();
255261
const auto severity = ::onnxruntime::logging::Severity::kVERBOSE;
256262
const auto data_type = ::onnxruntime::logging::DataType::SYSTEM;
@@ -273,6 +279,9 @@ Status QnnBackendManager::InitializeQnnLog(const logging::Logger& logger) {
273279
QnnLog_Level_t qnn_log_level = MapOrtSeverityToQNNLogLevel(ort_log_level);
274280
LOGS(*logger_, VERBOSE) << "Set Qnn log level: " << qnn_log_level;
275281

282+
// NOTE: Even if logCreate() fails and QNN does not return a valid log_handle_, QNN may still
283+
// call the QnnLogging() callback. So, we have to make sure that QnnLogging() can handle calls
284+
// in which ORT logging is not available.
276285
Qnn_ErrorHandle_t result = qnn_interface_.logCreate(QnnLogging, qnn_log_level, &log_handle_);
277286

278287
if (result != QNN_SUCCESS) {
@@ -879,6 +888,10 @@ Status QnnBackendManager::SetupBackend(const logging::Logger& logger,
879888
}
880889

881890
Status QnnBackendManager::CreateHtpPowerCfgId(uint32_t device_id, uint32_t core_id, uint32_t& htp_power_config_id) {
891+
// This function is called in QNN EP's OnRunStart() even if QNN backend setup failed and the model is assigned
892+
// to a different EP. Therefore, we have to check that backend setup actually completed before trying to
893+
// create an HTP power config ID. Otherwise, this causes a segfault because the QNN backend lib is unloaded.
894+
ORT_RETURN_IF_NOT(backend_setup_completed_, "Cannot create HTP power config ID if backend setup is not complete.");
882895
QnnDevice_Infrastructure_t qnn_device_infra = nullptr;
883896
auto status = qnn_interface_.deviceGetInfrastructure(&qnn_device_infra);
884897
ORT_RETURN_IF(QNN_SUCCESS != status, "backendGetPerfInfrastructure failed.");
@@ -896,6 +909,10 @@ Status QnnBackendManager::CreateHtpPowerCfgId(uint32_t device_id, uint32_t core_
896909

897910
Status QnnBackendManager::SetHtpPowerConfig(uint32_t htp_power_config_client_id,
898911
HtpPerformanceMode htp_performance_mode) {
912+
// This function is called in QNN EP's OnRunStart() even if QNN backend setup failed and the model is assigned
913+
// to a different EP. Therefore, we have to check that backend setup actually completed before trying to
914+
// set an HTP power config ID. Otherwise, this causes a segfault because the QNN backend lib is unloaded.
915+
ORT_RETURN_IF_NOT(backend_setup_completed_, "Cannot set HTP power config ID if backend setup is not complete.");
899916
QnnDevice_Infrastructure_t qnn_device_infra = nullptr;
900917
auto status = qnn_interface_.deviceGetInfrastructure(&qnn_device_infra);
901918
ORT_RETURN_IF(QNN_SUCCESS != status, "backendGetPerfInfrastructure failed.");
@@ -1037,6 +1054,10 @@ Status QnnBackendManager::SetHtpPowerConfig(uint32_t htp_power_config_client_id,
10371054

10381055
Status QnnBackendManager::SetRpcControlLatency(uint32_t htp_power_config_client_id,
10391056
uint32_t rpc_control_latency) {
1057+
// This function is called in QNN EP's OnRunStart() even if QNN backend setup failed and the model is assigned
1058+
// to a different EP. Therefore, we have to check that backend setup actually completed before trying to
1059+
// set RPC control latency. Otherwise, this causes a segfault because the QNN backend library is unloaded.
1060+
ORT_RETURN_IF_NOT(backend_setup_completed_, "Cannot set HTP RPC control latency if backend setup is not complete.");
10401061
if (rpc_control_latency != 0) {
10411062
QnnDevice_Infrastructure_t qnn_device_infra = nullptr;
10421063
auto status = qnn_interface_.deviceGetInfrastructure(&qnn_device_infra);
@@ -1099,39 +1120,35 @@ Status QnnBackendManager::TerminateQnnLog() {
10991120
}
11001121

11011122
void QnnBackendManager::ReleaseResources() {
1102-
if (!backend_setup_completed_) {
1103-
return;
1104-
}
1105-
11061123
auto result = ReleaseContext();
11071124
if (Status::OK() != result) {
1108-
LOGS_DEFAULT(ERROR) << "Failed to ReleaseContext.";
1125+
LOGS_DEFAULT(ERROR) << "Failed to ReleaseContext: " << result.ErrorMessage();
11091126
}
11101127

11111128
result = ReleaseProfilehandle();
11121129
if (Status::OK() != result) {
1113-
LOGS_DEFAULT(ERROR) << "Failed to ReleaseProfilehandle.";
1130+
LOGS_DEFAULT(ERROR) << "Failed to ReleaseProfilehandle: " << result.ErrorMessage();
11141131
}
11151132

11161133
result = ReleaseDevice();
11171134
if (Status::OK() != result) {
1118-
LOGS_DEFAULT(ERROR) << "Failed to ReleaseDevice.";
1135+
LOGS_DEFAULT(ERROR) << "Failed to ReleaseDevice: " << result.ErrorMessage();
11191136
}
11201137

11211138
result = ShutdownBackend();
11221139
if (Status::OK() != result) {
1123-
LOGS_DEFAULT(ERROR) << "Failed to ShutdownBackend.";
1140+
LOGS_DEFAULT(ERROR) << "Failed to ShutdownBackend: " << result.ErrorMessage();
11241141
}
11251142

11261143
result = TerminateQnnLog();
11271144
if (Status::OK() != result) {
1128-
LOGS_DEFAULT(ERROR) << "Failed to TerminateQnnLog.";
1145+
LOGS_DEFAULT(ERROR) << "Failed to TerminateQnnLog: " << result.ErrorMessage();
11291146
}
11301147

11311148
if (backend_lib_handle_) {
11321149
result = UnloadLib(backend_lib_handle_);
11331150
if (Status::OK() != result) {
1134-
LOGS_DEFAULT(ERROR) << "Failed to unload backend library.";
1151+
LOGS_DEFAULT(ERROR) << "Failed to unload backend library: " << result.ErrorMessage();
11351152
}
11361153
}
11371154

0 commit comments

Comments
 (0)