Skip to content

Commit 907ede2

Browse files
authored
Adds automatic loading of the external tensorrt_plugins library in the TensorRT-RTX Execution Provider to support custom transformer plugins (#26558)
External TensorRT plugins for transformer models were previously only available as internal TensorRT components. To enable flexible deployment these plugins have been externalized into a separate library.
1 parent 5955798 commit 907ede2

File tree

1 file changed

+17
-0
lines changed

1 file changed

+17
-0
lines changed

onnxruntime/core/providers/nv_tensorrt_rtx/nv_execution_provider_custom_ops.cc

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,23 @@ common::Status CreateTensorRTCustomOpDomainList(std::vector<OrtCustomOpDomain*>&
6767
is_loaded = true;
6868
}
6969

70+
// Load external tensorrt_plugins library from EP directory (same location as nvinfer_plugin)
71+
// This library contains GroupQueryAttention and RotaryEmbedding plugins for transformer models
72+
try {
73+
const auto& env = onnxruntime::GetDefaultEnv();
74+
auto external_plugin_path = env.GetRuntimePath() +
75+
PathString(LIBRARY_PREFIX ORT_TSTR("tensorrt_plugins") LIBRARY_EXTENSION);
76+
void* external_plugin_handle = nullptr;
77+
auto status = env.LoadDynamicLibrary(external_plugin_path, false, &external_plugin_handle);
78+
if (status.IsOK()) {
79+
LOGS_DEFAULT(INFO) << "[NvTensorRTRTX EP] External plugins loaded: tensorrt_plugins (GQA + RotaryEmbedding)";
80+
} else {
81+
LOGS_DEFAULT(VERBOSE) << "[NvTensorRTRTX EP] tensorrt_plugins library not found in runtime path (optional)";
82+
}
83+
} catch (const std::exception& e) {
84+
LOGS_DEFAULT(VERBOSE) << "[NvTensorRTRTX EP] tensorrt_plugins library not available: " << e.what();
85+
}
86+
7087
try {
7188
// Get all registered TRT plugins from registry
7289
LOGS_DEFAULT(VERBOSE) << "[NvTensorRTRTX EP] Getting all registered TRT plugins from TRT plugin registry ...";

0 commit comments

Comments
 (0)