add option to force reload TRT EP library

Author: chilo-ms

onnxruntime/core/providers/tensorrt/tensorrt_provider_factory_creator.h

@@ -16,5 +16,6 @@ struct TensorrtProviderFactoryCreator {
   static std::shared_ptr<IExecutionProviderFactory> Create(int device_id);
   static std::shared_ptr<IExecutionProviderFactory> Create(const OrtTensorRTProviderOptions* provider_options);
   static std::shared_ptr<IExecutionProviderFactory> Create(const OrtTensorRTProviderOptionsV2* provider_options);
+  static void UnloadLibrary();
 };
 }  // namespace onnxruntime

onnxruntime/core/session/provider_bridge_ort.cc

@@ -2214,6 +2214,10 @@ std::shared_ptr<IExecutionProviderFactory> TensorrtProviderFactoryCreator::Creat
   return nullptr;
 }
 
+void TensorrtProviderFactoryCreator::UnloadLibrary() {
+  s_library_tensorrt.Unload();
+}
+
 std::shared_ptr<IExecutionProviderFactory> NvProviderFactoryCreator::Create(int device_id) try {
   return s_library_nv.Get().CreateExecutionProviderFactory(device_id);
 } catch (const std::exception& exception) {

onnxruntime/test/providers/tensorrt/tensorrt_basic_test.cc

@@ -1425,7 +1425,11 @@ TEST(TensorrtExecutionProviderTest, PartiallySupportedModel_MemcpyOpsOnCPU_Infer
   OrtTensorRTProviderOptionsV2 params;
   params.trt_use_cpu_ep_memcpy_kernels = true;
   params.trt_op_types_to_exclude = "Mul";
-  std::unique_ptr<IExecutionProvider> execution_provider = TensorrtExecutionProviderWithOptions(&params);
+
+  // Previous unit tests alread loaded TRT EP library that registered the Memcpy kernels to TRT EP, we need
+  // to reload the TRT EP library after setting the trt_use_cpu_ep_memcpy_kernels option to make sure the
+  // Memcpy kernels are registered to CPU EP instead of TRT EP.
+  std::unique_ptr<IExecutionProvider> execution_provider = TensorrtExecutionProviderWithOptions(&params, /*force_reload_library*/ true);
   EXPECT_TRUE(session_object.RegisterExecutionProvider(std::move(execution_provider)).IsOK());
 
   auto status = session_object.Load(ORT_TSTR("testdata/add_mul_add.onnx"));

onnxruntime/test/util/default_providers.cc

@@ -70,8 +70,12 @@ std::unique_ptr<IExecutionProvider> TensorrtExecutionProviderWithOptions(const O
   return nullptr;
 }
 
-std::unique_ptr<IExecutionProvider> TensorrtExecutionProviderWithOptions(const OrtTensorRTProviderOptionsV2* params) {
+std::unique_ptr<IExecutionProvider> TensorrtExecutionProviderWithOptions(const OrtTensorRTProviderOptionsV2* params,
+                                                                         bool force_reload_library) {
 #ifdef USE_TENSORRT
+  if (force_reload_library) {
+    TensorrtProviderFactoryCreator::UnloadLibrary();
+  }
   if (auto factory = TensorrtProviderFactoryCreator::Create(params))
     return factory->CreateProvider();
 #else

onnxruntime/test/util/include/default_providers.h

@@ -46,7 +46,7 @@ std::unique_ptr<IExecutionProvider> DnnlExecutionProviderWithOptions(const OrtDn
 std::unique_ptr<IExecutionProvider> DefaultTensorrtExecutionProvider();
 std::unique_ptr<IExecutionProvider> DefaultNvTensorRTRTXExecutionProvider();
 std::unique_ptr<IExecutionProvider> TensorrtExecutionProviderWithOptions(const OrtTensorRTProviderOptions* params);
-std::unique_ptr<IExecutionProvider> TensorrtExecutionProviderWithOptions(const OrtTensorRTProviderOptionsV2* params);
+std::unique_ptr<IExecutionProvider> TensorrtExecutionProviderWithOptions(const OrtTensorRTProviderOptionsV2* params, bool force_reload_library = false);
 std::unique_ptr<IExecutionProvider> DefaultMIGraphXExecutionProvider();
 std::unique_ptr<IExecutionProvider> MIGraphXExecutionProviderWithOptions(const OrtMIGraphXProviderOptions* params);
 std::unique_ptr<IExecutionProvider> OpenVINOExecutionProviderWithOptions(const ProviderOptions* params, const SessionOptions* session_options = nullptr);