Merge pull request #6 from FrancescoOlivaTC/develop

Refactor gRPC client methods to use  Nvidia proto requests

Author: StefanoLusardiTC

inference_client/examples/client_example_model_info.cpp

@@ -25,7 +25,7 @@ int main(int, char**)
     auto client = tc::infer::create_client("localhost:8001");
 
     auto model_name = "simple_int32";
-    auto model_version = "1";
+    auto model_version = "";
 
     try
     {
@@ -34,6 +34,24 @@ int main(int, char**)
         spdlog::debug("\n");
         spdlog::debug("is_model_ready: {}", is_model_ready);
 
+        // Model List
+        auto model_list = client->model_list();
+        spdlog::debug("\n");
+        spdlog::debug("model_list:");
+        for (auto&& model : model_list)
+            spdlog::debug(model);
+
+        // Model Load
+        auto is_model_loaded = client->model_load(model_name, model_version);
+        spdlog::debug("\n");
+        spdlog::debug("model_load: {}", is_model_loaded);
+
+        if (!is_model_loaded)
+        {
+            spdlog::error("Model {} version {} not loaded", model_name, model_version);
+            return EXIT_FAILURE;
+        }
+
         // Model Metadata
         auto model_metadata = client->model_metadata(model_name, model_version);
         spdlog::debug("\n");
@@ -59,25 +77,10 @@ int main(int, char**)
             spdlog::debug("    shape: [{}]", fmt::join(output.shape, ","));
         }
 
-        // Model List
-        /* CURRENTLY NOT IMPLEMENTED IN TRITON INFERENCE SERVER (AMD SERVER ONLY) */
-        // auto model_list = client->model_list();
-        // spdlog::debug("\n");
-        // spdlog::debug("model_list:");
-        // for (auto&& model : model_list)
-        //     spdlog::debug(model);
-
-        // Model Load
-        /* CURRENTLY NOT IMPLEMENTED IN TRITON INFERENCE SERVER (AMD SERVER ONLY) */
-        // auto is_model_loaded = client->model_load(model_name, model_version);
-        // spdlog::debug("\n");
-        // spdlog::debug("model_load: {}", is_model_loaded);
-
         // Model Unload
-        /* CURRENTLY NOT IMPLEMENTED IN TRITON INFERENCE SERVER (AMD SERVER ONLY) */
-        // auto is_model_unloaded = client->model_unload(model_name, model_version);
-        // spdlog::debug("\n");
-        // spdlog::debug("model_unload: {}", is_model_unloaded);
+        auto is_model_unloaded = client->model_unload(model_name, model_version);
+        spdlog::debug("\n");
+        spdlog::debug("model_unload: {}", is_model_unloaded);
     }
     catch (const std::runtime_error& ex)
     {

inference_client/src/grpc_client.cpp

@@ -107,45 +107,56 @@ bool grpc_client::is_model_ready(const std::string& model_name, const std::strin
 
 std::vector<std::string> grpc_client::model_list()
 {
-    inference::ModelListRequest request;
-    inference::ModelListResponse response;
+    inference::RepositoryIndexRequest request;
+    inference::RepositoryIndexResponse response;
     grpc::ClientContext context;
 
+    // Set repository_name to empty string for default repository
+    request.set_repository_name("");
+    request.set_ready(false); // Get all models, not just ready ones
+
     context.set_deadline(std::chrono::system_clock::now() + _rpc_timeout);
-    grpc::Status rpc_status = _stub->ModelList(&context, request, &response);
+    grpc::Status rpc_status = _stub->RepositoryIndex(&context, request, &response);
     check_status(rpc_status);
 
-    const auto models = response.models();
-    const std::vector<std::string> model_list(models.begin(), models.end());
+    std::vector<std::string> model_list;
+    for (const auto& model : response.models())
+    {
+        model_list.push_back(model.name());
+    }
 
     return model_list;
 }
 
 bool grpc_client::model_load(const std::string& model_name, const std::string&)
 {
-    inference::ModelLoadRequest request;
-    inference::ModelLoadResponse response;
+    inference::RepositoryModelLoadRequest request;
+    inference::RepositoryModelLoadResponse response;
     grpc::ClientContext context;
 
-    request.set_name(model_name);
+    // Set repository_name to empty string for default repository
+    request.set_repository_name("");
+    request.set_model_name(model_name);
 
     context.set_deadline(std::chrono::system_clock::now() + _rpc_timeout);
-    grpc::Status rpc_status = _stub->ModelLoad(&context, request, &response);
+    grpc::Status rpc_status = _stub->RepositoryModelLoad(&context, request, &response);
     check_status(rpc_status);
 
     return true;
 }
 
 bool grpc_client::model_unload(const std::string& model_name, const std::string&)
 {
-    inference::ModelUnloadRequest request;
-    inference::ModelUnloadResponse response;
+    inference::RepositoryModelUnloadRequest request;
+    inference::RepositoryModelUnloadResponse response;
     grpc::ClientContext context;
 
-    request.set_name(model_name);
+    // Set repository_name to empty string for default repository
+    request.set_repository_name("");
+    request.set_model_name(model_name);
 
     context.set_deadline(std::chrono::system_clock::now() + _rpc_timeout);
-    grpc::Status rpc_status = _stub->ModelUnload(&context, request, &response);
+    grpc::Status rpc_status = _stub->RepositoryModelUnload(&context, request, &response);
     check_status(rpc_status);
 
     return true;

proto/services.proto

@@ -6,12 +6,11 @@ service GRPCInferenceService
     rpc ServerLive(ServerLiveRequest) returns (ServerLiveResponse) {}
     rpc ServerReady(ServerReadyRequest) returns (ServerReadyResponse) {}
     rpc ServerMetadata(ServerMetadataRequest) returns (ServerMetadataResponse) {}
-    
     rpc ModelReady(ModelReadyRequest) returns (ModelReadyResponse) {}
-    rpc ModelList(ModelListRequest) returns (ModelListResponse) {}
+    rpc RepositoryIndex(RepositoryIndexRequest) returns (RepositoryIndexResponse) {}
     rpc ModelMetadata(ModelMetadataRequest) returns (ModelMetadataResponse) {}
-    rpc ModelLoad(ModelLoadRequest) returns (ModelLoadResponse) {}
-    rpc ModelUnload(ModelUnloadRequest) returns (ModelUnloadResponse) {}
+    rpc RepositoryModelLoad(RepositoryModelLoadRequest) returns (RepositoryModelLoadResponse) {}
+    rpc RepositoryModelUnload(RepositoryModelUnloadRequest) returns (RepositoryModelUnloadResponse) {}
     rpc ModelInfer(ModelInferRequest) returns (ModelInferResponse) {}
 }
 
@@ -37,10 +36,21 @@ message ModelReadyResponse
     bool ready = 1;
 }
 
-message ModelListRequest {}
-message ModelListResponse
+message RepositoryIndexRequest
 {
-    repeated string models = 1;
+    string repository_name = 1;
+    bool ready = 2;
+}
+message RepositoryIndexResponse
+{
+    message ModelIndex
+    {
+        string name = 1;
+        string version = 2;
+        string state = 3;
+        string reason = 4;
+    }
+    repeated ModelIndex models = 1;
 }
 
 message ServerMetadataRequest {}
@@ -71,18 +81,32 @@ message ModelMetadataResponse
     repeated TensorMetadata outputs = 5;
 }
 
-message ModelLoadRequest
+message ModelRepositoryParameter
 {
-    string name = 1;
-    map<string, InferParameter> parameters = 2;
+    oneof parameter_choice
+    {
+        bool bool_param = 1;
+        int64 int64_param = 2;
+        string string_param = 3;
+        bytes bytes_param = 4;
+    }
 }
-message ModelLoadResponse {}
 
-message ModelUnloadRequest
+message RepositoryModelLoadRequest
 {
-    string name = 1;
+    string repository_name = 1;
+    string model_name = 2;
+    map<string, ModelRepositoryParameter> parameters = 3;
+}
+message RepositoryModelLoadResponse {}
+
+message RepositoryModelUnloadRequest
+{
+    string repository_name = 1;
+    string model_name = 2;
+    map<string, ModelRepositoryParameter> parameters = 3;
 }
-message ModelUnloadResponse {}
+message RepositoryModelUnloadResponse {}
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////