From b77f2193d53fed67dbdf1df7b45d6fb0ef9666ad Mon Sep 17 00:00:00 2001 From: Damian Kalinowski Date: Thu, 6 Mar 2025 11:01:14 +0100 Subject: [PATCH 01/24] save --- src/BUILD | 37 ++++++++++++++++++- .../multi_part_parser_drogon_impl.hpp | 36 ++++++++++++++++++ src/http_payload.hpp | 2 + src/multi_part_parser.hpp | 27 ++++++++++++++ 4 files changed, 100 insertions(+), 2 deletions(-) create mode 100644 src/http_frontend/multi_part_parser_drogon_impl.hpp create mode 100644 src/multi_part_parser.hpp diff --git a/src/BUILD b/src/BUILD index 030a93069c..31b93f66c2 100644 --- a/src/BUILD +++ b/src/BUILD @@ -462,8 +462,14 @@ cc_library( "//src/embeddings:embeddingscalculator", "//src/rerank:rerankcalculator",], }) + select({ - "//:enable_drogon": ["libdrogon_http_server"], - "//conditions:default" : ["libnet_http_server"], + "//:enable_drogon": [ + "libdrogon_http_server", + "libovmsmulti_part_parser_drogon_impl", + ], + "//conditions:default" : [ + "libnet_http_server", + # TODO + ], }) + [ "cpp_headers", "libovms_capi_servable_metadata", @@ -615,6 +621,7 @@ cc_library( deps = [ "@com_github_tencent_rapidjson//:rapidjson", "libovmsclient_connection", + "libovmsmulti_part_parser", ], visibility = ["//visibility:public"], copts = COPTS_ADJUSTED, @@ -633,6 +640,7 @@ cc_library( linkopts = LINKOPTS_ADJUSTED, alwayslink = 1, ) + cc_library( name = "libhttpclientconnection", hdrs = ["http_frontend/http_client_connection.hpp"], @@ -647,6 +655,31 @@ cc_library( alwayslink = 1, ) +cc_library( + name = "libovmsmulti_part_parser", + hdrs = ["multi_part_parser.hpp"], + deps = [ + ], + visibility = ["//visibility:public",], + local_defines = COMMON_LOCAL_DEFINES, + copts = COPTS_ADJUSTED, + linkopts = LINKOPTS_ADJUSTED, + alwayslink = 1, +) + +cc_library( + name = "libovmsmulti_part_parser_drogon_impl", + hdrs = ["http_frontend/multi_part_parser_drogon_impl.hpp"], + deps = [ + "libovmsmulti_part_parser", + ], + visibility = ["//visibility:public",], + local_defines = COMMON_LOCAL_DEFINES, + copts = COPTS_ADJUSTED, + linkopts = LINKOPTS_ADJUSTED, + alwayslink = 1, +) + cc_library( name = "libovms_module", hdrs = ["module.hpp"], diff --git a/src/http_frontend/multi_part_parser_drogon_impl.hpp b/src/http_frontend/multi_part_parser_drogon_impl.hpp new file mode 100644 index 0000000000..9491c7eb82 --- /dev/null +++ b/src/http_frontend/multi_part_parser_drogon_impl.hpp @@ -0,0 +1,36 @@ +//***************************************************************************** +// Copyright 2024 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include "../multi_part_parser.hpp" + +namespace ovms { + +class DrogonMultiPartParser : public MultiPartParser { + //std::shared_ptr serverReaderWriter; + +public: + //DrogonMultiPartParser(std::shared_ptr serverReaderWriter) : + DrogonMultiPartParser() {} + //serverReaderWriter(std::move(serverReaderWriter)) {} + + bool parse() override {} + std::string getFieldByName(const std::string& name) { + return ""; + }; +}; + +} // namespace ovms diff --git a/src/http_payload.hpp b/src/http_payload.hpp index 8a6b69426f..4af19ec3ad 100644 --- a/src/http_payload.hpp +++ b/src/http_payload.hpp @@ -25,6 +25,7 @@ #pragma warning(pop) #include "client_connection.hpp" +#include "multi_part_parser.hpp" namespace ovms { @@ -34,6 +35,7 @@ struct HttpPayload { std::string body; // always std::shared_ptr parsedJson; // pre-parsed body = null std::shared_ptr client; + std::shared_ptr multipartParser; }; } // namespace ovms diff --git a/src/multi_part_parser.hpp b/src/multi_part_parser.hpp new file mode 100644 index 0000000000..22988993ee --- /dev/null +++ b/src/multi_part_parser.hpp @@ -0,0 +1,27 @@ +//***************************************************************************** +// Copyright 2025 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#pragma once + +#include + +namespace ovms { + +class MultiPartParser { +public: + virtual bool parse() = 0; + virtual std::string getFieldByName(const std::string& name) = 0; +}; + +} // namespace ovms From 78a0e1e7b0d7d6cf8a2815a56ecb4bb5f30e7360 Mon Sep 17 00:00:00 2001 From: Damian Kalinowski Date: Thu, 6 Mar 2025 11:16:49 +0100 Subject: [PATCH 02/24] save --- src/BUILD | 2 ++ .../multi_part_parser_drogon_impl.cpp | 28 +++++++++++++++++++ .../multi_part_parser_drogon_impl.hpp | 24 ++++++++++------ 3 files changed, 45 insertions(+), 9 deletions(-) create mode 100644 src/http_frontend/multi_part_parser_drogon_impl.cpp diff --git a/src/BUILD b/src/BUILD index 31b93f66c2..0ae09d091f 100644 --- a/src/BUILD +++ b/src/BUILD @@ -670,8 +670,10 @@ cc_library( cc_library( name = "libovmsmulti_part_parser_drogon_impl", hdrs = ["http_frontend/multi_part_parser_drogon_impl.hpp"], + srcs = ["http_frontend/multi_part_parser_drogon_impl.cpp"], deps = [ "libovmsmulti_part_parser", + "@drogon//:drogon_cmake", ], visibility = ["//visibility:public",], local_defines = COMMON_LOCAL_DEFINES, diff --git a/src/http_frontend/multi_part_parser_drogon_impl.cpp b/src/http_frontend/multi_part_parser_drogon_impl.cpp new file mode 100644 index 0000000000..4da050d73b --- /dev/null +++ b/src/http_frontend/multi_part_parser_drogon_impl.cpp @@ -0,0 +1,28 @@ +//***************************************************************************** +// Copyright 2025 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "multi_part_parser_drogon_impl.hpp" + +namespace ovms { + +bool DrogonMultiPartParser::parse() { + return this->parser->parse(request) == 0; +} + +std::string DrogonMultiPartParser::getFieldByName(const std::string& name) { + return this->parser->getParameter(name); +} + +} // namespace ovms diff --git a/src/http_frontend/multi_part_parser_drogon_impl.hpp b/src/http_frontend/multi_part_parser_drogon_impl.hpp index 9491c7eb82..771999eef6 100644 --- a/src/http_frontend/multi_part_parser_drogon_impl.hpp +++ b/src/http_frontend/multi_part_parser_drogon_impl.hpp @@ -17,20 +17,26 @@ #include "../multi_part_parser.hpp" +#pragma warning(push) +#pragma warning(disable : 6326) +#include +#pragma warning(pop) + +#include +#include + namespace ovms { class DrogonMultiPartParser : public MultiPartParser { - //std::shared_ptr serverReaderWriter; + const drogon::HttpRequestPtr request{nullptr}; + const std::shared_ptr parser{nullptr}; public: - //DrogonMultiPartParser(std::shared_ptr serverReaderWriter) : - DrogonMultiPartParser() {} - //serverReaderWriter(std::move(serverReaderWriter)) {} - - bool parse() override {} - std::string getFieldByName(const std::string& name) { - return ""; - }; + DrogonMultiPartParser(const drogon::HttpRequestPtr& request) + : request(request), parser(std::make_shared()) {} + + bool parse() override; + std::string getFieldByName(const std::string& name); }; } // namespace ovms From 9c070cb15d90d67e3f3b0cc268068bb35af6eb92 Mon Sep 17 00:00:00 2001 From: Damian Kalinowski Date: Thu, 6 Mar 2025 11:44:40 +0100 Subject: [PATCH 03/24] save --- src/http_rest_api_handler.cpp | 40 ++++++++++++++++++----------------- src/http_rest_api_handler.hpp | 18 ++++++++++++---- src/http_server.cpp | 5 ++++- 3 files changed, 39 insertions(+), 24 deletions(-) diff --git a/src/http_rest_api_handler.cpp b/src/http_rest_api_handler.cpp index 099df1f4ad..37f25c06ec 100644 --- a/src/http_rest_api_handler.cpp +++ b/src/http_rest_api_handler.cpp @@ -159,7 +159,7 @@ void HttpRestApiHandler::registerHandler(RequestType type, HandlerCallbackFn f) } void HttpRestApiHandler::registerAll() { - registerHandler(Predict, [this](const std::string_view uri, const HttpRequestComponents& request_components, std::string& response, const std::string& request_body, HttpResponseComponents& response_components, std::shared_ptr serverReaderWriter) -> Status { + registerHandler(Predict, [this](const std::string_view uri, const HttpRequestComponents& request_components, std::string& response, const std::string& request_body, HttpResponseComponents& response_components, std::shared_ptr serverReaderWriter, std::shared_ptr multiPartParser) -> Status { if (request_components.processing_method == "predict") { return processPredictRequest(request_components.model_name, request_components.model_version, request_components.model_version_label, request_body, &response); @@ -169,44 +169,44 @@ void HttpRestApiHandler::registerAll() { } }); - registerHandler(GetModelMetadata, [this](const std::string_view uri, const HttpRequestComponents& request_components, std::string& response, const std::string& request_body, HttpResponseComponents& response_components, std::shared_ptr serverReaderWriter) { + registerHandler(GetModelMetadata, [this](const std::string_view uri, const HttpRequestComponents& request_components, std::string& response, const std::string& request_body, HttpResponseComponents& response_components, std::shared_ptr serverReaderWriter, std::shared_ptr multiPartParser) { return processModelMetadataRequest(request_components.model_name, request_components.model_version, request_components.model_version_label, &response); }); - registerHandler(GetModelStatus, [this](const std::string_view uri, const HttpRequestComponents& request_components, std::string& response, const std::string& request_body, HttpResponseComponents& response_components, std::shared_ptr serverReaderWriter) { + registerHandler(GetModelStatus, [this](const std::string_view uri, const HttpRequestComponents& request_components, std::string& response, const std::string& request_body, HttpResponseComponents& response_components, std::shared_ptr serverReaderWriter, std::shared_ptr multiPartParser) { return processModelStatusRequest(request_components.model_name, request_components.model_version, request_components.model_version_label, &response); }); - registerHandler(ConfigReload, [this](const std::string_view uri, const HttpRequestComponents& request_components, std::string& response, const std::string& request_body, HttpResponseComponents& response_components, std::shared_ptr serverReaderWriter) -> Status { + registerHandler(ConfigReload, [this](const std::string_view uri, const HttpRequestComponents& request_components, std::string& response, const std::string& request_body, HttpResponseComponents& response_components, std::shared_ptr serverReaderWriter, std::shared_ptr multiPartParser) -> Status { return processConfigReloadRequest(response, this->modelManager); }); - registerHandler(ConfigStatus, [this](const std::string_view uri, const HttpRequestComponents& request_components, std::string& response, const std::string& request_body, HttpResponseComponents& response_components, std::shared_ptr serverReaderWriter) -> Status { + registerHandler(ConfigStatus, [this](const std::string_view uri, const HttpRequestComponents& request_components, std::string& response, const std::string& request_body, HttpResponseComponents& response_components, std::shared_ptr serverReaderWriter, std::shared_ptr multiPartParser) -> Status { return processConfigStatusRequest(response, this->modelManager); }); - registerHandler(KFS_GetModelReady, [this](const std::string_view uri, const HttpRequestComponents& request_components, std::string& response, const std::string& request_body, HttpResponseComponents& response_components, std::shared_ptr serverReaderWriter) -> Status { + registerHandler(KFS_GetModelReady, [this](const std::string_view uri, const HttpRequestComponents& request_components, std::string& response, const std::string& request_body, HttpResponseComponents& response_components, std::shared_ptr serverReaderWriter, std::shared_ptr multiPartParser) -> Status { return processModelReadyKFSRequest(request_components, response, request_body); }); - registerHandler(KFS_GetModelMetadata, [this](const std::string_view uri, const HttpRequestComponents& request_components, std::string& response, const std::string& request_body, HttpResponseComponents& response_components, std::shared_ptr serverReaderWriter) -> Status { + registerHandler(KFS_GetModelMetadata, [this](const std::string_view uri, const HttpRequestComponents& request_components, std::string& response, const std::string& request_body, HttpResponseComponents& response_components, std::shared_ptr serverReaderWriter, std::shared_ptr multiPartParser) -> Status { return processModelMetadataKFSRequest(request_components, response, request_body); }); - registerHandler(KFS_Infer, [this](const std::string_view uri, const HttpRequestComponents& request_components, std::string& response, const std::string& request_body, HttpResponseComponents& response_components, std::shared_ptr serverReaderWriter) -> Status { + registerHandler(KFS_Infer, [this](const std::string_view uri, const HttpRequestComponents& request_components, std::string& response, const std::string& request_body, HttpResponseComponents& response_components, std::shared_ptr serverReaderWriter, std::shared_ptr multiPartParser) -> Status { return processInferKFSRequest(request_components, response, request_body, response_components.inferenceHeaderContentLength); }); - registerHandler(KFS_GetServerReady, [this](const std::string_view uri, const HttpRequestComponents& request_components, std::string& response, const std::string& request_body, HttpResponseComponents& response_components, std::shared_ptr serverReaderWriter) -> Status { + registerHandler(KFS_GetServerReady, [this](const std::string_view uri, const HttpRequestComponents& request_components, std::string& response, const std::string& request_body, HttpResponseComponents& response_components, std::shared_ptr serverReaderWriter, std::shared_ptr multiPartParser) -> Status { return processServerReadyKFSRequest(request_components, response, request_body); }); - registerHandler(KFS_GetServerLive, [this](const std::string_view uri, const HttpRequestComponents& request_components, std::string& response, const std::string& request_body, HttpResponseComponents& response_components, std::shared_ptr serverReaderWriter) -> Status { + registerHandler(KFS_GetServerLive, [this](const std::string_view uri, const HttpRequestComponents& request_components, std::string& response, const std::string& request_body, HttpResponseComponents& response_components, std::shared_ptr serverReaderWriter, std::shared_ptr multiPartParser) -> Status { return processServerLiveKFSRequest(request_components, response, request_body); }); - registerHandler(KFS_GetServerMetadata, [this](const std::string_view uri, const HttpRequestComponents& request_components, std::string& response, const std::string& request_body, HttpResponseComponents& response_components, std::shared_ptr serverReaderWriter) -> Status { + registerHandler(KFS_GetServerMetadata, [this](const std::string_view uri, const HttpRequestComponents& request_components, std::string& response, const std::string& request_body, HttpResponseComponents& response_components, std::shared_ptr serverReaderWriter, std::shared_ptr multiPartParser) -> Status { return processServerMetadataKFSRequest(request_components, response, request_body); }); - registerHandler(V3, [this](const std::string_view uri, const HttpRequestComponents& request_components, std::string& response, const std::string& request_body, HttpResponseComponents& response_components, std::shared_ptr serverReaderWriter) -> Status { + registerHandler(V3, [this](const std::string_view uri, const HttpRequestComponents& request_components, std::string& response, const std::string& request_body, HttpResponseComponents& response_components, std::shared_ptr serverReaderWriter, std::shared_ptr multiPartParser) -> Status { OVMS_PROFILE_FUNCTION(); - return processV3(uri, request_components, response, request_body, std::move(serverReaderWriter)); + return processV3(uri, request_components, response, request_body, std::move(serverReaderWriter), std::move(multiPartParser)); }); - registerHandler(Metrics, [this](const std::string_view uri, const HttpRequestComponents& request_components, std::string& response, const std::string& request_body, HttpResponseComponents& response_components, std::shared_ptr serverReaderWriter) -> Status { + registerHandler(Metrics, [this](const std::string_view uri, const HttpRequestComponents& request_components, std::string& response, const std::string& request_body, HttpResponseComponents& response_components, std::shared_ptr serverReaderWriter, std::shared_ptr multiPartParser) -> Status { return processMetrics(request_components, response, request_body); }); } @@ -437,18 +437,19 @@ Status HttpRestApiHandler::dispatchToProcessor( std::string* response, const HttpRequestComponents& request_components, HttpResponseComponents& response_components, - std::shared_ptr serverReaderWriter) { + std::shared_ptr serverReaderWriter, + std::shared_ptr multiPartParser) { auto handler = handlers.find(request_components.type); if (handler != handlers.end()) { - return handler->second(uri, request_components, *response, request_body, response_components, std::move(serverReaderWriter)); + return handler->second(uri, request_components, *response, request_body, response_components, std::move(serverReaderWriter), std::move(multiPartParser)); } else { return StatusCode::UNKNOWN_REQUEST_COMPONENTS_TYPE; } return StatusCode::UNKNOWN_REQUEST_COMPONENTS_TYPE; } -Status HttpRestApiHandler::processV3(const std::string_view uri, const HttpRequestComponents& request_components, std::string& response, const std::string& request_body, std::shared_ptr serverReaderWriter) { +Status HttpRestApiHandler::processV3(const std::string_view uri, const HttpRequestComponents& request_components, std::string& response, const std::string& request_body, std::shared_ptr serverReaderWriter, std::shared_ptr multiPartParser) { #if (MEDIAPIPE_DISABLE == 0) OVMS_PROFILE_FUNCTION(); HttpPayload request; @@ -820,7 +821,8 @@ Status HttpRestApiHandler::processRequest( std::vector>* headers, std::string* response, HttpResponseComponents& responseComponents, - std::shared_ptr serverReaderWriter) { + std::shared_ptr serverReaderWriter, + std::shared_ptr multiPartParser) { std::smatch sm; std::string request_path_str(request_path); @@ -838,7 +840,7 @@ Status HttpRestApiHandler::processRequest( if (!status.ok()) return status; - return dispatchToProcessor(request_path, request_body, response, requestComponents, responseComponents, std::move(serverReaderWriter)); + return dispatchToProcessor(request_path, request_body, response, requestComponents, responseComponents, std::move(serverReaderWriter), std::move(multiPartParser)); } Status HttpRestApiHandler::processPredictRequest( diff --git a/src/http_rest_api_handler.hpp b/src/http_rest_api_handler.hpp index c235e6a1f4..2058cb95cf 100644 --- a/src/http_rest_api_handler.hpp +++ b/src/http_rest_api_handler.hpp @@ -33,6 +33,7 @@ #pragma warning(pop) #include "http_async_writer_interface.hpp" +#include "multi_part_parser.hpp" #include "rest_parser.hpp" #include "status.hpp" @@ -73,7 +74,14 @@ struct HttpResponseComponents { std::optional inferenceHeaderContentLength; }; -using HandlerCallbackFn = std::function)>; +using HandlerCallbackFn = std::function, + std::shared_ptr)>; std::string urlDecode(const std::string& encoded); @@ -119,7 +127,8 @@ class HttpRestApiHandler { std::string* response, const HttpRequestComponents& request_components, HttpResponseComponents& response_components, - std::shared_ptr writer); + std::shared_ptr writer, + std::shared_ptr multiPartParser); /** * @brief Process Request @@ -139,7 +148,8 @@ class HttpRestApiHandler { std::vector>* headers, std::string* response, HttpResponseComponents& responseComponents, - std::shared_ptr writer); + std::shared_ptr writer, + std::shared_ptr multiPartParser); /** * @brief Process predict request @@ -220,7 +230,7 @@ class HttpRestApiHandler { Status processServerLiveKFSRequest(const HttpRequestComponents& request_components, std::string& response, const std::string& request_body); Status processServerMetadataKFSRequest(const HttpRequestComponents& request_components, std::string& response, const std::string& request_body); - Status processV3(const std::string_view uri, const HttpRequestComponents& request_components, std::string& response, const std::string& request_body, std::shared_ptr serverReaderWriter); + Status processV3(const std::string_view uri, const HttpRequestComponents& request_components, std::string& response, const std::string& request_body, std::shared_ptr serverReaderWriter, std::shared_ptr multiPartParser); private: const std::regex predictionRegex; diff --git a/src/http_server.cpp b/src/http_server.cpp index 1cc3542344..2f23936bc6 100644 --- a/src/http_server.cpp +++ b/src/http_server.cpp @@ -55,6 +55,7 @@ #include #include "drogon_http_async_writer_impl.hpp" +#include "http_frontend/multi_part_parser_drogon_impl.hpp" // TODO: net_http #endif namespace ovms { @@ -206,6 +207,7 @@ std::unique_ptr createAndStartDrogonHttpServer(const std::stri std::string output; HttpResponseComponents responseComponents; std::shared_ptr writer = std::make_shared(callback, pool, req); + std::shared_ptr multiPartParser = std::make_shared(req); const auto status = handler->processRequest( drogon::to_string_view(req->getMethod()), @@ -214,7 +216,8 @@ std::unique_ptr createAndStartDrogonHttpServer(const std::stri &headers, &output, responseComponents, - writer); + writer, + multiPartParser); if (status == StatusCode::PARTIAL_END) { // No further messaging is required. // Partial responses were delivered via "req" object. From eddcb01018fb3367ad5d1e1d74e03055cfebc0ae Mon Sep 17 00:00:00 2001 From: Damian Kalinowski Date: Thu, 6 Mar 2025 12:21:26 +0100 Subject: [PATCH 04/24] save --- src/http_payload.hpp | 4 +++- src/http_rest_api_handler.cpp | 17 ++++++++++++----- src/http_rest_api_handler.hpp | 9 ++++++--- src/http_server.cpp | 12 +++++++----- 4 files changed, 28 insertions(+), 14 deletions(-) diff --git a/src/http_payload.hpp b/src/http_payload.hpp index 4af19ec3ad..84038e0ec3 100644 --- a/src/http_payload.hpp +++ b/src/http_payload.hpp @@ -17,6 +17,7 @@ #include #include +#include #include #include #pragma warning(push) @@ -31,7 +32,8 @@ namespace ovms { struct HttpPayload { std::string uri; - std::vector> headers; + //std::vector> headers; + std::unordered_map headers; std::string body; // always std::shared_ptr parsedJson; // pre-parsed body = null std::shared_ptr client; diff --git a/src/http_rest_api_handler.cpp b/src/http_rest_api_handler.cpp index 37f25c06ec..0955b35dc5 100644 --- a/src/http_rest_api_handler.cpp +++ b/src/http_rest_api_handler.cpp @@ -461,11 +461,14 @@ Status HttpRestApiHandler::processV3(const std::string_view uri, const HttpReque doc->Parse(request_body.c_str()); } { + SPDLOG_INFO("AAAAAAAAAAAAA [{}]", request_components.headers.at("content-type")); + OVMS_PROFILE_SCOPE("rapidjson validate"); if (doc->HasParseError()) { return Status(StatusCode::JSON_INVALID, "Cannot parse JSON body"); } + if (!doc->IsObject()) { return Status(StatusCode::JSON_INVALID, "JSON body must be an object"); } @@ -658,7 +661,8 @@ Status HttpRestApiHandler::processModelMetadataKFSRequest(const HttpRequestCompo } static Status parseInferenceHeaderContentLength(HttpRequestComponents& requestComponents, - const std::vector>& headers) { + //const std::vector>& headers) { + const std::unordered_map& headers) { for (auto& header : headers) { if (toLower(header.first) == "inference-header-content-length") { // drogon automatically converts all headers to lowercase, net_http does not requestComponents.inferenceHeaderContentLength = stoi32(header.second); @@ -673,8 +677,9 @@ static Status parseInferenceHeaderContentLength(HttpRequestComponents& requestCo Status HttpRestApiHandler::parseRequestComponents(HttpRequestComponents& requestComponents, const std::string_view http_method, const std::string& request_path, - const std::vector>& headers) { - std::smatch sm; + //const std::vector>& headers) { + const std::unordered_map& headers) { + std::smatch sm; requestComponents.http_method = http_method; if (http_method != "POST" && http_method != "GET") { return StatusCode::REST_UNSUPPORTED_METHOD; @@ -818,7 +823,8 @@ Status HttpRestApiHandler::processRequest( const std::string_view http_method, const std::string_view request_path, const std::string& request_body, - std::vector>* headers, + // std::vector>* headers, + std::unordered_map* headers, std::string* response, HttpResponseComponents& responseComponents, std::shared_ptr serverReaderWriter, @@ -836,7 +842,8 @@ Status HttpRestApiHandler::processRequest( headers->clear(); response->clear(); - headers->push_back({"Content-Type", "application/json"}); + //headers->push_back({"Content-Type", "application/json"}); + (*headers)["content-type"] = "application/json"; // ? if (!status.ok()) return status; diff --git a/src/http_rest_api_handler.hpp b/src/http_rest_api_handler.hpp index 2058cb95cf..4cd94693b1 100644 --- a/src/http_rest_api_handler.hpp +++ b/src/http_rest_api_handler.hpp @@ -67,7 +67,8 @@ struct HttpRequestComponents { std::string processing_method; std::string model_subresource; std::optional inferenceHeaderContentLength; - std::vector> headers; + //std::vector> headers; + std::unordered_map headers; }; struct HttpResponseComponents { @@ -113,7 +114,8 @@ class HttpRestApiHandler { Status parseRequestComponents(HttpRequestComponents& components, const std::string_view http_method, const std::string& request_path, - const std::vector>& headers = {}); + //const std::vector>& headers = {}); + const std::unordered_map& headers = {}); Status parseModelVersion(std::string& model_version_str, std::optional& model_version); static Status prepareGrpcRequest(const std::string modelName, const std::optional& modelVersion, const std::string& request_body, ::KFSRequest& grpc_request, const std::optional& inferenceHeaderContentLength = {}); @@ -145,7 +147,8 @@ class HttpRestApiHandler { const std::string_view http_method, const std::string_view request_path, const std::string& request_body, - std::vector>* headers, + //std::vector>* headers, + std::unordered_map* headers, std::string* response, HttpResponseComponents& responseComponents, std::shared_ptr writer, diff --git a/src/http_server.cpp b/src/http_server.cpp index 2f23936bc6..2da92cd6ef 100644 --- a/src/http_server.cpp +++ b/src/http_server.cpp @@ -192,10 +192,11 @@ std::unique_ptr createAndStartDrogonHttpServer(const std::stri server->registerRequestDispatcher([handler, &pool](const drogon::HttpRequestPtr& req, std::function&& callback) { SPDLOG_DEBUG("REST request {}", req->getOriginalPath()); - std::vector> headers; + //std::vector> headers; + std::unordered_map headers; for (const auto& header : req->headers()) { - headers.emplace_back(header.first, header.second); + headers[header.first] = header.second; } SPDLOG_DEBUG("Processing HTTP request: {} {} body: {} bytes", @@ -236,8 +237,9 @@ std::unique_ptr createAndStartDrogonHttpServer(const std::stri resp->setContentTypeCode(drogon::CT_APPLICATION_JSON); if (responseComponents.inferenceHeaderContentLength.has_value()) { - std::pair header{"Inference-Header-Content-Length", std::to_string(responseComponents.inferenceHeaderContentLength.value())}; - headers.emplace_back(header); + //std::pair header{"Inference-Header-Content-Length", std::to_string(responseComponents.inferenceHeaderContentLength.value())}; + //headers.emplace_back(header); + headers["inference-header-content-length"] = std::to_string(responseComponents.inferenceHeaderContentLength.value()); } for (const auto& [key, value] : headers) { resp->addHeader(key, value); @@ -321,7 +323,7 @@ class RestApiRequestDispatcher { body.size()); HttpResponseComponents responseComponents; std::shared_ptr writer = std::make_shared(req); - const auto status = handler_->processRequest(req->http_method(), req->uri_path(), body, &headers, &output, responseComponents, writer); + const auto status = handler_->processRequest(req->http_method(), req->uri_path(), body, &headers, &output, responseComponents, writer); // TODO: vector of headers is no longer a vector if (status == StatusCode::PARTIAL_END) { // No further messaging is required. // Partial responses were delivered via "req" object. From a17f3a118e09d5687c6b221f429912cb3f39dea7 Mon Sep 17 00:00:00 2001 From: Damian Kalinowski Date: Thu, 6 Mar 2025 13:18:32 +0100 Subject: [PATCH 05/24] save --- src/http_rest_api_handler.cpp | 6 +++++- src/http_server.cpp | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/http_rest_api_handler.cpp b/src/http_rest_api_handler.cpp index 0955b35dc5..07935e229d 100644 --- a/src/http_rest_api_handler.cpp +++ b/src/http_rest_api_handler.cpp @@ -461,7 +461,11 @@ Status HttpRestApiHandler::processV3(const std::string_view uri, const HttpReque doc->Parse(request_body.c_str()); } { - SPDLOG_INFO("AAAAAAAAAAAAA [{}]", request_components.headers.at("content-type")); + SPDLOG_INFO("Headers in processV3:"); + + for (const auto& [key, value] : request_components.headers) { + SPDLOG_INFO("\t[{}]->[{}]", key, value); + } OVMS_PROFILE_SCOPE("rapidjson validate"); if (doc->HasParseError()) { diff --git a/src/http_server.cpp b/src/http_server.cpp index 2da92cd6ef..9f2bcb793c 100644 --- a/src/http_server.cpp +++ b/src/http_server.cpp @@ -195,7 +195,9 @@ std::unique_ptr createAndStartDrogonHttpServer(const std::stri //std::vector> headers; std::unordered_map headers; + SPDLOG_ERROR("Drogon headers:"); for (const auto& header : req->headers()) { + SPDLOG_ERROR("\t[{}]->[{}]", header.first, header.second); headers[header.first] = header.second; } From 20f4818529ccaf5631fed455a3f2e5582deaa10e Mon Sep 17 00:00:00 2001 From: Damian Kalinowski Date: Thu, 6 Mar 2025 13:56:23 +0100 Subject: [PATCH 06/24] save --- src/http_rest_api_handler.cpp | 91 ++++++++++++++++++++++++----------- 1 file changed, 64 insertions(+), 27 deletions(-) diff --git a/src/http_rest_api_handler.cpp b/src/http_rest_api_handler.cpp index 07935e229d..5dd4102675 100644 --- a/src/http_rest_api_handler.cpp +++ b/src/http_rest_api_handler.cpp @@ -453,13 +453,13 @@ Status HttpRestApiHandler::processV3(const std::string_view uri, const HttpReque #if (MEDIAPIPE_DISABLE == 0) OVMS_PROFILE_FUNCTION(); HttpPayload request; - std::shared_ptr doc = std::make_shared(); + std::shared_ptr doc;// = std::make_shared(); std::shared_ptr executor; bool streamFieldVal = false; - { - OVMS_PROFILE_SCOPE("rapidjson parse body"); - doc->Parse(request_body.c_str()); - } + // { + // OVMS_PROFILE_SCOPE("rapidjson parse body"); + // doc->Parse(request_body.c_str()); + // } { SPDLOG_INFO("Headers in processV3:"); @@ -467,36 +467,72 @@ Status HttpRestApiHandler::processV3(const std::string_view uri, const HttpReque SPDLOG_INFO("\t[{}]->[{}]", key, value); } - OVMS_PROFILE_SCOPE("rapidjson validate"); - if (doc->HasParseError()) { - return Status(StatusCode::JSON_INVALID, "Cannot parse JSON body"); - } + auto it = request_components.headers.find("content-type"); + bool isApplicationJson = it != request_components.headers.end() && it->second.find("application/json") != std::string::npos; + bool isMultiPart = it != request_components.headers.end() && it->second.find("multipart/form-data") != std::string::npos; + bool isDefault = !isApplicationJson && !isMultiPart; + std::string model_name; - if (!doc->IsObject()) { - return Status(StatusCode::JSON_INVALID, "JSON body must be an object"); - } + if (isMultiPart) { + if (!multiPartParser->parse()) { + return StatusCode::REST_INVALID_URL; // TODO: Better errror? + } + model_name = multiPartParser->getFieldByName("model"); + if (model_name.empty()) { + isDefault = true; + } else { + SPDLOG_INFO("Model name from Multipart Field: {}", model_name); + } + } else if (isApplicationJson) { + doc = std::make_shared(); - auto modelNameIt = doc->FindMember("model"); - if (modelNameIt == doc->MemberEnd()) { - return Status(StatusCode::JSON_INVALID, "model field is missing in JSON body"); - } + doc->Parse(request_body.c_str()); - if (!modelNameIt->value.IsString()) { - return Status(StatusCode::JSON_INVALID, "model field is not a string"); - } + OVMS_PROFILE_SCOPE("rapidjson validate"); + if (doc->HasParseError()) { + return Status(StatusCode::JSON_INVALID, "Cannot parse JSON body"); + } - const std::string model_name = modelNameIt->value.GetString(); + if (!doc->IsObject()) { + return Status(StatusCode::JSON_INVALID, "JSON body must be an object"); + } - bool isTextGenerationEndpoint = uri.find("completions") != std::string_view::npos; - if (isTextGenerationEndpoint) { - auto streamIt = doc->FindMember("stream"); - if (streamIt != doc->MemberEnd()) { - if (!streamIt->value.IsBool()) { - return Status(StatusCode::JSON_INVALID, "stream field is not a boolean"); + auto modelNameIt = doc->FindMember("model"); + if (modelNameIt == doc->MemberEnd()) { + return Status(StatusCode::JSON_INVALID, "model field is missing in JSON body"); + } + + if (!modelNameIt->value.IsString()) { + return Status(StatusCode::JSON_INVALID, "model field is not a string"); + } + + bool isTextGenerationEndpoint = uri.find("completions") != std::string_view::npos; + if (isTextGenerationEndpoint) { + auto streamIt = doc->FindMember("stream"); + if (streamIt != doc->MemberEnd()) { + if (!streamIt->value.IsBool()) { + return Status(StatusCode::JSON_INVALID, "stream field is not a boolean"); + } + streamFieldVal = streamIt->value.GetBool(); } - streamFieldVal = streamIt->value.GetBool(); } + + model_name = modelNameIt->value.GetString(); + if (model_name.empty()) { + isDefault = true; + } else { + SPDLOG_INFO("Model name from Application Json: {}", model_name); + } + } + + // Deduce Graph Name from URI + if (isDefault) { + if (uri.size() <= 4) { // nothing after "/v3/..." + return StatusCode::REST_INVALID_URL; + } + model_name = std::string(uri.substr(4)); + SPDLOG_INFO("Model name from URI: {}", model_name); } auto status = this->modelManager.createPipeline(executor, model_name); @@ -509,6 +545,7 @@ Status HttpRestApiHandler::processV3(const std::string_view uri, const HttpReque request.parsedJson = std::move(doc); request.uri = std::string(uri); request.client = std::make_shared(serverReaderWriter); + request.multipartParser = std::move(multiPartParser); } if (streamFieldVal == false) { ExecutionContext executionContext{ExecutionContext::Interface::REST, ExecutionContext::Method::V3Unary}; From e2d712ab974e73e957f7911769910b1433a4f4da Mon Sep 17 00:00:00 2001 From: Damian Kalinowski Date: Thu, 6 Mar 2025 14:01:47 +0100 Subject: [PATCH 07/24] save --- src/llm/servable.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/llm/servable.cpp b/src/llm/servable.cpp index 12bfa3acf8..7949dbe33b 100644 --- a/src/llm/servable.cpp +++ b/src/llm/servable.cpp @@ -46,6 +46,9 @@ static std::string wrapTextInServerSideEventMessage(const std::string& text) { absl::Status GenAiServable::loadRequest(std::shared_ptr& executionContext, const ovms::HttpPayload& payload) { SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Request body: {}", payload.body); SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Request uri: {}", payload.uri); + if (!payload.parsedJson || payload.parsedJson->HasParseError()) { + return absl::InvalidArgumentError("Cannot parse JSON body"); + } if (payload.uri == "/v3/chat/completions" || payload.uri == "/v3/v1/chat/completions") { executionContext->endpoint = Endpoint::CHAT_COMPLETIONS; } else if (payload.uri == "/v3/completions" || payload.uri == "/v3/v1/completions") { From 5b40e69aa54fab15ce7f6db908b779a6981b6c63 Mon Sep 17 00:00:00 2001 From: Damian Kalinowski Date: Thu, 6 Mar 2025 15:08:41 +0100 Subject: [PATCH 08/24] save --- .../multi_part_parser_drogon_impl.cpp | 17 +++++++++++++++-- .../multi_part_parser_drogon_impl.hpp | 9 ++++++++- src/http_rest_api_handler.cpp | 4 +--- src/llm/servable.cpp | 2 +- src/multi_part_parser.hpp | 6 +++++- 5 files changed, 30 insertions(+), 8 deletions(-) diff --git a/src/http_frontend/multi_part_parser_drogon_impl.cpp b/src/http_frontend/multi_part_parser_drogon_impl.cpp index 4da050d73b..452106a989 100644 --- a/src/http_frontend/multi_part_parser_drogon_impl.cpp +++ b/src/http_frontend/multi_part_parser_drogon_impl.cpp @@ -18,11 +18,24 @@ namespace ovms { bool DrogonMultiPartParser::parse() { - return this->parser->parse(request) == 0; + this->hasParseError_ = this->parser->parse(request) != 0; + return !this->hasParseError_; } -std::string DrogonMultiPartParser::getFieldByName(const std::string& name) { +bool DrogonMultiPartParser::hasParseError() const { + return this->hasParseError_; +} + +std::string DrogonMultiPartParser::getFieldByName(const std::string& name) const { return this->parser->getParameter(name); } +std::string_view DrogonMultiPartParser::getFileContentByName(const std::string& name) const { + auto it = this->parser->getFilesMap().find(name); + if (it == this->parser->getFilesMap().end()) { + return ""; + } + return it->second.fileContent(); +} + } // namespace ovms diff --git a/src/http_frontend/multi_part_parser_drogon_impl.hpp b/src/http_frontend/multi_part_parser_drogon_impl.hpp index 771999eef6..d1f6b3e1f6 100644 --- a/src/http_frontend/multi_part_parser_drogon_impl.hpp +++ b/src/http_frontend/multi_part_parser_drogon_impl.hpp @@ -23,11 +23,13 @@ #pragma warning(pop) #include +#include #include namespace ovms { class DrogonMultiPartParser : public MultiPartParser { + bool hasParseError_{true}; const drogon::HttpRequestPtr request{nullptr}; const std::shared_ptr parser{nullptr}; @@ -36,7 +38,12 @@ class DrogonMultiPartParser : public MultiPartParser { : request(request), parser(std::make_shared()) {} bool parse() override; - std::string getFieldByName(const std::string& name); + + bool hasParseError() const override; + + std::string getFieldByName(const std::string& name) const override; + std::string_view getFileContentByName(const std::string& name) const override; + }; } // namespace ovms diff --git a/src/http_rest_api_handler.cpp b/src/http_rest_api_handler.cpp index 5dd4102675..81eec46aea 100644 --- a/src/http_rest_api_handler.cpp +++ b/src/http_rest_api_handler.cpp @@ -453,7 +453,7 @@ Status HttpRestApiHandler::processV3(const std::string_view uri, const HttpReque #if (MEDIAPIPE_DISABLE == 0) OVMS_PROFILE_FUNCTION(); HttpPayload request; - std::shared_ptr doc;// = std::make_shared(); + std::shared_ptr doc = std::make_shared(); std::shared_ptr executor; bool streamFieldVal = false; // { @@ -485,8 +485,6 @@ Status HttpRestApiHandler::processV3(const std::string_view uri, const HttpReque SPDLOG_INFO("Model name from Multipart Field: {}", model_name); } } else if (isApplicationJson) { - doc = std::make_shared(); - doc->Parse(request_body.c_str()); OVMS_PROFILE_SCOPE("rapidjson validate"); diff --git a/src/llm/servable.cpp b/src/llm/servable.cpp index 7949dbe33b..a58fdaa44b 100644 --- a/src/llm/servable.cpp +++ b/src/llm/servable.cpp @@ -46,7 +46,7 @@ static std::string wrapTextInServerSideEventMessage(const std::string& text) { absl::Status GenAiServable::loadRequest(std::shared_ptr& executionContext, const ovms::HttpPayload& payload) { SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Request body: {}", payload.body); SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Request uri: {}", payload.uri); - if (!payload.parsedJson || payload.parsedJson->HasParseError()) { + if (payload.parsedJson->HasParseError()) { return absl::InvalidArgumentError("Cannot parse JSON body"); } if (payload.uri == "/v3/chat/completions" || payload.uri == "/v3/v1/chat/completions") { diff --git a/src/multi_part_parser.hpp b/src/multi_part_parser.hpp index 22988993ee..6e65922893 100644 --- a/src/multi_part_parser.hpp +++ b/src/multi_part_parser.hpp @@ -21,7 +21,11 @@ namespace ovms { class MultiPartParser { public: virtual bool parse() = 0; - virtual std::string getFieldByName(const std::string& name) = 0; + + virtual bool hasParseError() const = 0; + + virtual std::string getFieldByName(const std::string& name) const = 0; + virtual std::string_view getFileContentByName(const std::string& name) const = 0; }; } // namespace ovms From dafa7d1aef95b8ad5580db0ee882086a1db00dee Mon Sep 17 00:00:00 2001 From: Damian Kalinowski Date: Thu, 6 Mar 2025 15:12:33 +0100 Subject: [PATCH 09/24] save --- src/test/mediapipe/calculators/BUILD | 1 + .../multipart_accepting_calculator.cpp | 59 +++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 src/test/mediapipe/calculators/multipart_accepting_calculator.cpp diff --git a/src/test/mediapipe/calculators/BUILD b/src/test/mediapipe/calculators/BUILD index 8ffc1952dc..e8ef37fca3 100644 --- a/src/test/mediapipe/calculators/BUILD +++ b/src/test/mediapipe/calculators/BUILD @@ -49,6 +49,7 @@ cc_library( "streaming_test_calculator.cpp", "two_input_calculator.cpp", "openai_chat_completions_mock_calculator.cpp", + "multipart_accepting_calculator.cpp", ], copts = select({ "//conditions:default": [ diff --git a/src/test/mediapipe/calculators/multipart_accepting_calculator.cpp b/src/test/mediapipe/calculators/multipart_accepting_calculator.cpp new file mode 100644 index 0000000000..516594f4f4 --- /dev/null +++ b/src/test/mediapipe/calculators/multipart_accepting_calculator.cpp @@ -0,0 +1,59 @@ +//***************************************************************************** +// Copyright 2025 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +#include "../../../http_payload.hpp" + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/port/canonical_errors.h" +#pragma GCC diagnostic pop + +namespace mediapipe { + +class MultipartAcceptingCalculator : public CalculatorBase { + static const std::string INPUT_TAG_NAME; + static const std::string OUTPUT_TAG_NAME; + +public: + static absl::Status GetContract(CalculatorContract* cc) { + RET_CHECK(!cc->Inputs().GetTags().empty()); + RET_CHECK(!cc->Outputs().GetTags().empty()); + cc->Inputs().Tag(INPUT_TAG_NAME).Set(); + cc->Outputs().Tag(OUTPUT_TAG_NAME).Set(); + return absl::OkStatus(); + } + + absl::Status Close(CalculatorContext* cc) final { + return absl::OkStatus(); + } + + absl::Status Open(CalculatorContext* cc) final { + return absl::OkStatus(); + } + + absl::Status Process(CalculatorContext* cc) final { + + return absl::OkStatus(); + } +}; +#pragma GCC diagnostic pop + +const std::string MultipartAcceptingCalculator::INPUT_TAG_NAME{"HTTP_REQUEST_PAYLOAD"}; +const std::string MultipartAcceptingCalculator::OUTPUT_TAG_NAME{"HTTP_RESPONSE_PAYLOAD"}; + +REGISTER_CALCULATOR(MultipartAcceptingCalculator); +} // namespace mediapipe From 4c93d9e9bc3bc293d2e9aaf3f59a62e38fffde0f Mon Sep 17 00:00:00 2001 From: Damian Kalinowski Date: Thu, 6 Mar 2025 15:20:17 +0100 Subject: [PATCH 10/24] adjust unit tests --- src/test/disabled_mediapipe_test.cpp | 2 +- src/test/embeddingsnode_test.cpp | 4 ++-- src/test/http_openai_handler_test.cpp | 6 +++--- src/test/kfs_rest_test.cpp | 27 +++++++++++---------------- src/test/llmnode_test.cpp | 4 ++-- src/test/llmtemplate_test.cpp | 2 +- src/test/reranknode_test.cpp | 2 +- 7 files changed, 21 insertions(+), 26 deletions(-) diff --git a/src/test/disabled_mediapipe_test.cpp b/src/test/disabled_mediapipe_test.cpp index 8a251bdbae..6b5378863f 100644 --- a/src/test/disabled_mediapipe_test.cpp +++ b/src/test/disabled_mediapipe_test.cpp @@ -33,7 +33,7 @@ class MediapipeDisabledTest : public ::testing::Test { public: std::unique_ptr handler; - std::vector> headers; + std::unordered_map headers; ovms::HttpRequestComponents comp; const std::string endpointChatCompletions = "/v3/chat/completions"; const std::string endpointCompletions = "/v3/completions"; diff --git a/src/test/embeddingsnode_test.cpp b/src/test/embeddingsnode_test.cpp index b2cce9bad1..1620e5fb05 100644 --- a/src/test/embeddingsnode_test.cpp +++ b/src/test/embeddingsnode_test.cpp @@ -30,7 +30,7 @@ class V3HttpTest : public ::testing::Test { public: std::unique_ptr handler; - std::vector> headers; + std::unordered_map headers; ovms::HttpRequestComponents comp; const std::string endpointEmbeddings = "/v3/embeddings"; const std::string endpointRerank = "/v3/rerank"; @@ -345,7 +345,7 @@ class EmbeddingsExtensionTest : public ::testing::Test { public: std::unique_ptr handler; - std::vector> headers; + std::unordered_map headers; ovms::HttpRequestComponents comp; const std::string endpointEmbeddings = "/v3/embeddings"; std::shared_ptr writer; diff --git a/src/test/http_openai_handler_test.cpp b/src/test/http_openai_handler_test.cpp index 32d4a1f09e..ab4e35fb33 100644 --- a/src/test/http_openai_handler_test.cpp +++ b/src/test/http_openai_handler_test.cpp @@ -37,7 +37,7 @@ class HttpOpenAIHandlerTest : public ::testing::Test { std::unique_ptr t; std::string port = "9173"; - std::vector> headers; + std::unordered_map headers; ovms::HttpRequestComponents comp; const std::string endpoint = "/v3/chat/completions"; std::shared_ptr writer; @@ -100,8 +100,8 @@ TEST_F(HttpOpenAIHandlerTest, UnaryWithHeaders) { "messages": [] } )"; - comp.headers.push_back(std::pair("test1", "header")); - comp.headers.push_back(std::pair("test2", "header")); + comp.headers["test1"] = "header"; + comp.headers["test2"] = "header"; ASSERT_EQ( handler->dispatchToProcessor("/v3/completions/", requestBody, &response, comp, responseComponents, writer), diff --git a/src/test/kfs_rest_test.cpp b/src/test/kfs_rest_test.cpp index 3c49a0d21d..12a634f219 100644 --- a/src/test/kfs_rest_test.cpp +++ b/src/test/kfs_rest_test.cpp @@ -190,9 +190,8 @@ std::unique_ptr HttpRestApiHandlerTest::thread = nullptr; #pragma GCC diagnostic ignored "-Wnarrowing" static void testInference(int headerLength, std::string& request_body, std::unique_ptr& handler, const std::string endpoint = "/v2/models/mediapipeAdd/versions/1/infer") { - std::vector> headers; - std::pair binaryInputsHeader{"inference-header-content-length", std::to_string(headerLength)}; - headers.emplace_back(binaryInputsHeader); + std::unordered_map headers; + headers["inference-header-content-length"] = std::to_string(headerLength); ovms::HttpRequestComponents comp; @@ -222,9 +221,8 @@ static void testInference(int headerLength, std::string& request_body, std::uniq static void testInferenceNegative(int headerLength, std::string& request_body, std::unique_ptr& handler, ovms::Status processorStatus) { std::string request = "/v2/models/mediapipeAdd/versions/1/infer"; - std::vector> headers; - std::pair binaryInputsHeader{"inference-header-content-length", std::to_string(headerLength)}; - headers.emplace_back(binaryInputsHeader); + std::unordered_map headers; + headers["inference-header-content-length"] = std::to_string(headerLength); ovms::HttpRequestComponents comp; @@ -535,27 +533,24 @@ TEST_F(HttpRestApiHandlerTest, RegexParseServerLive) { TEST_F(HttpRestApiHandlerTest, RegexParseInferWithBinaryInputs) { std::string request = "/v2/models/dummy/versions/1/infer"; ovms::HttpRequestComponents comp; - std::vector> headers; - std::pair binaryInputsHeader{"inference-header-content-length", "15"}; - headers.emplace_back(binaryInputsHeader); + std::unordered_map headers; + headers["inference-header-content-length"] = "15"; ASSERT_EQ(handler->parseRequestComponents(comp, "POST", request, headers), StatusCode::OK); } TEST_F(HttpRestApiHandlerTest, RegexParseInferWithBinaryInputsSizeNegative) { std::string request = "/v2/models/dummy/versions/1/infer"; ovms::HttpRequestComponents comp; - std::vector> headers; - std::pair binaryInputsHeader{"inference-header-content-length", "-15"}; - headers.emplace_back(binaryInputsHeader); + std::unordered_map headers; + headers["inference-header-content-length"] = "-15"; ASSERT_EQ(handler->parseRequestComponents(comp, "POST", request, headers), StatusCode::REST_INFERENCE_HEADER_CONTENT_LENGTH_INVALID); } TEST_F(HttpRestApiHandlerTest, RegexParseInferWithBinaryInputsSizeNotInt) { std::string request = "/v2/models/dummy/versions/1/infer"; ovms::HttpRequestComponents comp; - std::vector> headers; - std::pair binaryInputsHeader{"inference-header-content-length", "value"}; - headers.emplace_back(binaryInputsHeader); + std::unordered_map headers; + headers["inference-header-content-length"] = "value"; ASSERT_EQ(handler->parseRequestComponents(comp, "POST", request, headers), StatusCode::REST_INFERENCE_HEADER_CONTENT_LENGTH_INVALID); } @@ -1572,7 +1567,7 @@ TEST_F(HttpRestApiHandlerWithStringModelTest, positivePassthrough_binaryInput) { std::string binaryInputData{0x05, 0x00, 0x00, 0x00, 'H', 'e', 'l', 'l', 'o', 0x02, 0x00, 0x00, 0x00, '1', '2'}; request_body += binaryInputData; - std::vector> headers{ + std::unordered_map headers{ {"inference-header-content-length", std::to_string(jsonEnd)}, {"Content-Type", "application/json"}, }; diff --git a/src/test/llmnode_test.cpp b/src/test/llmnode_test.cpp index 9ef0723d8d..e0460c5eca 100644 --- a/src/test/llmnode_test.cpp +++ b/src/test/llmnode_test.cpp @@ -57,7 +57,7 @@ class LLMFlowHttpTest : public ::testing::Test { public: std::unique_ptr handler; - std::vector> headers; + std::unordered_map headers; ovms::HttpRequestComponents comp; const std::string endpointChatCompletions = "/v3/chat/completions"; const std::string endpointCompletions = "/v3/completions"; @@ -655,7 +655,7 @@ TEST_F(LLMFlowHttpTest, KFSApiRequestToChatCompletionsGraph) { } ] })"; - std::vector> headers; + std::unordered_map headers; ASSERT_EQ(handler->parseRequestComponents(comp, "POST", "/v2/models/llmDummyKFS/versions/1/infer", headers), ovms::StatusCode::OK); ASSERT_EQ( handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), diff --git a/src/test/llmtemplate_test.cpp b/src/test/llmtemplate_test.cpp index 66cd525e8d..18cd68856e 100644 --- a/src/test/llmtemplate_test.cpp +++ b/src/test/llmtemplate_test.cpp @@ -551,7 +551,7 @@ class LLMChatTemplateHttpTest : public TestWithTempDir { public: std::unique_ptr handler; - std::vector> headers; + std::unordered_map headers; ovms::HttpRequestComponents comp; const std::string endpointChatCompletions = "/v3/chat/completions"; const std::string endpointCompletions = "/v3/completions"; diff --git a/src/test/reranknode_test.cpp b/src/test/reranknode_test.cpp index c2b0ff8d3e..f70f42262e 100644 --- a/src/test/reranknode_test.cpp +++ b/src/test/reranknode_test.cpp @@ -33,7 +33,7 @@ class V3HttpTest : public ::testing::Test { public: std::unique_ptr handler; - std::vector> headers; + std::unordered_map headers; ovms::HttpRequestComponents comp; const std::string endpointEmbeddings = "/v3/embeddings"; const std::string endpointRerank = "/v3/rerank"; From 0f5fcd70fcedbef72f24c72943cc180bac65082b Mon Sep 17 00:00:00 2001 From: Damian Kalinowski Date: Thu, 6 Mar 2025 15:27:22 +0100 Subject: [PATCH 11/24] test --- src/test/embeddingsnode_test.cpp | 30 +++++++++++++++++------------- src/test/test_http_utils.hpp | 11 +++++++++++ 2 files changed, 28 insertions(+), 13 deletions(-) diff --git a/src/test/embeddingsnode_test.cpp b/src/test/embeddingsnode_test.cpp index 1620e5fb05..f1d650b63d 100644 --- a/src/test/embeddingsnode_test.cpp +++ b/src/test/embeddingsnode_test.cpp @@ -35,6 +35,7 @@ class V3HttpTest : public ::testing::Test { const std::string endpointEmbeddings = "/v3/embeddings"; const std::string endpointRerank = "/v3/rerank"; std::shared_ptr writer; + std::shared_ptr multiPartParser; std::string response; ovms::HttpResponseComponents responseComponents; @@ -52,6 +53,7 @@ class V3HttpTest : public ::testing::Test { void SetUp() { writer = std::make_shared(); + multiPartParser = std::make_shared(); ovms::Server& server = ovms::Server::instance(); handler = std::make_unique(server, 5); ASSERT_EQ(handler->parseRequestComponents(comp, "POST", endpointEmbeddings, headers), ovms::StatusCode::OK); @@ -96,7 +98,7 @@ TEST_F(EmbeddingsHttpTest, simplePositive) { } )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointEmbeddings, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointEmbeddings, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); rapidjson::Document d; rapidjson::ParseResult ok = d.Parse(response.c_str()); @@ -130,7 +132,7 @@ TEST_F(EmbeddingsHttpTest, simplePositiveNoNorm) { } )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointEmbeddings, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointEmbeddings, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); rapidjson::Document d; rapidjson::ParseResult ok = d.Parse(response.c_str()); @@ -165,7 +167,7 @@ TEST_F(EmbeddingsHttpTest, simplePositiveBase64) { } )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointEmbeddings, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointEmbeddings, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); rapidjson::Document d; rapidjson::ParseResult ok = d.Parse(response.c_str()); @@ -192,7 +194,7 @@ TEST_F(EmbeddingsHttpTest, simplePositiveInt) { "input": [111, 222, 121] } )"; - Status status = handler->dispatchToProcessor(endpointEmbeddings, requestBody, &response, comp, responseComponents, writer); + Status status = handler->dispatchToProcessor(endpointEmbeddings, requestBody, &response, comp, responseComponents, writer, multiPartParser); ASSERT_EQ(status, ovms::StatusCode::OK) << status.string(); @@ -214,7 +216,7 @@ TEST_F(EmbeddingsHttpTest, simplePositiveMultipleInts) { "input": [[111, 222, 121], [123, 221, 311]] } )"; - Status status = handler->dispatchToProcessor(endpointEmbeddings, requestBody, &response, comp, responseComponents, writer); + Status status = handler->dispatchToProcessor(endpointEmbeddings, requestBody, &response, comp, responseComponents, writer, multiPartParser); ASSERT_EQ(status, ovms::StatusCode::OK) << status.string(); @@ -239,7 +241,7 @@ TEST_F(EmbeddingsHttpTest, simplePositiveMultipleIntLengths) { "input": [[1, 2, 3, 4, 5, 6], [4, 5, 6, 7], [7, 8]] } )"; - Status status = handler->dispatchToProcessor(endpointEmbeddings, requestBody, &response, comp, responseComponents, writer); + Status status = handler->dispatchToProcessor(endpointEmbeddings, requestBody, &response, comp, responseComponents, writer, multiPartParser); ASSERT_EQ(status, ovms::StatusCode::OK) << status.string(); @@ -266,7 +268,7 @@ TEST_F(EmbeddingsHttpTest, simplePositiveMultipleStrings) { "input": ["one", "two"] } )"; - Status status = handler->dispatchToProcessor(endpointEmbeddings, requestBody, &response, comp, responseComponents, writer); + Status status = handler->dispatchToProcessor(endpointEmbeddings, requestBody, &response, comp, responseComponents, writer, multiPartParser); ASSERT_EQ(status, ovms::StatusCode::OK) << status.string(); @@ -291,7 +293,7 @@ TEST_F(EmbeddingsHttpTest, positiveLongInput) { } std::string requestBody = "{ \"model\": \"embeddings\", \"input\": \"" + words + " \"}"; - Status status = handler->dispatchToProcessor(endpointEmbeddings, requestBody, &response, comp, responseComponents, writer); + Status status = handler->dispatchToProcessor(endpointEmbeddings, requestBody, &response, comp, responseComponents, writer, multiPartParser); ASSERT_EQ(status, ovms::StatusCode::OK) << status.string(); @@ -310,7 +312,7 @@ TEST_F(EmbeddingsHttpTest, negativeTooLongInput) { } std::string requestBody = "{ \"model\": \"embeddings\", \"input\": \"" + words + " \"}"; - Status status = handler->dispatchToProcessor(endpointEmbeddings, requestBody, &response, comp, responseComponents, writer); + Status status = handler->dispatchToProcessor(endpointEmbeddings, requestBody, &response, comp, responseComponents, writer, multiPartParser); ASSERT_EQ(status, ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR) << status.string(); @@ -328,7 +330,7 @@ TEST_F(EmbeddingsHttpTest, negativeTooLongInputPair) { } std::string requestBody = "{ \"model\": \"embeddings\", \"input\": [\"" + words + " \", \"short prompt\"]}"; - Status status = handler->dispatchToProcessor(endpointEmbeddings, requestBody, &response, comp, responseComponents, writer); + Status status = handler->dispatchToProcessor(endpointEmbeddings, requestBody, &response, comp, responseComponents, writer, multiPartParser); ASSERT_EQ(status, ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR) << status.string(); @@ -349,6 +351,7 @@ class EmbeddingsExtensionTest : public ::testing::Test { ovms::HttpRequestComponents comp; const std::string endpointEmbeddings = "/v3/embeddings"; std::shared_ptr writer; + std::shared_ptr multiPartParser; std::string response; ovms::HttpResponseComponents responseComponents; @@ -385,6 +388,7 @@ class EmbeddingsExtensionTest : public ::testing::Test { GTEST_SKIP() << "Skipping test because we have no custom extension built for Windows"; #endif writer = std::make_shared(); + multiPartParser = std::make_shared(); ovms::Server& server = ovms::Server::instance(); handler = std::make_unique(server, 5); ASSERT_EQ(handler->parseRequestComponents(comp, "POST", endpointEmbeddings, headers), ovms::StatusCode::OK); @@ -416,7 +420,7 @@ TEST_F(EmbeddingsExtensionTest, simplePositive) { } )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointEmbeddings, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointEmbeddings, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); rapidjson::Document d; rapidjson::ParseResult ok = d.Parse(response.c_str()); @@ -454,7 +458,7 @@ TEST_F(EmbeddingsInvalidConfigTest, simpleNegative) { "input": "dummyInput" } )"; - Status status = handler->dispatchToProcessor(endpointEmbeddings, requestBody, &response, comp, responseComponents, writer); + Status status = handler->dispatchToProcessor(endpointEmbeddings, requestBody, &response, comp, responseComponents, writer, multiPartParser); ASSERT_EQ(status, ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR) << status.string(); } @@ -482,6 +486,6 @@ TEST_F(EmbeddingsInvalidTokenizerConfigTest, simpleNegative) { "input": "dummyInput" } )"; - Status status = handler->dispatchToProcessor(endpointEmbeddings, requestBody, &response, comp, responseComponents, writer); + Status status = handler->dispatchToProcessor(endpointEmbeddings, requestBody, &response, comp, responseComponents, writer, multiPartParser); ASSERT_EQ(status, ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR) << status.string(); } diff --git a/src/test/test_http_utils.hpp b/src/test/test_http_utils.hpp index c78c40b2f3..ac45a6c33e 100644 --- a/src/test/test_http_utils.hpp +++ b/src/test/test_http_utils.hpp @@ -16,6 +16,7 @@ #pragma once #include #include +#include #include #include @@ -24,6 +25,7 @@ #include "../http_async_writer_interface.hpp" #include "../http_server.hpp" #include "../http_status_code.hpp" +#include "../multi_part_parser.hpp" class MockedServerRequestInterface final : public ovms::HttpAsyncWriter { public: @@ -35,3 +37,12 @@ class MockedServerRequestInterface final : public ovms::HttpAsyncWriter { MOCK_METHOD(void, RegisterDisconnectionCallback, (std::function), (override)); MOCK_METHOD(void, PartialReplyBegin, (std::function), (override)); }; + +class MockedMultiPartParser final : public ovms::MultiPartParser { + public: + MOCK_METHOD(bool, parse, (), (override)); + MOCK_METHOD(bool, hasParseError, (), (const override)); + MOCK_METHOD(std::string, getFieldByName, (const std::string&), (const override)); + MOCK_METHOD(std::string_view, getFileContentByName, (const std::string&), (const override)); + }; + From ab09d595a22d55f32ee9942b3e47f345962a86c7 Mon Sep 17 00:00:00 2001 From: Damian Kalinowski Date: Thu, 6 Mar 2025 15:34:49 +0100 Subject: [PATCH 12/24] test --- src/test/kfs_rest_test.cpp | 83 ++++++++++++++++++++++++------------ src/test/reranknode_test.cpp | 16 ++++--- 2 files changed, 64 insertions(+), 35 deletions(-) diff --git a/src/test/kfs_rest_test.cpp b/src/test/kfs_rest_test.cpp index 12a634f219..6fd90ca921 100644 --- a/src/test/kfs_rest_test.cpp +++ b/src/test/kfs_rest_test.cpp @@ -200,7 +200,8 @@ static void testInference(int headerLength, std::string& request_body, std::uniq std::string response; ovms::HttpResponseComponents responseComponents; std::shared_ptr writer{nullptr}; - ASSERT_EQ(handler->dispatchToProcessor("", request_body, &response, comp, responseComponents, writer), ovms::StatusCode::OK); + std::shared_ptr multiPartParser{nullptr}; + ASSERT_EQ(handler->dispatchToProcessor("", request_body, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); rapidjson::Document doc; doc.Parse(response.c_str()); @@ -231,7 +232,8 @@ static void testInferenceNegative(int headerLength, std::string& request_body, s std::string response; ovms::HttpResponseComponents responseComponents; std::shared_ptr writer{nullptr}; - ASSERT_EQ(handler->dispatchToProcessor("", request_body, &response, comp, responseComponents, writer), processorStatus); + std::shared_ptr multiPartParser{nullptr}; + ASSERT_EQ(handler->dispatchToProcessor("", request_body, &response, comp, responseComponents, writer, multiPartParser), processorStatus); } class HttpRestApiHandlerWithMediapipe : public ::testing::TestWithParam { @@ -381,7 +383,8 @@ TEST_F(HttpRestApiHandlerWithMediapipePassthrough, inferRequestBYTES) { std::string response; ovms::HttpResponseComponents responseComponents; std::shared_ptr writer{nullptr}; - ASSERT_EQ(handler->dispatchToProcessor("", request_body, &response, comp, responseComponents, writer), ovms::StatusCode::OK); + std::shared_ptr multiPartParser{nullptr}; + ASSERT_EQ(handler->dispatchToProcessor("", request_body, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); rapidjson::Document doc; doc.Parse(response.c_str()); @@ -567,7 +570,8 @@ TEST_F(HttpRestApiHandlerTest, dispatchMetadata) { std::string discard; ovms::HttpResponseComponents responseComponents; std::shared_ptr writer{nullptr}; - handler->dispatchToProcessor("", std::string(), &discard, comp, responseComponents, writer); + std::shared_ptr multiPartParser{nullptr}; + handler->dispatchToProcessor("", std::string(), &discard, comp, responseComponents, writer, multiPartParser); ASSERT_EQ(c, 1); } @@ -577,7 +581,7 @@ TEST_F(HttpRestApiHandlerTest, dispatchReady) { ovms::HttpRequestComponents comp; int c = 0; - handler->registerHandler(KFS_GetModelReady, [&](const std::string_view, const ovms::HttpRequestComponents& request_components, std::string& response, const std::string& request_body, ovms::HttpResponseComponents& response_components, std::shared_ptr writer) { + handler->registerHandler(KFS_GetModelReady, [&](const std::string_view, const ovms::HttpRequestComponents& request_components, std::string& response, const std::string& request_body, ovms::HttpResponseComponents& response_components, std::shared_ptr writer, multiPartParser) { c++; return ovms::StatusCode::OK; }); @@ -585,7 +589,8 @@ TEST_F(HttpRestApiHandlerTest, dispatchReady) { std::string discard; ovms::HttpResponseComponents responseComponents; std::shared_ptr writer{nullptr}; - handler->dispatchToProcessor("", std::string(), &discard, comp, responseComponents, writer); + std::shared_ptr multiPartParser{nullptr}; + handler->dispatchToProcessor("", std::string(), &discard, comp, responseComponents, writer, multiPartParser); ASSERT_EQ(c, 1); } @@ -598,7 +603,8 @@ TEST_F(HttpRestApiHandlerTest, modelMetadataRequest) { std::string response; ovms::HttpResponseComponents responseComponents; std::shared_ptr writer{nullptr}; - ASSERT_EQ(handler->dispatchToProcessor("", std::string(), &response, comp, responseComponents, writer), ovms::StatusCode::OK); + std::shared_ptr multiPartParser{nullptr}; + ASSERT_EQ(handler->dispatchToProcessor("", std::string(), &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); rapidjson::Document doc; doc.Parse(response.c_str()); @@ -634,7 +640,8 @@ TEST_F(HttpRestApiHandlerWithScalarModelTest, modelMetadataRequest) { std::string response; ovms::HttpResponseComponents responseComponents; std::shared_ptr writer{nullptr}; - ASSERT_EQ(handler->dispatchToProcessor("", std::string(), &response, comp, responseComponents, writer), ovms::StatusCode::OK); + std::shared_ptr multiPartParser{nullptr}; + ASSERT_EQ(handler->dispatchToProcessor("", std::string(), &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); rapidjson::Document doc; doc.Parse(response.c_str()); @@ -661,7 +668,8 @@ TEST_F(HttpRestApiHandlerTest, inferRequestWithMultidimensionalMatrix) { std::string response; ovms::HttpResponseComponents responseComponents; std::shared_ptr writer{nullptr}; - ASSERT_EQ(handler->dispatchToProcessor("", request_body, &response, comp, responseComponents, writer), ovms::StatusCode::OK); + std::shared_ptr multiPartParser{nullptr}; + ASSERT_EQ(handler->dispatchToProcessor("", request_body, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); rapidjson::Document doc; doc.Parse(response.c_str()); @@ -682,7 +690,8 @@ TEST_F(HttpRestApiHandlerTest, inferRequest) { std::string response; ovms::HttpResponseComponents responseComponents; std::shared_ptr writer{nullptr}; - ASSERT_EQ(handler->dispatchToProcessor("", request_body, &response, comp, responseComponents, writer), ovms::StatusCode::OK); + std::shared_ptr multiPartParser{nullptr}; + ASSERT_EQ(handler->dispatchToProcessor("", request_body, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); rapidjson::Document doc; doc.Parse(response.c_str()); @@ -705,7 +714,8 @@ TEST_F(HttpRestApiHandlerTest, inferRequestWithSpecificBinaryOutputNotBool) { std::string response; ovms::HttpResponseComponents responseComponents; std::shared_ptr writer{nullptr}; - ASSERT_EQ(handler->dispatchToProcessor("", request_body, &response, comp, responseComponents, writer), ovms::StatusCode::OK); + std::shared_ptr multiPartParser{nullptr}; + ASSERT_EQ(handler->dispatchToProcessor("", request_body, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); rapidjson::Document doc; doc.Parse(response.c_str()); ASSERT_EQ(doc["model_name"].GetString(), std::string("dummy")); @@ -727,7 +737,8 @@ TEST_F(HttpRestApiHandlerTest, inferRequestWithDefaultBinaryOutputFalse) { std::string response; ovms::HttpResponseComponents responseComponents; std::shared_ptr writer{nullptr}; - ASSERT_EQ(handler->dispatchToProcessor("", request_body, &response, comp, responseComponents, writer), ovms::StatusCode::OK); + std::shared_ptr multiPartParser{nullptr}; + ASSERT_EQ(handler->dispatchToProcessor("", request_body, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); rapidjson::Document doc; doc.Parse(response.c_str()); ASSERT_EQ(doc["model_name"].GetString(), std::string("dummy")); @@ -749,7 +760,8 @@ TEST_F(HttpRestApiHandlerTest, inferRequestWithSpecificBinaryOutputFalse) { std::string response; ovms::HttpResponseComponents responseComponents; std::shared_ptr writer{nullptr}; - ASSERT_EQ(handler->dispatchToProcessor("", request_body, &response, comp, responseComponents, writer), ovms::StatusCode::OK); + std::shared_ptr multiPartParser{nullptr}; + ASSERT_EQ(handler->dispatchToProcessor("", request_body, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); rapidjson::Document doc; doc.Parse(response.c_str()); ASSERT_EQ(doc["model_name"].GetString(), std::string("dummy")); @@ -771,7 +783,8 @@ TEST_F(HttpRestApiHandlerTest, inferRequestWithDefaultBinaryOutputTrue) { std::string response; ovms::HttpResponseComponents responseComponents; std::shared_ptr writer{nullptr}; - ASSERT_EQ(handler->dispatchToProcessor("", request_body, &response, comp, responseComponents, writer), ovms::StatusCode::OK); + std::shared_ptr multiPartParser{nullptr}; + ASSERT_EQ(handler->dispatchToProcessor("", request_body, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); rapidjson::Document doc; doc.Parse(response.c_str()); ASSERT_EQ(doc["model_name"].GetString(), std::string("dummy")); @@ -795,7 +808,8 @@ TEST_F(HttpRestApiHandlerTest, inferRequestWithSpecificBinaryOutputTrue) { std::string response; ovms::HttpResponseComponents responseComponents; std::shared_ptr writer{nullptr}; - ASSERT_EQ(handler->dispatchToProcessor("", request_body, &response, comp, responseComponents, writer), ovms::StatusCode::OK); + std::shared_ptr multiPartParser{nullptr}; + ASSERT_EQ(handler->dispatchToProcessor("", request_body, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); rapidjson::Document doc; doc.Parse(response.c_str()); ASSERT_EQ(doc["model_name"].GetString(), std::string("dummy")); @@ -819,7 +833,8 @@ TEST_F(HttpRestApiHandlerTest, inferRequestWithSpecificBinaryOutputTrueDefaultFa std::string response; ovms::HttpResponseComponents responseComponents; std::shared_ptr writer{nullptr}; - ASSERT_EQ(handler->dispatchToProcessor("", request_body, &response, comp, responseComponents, writer), ovms::StatusCode::OK); + std::shared_ptr multiPartParser{nullptr}; + ASSERT_EQ(handler->dispatchToProcessor("", request_body, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); rapidjson::Document doc; doc.Parse(response.c_str()); ASSERT_EQ(doc["model_name"].GetString(), std::string("dummy")); @@ -843,7 +858,8 @@ TEST_F(HttpRestApiHandlerWithScalarModelTest, inferRequestScalar) { std::string response; ovms::HttpResponseComponents responseComponents; std::shared_ptr writer{nullptr}; - ASSERT_EQ(handler->dispatchToProcessor("", request_body, &response, comp, responseComponents, writer), ovms::StatusCode::OK); + std::shared_ptr multiPartParser{nullptr}; + ASSERT_EQ(handler->dispatchToProcessor("", request_body, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); rapidjson::Document doc; doc.Parse(response.c_str()); @@ -866,7 +882,8 @@ TEST_F(HttpRestApiHandlerWithDynamicModelTest, inferRequestZeroBatch) { std::string response; ovms::HttpResponseComponents responseComponents; std::shared_ptr writer{nullptr}; - ASSERT_EQ(handler->dispatchToProcessor("", request_body, &response, comp, responseComponents, writer), ovms::StatusCode::OK); + std::shared_ptr multiPartParser{nullptr}; + ASSERT_EQ(handler->dispatchToProcessor("", request_body, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); rapidjson::Document doc; doc.Parse(response.c_str()); @@ -890,7 +907,8 @@ TEST_F(HttpRestApiHandlerWithDynamicModelTest, inferRequestZeroDim) { std::string response; ovms::HttpResponseComponents responseComponents; std::shared_ptr writer{nullptr}; - ASSERT_EQ(handler->dispatchToProcessor("", request_body, &response, comp, responseComponents, writer), ovms::StatusCode::OK); + std::shared_ptr multiPartParser{nullptr}; + ASSERT_EQ(handler->dispatchToProcessor("", request_body, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); rapidjson::Document doc; doc.Parse(response.c_str()); @@ -1453,7 +1471,8 @@ TEST_F(HttpRestApiHandlerWithStringModelTest, invalidPrecision) { std::string response; ovms::HttpResponseComponents responseComponents; std::shared_ptr writer{nullptr}; - ASSERT_EQ(handler->dispatchToProcessor("", request_body, &response, comp, responseComponents, writer), ovms::StatusCode::REST_COULD_NOT_PARSE_INPUT); + std::shared_ptr multiPartParser{nullptr}; + ASSERT_EQ(handler->dispatchToProcessor("", request_body, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::REST_COULD_NOT_PARSE_INPUT); } TEST_F(HttpRestApiHandlerWithStringModelTest, invalidShape) { @@ -1465,7 +1484,8 @@ TEST_F(HttpRestApiHandlerWithStringModelTest, invalidShape) { std::string response; ovms::HttpResponseComponents responseComponents; std::shared_ptr writer{nullptr}; - ASSERT_EQ(handler->dispatchToProcessor("", request_body, &response, comp, responseComponents, writer), ovms::StatusCode::INVALID_VALUE_COUNT); + std::shared_ptr multiPartParser{nullptr}; + ASSERT_EQ(handler->dispatchToProcessor("", request_body, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::INVALID_VALUE_COUNT); } TEST_F(HttpRestApiHandlerWithStringModelTest, invalidShape_noData) { @@ -1477,7 +1497,8 @@ TEST_F(HttpRestApiHandlerWithStringModelTest, invalidShape_noData) { std::string response; ovms::HttpResponseComponents responseComponents; std::shared_ptr writer{nullptr}; - ASSERT_EQ(handler->dispatchToProcessor("", request_body, &response, comp, responseComponents, writer), ovms::StatusCode::INVALID_VALUE_COUNT); + std::shared_ptr multiPartParser{nullptr}; + ASSERT_EQ(handler->dispatchToProcessor("", request_body, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::INVALID_VALUE_COUNT); } TEST_F(HttpRestApiHandlerWithStringModelTest, invalidShape_emptyData) { @@ -1489,7 +1510,8 @@ TEST_F(HttpRestApiHandlerWithStringModelTest, invalidShape_emptyData) { std::string response; ovms::HttpResponseComponents responseComponents; std::shared_ptr writer{nullptr}; - ASSERT_EQ(handler->dispatchToProcessor("", request_body, &response, comp, responseComponents, writer), ovms::StatusCode::INVALID_VALUE_COUNT); + std::shared_ptr multiPartParser{nullptr}; + ASSERT_EQ(handler->dispatchToProcessor("", request_body, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::INVALID_VALUE_COUNT); } void assertStringMetadataOutput(rapidjson::Document& doc) { @@ -1524,7 +1546,8 @@ TEST_F(HttpRestApiHandlerWithStringModelTest, positivePassthrough) { std::string response; ovms::HttpResponseComponents responseComponents; std::shared_ptr writer{nullptr}; - ASSERT_EQ(handler->dispatchToProcessor("", request_body, &response, comp, responseComponents, writer), ovms::StatusCode::OK); + std::shared_ptr multiPartParser{nullptr}; + ASSERT_EQ(handler->dispatchToProcessor("", request_body, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); rapidjson::Document doc; doc.Parse(response.c_str()); @@ -1574,7 +1597,8 @@ TEST_F(HttpRestApiHandlerWithStringModelTest, positivePassthrough_binaryInput) { ovms::HttpResponseComponents responseComponents; std::string output; std::shared_ptr writer{nullptr}; - ASSERT_EQ(handler->processRequest("POST", request, request_body, &headers, &output, responseComponents, writer), ovms::StatusCode::OK); + std::shared_ptr multiPartParser{nullptr}; + ASSERT_EQ(handler->processRequest("POST", request, request_body, &headers, &output, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); ASSERT_TRUE(responseComponents.inferenceHeaderContentLength.has_value()); ASSERT_EQ(responseComponents.inferenceHeaderContentLength.value(), 272); @@ -1601,7 +1625,8 @@ TEST_F(HttpRestApiHandlerTest, serverReady) { std::string response; ovms::HttpResponseComponents responseComponents; std::shared_ptr writer{nullptr}; - ovms::Status status = handler->dispatchToProcessor("", request, &response, comp, responseComponents, writer); + std::shared_ptr multiPartParser{nullptr}; + ovms::Status status = handler->dispatchToProcessor("", request, &response, comp, responseComponents, writer, multiPartParser); ASSERT_EQ(status, ovms::StatusCode::OK); } @@ -1613,7 +1638,8 @@ TEST_F(HttpRestApiHandlerTest, serverLive) { std::string response; ovms::HttpResponseComponents responseComponents; std::shared_ptr writer{nullptr}; - ovms::Status status = handler->dispatchToProcessor("", request, &response, comp, responseComponents, writer); + std::shared_ptr multiPartParser{nullptr}; + ovms::Status status = handler->dispatchToProcessor("", request, &response, comp, responseComponents, writer, multiPartParser); ASSERT_EQ(status, ovms::StatusCode::OK); } @@ -1625,7 +1651,8 @@ TEST_F(HttpRestApiHandlerTest, serverMetadata) { std::string response; ovms::HttpResponseComponents responseComponents; std::shared_ptr writer{nullptr}; - ovms::Status status = handler->dispatchToProcessor("", request, &response, comp, responseComponents, writer); + std::shared_ptr multiPartParser{nullptr}; + ovms::Status status = handler->dispatchToProcessor("", request, &response, comp, responseComponents, writer, multiPartParser); rapidjson::Document doc; doc.Parse(response.c_str()); diff --git a/src/test/reranknode_test.cpp b/src/test/reranknode_test.cpp index f70f42262e..0ad153f3c1 100644 --- a/src/test/reranknode_test.cpp +++ b/src/test/reranknode_test.cpp @@ -38,6 +38,7 @@ class V3HttpTest : public ::testing::Test { const std::string endpointEmbeddings = "/v3/embeddings"; const std::string endpointRerank = "/v3/rerank"; std::shared_ptr writer; + std::shared_ptr multiPartParser; std::string response; ovms::HttpResponseComponents responseComponents; @@ -55,6 +56,7 @@ class V3HttpTest : public ::testing::Test { void SetUp() { writer = std::make_shared(); + multiPartParser = std::make_shared(); ovms::Server& server = ovms::Server::instance(); handler = std::make_unique(server, 5); ASSERT_EQ(handler->parseRequestComponents(comp, "POST", endpointEmbeddings, headers), ovms::StatusCode::OK); @@ -102,7 +104,7 @@ TEST_F(RerankHttpTest, simplePositive) { } )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointRerank, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointRerank, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); rapidjson::Document d; rapidjson::ParseResult ok = d.Parse(response.c_str()); @@ -134,7 +136,7 @@ TEST_F(RerankHttpTest, positiveTopN) { } )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointRerank, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointRerank, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); rapidjson::Document d; rapidjson::ParseResult ok = d.Parse(response.c_str()); @@ -166,7 +168,7 @@ TEST_F(RerankHttpTest, positiveReturnDocuments) { } )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointRerank, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointRerank, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); rapidjson::Document d; rapidjson::ParseResult ok = d.Parse(response.c_str()); @@ -242,7 +244,7 @@ TEST_F(RerankWithParamsHttpTest, PositiveMaxAllowedChunksNotExceeded) { std::string requestBody = buffer.GetString(); ASSERT_EQ( - handler->dispatchToProcessor(endpointRerank, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointRerank, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); } @@ -270,7 +272,7 @@ TEST_F(RerankWithParamsHttpTest, MaxAllowedChunksExceededByDocumentsBeforeChunki document.Accept(wr); std::string requestBody = buffer.GetString(); - auto status = handler->dispatchToProcessor(endpointRerank, requestBody, &response, comp, responseComponents, writer); + auto status = handler->dispatchToProcessor(endpointRerank, requestBody, &response, comp, responseComponents, writer, multiPartParser); ASSERT_EQ(status, ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); ASSERT_THAT(status.string(), ::testing::HasSubstr("Number of documents exceeds max_allowed_chunks")); // 5 because we prepared 1 document more than allowed } @@ -302,7 +304,7 @@ TEST_F(RerankWithParamsHttpTest, MaxAllowedChunksExceededAfterChunking) { document.Accept(wr); std::string requestBody = buffer.GetString(); - auto status = handler->dispatchToProcessor(endpointRerank, requestBody, &response, comp, responseComponents, writer); + auto status = handler->dispatchToProcessor(endpointRerank, requestBody, &response, comp, responseComponents, writer, multiPartParser); ASSERT_EQ(status, ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR) << status.string(); ASSERT_THAT(status.string(), ::testing::HasSubstr("Chunking failed: exceeding max_allowed_chunks after chunking limit: 4; actual: 8")); // 8 because of the last document which was chunked to 5 documents, 3 + 5 = 8 } @@ -357,7 +359,7 @@ TEST_F(RerankWithInvalidParamsHttpTest, AnyRequestNegativeWithInvalidSetup) { document.Accept(wr); std::string requestBody = buffer.GetString(); - auto status = handler->dispatchToProcessor(endpointRerank, requestBody, &response, comp, responseComponents, writer); + auto status = handler->dispatchToProcessor(endpointRerank, requestBody, &response, comp, responseComponents, writer, multiPartParser); ASSERT_EQ(status, ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); ASSERT_THAT(status.string(), ::testing::HasSubstr("max_position_embeddings should be larger than 2 * NUMBER_OF_SPECIAL_TOKENS")); } From f4e02d94435568d12aa6401f2d301cf65ff733dc Mon Sep 17 00:00:00 2001 From: Damian Kalinowski Date: Thu, 6 Mar 2025 15:36:23 +0100 Subject: [PATCH 13/24] test --- src/test/kfs_rest_test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/kfs_rest_test.cpp b/src/test/kfs_rest_test.cpp index 6fd90ca921..86ea869619 100644 --- a/src/test/kfs_rest_test.cpp +++ b/src/test/kfs_rest_test.cpp @@ -562,7 +562,7 @@ TEST_F(HttpRestApiHandlerTest, dispatchMetadata) { ovms::HttpRequestComponents comp; int c = 0; - handler->registerHandler(KFS_GetModelMetadata, [&](const std::string_view uri, const ovms::HttpRequestComponents& request_components, std::string& response, const std::string& request_body, ovms::HttpResponseComponents& response_components, std::shared_ptr) { + handler->registerHandler(KFS_GetModelMetadata, [&](const std::string_view uri, const ovms::HttpRequestComponents& request_components, std::string& response, const std::string& request_body, ovms::HttpResponseComponents& response_components, std::shared_ptr, std::shared_ptr) { c++; return ovms::StatusCode::OK; }); @@ -581,7 +581,7 @@ TEST_F(HttpRestApiHandlerTest, dispatchReady) { ovms::HttpRequestComponents comp; int c = 0; - handler->registerHandler(KFS_GetModelReady, [&](const std::string_view, const ovms::HttpRequestComponents& request_components, std::string& response, const std::string& request_body, ovms::HttpResponseComponents& response_components, std::shared_ptr writer, multiPartParser) { + handler->registerHandler(KFS_GetModelReady, [&](const std::string_view, const ovms::HttpRequestComponents& request_components, std::string& response, const std::string& request_body, ovms::HttpResponseComponents& response_components, std::shared_ptr, std::shared_ptr) { c++; return ovms::StatusCode::OK; }); From 3a4a6e79d9a2110aeb6b653b07a31aed862d754c Mon Sep 17 00:00:00 2001 From: Damian Kalinowski Date: Thu, 6 Mar 2025 15:48:59 +0100 Subject: [PATCH 14/24] Test --- src/test/http_openai_handler_test.cpp | 22 +-- src/test/llmnode_test.cpp | 240 +++++++++++++------------- src/test/llmtemplate_test.cpp | 12 +- src/test/metrics_flow_test.cpp | 20 ++- 4 files changed, 152 insertions(+), 142 deletions(-) diff --git a/src/test/http_openai_handler_test.cpp b/src/test/http_openai_handler_test.cpp index ab4e35fb33..3eaa0b6439 100644 --- a/src/test/http_openai_handler_test.cpp +++ b/src/test/http_openai_handler_test.cpp @@ -41,6 +41,7 @@ class HttpOpenAIHandlerTest : public ::testing::Test { ovms::HttpRequestComponents comp; const std::string endpoint = "/v3/chat/completions"; std::shared_ptr writer; + std::shared_ptr multiPartParser; std::string response; ovms::HttpResponseComponents responseComponents; @@ -56,6 +57,7 @@ class HttpOpenAIHandlerTest : public ::testing::Test { void SetUp() { writer = std::make_shared(); + multiPartParser = std::make_shared(); SetUpServer(getGenericFullPathForSrcTest("/ovms/src/test/mediapipe/config_mediapipe_openai_chat_completions_mock.json").c_str()); ASSERT_EQ(handler->parseRequestComponents(comp, "POST", endpoint, headers), ovms::StatusCode::OK); } @@ -78,7 +80,7 @@ TEST_F(HttpOpenAIHandlerTest, Unary) { )"; ASSERT_EQ( - handler->dispatchToProcessor("/v3/v1/completions/", requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor("/v3/v1/completions/", requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); std::string expectedResponse = R"(/v3/v1/completions/ @@ -104,7 +106,7 @@ TEST_F(HttpOpenAIHandlerTest, UnaryWithHeaders) { comp.headers["test2"] = "header"; ASSERT_EQ( - handler->dispatchToProcessor("/v3/completions/", requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor("/v3/completions/", requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); std::string expectedResponse = R"(/v3/completions/ @@ -133,7 +135,7 @@ TEST_F(HttpOpenAIHandlerTest, Stream) { EXPECT_CALL(*writer, IsDisconnected()).Times(9); ASSERT_EQ( - handler->dispatchToProcessor("/v3/completions", requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor("/v3/completions", requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::PARTIAL_END); ASSERT_EQ(response, ""); @@ -146,7 +148,7 @@ TEST_F(HttpOpenAIHandlerTest, BodyNotAJson) { EXPECT_CALL(*writer, PartialReply(::testing::_)).Times(0); EXPECT_CALL(*writer, IsDisconnected()).Times(0); - auto status = handler->dispatchToProcessor("/v3/completions", requestBody, &response, comp, responseComponents, writer); + auto status = handler->dispatchToProcessor("/v3/completions", requestBody, &response, comp, responseComponents, writer, multiPartParser); ASSERT_EQ(status, ovms::StatusCode::JSON_INVALID); ASSERT_EQ(status.string(), "The file is not valid json - Cannot parse JSON body"); } @@ -158,7 +160,7 @@ TEST_F(HttpOpenAIHandlerTest, JsonBodyValidButNotAnObject) { EXPECT_CALL(*writer, PartialReply(::testing::_)).Times(0); EXPECT_CALL(*writer, IsDisconnected()).Times(0); - auto status = handler->dispatchToProcessor("/v3/completions", requestBody, &response, comp, responseComponents, writer); + auto status = handler->dispatchToProcessor("/v3/completions", requestBody, &response, comp, responseComponents, writer, multiPartParser); ASSERT_EQ(status, ovms::StatusCode::JSON_INVALID); ASSERT_EQ(status.string(), "The file is not valid json - JSON body must be an object"); } @@ -175,7 +177,7 @@ TEST_F(HttpOpenAIHandlerTest, ModelFieldMissing) { EXPECT_CALL(*writer, PartialReply(::testing::_)).Times(0); EXPECT_CALL(*writer, IsDisconnected()).Times(0); - auto status = handler->dispatchToProcessor("/v3/completions", requestBody, &response, comp, responseComponents, writer); + auto status = handler->dispatchToProcessor("/v3/completions", requestBody, &response, comp, responseComponents, writer, multiPartParser); ASSERT_EQ(status, ovms::StatusCode::JSON_INVALID); ASSERT_EQ(status.string(), "The file is not valid json - model field is missing in JSON body"); } @@ -193,7 +195,7 @@ TEST_F(HttpOpenAIHandlerTest, ModelFieldNotAString) { EXPECT_CALL(*writer, PartialReply(::testing::_)).Times(0); EXPECT_CALL(*writer, IsDisconnected()).Times(0); - auto status = handler->dispatchToProcessor("/v3/completions", requestBody, &response, comp, responseComponents, writer); + auto status = handler->dispatchToProcessor("/v3/completions", requestBody, &response, comp, responseComponents, writer, multiPartParser); ASSERT_EQ(status, ovms::StatusCode::JSON_INVALID); ASSERT_EQ(status.string(), "The file is not valid json - model field is not a string"); } @@ -212,7 +214,7 @@ TEST_F(HttpOpenAIHandlerTest, StreamFieldNotABoolean) { EXPECT_CALL(*writer, PartialReply(::testing::_)).Times(0); EXPECT_CALL(*writer, IsDisconnected()).Times(0); - auto status = handler->dispatchToProcessor("/v3/completions", requestBody, &response, comp, responseComponents, writer); + auto status = handler->dispatchToProcessor("/v3/completions", requestBody, &response, comp, responseComponents, writer, multiPartParser); ASSERT_EQ(status, ovms::StatusCode::JSON_INVALID); ASSERT_EQ(status.string(), "The file is not valid json - stream field is not a boolean"); } @@ -230,7 +232,7 @@ TEST_F(HttpOpenAIHandlerTest, GraphWithANameDoesNotExist) { EXPECT_CALL(*writer, PartialReply(::testing::_)).Times(0); EXPECT_CALL(*writer, IsDisconnected()).Times(0); - auto status = handler->dispatchToProcessor("/v3/completions", requestBody, &response, comp, responseComponents, writer); + auto status = handler->dispatchToProcessor("/v3/completions", requestBody, &response, comp, responseComponents, writer, multiPartParser); ASSERT_EQ(status, ovms::StatusCode::MEDIAPIPE_DEFINITION_NAME_MISSING); } @@ -502,6 +504,6 @@ TEST_F(HttpOpenAIHandlerTest, V3ApiWithNonLLMCalculator) { EXPECT_CALL(*writer, PartialReply(::testing::_)).Times(0); EXPECT_CALL(*writer, IsDisconnected()).Times(0); - auto status = handler->dispatchToProcessor("/v3/completions", requestBody, &response, comp, responseComponents, writer); + auto status = handler->dispatchToProcessor("/v3/completions", requestBody, &response, comp, responseComponents, writer, multiPartParser); ASSERT_EQ(status, ovms::StatusCode::MEDIAPIPE_GRAPH_ADD_PACKET_INPUT_STREAM); } diff --git a/src/test/llmnode_test.cpp b/src/test/llmnode_test.cpp index e0460c5eca..f2dbf1ec57 100644 --- a/src/test/llmnode_test.cpp +++ b/src/test/llmnode_test.cpp @@ -62,6 +62,7 @@ class LLMFlowHttpTest : public ::testing::Test { const std::string endpointChatCompletions = "/v3/chat/completions"; const std::string endpointCompletions = "/v3/completions"; std::shared_ptr writer; + std::shared_ptr multiPartParser; std::string response; rapidjson::Document parsedResponse; ovms::HttpResponseComponents responseComponents; @@ -137,6 +138,7 @@ class LLMFlowHttpTest : public ::testing::Test { void SetUp() { writer = std::make_shared(); + multiPartParser = std::make_shared(); ON_CALL(*writer, PartialReplyBegin(::testing::_)).WillByDefault(testing::Invoke([](std::function fn) { fn(); })); // make the streaming flow sequential ovms::Server& server = ovms::Server::instance(); handler = std::make_unique(server, 5); @@ -191,7 +193,7 @@ TEST_F(LLMFlowHttpTest, unaryCompletionsJson) { )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); parsedResponse.Parse(response.c_str()); ASSERT_TRUE(parsedResponse["choices"].IsArray()); @@ -232,7 +234,7 @@ TEST_F(LLMFlowHttpTest, unaryCompletionsJsonSpeculativeDecoding) { )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); parsedResponse.Parse(response.c_str()); ASSERT_TRUE(parsedResponse["choices"].IsArray()); @@ -254,7 +256,7 @@ TEST_F(LLMFlowHttpTest, unaryCompletionsJsonSpeculativeDecoding) { )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); parsedResponse.Parse(response.c_str()); ASSERT_TRUE(parsedResponse["choices"].IsArray()); @@ -284,7 +286,7 @@ TEST_F(LLMFlowHttpTest, unaryCompletionsJsonEchoWithCompletion) { )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); parsedResponse.Parse(response.c_str()); ASSERT_TRUE(parsedResponse["choices"].IsArray()); @@ -348,7 +350,7 @@ TEST_F(LLMFlowHttpTest, streamCompletionsEchoWithCompletion) { }); ASSERT_EQ( - handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::PARTIAL_END); // Since prompt is treated as a single entity and streamer returns chunk only after space or newline @@ -370,7 +372,7 @@ TEST_F(LLMFlowHttpTest, unaryCompletionsJsonEchoOnly) { )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); parsedResponse.Parse(response.c_str()); ASSERT_TRUE(parsedResponse["choices"].IsArray()); @@ -444,7 +446,7 @@ TEST_F(LLMFlowHttpTest, streamCompletionsEchoOnly) { EXPECT_STREQ(d["object"].GetString(), "text_completion.chunk"); }); ASSERT_EQ( - handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::PARTIAL_END); } @@ -460,7 +462,7 @@ TEST_F(LLMFlowHttpTest, unaryCompletionsJsonFinishReasonLength) { )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); parsedResponse.Parse(response.c_str()); ASSERT_TRUE(parsedResponse["choices"].IsArray()); @@ -491,7 +493,7 @@ TEST_F(LLMFlowHttpTest, unaryCompletionsJsonSingleStopString) { )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); parsedResponse.Parse(response.c_str()); ASSERT_TRUE(parsedResponse["choices"].IsArray()); @@ -524,7 +526,7 @@ TEST_F(LLMFlowHttpTest, unaryCompletionsJsonNFail) { )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } TEST_F(LLMFlowHttpTest, unaryCompletionsJsonN) { @@ -548,7 +550,7 @@ TEST_F(LLMFlowHttpTest, unaryCompletionsJsonN) { )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); parsedResponse.Parse(response.c_str()); ASSERT_TRUE(parsedResponse["choices"].IsArray()); @@ -589,7 +591,7 @@ TEST_F(LLMFlowHttpTest, unaryChatCompletionsJsonNFail) { )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } @@ -619,7 +621,7 @@ TEST_F(LLMFlowHttpTest, unaryChatCompletionsJsonN) { )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); parsedResponse.Parse(response.c_str()); ASSERT_TRUE(parsedResponse["choices"].IsArray()); @@ -658,7 +660,7 @@ TEST_F(LLMFlowHttpTest, KFSApiRequestToChatCompletionsGraph) { std::unordered_map headers; ASSERT_EQ(handler->parseRequestComponents(comp, "POST", "/v2/models/llmDummyKFS/versions/1/infer", headers), ovms::StatusCode::OK); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_GRAPH_ADD_PACKET_INPUT_STREAM); } @@ -680,7 +682,7 @@ TEST_F(LLMFlowHttpTest, unaryChatCompletionsJson) { )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); parsedResponse.Parse(response.c_str()); ASSERT_TRUE(parsedResponse["choices"].IsArray()); @@ -732,7 +734,7 @@ TEST_F(LLMFlowHttpTest, unaryChatCompletionsJsonSpeculativeDecoding) { )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); parsedResponse.Parse(response.c_str()); ASSERT_TRUE(parsedResponse["choices"].IsArray()); @@ -760,7 +762,7 @@ TEST_F(LLMFlowHttpTest, unaryChatCompletionsJsonSpeculativeDecoding) { )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); parsedResponse.Parse(response.c_str()); ASSERT_TRUE(parsedResponse["choices"].IsArray()); @@ -789,7 +791,7 @@ TEST_F(LLMFlowHttpTest, unaryChatCompletionsJsonContentArray) { )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); parsedResponse.Parse(response.c_str()); ASSERT_TRUE(parsedResponse["choices"].IsArray()); @@ -832,7 +834,7 @@ TEST_F(LLMFlowHttpTest, unaryChatCompletionsJsonContentArrayWithImage) { )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); parsedResponse.Parse(response.c_str()); ASSERT_TRUE(parsedResponse["choices"].IsArray()); @@ -878,7 +880,7 @@ TEST_F(LLMFlowHttpTest, unaryChatCompletionsJsonNMultipleStopStrings) { )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); parsedResponse.Parse(response.c_str()); ASSERT_TRUE(parsedResponse["choices"].IsArray()); @@ -917,7 +919,7 @@ TEST_F(LLMFlowHttpTest, DISABLED_unaryChatCompletionsJsonLogprobs) { )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); parsedResponse.Parse(response.c_str()); ASSERT_TRUE(parsedResponse["choices"].IsArray()); @@ -948,7 +950,7 @@ TEST_F(LLMFlowHttpTest, unaryCompletionsJsonLogprobs) { )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); parsedResponse.Parse(response.c_str()); ASSERT_TRUE(parsedResponse["choices"].IsArray()); @@ -983,7 +985,7 @@ TEST_F(LLMFlowHttpTest, ChatCompletionsJsonLogprobsStream) { )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::PARTIAL_END); } @@ -1000,7 +1002,7 @@ TEST_F(LLMFlowHttpTest, CompletionsJsonLogprobsStream) { )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } @@ -1022,7 +1024,7 @@ TEST_F(LLMFlowHttpTest, unaryChatCompletionsStopStringBadType) { )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } @@ -1045,7 +1047,7 @@ TEST_F(LLMFlowHttpTest, unaryChatCompletionsIncludeStopStringInOutputBadType) { )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } @@ -1062,7 +1064,7 @@ TEST_F(LLMFlowHttpTest, unaryCompletionsStopStringElementBadType) { )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } @@ -1084,7 +1086,7 @@ TEST_F(LLMFlowHttpTest, unaryChatCompletionsStopStringExceedingSize) { )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } @@ -1101,7 +1103,7 @@ TEST_F(LLMFlowHttpTest, unaryCompletionsStopStringEmpty) { )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); } @@ -1116,7 +1118,7 @@ TEST_F(LLMFlowHttpTest, streamBeamSearchCompletionsFail) { )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::PARTIAL_END); } @@ -1136,7 +1138,7 @@ TEST_F(LLMFlowHttpTest, streamBeamSearchChatCompletionsFail) { )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::PARTIAL_END); } @@ -1176,7 +1178,7 @@ TEST_F(LLMFlowHttpTest, inferCompletionsStream) { EXPECT_STREQ(d["object"].GetString(), "text_completion.chunk"); }); ASSERT_EQ( - handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::PARTIAL_END); } @@ -1222,7 +1224,7 @@ TEST_F(LLMFlowHttpTest, inferChatCompletionsStream) { EXPECT_STREQ(d["object"].GetString(), "chat.completion.chunk"); }); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::PARTIAL_END); } @@ -1244,7 +1246,7 @@ TEST_F(LLMFlowHttpTest, unaryChatCompletionsStreamOptionsSetFail) { )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } @@ -1261,7 +1263,7 @@ TEST_F(LLMFlowHttpTest, unaryCompletionsStreamOptionsSetFail) { )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } @@ -1290,7 +1292,7 @@ TEST_F(LLMFlowHttpTest, streamChatCompletionsFinishReasonLength) { }); EXPECT_CALL(*writer, PartialReplyEnd()).Times(1); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::PARTIAL_END); ASSERT_TRUE(responses.back().find("\"finish_reason\":\"length\"") != std::string::npos); } @@ -1323,7 +1325,7 @@ TEST_F(LLMFlowHttpTest, streamChatCompletionsSingleStopString) { }); EXPECT_CALL(*writer, PartialReplyEnd()).Times(1); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::PARTIAL_END); ASSERT_TRUE(responses.back().find("\"finish_reason\":\"stop\"") != std::string::npos); std::regex content_regex("\"content\":\".*\\.[ ]{0,1}\""); @@ -1350,7 +1352,7 @@ TEST_F(LLMFlowHttpTest, streamCompletionsFinishReasonLength) { }); EXPECT_CALL(*writer, PartialReplyEnd()).Times(1); ASSERT_EQ( - handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::PARTIAL_END); ASSERT_TRUE(responses.back().find("\"finish_reason\":\"length\"") != std::string::npos); } @@ -1379,7 +1381,7 @@ TEST_F(LLMFlowHttpTest, streamCompletionsSingleStopString) { }); EXPECT_CALL(*writer, PartialReplyEnd()).Times(1); ASSERT_EQ( - handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::PARTIAL_END); ASSERT_TRUE(responses.back().find("\"finish_reason\":\"stop\"") != std::string::npos); std::regex content_regex("\"text\":\".*\\.[ ]{0,1}\""); @@ -1412,7 +1414,7 @@ TEST_F(LLMFlowHttpTest, streamChatCompletionsUsage) { }); EXPECT_CALL(*writer, PartialReplyEnd()).Times(1); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::PARTIAL_END); ASSERT_TRUE(responses.back().find("\"completion_tokens\":5") != std::string::npos); ASSERT_TRUE(responses.back().find("\"prompt_tokens\"") != std::string::npos); @@ -1441,7 +1443,7 @@ TEST_F(LLMFlowHttpTest, streamCompletionsUsage) { }); EXPECT_CALL(*writer, PartialReplyEnd()).Times(1); ASSERT_EQ( - handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::PARTIAL_END); ASSERT_TRUE(responses.back().find("\"completion_tokens\":5") != std::string::npos); ASSERT_TRUE(responses.back().find("\"prompt_tokens\"") != std::string::npos); @@ -1478,7 +1480,7 @@ TEST_F(LLMFlowHttpTest, streamChatCompletionsBadStopStringType) { }); EXPECT_CALL(*writer, PartialReplyEnd()).Times(1); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::PARTIAL_END); } @@ -1505,7 +1507,7 @@ TEST_F(LLMFlowHttpTest, streamCompletionsBadStopStringElementType) { }); EXPECT_CALL(*writer, PartialReplyEnd()).Times(1); ASSERT_EQ( - handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::PARTIAL_END); } @@ -1538,7 +1540,7 @@ TEST_F(LLMFlowHttpTest, streamCompletionsIncludeStopStrInOutputFalse) { }); EXPECT_CALL(*writer, PartialReplyEnd()).Times(1); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::PARTIAL_END); } @@ -1566,7 +1568,7 @@ TEST_F(LLMFlowHttpTest, streamCompletionsBadIncludeStopStrInOutputType) { }); EXPECT_CALL(*writer, PartialReplyEnd()).Times(1); ASSERT_EQ( - handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::PARTIAL_END); } @@ -1598,7 +1600,7 @@ TEST_F(LLMFlowHttpTest, streamChatCompletionsBadStreamOptionsBadType) { }); EXPECT_CALL(*writer, PartialReplyEnd()).Times(1); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::PARTIAL_END); } @@ -1625,7 +1627,7 @@ TEST_F(LLMFlowHttpTest, streamCompletionsStreamOptionsBadType) { }); EXPECT_CALL(*writer, PartialReplyEnd()).Times(1); ASSERT_EQ( - handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::PARTIAL_END); } @@ -1657,7 +1659,7 @@ TEST_F(LLMFlowHttpTest, streamChatCompletionsStreamOptionsBadContent) { }); EXPECT_CALL(*writer, PartialReplyEnd()).Times(1); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::PARTIAL_END); } @@ -1684,7 +1686,7 @@ TEST_F(LLMFlowHttpTest, streamCompletionsStreamOptionsBadContent) { }); EXPECT_CALL(*writer, PartialReplyEnd()).Times(1); ASSERT_EQ( - handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::PARTIAL_END); } @@ -1716,7 +1718,7 @@ TEST_F(LLMFlowHttpTest, streamChatCompletionsBadIncludeUsage) { }); EXPECT_CALL(*writer, PartialReplyEnd()).Times(1); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::PARTIAL_END); } @@ -1743,7 +1745,7 @@ TEST_F(LLMFlowHttpTest, streamCompletionsBadIncludeUsage) { }); EXPECT_CALL(*writer, PartialReplyEnd()).Times(1); ASSERT_EQ( - handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::PARTIAL_END); } @@ -1770,7 +1772,7 @@ TEST_F(LLMFlowHttpTest, inferChatCompletionsUnaryClientDisconnectedImmediately) fn(); // disconnect immediately, even before read_all is called }); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } @@ -1808,7 +1810,7 @@ TEST_F(LLMFlowHttpTest, inferChatCompletionsStreamClientDisconnectedImmediately) }); // no results ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::PARTIAL_END); ASSERT_EQ(i, 1); ASSERT_EQ(response, ""); @@ -1843,7 +1845,7 @@ TEST_F(LLMFlowHttpTest, inferCompletionsStreamClientDisconnectedImmediately) { }); // no results ASSERT_EQ( - handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::PARTIAL_END); ASSERT_EQ(i, 1); ASSERT_EQ(response, ""); @@ -1886,7 +1888,7 @@ TEST_F(LLMHttpParametersValidationTest, maxTokensInvalid) { )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } @@ -1906,7 +1908,7 @@ TEST_F(LLMHttpParametersValidationTest, maxTokensExceedsUint32Size) { )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } @@ -1927,7 +1929,7 @@ TEST_F(LLMHttpParametersValidationTest, maxTokensExceeds4000WhenIgnoreEosTrue) { )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } @@ -1947,7 +1949,7 @@ TEST_F(LLMHttpParametersValidationTest, maxCompletionsTokensInvalid) { )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } @@ -1967,7 +1969,7 @@ TEST_F(LLMHttpParametersValidationTest, maxCompletionsTokensExceedsUint32Size) { )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } @@ -1988,7 +1990,7 @@ TEST_F(LLMHttpParametersValidationTest, maxCompletionsTokensExceeds4000WhenIgnor )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } @@ -1996,7 +1998,7 @@ TEST_F(LLMHttpParametersValidationTest, streamInvalid) { std::string requestBody = validRequestBodyWithParameter("stream", "\"INVALID\""); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::JSON_INVALID); } @@ -2011,7 +2013,7 @@ TEST_F(LLMHttpParametersValidationTest, messagesInvalid) { )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } @@ -2025,7 +2027,7 @@ TEST_F(LLMHttpParametersValidationTest, messagesMissing) { )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } @@ -2042,7 +2044,7 @@ TEST_F(LLMHttpParametersValidationTest, messageNotAnObject) { )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } @@ -2062,7 +2064,7 @@ TEST_F(LLMHttpParametersValidationTest, messageNotAString) { )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } @@ -2082,7 +2084,7 @@ TEST_F(LLMHttpParametersValidationTest, roleNotAString) { )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } @@ -2097,7 +2099,7 @@ TEST_F(LLMHttpParametersValidationTest, promptInvalid) { )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } @@ -2111,7 +2113,7 @@ TEST_F(LLMHttpParametersValidationTest, promptMissing) { )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } @@ -2130,7 +2132,7 @@ TEST_F(LLMHttpParametersValidationTest, modelMissing) { )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::JSON_INVALID); } @@ -2150,7 +2152,7 @@ TEST_F(LLMHttpParametersValidationTest, modelInvalid) { )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::JSON_INVALID); } @@ -2158,7 +2160,7 @@ TEST_F(LLMHttpParametersValidationTest, ignoreEosValid) { std::string requestBody = validRequestBodyWithParameter("ignore_eos", "false"); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); } @@ -2166,7 +2168,7 @@ TEST_F(LLMHttpParametersValidationTest, ignoreEosInvalid) { std::string requestBody = validRequestBodyWithParameter("ignore_eos", "\"INVALID\""); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } @@ -2174,13 +2176,13 @@ TEST_F(LLMHttpParametersValidationTest, repetitionPenaltyValid) { std::string requestBody = validRequestBodyWithParameter("repetition_penalty", "2.0"); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); requestBody = validRequestBodyWithParameter("repetition_penalty", "1"); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); } @@ -2188,7 +2190,7 @@ TEST_F(LLMHttpParametersValidationTest, repetitionPenaltyInvalid) { std::string requestBody = validRequestBodyWithParameter("repetition_penalty", "\"INVALID\""); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } @@ -2196,7 +2198,7 @@ TEST_F(LLMHttpParametersValidationTest, diversityPenaltyValid) { std::string requestBody = validRequestBodyWithParameter("diversity_penalty", "2.0"); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); } @@ -2204,7 +2206,7 @@ TEST_F(LLMHttpParametersValidationTest, diversityPenaltyInvalid) { std::string requestBody = validRequestBodyWithParameter("diversity_penalty", "\"INVALID\""); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } @@ -2212,13 +2214,13 @@ TEST_F(LLMHttpParametersValidationTest, lengthPenaltyValid) { std::string requestBody = validRequestBodyWithParameter("length_penalty", "2.0"); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); requestBody = validRequestBodyWithParameter("length_penalty", "2"); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); } @@ -2226,7 +2228,7 @@ TEST_F(LLMHttpParametersValidationTest, lengthPenaltyInvalid) { std::string requestBody = validRequestBodyWithParameter("length_penalty", "\"INVALID\""); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } @@ -2234,19 +2236,19 @@ TEST_F(LLMHttpParametersValidationTest, temperatureValid) { std::string requestBody = validRequestBodyWithParameter("temperature", "1.5"); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); requestBody = validRequestBodyWithParameter("temperature", "0"); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); requestBody = validRequestBodyWithParameter("temperature", "2"); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); } @@ -2254,7 +2256,7 @@ TEST_F(LLMHttpParametersValidationTest, temperatureInvalid) { std::string requestBody = validRequestBodyWithParameter("temperature", "\"INVALID\""); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } @@ -2262,7 +2264,7 @@ TEST_F(LLMHttpParametersValidationTest, temperatureOutOfRange) { std::string requestBody = validRequestBodyWithParameter("temperature", "3.0"); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } @@ -2270,13 +2272,13 @@ TEST_F(LLMHttpParametersValidationTest, frequencyPenaltyValid) { std::string requestBody = validRequestBodyWithParameter("frequency_penalty", "1.5"); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); requestBody = validRequestBodyWithParameter("frequency_penalty", "1"); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); } @@ -2284,7 +2286,7 @@ TEST_F(LLMHttpParametersValidationTest, frequencyPenaltyInvalid) { std::string requestBody = validRequestBodyWithParameter("frequency_penalty", "\"INVALID\""); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } @@ -2292,7 +2294,7 @@ TEST_F(LLMHttpParametersValidationTest, frequencyPenaltyOutOfRange) { std::string requestBody = validRequestBodyWithParameter("frequency_penalty", "3.0"); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } @@ -2300,13 +2302,13 @@ TEST_F(LLMHttpParametersValidationTest, presencePenaltyValid) { std::string requestBody = validRequestBodyWithParameter("presence_penalty", "1.5"); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); requestBody = validRequestBodyWithParameter("presence_penalty", "1"); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); } @@ -2314,7 +2316,7 @@ TEST_F(LLMHttpParametersValidationTest, presencePenaltyInvalid) { std::string requestBody = validRequestBodyWithParameter("presence_penalty", "\"INVALID\""); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } @@ -2322,7 +2324,7 @@ TEST_F(LLMHttpParametersValidationTest, presencePenaltyOutOfRange) { std::string requestBody = validRequestBodyWithParameter("presence_penalty", "3.0"); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } @@ -2330,13 +2332,13 @@ TEST_F(LLMHttpParametersValidationTest, topPValid) { std::string requestBody = validRequestBodyWithParameter("top_p", "0.5"); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); requestBody = validRequestBodyWithParameter("top_p", "1"); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); } @@ -2344,7 +2346,7 @@ TEST_F(LLMHttpParametersValidationTest, topPInvalid) { std::string requestBody = validRequestBodyWithParameter("top_p", "\"INVALID\""); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } @@ -2352,7 +2354,7 @@ TEST_F(LLMHttpParametersValidationTest, topPOutOfRange) { std::string requestBody = validRequestBodyWithParameter("top_p", "3.0"); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } @@ -2360,7 +2362,7 @@ TEST_F(LLMHttpParametersValidationTest, topKValid) { std::string requestBody = validRequestBodyWithParameter("top_k", "2"); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); } @@ -2368,7 +2370,7 @@ TEST_F(LLMHttpParametersValidationTest, topKInvalid) { std::string requestBody = validRequestBodyWithParameter("top_k", "\"INVALID\""); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } @@ -2376,7 +2378,7 @@ TEST_F(LLMHttpParametersValidationTest, seedValid) { std::string requestBody = validRequestBodyWithParameter("seed", "1"); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); } @@ -2384,7 +2386,7 @@ TEST_F(LLMHttpParametersValidationTest, seedInvalid) { std::string requestBody = validRequestBodyWithParameter("seed", "\"INVALID\""); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } @@ -2392,7 +2394,7 @@ TEST_F(LLMHttpParametersValidationTest, bestOfValid) { std::string requestBody = validRequestBodyWithParameter("best_of", "1"); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); } @@ -2400,7 +2402,7 @@ TEST_F(LLMHttpParametersValidationTest, bestOfInvalid) { std::string requestBody = validRequestBodyWithParameter("best_of", "\"INVALID\""); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } @@ -2408,7 +2410,7 @@ TEST_F(LLMHttpParametersValidationTest, bestOfNegative) { std::string requestBody = validRequestBodyWithParameter("best_of", "-1"); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } @@ -2416,7 +2418,7 @@ TEST_F(LLMHttpParametersValidationTest, bestOfExceedsLimit) { std::string requestBody = validRequestBodyWithParameter("best_of", "40"); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } @@ -2424,7 +2426,7 @@ TEST_F(LLMHttpParametersValidationTest, nValid) { std::string requestBody = validRequestBodyWithParameter("n", "1"); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); } @@ -2432,7 +2434,7 @@ TEST_F(LLMHttpParametersValidationTest, nInvalid) { std::string requestBody = validRequestBodyWithParameter("n", "\"INVALID\""); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } @@ -2440,7 +2442,7 @@ TEST_F(LLMHttpParametersValidationTest, nNegative) { std::string requestBody = validRequestBodyWithParameter("best_of", "-1"); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } @@ -2462,7 +2464,7 @@ TEST_F(LLMHttpParametersValidationTest, nGreaterThanBestOf) { )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } @@ -2476,7 +2478,7 @@ TEST_F(LLMHttpParametersValidationTest, MessagesEmpty) { )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } @@ -2489,7 +2491,7 @@ TEST_F(LLMHttpParametersValidationTest, MessagesWithEmptyObject) { )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } @@ -2502,7 +2504,7 @@ TEST_F(LLMHttpParametersValidationTest, EmptyPrompt) { )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } @@ -2515,7 +2517,7 @@ TEST_F(LLMHttpParametersValidationTest, MessagesWithOnlyRole) { )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } @@ -2530,7 +2532,7 @@ TEST_F(LLMHttpParametersValidationTest, SpeculativeDecodingExclusiveParametersPr )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } @@ -2545,7 +2547,7 @@ TEST_F(LLMHttpParametersValidationTest, SpeculativeDecodingExclusiveParametersPr )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } @@ -2558,7 +2560,7 @@ TEST_F(LLMHttpParametersValidationTest, SpeculativeDecodingNoSDSpecificParameter )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } @@ -2571,7 +2573,7 @@ TEST_F(LLMHttpParametersValidationTest, SpeculativeDecodingNoSDSpecificParameter )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } @@ -2585,7 +2587,7 @@ TEST_F(LLMHttpParametersValidationTest, MessagesWithOnlyContent) { )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); } @@ -2599,7 +2601,7 @@ TEST_F(LLMHttpParametersValidationTest, MessagesWithMoreMessageFields) { )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); } diff --git a/src/test/llmtemplate_test.cpp b/src/test/llmtemplate_test.cpp index 18cd68856e..3ec1f24b25 100644 --- a/src/test/llmtemplate_test.cpp +++ b/src/test/llmtemplate_test.cpp @@ -556,11 +556,13 @@ class LLMChatTemplateHttpTest : public TestWithTempDir { const std::string endpointChatCompletions = "/v3/chat/completions"; const std::string endpointCompletions = "/v3/completions"; std::shared_ptr writer; + std::shared_ptr multiPartParser; std::string response; ovms::HttpResponseComponents responseComponents; void SetUp() { writer = std::make_shared(); + multiPartParser = std::make_shared(); ON_CALL(*writer, PartialReplyBegin(::testing::_)).WillByDefault(testing::Invoke([](std::function fn) { fn(); })); TestWithTempDir::SetUp(); tokenizerConfigFilePath = directoryPath + "/tokenizer_config.json"; @@ -668,7 +670,7 @@ TEST_F(LLMJinjaChatTemplateHttpTest, inferChatCompletionsUnary) { )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); // Assertion split in two parts to avoid timestamp mismatch // const size_t timestampLength = 10; @@ -690,7 +692,7 @@ TEST_F(LLMJinjaChatTemplateHttpTest, inferCompletionsUnary) { )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); // Assertion split in two parts to avoid timestamp mismatch // const size_t timestampLength = 10; @@ -732,7 +734,7 @@ TEST_F(LLMJinjaChatTemplateHttpTest, inferChatCompletionsStream) { // TODO: New output EXPECT_CALL(writer, IsDisconnected()).Times(7); ASSERT_EQ( - handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::PARTIAL_END); ASSERT_EQ(response, ""); @@ -772,7 +774,7 @@ TEST_F(LLMJinjaChatTemplateHttpTest, inferCompletionsStream) { // TODO: New output EXPECT_CALL(writer, IsDisconnected()).Times(7); ASSERT_EQ( - handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::PARTIAL_END); ASSERT_EQ(response, ""); @@ -798,7 +800,7 @@ TEST_F(LLMJinjaChatTemplateHttpTest, inferDefaultChatCompletionsUnary) { )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); // Assertion split in two parts to avoid timestamp mismatch // const size_t timestampLength = 10; diff --git a/src/test/metrics_flow_test.cpp b/src/test/metrics_flow_test.cpp index 3c5bc90768..cb13956d6b 100644 --- a/src/test/metrics_flow_test.cpp +++ b/src/test/metrics_flow_test.cpp @@ -824,6 +824,7 @@ TEST_F(MetricFlowTest, ModelReady) { TEST_F(MetricFlowTest, RestV3Unary) { HttpRestApiHandler handler(server, 0); std::shared_ptr stream = std::make_shared(); + std::shared_ptr multiPartParser = std::make_shared(); EXPECT_CALL(*stream, IsDisconnected()) .WillRepeatedly(::testing::Return(false)); @@ -833,9 +834,9 @@ TEST_F(MetricFlowTest, RestV3Unary) { std::string response; HttpRequestComponents comps; auto streamPtr = std::static_pointer_cast(stream); - auto status = handler.processV3("/v3/completions", comps, response, request, streamPtr); + auto status = handler.processV3("/v3/completions", comps, response, request, streamPtr, multiPartParser); ASSERT_EQ(status, ovms::StatusCode::OK) << status.string(); - status = handler.processV3("/v3/v1/completions", comps, response, request, streamPtr); + status = handler.processV3("/v3/v1/completions", comps, response, request, streamPtr, multiPartParser); ASSERT_EQ(status, ovms::StatusCode::OK) << status.string(); } @@ -850,6 +851,7 @@ TEST_F(MetricFlowTest, RestV3Unary) { TEST_F(MetricFlowTest, RestV3UnaryError) { HttpRestApiHandler handler(server, 0); std::shared_ptr stream = std::make_shared(); + std::shared_ptr multiPartParser = std::make_shared(); auto streamPtr = std::static_pointer_cast(stream); EXPECT_CALL(*stream, IsDisconnected()) @@ -861,9 +863,9 @@ TEST_F(MetricFlowTest, RestV3UnaryError) { std::string request = R"({"model": "dummy_gpt", "prompt":"ReturnError"})"; std::string response; HttpRequestComponents comps; - auto status = handler.processV3("/v3/completions", comps, response, request, streamPtr); + auto status = handler.processV3("/v3/completions", comps, response, request, streamPtr, multiPartParser); ASSERT_EQ(status, ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR) << status.string(); - status = handler.processV3("/v3/v1/completions", comps, response, request, streamPtr); + status = handler.processV3("/v3/v1/completions", comps, response, request, streamPtr, multiPartParser); ASSERT_EQ(status, ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR) << status.string(); } @@ -875,6 +877,7 @@ TEST_F(MetricFlowTest, RestV3UnaryError) { TEST_F(MetricFlowTest, RestV3Stream) { HttpRestApiHandler handler(server, 0); std::shared_ptr stream = std::make_shared(); + std::shared_ptr multiPartParser = std::make_shared(); ON_CALL(*stream, PartialReplyBegin(::testing::_)).WillByDefault(testing::Invoke([](std::function fn) { fn(); })); // make the streaming flow sequential EXPECT_CALL(*stream, IsDisconnected()) @@ -885,9 +888,9 @@ TEST_F(MetricFlowTest, RestV3Stream) { std::string response; HttpRequestComponents comps; auto streamPtr = std::static_pointer_cast(stream); - auto status = handler.processV3("/v3/completions", comps, response, request, streamPtr); + auto status = handler.processV3("/v3/completions", comps, response, request, streamPtr, multiPartParser); ASSERT_EQ(status, ovms::StatusCode::PARTIAL_END) << status.string(); - status = handler.processV3("/v3/v1/completions", comps, response, request, streamPtr); + status = handler.processV3("/v3/v1/completions", comps, response, request, streamPtr, multiPartParser); ASSERT_EQ(status, ovms::StatusCode::PARTIAL_END) << status.string(); } @@ -906,6 +909,7 @@ TEST_F(MetricFlowTest, RestV3Stream) { TEST_F(MetricFlowTest, RestV3StreamError) { HttpRestApiHandler handler(server, 0); std::shared_ptr stream = std::make_shared(); + std::shared_ptr multiPartParser = std::make_shared(); auto streamPtr = std::static_pointer_cast(stream); ON_CALL(*stream, PartialReplyBegin(::testing::_)).WillByDefault(testing::Invoke([](std::function fn) { fn(); })); @@ -918,9 +922,9 @@ TEST_F(MetricFlowTest, RestV3StreamError) { std::string request = R"({"model": "dummy_gpt", "stream": true, "prompt": "ReturnError"})"; std::string response; HttpRequestComponents comps; - auto status = handler.processV3("/v3/completions", comps, response, request, streamPtr); + auto status = handler.processV3("/v3/completions", comps, response, request, streamPtr, multiPartParser); ASSERT_EQ(status, ovms::StatusCode::PARTIAL_END) << status.string(); - status = handler.processV3("/v3/v1/completions", comps, response, request, streamPtr); + status = handler.processV3("/v3/v1/completions", comps, response, request, streamPtr, multiPartParser); ASSERT_EQ(status, ovms::StatusCode::PARTIAL_END) << status.string(); } From b9292a005ac22f13b779d1edf699727412fa324b Mon Sep 17 00:00:00 2001 From: Damian Kalinowski Date: Wed, 16 Apr 2025 12:30:35 +0200 Subject: [PATCH 15/24] ut --- .../complete_flow_test.cpp | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/test/llm/visual_language_model/complete_flow_test.cpp b/src/test/llm/visual_language_model/complete_flow_test.cpp index 28f666023d..fc5fde62e5 100644 --- a/src/test/llm/visual_language_model/complete_flow_test.cpp +++ b/src/test/llm/visual_language_model/complete_flow_test.cpp @@ -46,10 +46,11 @@ class VLMServableExecutionTest : public ::testing::Test { public: std::unique_ptr handler; - std::vector> headers; + std::unordered_map headers; ovms::HttpRequestComponents comp; const std::string endpointChatCompletions = "/v3/chat/completions"; std::shared_ptr writer; + std::shared_ptr multiPartParser; std::string response; rapidjson::Document parsedResponse; ovms::HttpResponseComponents responseComponents; @@ -61,6 +62,7 @@ class VLMServableExecutionTest : public ::testing::Test { void SetUp() { writer = std::make_shared(); + multiPartParser = std::make_shared(); ON_CALL(*writer, PartialReplyBegin(::testing::_)).WillByDefault(testing::Invoke([](std::function fn) { fn(); })); // make the streaming flow sequential ovms::Server& server = ovms::Server::instance(); handler = std::make_unique(server, 5); @@ -141,7 +143,7 @@ TEST_P(VLMServableExecutionTestParameterized, unaryBasic) { std::string requestBody = createRequestBody(modelName, fields); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); parsedResponse.Parse(response.c_str()); ASSERT_TRUE(parsedResponse["choices"].IsArray()); @@ -179,7 +181,7 @@ TEST_P(VLMServableExecutionTestParameterized, unaryBasicOnlyImage) { std::string requestBody = createRequestBody(modelName, fields, false, 1); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); parsedResponse.Parse(response.c_str()); ASSERT_TRUE(parsedResponse["choices"].IsArray()); @@ -217,7 +219,7 @@ TEST_P(VLMServableExecutionTestParameterized, unaryMultipleImageTagOrderPasses) std::string requestBody = createRequestBody(modelName, fields, false, 3); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); parsedResponse.Parse(response.c_str()); ASSERT_TRUE(parsedResponse["choices"].IsArray()); @@ -254,7 +256,7 @@ TEST_P(VLMServableExecutionTestParameterized, UnaryRestrictedTagUsed) { std::string requestBody = createRequestBody(modelName, fields, true, 1, ""); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } @@ -277,7 +279,7 @@ TEST_P(VLMServableExecutionTestParameterized, streamBasic) { }); EXPECT_CALL(*writer, PartialReplyEnd()).Times(1); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::PARTIAL_END); if (modelName.find("legacy") == std::string::npos) { ASSERT_TRUE(responses.back().find("\"finish_reason\":\"length\"") != std::string::npos); @@ -302,7 +304,7 @@ TEST_P(VLMServableExecutionTestParameterized, streamBasicOnlyImage) { }); EXPECT_CALL(*writer, PartialReplyEnd()).Times(1); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::PARTIAL_END); if (modelName.find("legacy") == std::string::npos) { ASSERT_TRUE(responses.back().find("\"finish_reason\":\"length\"") != std::string::npos); @@ -327,7 +329,7 @@ TEST_P(VLMServableExecutionTestParameterized, streamMultipleImageTagOrderPasses) }); EXPECT_CALL(*writer, PartialReplyEnd()).Times(1); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::PARTIAL_END); if (modelName.find("legacy") == std::string::npos) { ASSERT_TRUE(responses.back().find("\"finish_reason\":\"length\"") != std::string::npos); @@ -355,7 +357,7 @@ TEST_P(VLMServableExecutionTestParameterized, streamRestrictedTagUsed) { }); EXPECT_CALL(*writer, PartialReplyEnd()).Times(1); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::PARTIAL_END); } From c11d88580015d10e4375e42908a9274805631ea2 Mon Sep 17 00:00:00 2001 From: Damian Kalinowski Date: Wed, 16 Apr 2025 13:07:23 +0200 Subject: [PATCH 16/24] compilable but failing unit tests --- src/test/disabled_mediapipe_test.cpp | 4 ++-- src/test/llm/llmnode_test.cpp | 24 ++++++++++++------------ 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/test/disabled_mediapipe_test.cpp b/src/test/disabled_mediapipe_test.cpp index 209be474ff..dcf273d1e6 100644 --- a/src/test/disabled_mediapipe_test.cpp +++ b/src/test/disabled_mediapipe_test.cpp @@ -83,7 +83,7 @@ TEST_F(MediapipeDisabledTest, completionsRequest) { )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::NOT_IMPLEMENTED); } @@ -106,6 +106,6 @@ TEST_F(MediapipeDisabledTest, chatCompletionsRequest) { )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::NOT_IMPLEMENTED); } diff --git a/src/test/llm/llmnode_test.cpp b/src/test/llm/llmnode_test.cpp index 8710a55668..8a0e50efe7 100644 --- a/src/test/llm/llmnode_test.cpp +++ b/src/test/llm/llmnode_test.cpp @@ -230,7 +230,7 @@ TEST_P(LLMFlowHttpTestParameterized, unaryCompletionsJson) { EXPECT_STREQ(parsedResponse["object"].GetString(), "text_completion"); } else { // Completions endpoint not supported for VLM servable ASSERT_EQ( - handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } } @@ -658,7 +658,7 @@ TEST_P(LLMFlowHttpTestParameterized, unaryCompletionsJsonSpaceStopString) { )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); parsedResponse.Parse(response.c_str()); ASSERT_TRUE(parsedResponse.HasMember("choices")); @@ -1369,7 +1369,7 @@ TEST_P(LLMFlowHttpTestParameterized, unaryChatCompletionsPromptTokensWithMaxToke )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } @@ -1401,7 +1401,7 @@ TEST_P(LLMFlowHttpTestParameterized, unaryChatCompletionsPromptTokensWithMaxComp )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } @@ -1432,7 +1432,7 @@ TEST_P(LLMFlowHttpTestParameterized, unaryChatCompletionsPromptTokensEqualToMaxM )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } @@ -1463,7 +1463,7 @@ TEST_P(LLMFlowHttpTestParameterized, unaryChatCompletionsStoppedByMaxModelLength )"; ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); // parsedResponse.Parse(response.c_str()); // ASSERT_TRUE(parsedResponse["usage"].IsObject()); @@ -1875,7 +1875,7 @@ TEST_P(LLMFlowHttpTestParameterized, streamCompletionsSpaceStopString) { }); EXPECT_CALL(*writer, PartialReplyEnd()).Times(1); ASSERT_EQ( - handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::PARTIAL_END); ASSERT_GE(responses.size(), 1); if (params.checkFinishReason) { @@ -2764,7 +2764,7 @@ TEST_P(LLMHttpParametersValidationTest, lengthPenaltyValid) { std::string requestBody = validRequestBodyWithParameter(params.modelName, "length_penalty", "2.0"); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); requestBody = validRequestBodyWithParameter(params.modelName, "length_penalty", "2"); @@ -2794,7 +2794,7 @@ TEST_P(LLMHttpParametersValidationTest, temperatureValid) { requestBody = validRequestBodyWithParameter(params.modelName, "temperature", "0"); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); requestBody = validRequestBodyWithParameter(params.modelName, "temperature", "2"); @@ -2818,7 +2818,7 @@ TEST_P(LLMHttpParametersValidationTest, temperatureOutOfRange) { std::string requestBody = validRequestBodyWithParameter(params.modelName, "temperature", "3.0"); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } @@ -2980,7 +2980,7 @@ TEST_P(LLMHttpParametersValidationTest, bestOfNegative) { std::string requestBody = validRequestBodyWithParameter(params.modelName, "best_of", "-1"); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } @@ -2989,7 +2989,7 @@ TEST_P(LLMHttpParametersValidationTest, bestOfExceedsLimit) { std::string requestBody = validRequestBodyWithParameter(params.modelName, "best_of", "40"); ASSERT_EQ( - handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer), + handler->dispatchToProcessor(endpointChatCompletions, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR); } From e91b214b5f365d35706f20750d5533d463a7c8b6 Mon Sep 17 00:00:00 2001 From: Damian Kalinowski Date: Wed, 16 Apr 2025 14:30:11 +0200 Subject: [PATCH 17/24] 3 fails remaining --- src/test/disabled_mediapipe_test.cpp | 2 +- src/test/embeddingsnode_test.cpp | 4 ++-- src/test/http_openai_handler_test.cpp | 2 +- src/test/llm/llmnode_test.cpp | 2 +- src/test/llm/llmtemplate_test.cpp | 2 +- src/test/llm/visual_language_model/complete_flow_test.cpp | 2 +- src/test/metrics_flow_test.cpp | 4 ++++ src/test/reranknode_test.cpp | 2 +- 8 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/test/disabled_mediapipe_test.cpp b/src/test/disabled_mediapipe_test.cpp index dcf273d1e6..9d51c1524b 100644 --- a/src/test/disabled_mediapipe_test.cpp +++ b/src/test/disabled_mediapipe_test.cpp @@ -33,7 +33,7 @@ class MediapipeDisabledTest : public ::testing::Test { public: std::unique_ptr handler; - std::unordered_map headers; + std::unordered_map headers{{"content-type", "application/json"}}; ovms::HttpRequestComponents comp; const std::string endpointChatCompletions = "/v3/chat/completions"; const std::string endpointCompletions = "/v3/completions"; diff --git a/src/test/embeddingsnode_test.cpp b/src/test/embeddingsnode_test.cpp index 3e83c2cc7f..de599c7200 100644 --- a/src/test/embeddingsnode_test.cpp +++ b/src/test/embeddingsnode_test.cpp @@ -30,7 +30,7 @@ class V3HttpTest : public ::testing::Test { public: std::unique_ptr handler; - std::unordered_map headers; + std::unordered_map headers{{"content-type", "application/json"}}; ovms::HttpRequestComponents comp; const std::string endpointEmbeddings = "/v3/embeddings"; const std::string endpointRerank = "/v3/rerank"; @@ -342,7 +342,7 @@ class EmbeddingsExtensionTest : public ::testing::Test { public: std::unique_ptr handler; - std::unordered_map headers; + std::unordered_map headers{{"content-type", "application/json"}}; ovms::HttpRequestComponents comp; const std::string endpointEmbeddings = "/v3/embeddings"; std::shared_ptr writer; diff --git a/src/test/http_openai_handler_test.cpp b/src/test/http_openai_handler_test.cpp index 7ca417321a..126689756d 100644 --- a/src/test/http_openai_handler_test.cpp +++ b/src/test/http_openai_handler_test.cpp @@ -37,7 +37,7 @@ class HttpOpenAIHandlerTest : public ::testing::Test { std::unique_ptr t; std::string port = "9173"; - std::unordered_map headers; + std::unordered_map headers{{"content-type", "application/json"}}; ovms::HttpRequestComponents comp; const std::string endpoint = "/v3/chat/completions"; std::shared_ptr writer; diff --git a/src/test/llm/llmnode_test.cpp b/src/test/llm/llmnode_test.cpp index 8a0e50efe7..f885142124 100644 --- a/src/test/llm/llmnode_test.cpp +++ b/src/test/llm/llmnode_test.cpp @@ -65,7 +65,7 @@ class LLMFlowHttpTest : public ::testing::Test { public: std::unique_ptr handler; - std::unordered_map headers; + std::unordered_map headers{{"content-type", "application/json"}}; ovms::HttpRequestComponents comp; const std::string endpointChatCompletions = "/v3/chat/completions"; const std::string endpointCompletions = "/v3/completions"; diff --git a/src/test/llm/llmtemplate_test.cpp b/src/test/llm/llmtemplate_test.cpp index 203084e304..111e765c6e 100644 --- a/src/test/llm/llmtemplate_test.cpp +++ b/src/test/llm/llmtemplate_test.cpp @@ -551,7 +551,7 @@ class LLMChatTemplateHttpTest : public TestWithTempDir { public: std::unique_ptr handler; - std::unordered_map headers; + std::unordered_map headers{{"content-type", "application/json"}}; ovms::HttpRequestComponents comp; const std::string endpointChatCompletions = "/v3/chat/completions"; const std::string endpointCompletions = "/v3/completions"; diff --git a/src/test/llm/visual_language_model/complete_flow_test.cpp b/src/test/llm/visual_language_model/complete_flow_test.cpp index fc5fde62e5..4afbd99f31 100644 --- a/src/test/llm/visual_language_model/complete_flow_test.cpp +++ b/src/test/llm/visual_language_model/complete_flow_test.cpp @@ -46,7 +46,7 @@ class VLMServableExecutionTest : public ::testing::Test { public: std::unique_ptr handler; - std::unordered_map headers; + std::unordered_map headers{{"content-type", "application/json"}}; ovms::HttpRequestComponents comp; const std::string endpointChatCompletions = "/v3/chat/completions"; std::shared_ptr writer; diff --git a/src/test/metrics_flow_test.cpp b/src/test/metrics_flow_test.cpp index 408fae0c75..884e62ba4a 100644 --- a/src/test/metrics_flow_test.cpp +++ b/src/test/metrics_flow_test.cpp @@ -832,6 +832,7 @@ TEST_F(MetricFlowTest, RestV3Unary) { std::string request = R"({"model": "dummy_gpt", "prompt": "Hello World"})"; std::string response; HttpRequestComponents comps; + comps.headers = {{"content-type", "application/json"}}; auto streamPtr = std::static_pointer_cast(stream); auto status = handler.processV3("/v3/completions", comps, response, request, streamPtr, multiPartParser); ASSERT_EQ(status, ovms::StatusCode::OK) << status.string(); @@ -862,6 +863,7 @@ TEST_F(MetricFlowTest, RestV3UnaryError) { std::string request = R"({"model": "dummy_gpt", "prompt":"ReturnError"})"; std::string response; HttpRequestComponents comps; + comps.headers = {{"content-type", "application/json"}}; auto status = handler.processV3("/v3/completions", comps, response, request, streamPtr, multiPartParser); ASSERT_EQ(status, ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR) << status.string(); status = handler.processV3("/v3/v1/completions", comps, response, request, streamPtr, multiPartParser); @@ -886,6 +888,7 @@ TEST_F(MetricFlowTest, RestV3Stream) { std::string request = R"({"model": "dummy_gpt", "stream": true, "prompt": "Hello World"})"; std::string response; HttpRequestComponents comps; + comps.headers = {{"content-type", "application/json"}}; auto streamPtr = std::static_pointer_cast(stream); auto status = handler.processV3("/v3/completions", comps, response, request, streamPtr, multiPartParser); ASSERT_EQ(status, ovms::StatusCode::PARTIAL_END) << status.string(); @@ -921,6 +924,7 @@ TEST_F(MetricFlowTest, RestV3StreamError) { std::string request = R"({"model": "dummy_gpt", "stream": true, "prompt": "ReturnError"})"; std::string response; HttpRequestComponents comps; + comps.headers = {{"content-type", "application/json"}}; auto status = handler.processV3("/v3/completions", comps, response, request, streamPtr, multiPartParser); ASSERT_EQ(status, ovms::StatusCode::PARTIAL_END) << status.string(); status = handler.processV3("/v3/v1/completions", comps, response, request, streamPtr, multiPartParser); diff --git a/src/test/reranknode_test.cpp b/src/test/reranknode_test.cpp index bcdffc20d5..c2e15128ef 100644 --- a/src/test/reranknode_test.cpp +++ b/src/test/reranknode_test.cpp @@ -33,7 +33,7 @@ class V3HttpTest : public ::testing::Test { public: std::unique_ptr handler; - std::unordered_map headers; + std::unordered_map headers{{"content-type", "application/json"}}; ovms::HttpRequestComponents comp; const std::string endpointEmbeddings = "/v3/embeddings"; const std::string endpointRerank = "/v3/rerank"; From 684b0eb69d21aaa55b13b823365e03cfe0d30ad7 Mon Sep 17 00:00:00 2001 From: Damian Kalinowski Date: Wed, 16 Apr 2025 15:32:27 +0200 Subject: [PATCH 18/24] all working --- src/http_frontend/multi_part_parser_drogon_impl.hpp | 6 +++--- src/http_rest_api_handler.cpp | 4 ++-- src/test/http_openai_handler_test.cpp | 4 ++-- .../openai_chat_completions_mock_calculator.cpp | 13 ++++++++++++- src/test/mediapipeflow_test.cpp | 1 + src/test/test_http_utils.hpp | 13 ++++++------- 6 files changed, 26 insertions(+), 15 deletions(-) diff --git a/src/http_frontend/multi_part_parser_drogon_impl.hpp b/src/http_frontend/multi_part_parser_drogon_impl.hpp index d1f6b3e1f6..33c5e26af0 100644 --- a/src/http_frontend/multi_part_parser_drogon_impl.hpp +++ b/src/http_frontend/multi_part_parser_drogon_impl.hpp @@ -34,8 +34,9 @@ class DrogonMultiPartParser : public MultiPartParser { const std::shared_ptr parser{nullptr}; public: - DrogonMultiPartParser(const drogon::HttpRequestPtr& request) - : request(request), parser(std::make_shared()) {} + DrogonMultiPartParser(const drogon::HttpRequestPtr& request) : + request(request), + parser(std::make_shared()) {} bool parse() override; @@ -43,7 +44,6 @@ class DrogonMultiPartParser : public MultiPartParser { std::string getFieldByName(const std::string& name) const override; std::string_view getFileContentByName(const std::string& name) const override; - }; } // namespace ovms diff --git a/src/http_rest_api_handler.cpp b/src/http_rest_api_handler.cpp index 9c3152e80d..41d1ce8851 100644 --- a/src/http_rest_api_handler.cpp +++ b/src/http_rest_api_handler.cpp @@ -480,7 +480,7 @@ Status HttpRestApiHandler::processV3(const std::string_view uri, const HttpReque if (isMultiPart) { if (!multiPartParser->parse()) { - return StatusCode::REST_INVALID_URL; // TODO: Better errror? + return StatusCode::REST_INVALID_URL; // TODO: Better error? } model_name = multiPartParser->getFieldByName("model"); if (model_name.empty()) { @@ -722,7 +722,7 @@ Status HttpRestApiHandler::parseRequestComponents(HttpRequestComponents& request const std::string& request_path, //const std::vector>& headers) { const std::unordered_map& headers) { - std::smatch sm; + std::smatch sm; requestComponents.http_method = http_method; if (http_method != "POST" && http_method != "GET") { return StatusCode::REST_UNSUPPORTED_METHOD; diff --git a/src/test/http_openai_handler_test.cpp b/src/test/http_openai_handler_test.cpp index 126689756d..76d1b1d3c4 100644 --- a/src/test/http_openai_handler_test.cpp +++ b/src/test/http_openai_handler_test.cpp @@ -80,7 +80,7 @@ TEST_F(HttpOpenAIHandlerTest, Unary) { ovms::StatusCode::OK); std::string expectedResponse = R"(/v3/v1/completions/ - +content-typeapplication/json { "model": "gpt", "stream": false, @@ -106,7 +106,7 @@ TEST_F(HttpOpenAIHandlerTest, UnaryWithHeaders) { ovms::StatusCode::OK); std::string expectedResponse = R"(/v3/completions/ -test1headertest2header +content-typeapplication/jsontest1headertest2header { "model": "gpt", "stream": false, diff --git a/src/test/mediapipe/calculators/openai_chat_completions_mock_calculator.cpp b/src/test/mediapipe/calculators/openai_chat_completions_mock_calculator.cpp index d62c5a4b42..a1fa7d9d10 100644 --- a/src/test/mediapipe/calculators/openai_chat_completions_mock_calculator.cpp +++ b/src/test/mediapipe/calculators/openai_chat_completions_mock_calculator.cpp @@ -75,7 +75,18 @@ class OpenAIChatCompletionsMockCalculator : public CalculatorBase { // - Request body // - timestamps 0-8 (appended in cycles) this->body = data.uri + std::string{"\n"}; - for (auto header : data.headers) { + + // Sort alphabetically to get determinism + std::vector> sorted_elements( + data.headers.begin(), data.headers.end()); + + // Sort the vector by key + std::sort(sorted_elements.begin(), sorted_elements.end(), + [](const auto& a, const auto& b) { + return a.first < b.first; + }); + + for (auto header : sorted_elements) { this->body += header.first; this->body += header.second; } diff --git a/src/test/mediapipeflow_test.cpp b/src/test/mediapipeflow_test.cpp index 8f67aecb60..be3f0e0a7c 100644 --- a/src/test/mediapipeflow_test.cpp +++ b/src/test/mediapipeflow_test.cpp @@ -3827,6 +3827,7 @@ TEST(WhitelistRegistered, MediapipeCalculatorsList) { "ModelInferHttpRequestCalculator", "ModelInferRequestImageCalculator", "MotionAnalysisCalculator", + "MultipartAcceptingCalculator", "MuxCalculator", "NegativeCalculator", "NoOutputStreamsProducedCalculator", diff --git a/src/test/test_http_utils.hpp b/src/test/test_http_utils.hpp index ac45a6c33e..aace1f2c11 100644 --- a/src/test/test_http_utils.hpp +++ b/src/test/test_http_utils.hpp @@ -39,10 +39,9 @@ class MockedServerRequestInterface final : public ovms::HttpAsyncWriter { }; class MockedMultiPartParser final : public ovms::MultiPartParser { - public: - MOCK_METHOD(bool, parse, (), (override)); - MOCK_METHOD(bool, hasParseError, (), (const override)); - MOCK_METHOD(std::string, getFieldByName, (const std::string&), (const override)); - MOCK_METHOD(std::string_view, getFileContentByName, (const std::string&), (const override)); - }; - +public: + MOCK_METHOD(bool, parse, (), (override)); + MOCK_METHOD(bool, hasParseError, (), (const override)); + MOCK_METHOD(std::string, getFieldByName, (const std::string&), (const override)); + MOCK_METHOD(std::string_view, getFileContentByName, (const std::string&), (const override)); +}; From 4980010f8d00c22c55a824273cc172d333954a34 Mon Sep 17 00:00:00 2001 From: Damian Kalinowski Date: Thu, 17 Apr 2025 16:34:23 +0200 Subject: [PATCH 19/24] add test --- .gitignore | 1 + src/BUILD | 3 + .../multipart_accepting_calculator.cpp | 2 +- .../config_mediapipe_multipart_mock.json | 9 +++ src/test/mediapipe/graph_multipart.pbtxt | 23 ++++++ src/test/multipart_calculator_test.cpp | 79 +++++++++++++++++++ 6 files changed, 116 insertions(+), 1 deletion(-) create mode 100644 src/test/mediapipe/config_mediapipe_multipart_mock.json create mode 100644 src/test/mediapipe/graph_multipart.pbtxt create mode 100644 src/test/multipart_calculator_test.cpp diff --git a/.gitignore b/.gitignore index 7f17193c13..5ec33f756f 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,4 @@ out *.errors.txt tmp/ *.zip +*.tar.gz \ No newline at end of file diff --git a/src/BUILD b/src/BUILD index 1058a1d429..bbfb0f282f 100644 --- a/src/BUILD +++ b/src/BUILD @@ -2791,6 +2791,7 @@ cc_test( "test/get_mediapipe_graph_metadata_response_test.cpp", "test/mediapipe_framework_test.cpp", "test/http_openai_handler_test.cpp", + "test/multipart_calculator_test.cpp", ], "//:disable_mediapipe" : [ "test/disabled_mediapipe_test.cpp", @@ -2858,6 +2859,7 @@ cc_test( "test/mediapipe/config_mediapipe_graph_with_side_packets.json", "test/mediapipe/config_mediapipe_two_inputs.json", "test/mediapipe/config_mediapipe_two_outputs_dag.json", + "test/mediapipe/config_mediapipe_multipart_mock.json", "test/mediapipe/config_mp_tf_passthrough.json", "test/mediapipe/config_standard_add.json", "test/mediapipe/config_standard_dummy.json", @@ -2876,6 +2878,7 @@ cc_test( "test/mediapipe/graphdummyadapterfull_dummyinputnames.pbtxt", "test/mediapipe/graphadapterfull_two_outputs_dag.pbtxt", "test/mediapipe/graphdummyadapterfull_two_outputs.pbtxt", + "test/mediapipe/graph_multipart.pbtxt", "test/mediapipe/negative/config_exception_during_process.json", "test/mediapipe/negative/config_no_calc_output_stream.json", "test/mediapipe/negative/graph_exception_during_process.pbtxt", diff --git a/src/test/mediapipe/calculators/multipart_accepting_calculator.cpp b/src/test/mediapipe/calculators/multipart_accepting_calculator.cpp index 516594f4f4..854eb71442 100644 --- a/src/test/mediapipe/calculators/multipart_accepting_calculator.cpp +++ b/src/test/mediapipe/calculators/multipart_accepting_calculator.cpp @@ -46,7 +46,7 @@ class MultipartAcceptingCalculator : public CalculatorBase { } absl::Status Process(CalculatorContext* cc) final { - + cc->Outputs().Tag(OUTPUT_TAG_NAME).Add(new std::string{"Out!"}, cc->InputTimestamp()); return absl::OkStatus(); } }; diff --git a/src/test/mediapipe/config_mediapipe_multipart_mock.json b/src/test/mediapipe/config_mediapipe_multipart_mock.json new file mode 100644 index 0000000000..e070f5882a --- /dev/null +++ b/src/test/mediapipe/config_mediapipe_multipart_mock.json @@ -0,0 +1,9 @@ +{ + "model_config_list": [], + "mediapipe_config_list": [ + { + "name": "multipart", + "graph_path": "/ovms/src/test/mediapipe/graph_multipart.pbtxt" + } + ] +} \ No newline at end of file diff --git a/src/test/mediapipe/graph_multipart.pbtxt b/src/test/mediapipe/graph_multipart.pbtxt new file mode 100644 index 0000000000..5051c04c4a --- /dev/null +++ b/src/test/mediapipe/graph_multipart.pbtxt @@ -0,0 +1,23 @@ +# +# Copyright 2025 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +input_stream: "HTTP_REQUEST_PAYLOAD:input" +output_stream: "HTTP_RESPONSE_PAYLOAD:output" + +node: { + calculator: "MultipartAcceptingCalculator" + input_stream: "HTTP_REQUEST_PAYLOAD:input" + output_stream: "HTTP_RESPONSE_PAYLOAD:output" +} diff --git a/src/test/multipart_calculator_test.cpp b/src/test/multipart_calculator_test.cpp new file mode 100644 index 0000000000..3de9d78966 --- /dev/null +++ b/src/test/multipart_calculator_test.cpp @@ -0,0 +1,79 @@ +//***************************************************************************** +// Copyright 2025 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include +#include + +#include "../http_rest_api_handler.hpp" +#include "../http_payload.hpp" +#include "../module_names.hpp" +#include "../servablemanagermodule.hpp" +#include "../server.hpp" +#include "test_http_utils.hpp" +#include "test_utils.hpp" + +class MultiPartCalculatorTest : public ::testing::Test { +protected: + ovms::Server& server = ovms::Server::instance(); + std::unique_ptr handler; + + std::unique_ptr t; + std::string port = "9173"; + + std::unordered_map headers{{"content-type", "application/json"}}; + ovms::HttpRequestComponents comp; + const std::string endpoint = "/v3/chat/completions"; + std::shared_ptr writer; + std::shared_ptr multiPartParser; + std::string response; + ovms::HttpResponseComponents responseComponents; + + void SetUpServer(const char* configPath) { + ::SetUpServer(this->t, this->server, this->port, configPath); + EnsureServerStartedWithTimeout(this->server, 5); + handler = std::make_unique(server, 5); + } + + void SetUp() { + writer = std::make_shared(); + multiPartParser = std::make_shared(); + SetUpServer(getGenericFullPathForSrcTest("/ovms/src/test/mediapipe/config_mediapipe_multipart_mock.json").c_str()); + ASSERT_EQ(handler->parseRequestComponents(comp, "POST", endpoint, headers), ovms::StatusCode::OK); + } + + void TearDown() { + handler.reset(); + server.setShutdownRequest(1); + t->join(); + server.setShutdownRequest(0); + } +}; + +TEST_F(MultiPartCalculatorTest, Unary) { + std::string requestBody = R"( + { + "model": "multipart", + "stream": false, + "messages": [] + } + )"; + + ASSERT_EQ( + handler->dispatchToProcessor("/v3/v1/completions/", requestBody, &response, comp, responseComponents, writer, multiPartParser), + ovms::StatusCode::OK); + + std::string expectedResponse = R"(Out!)"; + ASSERT_EQ(response, expectedResponse); +} From e17afe4128ff06c32ec5dc2941bfd77a1ff51b3d Mon Sep 17 00:00:00 2001 From: Damian Kalinowski Date: Fri, 18 Apr 2025 14:34:16 +0200 Subject: [PATCH 20/24] push --- src/test/multipart_calculator_test.cpp | 29 ++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/src/test/multipart_calculator_test.cpp b/src/test/multipart_calculator_test.cpp index 3de9d78966..7eaf0831f0 100644 --- a/src/test/multipart_calculator_test.cpp +++ b/src/test/multipart_calculator_test.cpp @@ -62,13 +62,30 @@ class MultiPartCalculatorTest : public ::testing::Test { }; TEST_F(MultiPartCalculatorTest, Unary) { + headers["content-type"] = "multipart/form-data; boundary=----WebKitFormBoundary7MA4YWxkTrZu0gW"; + + comp = ovms::HttpRequestComponents(); + ASSERT_EQ(handler->parseRequestComponents(comp, "POST", endpoint, headers), ovms::StatusCode::OK); + std::string requestBody = R"( - { - "model": "multipart", - "stream": false, - "messages": [] - } - )"; +------WebKitFormBoundary7MA4YWxkTrZu0gW +Content-Disposition: form-data; name="username" + +john_doe +------WebKitFormBoundary7MA4YWxkTrZu0gW +Content-Disposition: form-data; name="email" + +john@example.com +------WebKitFormBoundary7MA4YWxkTrZu0gW +Content-Disposition: form-data; name="model" + +multipart +------WebKitFormBoundary7MA4YWxkTrZu0gW--)"; + + EXPECT_CALL(*multiPartParser, parse()).WillOnce(::testing::Return(true)); + EXPECT_CALL(*multiPartParser, getFieldByName(::testing::Eq("model"))).WillOnce([](const std::string& name) { + return "multipart"; + }); ASSERT_EQ( handler->dispatchToProcessor("/v3/v1/completions/", requestBody, &response, comp, responseComponents, writer, multiPartParser), From 156bcc7a863aa5b726856d4df82e64827e906a13 Mon Sep 17 00:00:00 2001 From: Damian Kalinowski Date: Fri, 18 Apr 2025 16:28:46 +0200 Subject: [PATCH 21/24] save --- .../multipart_accepting_calculator.cpp | 8 +++++++- src/test/multipart_calculator_test.cpp | 18 +++++++++++++++--- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/src/test/mediapipe/calculators/multipart_accepting_calculator.cpp b/src/test/mediapipe/calculators/multipart_accepting_calculator.cpp index 854eb71442..2fc3b983f7 100644 --- a/src/test/mediapipe/calculators/multipart_accepting_calculator.cpp +++ b/src/test/mediapipe/calculators/multipart_accepting_calculator.cpp @@ -46,7 +46,13 @@ class MultipartAcceptingCalculator : public CalculatorBase { } absl::Status Process(CalculatorContext* cc) final { - cc->Outputs().Tag(OUTPUT_TAG_NAME).Add(new std::string{"Out!"}, cc->InputTimestamp()); + auto payload = cc->Inputs().Tag(INPUT_TAG_NAME).Get(); + RET_CHECK(payload.multipartParser != nullptr); + std::string email = payload.multipartParser->getFieldByName("email"); + std::string username = payload.multipartParser->getFieldByName("username"); + std::string_view fileContent = payload.multipartParser->getFileContentByName("file"); + + cc->Outputs().Tag(OUTPUT_TAG_NAME).Add(new std::string{email + std::string{"+"} + username + std::string{"\n"} + std::string(fileContent)}, cc->InputTimestamp()); return absl::OkStatus(); } }; diff --git a/src/test/multipart_calculator_test.cpp b/src/test/multipart_calculator_test.cpp index 7eaf0831f0..54af589a0a 100644 --- a/src/test/multipart_calculator_test.cpp +++ b/src/test/multipart_calculator_test.cpp @@ -80,17 +80,29 @@ john@example.com Content-Disposition: form-data; name="model" multipart +------WebKitFormBoundary7MA4YWxkTrZu0gW +Content-Disposition: form-data; name="doc"; filename="notes.txt" +Content-Type: text/plain + +this is file content +It has two lines. ------WebKitFormBoundary7MA4YWxkTrZu0gW--)"; EXPECT_CALL(*multiPartParser, parse()).WillOnce(::testing::Return(true)); - EXPECT_CALL(*multiPartParser, getFieldByName(::testing::Eq("model"))).WillOnce([](const std::string& name) { - return "multipart"; + EXPECT_CALL(*multiPartParser, getFieldByName(::testing::Eq("model"))).WillOnce(::testing::Return("multipart")); + EXPECT_CALL(*multiPartParser, getFieldByName(::testing::Eq("email"))).WillOnce(::testing::Return("john@example.com")); + EXPECT_CALL(*multiPartParser, getFieldByName(::testing::Eq("username"))).WillOnce(::testing::Return("john_doe")); + EXPECT_CALL(*multiPartParser, getFileContentByName(::testing::Eq("file"))).WillOnce([] (const std::string& name) { + static std::string retval{"this is file content\nIt has two lines."}; + return std::string_view(retval); }); ASSERT_EQ( handler->dispatchToProcessor("/v3/v1/completions/", requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::OK); - std::string expectedResponse = R"(Out!)"; + std::string expectedResponse = R"(john@example.com+john_doe +this is file content +It has two lines.)"; ASSERT_EQ(response, expectedResponse); } From 96eeb09a3103237709cfaf39352e68ea8071f482 Mon Sep 17 00:00:00 2001 From: Damian Kalinowski Date: Fri, 18 Apr 2025 16:40:37 +0200 Subject: [PATCH 22/24] save --- .../multi_part_parser_drogon_impl.cpp | 1 + src/http_rest_api_handler.cpp | 30 ++++++++----------- 2 files changed, 13 insertions(+), 18 deletions(-) diff --git a/src/http_frontend/multi_part_parser_drogon_impl.cpp b/src/http_frontend/multi_part_parser_drogon_impl.cpp index 452106a989..cab6c2177a 100644 --- a/src/http_frontend/multi_part_parser_drogon_impl.cpp +++ b/src/http_frontend/multi_part_parser_drogon_impl.cpp @@ -17,6 +17,7 @@ namespace ovms { +// TODO (dkalinow): Test actual drogon parser with mocked request bool DrogonMultiPartParser::parse() { this->hasParseError_ = this->parser->parse(request) != 0; return !this->hasParseError_; diff --git a/src/http_rest_api_handler.cpp b/src/http_rest_api_handler.cpp index 41d1ce8851..2c9112db06 100644 --- a/src/http_rest_api_handler.cpp +++ b/src/http_rest_api_handler.cpp @@ -460,17 +460,7 @@ Status HttpRestApiHandler::processV3(const std::string_view uri, const HttpReque std::shared_ptr doc = std::make_shared(); std::shared_ptr executor; bool streamFieldVal = false; - // { - // OVMS_PROFILE_SCOPE("rapidjson parse body"); - // doc->Parse(request_body.c_str()); - // } { - SPDLOG_INFO("Headers in processV3:"); - - for (const auto& [key, value] : request_components.headers) { - SPDLOG_INFO("\t[{}]->[{}]", key, value); - } - auto it = request_components.headers.find("content-type"); bool isApplicationJson = it != request_components.headers.end() && it->second.find("application/json") != std::string::npos; bool isMultiPart = it != request_components.headers.end() && it->second.find("multipart/form-data") != std::string::npos; @@ -478,19 +468,23 @@ Status HttpRestApiHandler::processV3(const std::string_view uri, const HttpReque std::string model_name; + // TODO (dkalinow): Test new routing if (isMultiPart) { + OVMS_PROFILE_SCOPE("multipart parse"); if (!multiPartParser->parse()) { - return StatusCode::REST_INVALID_URL; // TODO: Better error? + return StatusCode::REST_INVALID_URL; } model_name = multiPartParser->getFieldByName("model"); if (model_name.empty()) { isDefault = true; } else { - SPDLOG_INFO("Model name from Multipart Field: {}", model_name); + SPDLOG_DEBUG("Model name from deduced from MultiPart field: {}", model_name); } } else if (isApplicationJson) { - doc->Parse(request_body.c_str()); - + { + OVMS_PROFILE_SCOPE("rapidjson parse"); + doc->Parse(request_body.c_str()); + } OVMS_PROFILE_SCOPE("rapidjson validate"); if (doc->HasParseError()) { return Status(StatusCode::JSON_INVALID, "Cannot parse JSON body"); @@ -524,24 +518,24 @@ Status HttpRestApiHandler::processV3(const std::string_view uri, const HttpReque if (model_name.empty()) { isDefault = true; } else { - SPDLOG_INFO("Model name from Application Json: {}", model_name); + SPDLOG_DEBUG("Model name from deduced from JSON: {}", model_name); } } - // Deduce Graph Name from URI + // Deduce Graph Name from URI since there is no info in JSON or MultiPart if (isDefault) { if (uri.size() <= 4) { // nothing after "/v3/..." return StatusCode::REST_INVALID_URL; } model_name = std::string(uri.substr(4)); - SPDLOG_INFO("Model name from URI: {}", model_name); + SPDLOG_DEBUG("Model name from deduced from URI: {}", model_name); } auto status = this->modelManager.createPipeline(executor, model_name); if (!status.ok()) { return status; } - // TODO: Possibly avoid making copy + request.headers = request_components.headers; request.body = request_body; request.parsedJson = std::move(doc); From b46350466558a8cb7e1d97e5b9fe58c958d931a8 Mon Sep 17 00:00:00 2001 From: Damian Kalinowski Date: Fri, 18 Apr 2025 16:54:45 +0200 Subject: [PATCH 23/24] save --- src/test/multipart_calculator_test.cpp | 121 ++++++++++++++++++++++++- 1 file changed, 120 insertions(+), 1 deletion(-) diff --git a/src/test/multipart_calculator_test.cpp b/src/test/multipart_calculator_test.cpp index 54af589a0a..0d1ccba6af 100644 --- a/src/test/multipart_calculator_test.cpp +++ b/src/test/multipart_calculator_test.cpp @@ -61,7 +61,7 @@ class MultiPartCalculatorTest : public ::testing::Test { } }; -TEST_F(MultiPartCalculatorTest, Unary) { +TEST_F(MultiPartCalculatorTest, UnaryWithModelField) { // only unary, there is no way to stream headers["content-type"] = "multipart/form-data; boundary=----WebKitFormBoundary7MA4YWxkTrZu0gW"; comp = ovms::HttpRequestComponents(); @@ -106,3 +106,122 @@ this is file content It has two lines.)"; ASSERT_EQ(response, expectedResponse); } + +TEST_F(MultiPartCalculatorTest, UnaryWithMissingModelFieldDefaultRouting) { + headers["content-type"] = "multipart/form-data; boundary=----WebKitFormBoundary7MA4YWxkTrZu0gW"; + + comp = ovms::HttpRequestComponents(); + ASSERT_EQ(handler->parseRequestComponents(comp, "POST", endpoint, headers), ovms::StatusCode::OK); + + std::string requestBody = R"( +------WebKitFormBoundary7MA4YWxkTrZu0gW +Content-Disposition: form-data; name="username" + +john_doe +------WebKitFormBoundary7MA4YWxkTrZu0gW +Content-Disposition: form-data; name="email" + +john@example.com +------WebKitFormBoundary7MA4YWxkTrZu0gW +Content-Disposition: form-data; name="doc"; filename="notes.txt" +Content-Type: text/plain + +this is file content +It has two lines. +------WebKitFormBoundary7MA4YWxkTrZu0gW--)"; + + EXPECT_CALL(*multiPartParser, parse()).WillOnce(::testing::Return(true)); + EXPECT_CALL(*multiPartParser, getFieldByName(::testing::Eq("model"))).WillOnce(::testing::Return("")); + EXPECT_CALL(*multiPartParser, getFieldByName(::testing::Eq("email"))).WillOnce(::testing::Return("john@example.com")); + EXPECT_CALL(*multiPartParser, getFieldByName(::testing::Eq("username"))).WillOnce(::testing::Return("john_doe")); + EXPECT_CALL(*multiPartParser, getFileContentByName(::testing::Eq("file"))).WillOnce([] (const std::string& name) { + static std::string retval{"this is file content\nIt has two lines."}; + return std::string_view(retval); + }); + + // Default routing uses everything that comes after /v3/ as graph name + const std::string URI = "/v3/multipart"; + + ASSERT_EQ( + handler->dispatchToProcessor(URI, requestBody, &response, comp, responseComponents, writer, multiPartParser), + ovms::StatusCode::OK); + + std::string expectedResponse = R"(john@example.com+john_doe +this is file content +It has two lines.)"; + ASSERT_EQ(response, expectedResponse); +} + +TEST_F(MultiPartCalculatorTest, UnaryWithMissingModelFieldDefaultRoutingWrongGraphName) { + headers["content-type"] = "multipart/form-data; boundary=----WebKitFormBoundary7MA4YWxkTrZu0gW"; + + comp = ovms::HttpRequestComponents(); + ASSERT_EQ(handler->parseRequestComponents(comp, "POST", endpoint, headers), ovms::StatusCode::OK); + + std::string requestBody = R"( +------WebKitFormBoundary7MA4YWxkTrZu0gW +Content-Disposition: form-data; name="username" + +john_doe +------WebKitFormBoundary7MA4YWxkTrZu0gW +Content-Disposition: form-data; name="email" + +john@example.com +------WebKitFormBoundary7MA4YWxkTrZu0gW +Content-Disposition: form-data; name="doc"; filename="notes.txt" +Content-Type: text/plain + +this is file content +It has two lines. +------WebKitFormBoundary7MA4YWxkTrZu0gW--)"; + + EXPECT_CALL(*multiPartParser, parse()).WillOnce(::testing::Return(true)); + EXPECT_CALL(*multiPartParser, getFieldByName(::testing::_)).Times(0); + EXPECT_CALL(*multiPartParser, getFieldByName(::testing::Eq("model"))).WillOnce(::testing::Return("")); + EXPECT_CALL(*multiPartParser, getFileContentByName(::testing::_)).Times(0); + + // Default routing uses everything that comes after /v3/ as graph name + const std::string URI = "/v3/NON_EXISTENT"; + + ASSERT_EQ( + handler->dispatchToProcessor(URI, requestBody, &response, comp, responseComponents, writer, multiPartParser), + ovms::StatusCode::MEDIAPIPE_DEFINITION_NAME_MISSING); +} + +TEST_F(MultiPartCalculatorTest, UnaryWithMissingModelFieldDefaultRoutingMissingGraphNameInURI) { + headers["content-type"] = "multipart/form-data; boundary=----WebKitFormBoundary7MA4YWxkTrZu0gW"; + + comp = ovms::HttpRequestComponents(); + ASSERT_EQ(handler->parseRequestComponents(comp, "POST", endpoint, headers), ovms::StatusCode::OK); + + std::string requestBody = R"( +------WebKitFormBoundary7MA4YWxkTrZu0gW +Content-Disposition: form-data; name="username" + +john_doe +------WebKitFormBoundary7MA4YWxkTrZu0gW +Content-Disposition: form-data; name="email" + +john@example.com +------WebKitFormBoundary7MA4YWxkTrZu0gW +Content-Disposition: form-data; name="doc"; filename="notes.txt" +Content-Type: text/plain + +this is file content +It has two lines. +------WebKitFormBoundary7MA4YWxkTrZu0gW--)"; + + EXPECT_CALL(*multiPartParser, parse()).WillOnce(::testing::Return(true)); + EXPECT_CALL(*multiPartParser, getFieldByName(::testing::_)).Times(0); + EXPECT_CALL(*multiPartParser, getFieldByName(::testing::Eq("model"))).WillOnce(::testing::Return("")); + EXPECT_CALL(*multiPartParser, getFileContentByName(::testing::_)).Times(0); + + // Default routing uses everything that comes after /v3/ as graph name + const std::string URI = "/v3/"; + + ASSERT_EQ( + handler->dispatchToProcessor(URI, requestBody, &response, comp, responseComponents, writer, multiPartParser), + ovms::StatusCode::REST_INVALID_URL); +} + + From bee88e90d3183bfda8a79a8fc708b42c9c0d0b10 Mon Sep 17 00:00:00 2001 From: Damian Kalinowski Date: Fri, 18 Apr 2025 16:56:20 +0200 Subject: [PATCH 24/24] save --- src/http_payload.hpp | 1 - src/http_rest_api_handler.cpp | 5 +---- src/http_rest_api_handler.hpp | 4 +--- src/http_server.cpp | 3 --- src/test/multipart_calculator_test.cpp | 6 ++---- 5 files changed, 4 insertions(+), 15 deletions(-) diff --git a/src/http_payload.hpp b/src/http_payload.hpp index 84038e0ec3..b4415a1616 100644 --- a/src/http_payload.hpp +++ b/src/http_payload.hpp @@ -32,7 +32,6 @@ namespace ovms { struct HttpPayload { std::string uri; - //std::vector> headers; std::unordered_map headers; std::string body; // always std::shared_ptr parsedJson; // pre-parsed body = null diff --git a/src/http_rest_api_handler.cpp b/src/http_rest_api_handler.cpp index 2c9112db06..0f587488ff 100644 --- a/src/http_rest_api_handler.cpp +++ b/src/http_rest_api_handler.cpp @@ -698,7 +698,6 @@ Status HttpRestApiHandler::processModelMetadataKFSRequest(const HttpRequestCompo } static Status parseInferenceHeaderContentLength(HttpRequestComponents& requestComponents, - //const std::vector>& headers) { const std::unordered_map& headers) { for (auto& header : headers) { if (toLower(header.first) == "inference-header-content-length") { // drogon automatically converts all headers to lowercase, net_http does not @@ -714,7 +713,6 @@ static Status parseInferenceHeaderContentLength(HttpRequestComponents& requestCo Status HttpRestApiHandler::parseRequestComponents(HttpRequestComponents& requestComponents, const std::string_view http_method, const std::string& request_path, - //const std::vector>& headers) { const std::unordered_map& headers) { std::smatch sm; requestComponents.http_method = http_method; @@ -879,8 +877,7 @@ Status HttpRestApiHandler::processRequest( headers->clear(); response->clear(); - //headers->push_back({"Content-Type", "application/json"}); - (*headers)["content-type"] = "application/json"; // ? + (*headers)["content-type"] = "application/json"; if (!status.ok()) return status; diff --git a/src/http_rest_api_handler.hpp b/src/http_rest_api_handler.hpp index 4cd94693b1..8535e1ee86 100644 --- a/src/http_rest_api_handler.hpp +++ b/src/http_rest_api_handler.hpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -67,7 +68,6 @@ struct HttpRequestComponents { std::string processing_method; std::string model_subresource; std::optional inferenceHeaderContentLength; - //std::vector> headers; std::unordered_map headers; }; @@ -114,7 +114,6 @@ class HttpRestApiHandler { Status parseRequestComponents(HttpRequestComponents& components, const std::string_view http_method, const std::string& request_path, - //const std::vector>& headers = {}); const std::unordered_map& headers = {}); Status parseModelVersion(std::string& model_version_str, std::optional& model_version); @@ -147,7 +146,6 @@ class HttpRestApiHandler { const std::string_view http_method, const std::string_view request_path, const std::string& request_body, - //std::vector>* headers, std::unordered_map* headers, std::string* response, HttpResponseComponents& responseComponents, diff --git a/src/http_server.cpp b/src/http_server.cpp index 066452363c..c567a8c592 100644 --- a/src/http_server.cpp +++ b/src/http_server.cpp @@ -192,7 +192,6 @@ std::unique_ptr createAndStartDrogonHttpServer(const std::stri server->registerRequestDispatcher([handler, &pool](const drogon::HttpRequestPtr& req, std::function drogonResponseInitializeCallback) { SPDLOG_DEBUG("REST request {}", req->getOriginalPath()); - //std::vector> headers; std::unordered_map headers; SPDLOG_ERROR("Drogon headers:"); @@ -239,8 +238,6 @@ std::unique_ptr createAndStartDrogonHttpServer(const std::stri resp->setContentTypeCode(drogon::CT_APPLICATION_JSON); if (responseComponents.inferenceHeaderContentLength.has_value()) { - //std::pair header{"Inference-Header-Content-Length", std::to_string(responseComponents.inferenceHeaderContentLength.value())}; - //headers.emplace_back(header); headers["inference-header-content-length"] = std::to_string(responseComponents.inferenceHeaderContentLength.value()); } for (const auto& [key, value] : headers) { diff --git a/src/test/multipart_calculator_test.cpp b/src/test/multipart_calculator_test.cpp index 0d1ccba6af..9f4430bd5c 100644 --- a/src/test/multipart_calculator_test.cpp +++ b/src/test/multipart_calculator_test.cpp @@ -92,7 +92,7 @@ It has two lines. EXPECT_CALL(*multiPartParser, getFieldByName(::testing::Eq("model"))).WillOnce(::testing::Return("multipart")); EXPECT_CALL(*multiPartParser, getFieldByName(::testing::Eq("email"))).WillOnce(::testing::Return("john@example.com")); EXPECT_CALL(*multiPartParser, getFieldByName(::testing::Eq("username"))).WillOnce(::testing::Return("john_doe")); - EXPECT_CALL(*multiPartParser, getFileContentByName(::testing::Eq("file"))).WillOnce([] (const std::string& name) { + EXPECT_CALL(*multiPartParser, getFileContentByName(::testing::Eq("file"))).WillOnce([](const std::string& name) { static std::string retval{"this is file content\nIt has two lines."}; return std::string_view(retval); }); @@ -134,7 +134,7 @@ It has two lines. EXPECT_CALL(*multiPartParser, getFieldByName(::testing::Eq("model"))).WillOnce(::testing::Return("")); EXPECT_CALL(*multiPartParser, getFieldByName(::testing::Eq("email"))).WillOnce(::testing::Return("john@example.com")); EXPECT_CALL(*multiPartParser, getFieldByName(::testing::Eq("username"))).WillOnce(::testing::Return("john_doe")); - EXPECT_CALL(*multiPartParser, getFileContentByName(::testing::Eq("file"))).WillOnce([] (const std::string& name) { + EXPECT_CALL(*multiPartParser, getFileContentByName(::testing::Eq("file"))).WillOnce([](const std::string& name) { static std::string retval{"this is file content\nIt has two lines."}; return std::string_view(retval); }); @@ -223,5 +223,3 @@ It has two lines. handler->dispatchToProcessor(URI, requestBody, &response, comp, responseComponents, writer, multiPartParser), ovms::StatusCode::REST_INVALID_URL); } - -