From 62db2ab059cb4de450e4aac7411e6cb3f4ac1d68 Mon Sep 17 00:00:00 2001 From: Anai-Guo Date: Sun, 12 Jul 2026 12:00:00 +0000 Subject: [PATCH 1/3] fix(rag): serve streaming chat completions as text/event-stream (#2837) rag_framework's _stream_completion returned the SSE stream with media_type="text/plain" while already emitting correct SSE framing (data: {...} chunks terminated by data: [DONE]). Strict OpenAI clients (e.g. Open WebUI) require content-type: text/event-stream and reject the stream otherwise, appearing to hang and then failing with a decode error even though the model server is generating tokens normally. Set the media type to text/event-stream to match the OpenAI streaming contract. Signed-off-by: Anai-Guo --- container-images/scripts/rag_framework | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) mode change 100755 => 100644 container-images/scripts/rag_framework diff --git a/container-images/scripts/rag_framework b/container-images/scripts/rag_framework old mode 100755 new mode 100644 index 16014004b..7aa5ad215 --- a/container-images/scripts/rag_framework +++ b/container-images/scripts/rag_framework @@ -338,7 +338,7 @@ class RagService: return StreamingResponse( generate(), - media_type="text/plain", + media_type="text/event-stream", headers={"Cache-Control": "no-cache", "Connection": "keep-alive"}, ) From cbcbd178344ddc268434ae7e48d1614b6c1bd07d Mon Sep 17 00:00:00 2001 From: Anai-Guo Date: Wed, 15 Jul 2026 03:23:40 -0700 Subject: [PATCH 2/3] test(rag): assert streaming completions advertise text/event-stream Adds a regression test for #2837 that loads the standalone rag_framework container script and checks _stream_completion sets media_type to text/event-stream (not text/plain). Container-only deps (fastapi, pydantic, openai, qdrant_client, uvicorn, aiohttp) are stubbed when absent so the test runs in the unit-test environment. Also restores the executable bit on the rag_framework script. Signed-off-by: Anai-Guo --- container-images/scripts/rag_framework | 0 test/unit/test_rag_stream_content_type.py | 118 ++++++++++++++++++++++ 2 files changed, 118 insertions(+) mode change 100644 => 100755 container-images/scripts/rag_framework create mode 100644 test/unit/test_rag_stream_content_type.py diff --git a/container-images/scripts/rag_framework b/container-images/scripts/rag_framework old mode 100644 new mode 100755 diff --git a/test/unit/test_rag_stream_content_type.py b/test/unit/test_rag_stream_content_type.py new file mode 100644 index 000000000..278ccf393 --- /dev/null +++ b/test/unit/test_rag_stream_content_type.py @@ -0,0 +1,118 @@ +import asyncio +import importlib +import importlib.util +import sys +from importlib.machinery import SourceFileLoader +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +# The RAG proxy is a standalone container script (no ``.py`` extension) that +# imports a handful of packages which are only installed inside the RAG +# container image, not in the unit-test environment. We stub the ones that +# are missing so the module can be imported and its response-construction +# logic exercised in isolation. +RAG_FRAMEWORK = Path(__file__).resolve().parents[2] / "container-images" / "scripts" / "rag_framework" + + +def _stub_pydantic(): + module = type(sys)("pydantic") + + class BaseModel: # minimal stand-in; models are never instantiated here + pass + + def Field(default=None, *args, **kwargs): + return default + + module.BaseModel = BaseModel + module.Field = Field + return module + + +def _stub_fastapi(): + module = type(sys)("fastapi") + + class _App: + def __init__(self, *args, **kwargs): + pass + + def _decorator(self, *args, **kwargs): + def wrap(func): + return func + + return wrap + + get = post = put = delete = middleware = _decorator + + class HTTPException(Exception): + def __init__(self, *args, **kwargs): + super().__init__(kwargs.get("detail", "")) + + module.FastAPI = lambda *a, **k: _App() + module.HTTPException = HTTPException + return module + + +def _stub_fastapi_responses(): + module = type(sys)("fastapi.responses") + + class StreamingResponse: + def __init__(self, content, media_type=None, headers=None, **kwargs): + self.body_iterator = content + self.media_type = media_type + self.headers = headers or {} + + module.StreamingResponse = StreamingResponse + return module + + +def _load_rag_framework(): + # Optional heavy deps that only ship in the container image. + for name in ("aiohttp", "openai", "qdrant_client", "uvicorn"): + if name not in sys.modules: + try: + importlib.import_module(name) + except ImportError: + sys.modules[name] = MagicMock() + + if "pydantic" not in sys.modules: + try: + importlib.import_module("pydantic") + except ImportError: + sys.modules["pydantic"] = _stub_pydantic() + + try: + importlib.import_module("fastapi") + importlib.import_module("fastapi.responses") + except ImportError: + sys.modules["fastapi"] = _stub_fastapi() + sys.modules["fastapi.responses"] = _stub_fastapi_responses() + + loader = SourceFileLoader("ramalama_rag_framework", str(RAG_FRAMEWORK)) + spec = importlib.util.spec_from_loader(loader.name, loader) + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + return module + + +def test_stream_completion_uses_sse_media_type(): + """Regression test for #2837: streaming chat completions must advertise + ``text/event-stream`` so strict OpenAI clients (e.g. Open WebUI) accept + the SSE body instead of rejecting it as ``text/plain``.""" + rag_framework = _load_rag_framework() + + # Bypass __init__: it would try to open Qdrant/OpenAI clients. The + # streaming generator is not iterated, so ``self`` is never dereferenced. + service = rag_framework.RagService.__new__(rag_framework.RagService) + + response = asyncio.run(service._stream_completion("cmpl-id", 0, MagicMock(), [])) + + assert response.media_type == "text/event-stream" + # SSE responses must not be cached. + headers = {key.lower(): value for key, value in dict(response.headers).items()} + assert headers.get("cache-control") == "no-cache" + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-v"])) From 0755fcd0f29442eaf62ba56b0d078c5ce652b7b1 Mon Sep 17 00:00:00 2001 From: Anai-Guo Date: Thu, 30 Jul 2026 01:11:36 +0000 Subject: [PATCH 3/3] test(rag): skip the SSE media-type test on Python 3.9 Signed-off-by: Anai-Guo --- test/unit/test_rag_stream_content_type.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/unit/test_rag_stream_content_type.py b/test/unit/test_rag_stream_content_type.py index 278ccf393..e3d774b53 100644 --- a/test/unit/test_rag_stream_content_type.py +++ b/test/unit/test_rag_stream_content_type.py @@ -96,6 +96,10 @@ def _load_rag_framework(): return module +@pytest.mark.skipif( + sys.version_info < (3, 10), + reason="rag_framework uses PEP 604 unions in class bodies; it only runs in the RAG container image (Python 3.11+)", +) def test_stream_completion_uses_sse_media_type(): """Regression test for #2837: streaming chat completions must advertise ``text/event-stream`` so strict OpenAI clients (e.g. Open WebUI) accept