Merge pull request #160 from danielferr85/main

Fix streaming response in rkllama_client (mention in issue #154)

Author: NotPunchnox

README.md

@@ -1,6 +1,6 @@
 # RKLLama: LLM Server and Client for Rockchip 3588/3576
 
-### [Version: 0.0.71](#New-Version)
+### [Version: 0.0.72](#New-Version)
 
 Video demo ( version 0.0.1 ):
 

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "rkllama"
-version = "0.0.71"
+version = "0.0.72"
 authors = [
     { name="NotPunchnox", email="punchnoxpro@gmail.com" },
     { name="TomJacobsUK", email="tom@tomjacobs.co.uk" },

src/rkllama/client/client.py

@@ -241,10 +241,12 @@ def send_message(model, message):
                                     final_json = response_json
 
                                     if len(response_json["choices"]) > 0 and "delta" in response_json["choices"][0].keys():
-                                        content_chunk = response_json["choices"][0]["delta"]["content"]
-                                        sys.stdout.write(content_chunk)
-                                        sys.stdout.flush()
-                                        assistant_message += content_chunk
+                                        if "content" in response_json["choices"][0]["delta"].keys():                                        
+                                            content_chunk = response_json["choices"][0]["delta"]["content"]
+                                            if content_chunk:
+                                                sys.stdout.write(content_chunk)
+                                                sys.stdout.flush()
+                                                assistant_message += content_chunk
                                         fr = response_json["choices"][0].get("finish_reason")
                                         if fr:
                                             finish_reason = fr

src/rkllama/server/server.py

@@ -1337,7 +1337,7 @@ def embeddings_ollama():
 def ollama_version():
     """Return a dummy version to be compatible with Ollama clients"""
     return jsonify({
-        "version": "0.0.71"
+        "version": "0.0.72"
     }), 200