From c092633cd7c442dd0a3f8329c3bfcf2a4a2340a2 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Wed, 6 Aug 2025 10:36:53 +0200 Subject: [PATCH 1/4] feat(models): add support to qwen-image (#5975) Signed-off-by: Ettore Di Giacinto --- backend/python/diffusers/backend.py | 29 +++++++++++++++++++++++++++++ gallery/index.yaml | 13 +++++++++++++ gallery/qwen-image.yaml | 19 +++++++++++++++++++ 3 files changed, 61 insertions(+) create mode 100644 gallery/qwen-image.yaml diff --git a/backend/python/diffusers/backend.py b/backend/python/diffusers/backend.py index 1a5f1785a2bc..17a71694a453 100755 --- a/backend/python/diffusers/backend.py +++ b/backend/python/diffusers/backend.py @@ -65,6 +65,19 @@ def sc(self, clip_input, images): return images, [False for i in images] UniPCMultistepScheduler, ) +def is_float(s): + try: + float(s) + return True + except ValueError: + return False + +def is_int(s): + try: + int(s) + return True + except ValueError: + return False # The scheduler list mapping was taken from here: https://github.com/neggles/animatediff-cli/blob/6f336f5f4b5e38e85d7f06f1744ef42d0a45f2a7/src/animatediff/schedulers.py#L39 # Credits to https://github.com/neggles @@ -169,8 +182,24 @@ def LoadModel(self, request, context): if ":" not in opt: continue key, value = opt.split(":") + # if value is a number, convert it to the appropriate type + if is_float(value): + value = float(value) + elif is_int(value): + value = int(value) self.options[key] = value + # From options, extract if present "torch_dtype" and set it to the appropriate type + if "torch_dtype" in self.options: + if self.options["torch_dtype"] == "fp16": + torchType = torch.float16 + elif self.options["torch_dtype"] == "bf16": + torchType = torch.bfloat16 + elif self.options["torch_dtype"] == "fp32": + torchType = torch.float32 + # remove it from options + del self.options["torch_dtype"] + print(f"Options: {self.options}", file=sys.stderr) local = False diff --git a/gallery/index.yaml b/gallery/index.yaml index 3cc71b9995d8..936f8ff5282b 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -1,4 +1,17 @@ --- +- &qwenimage + name: "qwen-image" + url: "github:mudler/LocalAI/gallery/qwen-image.yaml@master" + urls: + - https://huggingface.co/Qwen/Qwen-Image + icon: https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-Image/qwen_image_logo.png + license: apache-2.0 + tags: + - qwen-image + - gpu + - text-to-image + description: | + We are thrilled to release Qwen-Image, an image generation foundation model in the Qwen series that achieves significant advances in complex text rendering and precise image editing. Experiments show strong general capabilities in both image generation and editing, with exceptional performance in text rendering, especially for Chinese. - &gptoss name: "gpt-oss-20b" url: "github:mudler/LocalAI/gallery/harmony.yaml@master" diff --git a/gallery/qwen-image.yaml b/gallery/qwen-image.yaml new file mode 100644 index 000000000000..4786138dd930 --- /dev/null +++ b/gallery/qwen-image.yaml @@ -0,0 +1,19 @@ +--- +name: "flux" + +config_file: | + backend: diffusers + cfg_scale: 0 + diffusers: + cuda: true + enable_parameters: num_inference_steps + pipeline_type: DiffusionPipeline + f16: true + low_vram: true + name: qwen-image + parameters: + model: Qwen/Qwen-Image + step: 50 + options: + - true_cfg_scale:4.0 + - torch_dtype:bf16 From da6f37f00026c9a130ca3e6de98822677bff6d51 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Wed, 6 Aug 2025 10:40:46 +0200 Subject: [PATCH 2/4] Update qwen-image.yaml Signed-off-by: Ettore Di Giacinto --- gallery/qwen-image.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gallery/qwen-image.yaml b/gallery/qwen-image.yaml index 4786138dd930..6807b5508cca 100644 --- a/gallery/qwen-image.yaml +++ b/gallery/qwen-image.yaml @@ -1,5 +1,5 @@ --- -name: "flux" +name: "qwen-image" config_file: | backend: diffusers From 3295a298f4f44fb4e0aaf73c327c01dd141956aa Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Wed, 6 Aug 2025 12:38:02 +0200 Subject: [PATCH 3/4] feat(webui): allow to specify image size (#5976) Signed-off-by: Ettore Di Giacinto --- core/http/static/image.js | 12 +++++++++++- core/http/views/text2image.html | 24 ++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/core/http/static/image.js b/core/http/static/image.js index 0b85ad61f694..24a06557875b 100644 --- a/core/http/static/image.js +++ b/core/http/static/image.js @@ -11,6 +11,7 @@ async function promptDallE(input) { document.getElementById("input").disabled = true; const model = document.getElementById("image-model").value; + const size = document.getElementById("image-size").value; const response = await fetch("v1/images/generations", { method: "POST", headers: { @@ -21,7 +22,7 @@ async function promptDallE(input) { steps: 10, prompt: input, n: 1, - size: "512x512", + size: size, }), }); const json = await response.json(); @@ -48,4 +49,13 @@ async function promptDallE(input) { document.getElementById("input").focus(); document.getElementById("genimage").addEventListener("submit", genImage); + +// Handle Enter key press in the prompt input +document.getElementById("input").addEventListener("keypress", function(event) { + if (event.key === "Enter") { + event.preventDefault(); + genImage(event); + } +}); + document.getElementById("loader").style.display = "none"; diff --git a/core/http/views/text2image.html b/core/http/views/text2image.html index 0424eefe9c21..665e546e13f5 100644 --- a/core/http/views/text2image.html +++ b/core/http/views/text2image.html @@ -91,6 +91,30 @@

Generate an Image

+ + +
+ + +
+ + +
+ +
From 20a70e1244dc43b38ce7cb24fb808e7b878dbb6f Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Wed, 6 Aug 2025 12:38:45 +0200 Subject: [PATCH 4/4] feat(backends): add KittenTTS (#5977) Signed-off-by: Ettore Di Giacinto --- .github/workflows/backend.yml | 13 +++ Makefile | 9 ++ backend/index.yaml | 18 +++ backend/python/kitten-tts/Makefile | 29 +++++ backend/python/kitten-tts/backend.py | 121 +++++++++++++++++++++ backend/python/kitten-tts/install.sh | 19 ++++ backend/python/kitten-tts/requirements.txt | 5 + backend/python/kitten-tts/run.sh | 9 ++ backend/python/kitten-tts/test.py | 82 ++++++++++++++ backend/python/kitten-tts/test.sh | 11 ++ gallery/index.yaml | 22 ++++ 11 files changed, 338 insertions(+) create mode 100644 backend/python/kitten-tts/Makefile create mode 100644 backend/python/kitten-tts/backend.py create mode 100755 backend/python/kitten-tts/install.sh create mode 100644 backend/python/kitten-tts/requirements.txt create mode 100755 backend/python/kitten-tts/run.sh create mode 100644 backend/python/kitten-tts/test.py create mode 100755 backend/python/kitten-tts/test.sh diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index fb05611e0f28..839d09c47425 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -920,6 +920,19 @@ jobs: # backend: "rfdetr" # dockerfile: "./backend/Dockerfile.python" # context: "./backend" + # kitten-tts + - build-type: '' + cuda-major-version: "" + cuda-minor-version: "" + platforms: 'linux/amd64,linux/arm64' + tag-latest: 'auto' + tag-suffix: '-kitten-tts' + runs-on: 'ubuntu-latest' + base-image: "ubuntu:22.04" + skip-drivers: 'false' + backend: "kitten-tts" + dockerfile: "./backend/Dockerfile.python" + context: "./backend" llama-cpp-darwin: runs-on: macOS-14 strategy: diff --git a/Makefile b/Makefile index ef4f56725ae4..16703f5e897e 100644 --- a/Makefile +++ b/Makefile @@ -156,6 +156,9 @@ backends/huggingface: docker-build-huggingface docker-save-huggingface build backends/rfdetr: docker-build-rfdetr docker-save-rfdetr build ./local-ai backends install "ocifile://$(abspath ./backend-images/rfdetr.tar)" +backends/kitten-tts: docker-build-kitten-tts docker-save-kitten-tts build + ./local-ai backends install "ocifile://$(abspath ./backend-images/kitten-tts.tar)" + ######################################################## ## AIO tests ######################################################## @@ -369,6 +372,12 @@ docker-build-huggingface: docker-build-rfdetr: docker build --build-arg BUILD_TYPE=$(BUILD_TYPE) --build-arg BASE_IMAGE=$(BASE_IMAGE) -t local-ai-backend:rfdetr -f backend/Dockerfile.python --build-arg BACKEND=rfdetr ./backend +docker-build-kitten-tts: + docker build --build-arg BUILD_TYPE=$(BUILD_TYPE) --build-arg BASE_IMAGE=$(BASE_IMAGE) -t local-ai-backend:kitten-tts -f backend/Dockerfile.python --build-arg BACKEND=kitten-tts ./backend + +docker-save-kitten-tts: backend-images + docker save local-ai-backend:kitten-tts -o backend-images/kitten-tts.tar + docker-save-rfdetr: backend-images docker save local-ai-backend:rfdetr -o backend-images/rfdetr.tar diff --git a/backend/index.yaml b/backend/index.yaml index 17061106c9f4..9167bbd2b60e 100644 --- a/backend/index.yaml +++ b/backend/index.yaml @@ -356,6 +356,24 @@ - LLM - huggingface license: MIT +- &kitten-tts + name: "kitten-tts" + uri: "quay.io/go-skynet/local-ai-backends:latest-kitten-tts" + mirrors: + - localai/localai-backends:latest-kitten-tts + urls: + - https://github.com/KittenML/KittenTTS + description: | + Kitten TTS is a text-to-speech model that can generate speech from text. + tags: + - text-to-speech + - TTS + license: apache-2.0 +- !!merge <<: *kitten-tts + name: "kitten-tts-development" + uri: "quay.io/go-skynet/local-ai-backends:master-kitten-tts" + mirrors: + - localai/localai-backends:master-kitten-tts - !!merge <<: *huggingface name: "huggingface-development" uri: "quay.io/go-skynet/local-ai-backends:master-huggingface" diff --git a/backend/python/kitten-tts/Makefile b/backend/python/kitten-tts/Makefile new file mode 100644 index 000000000000..f05fc191698f --- /dev/null +++ b/backend/python/kitten-tts/Makefile @@ -0,0 +1,29 @@ +.PHONY: kitten-tts +kitten-tts: protogen + bash install.sh + +.PHONY: run +run: protogen + @echo "Running kitten-tts..." + bash run.sh + @echo "kitten-tts run." + +.PHONY: test +test: protogen + @echo "Testing kitten-tts..." + bash test.sh + @echo "kitten-tts tested." + +.PHONY: protogen +protogen: backend_pb2_grpc.py backend_pb2.py + +.PHONY: protogen-clean +protogen-clean: + $(RM) backend_pb2_grpc.py backend_pb2.py + +backend_pb2_grpc.py backend_pb2.py: + python3 -m grpc_tools.protoc -I../.. -I./ --python_out=. --grpc_python_out=. backend.proto + +.PHONY: clean +clean: protogen-clean + rm -rf venv __pycache__ \ No newline at end of file diff --git a/backend/python/kitten-tts/backend.py b/backend/python/kitten-tts/backend.py new file mode 100644 index 000000000000..775f85f57d0b --- /dev/null +++ b/backend/python/kitten-tts/backend.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +""" +This is an extra gRPC server of LocalAI for Kitten TTS +""" +from concurrent import futures +import time +import argparse +import signal +import sys +import os +import backend_pb2 +import backend_pb2_grpc + +import torch +from kittentts import KittenTTS +import soundfile as sf + +import grpc + + +_ONE_DAY_IN_SECONDS = 60 * 60 * 24 + +# If MAX_WORKERS are specified in the environment use it, otherwise default to 1 +MAX_WORKERS = int(os.environ.get('PYTHON_GRPC_MAX_WORKERS', '1')) +KITTEN_LANGUAGE = os.environ.get('KITTEN_LANGUAGE', None) + +# Implement the BackendServicer class with the service methods +class BackendServicer(backend_pb2_grpc.BackendServicer): + """ + BackendServicer is the class that implements the gRPC service + """ + def Health(self, request, context): + return backend_pb2.Reply(message=bytes("OK", 'utf-8')) + def LoadModel(self, request, context): + + # Get device + # device = "cuda" if request.CUDA else "cpu" + if torch.cuda.is_available(): + print("CUDA is available", file=sys.stderr) + device = "cuda" + else: + print("CUDA is not available", file=sys.stderr) + device = "cpu" + + if not torch.cuda.is_available() and request.CUDA: + return backend_pb2.Result(success=False, message="CUDA is not available") + + self.AudioPath = None + # List available KittenTTS models + print("Available KittenTTS voices: expr-voice-2-m, expr-voice-2-f, expr-voice-3-m, expr-voice-3-f, expr-voice-4-m, expr-voice-4-f, expr-voice-5-m, expr-voice-5-f") + if os.path.isabs(request.AudioPath): + self.AudioPath = request.AudioPath + elif request.AudioPath and request.ModelFile != "" and not os.path.isabs(request.AudioPath): + # get base path of modelFile + modelFileBase = os.path.dirname(request.ModelFile) + # modify LoraAdapter to be relative to modelFileBase + self.AudioPath = os.path.join(modelFileBase, request.AudioPath) + + try: + print("Preparing KittenTTS model, please wait", file=sys.stderr) + # Use the model name from request.Model, defaulting to "KittenML/kitten-tts-nano-0.1" if not specified + model_name = request.Model if request.Model else "KittenML/kitten-tts-nano-0.1" + self.tts = KittenTTS(model_name) + except Exception as err: + return backend_pb2.Result(success=False, message=f"Unexpected {err=}, {type(err)=}") + # Implement your logic here for the LoadModel service + # Replace this with your desired response + return backend_pb2.Result(message="Model loaded successfully", success=True) + + def TTS(self, request, context): + try: + # KittenTTS doesn't use language parameter like TTS, so we ignore it + # For multi-speaker models, use voice parameter + voice = request.voice if request.voice else "expr-voice-2-f" + + # Generate audio using KittenTTS + audio = self.tts.generate(request.text, voice=voice) + + # Save the audio using soundfile + sf.write(request.dst, audio, 24000) + + except Exception as err: + return backend_pb2.Result(success=False, message=f"Unexpected {err=}, {type(err)=}") + return backend_pb2.Result(success=True) + +def serve(address): + server = grpc.server(futures.ThreadPoolExecutor(max_workers=MAX_WORKERS), + options=[ + ('grpc.max_message_length', 50 * 1024 * 1024), # 50MB + ('grpc.max_send_message_length', 50 * 1024 * 1024), # 50MB + ('grpc.max_receive_message_length', 50 * 1024 * 1024), # 50MB + ]) + backend_pb2_grpc.add_BackendServicer_to_server(BackendServicer(), server) + server.add_insecure_port(address) + server.start() + print("Server started. Listening on: " + address, file=sys.stderr) + + # Define the signal handler function + def signal_handler(sig, frame): + print("Received termination signal. Shutting down...") + server.stop(0) + sys.exit(0) + + # Set the signal handlers for SIGINT and SIGTERM + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + + try: + while True: + time.sleep(_ONE_DAY_IN_SECONDS) + except KeyboardInterrupt: + server.stop(0) + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Run the gRPC server.") + parser.add_argument( + "--addr", default="localhost:50051", help="The address to bind the server to." + ) + args = parser.parse_args() + + serve(args.addr) diff --git a/backend/python/kitten-tts/install.sh b/backend/python/kitten-tts/install.sh new file mode 100755 index 000000000000..32befa8e6c03 --- /dev/null +++ b/backend/python/kitten-tts/install.sh @@ -0,0 +1,19 @@ +#!/bin/bash +set -e + +backend_dir=$(dirname $0) +if [ -d $backend_dir/common ]; then + source $backend_dir/common/libbackend.sh +else + source $backend_dir/../common/libbackend.sh +fi + +# This is here because the Intel pip index is broken and returns 200 status codes for every package name, it just doesn't return any package links. +# This makes uv think that the package exists in the Intel pip index, and by default it stops looking at other pip indexes once it finds a match. +# We need uv to continue falling through to the pypi default index to find optimum[openvino] in the pypi index +# the --upgrade actually allows us to *downgrade* torch to the version provided in the Intel pip index +if [ "x${BUILD_PROFILE}" == "xintel" ]; then + EXTRA_PIP_INSTALL_FLAGS+=" --upgrade --index-strategy=unsafe-first-match" +fi + +installRequirements diff --git a/backend/python/kitten-tts/requirements.txt b/backend/python/kitten-tts/requirements.txt new file mode 100644 index 000000000000..23439f8e5784 --- /dev/null +++ b/backend/python/kitten-tts/requirements.txt @@ -0,0 +1,5 @@ +grpcio==1.71.0 +protobuf +certifi +packaging==24.1 +https://github.com/KittenML/KittenTTS/releases/download/0.1/kittentts-0.1.0-py3-none-any.whl \ No newline at end of file diff --git a/backend/python/kitten-tts/run.sh b/backend/python/kitten-tts/run.sh new file mode 100755 index 000000000000..82b7b09ecc7d --- /dev/null +++ b/backend/python/kitten-tts/run.sh @@ -0,0 +1,9 @@ +#!/bin/bash +backend_dir=$(dirname $0) +if [ -d $backend_dir/common ]; then + source $backend_dir/common/libbackend.sh +else + source $backend_dir/../common/libbackend.sh +fi + +startBackend $@ \ No newline at end of file diff --git a/backend/python/kitten-tts/test.py b/backend/python/kitten-tts/test.py new file mode 100644 index 000000000000..e0b1a0bdd124 --- /dev/null +++ b/backend/python/kitten-tts/test.py @@ -0,0 +1,82 @@ +""" +A test script to test the gRPC service +""" +import unittest +import subprocess +import time +import backend_pb2 +import backend_pb2_grpc + +import grpc + + +class TestBackendServicer(unittest.TestCase): + """ + TestBackendServicer is the class that tests the gRPC service + """ + def setUp(self): + """ + This method sets up the gRPC service by starting the server + """ + self.service = subprocess.Popen(["python3", "backend.py", "--addr", "localhost:50051"]) + time.sleep(30) + + def tearDown(self) -> None: + """ + This method tears down the gRPC service by terminating the server + """ + self.service.terminate() + self.service.wait() + + def test_server_startup(self): + """ + This method tests if the server starts up successfully + """ + try: + self.setUp() + with grpc.insecure_channel("localhost:50051") as channel: + stub = backend_pb2_grpc.BackendStub(channel) + response = stub.Health(backend_pb2.HealthMessage()) + self.assertEqual(response.message, b'OK') + except Exception as err: + print(err) + self.fail("Server failed to start") + finally: + self.tearDown() + + def test_load_model(self): + """ + This method tests if the model is loaded successfully + """ + try: + self.setUp() + with grpc.insecure_channel("localhost:50051") as channel: + stub = backend_pb2_grpc.BackendStub(channel) + response = stub.LoadModel(backend_pb2.ModelOptions(Model="tts_models/en/vctk/vits")) + print(response) + self.assertTrue(response.success) + self.assertEqual(response.message, "Model loaded successfully") + except Exception as err: + print(err) + self.fail("LoadModel service failed") + finally: + self.tearDown() + + def test_tts(self): + """ + This method tests if the embeddings are generated successfully + """ + try: + self.setUp() + with grpc.insecure_channel("localhost:50051") as channel: + stub = backend_pb2_grpc.BackendStub(channel) + response = stub.LoadModel(backend_pb2.ModelOptions(Model="tts_models/en/vctk/vits")) + self.assertTrue(response.success) + tts_request = backend_pb2.TTSRequest(text="80s TV news production music hit for tonight's biggest story") + tts_response = stub.TTS(tts_request) + self.assertIsNotNone(tts_response) + except Exception as err: + print(err) + self.fail("TTS service failed") + finally: + self.tearDown() \ No newline at end of file diff --git a/backend/python/kitten-tts/test.sh b/backend/python/kitten-tts/test.sh new file mode 100755 index 000000000000..eb59f2aaf3f3 --- /dev/null +++ b/backend/python/kitten-tts/test.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -e + +backend_dir=$(dirname $0) +if [ -d $backend_dir/common ]; then + source $backend_dir/common/libbackend.sh +else + source $backend_dir/../common/libbackend.sh +fi + +runUnittests diff --git a/gallery/index.yaml b/gallery/index.yaml index 936f8ff5282b..9231cf15226f 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -1,4 +1,26 @@ --- +- name: "kitten-tts" + url: "github:mudler/LocalAI/gallery/virtual.yaml@master" + urls: + - https://github.com/KittenML/KittenTTS + license: apache-2.0 + tags: + - tts + - kitten-tts + - gpu + - cpu + - text-to-speech + description: | + Kitten TTS is an open-source realistic text-to-speech model with just 15 million parameters, designed for lightweight deployment and high-quality voice synthesis. + overrides: + backend: "kitten-tts" + name: "kitten-tts" + description: "Kitten TTS is a text-to-speech model that can generate speech from text." + parameters: + model: "KittenML/kitten-tts-nano-0.1" + voice: "expr-voice-5-f" + known_usecases: + - tts - &qwenimage name: "qwen-image" url: "github:mudler/LocalAI/gallery/qwen-image.yaml@master"