diff --git a/backend/index.yaml b/backend/index.yaml index 9167bbd2b60e..2bca5bf9122f 100644 --- a/backend/index.yaml +++ b/backend/index.yaml @@ -294,6 +294,7 @@ license: MIT icon: https://private-user-images.githubusercontent.com/660224/448166653-bd8c5f03-e91d-4ee5-b680-57355da204d1.png?jwt=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJnaXRodWIuY29tIiwiYXVkIjoicmF3LmdpdGh1YnVzZXJjb250ZW50LmNvbSIsImtleSI6ImtleTUiLCJleHAiOjE3NTAxOTE0MDAsIm5iZiI6MTc1MDE5MTEwMCwicGF0aCI6Ii82NjAyMjQvNDQ4MTY2NjUzLWJkOGM1ZjAzLWU5MWQtNGVlNS1iNjgwLTU3MzU1ZGEyMDRkMS5wbmc_WC1BbXotQWxnb3JpdGhtPUFXUzQtSE1BQy1TSEEyNTYmWC1BbXotQ3JlZGVudGlhbD1BS0lBVkNPRFlMU0E1M1BRSzRaQSUyRjIwMjUwNjE3JTJGdXMtZWFzdC0xJTJGczMlMkZhd3M0X3JlcXVlc3QmWC1BbXotRGF0ZT0yMDI1MDYxN1QyMDExNDBaJlgtQW16LUV4cGlyZXM9MzAwJlgtQW16LVNpZ25hdHVyZT1hMmI1NGY3OGFiZTlhNGFkNTVlYTY4NTIwMWEzODRiZGE4YzdhNGQ5MGNhNzE3MDYyYTA2NDIxYTkyYzhiODkwJlgtQW16LVNpZ25lZEhlYWRlcnM9aG9zdCJ9.mR9kM9xX0TdzPuSpuspCllHYQiq79dFQ2rtuNvjrl6w name: "chatterbox" + alias: "chatterbox" capabilities: nvidia: "cuda12-chatterbox" - &piper diff --git a/backend/python/transformers/backend.py b/backend/python/transformers/backend.py index 3d34132fded2..ef8a2fd40b6e 100644 --- a/backend/python/transformers/backend.py +++ b/backend/python/transformers/backend.py @@ -229,8 +229,13 @@ def LoadModel(self, request, context): self.model = MusicgenForConditionalGeneration.from_pretrained(model_name) elif request.Type == "DiaForConditionalGeneration": autoTokenizer = False + print("DiaForConditionalGeneration", file=sys.stderr) self.processor = AutoProcessor.from_pretrained(model_name) self.model = DiaForConditionalGeneration.from_pretrained(model_name) + if self.CUDA: + self.model = self.model.to("cuda") + self.processor = self.processor.to("cuda") + print("DiaForConditionalGeneration loaded", file=sys.stderr) self.DiaTTS = True elif request.Type == "OuteTTS": autoTokenizer = False @@ -536,7 +541,7 @@ def SoundGeneration(self, request, context): return backend_pb2.Result(success=True) - def DiaTTS(self, request, context): + def CallDiaTTS(self, request, context): """ Generates dialogue audio using the Dia model. @@ -581,7 +586,7 @@ def DiaTTS(self, request, context): return backend_pb2.Result(success=True) - def OuteTTS(self, request, context): + def CallOuteTTS(self, request, context): try: print("[OuteTTS] generating TTS", file=sys.stderr) gen_cfg = outetts.GenerationConfig( @@ -603,10 +608,11 @@ def OuteTTS(self, request, context): # The TTS endpoint is older, and provides fewer features, but exists for compatibility reasons def TTS(self, request, context): if self.OuteTTS: - return self.OuteTTS(request, context) + return self.CallOuteTTS(request, context) if self.DiaTTS: - return self.DiaTTS(request, context) + print("DiaTTS", file=sys.stderr) + return self.CallDiaTTS(request, context) model_name = request.model try: diff --git a/docs/content/docs/advanced/advanced-usage.md b/docs/content/docs/advanced/advanced-usage.md index 2ddea421b64c..8feaf95188ca 100644 --- a/docs/content/docs/advanced/advanced-usage.md +++ b/docs/content/docs/advanced/advanced-usage.md @@ -95,7 +95,7 @@ Specifying a `config-file` via CLI allows to declare models in a single file as chat: chat ``` -See also [chatbot-ui](https://github.com/go-skynet/LocalAI/tree/master/examples/chatbot-ui) as an example on how to use config files. +See also [chatbot-ui](https://github.com/mudler/LocalAI-examples/tree/main/chatbot-ui) as an example on how to use config files. It is possible to specify a full URL or a short-hand URL to a YAML model configuration file and use it on start with local-ai, for example to use phi-2: @@ -341,7 +341,7 @@ Below is an instruction that describes a task, paired with an input that provide Instead of installing models manually, you can use the LocalAI API endpoints and a model definition to install programmatically via API models in runtime. -A curated collection of model files is in the [model-gallery](https://github.com/go-skynet/model-gallery) (work in progress!). The files of the model gallery are different from the model files used to configure LocalAI models. The model gallery files contains information about the model setup, and the files necessary to run the model locally. +A curated collection of model files is in the [model-gallery](https://github.com/mudler/LocalAI/tree/master/gallery). The files of the model gallery are different from the model files used to configure LocalAI models. The model gallery files contains information about the model setup, and the files necessary to run the model locally. To install for example `lunademo`, you can send a POST call to the `/models/apply` endpoint with the model definition url (`url`) and the name of the model should have in LocalAI (`name`, optional): diff --git a/docs/content/docs/faq.md b/docs/content/docs/faq.md index c1dc24ec7759..46fd8c849c1b 100644 --- a/docs/content/docs/faq.md +++ b/docs/content/docs/faq.md @@ -46,7 +46,7 @@ There is the availability of localai-webui and chatbot-ui in the examples sectio ### Does it work with AutoGPT? -Yes, see the [examples](https://github.com/go-skynet/LocalAI/tree/master/examples/)! +Yes, see the [examples](https://github.com/mudler/LocalAI-examples)! ### How can I troubleshoot when something is wrong? diff --git a/docs/content/docs/features/embeddings.md b/docs/content/docs/features/embeddings.md index 7e0f3abf48ac..e6464634fd67 100644 --- a/docs/content/docs/features/embeddings.md +++ b/docs/content/docs/features/embeddings.md @@ -75,4 +75,4 @@ curl http://localhost:8080/embeddings -X POST -H "Content-Type: application/json ## 💡 Examples -- Example that uses LLamaIndex and LocalAI as embedding: [here](https://github.com/go-skynet/LocalAI/tree/master/examples/query_data/). +- Example that uses LLamaIndex and LocalAI as embedding: [here](https://github.com/mudler/LocalAI-examples/tree/main/query_data). diff --git a/docs/content/docs/features/openai-functions.md b/docs/content/docs/features/openai-functions.md index 5d43ece03f7c..2f0c8c419f88 100644 --- a/docs/content/docs/features/openai-functions.md +++ b/docs/content/docs/features/openai-functions.md @@ -263,4 +263,4 @@ Grammars and function tools can be used as well in conjunction with vision APIs: ## 💡 Examples -A full e2e example with `docker-compose` is available [here](https://github.com/go-skynet/LocalAI/tree/master/examples/functions). +A full e2e example with `docker-compose` is available [here](https://github.com/mudler/LocalAI-examples/tree/main/functions). diff --git a/docs/content/docs/getting-started/models.md b/docs/content/docs/getting-started/models.md index a57fa9942776..08a25e982dbb 100644 --- a/docs/content/docs/getting-started/models.md +++ b/docs/content/docs/getting-started/models.md @@ -207,4 +207,4 @@ For instructions on building LocalAI from source, see the [Build Section]({{% re {{% /tab %}} {{< /tabs >}} -For more model configurations, visit the [Examples Section](https://github.com/mudler/LocalAI/tree/master/examples/configurations). +For more model configurations, visit the [Examples Section](https://github.com/mudler/LocalAI-examples/tree/main/configurations). diff --git a/docs/content/docs/whats-new.md b/docs/content/docs/whats-new.md index e4f7ab25cb51..320d0dca198e 100644 --- a/docs/content/docs/whats-new.md +++ b/docs/content/docs/whats-new.md @@ -99,8 +99,8 @@ Thanks to the community efforts now we have a new [how-to website](https://io.mi #### 💡 More examples! -- Open source autopilot? See the new addition by {{< github "gruberdev" >}} in our [examples](https://github.com/go-skynet/LocalAI/tree/master/examples/continue) on how to use Continue with LocalAI! -- Want to try LocalAI with Insomnia? Check out the new [Insomnia example](https://github.com/go-skynet/LocalAI/tree/master/examples/insomnia) by {{< github "dave-gray101" >}}! +- Open source autopilot? See the new addition by {{< github "gruberdev" >}} in our [examples](https://github.com/mudler/LocalAI-examples/tree/main/continue) on how to use Continue with LocalAI! +- Want to try LocalAI with Insomnia? Check out the new [Insomnia example](https://github.com/mudler/LocalAI-examples/tree/main/insomnia) by {{< github "dave-gray101" >}}! #### LocalAGI in discord! @@ -258,7 +258,7 @@ And here when it actually picks to reply to the user instead of using functions! Note: functions are supported only with `llama.cpp`-compatible models. -A full example is available here: https://github.com/go-skynet/LocalAI/tree/master/examples/functions +A full example is available here: https://github.com/mudler/LocalAI-examples/tree/main/functions ### gRPC backends @@ -377,9 +377,9 @@ We now support a vast variety of models, while being backward compatible with pr ### Examples -- 💡 [AutoGPT](https://github.com/go-skynet/LocalAI/tree/master/examples/autoGPT) example ( [mudler](https://github.com/mudler) ) -- 💡 [PrivateGPT](https://github.com/go-skynet/LocalAI/tree/master/examples/privateGPT) example ( [mudler](https://github.com/mudler) ) -- 💡 [Flowise](https://github.com/go-skynet/LocalAI/tree/master/examples/flowise) example ( [mudler](https://github.com/mudler) ) +- 💡 [AutoGPT](https://github.com/mudler/LocalAI-examples/tree/main/autoGPT) example ( [mudler](https://github.com/mudler) ) +- 💡 [PrivateGPT](https://github.com/mudler/LocalAI-examples/tree/main/privateGPT) example ( [mudler](https://github.com/mudler) ) +- 💡 [Flowise](https://github.com/mudler/LocalAI-examples/tree/main/flowise) example ( [mudler](https://github.com/mudler) ) Two new projects offer now direct integration with LocalAI! @@ -449,7 +449,7 @@ Now LocalAI can generate images too: - 14-05-2023: __v1.11.1__ released! `rwkv` backend patch release - 13-05-2023: __v1.11.0__ released! 🔥 Updated `llama.cpp` bindings: This update includes a breaking change in the model files ( https://github.com/ggerganov/llama.cpp/pull/1405 ) - old models should still work with the `gpt4all-llama` backend. -- 12-05-2023: __v1.10.0__ released! 🔥🔥 Updated `gpt4all` bindings. Added support for GPTNeox (experimental), RedPajama (experimental), Starcoder (experimental), Replit (experimental), MosaicML MPT. Also now `embeddings` endpoint supports tokens arrays. See the [langchain-chroma](https://github.com/go-skynet/LocalAI/tree/master/examples/langchain-chroma) example! Note - this update does NOT include https://github.com/ggerganov/llama.cpp/pull/1405 which makes models incompatible. +- 12-05-2023: __v1.10.0__ released! 🔥🔥 Updated `gpt4all` bindings. Added support for GPTNeox (experimental), RedPajama (experimental), Starcoder (experimental), Replit (experimental), MosaicML MPT. Also now `embeddings` endpoint supports tokens arrays. See the [langchain-chroma](https://github.com/mudler/LocalAI-examples/tree/main/langchain-chroma) example! Note - this update does NOT include https://github.com/ggerganov/llama.cpp/pull/1405 which makes models incompatible. - 11-05-2023: __v1.9.0__ released! 🔥 Important whisper updates ( {{< pr "233" >}} {{< pr "229" >}} ) and extended gpt4all model families support ( {{< pr "232" >}} ). Redpajama/dolly experimental ( {{< pr "214" >}} ) - 10-05-2023: __v1.8.0__ released! 🔥 Added support for fast and accurate embeddings with `bert.cpp` ( {{< pr "222" >}} ) - 09-05-2023: Added experimental support for transcriptions endpoint ( {{< pr "211" >}} ) diff --git a/gallery/index.yaml b/gallery/index.yaml index 1a268c47e595..6662e68e5dba 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -151,6 +151,103 @@ - filename: OpenAI-20B-NEO-MXFP4_MOE4.gguf sha256: 066c84a0844b1f1f4515e5c64095fe4c67e86d5eb70db4e368e283b1134d9c1e uri: huggingface://DavidAU/Openai_gpt-oss-20b-NEO-GGUF/OpenAI-20B-NEO-MXFP4_MOE4.gguf +- !!merge <<: *gptoss + name: "huihui-ai_huihui-gpt-oss-20b-bf16-abliterated" + urls: + - https://huggingface.co/huihui-ai/Huihui-gpt-oss-20b-BF16-abliterated + - https://huggingface.co/bartowski/huihui-ai_Huihui-gpt-oss-20b-BF16-abliterated-GGUF + description: | + This is an uncensored version of unsloth/gpt-oss-20b-BF16 created with abliteration (see remove-refusals-with-transformers to know more about it). + overrides: + parameters: + model: huihui-ai_Huihui-gpt-oss-20b-BF16-abliterated-MXFP4_MOE.gguf + files: + - filename: huihui-ai_Huihui-gpt-oss-20b-BF16-abliterated-MXFP4_MOE.gguf + sha256: abca50d1bd95c49d71db36aad0f38090ea5465ce148634c496a48bc87030bdd9 + uri: huggingface://bartowski/huihui-ai_Huihui-gpt-oss-20b-BF16-abliterated-GGUF/huihui-ai_Huihui-gpt-oss-20b-BF16-abliterated-MXFP4_MOE.gguf +- !!merge <<: *gptoss + name: "openai-gpt-oss-20b-abliterated-uncensored-neo-imatrix" + icon: https://huggingface.co/DavidAU/OpenAi-GPT-oss-20b-abliterated-uncensored-NEO-Imatrix-gguf/resolve/main/power-the-matrix.gif + urls: + - https://huggingface.co/DavidAU/OpenAi-GPT-oss-20b-abliterated-uncensored-NEO-Imatrix-gguf + description: | + These are NEO Imatrix GGUFs, NEO dataset by DavidAU. + + NEO dataset improves overall performance, and is for all use cases. + + This model uses Huihui-gpt-oss-20b-BF16-abliterated as a base which DE-CENSORS the model and removes refusals. + + Example output below (creative; IQ4_NL), using settings below. + + This model can be a little rough around the edges (due to abliteration) ; make sure you see the settings below for best operation. + + It can also be creative, off the shelf crazy and rational too. + + Enjoy! + overrides: + parameters: + model: OpenAI-20B-NEOPlus-Uncensored-IQ4_NL.gguf + files: + - filename: OpenAI-20B-NEOPlus-Uncensored-IQ4_NL.gguf + sha256: 274ffaaf0783270c071006842ffe60af73600fc63c2b6153c0701b596fc3b122 + uri: huggingface://DavidAU/OpenAi-GPT-oss-20b-abliterated-uncensored-NEO-Imatrix-gguf/OpenAI-20B-NEOPlus-Uncensored-IQ4_NL.gguf +- name: "chatterbox" + url: "github:mudler/LocalAI/gallery/virtual.yaml@master" + icon: https://private-user-images.githubusercontent.com/660224/448166653-bd8c5f03-e91d-4ee5-b680-57355da204d1.png + license: "mit" + urls: + - https://github.com/resemble-ai/chatterbox + tags: + - tts + - dia + - gpu + - text-to-speech + description: | + Chatterbox, Resemble AI's first production-grade open source TTS model. Licensed under MIT, Chatterbox has been benchmarked against leading closed-source systems like ElevenLabs, and is consistently preferred in side-by-side evaluations. + overrides: + backend: "chatterbox" + name: "chatterbox" + known_usecases: + - tts +- name: "dia" + url: "github:mudler/LocalAI/gallery/virtual.yaml@master" + icon: https://github.com/nari-labs/dia/raw/main/dia/static/images/banner.png + urls: + - https://github.com/nari-labs/dia + - https://huggingface.co/nari-labs/Dia-1.6B-0626 + license: apache-2.0 + tags: + - tts + - dia + - gpu + - text-to-speech + overrides: + backend: "transformers" + name: "dia" + description: "Dia is a 1.6B parameter text to speech model created by Nari Labs." + parameters: + model: nari-labs/Dia-1.6B-0626 + type: DiaForConditionalGeneration + known_usecases: + - tts +- name: "outetts" + url: "github:mudler/LocalAI/gallery/virtual.yaml@master" + urls: + - https://github.com/edwko/OuteTTS + license: apache-2.0 + tags: + - tts + - gpu + - text-to-speech + overrides: + backend: "transformers" + name: "outetts" + description: "OuteTTS is a 1.6B parameter text to speech model created by OuteAI." + parameters: + model: OuteAI/OuteTTS-0.3-1B + type: OuteTTS + known_usecases: + - tts - &afm name: "arcee-ai_afm-4.5b" url: "github:mudler/LocalAI/gallery/chatml.yaml@master" @@ -11832,6 +11929,29 @@ - filename: ockerman0_AnubisLemonade-70B-v1.1-Q4_K_M.gguf sha256: e217b2c39d4fae8499ca2a24ff8c7025ec93cd16883aa57f43ac9240222c4754 uri: huggingface://bartowski/ockerman0_AnubisLemonade-70B-v1.1-GGUF/ockerman0_AnubisLemonade-70B-v1.1-Q4_K_M.gguf +- !!merge <<: *llama31 + name: "tarek07_nomad-llama-70b" + icon: https://cdn-uploads.huggingface.co/production/uploads/64909c086073a0cd172d0411/5F7S8kdO8NTMua6iCRTUO.png + urls: + - https://huggingface.co/Tarek07/Nomad-LLaMa-70B + - https://huggingface.co/bartowski/Tarek07_Nomad-LLaMa-70B-GGUF + description: | + I decided to make a simple model for a change, with some models I was curious to see work together. + models: + - model: ArliAI/DS-R1-Distill-70B-ArliAI-RpR-v4-Large + - model: TheDrummer/Anubis-70B-v1.1 + - model: Mawdistical/Vulpine-Seduction-70B + - model: Darkhn/L3.3-70B-Animus-V5-Pro + - model: zerofata/L3.3-GeneticLemonade-Unleashed-v3-70B + - model: Sao10K/Llama-3.3-70B-Vulpecula-r1 + base_model: nbeerbower/Llama-3.1-Nemotron-lorablated-70B + overrides: + parameters: + model: Tarek07_Nomad-LLaMa-70B-Q4_K_M.gguf + files: + - filename: Tarek07_Nomad-LLaMa-70B-Q4_K_M.gguf + sha256: 734c7042a84cd6c059c4ddd3ffb84b23752aeaaf670c5cbb0031f8128ec5ffc8 + uri: huggingface://bartowski/Tarek07_Nomad-LLaMa-70B-GGUF/Tarek07_Nomad-LLaMa-70B-Q4_K_M.gguf - &deepseek url: "github:mudler/LocalAI/gallery/deepseek.yaml@master" ## Deepseek name: "deepseek-coder-v2-lite-instruct"