diff --git a/.github/workflows/bump_deps.yaml b/.github/workflows/bump_deps.yaml index f15d62f7f520..2b7941112c55 100644 --- a/.github/workflows/bump_deps.yaml +++ b/.github/workflows/bump_deps.yaml @@ -21,7 +21,7 @@ jobs: variable: "BARKCPP_VERSION" branch: "main" file: "Makefile" - - repository: "richiejp/stable-diffusion.cpp" + - repository: "leejet/stable-diffusion.cpp" variable: "STABLEDIFFUSION_GGML_VERSION" branch: "master" file: "backend/go/stablediffusion-ggml/Makefile" diff --git a/Makefile b/Makefile index 815692c24b5b..03496c202ad5 100644 --- a/Makefile +++ b/Makefile @@ -145,7 +145,7 @@ backends/stablediffusion-ggml: docker-build-stablediffusion-ggml docker-save-sta backends/whisper: docker-build-whisper docker-save-whisper build ./local-ai backends install "ocifile://$(abspath ./backend-images/whisper.tar)" - + backends/silero-vad: docker-build-silero-vad docker-save-silero-vad build ./local-ai backends install "ocifile://$(abspath ./backend-images/silero-vad.tar)" @@ -359,19 +359,19 @@ backend-images: mkdir -p backend-images docker-build-llama-cpp: - docker build --build-arg BUILD_TYPE=$(BUILD_TYPE) --build-arg IMAGE_BASE=$(IMAGE_BASE) -t local-ai-backend:llama-cpp -f backend/Dockerfile.llama-cpp . + docker build --build-arg BUILD_TYPE=$(BUILD_TYPE) --build-arg BASE_IMAGE=$(BASE_IMAGE) -t local-ai-backend:llama-cpp -f backend/Dockerfile.llama-cpp . docker-build-bark-cpp: - docker build -t local-ai-backend:bark-cpp -f backend/Dockerfile.golang --build-arg BACKEND=bark-cpp . + docker build --build-arg BUILD_TYPE=$(BUILD_TYPE) --build-arg BASE_IMAGE=$(BASE_IMAGE) -t local-ai-backend:bark-cpp -f backend/Dockerfile.golang --build-arg BACKEND=bark-cpp . docker-build-piper: - docker build -t local-ai-backend:piper -f backend/Dockerfile.golang --build-arg BACKEND=piper . + docker build --build-arg BUILD_TYPE=$(BUILD_TYPE) --build-arg BASE_IMAGE=$(BASE_IMAGE) -t local-ai-backend:piper -f backend/Dockerfile.golang --build-arg BACKEND=piper . docker-build-local-store: - docker build -t local-ai-backend:local-store -f backend/Dockerfile.golang --build-arg BACKEND=local-store . + docker build --build-arg BUILD_TYPE=$(BUILD_TYPE) --build-arg BASE_IMAGE=$(BASE_IMAGE) -t local-ai-backend:local-store -f backend/Dockerfile.golang --build-arg BACKEND=local-store . docker-build-huggingface: - docker build -t local-ai-backend:huggingface -f backend/Dockerfile.golang --build-arg BACKEND=huggingface . + docker build --build-arg BUILD_TYPE=$(BUILD_TYPE) --build-arg BASE_IMAGE=$(BASE_IMAGE) -t local-ai-backend:huggingface -f backend/Dockerfile.golang --build-arg BACKEND=huggingface . docker-save-huggingface: backend-images docker save local-ai-backend:huggingface -o backend-images/huggingface.tar @@ -380,7 +380,7 @@ docker-save-local-store: backend-images docker save local-ai-backend:local-store -o backend-images/local-store.tar docker-build-silero-vad: - docker build -t local-ai-backend:silero-vad -f backend/Dockerfile.golang --build-arg BACKEND=silero-vad . + docker build --build-arg BUILD_TYPE=$(BUILD_TYPE) --build-arg BASE_IMAGE=$(BASE_IMAGE) -t local-ai-backend:silero-vad -f backend/Dockerfile.golang --build-arg BACKEND=silero-vad . docker-save-silero-vad: backend-images docker save local-ai-backend:silero-vad -o backend-images/silero-vad.tar @@ -395,25 +395,25 @@ docker-save-bark-cpp: backend-images docker save local-ai-backend:bark-cpp -o backend-images/bark-cpp.tar docker-build-stablediffusion-ggml: - docker build -t local-ai-backend:stablediffusion-ggml -f backend/Dockerfile.golang --build-arg BACKEND=stablediffusion-ggml . + docker build --build-arg BUILD_TYPE=$(BUILD_TYPE) --build-arg BASE_IMAGE=$(BASE_IMAGE) -t local-ai-backend:stablediffusion-ggml -f backend/Dockerfile.golang --build-arg BACKEND=stablediffusion-ggml . docker-save-stablediffusion-ggml: backend-images docker save local-ai-backend:stablediffusion-ggml -o backend-images/stablediffusion-ggml.tar docker-build-rerankers: - docker build -t local-ai-backend:rerankers -f backend/Dockerfile.python --build-arg BACKEND=rerankers . + docker build --build-arg BUILD_TYPE=$(BUILD_TYPE) --build-arg BASE_IMAGE=$(BASE_IMAGE) -t local-ai-backend:rerankers -f backend/Dockerfile.python --build-arg BACKEND=rerankers . docker-build-vllm: - docker build -t local-ai-backend:vllm -f backend/Dockerfile.python --build-arg BACKEND=vllm . + docker build --build-arg BUILD_TYPE=$(BUILD_TYPE) --build-arg BASE_IMAGE=$(BASE_IMAGE) -t local-ai-backend:vllm -f backend/Dockerfile.python --build-arg BACKEND=vllm . docker-build-transformers: - docker build -t local-ai-backend:transformers -f backend/Dockerfile.python --build-arg BACKEND=transformers . + docker build --build-arg BUILD_TYPE=$(BUILD_TYPE) --build-arg BASE_IMAGE=$(BASE_IMAGE) -t local-ai-backend:transformers -f backend/Dockerfile.python --build-arg BACKEND=transformers . docker-build-diffusers: - docker build -t local-ai-backend:diffusers -f backend/Dockerfile.python --build-arg BACKEND=diffusers . + docker build --build-arg BUILD_TYPE=$(BUILD_TYPE) --build-arg BASE_IMAGE=$(BASE_IMAGE) -t local-ai-backend:diffusers -f backend/Dockerfile.python --build-arg BACKEND=diffusers . docker-build-kokoro: - docker build -t local-ai-backend:kokoro -f backend/Dockerfile.python --build-arg BACKEND=kokoro . + docker build --build-arg BUILD_TYPE=$(BUILD_TYPE) --build-arg BASE_IMAGE=$(BASE_IMAGE) -t local-ai-backend:kokoro -f backend/Dockerfile.python --build-arg BACKEND=kokoro . docker-build-whisper: docker build --build-arg BUILD_TYPE=$(BUILD_TYPE) --build-arg BASE_IMAGE=$(BASE_IMAGE) -t local-ai-backend:whisper -f backend/Dockerfile.golang --build-arg BACKEND=whisper . @@ -422,19 +422,19 @@ docker-save-whisper: backend-images docker save local-ai-backend:whisper -o backend-images/whisper.tar docker-build-faster-whisper: - docker build -t local-ai-backend:faster-whisper -f backend/Dockerfile.python --build-arg BACKEND=faster-whisper . + docker build --build-arg BUILD_TYPE=$(BUILD_TYPE) --build-arg BASE_IMAGE=$(BASE_IMAGE) -t local-ai-backend:faster-whisper -f backend/Dockerfile.python --build-arg BACKEND=faster-whisper . docker-build-coqui: - docker build -t local-ai-backend:coqui -f backend/Dockerfile.python --build-arg BACKEND=coqui . + docker build --build-arg BUILD_TYPE=$(BUILD_TYPE) --build-arg BASE_IMAGE=$(BASE_IMAGE) -t local-ai-backend:coqui -f backend/Dockerfile.python --build-arg BACKEND=coqui . docker-build-bark: - docker build -t local-ai-backend:bark -f backend/Dockerfile.python --build-arg BACKEND=bark . + docker build --build-arg BUILD_TYPE=$(BUILD_TYPE) --build-arg BASE_IMAGE=$(BASE_IMAGE) -t local-ai-backend:bark -f backend/Dockerfile.python --build-arg BACKEND=bark . docker-build-chatterbox: - docker build -t local-ai-backend:chatterbox -f backend/Dockerfile.python --build-arg BACKEND=chatterbox . + docker build --build-arg BUILD_TYPE=$(BUILD_TYPE) --build-arg BASE_IMAGE=$(BASE_IMAGE) -t local-ai-backend:chatterbox -f backend/Dockerfile.python --build-arg BACKEND=chatterbox . docker-build-exllama2: - docker build -t local-ai-backend:exllama2 -f backend/Dockerfile.python --build-arg BACKEND=exllama2 . + docker build --build-arg BUILD_TYPE=$(BUILD_TYPE) --build-arg BASE_IMAGE=$(BASE_IMAGE) -t local-ai-backend:exllama2 -f backend/Dockerfile.python --build-arg BACKEND=exllama2 . docker-build-backends: docker-build-llama-cpp docker-build-rerankers docker-build-vllm docker-build-transformers docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-coqui docker-build-bark docker-build-chatterbox docker-build-exllama2 diff --git a/backend/cpp/llama-cpp/Makefile b/backend/cpp/llama-cpp/Makefile index aee6e6c7ebfa..40a7399463ab 100644 --- a/backend/cpp/llama-cpp/Makefile +++ b/backend/cpp/llama-cpp/Makefile @@ -1,5 +1,5 @@ -LLAMA_VERSION?=acd6cb1c41676f6bbb25c2a76fa5abeb1719301e +LLAMA_VERSION?=a86f52b2859dae4db5a7a0bbc0f1ad9de6b43ec6 LLAMA_REPO?=https://github.com/ggerganov/llama.cpp CMAKE_ARGS?= diff --git a/backend/go/stablediffusion-ggml/Makefile b/backend/go/stablediffusion-ggml/Makefile index 9f15d55118ae..3d3e1427965e 100644 --- a/backend/go/stablediffusion-ggml/Makefile +++ b/backend/go/stablediffusion-ggml/Makefile @@ -18,8 +18,8 @@ GO_TAGS?= LD_FLAGS?= # stablediffusion.cpp (ggml) -STABLEDIFFUSION_GGML_REPO?=https://github.com/richiejp/stable-diffusion.cpp -STABLEDIFFUSION_GGML_VERSION?=10c6501bd05a697e014f1bee3a84e5664290c489 +STABLEDIFFUSION_GGML_REPO?=https://github.com/leejet/stable-diffusion.cpp +STABLEDIFFUSION_GGML_VERSION?=1896b28ef2fd5b3643120e66979bea487385439f # Disable Shared libs as we are linking on static gRPC and we can't mix shared and static CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF @@ -91,23 +91,18 @@ endif # (ggml can have different backends cpu, cuda, etc., each backend generates a .a archive) GGML_ARCHIVE_DIR := build/ggml/src/ ALL_ARCHIVES := $(shell find $(GGML_ARCHIVE_DIR) -type f -name '*.a') +ALL_OBJS := $(shell find $(GGML_ARCHIVE_DIR) -type f -name '*.o') # Name of the single merged library COMBINED_LIB := libggmlall.a -# Rule to merge all the .a files into one +# Instead of using the archives generated by GGML, use the object files directly to avoid overwriting objects with the same base name $(COMBINED_LIB): $(ALL_ARCHIVES) - @echo "Merging all .a into $(COMBINED_LIB)" + @echo "Merging all .o into $(COMBINED_LIB): $(ALL_OBJS)" rm -f $@ - mkdir -p merge-tmp - for a in $(ALL_ARCHIVES); do \ - ( cd merge-tmp && ar x ../$$a ); \ - done - ( cd merge-tmp && ar rcs ../$@ *.o ) + ar -qc $@ $(ALL_OBJS) # Ensure we have a proper index ranlib $@ - # Clean up - rm -rf merge-tmp build/libstable-diffusion.a: @echo "Building SD with $(BUILD_TYPE) build type and $(CMAKE_ARGS)" diff --git a/backend/go/stablediffusion-ggml/gosd.cpp b/backend/go/stablediffusion-ggml/gosd.cpp index 4c7c161a69fd..e3e665d7a866 100644 --- a/backend/go/stablediffusion-ggml/gosd.cpp +++ b/backend/go/stablediffusion-ggml/gosd.cpp @@ -53,9 +53,43 @@ sd_ctx_t* sd_c; sample_method_t sample_method; +// Copied from the upstream CLI +void sd_log_cb(enum sd_log_level_t level, const char* log, void* data) { + //SDParams* params = (SDParams*)data; + const char* level_str; + + if (!log /*|| (!params->verbose && level <= SD_LOG_DEBUG)*/) { + return; + } + + switch (level) { + case SD_LOG_DEBUG: + level_str = "DEBUG"; + break; + case SD_LOG_INFO: + level_str = "INFO"; + break; + case SD_LOG_WARN: + level_str = "WARN"; + break; + case SD_LOG_ERROR: + level_str = "ERROR"; + break; + default: /* Potential future-proofing */ + level_str = "?????"; + break; + } + + fprintf(stderr, "[%-5s] ", level_str); + fputs(log, stderr); + fflush(stderr); +} + int load_model(char *model, char* options[], int threads, int diff) { fprintf (stderr, "Loading model!\n"); + sd_set_log_callback(sd_log_cb, NULL); + char *stableDiffusionModel = ""; if (diff == 1 ) { stableDiffusionModel = model; @@ -70,6 +104,8 @@ int load_model(char *model, char* options[], int threads, int diff) { char *scheduler = ""; char *sampler = ""; + fprintf(stderr, "parsing options\n"); + // If options is not NULL, parse options for (int i = 0; options[i] != NULL; i++) { char *optname = strtok(options[i], ":"); @@ -98,10 +134,13 @@ int load_model(char *model, char* options[], int threads, int diff) { } } + fprintf(stderr, "parsed options\n"); + int sample_method_found = -1; - for (int m = 0; m < N_SAMPLE_METHODS; m++) { + for (int m = 0; m < SAMPLE_METHOD_COUNT; m++) { if (!strcmp(sampler, sample_method_str[m])) { sample_method_found = m; + fprintf(stderr, "Found sampler: %s\n", sampler); } } if (sample_method_found == -1) { @@ -111,7 +150,7 @@ int load_model(char *model, char* options[], int threads, int diff) { sample_method = (sample_method_t)sample_method_found; int schedule_found = -1; - for (int d = 0; d < N_SCHEDULES; d++) { + for (int d = 0; d < SCHEDULE_COUNT; d++) { if (!strcmp(scheduler, schedule_str[d])) { schedule_found = d; fprintf (stderr, "Found scheduler: %s\n", scheduler); @@ -125,30 +164,28 @@ int load_model(char *model, char* options[], int threads, int diff) { } schedule_t schedule = (schedule_t)schedule_found; - + fprintf (stderr, "Creating context\n"); - sd_ctx_t* sd_ctx = new_sd_ctx(model, - clip_l_path, - clip_g_path, - t5xxl_path, - stableDiffusionModel, - vae_path, - "", - "", - "", - "", - "", - false, - false, - false, - threads, - SD_TYPE_COUNT, - STD_DEFAULT_RNG, - schedule, - false, - false, - false, - false); + sd_ctx_params_t ctx_params; + sd_ctx_params_init(&ctx_params); + ctx_params.model_path = model; + ctx_params.clip_l_path = clip_l_path; + ctx_params.clip_g_path = clip_g_path; + ctx_params.t5xxl_path = t5xxl_path; + ctx_params.diffusion_model_path = stableDiffusionModel; + ctx_params.vae_path = vae_path; + ctx_params.taesd_path = ""; + ctx_params.control_net_path = ""; + ctx_params.lora_model_dir = ""; + ctx_params.embedding_dir = ""; + ctx_params.stacked_id_embed_dir = ""; + ctx_params.vae_decode_only = false; + ctx_params.vae_tiling = false; + ctx_params.free_params_immediately = false; + ctx_params.n_threads = threads; + ctx_params.rng_type = STD_DEFAULT_RNG; + ctx_params.schedule = schedule; + sd_ctx_t* sd_ctx = new_sd_ctx(&ctx_params); if (sd_ctx == NULL) { fprintf (stderr, "failed loading model (generic error)\n"); @@ -169,29 +206,22 @@ int gen_image(char *text, char *negativeText, int width, int height, int steps, fprintf (stderr, "Generating image\n"); - results = txt2img(sd_c, - text, - negativeText, - -1, //clip_skip - cfg_scale, // sfg_scale - 3.5f, - 0, // eta - width, - height, - sample_method, - steps, - seed, - 1, - NULL, - 0.9f, - 20.f, - false, - "", - skip_layers.data(), - skip_layers.size(), - 0, - 0.01, - 0.2); + sd_img_gen_params_t p; + sd_img_gen_params_init(&p); + + p.prompt = text; + p.negative_prompt = negativeText; + p.guidance.txt_cfg = cfg_scale; + p.guidance.slg.layers = skip_layers.data(); + p.guidance.slg.layer_count = skip_layers.size(); + p.width = width; + p.height = height; + p.sample_method = sample_method; + p.sample_steps = steps; + p.seed = seed; + p.input_id_images_path = ""; + + results = generate_image(sd_c, &p); if (results == NULL) { fprintf (stderr, "NO results\n"); diff --git a/backend/go/stablediffusion-ggml/gosd.go b/backend/go/stablediffusion-ggml/gosd.go index 7d8d770ffcf2..fa4d0e72e819 100644 --- a/backend/go/stablediffusion-ggml/gosd.go +++ b/backend/go/stablediffusion-ggml/gosd.go @@ -37,8 +37,8 @@ func (sd *SDGGML) Load(opts *pb.ModelOptions) error { size := C.size_t(unsafe.Sizeof((*C.char)(nil))) length := C.size_t(len(opts.Options)) - options = (**C.char)(C.malloc(length * size)) - view := (*[1 << 30]*C.char)(unsafe.Pointer(options))[0:len(opts.Options):len(opts.Options)] + options = (**C.char)(C.malloc((length + 1) * size)) + view := (*[1 << 30]*C.char)(unsafe.Pointer(options))[0:len(opts.Options) + 1:len(opts.Options) + 1] var diffusionModel int @@ -66,6 +66,7 @@ func (sd *SDGGML) Load(opts *pb.ModelOptions) error { for i, x := range oo { view[i] = C.CString(x) } + view[len(oo)] = nil sd.cfgScale = opts.CFGScale diff --git a/core/application/application.go b/core/application/application.go index 8c9842d948ab..a990866019a0 100644 --- a/core/application/application.go +++ b/core/application/application.go @@ -2,8 +2,8 @@ package application import ( "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/templates" "github.com/mudler/LocalAI/pkg/model" - "github.com/mudler/LocalAI/pkg/templates" ) type Application struct { diff --git a/core/application/startup.go b/core/application/startup.go index 59003799b526..983807bb278d 100644 --- a/core/application/startup.go +++ b/core/application/startup.go @@ -10,8 +10,8 @@ import ( "github.com/mudler/LocalAI/core/services" "github.com/mudler/LocalAI/internal" + coreStartup "github.com/mudler/LocalAI/core/startup" "github.com/mudler/LocalAI/pkg/model" - pkgStartup "github.com/mudler/LocalAI/pkg/startup" "github.com/mudler/LocalAI/pkg/xsysinfo" "github.com/rs/zerolog/log" ) @@ -55,11 +55,11 @@ func New(opts ...config.AppOption) (*Application, error) { } } - if err := pkgStartup.InstallModels(options.Galleries, options.BackendGalleries, options.ModelPath, options.BackendsPath, options.EnforcePredownloadScans, options.AutoloadBackendGalleries, nil, options.ModelsURL...); err != nil { + if err := coreStartup.InstallModels(options.Galleries, options.BackendGalleries, options.ModelPath, options.BackendsPath, options.EnforcePredownloadScans, options.AutoloadBackendGalleries, nil, options.ModelsURL...); err != nil { log.Error().Err(err).Msg("error installing models") } - if err := pkgStartup.InstallExternalBackends(options.BackendGalleries, options.BackendsPath, nil, options.ExternalBackends...); err != nil { + if err := coreStartup.InstallExternalBackends(options.BackendGalleries, options.BackendsPath, nil, options.ExternalBackends...); err != nil { log.Error().Err(err).Msg("error installing external backends") } diff --git a/core/cli/backends.go b/core/cli/backends.go index 480d6890140a..d5fa84878ee5 100644 --- a/core/cli/backends.go +++ b/core/cli/backends.go @@ -8,7 +8,7 @@ import ( "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/gallery" - "github.com/mudler/LocalAI/pkg/startup" + "github.com/mudler/LocalAI/core/startup" "github.com/rs/zerolog/log" "github.com/schollz/progressbar/v3" ) diff --git a/core/cli/models.go b/core/cli/models.go index 94a838961fa0..1dc018572159 100644 --- a/core/cli/models.go +++ b/core/cli/models.go @@ -9,8 +9,8 @@ import ( "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/gallery" + "github.com/mudler/LocalAI/core/startup" "github.com/mudler/LocalAI/pkg/downloader" - "github.com/mudler/LocalAI/pkg/startup" "github.com/rs/zerolog/log" "github.com/schollz/progressbar/v3" ) diff --git a/core/cli/util.go b/core/cli/util.go index a13a11590991..4e01ea2c25a8 100644 --- a/core/cli/util.go +++ b/core/cli/util.go @@ -72,7 +72,7 @@ func (u *CreateOCIImageCMD) Run(ctx *cliContext.Context) error { } func (u *GGUFInfoCMD) Run(ctx *cliContext.Context) error { - if u.Args == nil || len(u.Args) == 0 { + if len(u.Args) == 0 { return fmt.Errorf("no GGUF file provided") } // We try to guess only if we don't have a template defined already diff --git a/core/gallery/backend_types.go b/core/gallery/backend_types.go index e45b8c7c3059..439355a46cf9 100644 --- a/core/gallery/backend_types.go +++ b/core/gallery/backend_types.go @@ -2,7 +2,7 @@ package gallery import ( "github.com/mudler/LocalAI/core/config" - "github.com/mudler/LocalAI/core/system" + "github.com/mudler/LocalAI/pkg/system" ) // BackendMetadata represents the metadata stored in a JSON file for each installed backend diff --git a/core/gallery/backends.go b/core/gallery/backends.go index e696072ed7b3..1f26c56e046e 100644 --- a/core/gallery/backends.go +++ b/core/gallery/backends.go @@ -8,9 +8,9 @@ import ( "time" "github.com/mudler/LocalAI/core/config" - "github.com/mudler/LocalAI/core/system" "github.com/mudler/LocalAI/pkg/downloader" "github.com/mudler/LocalAI/pkg/model" + "github.com/mudler/LocalAI/pkg/system" "github.com/rs/zerolog/log" ) diff --git a/core/gallery/backends_test.go b/core/gallery/backends_test.go index 3e4df40f73c9..460677706850 100644 --- a/core/gallery/backends_test.go +++ b/core/gallery/backends_test.go @@ -7,7 +7,7 @@ import ( "runtime" "github.com/mudler/LocalAI/core/config" - "github.com/mudler/LocalAI/core/system" + "github.com/mudler/LocalAI/pkg/system" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "gopkg.in/yaml.v2" diff --git a/core/gallery/models.go b/core/gallery/models.go index 30ec2908e0b2..54afbfdfdfcf 100644 --- a/core/gallery/models.go +++ b/core/gallery/models.go @@ -10,8 +10,8 @@ import ( "dario.cat/mergo" "github.com/mudler/LocalAI/core/config" lconfig "github.com/mudler/LocalAI/core/config" - "github.com/mudler/LocalAI/core/system" "github.com/mudler/LocalAI/pkg/downloader" + "github.com/mudler/LocalAI/pkg/system" "github.com/mudler/LocalAI/pkg/utils" "github.com/rs/zerolog/log" diff --git a/core/http/endpoints/openai/chat.go b/core/http/endpoints/openai/chat.go index 1e33c977b42d..f43254c15a52 100644 --- a/core/http/endpoints/openai/chat.go +++ b/core/http/endpoints/openai/chat.go @@ -15,8 +15,8 @@ import ( "github.com/mudler/LocalAI/core/schema" "github.com/mudler/LocalAI/pkg/functions" + "github.com/mudler/LocalAI/core/templates" "github.com/mudler/LocalAI/pkg/model" - "github.com/mudler/LocalAI/pkg/templates" "github.com/rs/zerolog/log" "github.com/valyala/fasthttp" @@ -175,7 +175,7 @@ func ChatEndpoint(cl *config.BackendConfigLoader, ml *model.ModelLoader, evaluat textContentToReturn = "" id = uuid.New().String() created = int(time.Now().Unix()) - + input, ok := c.Locals(middleware.CONTEXT_LOCALS_KEY_LOCALAI_REQUEST).(*schema.OpenAIRequest) if !ok || input.Model == "" { return fiber.ErrBadRequest diff --git a/core/http/endpoints/openai/completion.go b/core/http/endpoints/openai/completion.go index 9620d2c12cb2..7dec8ca7e76d 100644 --- a/core/http/endpoints/openai/completion.go +++ b/core/http/endpoints/openai/completion.go @@ -15,9 +15,9 @@ import ( "github.com/gofiber/fiber/v2" "github.com/google/uuid" "github.com/mudler/LocalAI/core/schema" + "github.com/mudler/LocalAI/core/templates" "github.com/mudler/LocalAI/pkg/functions" "github.com/mudler/LocalAI/pkg/model" - "github.com/mudler/LocalAI/pkg/templates" "github.com/rs/zerolog/log" "github.com/valyala/fasthttp" ) diff --git a/core/http/endpoints/openai/edit.go b/core/http/endpoints/openai/edit.go index c4cb5e19ca62..5b29b1c9cd27 100644 --- a/core/http/endpoints/openai/edit.go +++ b/core/http/endpoints/openai/edit.go @@ -12,8 +12,8 @@ import ( "github.com/google/uuid" "github.com/mudler/LocalAI/core/schema" + "github.com/mudler/LocalAI/core/templates" "github.com/mudler/LocalAI/pkg/model" - "github.com/mudler/LocalAI/pkg/templates" "github.com/rs/zerolog/log" ) diff --git a/core/http/endpoints/openai/realtime.go b/core/http/endpoints/openai/realtime.go index b3e756bd822f..4c4adef6f1ed 100644 --- a/core/http/endpoints/openai/realtime.go +++ b/core/http/endpoints/openai/realtime.go @@ -16,12 +16,12 @@ import ( "github.com/mudler/LocalAI/core/application" "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/http/endpoints/openai/types" + "github.com/mudler/LocalAI/core/templates" laudio "github.com/mudler/LocalAI/pkg/audio" "github.com/mudler/LocalAI/pkg/functions" "github.com/mudler/LocalAI/pkg/grpc/proto" model "github.com/mudler/LocalAI/pkg/model" "github.com/mudler/LocalAI/pkg/sound" - "github.com/mudler/LocalAI/pkg/templates" "google.golang.org/grpc" @@ -29,8 +29,8 @@ import ( ) const ( - localSampleRate = 16000 - remoteSampleRate = 24000 + localSampleRate = 16000 + remoteSampleRate = 24000 ) // A model can be "emulated" that is: transcribe audio to text -> feed text to the LLM -> generate audio as result @@ -210,9 +210,9 @@ func registerRealtime(application *application.Application) func(c *websocket.Co // TODO: Need some way to pass this to the backend Threshold: 0.5, // TODO: This is ignored and the amount of padding is random at present - PrefixPaddingMs: 30, + PrefixPaddingMs: 30, SilenceDurationMs: 500, - CreateResponse: func() *bool { t := true; return &t }(), + CreateResponse: func() *bool { t := true; return &t }(), }, }, InputAudioTranscription: &types.InputAudioTranscription{ @@ -233,7 +233,7 @@ func registerRealtime(application *application.Application) func(c *websocket.Co // TODO: The API has no way to configure the VAD model or other models that make up a pipeline to fake any-to-any // So possibly we could have a way to configure a composite model that can be used in situations where any-to-any is expected pipeline := config.Pipeline{ - VAD: "silero-vad", + VAD: "silero-vad", Transcription: session.InputAudioTranscription.Model, } @@ -567,8 +567,8 @@ func updateTransSession(session *Session, update *types.ClientSession, cl *confi trCur := session.InputAudioTranscription if trUpd != nil && trUpd.Model != "" && trUpd.Model != trCur.Model { - pipeline := config.Pipeline { - VAD: "silero-vad", + pipeline := config.Pipeline{ + VAD: "silero-vad", Transcription: trUpd.Model, } @@ -684,7 +684,7 @@ func handleVAD(cfg *config.BackendConfig, evaluator *templates.Evaluator, sessio sendEvent(c, types.InputAudioBufferClearedEvent{ ServerEventBase: types.ServerEventBase{ EventID: "event_TODO", - Type: types.ServerEventTypeInputAudioBufferCleared, + Type: types.ServerEventTypeInputAudioBufferCleared, }, }) @@ -697,7 +697,7 @@ func handleVAD(cfg *config.BackendConfig, evaluator *templates.Evaluator, sessio sendEvent(c, types.InputAudioBufferSpeechStartedEvent{ ServerEventBase: types.ServerEventBase{ EventID: "event_TODO", - Type: types.ServerEventTypeInputAudioBufferSpeechStarted, + Type: types.ServerEventTypeInputAudioBufferSpeechStarted, }, AudioStartMs: time.Now().Sub(startTime).Milliseconds(), }) @@ -719,7 +719,7 @@ func handleVAD(cfg *config.BackendConfig, evaluator *templates.Evaluator, sessio sendEvent(c, types.InputAudioBufferSpeechStoppedEvent{ ServerEventBase: types.ServerEventBase{ EventID: "event_TODO", - Type: types.ServerEventTypeInputAudioBufferSpeechStopped, + Type: types.ServerEventTypeInputAudioBufferSpeechStopped, }, AudioEndMs: time.Now().Sub(startTime).Milliseconds(), }) @@ -728,9 +728,9 @@ func handleVAD(cfg *config.BackendConfig, evaluator *templates.Evaluator, sessio sendEvent(c, types.InputAudioBufferCommittedEvent{ ServerEventBase: types.ServerEventBase{ EventID: "event_TODO", - Type: types.ServerEventTypeInputAudioBufferCommitted, + Type: types.ServerEventTypeInputAudioBufferCommitted, }, - ItemID: generateItemID(), + ItemID: generateItemID(), PreviousItemID: "TODO", }) @@ -833,9 +833,9 @@ func commitUtterance(ctx context.Context, utt []byte, cfg *config.BackendConfig, func runVAD(ctx context.Context, session *Session, adata []int16) ([]*proto.VADSegment, error) { soundIntBuffer := &audio.IntBuffer{ - Format: &audio.Format{SampleRate: localSampleRate, NumChannels: 1}, + Format: &audio.Format{SampleRate: localSampleRate, NumChannels: 1}, SourceBitDepth: 16, - Data: sound.ConvertInt16ToInt(adata), + Data: sound.ConvertInt16ToInt(adata), } float32Data := soundIntBuffer.AsFloat32Buffer().Data diff --git a/core/http/middleware/request.go b/core/http/middleware/request.go index 3a9177f517ef..dccf55a0c251 100644 --- a/core/http/middleware/request.go +++ b/core/http/middleware/request.go @@ -11,9 +11,9 @@ import ( "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/schema" "github.com/mudler/LocalAI/core/services" + "github.com/mudler/LocalAI/core/templates" "github.com/mudler/LocalAI/pkg/functions" "github.com/mudler/LocalAI/pkg/model" - "github.com/mudler/LocalAI/pkg/templates" "github.com/mudler/LocalAI/pkg/utils" "github.com/gofiber/fiber/v2" diff --git a/core/services/backends.go b/core/services/backends.go index 855d1c0adf7b..fe6446b55276 100644 --- a/core/services/backends.go +++ b/core/services/backends.go @@ -2,7 +2,7 @@ package services import ( "github.com/mudler/LocalAI/core/gallery" - "github.com/mudler/LocalAI/core/system" + "github.com/mudler/LocalAI/pkg/system" "github.com/mudler/LocalAI/pkg/utils" "github.com/rs/zerolog/log" diff --git a/core/services/gallery.go b/core/services/gallery.go index b8306b30e07a..cace4c15218e 100644 --- a/core/services/gallery.go +++ b/core/services/gallery.go @@ -7,8 +7,8 @@ import ( "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/gallery" - "github.com/mudler/LocalAI/core/system" "github.com/mudler/LocalAI/pkg/model" + "github.com/mudler/LocalAI/pkg/system" "github.com/rs/zerolog/log" ) diff --git a/core/services/models.go b/core/services/models.go index beb981c8ed25..98b2ddc8d08c 100644 --- a/core/services/models.go +++ b/core/services/models.go @@ -7,7 +7,7 @@ import ( "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/gallery" - "github.com/mudler/LocalAI/core/system" + "github.com/mudler/LocalAI/pkg/system" "github.com/mudler/LocalAI/pkg/utils" "gopkg.in/yaml.v2" ) diff --git a/pkg/startup/backend_preload.go b/core/startup/backend_preload.go similarity index 97% rename from pkg/startup/backend_preload.go rename to core/startup/backend_preload.go index 37ca911d8df7..4eb57deea78a 100644 --- a/pkg/startup/backend_preload.go +++ b/core/startup/backend_preload.go @@ -8,8 +8,8 @@ import ( "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/gallery" - "github.com/mudler/LocalAI/core/system" "github.com/mudler/LocalAI/pkg/downloader" + "github.com/mudler/LocalAI/pkg/system" "github.com/rs/zerolog/log" ) diff --git a/pkg/startup/model_preload.go b/core/startup/model_preload.go similarity index 99% rename from pkg/startup/model_preload.go rename to core/startup/model_preload.go index 1b2e88cdef3c..78214eedef75 100644 --- a/pkg/startup/model_preload.go +++ b/core/startup/model_preload.go @@ -10,8 +10,8 @@ import ( "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/gallery" - "github.com/mudler/LocalAI/core/system" "github.com/mudler/LocalAI/pkg/downloader" + "github.com/mudler/LocalAI/pkg/system" "github.com/mudler/LocalAI/pkg/utils" "github.com/rs/zerolog/log" "gopkg.in/yaml.v2" diff --git a/pkg/startup/model_preload_test.go b/core/startup/model_preload_test.go similarity index 96% rename from pkg/startup/model_preload_test.go rename to core/startup/model_preload_test.go index d8e257d7a441..b99544d28d27 100644 --- a/pkg/startup/model_preload_test.go +++ b/core/startup/model_preload_test.go @@ -6,7 +6,7 @@ import ( "path/filepath" "github.com/mudler/LocalAI/core/config" - . "github.com/mudler/LocalAI/pkg/startup" + . "github.com/mudler/LocalAI/core/startup" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" diff --git a/pkg/startup/startup_suite_test.go b/core/startup/startup_suite_test.go similarity index 100% rename from pkg/startup/startup_suite_test.go rename to core/startup/startup_suite_test.go diff --git a/pkg/templates/cache.go b/core/templates/cache.go similarity index 100% rename from pkg/templates/cache.go rename to core/templates/cache.go diff --git a/pkg/templates/evaluator.go b/core/templates/evaluator.go similarity index 100% rename from pkg/templates/evaluator.go rename to core/templates/evaluator.go diff --git a/pkg/templates/evaluator_test.go b/core/templates/evaluator_test.go similarity index 99% rename from pkg/templates/evaluator_test.go rename to core/templates/evaluator_test.go index b58dd40ba985..04ef48d6e894 100644 --- a/pkg/templates/evaluator_test.go +++ b/core/templates/evaluator_test.go @@ -3,8 +3,8 @@ package templates_test import ( "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/schema" + . "github.com/mudler/LocalAI/core/templates" "github.com/mudler/LocalAI/pkg/functions" - . "github.com/mudler/LocalAI/pkg/templates" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" diff --git a/pkg/templates/multimodal.go b/core/templates/multimodal.go similarity index 100% rename from pkg/templates/multimodal.go rename to core/templates/multimodal.go diff --git a/pkg/templates/multimodal_test.go b/core/templates/multimodal_test.go similarity index 97% rename from pkg/templates/multimodal_test.go rename to core/templates/multimodal_test.go index adea6b524568..b8c009e9d428 100644 --- a/pkg/templates/multimodal_test.go +++ b/core/templates/multimodal_test.go @@ -1,7 +1,7 @@ package templates_test import ( - . "github.com/mudler/LocalAI/pkg/templates" // Update with your module path + . "github.com/mudler/LocalAI/core/templates" // Update with your module path // Update with your module path . "github.com/onsi/ginkgo/v2" diff --git a/pkg/templates/templates_suite_test.go b/core/templates/templates_suite_test.go similarity index 100% rename from pkg/templates/templates_suite_test.go rename to core/templates/templates_suite_test.go diff --git a/docs/content/docs/features/backends.md b/docs/content/docs/features/backends.md index 83b4cc1dd4cd..2c80b91a1080 100644 --- a/docs/content/docs/features/backends.md +++ b/docs/content/docs/features/backends.md @@ -96,8 +96,8 @@ Your backend container should: For getting started, see the available backends in LocalAI here: https://github.com/mudler/LocalAI/tree/master/backend . - For Python based backends there is a template that can be used as starting point: https://github.com/mudler/LocalAI/tree/master/backend/python/common/template . -- For Golang based backends, you can see the `bark-cpp` backend as an example: https://github.com/mudler/LocalAI/tree/master/backend/go/bark -- For C++ based backends, you can see the `llama-cpp` backend as an example: https://github.com/mudler/LocalAI/tree/master/backend/cpp/llama +- For Golang based backends, you can see the `bark-cpp` backend as an example: https://github.com/mudler/LocalAI/tree/master/backend/go/bark-cpp +- For C++ based backends, you can see the `llama-cpp` backend as an example: https://github.com/mudler/LocalAI/tree/master/backend/cpp/llama-cpp ### Publishing Your Backend diff --git a/docs/content/docs/getting-started/quickstart.md b/docs/content/docs/getting-started/quickstart.md index 39acf38d18aa..faa70914ab3c 100644 --- a/docs/content/docs/getting-started/quickstart.md +++ b/docs/content/docs/getting-started/quickstart.md @@ -154,7 +154,7 @@ For instructions on using AIO images, see [Using container images]({{% relref "d LocalAI is part of the Local family stack, along with LocalAGI and LocalRecall. -[LocalAGI](https://github.com/mudler/LocalAGI) is a powerful, self-hostable AI Agent platform designed for maximum privacy and flexibility which encompassess and uses all the softwre stack. It provides a complete drop-in replacement for OpenAI's Responses APIs with advanced agentic capabilities, working entirely locally on consumer-grade hardware (CPU and GPU). +[LocalAGI](https://github.com/mudler/LocalAGI) is a powerful, self-hostable AI Agent platform designed for maximum privacy and flexibility which encompassess and uses all the software stack. It provides a complete drop-in replacement for OpenAI's Responses APIs with advanced agentic capabilities, working entirely locally on consumer-grade hardware (CPU and GPU). ### Quick Start diff --git a/core/system/capabilities.go b/pkg/system/capabilities.go similarity index 100% rename from core/system/capabilities.go rename to pkg/system/capabilities.go