diff --git a/backend/cpp/llama-cpp/Makefile b/backend/cpp/llama-cpp/Makefile index 3aac12db066c..165dc869f199 100644 --- a/backend/cpp/llama-cpp/Makefile +++ b/backend/cpp/llama-cpp/Makefile @@ -1,5 +1,5 @@ -LLAMA_VERSION?=fd1234cb468935ea087d6929b2487926c3afff4b +LLAMA_VERSION?=e725a1a982ca870404a9c4935df52466327bbd02 LLAMA_REPO?=https://github.com/ggerganov/llama.cpp CMAKE_ARGS?= diff --git a/backend/cpp/llama-cpp/grpc-server.cpp b/backend/cpp/llama-cpp/grpc-server.cpp index 535bf5d578c3..872c9edb71c0 100644 --- a/backend/cpp/llama-cpp/grpc-server.cpp +++ b/backend/cpp/llama-cpp/grpc-server.cpp @@ -313,9 +313,11 @@ static void params_parse(const backend::ModelOptions* request, params.pooling_type = LLAMA_POOLING_TYPE_RANK; } + if (request->ropescaling() == "none") { params.rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_NONE; } else if (request->ropescaling() == "yarn") { params.rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_YARN; } - else { params.rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_LINEAR; } + else if (request->ropescaling() == "linear") { params.rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_LINEAR; } + if ( request->yarnextfactor() != 0.0f ) { params.yarn_ext_factor = request->yarnextfactor(); } diff --git a/core/http/endpoints/openai/chat.go b/core/http/endpoints/openai/chat.go index f43254c15a52..0d96e04cdecc 100644 --- a/core/http/endpoints/openai/chat.go +++ b/core/http/endpoints/openai/chat.go @@ -305,7 +305,7 @@ func ChatEndpoint(cl *config.BackendConfigLoader, ml *model.ModelLoader, evaluat // If we are using the tokenizer template, we don't need to process the messages // unless we are processing functions if !config.TemplateConfig.UseTokenizerTemplate || shouldUseFn { - predInput = evaluator.TemplateMessages(input.Messages, config, funcs, shouldUseFn) + predInput = evaluator.TemplateMessages(*input, input.Messages, config, funcs, shouldUseFn) log.Debug().Msgf("Prompt (after templating): %s", predInput) if config.Grammar != "" { diff --git a/core/http/endpoints/openai/completion.go b/core/http/endpoints/openai/completion.go index 7dec8ca7e76d..654166a1c48d 100644 --- a/core/http/endpoints/openai/completion.go +++ b/core/http/endpoints/openai/completion.go @@ -109,8 +109,10 @@ func CompletionEndpoint(cl *config.BackendConfigLoader, ml *model.ModelLoader, e predInput := config.PromptStrings[0] templatedInput, err := evaluator.EvaluateTemplateForPrompt(templates.CompletionPromptTemplate, *config, templates.PromptTemplateData{ - Input: predInput, - SystemPrompt: config.SystemPrompt, + Input: predInput, + SystemPrompt: config.SystemPrompt, + ReasoningEffort: input.ReasoningEffort, + Metadata: input.Metadata, }) if err == nil { predInput = templatedInput @@ -160,8 +162,10 @@ func CompletionEndpoint(cl *config.BackendConfigLoader, ml *model.ModelLoader, e for k, i := range config.PromptStrings { templatedInput, err := evaluator.EvaluateTemplateForPrompt(templates.CompletionPromptTemplate, *config, templates.PromptTemplateData{ - SystemPrompt: config.SystemPrompt, - Input: i, + SystemPrompt: config.SystemPrompt, + Input: i, + ReasoningEffort: input.ReasoningEffort, + Metadata: input.Metadata, }) if err == nil { i = templatedInput diff --git a/core/http/endpoints/openai/edit.go b/core/http/endpoints/openai/edit.go index 5b29b1c9cd27..fbcd398d2017 100644 --- a/core/http/endpoints/openai/edit.go +++ b/core/http/endpoints/openai/edit.go @@ -47,9 +47,11 @@ func EditEndpoint(cl *config.BackendConfigLoader, ml *model.ModelLoader, evaluat for _, i := range config.InputStrings { templatedInput, err := evaluator.EvaluateTemplateForPrompt(templates.EditPromptTemplate, *config, templates.PromptTemplateData{ - Input: i, - Instruction: input.Instruction, - SystemPrompt: config.SystemPrompt, + Input: i, + Instruction: input.Instruction, + SystemPrompt: config.SystemPrompt, + ReasoningEffort: input.ReasoningEffort, + Metadata: input.Metadata, }) if err == nil { i = templatedInput diff --git a/core/schema/openai.go b/core/schema/openai.go index 44b54d188ddd..5506231e560b 100644 --- a/core/schema/openai.go +++ b/core/schema/openai.go @@ -183,6 +183,10 @@ type OpenAIRequest struct { Backend string `json:"backend" yaml:"backend"` ModelBaseName string `json:"model_base_name" yaml:"model_base_name"` + + ReasoningEffort string `json:"reasoning_effort" yaml:"reasoning_effort"` + + Metadata map[string]string `json:"metadata" yaml:"metadata"` } type ModelsDataResponse struct { diff --git a/core/templates/evaluator.go b/core/templates/evaluator.go index 78de7582ea47..f9bd313afce5 100644 --- a/core/templates/evaluator.go +++ b/core/templates/evaluator.go @@ -21,6 +21,8 @@ type PromptTemplateData struct { Instruction string Functions []functions.Function MessageIndex int + ReasoningEffort string + Metadata map[string]string } type ChatMessageTemplateData struct { @@ -133,7 +135,7 @@ func (e *Evaluator) evaluateJinjaTemplateForPrompt(templateType TemplateType, te return e.cache.evaluateJinjaTemplate(templateType, templateName, conversation) } -func (e *Evaluator) TemplateMessages(messages []schema.Message, config *config.BackendConfig, funcs []functions.Function, shouldUseFn bool) string { +func (e *Evaluator) TemplateMessages(input schema.OpenAIRequest, messages []schema.Message, config *config.BackendConfig, funcs []functions.Function, shouldUseFn bool) string { if config.TemplateConfig.JinjaTemplate { var messageData []ChatMessageTemplateData @@ -283,6 +285,8 @@ func (e *Evaluator) TemplateMessages(messages []schema.Message, config *config.B SuppressSystemPrompt: suppressConfigSystemPrompt, Input: predInput, Functions: funcs, + ReasoningEffort: input.ReasoningEffort, + Metadata: input.Metadata, }) if err == nil { predInput = templatedInput diff --git a/core/templates/evaluator_test.go b/core/templates/evaluator_test.go index 04ef48d6e894..41c17e6a517d 100644 --- a/core/templates/evaluator_test.go +++ b/core/templates/evaluator_test.go @@ -219,7 +219,7 @@ var _ = Describe("Templates", func() { for key := range chatMLTestMatch { foo := chatMLTestMatch[key] It("renders correctly `"+key+"`", func() { - templated := evaluator.TemplateMessages(foo["messages"].([]schema.Message), foo["config"].(*config.BackendConfig), foo["functions"].([]functions.Function), foo["shouldUseFn"].(bool)) + templated := evaluator.TemplateMessages(schema.OpenAIRequest{}, foo["messages"].([]schema.Message), foo["config"].(*config.BackendConfig), foo["functions"].([]functions.Function), foo["shouldUseFn"].(bool)) Expect(templated).To(Equal(foo["expected"]), templated) }) } @@ -232,7 +232,7 @@ var _ = Describe("Templates", func() { for key := range llama3TestMatch { foo := llama3TestMatch[key] It("renders correctly `"+key+"`", func() { - templated := evaluator.TemplateMessages(foo["messages"].([]schema.Message), foo["config"].(*config.BackendConfig), foo["functions"].([]functions.Function), foo["shouldUseFn"].(bool)) + templated := evaluator.TemplateMessages(schema.OpenAIRequest{}, foo["messages"].([]schema.Message), foo["config"].(*config.BackendConfig), foo["functions"].([]functions.Function), foo["shouldUseFn"].(bool)) Expect(templated).To(Equal(foo["expected"]), templated) }) } @@ -245,7 +245,7 @@ var _ = Describe("Templates", func() { for key := range jinjaTest { foo := jinjaTest[key] It("renders correctly `"+key+"`", func() { - templated := evaluator.TemplateMessages(foo["messages"].([]schema.Message), foo["config"].(*config.BackendConfig), foo["functions"].([]functions.Function), foo["shouldUseFn"].(bool)) + templated := evaluator.TemplateMessages(schema.OpenAIRequest{}, foo["messages"].([]schema.Message), foo["config"].(*config.BackendConfig), foo["functions"].([]functions.Function), foo["shouldUseFn"].(bool)) Expect(templated).To(Equal(foo["expected"]), templated) }) } diff --git a/gallery/harmony.yaml b/gallery/harmony.yaml index a8b9b7eeeed7..2fe84750a2fa 100644 --- a/gallery/harmony.yaml +++ b/gallery/harmony.yaml @@ -5,23 +5,22 @@ config_file: | mmap: true backend: "llama-cpp" template: - chat_message: | - <|start|>{{ if .FunctionCall -}}functions.{{ .FunctionCall.Name }} to=assistant{{ else if eq .RoleName "assistant"}}<|channel|>final<|message|>{{else}}{{ .RoleName }}{{end}}<|message|> - {{ if .Content -}} - {{.Content }} - {{ end -}} - {{ if .FunctionCall -}} - {{toJson .FunctionCall}} - {{ end -}}<|end|> - function: | - <|im_start|>system - You are a function calling AI model. You are provided with functions to execute. You may call one or more functions to assist with the user query. Don't make assumptions about what values to plug into functions. Here are the available tools: - + chat_message: |- + <|start|>{{ if .FunctionCall -}}functions.{{ .FunctionCall.Name }} to=assistant{{ else if eq .RoleName "assistant"}}assistant<|channel|>final<|message|>{{else}}{{ .RoleName }}{{end}}<|message|> + {{- if .Content -}} + {{- .Content -}} + {{- end -}} + {{- if .FunctionCall -}} + {{- toJson .FunctionCall -}} + {{- end -}}<|end|> + function: |- <|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI. Knowledge cutoff: 2024-06 - Current date: {{ now | date }} + Current date: {{ now | date "Mon Jan 2 15:04:05 MST 2006" }} + + Reasoning: {{if eq .ReasoningEffort ""}}medium{{else}}{{.ReasoningEffort}}{{end}} - Reasoning: medium + # {{with .Metadata}}{{ if ne .system_prompt "" }}{{ .system_prompt }}{{ end }}{{else}}You are a friendly and helpful assistant.{{ end }}<|end|>{{- .Input -}}<|start|>assistant # Tools @@ -49,21 +48,15 @@ config_file: | # Instructions - <|end|> - {{.Input -}} - <|start|>assistant - chat: | + <|end|>{{.Input -}}<|start|>assistant + chat: |- <|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI. Knowledge cutoff: 2024-06 - Current date: {{ now | date }} + Current date: {{ now | date "Mon Jan 2 15:04:05 MST 2006" }} - Reasoning: medium - - # Instructions + Reasoning: {{if eq .ReasoningEffort ""}}medium{{else}}{{.ReasoningEffort}}{{end}} - <|end|> - {{.Input -}} - <|im_start|>assistant + # {{with .Metadata}}{{ if ne .system_prompt "" }}{{ .system_prompt }}{{ end }}{{else}}You are a friendly and helpful assistant.{{ end }}<|end|>{{- .Input -}}<|start|>assistant completion: | {{.Input}} context_size: 8192 diff --git a/gallery/index.yaml b/gallery/index.yaml index 7d53b5dbc9e6..04be07934ef2 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -127,6 +127,30 @@ - filename: gpt-oss-120b-mxfp4-00003-of-00003.gguf sha256: b326bfd8ac696c4b9a14e9e84d5529b2bb86847aea0e65443cbf075accba8b71 uri: huggingface://ggml-org/gpt-oss-120b-GGUF/gpt-oss-120b-mxfp4-00003-of-00003.gguf +- !!merge <<: *gptoss + name: "openai_gpt-oss-20b-neo" + icon: https://huggingface.co/DavidAU/Openai_gpt-oss-20b-NEO-GGUF/resolve/main/matrix1.gif + urls: + - https://huggingface.co/DavidAU/Openai_gpt-oss-20b-NEO-GGUF + description: | + These are NEO Imatrix GGUFs, NEO dataset by DavidAU. + + NEO dataset improves overall performance, and is for all use cases. + + Example output below (creative), using settings below. + + Model also passed "hard" coding test too (6 experts); no issues (IQ4_NL). + + (Forcing the model to create code with no dependencies and limits of coding short cuts, with multiple loops, and in real time with no blocking in a language that does not support it normally.) + + Due to quanting issues with this model (which result in oddball quant sizes / mixtures), only TESTED quants will be uploaded (at the moment). + overrides: + parameters: + model: OpenAI-20B-NEO-MXFP4_MOE4.gguf + files: + - filename: OpenAI-20B-NEO-MXFP4_MOE4.gguf + sha256: 066c84a0844b1f1f4515e5c64095fe4c67e86d5eb70db4e368e283b1134d9c1e + uri: huggingface://DavidAU/Openai_gpt-oss-20b-NEO-GGUF/OpenAI-20B-NEO-MXFP4_MOE4.gguf - &afm name: "arcee-ai_afm-4.5b" url: "github:mudler/LocalAI/gallery/chatml.yaml@master" diff --git a/swagger/docs.go b/swagger/docs.go index 132cd4f342f2..4c6d9b2b2d5a 100644 --- a/swagger/docs.go +++ b/swagger/docs.go @@ -1535,6 +1535,12 @@ const docTemplate = `{ "$ref": "#/definitions/schema.Message" } }, + "metadata": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, "mode": { "description": "Image (not supported by OpenAI)", "type": "integer" @@ -1567,6 +1573,9 @@ const docTemplate = `{ "quality": { "type": "string" }, + "reasoning_effort": { + "type": "string" + }, "ref_images": { "description": "Reference images for models that support them (e.g., Flux Kontext)", "type": "array", diff --git a/swagger/swagger.json b/swagger/swagger.json index 1f4e2ce93bde..4b46f78ea51c 100644 --- a/swagger/swagger.json +++ b/swagger/swagger.json @@ -1528,6 +1528,12 @@ "$ref": "#/definitions/schema.Message" } }, + "metadata": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, "mode": { "description": "Image (not supported by OpenAI)", "type": "integer" @@ -1560,6 +1566,9 @@ "quality": { "type": "string" }, + "reasoning_effort": { + "type": "string" + }, "ref_images": { "description": "Reference images for models that support them (e.g., Flux Kontext)", "type": "array", diff --git a/swagger/swagger.yaml b/swagger/swagger.yaml index 63310fff5858..8e01f3f97066 100644 --- a/swagger/swagger.yaml +++ b/swagger/swagger.yaml @@ -491,6 +491,10 @@ definitions: items: $ref: '#/definitions/schema.Message' type: array + metadata: + additionalProperties: + type: string + type: object mode: description: Image (not supported by OpenAI) type: integer @@ -514,6 +518,8 @@ definitions: description: Prompt is read only by completion/image API calls quality: type: string + reasoning_effort: + type: string ref_images: description: Reference images for models that support them (e.g., Flux Kontext) items: