Merge pull request #52 from BalanceBalls/images-support

Images support

Author: BalanceBalls

README.md

@@ -14,6 +14,8 @@ A tool for those who appreciate keyboard driven apps and terminal workflows
  * 📦 **Single binary** - lightweight, zero dependencies, use anywhere without any requirements
  * 🤖 **Support for OpenAI compatible APIs** (ChatGPT, Mistral, Ollama, LMStudio, llama-cpp and more)
  * 🌟 **Support for Gemini API**
+ * 🔀 **Support for OpenRouter API**
+ * 🖼️ **Images support**
  * 💬 **Chat sessions** management and quick chats
  * ⚙️ **Settings presets** (configure different personas with unique settings)
  * ✂️ **Convenient text selection** tool (vim-like line selection)
@@ -70,7 +72,7 @@ bash -c "$(curl -fsSL https://raw.githubusercontent.com/BalanceBalls/nekot/main/
 
 ## Setting API keys
 
-To use the app, you will need to set `OPENAI_API_KEY` or/and `GEMINI_API_KEY` env variables depending on your needs
+To use the app, you will need to set `OPENAI_API_KEY` or/and `GEMINI_API_KEY`, `OPENROUTER_API_KEY` env variables depending on your needs
 
 <details>
 
@@ -95,6 +97,14 @@ Set up your api key - [how to get an api key](https://aistudio.google.com/apikey
 ```bash
 export  GEMINI_API_KEY="some-key" # you would want to export this in your .zshrc or .bashrc
 ```
+
+### OpenRouter API
+
+Set up your api key - [how to get an api key](https://openrouter.ai/docs/api-reference/authentication)
+
+```bash
+export  OPENROUTER_API_KEY="some-key" # you would want to export this in your .zshrc or .bashrc
+```
 </details>
 
 ## Config
@@ -110,14 +120,18 @@ We provide a `config.json` file within your directory for easy access to essenti
   "systemMessage": "",
   "defaultModel": "",
   "colorScheme": "groove", // pink, blue, groove
-  "provider": "openai" // openai, gemini
+  "provider": "openai", // openai, gemini, openrouter
+  "maxAttachmentSizeMb": 3,
+  "includeReasoningTokensInContext": true
 }
 ```
 
  - `providerBaseUrl`: The url can be anything that follows OpenAI API standard ( [ollama](http://localhost:11434), [lmstudio](http://127.0.0.1:1234), etc)
  - `chatGPTApiUrl` [obsolete]: same as `providerBaseUrl`
  - `systemMessage` field is available for customizing system prompt messages. **Better to set it from the app**
  - `defaultModel` field sets the default model.  **Better to set it from the app**
+ - `maxAttachmentSizeMb` field sets maximum allowed image size
+ - `includeReasoningTokensInContext` field sets whether to include reasoning tokens in the next request or not.
 
 ### Providers
 
@@ -126,9 +140,10 @@ You can change API provider using the `provider` field.
 Available providers:
  * `openai` **default**
  * `gemini`
+ * `openrouter`
 
 To use **GeminiAPI**, just set `"provider": "gemini"` (make sure to set GEMINI_API_KEY env variable).
-When using the `gemini` provider, `providerBaseUrl` param is not used.
+When using the `gemini` or `openrouter` providers, `providerBaseUrl` param is not used.
 
 ### Themes
 You can change colorscheme using the `colorScheme` field.
@@ -159,6 +174,7 @@ To switch between openai and gemini APIs you can use `-p` flag:
 ```bash
 nekot -p openai
 nekot -p gemini
+nekot -p openrouter
 ```
 
 ### Provider url
@@ -201,6 +217,7 @@ nekot -t blue
     \```
 - `esc`: Exit insert mode for the prompt
     * When in 'Prompt editor' mode, pressing `esc` second time will close editor
+- `ctrl+a`: open file picker for attaching images. You can also attach images by typing: [img=/path/to/image]
 
 ## Chat Messages Pane
 
@@ -284,6 +301,7 @@ rc-nekot
   nice terminal layouts!
 - [bubbles](https://github.com/charmbracelet/bubbles): Some general use
   components for Bubble Tea apps!
+- [operouter-sdk](https://github.com/reVrost/go-openrouter) openrouter api sdk
 
 ## Contributors
 

clients/gemini.go

@@ -2,9 +2,11 @@ package clients
 
 import (
 	"context"
+	"encoding/base64"
 	"errors"
 	"fmt"
 	"os"
+	"path/filepath"
 	"strings"
 
 	"github.com/BalanceBalls/nekot/config"
@@ -36,7 +38,7 @@ func NewGeminiClient(systemMessage string) *GeminiClient {
 
 func (c GeminiClient) RequestCompletion(
 	ctx context.Context,
-	chatMsgs []util.MessageToSend,
+	chatMsgs []util.LocalStoreMessage,
 	modelSettings util.Settings,
 	resultChan chan util.ProcessApiCompletionResponse,
 ) tea.Cmd {
@@ -50,7 +52,8 @@ func (c GeminiClient) RequestCompletion(
 
 		client, err := genai.NewClient(ctx, option.WithAPIKey(os.Getenv("GEMINI_API_KEY")))
 		if err != nil {
-			return util.MakeErrorMsg(err.Error())
+			resultChan <- util.ProcessApiCompletionResponse{ID: util.ChunkIndexStart, Err: err, Final: true}
+			return nil
 		}
 		defer client.Close()
 
@@ -61,7 +64,10 @@ func (c GeminiClient) RequestCompletion(
 
 		currentPrompt := chatMsgs[len(chatMsgs)-1].Content
 		cs := model.StartChat()
-		cs.History = buildChatHistory(chatMsgs)
+		cs.History, err = buildChatHistory(chatMsgs, *config.IncludeReasoningTokensInContext)
+		if err != nil {
+			return util.MakeErrorMsg(err.Error())
+		}
 
 		iter := cs.SendMessageStream(ctx, genai.Text(currentPrompt))
 
@@ -133,7 +139,7 @@ func (c GeminiClient) RequestModelsList(ctx context.Context) util.ProcessModelsR
 	modelsIter := client.ListModels(ctx)
 
 	if ctx.Err() == context.DeadlineExceeded {
-		return util.ProcessModelsResponse{Err: errors.New("Timedout during fetching models")}
+		return util.ProcessModelsResponse{Err: errors.New("timed out during fetching models")}
 	}
 
 	var modelsList []util.ModelDescription
@@ -319,7 +325,7 @@ func handleFinishReason(reason genai.FinishReason) (string, error) {
 	return "", nil
 }
 
-func buildChatHistory(msgs []util.MessageToSend) []*genai.Content {
+func buildChatHistory(msgs []util.LocalStoreMessage, includeReasoning bool) ([]*genai.Content, error) {
 	chat := []*genai.Content{}
 
 	for _, singleMessage := range msgs {
@@ -329,7 +335,7 @@ func buildChatHistory(msgs []util.MessageToSend) []*genai.Content {
 		}
 
 		messageContent := ""
-		if singleMessage.Resoning != "" {
+		if singleMessage.Resoning != "" && includeReasoning {
 			messageContent += singleMessage.Resoning
 		}
 		if singleMessage.Content != "" {
@@ -343,11 +349,27 @@ func buildChatHistory(msgs []util.MessageToSend) []*genai.Content {
 				},
 				Role: role,
 			}
+
+			if len(singleMessage.Attachments) != 0 {
+				for _, item := range singleMessage.Attachments {
+					decodedBytes, err := base64.StdEncoding.DecodeString(item.Content)
+
+					if err != nil {
+						util.Slog.Error("failed to decode file bytes", "item", item.Path, "error", err.Error())
+						return nil, errors.New("could not prepare attachments for request")
+					}
+
+					extension := filepath.Ext(item.Path)
+					extension = strings.TrimPrefix(extension, ".")
+					part := genai.ImageData(extension, decodedBytes)
+					message.Parts = append(message.Parts, part)
+				}
+			}
 			chat = append(chat, &message)
 		}
 
 		util.Slog.Debug("constructed turn", "data", messageContent)
 	}
 
-	return chat
+	return chat, nil
 }

clients/openai.go

@@ -10,6 +10,7 @@ import (
 	"net/http"
 	"net/url"
 	"os"
+	"path/filepath"
 	"strings"
 
 	"github.com/BalanceBalls/nekot/config"
@@ -36,7 +37,7 @@ func NewOpenAiClient(apiUrl, systemMessage string) *OpenAiClient {
 
 func (c OpenAiClient) RequestCompletion(
 	ctx context.Context,
-	chatMsgs []util.MessageToSend,
+	chatMsgs []util.LocalStoreMessage,
 	modelSettings util.Settings,
 	resultChan chan util.ProcessApiCompletionResponse,
 ) tea.Cmd {
@@ -80,22 +81,54 @@ func (c OpenAiClient) RequestModelsList(ctx context.Context) util.ProcessModelsR
 	return processModelsListResponse(resp)
 }
 
-func ConstructUserMessage(content string) util.OpenAIConversationTurn {
+func constructUserMessage(msg util.LocalStoreMessage) util.OpenAIConversationTurn {
+	content := []util.OpenAiContent{
+		{
+			Type: "text",
+			Text: msg.Content,
+		},
+	}
+
+	if len(msg.Attachments) != 0 {
+		for _, attachment := range msg.Attachments {
+			data := getImageURLString(attachment)
+			image := util.OpenAiContent{
+				Type: "image_url",
+				ImageURL: util.OpenAiImage{
+					URL: data,
+				},
+			}
+			content = append(content, image)
+		}
+	}
+
 	return util.OpenAIConversationTurn{
-		Role:    "user",
+		Role:    msg.Role,
 		Content: content,
 	}
 }
 
+func getImageURLString(attachment util.Attachment) string {
+	extension := filepath.Ext(attachment.Path)
+	extension = strings.TrimPrefix(extension, ".")
+	content := "data:image/" + extension + ";base64," + attachment.Content
+	return content
+}
+
 func constructSystemMessage(content string) util.OpenAIConversationTurn {
 	return util.OpenAIConversationTurn{
-		Role:    "system",
-		Content: content,
+		Role: "system",
+		Content: []util.OpenAiContent{
+			{
+				Type: "text",
+				Text: content,
+			},
+		},
 	}
 }
 
 func (c OpenAiClient) constructCompletionRequestPayload(
-	chatMsgs []util.MessageToSend,
+	chatMsgs []util.LocalStoreMessage,
 	cfg config.Config,
 	settings util.Settings,
 ) ([]byte, error) {
@@ -114,7 +147,7 @@ func (c OpenAiClient) constructCompletionRequestPayload(
 
 	for _, singleMessage := range chatMsgs {
 		messageContent := ""
-		if singleMessage.Resoning != "" {
+		if singleMessage.Resoning != "" && *cfg.IncludeReasoningTokensInContext {
 			messageContent += singleMessage.Resoning
 		}
 
@@ -123,13 +156,8 @@ func (c OpenAiClient) constructCompletionRequestPayload(
 		}
 
 		if messageContent != "" {
-			conversationTurn := util.OpenAIConversationTurn{
-				Model:   singleMessage.Model,
-				Role:    singleMessage.Role,
-				Content: messageContent,
-			}
-
-			util.Slog.Debug("constructed turn", "data", conversationTurn)
+			singleMessage.Content = messageContent
+			conversationTurn := constructUserMessage(singleMessage)
 			messages = append(messages, conversationTurn)
 		}
 	}