diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs index 733528922..b86e59161 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs @@ -13,6 +13,7 @@ public interface IAgentService Task RefreshAgents(); Task> GetAgents(AgentFilter filter); Task> GetAgentOptions(List? agentIds = null, bool byName = false); + Task> GetAgentUtilityOptions(); /// /// Load agent configurations and trigger hooks diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/Models/AgentUtility.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/Models/AgentUtility.cs index 6233dfa73..3dc5247d4 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Agents/Models/AgentUtility.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/Models/AgentUtility.cs @@ -26,8 +26,7 @@ public override string ToString() public class UtilityItem { [JsonPropertyName("function_name")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? FunctionName { get; set; } + public string FunctionName { get; set; } = null!; [JsonPropertyName("template_name")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] @@ -36,4 +35,8 @@ public class UtilityItem [JsonPropertyName("visibility_expression")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? VisibilityExpression { get; set; } + + [JsonPropertyName("description")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Description { get; set; } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs index 8e255e763..ef40e11c3 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs @@ -7,9 +7,9 @@ public interface IFileStorageService #region Common string GetDirectory(string conversationId); IEnumerable GetFiles(string relativePath, string? searchQuery = null); - byte[] GetFileBytes(string fileStorageUrl); + BinaryData GetFileBytes(string fileStorageUrl); bool SaveFileStreamToPath(string filePath, Stream stream); - bool SaveFileBytesToPath(string filePath, byte[] bytes); + bool SaveFileBytesToPath(string filePath, BinaryData binary); string GetParentDir(string dir, int level = 1); bool ExistDirectory(string? dir); void CreateDirectory(string dir); diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/InstructFileModel.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/InstructFileModel.cs index 7eaddd5af..4d04a2fc7 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/InstructFileModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/InstructFileModel.cs @@ -3,16 +3,23 @@ namespace BotSharp.Abstraction.Files.Models; public class InstructFileModel : FileBase { /// - /// File extension without dot + /// File extension /// [JsonPropertyName("file_extension")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? FileExtension { get; set; } = string.Empty; /// - /// External file url + /// File url /// [JsonPropertyName("file_url")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? FileUrl { get; set; } = string.Empty; + + /// + /// File url + /// + [JsonPropertyName("content_type")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? ContentType { get; set; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs index db9ccb65c..9dc67fd54 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs @@ -11,11 +11,16 @@ public static class FileUtility /// /// /// - public static (string, byte[]) GetFileInfoFromData(string data) + public static (string?, BinaryData) GetFileInfoFromData(string data) { if (string.IsNullOrEmpty(data)) { - return (string.Empty, new byte[0]); + return (null, BinaryData.Empty); + } + + if (!data.StartsWith("data:")) + { + return (null, BinaryData.FromBytes(Convert.FromBase64String(data))); } var typeStartIdx = data.IndexOf(':'); @@ -25,13 +30,13 @@ public static (string, byte[]) GetFileInfoFromData(string data) var base64startIdx = data.IndexOf(','); var base64Str = data.Substring(base64startIdx + 1); - return (contentType, Convert.FromBase64String(base64Str)); + return (contentType, BinaryData.FromBytes(Convert.FromBase64String(base64Str))); } - public static string BuildFileDataFromFile(string fileName, byte[] bytes) + public static string BuildFileDataFromFile(string fileName, BinaryData binary) { var contentType = GetFileContentType(fileName); - var base64 = Convert.ToBase64String(bytes); + var base64 = Convert.ToBase64String(binary.ToArray()); return $"data:{contentType};base64,{base64}"; } diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.Utility.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.Utility.cs new file mode 100644 index 000000000..7537c80ae --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.Utility.cs @@ -0,0 +1,101 @@ +using BotSharp.Abstraction.Repositories.Settings; +using System.IO; + +namespace BotSharp.Core.Agents.Services; + +public partial class AgentService +{ + public async Task> GetAgentUtilityOptions() + { + var utilities = new List(); + var hooks = _services.GetServices(); + foreach (var hook in hooks) + { + hook.AddUtilities(utilities); + } + + utilities = utilities.Where(x => !string.IsNullOrWhiteSpace(x.Category) + && !string.IsNullOrWhiteSpace(x.Name) + && !x.Items.IsNullOrEmpty()).ToList(); + + var allItems = utilities.SelectMany(x => x.Items).ToList(); + var functionNames = allItems.Select(x => x.FunctionName).Distinct().ToList(); + var mapper = await GetAgentDocs(functionNames); + + allItems.ForEach(x => + { + if (mapper.ContainsKey(x.FunctionName)) + { + x.Description = mapper[x.FunctionName]; + } + }); + + return utilities; + } + + #region Private methods + private async ValueTask> GetAgentDocs(IEnumerable names) + { + var mapper = new Dictionary(); + if (names.IsNullOrEmpty()) + { + return mapper; + } + + var dir = GetAgentDocDir(BuiltInAgentId.UtilityAssistant); + if (string.IsNullOrEmpty(dir)) + { + return mapper; + } + + var matchDocs = Directory.GetFiles(dir, "*.md") + .Where(x => names.Contains(Path.GetFileNameWithoutExtension(x))) + .ToList(); + + if (matchDocs.IsNullOrEmpty()) + { + return mapper; + } + + await foreach (var item in GetUtilityDescriptions(matchDocs)) + { + mapper[item.Key] = item.Value; + } + + return mapper; + } + + private string GetAgentDocDir(string agentId) + { + var dbSettings = _services.GetRequiredService(); + var agentSettings = _services.GetRequiredService(); + var dir = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, dbSettings.FileRepository, agentSettings.DataDir, agentId, "docs"); + if (!Directory.Exists(dir)) + { + dir = string.Empty; + } + return dir; + } + + private async IAsyncEnumerable> GetUtilityDescriptions(IEnumerable docs) + { + foreach (var doc in docs) + { + var content = string.Empty; + try + { + content = await File.ReadAllTextAsync(doc); + } + catch { } + + if (string.IsNullOrWhiteSpace(content)) + { + continue; + } + + var fileName = Path.GetFileNameWithoutExtension(doc); + yield return new KeyValuePair(fileName, content); + } + } + #endregion +} diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs index e4d8f16d5..c86651795 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs @@ -145,11 +145,7 @@ public IConversationStateService SetState(string name, T value, bool isNeedVe newPair.Values = new List { newValue }; _curStates[name] = newPair; } - else if (isNoChange) - { - // do nothing - } - else + else if (!isNoChange) { _curStates[name].Values.Add(newValue); } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs index d61b61650..339cb0495 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs @@ -1,5 +1,4 @@ using BotSharp.Abstraction.Instructs.Models; -using System.IO; namespace BotSharp.Core.Files.Services; @@ -14,12 +13,11 @@ public async Task SpeechToText(InstructFileModel audio, string? text = n } var completion = CompletionProvider.GetAudioTranscriber(_services, provider: options?.Provider, model: options?.Model); - var audioBytes = await DownloadFile(audio); - using var stream = new MemoryStream(); - stream.Write(audioBytes, 0, audioBytes.Length); + var audioBinary = await DownloadFile(audio); + using var stream = audioBinary.ToStream(); stream.Position = 0; - var fileName = $"{audio.FileName ?? "audio"}.{audio.FileExtension ?? "wav"}"; + var fileName = BuildFileName(audio.FileName, audio.FileExtension, "audio", "wav"); var content = await completion.TranscriptTextAsync(stream, fileName, text ?? string.Empty); stream.Close(); return content; diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs index dd2e02d40..b2067e633 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs @@ -1,7 +1,5 @@ using BotSharp.Abstraction.Instructs.Models; using BotSharp.Abstraction.Instructs; -using System.IO; -using BotSharp.Abstraction.Infrastructures; namespace BotSharp.Core.Files.Services; @@ -21,7 +19,12 @@ public async Task ReadImages(string text, IEnumerable { new RoleDialogModel(AgentRole.User, text) { - Files = images?.Select(x => new BotSharpFile { FileUrl = x.FileUrl, FileData = x.FileData }).ToList() ?? [] + Files = images?.Select(x => new BotSharpFile + { + FileUrl = x.FileUrl, + FileData = x.FileData, + ContentType = x.ContentType + }).ToList() ?? [] } }); @@ -76,12 +79,11 @@ public async Task VaryImage(InstructFileModel image, InstructOp var innerAgentId = options?.AgentId ?? Guid.Empty.ToString(); var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "dall-e-2"); - var bytes = await DownloadFile(image); - using var stream = new MemoryStream(); - stream.Write(bytes, 0, bytes.Length); + var binary = await DownloadFile(image); + using var stream = binary.ToStream(); stream.Position = 0; - var fileName = $"{image.FileName ?? "image"}.{image.FileExtension ?? "png"}"; + var fileName = BuildFileName(image.FileName, image.FileExtension, "image", "png"); var message = await completion.GetImageVariation(new Agent() { Id = innerAgentId @@ -113,12 +115,11 @@ public async Task EditImage(string text, InstructFileModel imag var instruction = await GetAgentTemplate(innerAgentId, options?.TemplateName); var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "dall-e-2"); - var bytes = await DownloadFile(image); - using var stream = new MemoryStream(); - stream.Write(bytes, 0, bytes.Length); + var binary = await DownloadFile(image); + using var stream = binary.ToStream(); stream.Position = 0; - var fileName = $"{image.FileName ?? "image"}.{image.FileExtension ?? "png"}"; + var fileName = BuildFileName(image.FileName, image.FileExtension, "image", "png"); var message = await completion.GetImageEdits(new Agent() { Id = innerAgentId @@ -153,19 +154,17 @@ public async Task EditImage(string text, InstructFileModel imag var instruction = await GetAgentTemplate(innerAgentId, options?.TemplateName); var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "dall-e-2"); - var imageBytes = await DownloadFile(image); - var maskBytes = await DownloadFile(mask); + var imageBinary = await DownloadFile(image); + var maskBinary = await DownloadFile(mask); - using var imageStream = new MemoryStream(); - imageStream.Write(imageBytes, 0, imageBytes.Length); + using var imageStream = imageBinary.ToStream(); imageStream.Position = 0; - using var maskStream = new MemoryStream(); - maskStream.Write(maskBytes, 0, maskBytes.Length); + using var maskStream = maskBinary.ToStream(); maskStream.Position = 0; - var imageName = $"{image.FileName ?? "image"}.{image.FileExtension ?? "png"}"; - var maskName = $"{mask.FileName ?? "mask"}.{mask.FileExtension ?? "png"}"; + var imageName = BuildFileName(image.FileName, image.FileExtension, "image", "png"); + var maskName = BuildFileName(image.FileName, image.FileExtension, "mask", "png"); var message = await completion.GetImageEdits(new Agent() { Id = innerAgentId diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs index b593e9497..04e31092f 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs @@ -22,7 +22,7 @@ public async Task ReadPdf(string text, List files, In try { var provider = options?.Provider ?? "openai"; - var pdfFiles = await DownloadFiles(sessionDir, files); + var pdfFiles = await DownloadAndSaveFiles(sessionDir, files); var targetFiles = pdfFiles; if (provider != "google-ai") @@ -78,44 +78,38 @@ await hook.OnResponseGenerated(new InstructResponseModel } #region Private methods - private async Task> DownloadFiles(string dir, List files, string extension = "pdf") + private async Task> DownloadAndSaveFiles(string dir, List files, string extension = "pdf") { if (string.IsNullOrWhiteSpace(dir) || files.IsNullOrEmpty()) { return Enumerable.Empty(); } + var downloadTasks = files.Select(x => DownloadFile(x)); + await Task.WhenAll(downloadTasks); + var locs = new List(); - foreach (var file in files) + for (int i = 0; i < files.Count; i++) { - try + var binary = downloadTasks.ElementAt(i).Result; + if (binary == null || binary.IsEmpty) { - var bytes = new byte[0]; - if (!string.IsNullOrEmpty(file.FileUrl)) - { - var http = _services.GetRequiredService(); - using var client = http.CreateClient(); - bytes = await client.GetByteArrayAsync(file.FileUrl); - } - else if (!string.IsNullOrEmpty(file.FileData)) - { - (_, bytes) = FileUtility.GetFileInfoFromData(file.FileData); - } + continue; + } - if (!bytes.IsNullOrEmpty()) - { - var guid = Guid.NewGuid().ToString(); - var fileDir = _fileStorage.BuildDirectory(dir, guid); - DeleteIfExistDirectory(fileDir, true); + try + { + var guid = Guid.NewGuid().ToString(); + var fileDir = _fileStorage.BuildDirectory(dir, guid); + DeleteIfExistDirectory(fileDir, createNew: true); - var outputDir = _fileStorage.BuildDirectory(fileDir, $"{guid}.{extension}"); - _fileStorage.SaveFileBytesToPath(outputDir, bytes); - locs.Add(outputDir); - } + var outputDir = _fileStorage.BuildDirectory(fileDir, $"{guid}.{extension}"); + _fileStorage.SaveFileBytesToPath(outputDir, binary); + locs.Add(outputDir); } catch (Exception ex) { - _logger.LogWarning(ex, $"Error when saving pdf file."); + _logger.LogWarning(ex, $"Error when saving #{i + 1} {extension} file."); continue; } } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs index 9ff3b46a7..25deecd0a 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs @@ -1,4 +1,6 @@ +using static System.Net.Mime.MediaTypeNames; + namespace BotSharp.Core.Files.Services; public partial class FileInstructService : IFileInstructService @@ -32,21 +34,31 @@ private void DeleteIfExistDirectory(string? dir, bool createNew = false) } } - private async Task DownloadFile(InstructFileModel file) + private async Task DownloadFile(InstructFileModel file) { - var bytes = new byte[0]; - if (!string.IsNullOrEmpty(file.FileUrl)) + var binary = BinaryData.Empty; + + try { - var http = _services.GetRequiredService(); - using var client = http.CreateClient(); - bytes = await client.GetByteArrayAsync(file.FileUrl); + if (!string.IsNullOrEmpty(file.FileUrl)) + { + var http = _services.GetRequiredService(); + using var client = http.CreateClient(); + var bytes = await client.GetByteArrayAsync(file.FileUrl); + binary = BinaryData.FromBytes(bytes); + } + else if (!string.IsNullOrEmpty(file.FileData)) + { + (_, binary) = FileUtility.GetFileInfoFromData(file.FileData); + } + + return binary; } - else if (!string.IsNullOrEmpty(file.FileData)) + catch (Exception ex) { - (_, bytes) = FileUtility.GetFileInfoFromData(file.FileData); + _logger.LogWarning(ex, $"Error when downloading file {file.FileUrl}"); + return binary; } - - return bytes; } private async Task GetAgentTemplate(string agentId, string? templateName) @@ -66,5 +78,13 @@ private async Task DownloadFile(InstructFileModel file) var instruction = agentService.RenderedTemplate(agent, templateName); return instruction; } + + private string BuildFileName(string? name, string? extension, string defaultName, string defaultExtension) + { + var fname = name.IfNullOrEmptyAs(defaultName); + var fextension = extension.IfNullOrEmptyAs(defaultExtension); + fextension = fextension.StartsWith(".") ? fextension.Substring(1) : fextension; + return $"{name}.{fextension}"; + } #endregion } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Common.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Common.cs index b16c2c2cf..80a10deeb 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Common.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Common.cs @@ -30,12 +30,12 @@ public IEnumerable GetFiles(string relativePath, string? searchPattern = return Directory.GetFiles(path); } - public byte[] GetFileBytes(string fileStorageUrl) + public BinaryData GetFileBytes(string fileStorageUrl) { using var stream = File.OpenRead(fileStorageUrl); var bytes = new byte[stream.Length]; stream.Read(bytes, 0, (int)stream.Length); - return bytes; + return BinaryData.FromBytes(bytes); } public bool SaveFileStreamToPath(string filePath, Stream stream) @@ -49,11 +49,11 @@ public bool SaveFileStreamToPath(string filePath, Stream stream) return true; } - public bool SaveFileBytesToPath(string filePath, byte[] bytes) + public bool SaveFileBytesToPath(string filePath, BinaryData binary) { using (var fs = new FileStream(filePath, FileMode.Create)) { - fs.Write(bytes, 0, bytes.Length); + fs.Write(binary.ToArray(), 0, binary.Length); fs.Flush(); fs.Close(); } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs index c1accd30d..9aac4df3b 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs @@ -136,7 +136,7 @@ public bool SaveMessageFiles(string conversationId, string messageId, string sou try { - var (_, bytes) = FileUtility.GetFileInfoFromData(file.FileData); + var (_, binary) = FileUtility.GetFileInfoFromData(file.FileData); var subDir = Path.Combine(dir, source, $"{i + 1}"); if (!ExistDirectory(subDir)) { @@ -145,7 +145,7 @@ public bool SaveMessageFiles(string conversationId, string messageId, string sou using (var fs = new FileStream(Path.Combine(subDir, file.FileName), FileMode.Create)) { - fs.Write(bytes, 0, bytes.Length); + fs.Write(binary.ToArray(), 0, binary.Length); fs.Flush(true); fs.Close(); Thread.Sleep(100); diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.User.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.User.cs index 21f522b27..9384bd837 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.User.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.User.cs @@ -34,8 +34,8 @@ public bool SaveUserAvatar(FileDataModel file) } dir = GetUserAvatarDir(user?.Id, createNewDir: true); - var (_, bytes) = FileUtility.GetFileInfoFromData(file.FileData); - File.WriteAllBytes(Path.Combine(dir, file.FileName), bytes); + var (_, binary) = FileUtility.GetFileInfoFromData(file.FileData); + File.WriteAllBytes(Path.Combine(dir, file.FileName), binary.ToArray()); return true; } catch (Exception ex) diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs index 0e53c7463..2b2a72f66 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs @@ -159,17 +159,10 @@ public async Task> GetAgentOptions() } [HttpGet("/agent/utility/options")] - public IEnumerable GetAgentUtilityOptions() + public async Task> GetAgentUtilityOptions() { - var utilities = new List(); - var hooks = _services.GetServices(); - foreach (var hook in hooks) - { - hook.AddUtilities(utilities); - } - return utilities.Where(x => !string.IsNullOrWhiteSpace(x.Category) - && !string.IsNullOrWhiteSpace(x.Name) - && !x.Items.IsNullOrEmpty()).ToList(); + var agentService = _services.GetRequiredService(); + return await agentService.GetAgentUtilityOptions(); } [HttpGet("/agent/labels")] diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index 430a45633..73a4b9aec 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -1,11 +1,9 @@ using BotSharp.Abstraction.Agents.Models; using BotSharp.Abstraction.Files.Utilities; -using BotSharp.Abstraction.Infrastructures; using BotSharp.Abstraction.Instructs; using BotSharp.Abstraction.Instructs.Models; using BotSharp.Core.Infrastructures; using BotSharp.OpenAPI.ViewModels.Instructs; -using static System.Net.Mime.MediaTypeNames; namespace BotSharp.OpenAPI.Controllers; @@ -93,7 +91,12 @@ public async Task ChatCompletion([FromBody] IncomingInstructRequest inpu { new RoleDialogModel(AgentRole.User, input.Text) { - Files = input.Files?.Select(x => new BotSharpFile { FileUrl = x.FileUrl, FileData = x.FileData }).ToList() ?? [] + Files = input.Files?.Select(x => new BotSharpFile + { + FileUrl = x.FileUrl, + FileData = x.FileData, + ContentType = x.ContentType + }).ToList() ?? [] } }); @@ -115,10 +118,10 @@ await hook.OnResponseGenerated(new InstructResponseModel #region Read image [HttpPost("/instruct/multi-modal")] - public async Task MultiModalCompletion([FromBody] MultiModalRequest input) + public async Task MultiModalCompletion([FromBody] MultiModalFileRequest input) { var state = _services.GetRequiredService(); - input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + input.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); try { @@ -135,34 +138,32 @@ public async Task MultiModalCompletion([FromBody] MultiModalRequest inpu catch (Exception ex) { var error = $"Error in reading images. {ex.Message}"; - _logger.LogError(error); + _logger.LogError(ex, error); return error; } } [HttpPost("/instruct/multi-modal/upload")] - public async Task MultiModalCompletion(IFormFile file, [FromForm] string text, [FromForm] string? provider = null, - [FromForm] string? model = null, [FromForm] List? states = null, - [FromForm] string? agentId = null, [FromForm] string? templateName = null) + public async Task MultiModalCompletion([FromForm] IEnumerable files, [FromForm] MultiModalRequest request) { var state = _services.GetRequiredService(); - states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + request?.States?.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); var viewModel = new MultiModalViewModel(); try { - var data = FileUtility.BuildFileDataFromFile(file); - var files = new List + var fileModels = files.Select(x => new InstructFileModel { - new InstructFileModel { FileData = data } - }; + FileData = FileUtility.BuildFileDataFromFile(x) + }).ToList(); + var fileInstruct = _services.GetRequiredService(); - var content = await fileInstruct.ReadImages(text, files, new InstructOptions + var content = await fileInstruct.ReadImages(request?.Text ?? string.Empty, fileModels, new InstructOptions { - Provider = provider, - Model = model, - AgentId = agentId, - TemplateName = templateName + Provider = request?.Provider, + Model = request?.Model, + AgentId = request?.AgentId, + TemplateName = request?.TemplateName }); viewModel.Content = content; return viewModel; @@ -170,7 +171,7 @@ public async Task MultiModalCompletion(IFormFile file, [Fro catch (Exception ex) { var error = $"Error in reading image upload. {ex.Message}"; - _logger.LogError(error); + _logger.LogError(ex, error); viewModel.Message = error; return viewModel; } @@ -182,7 +183,7 @@ public async Task MultiModalCompletion(IFormFile file, [Fro public async Task ImageGeneration([FromBody] ImageGenerationRequest input) { var state = _services.GetRequiredService(); - input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + input.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); var imageViewModel = new ImageGenerationViewModel(); try @@ -202,7 +203,7 @@ public async Task ImageGeneration([FromBody] ImageGene catch (Exception ex) { var error = $"Error in image generation. {ex.Message}"; - _logger.LogError(error); + _logger.LogError(ex, error); imageViewModel.Message = error; return imageViewModel; } @@ -214,7 +215,7 @@ public async Task ImageGeneration([FromBody] ImageGene public async Task ImageVariation([FromBody] ImageVariationRequest input) { var state = _services.GetRequiredService(); - input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + input.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); var imageViewModel = new ImageGenerationViewModel(); try @@ -239,30 +240,34 @@ public async Task ImageVariation([FromBody] ImageVaria catch (Exception ex) { var error = $"Error in image variation. {ex.Message}"; - _logger.LogError(error); + _logger.LogError(ex, error); imageViewModel.Message = error; return imageViewModel; } } [HttpPost("/instruct/image-variation/upload")] - public async Task ImageVariation(IFormFile file, [FromForm] string? provider = null, - [FromForm] string? model = null, [FromForm] List? states = null, - [FromForm] string? agentId = null) + public async Task ImageVariation(IFormFile file, [FromForm] MultiModalRequest request) { var state = _services.GetRequiredService(); - states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + request?.States?.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); var imageViewModel = new ImageGenerationViewModel(); try { var fileInstruct = _services.GetRequiredService(); var fileData = FileUtility.BuildFileDataFromFile(file); - var message = await fileInstruct.VaryImage(new InstructFileModel { FileData = fileData }, new InstructOptions + var message = await fileInstruct.VaryImage(new InstructFileModel { - Provider = provider, - Model = model, - AgentId = agentId + FileData = fileData, + FileName = Path.GetFileNameWithoutExtension(file.FileName), + FileExtension = Path.GetExtension(file.FileName) + }, + new InstructOptions + { + Provider = request?.Provider, + Model = request?.Model, + AgentId = request?.AgentId }); imageViewModel.Content = message.Content; @@ -272,7 +277,7 @@ public async Task ImageVariation(IFormFile file, [From catch (Exception ex) { var error = $"Error in image variation upload. {ex.Message}"; - _logger.LogError(error); + _logger.LogError(ex, error); imageViewModel.Message = error; return imageViewModel; } @@ -283,7 +288,7 @@ public async Task ImageEdit([FromBody] ImageEditReques { var fileInstruct = _services.GetRequiredService(); var state = _services.GetRequiredService(); - input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + input.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); var imageViewModel = new ImageGenerationViewModel(); try @@ -306,31 +311,35 @@ public async Task ImageEdit([FromBody] ImageEditReques catch (Exception ex) { var error = $"Error in image edit. {ex.Message}"; - _logger.LogError(error); + _logger.LogError(ex, error); imageViewModel.Message = error; return imageViewModel; } } [HttpPost("/instruct/image-edit/upload")] - public async Task ImageEdit(IFormFile file, [FromForm] string text, [FromForm] string? provider = null, - [FromForm] string? model = null, [FromForm] List? states = null, - [FromForm] string? agentId = null, [FromForm] string? templateName = null) + public async Task ImageEdit(IFormFile file, [FromForm] MultiModalRequest request) { var fileInstruct = _services.GetRequiredService(); var state = _services.GetRequiredService(); - states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + request?.States?.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); var imageViewModel = new ImageGenerationViewModel(); try { var fileData = FileUtility.BuildFileDataFromFile(file); - var message = await fileInstruct.EditImage(text, new InstructFileModel { FileData = fileData }, new InstructOptions + var message = await fileInstruct.EditImage(request?.Text ?? string.Empty, new InstructFileModel { - Provider = provider, - Model = model, - AgentId = agentId, - TemplateName = templateName + FileData = fileData, + FileName = Path.GetFileNameWithoutExtension(file.FileName), + FileExtension = Path.GetExtension(file.FileName) + }, + new InstructOptions + { + Provider = request?.Provider, + Model = request?.Model, + AgentId = request?.AgentId, + TemplateName = request?.TemplateName }); imageViewModel.Content = message.Content; @@ -341,7 +350,7 @@ public async Task ImageEdit(IFormFile file, [FromForm] catch (Exception ex) { var error = $"Error in image edit upload. {ex.Message}"; - _logger.LogError(error); + _logger.LogError(ex, error); imageViewModel.Message = error; return imageViewModel; } @@ -352,7 +361,7 @@ public async Task ImageMaskEdit([FromBody] ImageMaskEd { var fileInstruct = _services.GetRequiredService(); var state = _services.GetRequiredService(); - input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + input.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); var imageViewModel = new ImageGenerationViewModel(); try @@ -377,35 +386,44 @@ public async Task ImageMaskEdit([FromBody] ImageMaskEd catch (Exception ex) { var error = $"Error in image mask edit. {ex.Message}"; - _logger.LogError(error); + _logger.LogError(ex, error); imageViewModel.Message = error; return imageViewModel; } } [HttpPost("/instruct/image-mask-edit/upload")] - public async Task ImageMaskEdit(IFormFile image, IFormFile mask, - [FromForm] string text, [FromForm] string? provider = null, [FromForm] string? model = null, - [FromForm] List? states = null, [FromForm] string? agentId = null, [FromForm] string? templateName = null) + public async Task ImageMaskEdit(IFormFile image, IFormFile mask, [FromForm] MultiModalRequest request) { var fileInstruct = _services.GetRequiredService(); var state = _services.GetRequiredService(); - states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + request?.States?.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); var imageViewModel = new ImageGenerationViewModel(); try { var imageData = FileUtility.BuildFileDataFromFile(image); var maskData = FileUtility.BuildFileDataFromFile(mask); - var message = await fileInstruct.EditImage(text, - new InstructFileModel { FileData = imageData }, - new InstructFileModel { FileData = maskData }, new InstructOptions - { - Provider = provider, - Model = model, - AgentId = agentId, - TemplateName = templateName - }); + var message = await fileInstruct.EditImage(request?.Text ?? string.Empty, + new InstructFileModel + { + FileData = imageData, + FileName = Path.GetFileNameWithoutExtension(image.FileName), + FileExtension = Path.GetExtension(image.FileName) + }, + new InstructFileModel + { + FileData = maskData, + FileName = Path.GetFileNameWithoutExtension(mask.FileName), + FileExtension = Path.GetExtension(mask.FileName) + }, + new InstructOptions + { + Provider = request?.Provider, + Model = request?.Model, + AgentId = request?.AgentId, + TemplateName = request?.TemplateName + }); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); @@ -415,7 +433,7 @@ public async Task ImageMaskEdit(IFormFile image, IForm catch (Exception ex) { var error = $"Error in image mask edit upload. {ex.Message}"; - _logger.LogError(error); + _logger.LogError(ex, error); imageViewModel.Message = error; return imageViewModel; } @@ -424,10 +442,10 @@ public async Task ImageMaskEdit(IFormFile image, IForm #region Pdf [HttpPost("/instruct/pdf-completion")] - public async Task PdfCompletion([FromBody] MultiModalRequest input) + public async Task PdfCompletion([FromBody] MultiModalFileRequest input) { var state = _services.GetRequiredService(); - input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + input.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); var viewModel = new PdfCompletionViewModel(); try @@ -446,36 +464,33 @@ public async Task PdfCompletion([FromBody] MultiModalReq catch (Exception ex) { var error = $"Error in pdf completion. {ex.Message}"; - _logger.LogError(error); + _logger.LogError(ex, error); viewModel.Message = error; return viewModel; } } [HttpPost("/instruct/pdf-completion/upload")] - public async Task PdfCompletion(IFormFile file, [FromForm] string text, - [FromForm] string? provider = null, [FromForm] string? model = null, [FromForm] List? states = null, - [FromForm] string? agentId = null, [FromForm] string? templateName = null) + public async Task PdfCompletion([FromForm] IEnumerable files, [FromForm] MultiModalRequest request) { var state = _services.GetRequiredService(); - states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + request?.States?.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); var viewModel = new PdfCompletionViewModel(); try { - var data = FileUtility.BuildFileDataFromFile(file); - var files = new List + var fileModels = files.Select(x => new InstructFileModel { - new InstructFileModel { FileData = data } - }; + FileData = FileUtility.BuildFileDataFromFile(x) + }).ToList(); var fileInstruct = _services.GetRequiredService(); - var content = await fileInstruct.ReadPdf(text, files, new InstructOptions + var content = await fileInstruct.ReadPdf(request?.Text ?? string.Empty, fileModels, new InstructOptions { - Provider = provider, - Model = model, - AgentId = agentId, - TemplateName = templateName + Provider = request?.Provider, + Model = request?.Model, + AgentId = request?.AgentId, + TemplateName = request?.TemplateName }); viewModel.Content = content; return viewModel; @@ -483,7 +498,7 @@ public async Task PdfCompletion(IFormFile file, [FromFor catch (Exception ex) { var error = $"Error in pdf completion upload. {ex.Message}"; - _logger.LogError(error); + _logger.LogError(ex, error); viewModel.Message = error; return viewModel; } @@ -496,7 +511,7 @@ public async Task SpeechToText([FromBody] SpeechToTextReq { var fileInstruct = _services.GetRequiredService(); var state = _services.GetRequiredService(); - input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + input.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); var viewModel = new SpeechToTextViewModel(); try @@ -519,32 +534,36 @@ public async Task SpeechToText([FromBody] SpeechToTextReq catch (Exception ex) { var error = $"Error in speech to text. {ex.Message}"; - _logger.LogError(error); + _logger.LogError(ex, error); viewModel.Message = error; return viewModel; } } [HttpPost("/instruct/speech-to-text/upload")] - public async Task SpeechToText(IFormFile file, - [FromForm] string? provider = null, [FromForm] string? model = null, - [FromForm] string? text = null, [FromForm] List? states = null, - [FromForm] string? agentId = null, [FromForm] string? templateName = null) + public async Task SpeechToText(IFormFile file, [FromForm] MultiModalRequest request) { var fileInstruct = _services.GetRequiredService(); var state = _services.GetRequiredService(); - states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + request?.States?.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); var viewModel = new SpeechToTextViewModel(); try { var audioData = FileUtility.BuildFileDataFromFile(file); - var content = await fileInstruct.SpeechToText(new InstructFileModel { FileData = audioData }, text, new InstructOptions + var content = await fileInstruct.SpeechToText(new InstructFileModel + { + FileData = audioData, + FileName = Path.GetFileNameWithoutExtension(file.FileName), + FileExtension = Path.GetExtension(file.FileName) + }, + request?.Text ?? string.Empty, + new InstructOptions { - Provider = provider, - Model = model, - AgentId = agentId, - TemplateName = templateName + Provider = request?.Provider, + Model = request?.Model, + AgentId = request?.AgentId, + TemplateName = request?.TemplateName }); viewModel.Content = content; @@ -553,7 +572,7 @@ public async Task SpeechToText(IFormFile file, catch (Exception ex) { var error = $"Error in speech-to-text upload. {ex.Message}"; - _logger.LogError(error); + _logger.LogError(ex, error); viewModel.Message = error; return viewModel; } @@ -563,7 +582,7 @@ public async Task SpeechToText(IFormFile file, public async Task TextToSpeech([FromBody] TextToSpeechRequest input) { var state = _services.GetRequiredService(); - input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + input.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); var completion = CompletionProvider.GetAudioSynthesizer(_services, provider: input.Provider, model: input.Model); var binaryData = await completion.GenerateAudioAsync(input.Text); diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs index 88a10838b..92e36f468 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs @@ -245,8 +245,8 @@ public IActionResult GetUserAvatar() private FileContentResult BuildFileResult(string file) { var fileStorage = _services.GetRequiredService(); - var bytes = fileStorage.GetFileBytes(file); - return File(bytes, "application/octet-stream", Path.GetFileName(file)); + var binary = fileStorage.GetFileBytes(file); + return File(binary.ToArray(), "application/octet-stream", Path.GetFileName(file)); } #endregion } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs index 80368a70a..4434da0f5 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructBaseRequest.cs @@ -1,3 +1,4 @@ +using BotSharp.OpenAPI.ViewModels.Instructs.Request; using System.Text.Json.Serialization; namespace BotSharp.Abstraction.Instructs.Models; @@ -17,14 +18,17 @@ public class InstructBaseRequest public virtual string? TemplateName { get; set; } [JsonPropertyName("states")] - public List States { get; set; } = []; + public List States { get; set; } = []; } public class MultiModalRequest : InstructBaseRequest { [JsonPropertyName("text")] public string Text { get; set; } = string.Empty; +} +public class MultiModalFileRequest : MultiModalRequest +{ [JsonPropertyName("files")] public List Files { get; set; } = []; } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructState.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructState.cs new file mode 100644 index 000000000..986280256 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructState.cs @@ -0,0 +1,7 @@ +namespace BotSharp.OpenAPI.ViewModels.Instructs.Request; + +public class InstructState +{ + public string Key { get; set; } + public string Value { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs index f20a675a8..a114c2fae 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs @@ -87,8 +87,8 @@ private async Task GetResponeFromDialogs(List dialogs) var fileName = Path.GetFileName(file.FileStorageUrl); if (!ParseAudioFileType(fileName)) continue; - var bytes = _fileStorage.GetFileBytes(file.FileStorageUrl); - using var stream = new MemoryStream(bytes); + var binary = _fileStorage.GetFileBytes(file.FileStorageUrl); + using var stream = binary.ToStream(); stream.Position = 0; var result = await audioCompletion.TranscriptTextAsync(stream, fileName); diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs index 5c8a12d14..8e4a59fe1 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs @@ -89,8 +89,8 @@ private void LoadWhisperModel(GgmlType modelType) DownloadModel(modelType, modelLoc); } - var bytes = _fileStorage.GetFileBytes(modelLoc); - _whisperProcessor = WhisperFactory.FromBuffer(bytes).CreateBuilder().WithLanguage("auto").Build(); + var binary = _fileStorage.GetFileBytes(modelLoc); + _whisperProcessor = WhisperFactory.FromBuffer(binary.ToArray()).CreateBuilder().WithLanguage("auto").Build(); } catch (Exception ex) { diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs index 114ea69d1..ab7135ef8 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -308,7 +308,7 @@ await onMessageReceived(new RoleDialogModel(choice.Role?.ToString() ?? ChatMessa ChatToolCall.CreateFunctionToolCall(message.ToolCallId.IfNullOrEmptyAs(message.FunctionName), message.FunctionName, BinaryData.FromString(message.FunctionArgs ?? "{}")) })); - messages.Add(new ToolChatMessage(message.ToolCallId ?? message.FunctionName, message.Content)); + messages.Add(new ToolChatMessage(message.ToolCallId.IfNullOrEmptyAs(message.FunctionName), message.Content)); } else if (message.Role == AgentRole.User) { @@ -322,15 +322,15 @@ await onMessageReceived(new RoleDialogModel(choice.Role?.ToString() ?? ChatMessa { if (!string.IsNullOrEmpty(file.FileData)) { - var (contentType, bytes) = FileUtility.GetFileInfoFromData(file.FileData); - var contentPart = ChatMessageContentPart.CreateImagePart(BinaryData.FromBytes(bytes), contentType, ChatImageDetailLevel.Auto); + var (contentType, binary) = FileUtility.GetFileInfoFromData(file.FileData); + var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), ChatImageDetailLevel.Auto); contentParts.Add(contentPart); } else if (!string.IsNullOrEmpty(file.FileStorageUrl)) { var contentType = FileUtility.GetFileContentType(file.FileStorageUrl); - var bytes = fileStorage.GetFileBytes(file.FileStorageUrl); - var contentPart = ChatMessageContentPart.CreateImagePart(BinaryData.FromBytes(bytes), contentType, ChatImageDetailLevel.Auto); + var binary = fileStorage.GetFileBytes(file.FileStorageUrl); + var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), ChatImageDetailLevel.Auto); contentParts.Add(contentPart); } else if (!string.IsNullOrEmpty(file.FileUrl)) diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/SignalRHub.cs b/src/Plugins/BotSharp.Plugin.ChatHub/SignalRHub.cs index 12b595f22..2cb0d1627 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/SignalRHub.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/SignalRHub.cs @@ -28,7 +28,7 @@ public override async Task OnConnectedAsync() _logger.LogInformation($"SignalR Hub: {_user.FirstName} {_user.LastName} ({Context.User.Identity.Name}) connected in {Context.ConnectionId}"); var convService = _services.GetRequiredService(); - _context.HttpContext.Request.Query.TryGetValue("conversationId", out var conversationId); + _context.HttpContext.Request.Query.TryGetValue("conversation-id", out var conversationId); if (!string.IsNullOrEmpty(conversationId)) { diff --git a/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs index 853040541..42ce1ac93 100644 --- a/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs @@ -270,10 +270,10 @@ public void SetModelName(string model) { messages.Add(new AssistantChatMessage(new List { - ChatToolCall.CreateFunctionToolCall(message.FunctionName, message.FunctionName, BinaryData.FromString(message.FunctionArgs ?? string.Empty)) + ChatToolCall.CreateFunctionToolCall(message.ToolCallId.IfNullOrEmptyAs(message.FunctionName), message.FunctionName, BinaryData.FromString(message.FunctionArgs ?? string.Empty)) })); - messages.Add(new ToolChatMessage(message.FunctionName, message.Content)); + messages.Add(new ToolChatMessage(message.ToolCallId.IfNullOrEmptyAs(message.FunctionName), message.Content)); } else if (message.Role == AgentRole.User) { diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs index df46cdfe7..35f817f4c 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs @@ -90,8 +90,8 @@ private void BuildEmailAttachments(BodyBuilder builder, IEnumerable GetResponeFromDialogs(List dialogs) _currentFileName = Path.GetFileName(file.FileStorageUrl); - var bytes = _fileStorage.GetFileBytes(file.FileStorageUrl); - var workbook = ConvertToWorkBook(bytes); + var binary = _fileStorage.GetFileBytes(file.FileStorageUrl); + var workbook = ConvertToWorkBook(binary.ToArray()); var currentCommandList = _mySqlService.WriteExcelDataToDB(workbook); sqlCommandList.AddRange(currentCommandList); diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs index 2895f1520..aed98feaa 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs @@ -77,9 +77,8 @@ private async Task GetImageEditGeneration(RoleDialogModel message, strin }; var fileStorage = _services.GetRequiredService(); - var fileBytes = fileStorage.GetFileBytes(image.FileStorageUrl); - using var stream = new MemoryStream(); - stream.Write(fileBytes); + var fileBinary = fileStorage.GetFileBytes(image.FileStorageUrl); + using var stream = fileBinary.ToStream(); stream.Position = 0; var result = await completion.GetImageEdits(agent, dialog, stream, image.FileName ?? string.Empty); stream.Close(); diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs index 9a23d8174..1ead28e65 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs @@ -307,26 +307,26 @@ public void SetModelName(string model) { if (!string.IsNullOrEmpty(file.FileData)) { - var (contentType, bytes) = FileUtility.GetFileInfoFromData(file.FileData); + var (contentType, binary) = FileUtility.GetFileInfoFromData(file.FileData); contentParts.Add(new Part() { InlineData = new() { - MimeType = contentType, - Data = Convert.ToBase64String(bytes) + MimeType = contentType.IfNullOrEmptyAs(file.ContentType), + Data = Convert.ToBase64String(binary.ToArray()) } }); } else if (!string.IsNullOrEmpty(file.FileStorageUrl)) { var contentType = FileUtility.GetFileContentType(file.FileStorageUrl); - var bytes = fileStorage.GetFileBytes(file.FileStorageUrl); + var binary = fileStorage.GetFileBytes(file.FileStorageUrl); contentParts.Add(new Part() { InlineData = new() { - MimeType = contentType, - Data = Convert.ToBase64String(bytes) + MimeType = contentType.IfNullOrEmptyAs(file.ContentType), + Data = Convert.ToBase64String(binary.ToArray()) } }); } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs index a2461f18e..491261d85 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs @@ -48,12 +48,12 @@ public async Task UploadDocumentsToKnowledge(string col try { // Get document info - var (contentType, bytes) = await GetFileInfo(file); - var contents = await GetFileContent(contentType, bytes, option ?? ChunkOption.Default()); + var (contentType, binary) = await GetFileInfo(file); + var contents = await GetFileContent(contentType, binary, option ?? ChunkOption.Default()); // Save document var fileId = Guid.NewGuid(); - var saved = SaveDocument(collectionName, vectorStoreProvider, fileId, file.FileName, bytes); + var saved = SaveDocument(collectionName, vectorStoreProvider, fileId, file.FileName, binary); if (!saved) { failedFiles.Add(file.FileName); @@ -342,11 +342,11 @@ public async Task GetKnowledgeDocumentBinaryData(string col /// /// /// - private async Task<(string, byte[])> GetFileInfo(ExternalFileModel file) + private async Task<(string, BinaryData)> GetFileInfo(ExternalFileModel file) { if (file == null) { - return (string.Empty, new byte[0]); + return (string.Empty, BinaryData.Empty); } if (!string.IsNullOrWhiteSpace(file.FileUrl)) @@ -355,37 +355,38 @@ public async Task GetKnowledgeDocumentBinaryData(string col var contentType = FileUtility.GetFileContentType(file.FileName); using var client = http.CreateClient(); var bytes = await client.GetByteArrayAsync(file.FileUrl); - return (contentType, bytes); + return (contentType, BinaryData.FromBytes(bytes)); } else if (!string.IsNullOrWhiteSpace(file.FileData)) { - var (contentType, bytes) = FileUtility.GetFileInfoFromData(file.FileData); - return (contentType, bytes); + var (contentType, binary) = FileUtility.GetFileInfoFromData(file.FileData); + return (contentType, binary); } - return (string.Empty, new byte[0]); + return (string.Empty, BinaryData.Empty); } #region Read doc content - private async Task> GetFileContent(string contentType, byte[] bytes, ChunkOption option) + private async Task> GetFileContent(string contentType, BinaryData binary, ChunkOption option) { IEnumerable results = new List(); if (contentType.IsEqualTo(MediaTypeNames.Text.Plain)) { - results = await ReadTxt(bytes, option); + results = await ReadTxt(binary, option); } else if (contentType.IsEqualTo(MediaTypeNames.Application.Pdf)) { - results = await ReadPdf(bytes); + results = await ReadPdf(binary); } return results; } - private async Task> ReadTxt(byte[] bytes, ChunkOption option) + private async Task> ReadTxt(BinaryData binary, ChunkOption option) { - using var stream = new MemoryStream(bytes); + using var stream = binary.ToStream(); + stream.Position = 0; using var reader = new StreamReader(stream); var content = await reader.ReadToEndAsync(); reader.Close(); @@ -395,18 +396,17 @@ private async Task> ReadTxt(byte[] bytes, ChunkOption option return lines; } - private async Task> ReadPdf(byte[] bytes) + private async Task> ReadPdf(BinaryData binary) { return Enumerable.Empty(); } #endregion - private bool SaveDocument(string collectionName, string vectorStoreProvider, Guid fileId, string fileName, byte[] bytes) + private bool SaveDocument(string collectionName, string vectorStoreProvider, Guid fileId, string fileName, BinaryData binary) { var fileStoreage = _services.GetRequiredService(); - var data = BinaryData.FromBytes(bytes); - var saved = fileStoreage.SaveKnowledgeBaseFile(collectionName, vectorStoreProvider, fileId, fileName, data); + var saved = fileStoreage.SaveKnowledgeBaseFile(collectionName, vectorStoreProvider, fileId, fileName, binary); return saved; } diff --git a/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIChatCompletionProvider.cs index d32fbeceb..d95a25ae0 100644 --- a/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIChatCompletionProvider.cs @@ -128,8 +128,8 @@ public async Task GetChatCompletions(Agent agent, List OnUserAudioTranscriptionCompleted(RealtimeHu { if (!string.IsNullOrEmpty(file.FileData)) { - var (contentType, bytes) = FileUtility.GetFileInfoFromData(file.FileData); - var contentPart = ChatMessageContentPart.CreateImagePart(BinaryData.FromBytes(bytes), contentType, ChatImageDetailLevel.Auto); + var (contentType, binary) = FileUtility.GetFileInfoFromData(file.FileData); + var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType, ChatImageDetailLevel.Auto); contentParts.Add(contentPart); } else if (!string.IsNullOrEmpty(file.FileStorageUrl)) { var contentType = FileUtility.GetFileContentType(file.FileStorageUrl); - var bytes = fileStorage.GetFileBytes(file.FileStorageUrl); - var contentPart = ChatMessageContentPart.CreateImagePart(BinaryData.FromBytes(bytes), contentType, ChatImageDetailLevel.Auto); + var binary = fileStorage.GetFileBytes(file.FileStorageUrl); + var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType, ChatImageDetailLevel.Auto); contentParts.Add(contentPart); } else if (!string.IsNullOrEmpty(file.FileUrl)) diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs index 0659b0c82..fbc0a0048 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs @@ -25,16 +25,17 @@ public IEnumerable GetFiles(string relativePath, string? searchPattern = } } - public byte[] GetFileBytes(string fileStorageUrl) + public BinaryData GetFileBytes(string fileStorageUrl) { try { - return _cosClient.BucketClient.DownloadFileBytes(fileStorageUrl); + var bytes = _cosClient.BucketClient.DownloadFileBytes(fileStorageUrl); + return BinaryData.FromBytes(bytes); } catch (Exception ex) { _logger.LogWarning(ex, $"Error when getting file bytes (url: {fileStorageUrl})."); - return Array.Empty(); + return BinaryData.Empty; } } @@ -53,13 +54,13 @@ public bool SaveFileStreamToPath(string filePath, Stream stream) } } - public bool SaveFileBytesToPath(string filePath, byte[] bytes) + public bool SaveFileBytesToPath(string filePath, BinaryData binary) { if (string.IsNullOrEmpty(filePath)) return false; try { - return _cosClient.BucketClient.UploadBytes(filePath, bytes); + return _cosClient.BucketClient.UploadBytes(filePath, binary.ToArray()); } catch (Exception ex) { diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs index 4afec3937..dfd6781fb 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs @@ -132,9 +132,9 @@ public bool SaveMessageFiles(string conversationId, string messageId, string sou try { - var (_, bytes) = FileUtility.GetFileInfoFromData(file.FileData); + var (_, binary) = FileUtility.GetFileInfoFromData(file.FileData); var subDir = $"{dir}/{source}/{i + 1}"; - _cosClient.BucketClient.UploadBytes($"{subDir}/{file.FileName}", bytes); + _cosClient.BucketClient.UploadBytes($"{subDir}/{file.FileName}", binary.ToArray()); } catch (Exception ex) { diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs index 720765281..9d65d00fd 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs @@ -28,11 +28,11 @@ public bool SaveUserAvatar(FileDataModel file) if (string.IsNullOrEmpty(dir)) return false; - var (_, bytes) = FileUtility.GetFileInfoFromData(file.FileData); + var (_, binary) = FileUtility.GetFileInfoFromData(file.FileData); var extension = Path.GetExtension(file.FileName); var fileName = user?.Id == null ? file.FileName : $"{user?.Id}{extension}"; - return _cosClient.BucketClient.UploadBytes($"{dir}/{fileName}", bytes); + return _cosClient.BucketClient.UploadBytes($"{dir}/{fileName}", binary.ToArray()); } catch (Exception ex) { diff --git a/tests/BotSharp.LLM.Tests/Core/NullFileStorageService.cs b/tests/BotSharp.LLM.Tests/Core/NullFileStorageService.cs index 23514e4a5..bb1d0c444 100644 --- a/tests/BotSharp.LLM.Tests/Core/NullFileStorageService.cs +++ b/tests/BotSharp.LLM.Tests/Core/NullFileStorageService.cs @@ -1,9 +1,9 @@ -using BotSharp.Abstraction.Files; +using BotSharp.Abstraction.Files; using BotSharp.Abstraction.Files.Models; namespace BotSharp.Plugin.Google.Core { - public class NullFileStorageService:IFileStorageService + public class NullFileStorageService : IFileStorageService { public string GetDirectory(string conversationId) { @@ -15,9 +15,10 @@ public IEnumerable GetFiles(string relativePath, string? searchQuery = n return new List { "FakeFile1.txt", "FakeFile2.txt" }; } - public byte[] GetFileBytes(string fileStorageUrl) + public BinaryData GetFileBytes(string fileStorageUrl) { - return new byte[] { 0x00, 0x01, 0x02 }; + var bytes = new byte[] { 0x00, 0x01, 0x02 }; + return BinaryData.FromBytes(bytes); } public bool SaveFileStreamToPath(string filePath, Stream stream) @@ -25,7 +26,7 @@ public bool SaveFileStreamToPath(string filePath, Stream stream) return true; } - public bool SaveFileBytesToPath(string filePath, byte[] bytes) + public bool SaveFileBytesToPath(string filePath, BinaryData binary) { return true; } diff --git a/tests/BotSharp.LLM.Tests/Core/TestAgentService.cs b/tests/BotSharp.LLM.Tests/Core/TestAgentService.cs index afaf28998..50d34701d 100644 --- a/tests/BotSharp.LLM.Tests/Core/TestAgentService.cs +++ b/tests/BotSharp.LLM.Tests/Core/TestAgentService.cs @@ -6,7 +6,6 @@ using BotSharp.Abstraction.Plugins.Models; using BotSharp.Abstraction.Repositories.Filters; using BotSharp.Abstraction.Utilities; -using NetTopologySuite.Algorithm; namespace BotSharp.Plugin.Google.Core { @@ -111,5 +110,10 @@ public PluginDef GetPlugin(string agentId) { return new PluginDef(); } + + public Task> GetAgentUtilityOptions() + { + return Task.FromResult(Enumerable.Empty()); + } } } \ No newline at end of file