Skip to content

Features/refine file instruct #1068

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 14 commits into
base: master
Choose a base branch
from
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ public interface IAgentService
Task<string> RefreshAgents();
Task<PagedItems<Agent>> GetAgents(AgentFilter filter);
Task<List<IdName>> GetAgentOptions(List<string>? agentIds = null, bool byName = false);
Task<IEnumerable<AgentUtility>> GetAgentUtilityOptions();

/// <summary>
/// Load agent configurations and trigger hooks
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,7 @@ public override string ToString()
public class UtilityItem
{
[JsonPropertyName("function_name")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? FunctionName { get; set; }
public string FunctionName { get; set; } = null!;

[JsonPropertyName("template_name")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
Expand All @@ -36,4 +35,8 @@ public class UtilityItem
[JsonPropertyName("visibility_expression")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? VisibilityExpression { get; set; }

[JsonPropertyName("description")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Description { get; set; }
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@ public interface IFileStorageService
#region Common
string GetDirectory(string conversationId);
IEnumerable<string> GetFiles(string relativePath, string? searchQuery = null);
byte[] GetFileBytes(string fileStorageUrl);
BinaryData GetFileBytes(string fileStorageUrl);
bool SaveFileStreamToPath(string filePath, Stream stream);
bool SaveFileBytesToPath(string filePath, byte[] bytes);
bool SaveFileBytesToPath(string filePath, BinaryData binary);
string GetParentDir(string dir, int level = 1);
bool ExistDirectory(string? dir);
void CreateDirectory(string dir);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,23 @@ namespace BotSharp.Abstraction.Files.Models;
public class InstructFileModel : FileBase
{
/// <summary>
/// File extension without dot
/// File extension
/// </summary>
[JsonPropertyName("file_extension")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? FileExtension { get; set; } = string.Empty;

/// <summary>
/// External file url
/// File url
/// </summary>
[JsonPropertyName("file_url")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? FileUrl { get; set; } = string.Empty;

/// <summary>
/// File url
/// </summary>
[JsonPropertyName("content_type")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? ContentType { get; set; }
}
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,16 @@ public static class FileUtility
/// </summary>
/// <param name="data"></param>
/// <returns></returns>
public static (string, byte[]) GetFileInfoFromData(string data)
public static (string?, BinaryData) GetFileInfoFromData(string data)
{
if (string.IsNullOrEmpty(data))
{
return (string.Empty, new byte[0]);
return (null, BinaryData.Empty);
}

if (!data.StartsWith("data:"))
{
return (null, BinaryData.FromBytes(Convert.FromBase64String(data)));
}

var typeStartIdx = data.IndexOf(':');
Expand All @@ -25,13 +30,13 @@ public static (string, byte[]) GetFileInfoFromData(string data)
var base64startIdx = data.IndexOf(',');
var base64Str = data.Substring(base64startIdx + 1);

return (contentType, Convert.FromBase64String(base64Str));
return (contentType, BinaryData.FromBytes(Convert.FromBase64String(base64Str)));
}

public static string BuildFileDataFromFile(string fileName, byte[] bytes)
public static string BuildFileDataFromFile(string fileName, BinaryData binary)
{
var contentType = GetFileContentType(fileName);
var base64 = Convert.ToBase64String(bytes);
var base64 = Convert.ToBase64String(binary.ToArray());
return $"data:{contentType};base64,{base64}";
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
using BotSharp.Abstraction.Repositories.Settings;
using System.IO;

namespace BotSharp.Core.Agents.Services;

public partial class AgentService
{
public async Task<IEnumerable<AgentUtility>> GetAgentUtilityOptions()
{
var utilities = new List<AgentUtility>();
var hooks = _services.GetServices<IAgentUtilityHook>();
foreach (var hook in hooks)
{
hook.AddUtilities(utilities);
}

utilities = utilities.Where(x => !string.IsNullOrWhiteSpace(x.Category)
&& !string.IsNullOrWhiteSpace(x.Name)
&& !x.Items.IsNullOrEmpty()).ToList();

var allItems = utilities.SelectMany(x => x.Items).ToList();
var functionNames = allItems.Select(x => x.FunctionName).Distinct().ToList();
var mapper = await GetAgentDocs(functionNames);

allItems.ForEach(x =>
{
if (mapper.ContainsKey(x.FunctionName))
{
x.Description = mapper[x.FunctionName];
}
});

return utilities;
}

#region Private methods
private async ValueTask<IDictionary<string, string>> GetAgentDocs(IEnumerable<string> names)
{
var mapper = new Dictionary<string, string>();
if (names.IsNullOrEmpty())
{
return mapper;
}

var dir = GetAgentDocDir(BuiltInAgentId.UtilityAssistant);
if (string.IsNullOrEmpty(dir))
{
return mapper;
}

var matchDocs = Directory.GetFiles(dir, "*.md")
.Where(x => names.Contains(Path.GetFileNameWithoutExtension(x)))
.ToList();

if (matchDocs.IsNullOrEmpty())
{
return mapper;
}

await foreach (var item in GetUtilityDescriptions(matchDocs))
{
mapper[item.Key] = item.Value;
}

return mapper;
}

private string GetAgentDocDir(string agentId)
{
var dbSettings = _services.GetRequiredService<BotSharpDatabaseSettings>();
var agentSettings = _services.GetRequiredService<AgentSettings>();
var dir = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, dbSettings.FileRepository, agentSettings.DataDir, agentId, "docs");
if (!Directory.Exists(dir))
{
dir = string.Empty;
}
return dir;
}

private async IAsyncEnumerable<KeyValuePair<string, string>> GetUtilityDescriptions(IEnumerable<string> docs)
{
foreach (var doc in docs)
{
var content = string.Empty;
try
{
content = await File.ReadAllTextAsync(doc);
}
catch { }

if (string.IsNullOrWhiteSpace(content))
{
continue;
}

var fileName = Path.GetFileNameWithoutExtension(doc);
yield return new KeyValuePair<string, string>(fileName, content);
}
}
#endregion
}
Original file line number Diff line number Diff line change
Expand Up @@ -145,11 +145,7 @@ public IConversationStateService SetState<T>(string name, T value, bool isNeedVe
newPair.Values = new List<StateValue> { newValue };
_curStates[name] = newPair;
}
else if (isNoChange)
{
// do nothing
}
else
else if (!isNoChange)
{
_curStates[name].Values.Add(newValue);
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
using BotSharp.Abstraction.Instructs.Models;
using System.IO;

namespace BotSharp.Core.Files.Services;

Expand All @@ -14,12 +13,11 @@ public async Task<string> SpeechToText(InstructFileModel audio, string? text = n
}

var completion = CompletionProvider.GetAudioTranscriber(_services, provider: options?.Provider, model: options?.Model);
var audioBytes = await DownloadFile(audio);
using var stream = new MemoryStream();
stream.Write(audioBytes, 0, audioBytes.Length);
var audioBinary = await DownloadFile(audio);
using var stream = audioBinary.ToStream();
stream.Position = 0;

var fileName = $"{audio.FileName ?? "audio"}.{audio.FileExtension ?? "wav"}";
var fileName = BuildFileName(audio.FileName, audio.FileExtension, "audio", "wav");
var content = await completion.TranscriptTextAsync(stream, fileName, text ?? string.Empty);
stream.Close();
return content;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
using BotSharp.Abstraction.Instructs.Models;
using BotSharp.Abstraction.Instructs;
using System.IO;
using BotSharp.Abstraction.Infrastructures;

namespace BotSharp.Core.Files.Services;

Expand All @@ -21,7 +19,12 @@ public async Task<string> ReadImages(string text, IEnumerable<InstructFileModel>
{
new RoleDialogModel(AgentRole.User, text)
{
Files = images?.Select(x => new BotSharpFile { FileUrl = x.FileUrl, FileData = x.FileData }).ToList() ?? []
Files = images?.Select(x => new BotSharpFile
{
FileUrl = x.FileUrl,
FileData = x.FileData,
ContentType = x.ContentType
}).ToList() ?? []
}
});

Expand Down Expand Up @@ -76,12 +79,11 @@ public async Task<RoleDialogModel> VaryImage(InstructFileModel image, InstructOp

var innerAgentId = options?.AgentId ?? Guid.Empty.ToString();
var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "dall-e-2");
var bytes = await DownloadFile(image);
using var stream = new MemoryStream();
stream.Write(bytes, 0, bytes.Length);
var binary = await DownloadFile(image);
using var stream = binary.ToStream();
stream.Position = 0;

var fileName = $"{image.FileName ?? "image"}.{image.FileExtension ?? "png"}";
var fileName = BuildFileName(image.FileName, image.FileExtension, "image", "png");
var message = await completion.GetImageVariation(new Agent()
{
Id = innerAgentId
Expand Down Expand Up @@ -113,12 +115,11 @@ public async Task<RoleDialogModel> EditImage(string text, InstructFileModel imag
var instruction = await GetAgentTemplate(innerAgentId, options?.TemplateName);

var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "dall-e-2");
var bytes = await DownloadFile(image);
using var stream = new MemoryStream();
stream.Write(bytes, 0, bytes.Length);
var binary = await DownloadFile(image);
using var stream = binary.ToStream();
stream.Position = 0;

var fileName = $"{image.FileName ?? "image"}.{image.FileExtension ?? "png"}";
var fileName = BuildFileName(image.FileName, image.FileExtension, "image", "png");
var message = await completion.GetImageEdits(new Agent()
{
Id = innerAgentId
Expand Down Expand Up @@ -153,19 +154,17 @@ public async Task<RoleDialogModel> EditImage(string text, InstructFileModel imag
var instruction = await GetAgentTemplate(innerAgentId, options?.TemplateName);

var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "dall-e-2");
var imageBytes = await DownloadFile(image);
var maskBytes = await DownloadFile(mask);
var imageBinary = await DownloadFile(image);
var maskBinary = await DownloadFile(mask);

using var imageStream = new MemoryStream();
imageStream.Write(imageBytes, 0, imageBytes.Length);
using var imageStream = imageBinary.ToStream();
imageStream.Position = 0;

using var maskStream = new MemoryStream();
maskStream.Write(maskBytes, 0, maskBytes.Length);
using var maskStream = maskBinary.ToStream();
maskStream.Position = 0;

var imageName = $"{image.FileName ?? "image"}.{image.FileExtension ?? "png"}";
var maskName = $"{mask.FileName ?? "mask"}.{mask.FileExtension ?? "png"}";
var imageName = BuildFileName(image.FileName, image.FileExtension, "image", "png");
var maskName = BuildFileName(image.FileName, image.FileExtension, "mask", "png");
var message = await completion.GetImageEdits(new Agent()
{
Id = innerAgentId
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ public async Task<string> ReadPdf(string text, List<InstructFileModel> files, In
try
{
var provider = options?.Provider ?? "openai";
var pdfFiles = await DownloadFiles(sessionDir, files);
var pdfFiles = await DownloadAndSaveFiles(sessionDir, files);

var targetFiles = pdfFiles;
if (provider != "google-ai")
Expand Down Expand Up @@ -78,44 +78,38 @@ await hook.OnResponseGenerated(new InstructResponseModel
}

#region Private methods
private async Task<IEnumerable<string>> DownloadFiles(string dir, List<InstructFileModel> files, string extension = "pdf")
private async Task<IEnumerable<string>> DownloadAndSaveFiles(string dir, List<InstructFileModel> files, string extension = "pdf")
{
if (string.IsNullOrWhiteSpace(dir) || files.IsNullOrEmpty())
{
return Enumerable.Empty<string>();
}

var downloadTasks = files.Select(x => DownloadFile(x));
await Task.WhenAll(downloadTasks);

var locs = new List<string>();
foreach (var file in files)
for (int i = 0; i < files.Count; i++)
{
try
var binary = downloadTasks.ElementAt(i).Result;
if (binary == null || binary.IsEmpty)
{
var bytes = new byte[0];
if (!string.IsNullOrEmpty(file.FileUrl))
{
var http = _services.GetRequiredService<IHttpClientFactory>();
using var client = http.CreateClient();
bytes = await client.GetByteArrayAsync(file.FileUrl);
}
else if (!string.IsNullOrEmpty(file.FileData))
{
(_, bytes) = FileUtility.GetFileInfoFromData(file.FileData);
}
continue;
}

if (!bytes.IsNullOrEmpty())
{
var guid = Guid.NewGuid().ToString();
var fileDir = _fileStorage.BuildDirectory(dir, guid);
DeleteIfExistDirectory(fileDir, true);
try
{
var guid = Guid.NewGuid().ToString();
var fileDir = _fileStorage.BuildDirectory(dir, guid);
DeleteIfExistDirectory(fileDir, createNew: true);

var outputDir = _fileStorage.BuildDirectory(fileDir, $"{guid}.{extension}");
_fileStorage.SaveFileBytesToPath(outputDir, bytes);
locs.Add(outputDir);
}
var outputDir = _fileStorage.BuildDirectory(fileDir, $"{guid}.{extension}");
_fileStorage.SaveFileBytesToPath(outputDir, binary);
locs.Add(outputDir);
}
catch (Exception ex)
{
_logger.LogWarning(ex, $"Error when saving pdf file.");
_logger.LogWarning(ex, $"Error when saving #{i + 1} {extension} file.");
continue;
}
}
Expand Down
Loading
Loading