diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/Models/Agent.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/Models/Agent.cs index 4688042be..adf4d3347 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Agents/Models/Agent.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/Models/Agent.cs @@ -1,7 +1,6 @@ using BotSharp.Abstraction.Functions.Models; using BotSharp.Abstraction.Plugins.Models; using BotSharp.Abstraction.Tasks.Models; -using System.Collections.Concurrent; namespace BotSharp.Abstraction.Agents.Models; @@ -152,7 +151,7 @@ public class Agent /// For rendering deferral /// [JsonIgnore] - public ConcurrentDictionary TemplateDict { get; set; } = new(); + public Dictionary TemplateDict { get; set; } = new(); [JsonIgnore] public List SecondaryFunctions { get; set; } = []; diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.Rendering.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.Rendering.cs index 2f322e561..55980e963 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.Rendering.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.Rendering.cs @@ -170,12 +170,9 @@ public IDictionary CollectRenderData(Agent agent) var innerDict = new Dictionary(); var dict = new Dictionary(agent.TemplateDict ?? []); - if (dict != null) + foreach (var p in dict) { - foreach (var p in dict) - { - innerDict[p.Key] = p.Value; - } + innerDict[p.Key] = p.Value; } var states = new Dictionary(state.GetStates()); diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ImageGenerationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ImageGenerationController.cs index 9d30faa79..4d82e92d1 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ImageGenerationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ImageGenerationController.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Files.Utilities; using BotSharp.Abstraction.Instructs.Models; using BotSharp.Abstraction.Instructs.Options; using BotSharp.OpenAPI.ViewModels.Instructs; @@ -17,6 +18,7 @@ public ImageGenerationController(IServiceProvider services, ILogger ComposeImages([FromBody] ImageCompositionRequest request) { @@ -52,4 +54,275 @@ public async Task ComposeImages([FromBody] ImageCompos return imageViewModel; } } + #endregion + + #region Image generation + [HttpPost("/instruct/image-generation")] + public async Task ImageGeneration([FromBody] ImageGenerationRequest request) + { + var state = _services.GetRequiredService(); + request.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); + var imageViewModel = new ImageGenerationViewModel(); + + try + { + var fileInstruct = _services.GetRequiredService(); + var message = await fileInstruct.GenerateImage(request.Text, new InstructOptions + { + Provider = request.Provider, + Model = request.Model, + AgentId = request.AgentId, + TemplateName = request.TemplateName + }); + imageViewModel.Content = message.Content; + imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? []; + return imageViewModel; + } + catch (Exception ex) + { + var error = $"Error in image generation. {ex.Message}"; + _logger.LogError(ex, error); + imageViewModel.Message = error; + return imageViewModel; + } + } + #endregion + + #region Image variation + [HttpPost("/instruct/image-variation")] + public async Task ImageVariation([FromBody] ImageVariationFileRequest request) + { + var state = _services.GetRequiredService(); + request.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); + var imageViewModel = new ImageGenerationViewModel(); + + try + { + if (request.File == null) + { + return new ImageGenerationViewModel { Message = "Error! Cannot find an image!" }; + } + + var fileInstruct = _services.GetRequiredService(); + var message = await fileInstruct.VaryImage(request.File, new InstructOptions + { + Provider = request.Provider, + Model = request.Model, + AgentId = request.AgentId, + ImageConvertProvider = request.ImageConvertProvider + }); + + imageViewModel.Content = message.Content; + imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? []; + return imageViewModel; + } + catch (Exception ex) + { + var error = $"Error in image variation. {ex.Message}"; + _logger.LogError(ex, error); + imageViewModel.Message = error; + return imageViewModel; + } + } + + [HttpPost("/instruct/image-variation/form")] + public async Task ImageVariation(IFormFile file, [FromForm] ImageVariationRequest request) + { + var state = _services.GetRequiredService(); + request?.States?.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); + var imageViewModel = new ImageGenerationViewModel(); + + try + { + var fileInstruct = _services.GetRequiredService(); + var fileData = FileUtility.BuildFileDataFromFile(file); + var message = await fileInstruct.VaryImage(new InstructFileModel + { + FileData = fileData, + FileName = Path.GetFileNameWithoutExtension(file.FileName), + FileExtension = Path.GetExtension(file.FileName) + }, + new InstructOptions + { + Provider = request?.Provider, + Model = request?.Model, + AgentId = request?.AgentId, + ImageConvertProvider = request?.ImageConvertProvider + }); + + imageViewModel.Content = message.Content; + imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? []; + return imageViewModel; + } + catch (Exception ex) + { + var error = $"Error in image variation upload. {ex.Message}"; + _logger.LogError(ex, error); + imageViewModel.Message = error; + return imageViewModel; + } + } + #endregion + + #region Image edit + [HttpPost("/instruct/image-edit")] + public async Task ImageEdit([FromBody] ImageEditFileRequest request) + { + var fileInstruct = _services.GetRequiredService(); + var state = _services.GetRequiredService(); + request.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); + var imageViewModel = new ImageGenerationViewModel(); + + try + { + if (request.File == null) + { + return new ImageGenerationViewModel { Message = "Error! Cannot find a valid image file!" }; + } + var message = await fileInstruct.EditImage(request.Text, request.File, new InstructOptions + { + Provider = request.Provider, + Model = request.Model, + AgentId = request.AgentId, + TemplateName = request.TemplateName, + ImageConvertProvider = request.ImageConvertProvider + }); + imageViewModel.Content = message.Content; + imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? []; + return imageViewModel; + } + catch (Exception ex) + { + var error = $"Error in image edit. {ex.Message}"; + _logger.LogError(ex, error); + imageViewModel.Message = error; + return imageViewModel; + } + } + + [HttpPost("/instruct/image-edit/form")] + public async Task ImageEdit(IFormFile file, [FromForm] ImageEditRequest request) + { + var fileInstruct = _services.GetRequiredService(); + var state = _services.GetRequiredService(); + request?.States?.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); + var imageViewModel = new ImageGenerationViewModel(); + + try + { + var fileData = FileUtility.BuildFileDataFromFile(file); + var message = await fileInstruct.EditImage(request?.Text ?? string.Empty, new InstructFileModel + { + FileData = fileData, + FileName = Path.GetFileNameWithoutExtension(file.FileName), + FileExtension = Path.GetExtension(file.FileName) + }, + new InstructOptions + { + Provider = request?.Provider, + Model = request?.Model, + AgentId = request?.AgentId, + TemplateName = request?.TemplateName, + ImageConvertProvider = request?.ImageConvertProvider + }); + + imageViewModel.Content = message.Content; + imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? []; + return imageViewModel; + } + catch (Exception ex) + { + var error = $"Error in image edit upload. {ex.Message}"; + _logger.LogError(ex, error); + imageViewModel.Message = error; + return imageViewModel; + } + } + #endregion + + #region Image mask edit + [HttpPost("/instruct/image-mask-edit")] + public async Task ImageMaskEdit([FromBody] ImageMaskEditFileRequest request) + { + var fileInstruct = _services.GetRequiredService(); + var state = _services.GetRequiredService(); + request.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); + var imageViewModel = new ImageGenerationViewModel(); + + try + { + var image = request.File; + var mask = request.Mask; + if (image == null || mask == null) + { + return new ImageGenerationViewModel { Message = "Error! Cannot find a valid image or mask!" }; + } + var message = await fileInstruct.EditImage(request.Text, image, mask, new InstructOptions + { + Provider = request.Provider, + Model = request.Model, + AgentId = request.AgentId, + TemplateName = request.TemplateName, + ImageConvertProvider = request.ImageConvertProvider + }); + imageViewModel.Content = message.Content; + imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? []; + return imageViewModel; + } + catch (Exception ex) + { + var error = $"Error in image mask edit. {ex.Message}"; + _logger.LogError(ex, error); + imageViewModel.Message = error; + return imageViewModel; + } + } + + [HttpPost("/instruct/image-mask-edit/form")] + public async Task ImageMaskEdit(IFormFile image, IFormFile mask, [FromForm] ImageMaskEditRequest request) + { + var fileInstruct = _services.GetRequiredService(); + var state = _services.GetRequiredService(); + request?.States?.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); + var imageViewModel = new ImageGenerationViewModel(); + + try + { + var imageData = FileUtility.BuildFileDataFromFile(image); + var maskData = FileUtility.BuildFileDataFromFile(mask); + var message = await fileInstruct.EditImage(request?.Text ?? string.Empty, + new InstructFileModel + { + FileData = imageData, + FileName = Path.GetFileNameWithoutExtension(image.FileName), + FileExtension = Path.GetExtension(image.FileName) + }, + new InstructFileModel + { + FileData = maskData, + FileName = Path.GetFileNameWithoutExtension(mask.FileName), + FileExtension = Path.GetExtension(mask.FileName) + }, + new InstructOptions + { + Provider = request?.Provider, + Model = request?.Model, + AgentId = request?.AgentId, + TemplateName = request?.TemplateName, + ImageConvertProvider = request?.ImageConvertProvider + }); + + imageViewModel.Content = message.Content; + imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? []; + return imageViewModel; + } + catch (Exception ex) + { + var error = $"Error in image mask edit upload. {ex.Message}"; + _logger.LogError(ex, error); + imageViewModel.Message = error; + return imageViewModel; + } + } + #endregion } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index dab28ca6b..986d5dd00 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -184,272 +184,6 @@ public async Task MultiModalCompletion([FromForm] IEnumerab } #endregion - #region Generate image - [HttpPost("/instruct/image-generation")] - public async Task ImageGeneration([FromBody] ImageGenerationRequest request) - { - var state = _services.GetRequiredService(); - request.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); - var imageViewModel = new ImageGenerationViewModel(); - - try - { - var fileInstruct = _services.GetRequiredService(); - var message = await fileInstruct.GenerateImage(request.Text, new InstructOptions - { - Provider = request.Provider, - Model = request.Model, - AgentId = request.AgentId, - TemplateName = request.TemplateName - }); - imageViewModel.Content = message.Content; - imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? []; - return imageViewModel; - } - catch (Exception ex) - { - var error = $"Error in image generation. {ex.Message}"; - _logger.LogError(ex, error); - imageViewModel.Message = error; - return imageViewModel; - } - } - #endregion - - #region Edit image - [HttpPost("/instruct/image-variation")] - public async Task ImageVariation([FromBody] ImageVariationFileRequest request) - { - var state = _services.GetRequiredService(); - request.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); - var imageViewModel = new ImageGenerationViewModel(); - - try - { - if (request.File == null) - { - return new ImageGenerationViewModel { Message = "Error! Cannot find an image!" }; - } - - var fileInstruct = _services.GetRequiredService(); - var message = await fileInstruct.VaryImage(request.File, new InstructOptions - { - Provider = request.Provider, - Model = request.Model, - AgentId = request.AgentId, - ImageConvertProvider = request.ImageConvertProvider - }); - - imageViewModel.Content = message.Content; - imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? []; - return imageViewModel; - } - catch (Exception ex) - { - var error = $"Error in image variation. {ex.Message}"; - _logger.LogError(ex, error); - imageViewModel.Message = error; - return imageViewModel; - } - } - - [HttpPost("/instruct/image-variation/form")] - public async Task ImageVariation(IFormFile file, [FromForm] ImageVariationRequest request) - { - var state = _services.GetRequiredService(); - request?.States?.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); - var imageViewModel = new ImageGenerationViewModel(); - - try - { - var fileInstruct = _services.GetRequiredService(); - var fileData = FileUtility.BuildFileDataFromFile(file); - var message = await fileInstruct.VaryImage(new InstructFileModel - { - FileData = fileData, - FileName = Path.GetFileNameWithoutExtension(file.FileName), - FileExtension = Path.GetExtension(file.FileName) - }, - new InstructOptions - { - Provider = request?.Provider, - Model = request?.Model, - AgentId = request?.AgentId, - ImageConvertProvider = request?.ImageConvertProvider - }); - - imageViewModel.Content = message.Content; - imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? []; - return imageViewModel; - } - catch (Exception ex) - { - var error = $"Error in image variation upload. {ex.Message}"; - _logger.LogError(ex, error); - imageViewModel.Message = error; - return imageViewModel; - } - } - - [HttpPost("/instruct/image-edit")] - public async Task ImageEdit([FromBody] ImageEditFileRequest request) - { - var fileInstruct = _services.GetRequiredService(); - var state = _services.GetRequiredService(); - request.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); - var imageViewModel = new ImageGenerationViewModel(); - - try - { - if (request.File == null) - { - return new ImageGenerationViewModel { Message = "Error! Cannot find a valid image file!" }; - } - var message = await fileInstruct.EditImage(request.Text, request.File, new InstructOptions - { - Provider = request.Provider, - Model = request.Model, - AgentId = request.AgentId, - TemplateName = request.TemplateName, - ImageConvertProvider = request.ImageConvertProvider - }); - imageViewModel.Content = message.Content; - imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? []; - return imageViewModel; - } - catch (Exception ex) - { - var error = $"Error in image edit. {ex.Message}"; - _logger.LogError(ex, error); - imageViewModel.Message = error; - return imageViewModel; - } - } - - [HttpPost("/instruct/image-edit/form")] - public async Task ImageEdit(IFormFile file, [FromForm] ImageEditRequest request) - { - var fileInstruct = _services.GetRequiredService(); - var state = _services.GetRequiredService(); - request?.States?.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); - var imageViewModel = new ImageGenerationViewModel(); - - try - { - var fileData = FileUtility.BuildFileDataFromFile(file); - var message = await fileInstruct.EditImage(request?.Text ?? string.Empty, new InstructFileModel - { - FileData = fileData, - FileName = Path.GetFileNameWithoutExtension(file.FileName), - FileExtension = Path.GetExtension(file.FileName) - }, - new InstructOptions - { - Provider = request?.Provider, - Model = request?.Model, - AgentId = request?.AgentId, - TemplateName = request?.TemplateName, - ImageConvertProvider = request?.ImageConvertProvider - }); - - imageViewModel.Content = message.Content; - imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? []; - return imageViewModel; - } - catch (Exception ex) - { - var error = $"Error in image edit upload. {ex.Message}"; - _logger.LogError(ex, error); - imageViewModel.Message = error; - return imageViewModel; - } - } - - [HttpPost("/instruct/image-mask-edit")] - public async Task ImageMaskEdit([FromBody] ImageMaskEditFileRequest request) - { - var fileInstruct = _services.GetRequiredService(); - var state = _services.GetRequiredService(); - request.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); - var imageViewModel = new ImageGenerationViewModel(); - - try - { - var image = request.File; - var mask = request.Mask; - if (image == null || mask == null) - { - return new ImageGenerationViewModel { Message = "Error! Cannot find a valid image or mask!" }; - } - var message = await fileInstruct.EditImage(request.Text, image, mask, new InstructOptions - { - Provider = request.Provider, - Model = request.Model, - AgentId = request.AgentId, - TemplateName = request.TemplateName, - ImageConvertProvider = request.ImageConvertProvider - }); - imageViewModel.Content = message.Content; - imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? []; - return imageViewModel; - } - catch (Exception ex) - { - var error = $"Error in image mask edit. {ex.Message}"; - _logger.LogError(ex, error); - imageViewModel.Message = error; - return imageViewModel; - } - } - - [HttpPost("/instruct/image-mask-edit/form")] - public async Task ImageMaskEdit(IFormFile image, IFormFile mask, [FromForm] ImageMaskEditRequest request) - { - var fileInstruct = _services.GetRequiredService(); - var state = _services.GetRequiredService(); - request?.States?.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External)); - var imageViewModel = new ImageGenerationViewModel(); - - try - { - var imageData = FileUtility.BuildFileDataFromFile(image); - var maskData = FileUtility.BuildFileDataFromFile(mask); - var message = await fileInstruct.EditImage(request?.Text ?? string.Empty, - new InstructFileModel - { - FileData = imageData, - FileName = Path.GetFileNameWithoutExtension(image.FileName), - FileExtension = Path.GetExtension(image.FileName) - }, - new InstructFileModel - { - FileData = maskData, - FileName = Path.GetFileNameWithoutExtension(mask.FileName), - FileExtension = Path.GetExtension(mask.FileName) - }, - new InstructOptions - { - Provider = request?.Provider, - Model = request?.Model, - AgentId = request?.AgentId, - TemplateName = request?.TemplateName, - ImageConvertProvider = request?.ImageConvertProvider - }); - - imageViewModel.Content = message.Content; - imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? []; - return imageViewModel; - } - catch (Exception ex) - { - var error = $"Error in image mask edit upload. {ex.Message}"; - _logger.LogError(ex, error); - imageViewModel.Message = error; - return imageViewModel; - } - } - #endregion - #region Pdf [HttpPost("/instruct/pdf-completion")] public async Task PdfCompletion([FromBody] PdfReadFileRequest request) diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/ComposeImageFn.cs b/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/ComposeImageFn.cs index 177ab3c5b..f32313fc9 100644 --- a/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/ComposeImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/ComposeImageFn.cs @@ -105,7 +105,7 @@ private async Task GetImageEditGeneration(Agent agent, RoleDialogModel m return response.Content; } - return await GetImageEditResponse(agent, description); + return await AiResponseHelper.GetImageGenerationResponse(_services, agent, description, savedFiles); } catch (Exception ex) { @@ -115,11 +115,6 @@ private async Task GetImageEditGeneration(Agent agent, RoleDialogModel m } } - private async Task GetImageEditResponse(Agent agent, string description) - { - return await AiResponseHelper.GetImageGenerationResponse(_services, agent, description); - } - private (string, string) GetLlmProviderModel(Agent agent) { var provider = agent?.LlmConfig?.ImageEdit?.Provider; diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/EditImageFn.cs b/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/EditImageFn.cs index c171b93f2..913d922e0 100644 --- a/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/EditImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/EditImageFn.cs @@ -107,7 +107,7 @@ private async Task GetImageEdit(Agent agent, RoleDialogModel message, st return response.Content; } - return await GetImageEditResponse(agent, description); + return await AiResponseHelper.GetImageGenerationResponse(_services, agent, description, savedFiles); } catch (Exception ex) { @@ -117,11 +117,6 @@ private async Task GetImageEdit(Agent agent, RoleDialogModel message, st } } - private async Task GetImageEditResponse(Agent agent, string description) - { - return await AiResponseHelper.GetImageGenerationResponse(_services, agent, description); - } - private (string, string) GetLlmProviderModel(Agent agent) { var provider = agent?.LlmConfig?.ImageEdit?.Provider; diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/GenerateImageFn.cs b/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/GenerateImageFn.cs index 12fa23006..c354df35c 100644 --- a/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/GenerateImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/GenerateImageFn.cs @@ -68,7 +68,7 @@ private async Task GetImageGeneration(Agent agent, RoleDialogModel messa return result.Content; } - return await GetImageGenerationResponse(agent, description); + return await AiResponseHelper.GetImageGenerationResponse(_services, agent, description, savedFiles); } catch (Exception ex) { @@ -78,11 +78,6 @@ private async Task GetImageGeneration(Agent agent, RoleDialogModel messa } } - private async Task GetImageGenerationResponse(Agent agent, string description) - { - return await AiResponseHelper.GetImageGenerationResponse(_services, agent, description); - } - private (string, string) GetLlmProviderModel(Agent agent) { var provider = agent?.LlmConfig?.ImageGeneration?.Provider; diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/Helpers/AiResponseHelper.cs b/src/Plugins/BotSharp.Plugin.ImageHandler/Helpers/AiResponseHelper.cs index 3c49743c7..ce999e753 100644 --- a/src/Plugins/BotSharp.Plugin.ImageHandler/Helpers/AiResponseHelper.cs +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/Helpers/AiResponseHelper.cs @@ -2,7 +2,19 @@ namespace BotSharp.Plugin.ImageHandler.Helpers; internal static class AiResponseHelper { - internal static string GetDefaultResponse(IEnumerable files) + internal static async Task GetImageGenerationResponse(IServiceProvider services, Agent agent, string description, IEnumerable? files = null) + { + var text = $"Please generate a user-friendly response from the following description to " + + $"inform user that you have completed the required image: {description}"; + + var provider = agent?.LlmConfig?.Provider ?? "openai"; + var model = agent?.LlmConfig?.Model ?? "gpt-4o-mini"; + var completion = CompletionProvider.GetChatCompletion(services, provider: provider, model: model); + var response = await completion.GetChatCompletions(agent, [new RoleDialogModel(AgentRole.User, text)]); + return response.Content.IfNullOrEmptyAs(GetDefaultResponse(files)) ?? string.Empty; + } + + internal static string GetDefaultResponse(IEnumerable? files) { if (files.IsNullOrEmpty()) { @@ -16,16 +28,4 @@ internal static string GetDefaultResponse(IEnumerable files) return $"Here is the image you asked for: {string.Join(", ", files)}"; } - - internal static async Task GetImageGenerationResponse(IServiceProvider services, Agent agent, string description) - { - var text = $"Please generate a user-friendly response from the following description to " + - $"inform user that you have completed the required image: {description}"; - - var provider = agent?.LlmConfig?.Provider ?? "openai"; - var model = agent?.LlmConfig?.Model ?? "gpt-4o-mini"; - var completion = CompletionProvider.GetChatCompletion(services, provider: provider, model: model); - var response = await completion.GetChatCompletions(agent, [new RoleDialogModel(AgentRole.User, text)]); - return response.Content; - } } diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index 276f853e7..9283d9be4 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -186,6 +186,187 @@ "Embedding": { "Dimension": 1536 } + }, + { + "Id": "dall-e", + "Name": "dall-e-2", + "Version": "dall-e-2", + "ApiKey": "", + "Endpoint": "", + "Type": "image", + "ImageGeneration": true, + "Image": { + "Generation": { + "Size": { + "Default": "1024x1024", + "Options": [ "256x256", "512x512", "1024x1024" ] + }, + "Quality": { + "Default": "standard", + "Options": [ "standard" ] + }, + "ResponseFormat": { + "Default": "bytes", + "Options": [ "url", "bytes" ] + } + }, + "Edit": { + "Size": { + "Default": "1024x1024", + "Options": [ "256x256", "512x512", "1024x1024" ] + }, + "ResponseFormat": { + "Default": "bytes", + "Options": [ "url", "bytes" ] + } + }, + "Variation": { + "Size": { + "Default": "1024x1024", + "Options": [ "256x256", "512x512", "1024x1024" ] + }, + "ResponseFormat": { + "Default": "bytes", + "Options": [ "url", "bytes" ] + } + } + }, + "Cost": { + "TextInputCost": 0.01, + "CachedTextInputCost": 0, + "AudioInputCost": 0, + "CachedAudioInputCost": 0, + "TextOutputCost": 0.03, + "AudioOutputCost": 0 + } + }, + { + "Id": "dall-e", + "Name": "dall-e-3", + "Version": "dall-e-3", + "ApiKey": "", + "Type": "image", + "ImageGeneration": true, + "Image": { + "Generation": { + "Size": { + "Default": "1024x1024", + "Options": [ "1024x1024", "1792x1024", "1024x1792" ] + }, + "Quality": { + "Default": "standard", + "Options": [ "standard", "hd", "auto" ] + }, + "Style": { + "Default": "natural", + "Options": [ "natural", "vivid" ] + }, + "ResponseFormat": { + "Default": "bytes", + "Options": [ "url", "bytes" ] + } + } + }, + "Cost": { + "TextInputCost": 0.01, + "CachedTextInputCost": 0, + "AudioInputCost": 0, + "CachedAudioInputCost": 0, + "TextOutputCost": 0.03, + "AudioOutputCost": 0 + } + }, + { + "Id": "gpt-image", + "Name": "gpt-image-1", + "Version": "gpt-image-1", + "ApiKey": "", + "Type": "image", + "ImageGeneration": true, + "Image": { + "Generation": { + "Size": { + "Default": "1024x1024", + "Options": [ "1024x1024", "1536x1024", "1024x1536", "auto" ] + }, + "Quality": { + "Default": "medium", + "Options": [ "low", "medium", "high", "auto" ] + }, + "Background": { + "Default": "auto", + "Options": [ "auto", "transparent", "opaque" ] + } + }, + "Edit": { + "Size": { + "Default": "1024x1024", + "Options": [ "1024x1024", "1536x1024", "1024x1536", "auto" ] + }, + "Quality": { + "Default": "medium", + "Options": [ "low", "medium", "high", "auto" ] + }, + "Background": { + "Default": "auto", + "Options": [ "transparent", "opaque", "auto" ] + } + } + }, + "Cost": { + "TextInputCost": 0.01, + "CachedTextInputCost": 0, + "AudioInputCost": 0, + "CachedAudioInputCost": 0, + "TextOutputCost": 0.03, + "AudioOutputCost": 0 + } + }, + { + "Id": "gpt-image", + "Name": "gpt-image-1-mini", + "Version": "gpt-image-1-mini", + "ApiKey": "", + "Type": "image", + "ImageGeneration": true, + "Image": { + "Generation": { + "Size": { + "Default": "1024x1024", + "Options": [ "1024x1024", "1536x1024", "1024x1536", "auto" ] + }, + "Quality": { + "Default": "medium", + "Options": [ "low", "medium", "high", "auto" ] + }, + "Background": { + "Default": "auto", + "Options": [ "auto", "transparent", "opaque" ] + } + }, + "Edit": { + "Size": { + "Default": "1024x1024", + "Options": [ "1024x1024", "1536x1024", "1024x1536", "auto" ] + }, + "Quality": { + "Default": "medium", + "Options": [ "low", "medium", "high", "auto" ] + }, + "Background": { + "Default": "auto", + "Options": [ "transparent", "opaque", "auto" ] + } + } + }, + "Cost": { + "TextInputCost": 0.01, + "CachedTextInputCost": 0, + "AudioInputCost": 0, + "CachedAudioInputCost": 0, + "TextOutputCost": 0.03, + "AudioOutputCost": 0 + } } ] }, @@ -545,7 +726,7 @@ }, "MetaGLM": { - "ApiKey": "6b6c8b3fca3e5da21d633e350980744d.938gruOqrK4BDqW8", + "ApiKey": "", "BaseAddress": "http://localhost:8100/v1/", "ModelId": "chatglm3_6b", "Temperature": 0.7,