Skip to content

Commit 7d020bb

Browse files
iceljciceljc
authored
Merge pull request #1201 from iceljc/master
refine image response and settings
2 parents 391f54b + 194b7f4 commit 7d020bb

File tree

9 files changed

+474
-305
lines changed

9 files changed

+474
-305
lines changed

‎src/Infrastructure/BotSharp.Abstraction/Agents/Models/Agent.cs‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
using BotSharp.Abstraction.Functions.Models;
22
using BotSharp.Abstraction.Plugins.Models;
33
using BotSharp.Abstraction.Tasks.Models;
4-
using System.Collections.Concurrent;
54

65
namespace BotSharp.Abstraction.Agents.Models;
76

@@ -152,7 +151,7 @@ public class Agent
152151
/// For rendering deferral
153152
/// </summary>
154153
[JsonIgnore]
155-
public ConcurrentDictionary<string, object> TemplateDict { get; set; } = new();
154+
public Dictionary<string, object> TemplateDict { get; set; } = new();
156155

157156
[JsonIgnore]
158157
public List<FunctionDef> SecondaryFunctions { get; set; } = [];

‎src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.Rendering.cs‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -170,12 +170,9 @@ public IDictionary<string, object> CollectRenderData(Agent agent)
170170

171171
var innerDict = new Dictionary<string, object>();
172172
var dict = new Dictionary<string, object>(agent.TemplateDict ?? []);
173-
if (dict != null)
173+
foreach (var p in dict)
174174
{
175-
foreach (var p in dict)
176-
{
177-
innerDict[p.Key] = p.Value;
178-
}
175+
innerDict[p.Key] = p.Value;
179176
}
180177

181178
var states = new Dictionary<string, string>(state.GetStates());

‎src/Infrastructure/BotSharp.OpenAPI/Controllers/ImageGenerationController.cs‎

Lines changed: 273 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
using BotSharp.Abstraction.Files.Utilities;
12
using BotSharp.Abstraction.Instructs.Models;
23
using BotSharp.Abstraction.Instructs.Options;
34
using BotSharp.OpenAPI.ViewModels.Instructs;
@@ -17,6 +18,7 @@ public ImageGenerationController(IServiceProvider services, ILogger<InstructMode
1718
_logger = logger;
1819
}
1920

21+
#region Image composition
2022
[HttpPost("/instruct/image-composition")]
2123
public async Task<ImageGenerationViewModel> ComposeImages([FromBody] ImageCompositionRequest request)
2224
{
@@ -52,4 +54,275 @@ public async Task<ImageGenerationViewModel> ComposeImages([FromBody] ImageCompos
5254
return imageViewModel;
5355
}
5456
}
57+
#endregion
58+
59+
#region Image generation
60+
[HttpPost("/instruct/image-generation")]
61+
public async Task<ImageGenerationViewModel> ImageGeneration([FromBody] ImageGenerationRequest request)
62+
{
63+
var state = _services.GetRequiredService<IConversationStateService>();
64+
request.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External));
65+
var imageViewModel = new ImageGenerationViewModel();
66+
67+
try
68+
{
69+
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
70+
var message = await fileInstruct.GenerateImage(request.Text, new InstructOptions
71+
{
72+
Provider = request.Provider,
73+
Model = request.Model,
74+
AgentId = request.AgentId,
75+
TemplateName = request.TemplateName
76+
});
77+
imageViewModel.Content = message.Content;
78+
imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? [];
79+
return imageViewModel;
80+
}
81+
catch (Exception ex)
82+
{
83+
var error = $"Error in image generation. {ex.Message}";
84+
_logger.LogError(ex, error);
85+
imageViewModel.Message = error;
86+
return imageViewModel;
87+
}
88+
}
89+
#endregion
90+
91+
#region Image variation
92+
[HttpPost("/instruct/image-variation")]
93+
public async Task<ImageGenerationViewModel> ImageVariation([FromBody] ImageVariationFileRequest request)
94+
{
95+
var state = _services.GetRequiredService<IConversationStateService>();
96+
request.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External));
97+
var imageViewModel = new ImageGenerationViewModel();
98+
99+
try
100+
{
101+
if (request.File == null)
102+
{
103+
return new ImageGenerationViewModel { Message = "Error! Cannot find an image!" };
104+
}
105+
106+
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
107+
var message = await fileInstruct.VaryImage(request.File, new InstructOptions
108+
{
109+
Provider = request.Provider,
110+
Model = request.Model,
111+
AgentId = request.AgentId,
112+
ImageConvertProvider = request.ImageConvertProvider
113+
});
114+
115+
imageViewModel.Content = message.Content;
116+
imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? [];
117+
return imageViewModel;
118+
}
119+
catch (Exception ex)
120+
{
121+
var error = $"Error in image variation. {ex.Message}";
122+
_logger.LogError(ex, error);
123+
imageViewModel.Message = error;
124+
return imageViewModel;
125+
}
126+
}
127+
128+
[HttpPost("/instruct/image-variation/form")]
129+
public async Task<ImageGenerationViewModel> ImageVariation(IFormFile file, [FromForm] ImageVariationRequest request)
130+
{
131+
var state = _services.GetRequiredService<IConversationStateService>();
132+
request?.States?.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External));
133+
var imageViewModel = new ImageGenerationViewModel();
134+
135+
try
136+
{
137+
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
138+
var fileData = FileUtility.BuildFileDataFromFile(file);
139+
var message = await fileInstruct.VaryImage(new InstructFileModel
140+
{
141+
FileData = fileData,
142+
FileName = Path.GetFileNameWithoutExtension(file.FileName),
143+
FileExtension = Path.GetExtension(file.FileName)
144+
},
145+
new InstructOptions
146+
{
147+
Provider = request?.Provider,
148+
Model = request?.Model,
149+
AgentId = request?.AgentId,
150+
ImageConvertProvider = request?.ImageConvertProvider
151+
});
152+
153+
imageViewModel.Content = message.Content;
154+
imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? [];
155+
return imageViewModel;
156+
}
157+
catch (Exception ex)
158+
{
159+
var error = $"Error in image variation upload. {ex.Message}";
160+
_logger.LogError(ex, error);
161+
imageViewModel.Message = error;
162+
return imageViewModel;
163+
}
164+
}
165+
#endregion
166+
167+
#region Image edit
168+
[HttpPost("/instruct/image-edit")]
169+
public async Task<ImageGenerationViewModel> ImageEdit([FromBody] ImageEditFileRequest request)
170+
{
171+
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
172+
var state = _services.GetRequiredService<IConversationStateService>();
173+
request.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External));
174+
var imageViewModel = new ImageGenerationViewModel();
175+
176+
try
177+
{
178+
if (request.File == null)
179+
{
180+
return new ImageGenerationViewModel { Message = "Error! Cannot find a valid image file!" };
181+
}
182+
var message = await fileInstruct.EditImage(request.Text, request.File, new InstructOptions
183+
{
184+
Provider = request.Provider,
185+
Model = request.Model,
186+
AgentId = request.AgentId,
187+
TemplateName = request.TemplateName,
188+
ImageConvertProvider = request.ImageConvertProvider
189+
});
190+
imageViewModel.Content = message.Content;
191+
imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? [];
192+
return imageViewModel;
193+
}
194+
catch (Exception ex)
195+
{
196+
var error = $"Error in image edit. {ex.Message}";
197+
_logger.LogError(ex, error);
198+
imageViewModel.Message = error;
199+
return imageViewModel;
200+
}
201+
}
202+
203+
[HttpPost("/instruct/image-edit/form")]
204+
public async Task<ImageGenerationViewModel> ImageEdit(IFormFile file, [FromForm] ImageEditRequest request)
205+
{
206+
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
207+
var state = _services.GetRequiredService<IConversationStateService>();
208+
request?.States?.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External));
209+
var imageViewModel = new ImageGenerationViewModel();
210+
211+
try
212+
{
213+
var fileData = FileUtility.BuildFileDataFromFile(file);
214+
var message = await fileInstruct.EditImage(request?.Text ?? string.Empty, new InstructFileModel
215+
{
216+
FileData = fileData,
217+
FileName = Path.GetFileNameWithoutExtension(file.FileName),
218+
FileExtension = Path.GetExtension(file.FileName)
219+
},
220+
new InstructOptions
221+
{
222+
Provider = request?.Provider,
223+
Model = request?.Model,
224+
AgentId = request?.AgentId,
225+
TemplateName = request?.TemplateName,
226+
ImageConvertProvider = request?.ImageConvertProvider
227+
});
228+
229+
imageViewModel.Content = message.Content;
230+
imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? [];
231+
return imageViewModel;
232+
}
233+
catch (Exception ex)
234+
{
235+
var error = $"Error in image edit upload. {ex.Message}";
236+
_logger.LogError(ex, error);
237+
imageViewModel.Message = error;
238+
return imageViewModel;
239+
}
240+
}
241+
#endregion
242+
243+
#region Image mask edit
244+
[HttpPost("/instruct/image-mask-edit")]
245+
public async Task<ImageGenerationViewModel> ImageMaskEdit([FromBody] ImageMaskEditFileRequest request)
246+
{
247+
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
248+
var state = _services.GetRequiredService<IConversationStateService>();
249+
request.States.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External));
250+
var imageViewModel = new ImageGenerationViewModel();
251+
252+
try
253+
{
254+
var image = request.File;
255+
var mask = request.Mask;
256+
if (image == null || mask == null)
257+
{
258+
return new ImageGenerationViewModel { Message = "Error! Cannot find a valid image or mask!" };
259+
}
260+
var message = await fileInstruct.EditImage(request.Text, image, mask, new InstructOptions
261+
{
262+
Provider = request.Provider,
263+
Model = request.Model,
264+
AgentId = request.AgentId,
265+
TemplateName = request.TemplateName,
266+
ImageConvertProvider = request.ImageConvertProvider
267+
});
268+
imageViewModel.Content = message.Content;
269+
imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? [];
270+
return imageViewModel;
271+
}
272+
catch (Exception ex)
273+
{
274+
var error = $"Error in image mask edit. {ex.Message}";
275+
_logger.LogError(ex, error);
276+
imageViewModel.Message = error;
277+
return imageViewModel;
278+
}
279+
}
280+
281+
[HttpPost("/instruct/image-mask-edit/form")]
282+
public async Task<ImageGenerationViewModel> ImageMaskEdit(IFormFile image, IFormFile mask, [FromForm] ImageMaskEditRequest request)
283+
{
284+
var fileInstruct = _services.GetRequiredService<IFileInstructService>();
285+
var state = _services.GetRequiredService<IConversationStateService>();
286+
request?.States?.ForEach(x => state.SetState(x.Key, x.Value, source: StateSource.External));
287+
var imageViewModel = new ImageGenerationViewModel();
288+
289+
try
290+
{
291+
var imageData = FileUtility.BuildFileDataFromFile(image);
292+
var maskData = FileUtility.BuildFileDataFromFile(mask);
293+
var message = await fileInstruct.EditImage(request?.Text ?? string.Empty,
294+
new InstructFileModel
295+
{
296+
FileData = imageData,
297+
FileName = Path.GetFileNameWithoutExtension(image.FileName),
298+
FileExtension = Path.GetExtension(image.FileName)
299+
},
300+
new InstructFileModel
301+
{
302+
FileData = maskData,
303+
FileName = Path.GetFileNameWithoutExtension(mask.FileName),
304+
FileExtension = Path.GetExtension(mask.FileName)
305+
},
306+
new InstructOptions
307+
{
308+
Provider = request?.Provider,
309+
Model = request?.Model,
310+
AgentId = request?.AgentId,
311+
TemplateName = request?.TemplateName,
312+
ImageConvertProvider = request?.ImageConvertProvider
313+
});
314+
315+
imageViewModel.Content = message.Content;
316+
imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? [];
317+
return imageViewModel;
318+
}
319+
catch (Exception ex)
320+
{
321+
var error = $"Error in image mask edit upload. {ex.Message}";
322+
_logger.LogError(ex, error);
323+
imageViewModel.Message = error;
324+
return imageViewModel;
325+
}
326+
}
327+
#endregion
55328
}

0 commit comments

Comments
 (0)