From 06f66932163c52738482bef411af4ba44434cbf2 Mon Sep 17 00:00:00 2001 From: srebrek Date: Tue, 13 Jan 2026 10:13:41 +0100 Subject: [PATCH 1/2] feat: Implement tool calling to local LLMs --- .../Examples/Chat/ChatExampleToolsSimple.cs | 8 +- .../Chat/ChatExampleToolsSimpleLocalLLM.cs | 27 ++ Examples/Examples/Program.cs | 2 + .../Services/LLMService/LLMService.cs | 330 +++++++++++++++++- 4 files changed, 360 insertions(+), 7 deletions(-) create mode 100644 Examples/Examples/Chat/ChatExampleToolsSimpleLocalLLM.cs diff --git a/Examples/Examples/Chat/ChatExampleToolsSimple.cs b/Examples/Examples/Chat/ChatExampleToolsSimple.cs index 9e0226d..547bf7a 100644 --- a/Examples/Examples/Chat/ChatExampleToolsSimple.cs +++ b/Examples/Examples/Chat/ChatExampleToolsSimple.cs @@ -11,11 +11,13 @@ public async Task Start() { OpenAiExample.Setup(); //We need to provide OpenAi API key - Console.WriteLine("(OpenAi) ChatExample with tools is running!"); + Console.WriteLine("(OpenAi) ChatExample with tools is running!"); + + var model = AIHub.Model(); await AIHub.Chat() .WithModel("gpt-5-nano") - .WithMessage("What time is it right now?") + .WithMessage("What time is it right now? Use tool provided.") .WithTools(new ToolsConfigurationBuilder() .AddTool( name: "get_current_time", @@ -25,4 +27,4 @@ await AIHub.Chat() .Build()) .CompleteAsync(interactive: true); } -} \ No newline at end of file +} diff --git a/Examples/Examples/Chat/ChatExampleToolsSimpleLocalLLM.cs b/Examples/Examples/Chat/ChatExampleToolsSimpleLocalLLM.cs new file mode 100644 index 0000000..c947332 --- /dev/null +++ b/Examples/Examples/Chat/ChatExampleToolsSimpleLocalLLM.cs @@ -0,0 +1,27 @@ +using Examples.Utils; +using MaIN.Core.Hub; +using MaIN.Core.Hub.Utils; + +namespace Examples.Chat; + +public class ChatExampleToolsSimpleLocalLLM : IExample +{ + public async Task Start() + { + Console.WriteLine("Local LLM ChatExample with tools is running!"); + + var model = AIHub.Model(); + + await AIHub.Chat() + .WithModel("gemma3:4b") + .WithMessage("What time is it right now? Use tool provided.") + .WithTools(new ToolsConfigurationBuilder() + .AddTool( + name: "get_current_time", + description: "Get the current date and time", + execute: Tools.GetCurrentTime) + .WithToolChoice("auto") + .Build()) + .CompleteAsync(interactive: true); + } +} \ No newline at end of file diff --git a/Examples/Examples/Program.cs b/Examples/Examples/Program.cs index 80cc3d6..215fd3c 100644 --- a/Examples/Examples/Program.cs +++ b/Examples/Examples/Program.cs @@ -52,6 +52,7 @@ static void RegisterExamples(IServiceCollection services) services.AddTransient(); services.AddTransient(); services.AddTransient(); + services.AddTransient(); services.AddTransient(); services.AddTransient(); services.AddTransient(); @@ -146,6 +147,7 @@ public class ExampleRegistry(IServiceProvider serviceProvider) ("\u25a0 Chat with Files from stream", serviceProvider.GetRequiredService()), ("\u25a0 Chat with Vision", serviceProvider.GetRequiredService()), ("\u25a0 Chat with Tools (simple)", serviceProvider.GetRequiredService()), + ("\u25a0 Chat with Tools (simple) Local LLM", serviceProvider.GetRequiredService()), ("\u25a0 Chat with Image Generation", serviceProvider.GetRequiredService()), ("\u25a0 Chat from Existing", serviceProvider.GetRequiredService()), ("\u25a0 Chat with reasoning", serviceProvider.GetRequiredService()), diff --git a/src/MaIN.Services/Services/LLMService/LLMService.cs b/src/MaIN.Services/Services/LLMService/LLMService.cs index f45bcea..3bf9581 100644 --- a/src/MaIN.Services/Services/LLMService/LLMService.cs +++ b/src/MaIN.Services/Services/LLMService/LLMService.cs @@ -1,5 +1,6 @@ using System.Collections.Concurrent; using System.Text; +using System.Text.Json; using LLama; using LLama.Batched; using LLama.Common; @@ -7,6 +8,7 @@ using LLama.Sampling; using MaIN.Domain.Configuration; using MaIN.Domain.Entities; +using MaIN.Domain.Entities.Tools; using MaIN.Domain.Models; using MaIN.Services.Constants; using MaIN.Services.Services.Abstract; @@ -14,6 +16,7 @@ using MaIN.Services.Services.LLMService.Utils; using MaIN.Services.Services.Models; using MaIN.Services.Utils; +using Microsoft.Extensions.Logging; using Microsoft.KernelMemory; using Grammar = LLama.Sampling.Grammar; using InferenceParams = MaIN.Domain.Entities.InferenceParams; @@ -25,6 +28,7 @@ public class LLMService : ILLMService { private const string DEFAULT_MODEL_ENV_PATH = "MaIN_ModelsPath"; private static readonly ConcurrentDictionary _sessionCache = new(); + private const int MaxToolIterations = 5; private readonly MaINSettings options; private readonly INotificationService notificationService; @@ -61,6 +65,11 @@ public LLMService( return await AskMemory(chat, memoryOptions, requestOptions, cancellationToken); } + if (chat.ToolsConfiguration?.Tools != null && chat.ToolsConfiguration.Tools.Any()) + { + return await ProcessWithToolsAsync(chat, requestOptions, cancellationToken); + } + var model = KnownModels.GetModel(chat.Model); var tokens = await ProcessChatRequest(chat, model, lastMsg, requestOptions, cancellationToken); lastMsg.MarkProcessed(); @@ -318,16 +327,26 @@ private static void ProcessTextMessage(Conversation conversation, var template = new LLamaTemplate(llmModel); var finalPrompt = ChatHelper.GetFinalPrompt(lastMsg, model, isNewConversation); + var hasTools = chat.ToolsConfiguration?.Tools != null && chat.ToolsConfiguration.Tools.Any(); + if (isNewConversation) { - foreach (var messageToProcess in chat.Messages - .Where(x => x.Properties.ContainsKey(Message.UnprocessedMessageProperty)) - .SkipLast(1)) + var messagesToProcess = hasTools + ? chat.Messages.SkipLast(1) + : chat.Messages.Where(x => x.Properties.ContainsKey(Message.UnprocessedMessageProperty)).SkipLast(1); + + foreach (var messageToProcess in messagesToProcess) { template.Add(messageToProcess.Role, messageToProcess.Content); } } + if (hasTools) + { + var toolsPrompt = FormatToolsForPrompt(chat.ToolsConfiguration!); + finalPrompt = $"{toolsPrompt}\n\n{finalPrompt}"; + } + template.Add(ServiceConstants.Roles.User, finalPrompt); template.AddAssistant = true; @@ -339,6 +358,151 @@ private static void ProcessTextMessage(Conversation conversation, conversation.Prompt(tokens); } + private static string FormatToolsForPrompt(ToolsConfiguration toolsConfig) + { + var sb = new StringBuilder(); + sb.AppendLine("## TOOLS"); + sb.AppendLine("You can call these tools if needed. To call a tool, respond with a JSON object inside tags."); + + foreach (var tool in toolsConfig.Tools) + { + // TODO: refactor to not allow null Function + sb.AppendLine($"- {tool.Function.Name}: {tool.Function.Description}"); + sb.AppendLine($" Parameters: {JsonSerializer.Serialize(tool.Function.Parameters)}"); + } + + sb.AppendLine("\n## RESPONSE FORMAT"); + sb.AppendLine("1. For normal conversation, just respond with plain text."); + sb.AppendLine("2. For tool calls, use this format:"); + sb.AppendLine(""); + sb.AppendLine("{\"tool_calls\": [{\"id\": \"abc\", \"type\": \"function\", \"function\": {\"name\": \"fn\", \"arguments\": \"{\\\"p\\\":\\\"v\\\"}\"}}]}"); + sb.AppendLine(""); + + return sb.ToString(); + } + + private List? ParseToolCalls(string response) + { + if (string.IsNullOrWhiteSpace(response)) return null; + + try + { + string jsonContent = ExtractJsonContent(response); + if (string.IsNullOrEmpty(jsonContent)) return null; + + using var doc = JsonDocument.Parse(jsonContent); + var root = doc.RootElement; + + // OpenAI standard { "tool_calls": [...] } + if (root.ValueKind == JsonValueKind.Object && root.TryGetProperty("tool_calls", out var toolCallsProp)) + { + var calls = toolCallsProp.Deserialize>(new JsonSerializerOptions { PropertyNameCaseInsensitive = true }); + return NormalizeToolCalls(calls); + } + + // TODO: test if those formats are used by any model + // model returned table [ { ... }, { ... } ] + if (root.ValueKind == JsonValueKind.Array) + { + var calls = root.Deserialize>(new JsonSerializerOptions { PropertyNameCaseInsensitive = true }); + return NormalizeToolCalls(calls); + } + + // flat format { "tool_name": "...", "arguments": {...} } + if (root.ValueKind == JsonValueKind.Object && (root.TryGetProperty("tool_name", out _) || root.TryGetProperty("function", out _))) + { + var singleCall = ParseSingleLegacyCall(root); + if (singleCall != null) return new List { singleCall }; + } + } + catch (Exception) + { + // No tool calls found + } + + return null; + } + + private string ExtractJsonContent(string text) + { + text = text.Trim(); + + int firstBrace = text.IndexOf('{'); + int firstBracket = text.IndexOf('['); + int startIndex = (firstBrace >= 0 && firstBracket >= 0) ? Math.Min(firstBrace, firstBracket) : Math.Max(firstBrace, firstBracket); + + int lastBrace = text.LastIndexOf('}'); + int lastBracket = text.LastIndexOf(']'); + int endIndex = Math.Max(lastBrace, lastBracket); + + if (startIndex >= 0 && endIndex > startIndex) + { + return text.Substring(startIndex, endIndex - startIndex + 1); + } + + return text; + } + + private ToolCall? ParseSingleLegacyCall(JsonElement root) + { + string name = string.Empty; + if (root.TryGetProperty("tool_name", out var tn)) name = tn.GetString(); + else if (root.TryGetProperty("function", out var fn) && fn.ValueKind == JsonValueKind.String) name = fn.GetString(); + else if (root.TryGetProperty("function", out var fnObj) && fnObj.TryGetProperty("name", out var n)) name = n.GetString(); + + if (string.IsNullOrEmpty(name)) return null; + + string? args = "{}"; + if (root.TryGetProperty("arguments", out var argProp)) + { + args = argProp.ValueKind == JsonValueKind.String ? argProp.GetString() : argProp.GetRawText(); + } + else if (root.TryGetProperty("parameters", out var paramProp)) + { + args = paramProp.GetRawText(); + } + + return new ToolCall + { + Id = Guid.NewGuid().ToString().Substring(0, 8), + Type = "function", + Function = new FunctionCall { Name = name, Arguments = args! } + }; + } + + private List NormalizeToolCalls(List? calls) + { + if (calls == null) return new List(); + foreach (var call in calls) + { + if (string.IsNullOrEmpty(call.Id)) call.Id = Guid.NewGuid().ToString().Substring(0, 8); + if (string.IsNullOrEmpty(call.Type)) call.Type = "function"; + if (call.Function == null) call.Function = new FunctionCall(); + } + return calls; + } + + public class ToolCall + { + [System.Text.Json.Serialization.JsonPropertyName("id")] + public string Id { get; set; } = Guid.NewGuid().ToString(); + + [System.Text.Json.Serialization.JsonPropertyName("type")] + public string Type { get; set; } = "function"; + + [System.Text.Json.Serialization.JsonPropertyName("function")] + public FunctionCall Function { get; set; } = new(); + } + + public class FunctionCall + { + [System.Text.Json.Serialization.JsonPropertyName("name")] + public string Name { get; set; } = string.Empty; + + [System.Text.Json.Serialization.JsonPropertyName("arguments")] + public string Arguments { get; set; } = "{}"; + } + private async Task<(List Tokens, bool IsComplete, bool HasFailed)> ProcessTokens( Chat chat, Conversation conversation, @@ -479,4 +643,162 @@ await notificationService.DispatchNotification( NotificationMessageBuilder.CreateChatCompletion(chatId, token, isComplete), ServiceConstants.Notifications.ReceiveMessageUpdate); } -} \ No newline at end of file + + private async Task ProcessWithToolsAsync( + Chat chat, + ChatRequestOptions requestOptions, + CancellationToken cancellationToken) + { + var model = KnownModels.GetModel(chat.Model); + var tokens = new List(); + var fullResponseBuilder = new StringBuilder(); + var iterations = 0; + + while (iterations < MaxToolIterations) + { + if (iterations > 0 && requestOptions.InteractiveUpdates && fullResponseBuilder.Length > 0) + { + var spaceToken = new LLMTokenValue { Text = " ", Type = TokenType.Message }; + tokens.Add(spaceToken); + + requestOptions.TokenCallback?.Invoke(spaceToken); + + await notificationService.DispatchNotification( + NotificationMessageBuilder.CreateChatCompletion(chat.Id, spaceToken, false), + ServiceConstants.Notifications.ReceiveMessageUpdate); + } + + var lastMsg = chat.Messages.Last(); + var iterationTokens = await ProcessChatRequest(chat, model, lastMsg, requestOptions, cancellationToken); + + var responseText = string.Concat(iterationTokens.Select(x => x.Text)); + + if (fullResponseBuilder.Length > 0) + { + fullResponseBuilder.Append(" "); + } + fullResponseBuilder.Append(responseText); + tokens.AddRange(iterationTokens); + + var toolCalls = ParseToolCalls(responseText); + + if (toolCalls == null || !toolCalls.Any()) + { + break; + } + + var assistantMessage = new Message + { + Content = responseText, + Role = AuthorRole.Assistant.ToString(), + Type = MessageType.LocalLLM, + Tool = true + }; + assistantMessage.Properties[ToolCallsProperty] = JsonSerializer.Serialize(toolCalls); + chat.Messages.Add(assistantMessage.MarkProcessed()); + + foreach (var toolCall in toolCalls) + { + if (chat.Properties.CheckProperty(ServiceConstants.Properties.AgentIdProperty)) + { + await notificationService.DispatchNotification( + NotificationMessageBuilder.ProcessingTools( + chat.Properties[ServiceConstants.Properties.AgentIdProperty], + string.Empty, + toolCall.Function.Name), + ServiceConstants.Notifications.ReceiveAgentUpdate); + } + + var executor = chat.ToolsConfiguration?.GetExecutor(toolCall.Function.Name); + + if (executor == null) + { + var errorMessage = $"No executor found for tool: {toolCall.Function.Name}"; + throw new InvalidOperationException(errorMessage); + } + + + try + { + requestOptions.ToolCallback?.Invoke(new ToolInvocation + { + ToolName = toolCall.Function.Name, + Arguments = toolCall.Function.Arguments, + Done = false + }); + + var toolResult = await executor(toolCall.Function.Arguments); + + requestOptions.ToolCallback?.Invoke(new ToolInvocation + { + ToolName = toolCall.Function.Name, + Arguments = toolCall.Function.Arguments, + Done = true + }); + + var toolMessage = new Message + { + Content = $"Tool result for {toolCall.Function.Name}: {toolResult}", + Role = ServiceConstants.Roles.Tool, + Type = MessageType.LocalLLM, + Tool = true + }; + toolMessage.Properties[ToolCallIdProperty] = toolCall.Id; + toolMessage.Properties[ToolNameProperty] = toolCall.Function.Name; + chat.Messages.Add(toolMessage.MarkProcessed()); + } + catch (Exception ex) + { + var errorResult = JsonSerializer.Serialize(new { error = ex.Message }); + var toolMessage = new Message + { + Content = $"Tool error for {toolCall.Function.Name}: {errorResult}", + Role = ServiceConstants.Roles.Tool, + Type = MessageType.LocalLLM, + Tool = true + }; + toolMessage.Properties[ToolCallIdProperty] = toolCall.Id; + toolMessage.Properties[ToolNameProperty] = toolCall.Function.Name; + chat.Messages.Add(toolMessage.MarkProcessed()); + } + } + + iterations++; + } + + if (iterations >= MaxToolIterations) + { + } + + var finalResponse = fullResponseBuilder.ToString(); + var finalToken = new LLMTokenValue { Text = finalResponse, Type = TokenType.FullAnswer }; + tokens.Add(finalToken); + + if (requestOptions.InteractiveUpdates) + { + await notificationService.DispatchNotification( + NotificationMessageBuilder.CreateChatCompletion(chat.Id, finalToken, true), + ServiceConstants.Notifications.ReceiveMessageUpdate); + } + + chat.Messages.Last().MarkProcessed(); + + return new ChatResult + { + Done = true, + CreatedAt = DateTime.Now, + Model = chat.Model, + Message = new Message + { + Content = finalResponse, + Tokens = tokens, + Role = AuthorRole.Assistant.ToString(), + Type = MessageType.LocalLLM, + }.MarkProcessed() + }; + } + + private const string ToolCallsProperty = "ToolCalls"; + private const string ToolCallIdProperty = "ToolCallId"; + private const string ToolNameProperty = "ToolName"; +} From 4ae460dffa5c3a532965ac33551b5ed63c545233 Mon Sep 17 00:00:00 2001 From: srebrek Date: Wed, 21 Jan 2026 17:19:50 +0100 Subject: [PATCH 2/2] Fix tool calling logic in multiple iterations loop - Prevent tool definition duplication in the system prompt during subsequent loop iterations. - Refine system prompt to enforce format more effectively. --- .../Examples/Chat/ChatExampleToolsSimple.cs | 2 +- .../Chat/ChatExampleToolsSimpleLocalLLM.cs | 2 +- .../Services/LLMService/LLMService.cs | 84 +++++++++++-------- 3 files changed, 49 insertions(+), 39 deletions(-) diff --git a/Examples/Examples/Chat/ChatExampleToolsSimple.cs b/Examples/Examples/Chat/ChatExampleToolsSimple.cs index 547bf7a..14d1ecf 100644 --- a/Examples/Examples/Chat/ChatExampleToolsSimple.cs +++ b/Examples/Examples/Chat/ChatExampleToolsSimple.cs @@ -17,7 +17,7 @@ public async Task Start() await AIHub.Chat() .WithModel("gpt-5-nano") - .WithMessage("What time is it right now? Use tool provided.") + .WithMessage("What time is it right now?") .WithTools(new ToolsConfigurationBuilder() .AddTool( name: "get_current_time", diff --git a/Examples/Examples/Chat/ChatExampleToolsSimpleLocalLLM.cs b/Examples/Examples/Chat/ChatExampleToolsSimpleLocalLLM.cs index c947332..03ff7f3 100644 --- a/Examples/Examples/Chat/ChatExampleToolsSimpleLocalLLM.cs +++ b/Examples/Examples/Chat/ChatExampleToolsSimpleLocalLLM.cs @@ -14,7 +14,7 @@ public async Task Start() await AIHub.Chat() .WithModel("gemma3:4b") - .WithMessage("What time is it right now? Use tool provided.") + .WithMessage("What time is it right now?") .WithTools(new ToolsConfigurationBuilder() .AddTool( name: "get_current_time", diff --git a/src/MaIN.Services/Services/LLMService/LLMService.cs b/src/MaIN.Services/Services/LLMService/LLMService.cs index 3bf9581..452763b 100644 --- a/src/MaIN.Services/Services/LLMService/LLMService.cs +++ b/src/MaIN.Services/Services/LLMService/LLMService.cs @@ -16,7 +16,6 @@ using MaIN.Services.Services.LLMService.Utils; using MaIN.Services.Services.Models; using MaIN.Services.Utils; -using Microsoft.Extensions.Logging; using Microsoft.KernelMemory; using Grammar = LLama.Sampling.Grammar; using InferenceParams = MaIN.Domain.Entities.InferenceParams; @@ -36,6 +35,11 @@ public class LLMService : ILLMService private readonly IMemoryFactory memoryFactory; private readonly string modelsPath; + private readonly JsonSerializerOptions _jsonToolOptions = new() + { + PropertyNameCaseInsensitive = true, + }; + public LLMService( MaINSettings options, INotificationService notificationService, @@ -341,9 +345,10 @@ private static void ProcessTextMessage(Conversation conversation, } } - if (hasTools) + if (hasTools && isNewConversation) { var toolsPrompt = FormatToolsForPrompt(chat.ToolsConfiguration!); + // Dodaj to jako wiadomość systemową lub na początku pierwszego promptu użytkownika finalPrompt = $"{toolsPrompt}\n\n{finalPrompt}"; } @@ -371,11 +376,14 @@ private static string FormatToolsForPrompt(ToolsConfiguration toolsConfig) sb.AppendLine($" Parameters: {JsonSerializer.Serialize(tool.Function.Parameters)}"); } - sb.AppendLine("\n## RESPONSE FORMAT"); + sb.AppendLine("\n## RESPONSE FORMAT (YOU HAVE TO CHOOSE ONE FORMAT AND CANNOT MIX THEM)##"); sb.AppendLine("1. For normal conversation, just respond with plain text."); - sb.AppendLine("2. For tool calls, use this format:"); + sb.AppendLine("2. For tool calls, use this format. " + + "You cannot respond with plain text before or after format. " + + "If you want to call multiple functions, you have to combine them into one array." + + "Your response MUST contain only one tool call block:"); sb.AppendLine(""); - sb.AppendLine("{\"tool_calls\": [{\"id\": \"abc\", \"type\": \"function\", \"function\": {\"name\": \"fn\", \"arguments\": \"{\\\"p\\\":\\\"v\\\"}\"}}]}"); + sb.AppendLine("{\"tool_calls\": [{\"id\": \"call_1\", \"type\": \"function\", \"function\": {\"name\": \"tool_name\", \"arguments\": \"{\\\"param\\\":\\\"value\\\"}\"}},{\"id\": \"call_2\", \"type\": \"function\", \"function\": {\"name\": \"tool2_name\", \"arguments\": \"{\\\"param1\\\":\\\"value1\\\",\\\"param2\\\":\\\"value2\\\"}\"}}]}"); sb.AppendLine(""); return sb.ToString(); @@ -385,9 +393,9 @@ private static string FormatToolsForPrompt(ToolsConfiguration toolsConfig) { if (string.IsNullOrWhiteSpace(response)) return null; + string jsonContent = ExtractJsonContent(response); try { - string jsonContent = ExtractJsonContent(response); if (string.IsNullOrEmpty(jsonContent)) return null; using var doc = JsonDocument.Parse(jsonContent); @@ -396,7 +404,7 @@ private static string FormatToolsForPrompt(ToolsConfiguration toolsConfig) // OpenAI standard { "tool_calls": [...] } if (root.ValueKind == JsonValueKind.Object && root.TryGetProperty("tool_calls", out var toolCallsProp)) { - var calls = toolCallsProp.Deserialize>(new JsonSerializerOptions { PropertyNameCaseInsensitive = true }); + var calls = toolCallsProp.Deserialize>(_jsonToolOptions); return NormalizeToolCalls(calls); } @@ -417,7 +425,7 @@ private static string FormatToolsForPrompt(ToolsConfiguration toolsConfig) } catch (Exception) { - // No tool calls found + // No tool calls found no need to throw nor log } return null; @@ -429,14 +437,14 @@ private string ExtractJsonContent(string text) int firstBrace = text.IndexOf('{'); int firstBracket = text.IndexOf('['); - int startIndex = (firstBrace >= 0 && firstBracket >= 0) ? Math.Min(firstBrace, firstBracket) : Math.Max(firstBrace, firstBracket); + int startIndex = (firstBrace >= 0 && firstBracket >= 0) ? Math.Min(firstBrace, firstBracket) : Math.Max(firstBrace, firstBracket); int lastBrace = text.LastIndexOf('}'); int lastBracket = text.LastIndexOf(']'); - int endIndex = Math.Max(lastBrace, lastBracket); + int endIndex = Math.Max(lastBrace, lastBracket); - if (startIndex >= 0 && endIndex > startIndex) - { + if (startIndex >= 0 && endIndex > startIndex) + { return text.Substring(startIndex, endIndex - startIndex + 1); } @@ -648,34 +656,35 @@ private async Task ProcessWithToolsAsync( Chat chat, ChatRequestOptions requestOptions, CancellationToken cancellationToken) - { + { + NativeLogConfig.llama_log_set((level, message) => { + if (level == LLamaLogLevel.Error) + { + Console.Error.Write(message); + } + }); // Remove llama native logging + var model = KnownModels.GetModel(chat.Model); var tokens = new List(); var fullResponseBuilder = new StringBuilder(); var iterations = 0; while (iterations < MaxToolIterations) - { - if (iterations > 0 && requestOptions.InteractiveUpdates && fullResponseBuilder.Length > 0) - { - var spaceToken = new LLMTokenValue { Text = " ", Type = TokenType.Message }; - tokens.Add(spaceToken); - - requestOptions.TokenCallback?.Invoke(spaceToken); - - await notificationService.DispatchNotification( - NotificationMessageBuilder.CreateChatCompletion(chat.Id, spaceToken, false), - ServiceConstants.Notifications.ReceiveMessageUpdate); - } - + { var lastMsg = chat.Messages.Last(); + await SendNotification(chat.Id, new LLMTokenValue + { + Type = TokenType.FullAnswer, + Text = $"Processing with tools... iteration {iterations + 1}\n\n" + }, false); + requestOptions.InteractiveUpdates = false; var iterationTokens = await ProcessChatRequest(chat, model, lastMsg, requestOptions, cancellationToken); var responseText = string.Concat(iterationTokens.Select(x => x.Text)); if (fullResponseBuilder.Length > 0) { - fullResponseBuilder.Append(" "); + fullResponseBuilder.Append('\n'); } fullResponseBuilder.Append(responseText); tokens.AddRange(iterationTokens); @@ -684,6 +693,12 @@ await notificationService.DispatchNotification( if (toolCalls == null || !toolCalls.Any()) { + requestOptions.InteractiveUpdates = true; + await SendNotification(chat.Id, new LLMTokenValue + { + Type = TokenType.FullAnswer, + Text = responseText + }, false); break; } @@ -768,19 +783,14 @@ await notificationService.DispatchNotification( if (iterations >= MaxToolIterations) { + await SendNotification(chat.Id, new LLMTokenValue + { + Type = TokenType.FullAnswer, + Text = "Maximum tool invocation iterations reached. Ending the conversation." + }, false); } var finalResponse = fullResponseBuilder.ToString(); - var finalToken = new LLMTokenValue { Text = finalResponse, Type = TokenType.FullAnswer }; - tokens.Add(finalToken); - - if (requestOptions.InteractiveUpdates) - { - await notificationService.DispatchNotification( - NotificationMessageBuilder.CreateChatCompletion(chat.Id, finalToken, true), - ServiceConstants.Notifications.ReceiveMessageUpdate); - } - chat.Messages.Last().MarkProcessed(); return new ChatResult