diff --git a/AISmart.sln b/AISmart.sln index a4932af2..f374e5fe 100644 --- a/AISmart.sln +++ b/AISmart.sln @@ -93,6 +93,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AISmart.EventSourcing.Mongo EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AISmart.Core", "src\AISmart.Core\AISmart.Core.csproj", "{49868FC0-2F38-4EF4-9719-6578CDF9C453}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AISmart.Evaluate", "src\AISmart.Evaluate\AISmart.Evaluate.csproj", "{E6CED902-9306-4EC4-810D-A23937B4C9FC}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -267,6 +269,10 @@ Global {49868FC0-2F38-4EF4-9719-6578CDF9C453}.Debug|Any CPU.Build.0 = Debug|Any CPU {49868FC0-2F38-4EF4-9719-6578CDF9C453}.Release|Any CPU.ActiveCfg = Release|Any CPU {49868FC0-2F38-4EF4-9719-6578CDF9C453}.Release|Any CPU.Build.0 = Release|Any CPU + {E6CED902-9306-4EC4-810D-A23937B4C9FC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E6CED902-9306-4EC4-810D-A23937B4C9FC}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E6CED902-9306-4EC4-810D-A23937B4C9FC}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E6CED902-9306-4EC4-810D-A23937B4C9FC}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -315,6 +321,7 @@ Global {49868FC0-2F38-4EF4-9719-6578CDF9C453} = {CA9AC87F-097E-4F15-8393-4BC07735A5B0} {482AC386-F3A8-4791-A708-414E52A8177A} = {04DBDB01-70F4-4E06-B468-8F87850B22BE} {89A0820D-CBB0-4061-A939-F30EF486BA93} = {CA9AC87F-097E-4F15-8393-4BC07735A5B0} + {E6CED902-9306-4EC4-810D-A23937B4C9FC} = {CA9AC87F-097E-4F15-8393-4BC07735A5B0} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {28315BFD-90E7-4E14-A2EA-F3D23AF4126F} diff --git a/src/AISmart.Evaluate/AISmart.Evaluate.csproj b/src/AISmart.Evaluate/AISmart.Evaluate.csproj new file mode 100644 index 00000000..857bdbaf --- /dev/null +++ b/src/AISmart.Evaluate/AISmart.Evaluate.csproj @@ -0,0 +1,19 @@ + + + + + + net8.0 + enable + AISmart + + + + + + + + + + + \ No newline at end of file diff --git a/src/AISmart.Evaluate/AISmartEvaluateModule.cs b/src/AISmart.Evaluate/AISmartEvaluateModule.cs new file mode 100644 index 00000000..b138d8b6 --- /dev/null +++ b/src/AISmart.Evaluate/AISmartEvaluateModule.cs @@ -0,0 +1,23 @@ +using AISmart.Evaluate.Service; +using AISmart.Options; +using AISmart.Provider; +using AISmart.Rag; +using Microsoft.Extensions.DependencyInjection; +using Volo.Abp.Modularity; + +namespace AISmart.Evaluate; + +[DependsOn( + typeof(AISmartSimpleRagModule) +)] +public class AISmartEvaluateModule : AbpModule +{ + public override void ConfigureServices(ServiceConfigurationContext context) + { + var configuration = context.Services.GetConfiguration(); + // Configure(configuration.GetSection("EvaluateRag")); + // context.Services.AddSingleton(); + // context.Services.AddSingleton(); + // Configure("EvaluateRag", configuration.GetSection("Evaluate:Rag")); + } +} \ No newline at end of file diff --git a/src/AISmart.Evaluate/Options/EvaluateOptions.cs b/src/AISmart.Evaluate/Options/EvaluateOptions.cs new file mode 100644 index 00000000..14e0f9dc --- /dev/null +++ b/src/AISmart.Evaluate/Options/EvaluateOptions.cs @@ -0,0 +1,8 @@ +using AISmart.Options; + +namespace AISmart.Evaluate.Options; + +public class EvaluateOptions +{ + public RagOptions Rag { get; set; } +} \ No newline at end of file diff --git a/src/AISmart.Evaluate/Service/AISmartEvaluateService.cs b/src/AISmart.Evaluate/Service/AISmartEvaluateService.cs new file mode 100644 index 00000000..79a65142 --- /dev/null +++ b/src/AISmart.Evaluate/Service/AISmartEvaluateService.cs @@ -0,0 +1,42 @@ +using System; +using System.Threading.Tasks; +using AISmart.Provider; +using AISmart.Rag; +using Microsoft.Extensions.Logging; +using Volo.Abp.DependencyInjection; + +namespace AISmart.Evaluate.Service; + +public class AISmartEvaluateService : IAISmartEvaluateService, ISingletonDependency +{ + private readonly ILogger _logger; + private readonly IRagProvider _ragProvider; + // private readonly IOptionsMonitor _evaluateOptions; + private readonly IRagProviderFactory _ragProviderFactory; + + public AISmartEvaluateService(ILogger logger, IRagProviderFactory ragProviderFactory) + { + _logger = logger; + _ragProvider = ragProviderFactory.GetProvider("EvaluateRag"); + } + + public async Task EvaluateAsync(string task, string result) + { + throw new NotImplementedException(); + } + + public async Task AddExceptionMessageAsync(string task, string exceptionMessage) + { + var text = + $""" + Task is: {task} + Exception {exceptionMessage} was caught during execution, please pay attention! + """; + await _ragProvider.StoreTextAsync(text); + } + + public async Task GetAdviceAsync(string task) + { + return await _ragProvider.RetrieveAnswerAsync(task); + } +} \ No newline at end of file diff --git a/src/AISmart.Evaluate/Service/IAISmartEvaluateService.cs b/src/AISmart.Evaluate/Service/IAISmartEvaluateService.cs new file mode 100644 index 00000000..0edcc783 --- /dev/null +++ b/src/AISmart.Evaluate/Service/IAISmartEvaluateService.cs @@ -0,0 +1,10 @@ +using System.Threading.Tasks; + +namespace AISmart.Evaluate.Service; + +public interface IAISmartEvaluateService +{ + Task EvaluateAsync(string task, string result); + Task AddExceptionMessageAsync(string task, string exceptionMessage); + Task GetAdviceAsync(string task); +} \ No newline at end of file diff --git a/src/AISmart.GAgent.Autogen/AISmart.GAgent.Autogen.csproj b/src/AISmart.GAgent.Autogen/AISmart.GAgent.Autogen.csproj index 5544810a..0b7b4e2d 100644 --- a/src/AISmart.GAgent.Autogen/AISmart.GAgent.Autogen.csproj +++ b/src/AISmart.GAgent.Autogen/AISmart.GAgent.Autogen.csproj @@ -10,6 +10,7 @@ + diff --git a/src/AISmart.GAgent.Autogen/AISmartGAgentAutogenModule.cs b/src/AISmart.GAgent.Autogen/AISmartGAgentAutogenModule.cs index 9abfcad7..4e7c3361 100644 --- a/src/AISmart.GAgent.Autogen/AISmartGAgentAutogenModule.cs +++ b/src/AISmart.GAgent.Autogen/AISmartGAgentAutogenModule.cs @@ -1,3 +1,5 @@ +using AISmart.Evaluate; +using AISmart.Evaluate.Options; using AISmart.GAgent.Autogen.Options; using AISmart.GAgent.Core; using AISmart.Options; @@ -13,7 +15,8 @@ namespace AISmart.GAgent.Autogen; [DependsOn( typeof(AISmartApplicationContractsModule), - typeof(AISmartSimpleRagModule) + typeof(AISmartSimpleRagModule), + typeof(AISmartEvaluateModule) )] public class AISmartGAgentAutogenModule : AbpModule { @@ -26,6 +29,11 @@ public override void ConfigureServices(ServiceConfigurationContext context) // context.Services.AddTransient(op => new ChatClient(autogenConfig.Model, autogenConfig.ApiKey)); context.Services.AddTransient(); // context.Services.AddTransient(); - Configure(configuration.GetSection("AutogenConfig:AutoGenRag")); + // Configure(configuration.GetSection("AutogenConfig:AutoGenRag")); + Configure("AutogenRag", configuration.GetSection("AutogenConfig:AutoGenRag")); + Configure("EvaluateRag", configuration.GetSection("Evaluate:Rag")); + + + } } \ No newline at end of file diff --git a/src/AISmart.GAgent.Autogen/AutogenGAgent.cs b/src/AISmart.GAgent.Autogen/AutogenGAgent.cs index 19bb7f31..c32a79ea 100644 --- a/src/AISmart.GAgent.Autogen/AutogenGAgent.cs +++ b/src/AISmart.GAgent.Autogen/AutogenGAgent.cs @@ -31,9 +31,9 @@ public class AutogenGAgent : GAgentBase, IA private readonly int _maxRaiseEventCount = 50; public AutogenGAgent(ILogger logger, - IRagProvider ragProvider) : base(logger) + IRagProviderFactory ragProviderFactory) : base(logger) { - _ragProvider = ragProvider; + _ragProvider = ragProviderFactory.GetProvider("AutogenRag"); } public async Task RegisterAgentEvent(Type agent, List eventTypes) diff --git a/src/AISmart.GAgent.Autogen/Executor/AutoGenExecutor.cs b/src/AISmart.GAgent.Autogen/Executor/AutoGenExecutor.cs index de07c839..935b1776 100644 --- a/src/AISmart.GAgent.Autogen/Executor/AutoGenExecutor.cs +++ b/src/AISmart.GAgent.Autogen/Executor/AutoGenExecutor.cs @@ -2,6 +2,7 @@ using System.Text.Json.Serialization; using AISmart.Agents; using AISmart.Dapr; +using AISmart.Evaluate.Service; using AISmart.GAgent.Autogen.Common; using AISmart.GAgent.Autogen.DescriptionManager; using AISmart.GAgent.Autogen.Event; @@ -24,11 +25,15 @@ public class AutoGenExecutor : Grain, IAutoGenExecutor private const string BreakFlag = "break"; private Guid _taskId; private IAgentDescriptionManager _descriptionManager; + private IAISmartEvaluateService _evaluateService; + public string ExceptionMessage; - public AutoGenExecutor(ILogger logger, IChatAgentProvider chatAgentProvider) + public AutoGenExecutor(ILogger logger, IChatAgentProvider chatAgentProvider, + IAISmartEvaluateService evaluateService) { _logger = logger; _chatAgentProvider = chatAgentProvider; + _evaluateService = evaluateService; } public async Task ExecuteTaskAsync(ExecutorTaskInfo taskInfo) @@ -36,6 +41,18 @@ public async Task ExecuteTaskAsync(ExecutorTaskInfo taskInfo) try { await CallAutogen(taskInfo); + ExceptionMessage = string.Empty; + } + catch (JsonException e) + { + await _evaluateService.AddExceptionMessageAsync(GetTaskDescription(taskInfo.History), e.ToString()); + await PublishInternalEvent(new AutoGenExecutorEvent() + { + TaskId = _taskId, + ExecuteStatus = TaskExecuteStatus.Break, + EndContent = "Internal error", + }); + ExceptionMessage = e.ToString(); } catch (Exception e) { @@ -57,6 +74,8 @@ public async Task CallAutogen(ExecutorTaskInfo taskInfo) _descriptionManager = GrainFactory.GetGrain(taskInfo.AgentDescriptionManagerId); var history = ConvertMessage(taskInfo.History); + // var exceptionAdvice = await _evaluateService.GetAdviceAsync(GetTaskDescription(taskInfo.History)); + // history.Add(new TextMessage(Role.System, exceptionAdvice)); var responsibility = await GetAgentResponsibility(); _chatAgentProvider.SetAgent(AgentName, responsibility, GetMiddleware()); var response = await _chatAgentProvider.SendAsync(AgentName, "What should be done next?", history); @@ -139,6 +158,16 @@ private List ConvertMessage(List listAutoGenMessage) return result; } + + private string GetTaskDescription(List listAutoGenMessage) + { + var list = new List(); + foreach (var item in listAutoGenMessage) + { + list.Add(item.Role + ": " + item.Content); + } + return string.Join("\n", list); + } private Role GetRole(string roleName) { @@ -263,7 +292,7 @@ public async Task HandleEventAsync(string agentName, string eventName, s var descriptionDic = await _descriptionManager.GetAgentDescription(); if (descriptionDic.TryGetValue(agentName, out var eventDescription) == false) { - throw new AutogenException($"Event name:{agentName} not exist"); + throw new AutogenException($"Agent name:{agentName} not exist"); } var eventInfo = eventDescription.EventList.FirstOrDefault(f => f.EventName == eventName); diff --git a/src/AISmart.SimpleRag/Provider/QdrantVectorDatabase.cs b/src/AISmart.SimpleRag/Provider/QdrantVectorDatabase.cs index b153bc29..09c03d15 100644 --- a/src/AISmart.SimpleRag/Provider/QdrantVectorDatabase.cs +++ b/src/AISmart.SimpleRag/Provider/QdrantVectorDatabase.cs @@ -113,6 +113,7 @@ public async Task StoreBatchAsync(IEnumerable<(float[] vector, string text)> poi public async Task> RetrieveAsync(float[] queryEmbedding, int topK = 5) { + await EnsureCollectionExistsAsync(); var requestBody = new { vector = queryEmbedding, top = topK, with_payload = true }; var response = await _httpClient.PostAsJsonAsync($"{_qdrantUrl}/collections/{_collectionName}/points/search", requestBody); response.EnsureSuccessStatusCode(); diff --git a/src/AISmart.SimpleRag/Provider/RagProvider.cs b/src/AISmart.SimpleRag/Provider/RagProvider.cs index c49335fd..b2e77aa3 100644 --- a/src/AISmart.SimpleRag/Provider/RagProvider.cs +++ b/src/AISmart.SimpleRag/Provider/RagProvider.cs @@ -19,23 +19,23 @@ public class RagProvider : IRagProvider, ISingletonDependency private readonly IEmbeddingProvider _embeddingProvider; private readonly IVectorDatabase _vectorDatabase; private readonly ILogger _logger; - private readonly IOptionsMonitor _ragOptions; + private readonly IOptions _ragOptions; - public RagProvider(IOptionsMonitor ragOptions, ILogger logger) + public RagProvider(IOptions ragOptions, ILogger logger) { _ragOptions = ragOptions; _chunker = new SimpleChunker(); - _embeddingProvider = new OpenAIEmbeddingProvider(_ragOptions.CurrentValue.APIKey); - _vectorDatabase = new QdrantVectorDatabase(_ragOptions.CurrentValue.QdrantUrl, - _ragOptions.CurrentValue.CollectionName, - _ragOptions.CurrentValue.VectorSize); + _embeddingProvider = new OpenAIEmbeddingProvider(_ragOptions.Value.APIKey); + _vectorDatabase = new QdrantVectorDatabase(_ragOptions.Value.QdrantUrl, + _ragOptions.Value.CollectionName, + _ragOptions.Value.VectorSize); _logger = logger; } public async Task StoreTextAsync(string text) { _logger.LogInformation("store text {text}", text); - var chunkSize = _ragOptions.CurrentValue.ChunkSize; + var chunkSize = _ragOptions.Value.ChunkSize; var chunks = await _chunker.Chunk(text, chunkSize); foreach (var chunk in chunks) { @@ -67,7 +67,7 @@ public async Task RetrieveAnswerAsync(string query) _logger.LogInformation("retrieve text {query}", query); var queryEmbedding = await _embeddingProvider.GetEmbeddingAsync(query); var relevantChunks = await _vectorDatabase.RetrieveAsync(queryEmbedding, 3); - return string.Join(" ", relevantChunks); + return relevantChunks.IsNullOrEmpty() ? "" : string.Join(" ", relevantChunks); } private async Task BatchStoreFilesAsync(IList files) diff --git a/src/AISmart.SimpleRag/Provider/RagProviderFactory.cs b/src/AISmart.SimpleRag/Provider/RagProviderFactory.cs new file mode 100644 index 00000000..8b0cad21 --- /dev/null +++ b/src/AISmart.SimpleRag/Provider/RagProviderFactory.cs @@ -0,0 +1,39 @@ +using System.Collections.Concurrent; +using AISmart.Options; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Volo.Abp.DependencyInjection; + +namespace AISmart.Provider; + +public class RagProviderFactory : IRagProviderFactory, ISingletonDependency +{ + private readonly IOptionsMonitor _ragOptionsMonitor; + private readonly ConcurrentDictionary _providersDict; + private readonly ILogger _logger; + + public RagProviderFactory(IOptionsMonitor ragOptionsMonitor, ILogger logger) + { + _ragOptionsMonitor = ragOptionsMonitor; + _providersDict = new ConcurrentDictionary(); + _logger = logger; + } + + public RagProvider GetProvider(string configName) + { + var options = _ragOptionsMonitor.Get(configName); + if (_providersDict.TryGetValue(configName, out var provider)) + { + return provider; + } + + provider = new RagProvider(Microsoft.Extensions.Options.Options.Create(options), _logger); + _providersDict[configName] = provider; + return provider; + } +} + +public interface IRagProviderFactory +{ + RagProvider GetProvider(string configName); +} \ No newline at end of file diff --git a/test/AISmart.GAgents.Tests/AutoGen/AutoGenTest.cs b/test/AISmart.GAgents.Tests/AutoGen/AutoGenTest.cs index 38e20334..f4fb09d8 100644 --- a/test/AISmart.GAgents.Tests/AutoGen/AutoGenTest.cs +++ b/test/AISmart.GAgents.Tests/AutoGen/AutoGenTest.cs @@ -8,6 +8,7 @@ using AISmart.GAgents.Tests; using AISmart.Sender; using Orleans.TestKit; +using Shouldly; namespace AISmart.GAgents.Tests.AutoGenTest; @@ -46,4 +47,43 @@ await publishingGAgent.PublishEventAsync(new AutoGenCreatedEvent await Task.Delay(1000); } + + [Fact] + public async Task CorrectAutogenTest() + { + var groupGAgent = await Silo.CreateGrainAsync(Guid.NewGuid()); + var autogenGAgent = await Silo.CreateGrainAsync(Guid.NewGuid()); + var publishingGAgent = await Silo.CreateGrainAsync(Guid.NewGuid()); + var drawGAgent = await Silo.CreateGrainAsync(Guid.NewGuid()); + var mathGAgent = await Silo.CreateGrainAsync(Guid.NewGuid()); + var autoGenExecutor = await Silo.CreateGrainAsync(Guid.NewGuid()); + + autogenGAgent.RegisterAgentEvent(typeof(DrawOperationGAgent), [typeof(DrawOperateEvent)]); + autogenGAgent.RegisterAgentEvent(typeof(MathOperationGAgent), [typeof(AddNumberEvent), typeof(AddNumberResultEvent)]); + + AddProbesByGrainId(autogenGAgent, drawGAgent, mathGAgent, publishingGAgent, groupGAgent); + + await groupGAgent.Register(autogenGAgent); + await groupGAgent.Register(drawGAgent); + await groupGAgent.Register(mathGAgent); + + Silo.AddProbe(_ => publishingGAgent); + Silo.AddProbe(_ => autoGenExecutor); + + await publishingGAgent.PublishTo(groupGAgent); + + Silo.AddStreamProbe(); + + await publishingGAgent.PublishEventAsync(new AutoGenCreatedEvent + { + Content = "What is 3+3, and then generate the corresponding polygon?" + }); + autoGenExecutor.ExceptionMessage.ShouldContain("JsonException"); + + await publishingGAgent.PublishEventAsync(new AutoGenCreatedEvent + { + Content = "What is 3+3, and then generate the corresponding polygon?" + }); + autoGenExecutor.ExceptionMessage.ShouldBeEmpty(); + } } \ No newline at end of file diff --git a/test/OrleansTestKit/TestChatAgentProvider.cs b/test/OrleansTestKit/TestChatAgentProvider.cs index 9143ec31..00000dec 100644 --- a/test/OrleansTestKit/TestChatAgentProvider.cs +++ b/test/OrleansTestKit/TestChatAgentProvider.cs @@ -8,10 +8,27 @@ public class TestChatAgentProvider : IChatAgentProvider, ITransientDependency { public async Task SendAsync(string agentName, string message, IEnumerable chatHistory) { - string jsonResponse = @"{ - ""compete"": ""This is a mock completion message."" -}"; - var response = new Message(Role.System, jsonResponse); +// string jsonResponse = @"{ +// ""compete"": ""This is a mock completion message."" +// }"; +// var response = new Message(Role.System, jsonResponse); +// return response; + + Message response; + foreach (var chat in chatHistory) + { + var content = chat.GetContent(); + if (chat.GetContent().Contains("JsonException")) + { + var correctJsonResponse = "{\"agentName\":\"MathOperationGAgent\",\"eventName\":\"AddNumberEvent\",\"parameters\":\"xxx\"}"; + response = new Message(Role.System, correctJsonResponse); + return response; + } + } + + var wrongJsonResponse = "\"agentName\":\"MathOperationGAgent\",\"eventName\":\"AddNumberEvent\",\"parameters\":\"xxx\"}"; + response = new Message(Role.System, wrongJsonResponse); + return response; } diff --git a/test/OrleansTestKit/TestEvaluateService.cs b/test/OrleansTestKit/TestEvaluateService.cs new file mode 100644 index 00000000..1356f5f5 --- /dev/null +++ b/test/OrleansTestKit/TestEvaluateService.cs @@ -0,0 +1,28 @@ +using AISmart.Evaluate.Service; + +namespace Orleans.TestKit; + +public class TestEvaluateService : IAISmartEvaluateService +{ + private string taskWithException; + private string exceptionMessage; + public async Task EvaluateAsync(string task, string result) + { + return; + } + + public async Task AddExceptionMessageAsync(string task, string exception) + { + taskWithException = task; + exceptionMessage = exception; + } + + public async Task GetAdviceAsync(string task) + { + if (taskWithException == task) + { + return await Task.FromResult(exceptionMessage); + } + return await Task.FromResult(""); + } +} \ No newline at end of file diff --git a/test/OrleansTestKit/TestKitSilo.cs b/test/OrleansTestKit/TestKitSilo.cs index 6d921b85..8deebd4d 100644 --- a/test/OrleansTestKit/TestKitSilo.cs +++ b/test/OrleansTestKit/TestKitSilo.cs @@ -3,8 +3,10 @@ using AISmart.EventSourcing.Core; using AISmart.EventSourcing.Core.LogConsistency; using AISmart.EventSourcing.Core.Storage; +using AISmart.Evaluate.Service; using AISmart.GAgent.Autogen; using AISmart.GAgent.Autogen.Common; +using AISmart.GAgent.Autogen.DescriptionManager; using AISmart.Mock; using AISmart.Provider; using AutoGen.OpenAI; @@ -76,14 +78,14 @@ public TestKitSilo() GrainRuntime = new TestGrainRuntime(GrainFactory, TimerRegistry, ReminderRegistry, ServiceProvider, StorageManager); ServiceProvider.AddService(GrainRuntime); - _grainCreator = new TestGrainCreator(GrainRuntime, ReminderRegistry, TestGrainStorage, ServiceProvider); + _grainCreator = new TestGrainCreator(GrainRuntime, ReminderRegistry, ServiceProvider); ServiceProvider.AddService(new MockAElfNodeProvider()); - // var manager = new AgentDescriptionManager(); - // ServiceProvider.AddService(manager); - // ServiceProvider.AddService(new AutoGenExecutor(NullLogger.Instance, GrainFactory, manager, new TestChatAgentProvider())); - ServiceProvider.AddService(TestGrainStorage); + var manager = new AgentDescriptionManager(); + ServiceProvider.AddService(manager); + ServiceProvider.AddService(new AutoGenExecutor(NullLogger.Instance, GrainFactory, manager, new TestChatAgentProvider())); + var provider = new ServiceCollection() .AddSingleton() .AddSingleton()