[WIP]evaluate module

AISmart.sln

@@ -93,6 +93,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AISmart.EventSourcing.Mongo
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AISmart.Core", "src\AISmart.Core\AISmart.Core.csproj", "{49868FC0-2F38-4EF4-9719-6578CDF9C453}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AISmart.Evaluate", "src\AISmart.Evaluate\AISmart.Evaluate.csproj", "{E6CED902-9306-4EC4-810D-A23937B4C9FC}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -267,6 +269,10 @@ Global
 		{49868FC0-2F38-4EF4-9719-6578CDF9C453}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{49868FC0-2F38-4EF4-9719-6578CDF9C453}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{49868FC0-2F38-4EF4-9719-6578CDF9C453}.Release|Any CPU.Build.0 = Release|Any CPU
+		{E6CED902-9306-4EC4-810D-A23937B4C9FC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{E6CED902-9306-4EC4-810D-A23937B4C9FC}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{E6CED902-9306-4EC4-810D-A23937B4C9FC}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{E6CED902-9306-4EC4-810D-A23937B4C9FC}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -315,6 +321,7 @@ Global
 		{49868FC0-2F38-4EF4-9719-6578CDF9C453} = {CA9AC87F-097E-4F15-8393-4BC07735A5B0}
 		{482AC386-F3A8-4791-A708-414E52A8177A} = {04DBDB01-70F4-4E06-B468-8F87850B22BE}
 		{89A0820D-CBB0-4061-A939-F30EF486BA93} = {CA9AC87F-097E-4F15-8393-4BC07735A5B0}
+		{E6CED902-9306-4EC4-810D-A23937B4C9FC} = {CA9AC87F-097E-4F15-8393-4BC07735A5B0}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {28315BFD-90E7-4E14-A2EA-F3D23AF4126F}

src/AISmart.Evaluate/AISmart.Evaluate.csproj

@@ -0,0 +1,19 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <Import Project="..\..\common.props"/>
+
+    <PropertyGroup>
+        <TargetFramework>net8.0</TargetFramework>
+        <Nullable>enable</Nullable>
+        <RootNamespace>AISmart</RootNamespace>
+    </PropertyGroup>
+
+    <ItemGroup>
+        <ProjectReference Include="..\AISmart.SimpleRag\AISmart.SimpleRag.csproj" />
+    </ItemGroup>
+
+    <ItemGroup>
+        <PackageReference Include="Volo.Abp.ObjectExtending"/>
+    </ItemGroup>
+
+</Project>

src/AISmart.Evaluate/AISmartEvaluateModule.cs

@@ -0,0 +1,23 @@
+using AISmart.Evaluate.Service;
+using AISmart.Options;
+using AISmart.Provider;
+using AISmart.Rag;
+using Microsoft.Extensions.DependencyInjection;
+using Volo.Abp.Modularity;
+
+namespace AISmart.Evaluate;
+
+[DependsOn(
+    typeof(AISmartSimpleRagModule)
+)]
+public class AISmartEvaluateModule : AbpModule
+{
+    public override void ConfigureServices(ServiceConfigurationContext context)
+    {
+        var configuration = context.Services.GetConfiguration();
+        // Configure<RagOptions>(configuration.GetSection("EvaluateRag"));
+        // context.Services.AddSingleton<IRagProvider, RagProvider>();
+        // context.Services.AddSingleton<IAISmartEvaluateService, AISmartEvaluateService>();
+        // Configure<RagOptions>("EvaluateRag", configuration.GetSection("Evaluate:Rag"));
+    }
+}

src/AISmart.Evaluate/Options/EvaluateOptions.cs

@@ -0,0 +1,8 @@
+using AISmart.Options;
+
+namespace AISmart.Evaluate.Options;
+
+public class EvaluateOptions
+{
+    public RagOptions Rag { get; set; }
+}

src/AISmart.Evaluate/Service/AISmartEvaluateService.cs

@@ -0,0 +1,42 @@
+using System;
+using System.Threading.Tasks;
+using AISmart.Provider;
+using AISmart.Rag;
+using Microsoft.Extensions.Logging;
+using Volo.Abp.DependencyInjection;
+
+namespace AISmart.Evaluate.Service;
+
+public class AISmartEvaluateService : IAISmartEvaluateService, ISingletonDependency
+{
+    private readonly ILogger<AISmartEvaluateService> _logger;
+    private readonly IRagProvider _ragProvider;
+    // private readonly IOptionsMonitor<EvaluateOptions> _evaluateOptions;
+    private readonly IRagProviderFactory _ragProviderFactory;
+
+    public AISmartEvaluateService(ILogger<AISmartEvaluateService> logger, IRagProviderFactory ragProviderFactory)
+    {
+        _logger = logger;
+        _ragProvider = ragProviderFactory.GetProvider("EvaluateRag");
+    }
+
+    public async Task EvaluateAsync(string task, string result)
+    {
+        throw new NotImplementedException();
+    }
+
+    public async Task AddExceptionMessageAsync(string task, string exceptionMessage)
+    {
+        var text =
+            $"""
+             Task is: {task} 
+             Exception {exceptionMessage} was caught during execution, please pay attention!
+             """;
+        await _ragProvider.StoreTextAsync(text);
+    }
+
+    public async Task<string> GetAdviceAsync(string task)
+    {
+        return await _ragProvider.RetrieveAnswerAsync(task);
+    }
+}

src/AISmart.Evaluate/Service/IAISmartEvaluateService.cs

@@ -0,0 +1,10 @@
+using System.Threading.Tasks;
+
+namespace AISmart.Evaluate.Service;
+
+public interface IAISmartEvaluateService
+{
+    Task EvaluateAsync(string task, string result);
+    Task AddExceptionMessageAsync(string task, string exceptionMessage);
+    Task<string> GetAdviceAsync(string task);
+}

src/AISmart.GAgent.Autogen/AISmart.GAgent.Autogen.csproj

@@ -10,6 +10,7 @@
         <ProjectReference Include="..\AISmart.Application.Contracts\AISmart.Application.Contracts.csproj" />
         <ProjectReference Include="..\AISmart.Application.Grains\AISmart.Application.Grains.csproj" />
         <ProjectReference Include="..\AISmart.SimpleRag\AISmart.SimpleRag.csproj" />
+        <ProjectReference Include="..\AISmart.Evaluate\AISmart.Evaluate.csproj" />
     </ItemGroup>
 
     <ItemGroup>

src/AISmart.GAgent.Autogen/AISmartGAgentAutogenModule.cs

@@ -1,3 +1,5 @@
+using AISmart.Evaluate;
+using AISmart.Evaluate.Options;
 using AISmart.GAgent.Autogen.Options;
 using AISmart.GAgent.Core;
 using AISmart.Options;
@@ -13,7 +15,8 @@ namespace AISmart.GAgent.Autogen;
 
 [DependsOn(
     typeof(AISmartApplicationContractsModule),
-    typeof(AISmartSimpleRagModule)
+    typeof(AISmartSimpleRagModule),
+    typeof(AISmartEvaluateModule)
 )]
 public class AISmartGAgentAutogenModule : AbpModule
 {
@@ -26,6 +29,11 @@ public override void ConfigureServices(ServiceConfigurationContext context)
         // context.Services.AddTransient<ChatClient>(op => new ChatClient(autogenConfig.Model, autogenConfig.ApiKey));
         context.Services.AddTransient<IChatAgentProvider, ChatAgentProvider>();
         // context.Services.AddTransient<IChatService, ChatService>();
-        Configure<RagOptions>(configuration.GetSection("AutogenConfig:AutoGenRag"));
+        // Configure<RagOptions>(configuration.GetSection("AutogenConfig:AutoGenRag"));
+        Configure<RagOptions>("AutogenRag", configuration.GetSection("AutogenConfig:AutoGenRag"));
+        Configure<RagOptions>("EvaluateRag", configuration.GetSection("Evaluate:Rag"));
+
+
+
     }
 }

src/AISmart.GAgent.Autogen/AutogenGAgent.cs

@@ -31,9 +31,9 @@ public class AutogenGAgent : GAgentBase<AutoGenAgentState, AutogenEventBase>, IA
     private readonly int _maxRaiseEventCount = 50;
 
     public AutogenGAgent(ILogger<AutogenGAgent> logger,
-        IRagProvider ragProvider) : base(logger)
+        IRagProviderFactory ragProviderFactory) : base(logger)
     {
-        _ragProvider = ragProvider;
+        _ragProvider = ragProviderFactory.GetProvider("AutogenRag");
     }
 
     public async Task RegisterAgentEvent(Type agent, List<Type> eventTypes)

src/AISmart.GAgent.Autogen/Executor/AutoGenExecutor.cs

@@ -2,6 +2,7 @@
 using System.Text.Json.Serialization;
 using AISmart.Agents;
 using AISmart.Dapr;
+using AISmart.Evaluate.Service;
 using AISmart.GAgent.Autogen.Common;
 using AISmart.GAgent.Autogen.DescriptionManager;
 using AISmart.GAgent.Autogen.Event;
@@ -24,18 +25,34 @@ public class AutoGenExecutor : Grain, IAutoGenExecutor
     private const string BreakFlag = "break";
     private Guid _taskId;
     private IAgentDescriptionManager _descriptionManager;
+    private IAISmartEvaluateService _evaluateService;
+    public string ExceptionMessage;
 
-    public AutoGenExecutor(ILogger<AutoGenExecutor> logger, IChatAgentProvider chatAgentProvider)
+    public AutoGenExecutor(ILogger<AutoGenExecutor> logger, IChatAgentProvider chatAgentProvider, 
+        IAISmartEvaluateService evaluateService)
     {
         _logger = logger;
         _chatAgentProvider = chatAgentProvider;
+        _evaluateService = evaluateService;
     }
 
     public async Task ExecuteTaskAsync(ExecutorTaskInfo taskInfo)
     {
         try
         {
             await CallAutogen(taskInfo);
+            ExceptionMessage = string.Empty;
+        }
+        catch (JsonException e)
+        {
+            await _evaluateService.AddExceptionMessageAsync(GetTaskDescription(taskInfo.History), e.ToString());
+            await PublishInternalEvent(new AutoGenExecutorEvent()
+            {
+                TaskId = _taskId,
+                ExecuteStatus = TaskExecuteStatus.Break,
+                EndContent = "Internal error",
+            });
+            ExceptionMessage = e.ToString();
         }
         catch (Exception e)
         {
@@ -57,6 +74,8 @@ public async Task CallAutogen(ExecutorTaskInfo taskInfo)
         _descriptionManager = GrainFactory.GetGrain<IAgentDescriptionManager>(taskInfo.AgentDescriptionManagerId);
 
         var history = ConvertMessage(taskInfo.History);
+        // var exceptionAdvice = await _evaluateService.GetAdviceAsync(GetTaskDescription(taskInfo.History));
+        // history.Add(new TextMessage(Role.System, exceptionAdvice));
         var responsibility = await GetAgentResponsibility();
         _chatAgentProvider.SetAgent(AgentName, responsibility, GetMiddleware());
         var response = await _chatAgentProvider.SendAsync(AgentName, "What should be done next?", history);
@@ -139,6 +158,16 @@ private List<IMessage> ConvertMessage(List<AutogenMessage> listAutoGenMessage)
 
         return result;
     }
+
+    private string GetTaskDescription(List<AutogenMessage> listAutoGenMessage)
+    {
+        var list = new List<string>();
+        foreach (var item in listAutoGenMessage)
+        {
+            list.Add(item.Role + ": " + item.Content);
+        }
+        return string.Join("\n", list);
+    }
 
     private Role GetRole(string roleName)
     {
@@ -263,7 +292,7 @@ public async Task<string> HandleEventAsync(string agentName, string eventName, s
         var descriptionDic = await _descriptionManager.GetAgentDescription();
         if (descriptionDic.TryGetValue(agentName, out var eventDescription) == false)
         {
-            throw new AutogenException($"Event name:{agentName} not exist");
+            throw new AutogenException($"Agent name:{agentName} not exist");
         }
 
         var eventInfo = eventDescription.EventList.FirstOrDefault(f => f.EventName == eventName);

src/AISmart.SimpleRag/Provider/QdrantVectorDatabase.cs

@@ -113,6 +113,7 @@ public async Task StoreBatchAsync(IEnumerable<(float[] vector, string text)> poi
 
     public async Task<List<string>> RetrieveAsync(float[] queryEmbedding, int topK = 5)
     {
+        await EnsureCollectionExistsAsync();
         var requestBody = new { vector = queryEmbedding, top = topK, with_payload = true };
         var response = await _httpClient.PostAsJsonAsync($"{_qdrantUrl}/collections/{_collectionName}/points/search", requestBody);
         response.EnsureSuccessStatusCode();

src/AISmart.SimpleRag/Provider/RagProvider.cs

@@ -19,23 +19,23 @@ public class RagProvider : IRagProvider, ISingletonDependency
     private readonly IEmbeddingProvider _embeddingProvider;
     private readonly IVectorDatabase _vectorDatabase;
     private readonly ILogger<RagProvider> _logger;
-    private readonly IOptionsMonitor<RagOptions> _ragOptions;
+    private readonly IOptions<RagOptions> _ragOptions;
 
-    public RagProvider(IOptionsMonitor<RagOptions> ragOptions, ILogger<RagProvider> logger)
+    public RagProvider(IOptions<RagOptions> ragOptions, ILogger<RagProvider> logger)
     {
         _ragOptions = ragOptions;
         _chunker = new SimpleChunker();
-        _embeddingProvider = new OpenAIEmbeddingProvider(_ragOptions.CurrentValue.APIKey);
-        _vectorDatabase = new QdrantVectorDatabase(_ragOptions.CurrentValue.QdrantUrl, 
-            _ragOptions.CurrentValue.CollectionName, 
-            _ragOptions.CurrentValue.VectorSize);
+        _embeddingProvider = new OpenAIEmbeddingProvider(_ragOptions.Value.APIKey);
+        _vectorDatabase = new QdrantVectorDatabase(_ragOptions.Value.QdrantUrl, 
+            _ragOptions.Value.CollectionName, 
+            _ragOptions.Value.VectorSize);
         _logger = logger;
     }
 
     public async Task StoreTextAsync(string text)
     {
         _logger.LogInformation("store text {text}", text);
-        var chunkSize = _ragOptions.CurrentValue.ChunkSize;
+        var chunkSize = _ragOptions.Value.ChunkSize;
         var chunks = await _chunker.Chunk(text, chunkSize);
         foreach (var chunk in chunks)
         {
@@ -67,7 +67,7 @@ public async Task<string> RetrieveAnswerAsync(string query)
         _logger.LogInformation("retrieve text {query}", query);
         var queryEmbedding = await _embeddingProvider.GetEmbeddingAsync(query);
         var relevantChunks = await _vectorDatabase.RetrieveAsync(queryEmbedding, 3);
-        return string.Join(" ", relevantChunks);
+        return relevantChunks.IsNullOrEmpty() ? "" : string.Join(" ", relevantChunks);
     }
 
     private async Task BatchStoreFilesAsync(IList<string> files)

src/AISmart.SimpleRag/Provider/RagProviderFactory.cs

@@ -0,0 +1,39 @@
+using System.Collections.Concurrent;
+using AISmart.Options;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using Volo.Abp.DependencyInjection;
+
+namespace AISmart.Provider;
+
+public class RagProviderFactory : IRagProviderFactory, ISingletonDependency
+{
+    private readonly IOptionsMonitor<RagOptions> _ragOptionsMonitor;
+    private readonly ConcurrentDictionary<string, RagProvider> _providersDict;
+    private readonly ILogger<RagProvider> _logger;
+
+    public RagProviderFactory(IOptionsMonitor<RagOptions> ragOptionsMonitor,  ILogger<RagProvider> logger)
+    {
+        _ragOptionsMonitor = ragOptionsMonitor;
+        _providersDict = new ConcurrentDictionary<string, RagProvider>();
+        _logger = logger;
+    }
+
+    public RagProvider GetProvider(string configName)
+    {
+        var options = _ragOptionsMonitor.Get(configName);
+        if (_providersDict.TryGetValue(configName, out var provider))
+        {
+            return provider;
+        }
+
+        provider = new RagProvider(Microsoft.Extensions.Options.Options.Create(options), _logger);
+        _providersDict[configName] = provider;
+        return provider;
+    }
+}
+
+public interface IRagProviderFactory
+{
+    RagProvider GetProvider(string configName);
+}