feat: AI Assistant with natural language app control, LiteLLM capabilities, and multimodal chat

Endpoints/AiAssistantEndpoints.cs

@@ -0,0 +1,194 @@
+using System;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+using LocalLLMServerManager.Services;
+using LocalLLMServerManager.Shared.Interfaces;
+using LocalLLMServerManager.Shared.Models;
+using LocalLLMServerManager.Shared.Services;
+using Microsoft.AspNetCore.Builder;
+using Microsoft.AspNetCore.Http;
+
+namespace LocalLLMServerManager.Endpoints;
+
+public static class AiAssistantEndpoints
+{
+    private static readonly JsonSerializerOptions JsonOptions = new()
+    {
+        PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
+        PropertyNameCaseInsensitive = true,
+        Converters = { new JsonStringEnumConverter() }
+    };
+
+    public static void MapAiAssistantEndpoints(this WebApplication app)
+    {
+        // 1. Status
+        app.MapGet("/api/ai/status", async (
+            ISettingsService settingsService,
+            IComponentManagerService componentService,
+            IPromptManagementService promptService) =>
+        {
+            var settings = settingsService.LoadSettings();
+            var isInstalled = componentService.IsAiAssistantInstalled;
+            var promptDir = PromptManagementService.ResolvePromptDirectory(settings.AiAssistantPromptsDirectory);
+            var sections = await promptService.GetAllPromptsAsync(promptDir);
+
+            return Results.Ok(new
+            {
+                enabled = settings.AiAssistantEnabled,
+                isInstalled,
+                configured = !string.IsNullOrWhiteSpace(settings.AiAssistantEndpoint),
+                endpoint = settings.AiAssistantEndpoint,
+                model = settings.AiAssistantModel,
+                promptDirectory = promptDir,
+                loadedPromptSections = sections.Keys
+            });
+        });
+
+        // 2. Validate Connection
+        app.MapPost("/api/ai/validate", async (
+            HttpContext httpContext,
+            IAiAssistantService assistantService,
+            ISettingsService settingsService) =>
+        {
+            string? endpoint = null;
+            string? apiKey = null;
+            string? model = null;
+
+            if (httpContext.Request.HasJsonContentType())
+            {
+                try
+                {
+                    using var doc = await JsonDocument.ParseAsync(httpContext.Request.Body, cancellationToken: httpContext.RequestAborted);
+                    var root = doc.RootElement;
+                    if (root.TryGetProperty("endpoint", out var epEl)) endpoint = epEl.GetString();
+                    if (root.TryGetProperty("apiKey", out var keyEl)) apiKey = keyEl.GetString();
+                    if (root.TryGetProperty("model", out var modelEl)) model = modelEl.GetString();
+                }
+                catch (JsonException ex)
+                {
+                    return Results.BadRequest(new { error = $"Invalid JSON payload: {ex.Message}" });
+                }
+            }
+
+            var result = await assistantService.ValidateConnectionAsync(endpoint, apiKey, model, httpContext.RequestAborted);
+            return Results.Ok(result);
+        });
+
+        // 3. Models
+        app.MapGet("/api/ai/models", async (
+            HttpContext httpContext,
+            IAiAssistantService assistantService,
+            ISettingsService settingsService) =>
+        {
+            var query = httpContext.Request.Query;
+            string? endpoint = query.TryGetValue("endpoint", out var epVal) && !string.IsNullOrWhiteSpace(epVal)
+                ? epVal.ToString()
+                : null;
+
+            string? apiKey = query.TryGetValue("apiKey", out var keyVal) && !string.IsNullOrWhiteSpace(keyVal)
+                ? keyVal.ToString()
+                : null;
+
+            bool includeLocal = query.TryGetValue("includeLocal", out var localVal) &&
+                bool.TryParse(localVal.ToString(), out var parsedLocal) && parsedLocal;
+
+            try
+            {
+                var models = await assistantService.GetModelCapabilitiesAsync(endpoint, apiKey, includeLocal, httpContext.RequestAborted);
+                return Results.Ok(models);
+            }
+            catch (Exception ex)
+            {
+                return Results.BadRequest(new { error = ex.Message });
+            }
+        });
+
+        // 4. Prompts
+        app.MapGet("/api/ai/prompts", async (
+            ISettingsService settingsService,
+            IPromptManagementService promptService) =>
+        {
+            var settings = settingsService.LoadSettings();
+            var promptDir = PromptManagementService.ResolvePromptDirectory(settings.AiAssistantPromptsDirectory);
+            var sections = await promptService.GetAllPromptsAsync(promptDir);
+
+            return Results.Ok(new
+            {
+                promptsDirectory = promptDir,
+                sections
+            });
+        });
+
+        app.MapPost("/api/ai/prompts/reload", async (
+            ISettingsService settingsService,
+            IPromptManagementService promptService) =>
+        {
+            promptService.InvalidateCache();
+            var settings = settingsService.LoadSettings();
+            var promptDir = PromptManagementService.ResolvePromptDirectory(settings.AiAssistantPromptsDirectory);
+            var sections = await promptService.GetAllPromptsAsync(promptDir);
+
+            return Results.Ok(new
+            {
+                success = true,
+                promptsDirectory = promptDir,
+                sections
+            });
+        });
+
+        // 5. Chat completion (SSE stream or non-stream JSON)
+        app.MapPost("/api/ai/chat", async (
+            HttpContext httpContext,
+            AiChatRequest? request,
+            IAiAssistantService assistantService) =>
+        {
+            if (request == null || request.Messages == null || request.Messages.Count == 0)
+            {
+                return Results.BadRequest(new { error = "Request body must contain 'messages' array." });
+            }
+
+            bool wantsStream = request.Stream ||
+                httpContext.Request.Query.ContainsKey("stream") ||
+                (httpContext.Request.Headers.Accept.ToString().Contains("text/event-stream", StringComparison.OrdinalIgnoreCase));
+
+            if (wantsStream)
+            {
+                httpContext.Response.ContentType = "text/event-stream";
+                httpContext.Response.Headers.CacheControl = "no-cache";
+                httpContext.Response.Headers.Connection = "keep-alive";
+
+                try
+                {
+                    await foreach (var chunk in assistantService.StreamChatAsync(request, cancellationToken: httpContext.RequestAborted))
+                    {
+                        var json = JsonSerializer.Serialize(chunk, JsonOptions);
+                        await httpContext.Response.WriteAsync($"data: {json}\n\n", httpContext.RequestAborted);
+                        await httpContext.Response.Body.FlushAsync(httpContext.RequestAborted);
+                    }
+                }
+                catch (OperationCanceledException)
+                {
+                    // Request aborted by client
+                }
+                catch (Exception ex)
+                {
+                    var errorChunk = new AiChatChunk(Error: ex.Message, IsDone: true);
+                    var json = JsonSerializer.Serialize(errorChunk, JsonOptions);
+                    try
+                    {
+                        await httpContext.Response.WriteAsync($"data: {json}\n\n");
+                        await httpContext.Response.Body.FlushAsync();
+                    }
+                    catch { }
+                }
+
+                return Results.Empty;
+            }
+            else
+            {
+                var response = await assistantService.SendChatAsync(request, cancellationToken: httpContext.RequestAborted);
+                return Results.Ok(response);
+            }
+        });
+    }
+}

LocalLLMServerManager.Shared/Interfaces/IAiAppTools.cs

@@ -0,0 +1,72 @@
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace LocalLLMServerManager.Shared.Interfaces;
+
+/// <summary>
+/// Natural Language Integration Bridge: allows the AI Copilot to query telemetry,
+/// manage engines, update settings, trigger workflows, and query documentation.
+/// </summary>
+public interface IAiAppTools
+{
+    /// <summary>
+    /// Returns live GPU VRAM usage and hardware identity.
+    /// </summary>
+    Task<string> GetGpuVramTelemetryAsync(CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Checks health and connectivity of Ollama, Forge, ComfyUI, and Kokoro TTS runtimes.
+    /// </summary>
+    Task<string> CheckServicesHealthAsync(CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Lists all installed Ollama LLM models and local model assets.
+    /// </summary>
+    Task<string> ListInstalledModelsAsync(CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Starts a backend engine ('forge', 'comfyui', or 'ollama').
+    /// </summary>
+    Task<string> StartAiEngineAsync(string engine, CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Stops an active engine ('forge' or 'comfyui').
+    /// </summary>
+    Task<string> StopAiEngineAsync(string engine, CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Unloads all models from VRAM to free GPU memory.
+    /// </summary>
+    Task<string> UnloadVramAsync(CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Retrieves current application settings.
+    /// </summary>
+    Task<string> GetAppSettingsAsync(CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Modifies a specific application setting key and persists to disk.
+    /// </summary>
+    Task<string> UpdateAppSettingAsync(string key, string value, CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Evaluates whether a specified model fits in the user's GPU VRAM and RAM.
+    /// </summary>
+    Task<string> CalculateHardwareFitAsync(string modelName, double? parametersBillions = null, string? quantization = null, string? modality = null, CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Triggers an image generation workflow via Forge or ComfyUI.
+    /// </summary>
+    Task<string> GenerateImageAsync(string prompt, string? negativePrompt = null, string? engine = null, int width = 1024, int height = 1024, CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Synthesizes text to speech using local Kokoro TTS.
+    /// </summary>
+    Task<string> SynthesizeSpeechAsync(string text, string voice = "af_heart", string format = "mp3", CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Searches in-app technical documentation procedures and guides.
+    /// </summary>
+    Task<string> QueryAppDocumentationAsync(string query, CancellationToken cancellationToken = default);
+}

LocalLLMServerManager.Shared/Interfaces/IAiAssistantService.cs

@@ -0,0 +1,53 @@
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+using LocalLLMServerManager.Shared.Models;
+
+namespace LocalLLMServerManager.Shared.Interfaces;
+
+/// <summary>
+/// Orchestration service interface for OpenAI-compatible LLM routing, tool execution loop,
+/// and streaming chat completion.
+/// </summary>
+public interface IAiAssistantService
+{
+    /// <summary>
+    /// Validates connectivity, authentication, and model availability against the target OpenAI-compatible API endpoint.
+    /// </summary>
+    Task<AiValidationResult> ValidateConnectionAsync(
+        string? endpoint = null,
+        string? apiKey = null,
+        string? model = null,
+        CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Enumerate available model IDs from the remote endpoint (via /v1/models).
+    /// </summary>
+    Task<List<string>> GetAvailableModelsAsync(
+        string? endpoint = null,
+        string? apiKey = null,
+        CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Enumerate available model capabilities from the remote LiteLLM or OpenAI-compatible endpoint.
+    /// </summary>
+    Task<List<AiModelCapabilityInfo>> GetModelCapabilitiesAsync(
+        string? endpoint = null,
+        string? apiKey = null,
+        bool includeLocal = false,
+        CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Executes a full non-streaming multi-turn chat completion including automated tool calling.
+    /// </summary>
+    Task<AiChatResponse> SendChatAsync(
+        AiChatRequest request,
+        CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Streams tokens and tool execution events using async enumeration.
+    /// </summary>
+    IAsyncEnumerable<AiChatChunk> StreamChatAsync(
+        AiChatRequest request,
+        CancellationToken cancellationToken = default);
+}

LocalLLMServerManager.Shared/Interfaces/IPromptManagementService.cs

@@ -0,0 +1,46 @@
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace LocalLLMServerManager.Shared.Interfaces;
+
+/// <summary>
+/// Service responsible for managing, loading, and assembling living prompt documents from the filesystem.
+/// </summary>
+public interface IPromptManagementService
+{
+    /// <summary>
+    /// Gets the core system prompt including persona and tone.
+    /// </summary>
+    Task<string> GetSystemPromptAsync(string? customPromptDir = null, CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Gets the application capabilities reference prompt.
+    /// </summary>
+    Task<string> GetCapabilitiesPromptAsync(string? customPromptDir = null, CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Gets the workflows guide prompt.
+    /// </summary>
+    Task<string> GetWorkflowsPromptAsync(string? customPromptDir = null, CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Gets the natural language app-control guidelines prompt.
+    /// </summary>
+    Task<string> GetAppControlPromptAsync(string? customPromptDir = null, CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Assembles the complete consolidated system prompt containing system instructions, capabilities, workflows, and tool calling guidance.
+    /// </summary>
+    Task<string> BuildFullSystemPromptAsync(string? customPromptDir = null, CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Returns all living prompts as a key-value dictionary of document names to contents.
+    /// </summary>
+    Task<IReadOnlyDictionary<string, string>> GetAllPromptsAsync(string? customPromptDir = null, CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Clears any cached prompts so modifications on disk take effect immediately.
+    /// </summary>
+    void InvalidateCache();
+}

LocalLLMServerManager.Shared/Interfaces/ISettingsService.cs

@@ -0,0 +1,10 @@
+using LocalLLMServerManager.Shared.Models;
+
+namespace LocalLLMServerManager.Shared.Interfaces;
+
+public interface ISettingsService
+{
+    string SettingsFilePath();
+    AppSettings LoadSettings();
+    void SaveSettings(AppSettings settings);
+}