From 5e23078d6a83dc13877d039b4aefe24149ce57e8 Mon Sep 17 00:00:00 2001 From: Steven Pelech Date: Thu, 17 Sep 2026 22:33:01 -0500 Subject: [PATCH 01/18] docs(spec): add AI assistant in-app module design specification --- .../specs/2026-09-17-ai-assistant-design.md | 156 ++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 docs/superpowers/specs/2026-09-17-ai-assistant-design.md diff --git a/docs/superpowers/specs/2026-09-17-ai-assistant-design.md b/docs/superpowers/specs/2026-09-17-ai-assistant-design.md new file mode 100644 index 0000000..ff4d9a2 --- /dev/null +++ b/docs/superpowers/specs/2026-09-17-ai-assistant-design.md @@ -0,0 +1,156 @@ +# In-App AI Assistant & Natural Language App Control Design Specification + +## 1. Overview & Goals + +LocalLLMServerManager is a unified management application and web interface for hosting, orchestrating, and interacting with local AI inference runtimes (Ollama, Stable Diffusion Forge, ComfyUI, Kokoro TTS, etc.). + +When running locally, a user's machine may lack the spare GPU VRAM required to concurrently run both heavy generation workflows (like Wan 2.2 video or Flux image diffusion) and a massive local reasoning LLM. Furthermore, users often need conversational guidance on hardware requirements, workflow setup, model recommendations, and troubleshooting. + +The **AI Assistant Module** bridges this gap by introducing: +1. **External OpenAI-Compatible Routing**: Connects to external LLMs (e.g., LiteLLM proxy, OpenAI, Vertex AI Gemini Flash models) through a configurable API endpoint and key. +2. **Natural Language App Control (Stretch Goal / Bridge)**: Equips the LLM with native tool calling capabilities to query VRAM telemetry, start/stop engines, switch active settings, trigger image/video/speech workflows, and check hardware compatibility. +3. **Living Prompts & Documentation System**: Decouples prompt engineering from application source code by loading markdown-based system prompts, personas, and workflow manuals dynamically from the repository filesystem (`Prompts/`). +4. **Rich Conversational UI/UX**: Delivers an OpenWebUI/Claude-style chat interface in Avalonia XAML (with 100% desktop and WebAssembly web app parity) featuring user/assistant bubbles, tool call progress cards, suggestion chips, and live status indicators. +5. **Modular Component Architecture & Setup Wizard**: Packaged as an optional module in `ComponentManagerService` with a first-run connection validation wizard (pinging models, verifying credentials, and selecting target models). +6. **Dual Verification Pipeline**: Comprehensive unit tests covering all orchestration and tool bridges, alongside live integration tests against LiteLLM / Vertex Flash endpoints. + +--- + +## 2. Architecture & Tech Stack + +``` ++-----------------------------------------------------------------------------------+ +| User Interface Layer | +| +-------------------------------------+ +------------------------------------+ | +| | Desktop App (Avalonia 12 / Fluent) | | Web App (Avalonia WebAssembly) | | +| | AiAssistantTabControl | | AiAssistantTabControl | | +| +-------------------------------------+ +------------------------------------+ | +| | | +| v | +| AiAssistantViewModel | ++-----------------------------------------------------------------------------------+ + | + HTTP API Client (/api/ai/*) + | ++-----------------------------------------------------------------------------------+ +| ASP.NET Core Server (Port 5246) | +| | +| +-----------------------------------------------------------------------------+ | +| | AiAssistantEndpoints | | +| | POST /api/ai/chat GET /api/ai/validate GET /api/ai/models | | +| | GET /api/ai/prompts POST /api/ai/prompts/reload | | +| +-----------------------------------------------------------------------------+ | +| | | +| v | +| +-----------------------------------------------------------------------------+ | +| | Orchestration Service: IAiAssistantService | | +| | - Powered by Microsoft.Extensions.AI (net10.0) | | +| | - FunctionInvokingChatClient tool loop | | +| | - Streaming & Non-streaming SSE response generation | | +| +-----------------------------------------------------------------------------+ | +| | | | +| v v | +| +------------------------------+ +--------------------------------------+ | +| | PromptManagementService | | AiAppTools | | +| | Loads & templates markdown | | App Integration Bridge: | | +| | prompts from Prompts/ dir | | - IGpuTelemetryProvider (VRAM) | | +| +------------------------------+ | - IAiEngineManager (start/stop) | | +| | - ISettingsService (view/update) | | +| | - ICanIRunItService (hardware fit) | | +| | - WorkflowEndpoints / Http | | +| | - Documentation search | | +| +--------------------------------------+ | +| | | +| v | +| Local AI Engines & OS Subsystems | +| (Ollama, Forge, ComfyUI, Kokoro) | ++-----------------------------------------------------------------------------------+ + | + v + +----------------------------------+ + | External OpenAI-Compatible API | + | (LiteLLM / Vertex Flash) | + +----------------------------------+ +``` + +### 2.1 Why `Microsoft.Extensions.AI`? +- **Native .NET Standard & .NET 10 Alignment**: `Microsoft.Extensions.AI` and `Microsoft.Extensions.AI.OpenAI` (v10.10.0) are Microsoft's official abstractions for AI models and tool calling in .NET 10. +- **Provider Agnostic**: Targets any OpenAI-compatible HTTP endpoint (LiteLLM, Azure OpenAI, vLLM, Ollama, OpenAI) simply by setting `Endpoint` and `ApiKey`. +- **Automatic Function Invocation**: Built-in `FunctionInvokingChatClient` handles multi-turn tool loops cleanly, allowing the LLM to call multiple tools in sequence before replying. +- **Strong Typing**: Clean integration with dependency injection and cancellation tokens. + +--- + +## 3. Subsystem Breakdown + +### 3.1 Living Prompts System (`Prompts/`) +All prompt text, persona instructions, and tool usage documentation are maintained as distinct markdown documents in `Prompts/`: +1. `Prompts/system-prompt.md`: Core system instructions, persona (helpful local AI orchestrator and assistant), safety rules, and operating principles. +2. `Prompts/capabilities.md`: Detailed descriptions of what the host application supports (Ollama LLMs, SD Forge image generation, ComfyUI workflows, Kokoro TTS, hardware fit calculator). +3. `Prompts/workflows.md`: Instructions on how to guide users through image, video, audio, and text workflows. +4. `Prompts/app-control.md`: Guidelines and schemas for when and how the assistant should call tools to control the application. + +`IPromptManagementService` reads these files at startup and provides a live reload mechanism (`ReloadPromptsAsync`) so prompt engineers can tweak prompts without rebuilding or restarting the app. + +### 3.2 App Integration Bridge (`AiAppTools`) +The LLM can execute actions in the app via registered functions: +- `get_gpu_vram_telemetry()`: Live VRAM capacity, used bytes, free bytes, GPU model. +- `check_services_health()`: Online/offline status of Ollama, SD Forge, ComfyUI, Kokoro. +- `list_installed_models()`: Query local Ollama and HuggingFace/Civitai downloaded models. +- `start_ai_engine(engine)` / `stop_ai_engine(engine)`: Process control for `forge`, `comfyui`, `ollama`. +- `get_app_settings()` / `update_app_setting(key, value)`: Inspect and update application settings. +- `calculate_hardware_fit(modelName, sizeGb, quant)`: Run hardware fit checks before downloading models. +- `generate_image(prompt, negativePrompt, engine, width, height)`: Trigger text-to-image on Forge or ComfyUI. +- `synthesize_speech(text, voice)`: Call Kokoro TTS to generate voice audio. +- `query_app_documentation(query)`: Search built-in ASD-STE100 user documentation. + +### 3.3 Optional Module Registration & Setup Wizard +- In `ComponentManagerService`, the `ai-assistant` component pack is registered alongside video and audio packs. +- The module includes a **Setup Wizard** in the UI: + - Base URL field (default: `http://127.0.0.1:4000/v1` for LiteLLM). + - API Key field (masked). + - Model Name field (default: `google/gemini-2.5-flash` or configurable). + - "Validate Connection" button which sends a lightweight test ping to the endpoint. + - Success badge with available model list upon successful validation. + +### 3.4 Conversational UI / UX +- **Theme Consistency**: Uses Semi.Avalonia / Fluent dark tokens (`BgDarkBrush`, `BgSurfaceBrush`, `PrimaryBrush`, `BorderBrush`). +- **Chat Bubbles**: + - User messages: Right-aligned styled bubble. + - Assistant messages: Left-aligned markdown bubble with copy button. + - Tool Invocation Cards: Specialized expandable cards showing "Invoking [Tool Name]...", tool arguments, and results with status indicators (Running, Success, Error). +- **Quick Action Chips**: + - "πŸ“Š Check GPU VRAM" + - "πŸ” What models can I run?" + - "⚑ Check backend services" + - "🎨 Generate an image" +- **Controls**: + - Text input with Shift+Enter multi-line support. + - Send button and Stop Generation button. + - Clear conversation button. + - Model and connection status pill in header. + +--- + +## 4. Error Handling & Edge Cases +- **External Endpoint Down / Timeout**: Clear error message in chat bubble with actionable troubleshooting steps. +- **Invalid API Key**: Prompt the user to open the Setup Wizard and test credentials. +- **Tool Execution Failure**: Tool errors are safely caught, wrapped into JSON error objects, and returned to the LLM so it can explain the error gracefully to the user. +- **Prompts Missing**: `PromptManagementService` falls back to robust built-in default prompts if the `Prompts/` directory is missing. +- **Concurrent Generation**: The UI disables the Send button and displays an active generation indicator while awaiting stream completion. + +--- + +## 5. Testing & Verification Strategy +- **Unit Tests (`LocalLLMServerManager.Tests`)**: + - `PromptManagementServiceTests`: Test directory loading, missing file fallback, prompt composition, reload. + - `AiAssistantServiceTests`: Test client initialization, connection validation, tool execution routing, error formatting. + - `AiAppToolsTests`: Test all app tool functions with mocked providers. + - `AiAssistantEndpointsTests`: Test REST API endpoints (`/api/ai/chat`, `/api/ai/validate`, `/api/ai/prompts`). + - `AiAssistantViewModelTests`: Test ViewModel state machine, message list updates, tool card formatting, validation command. +- **Live LLM Integration Tests**: + - `AiAssistantLiveLlmTests`: Interactive test with LiteLLM / Vertex Flash endpoint (with environment variable override `LITELLM_API_KEY` / `LITELLM_ENDPOINT`). +- **Regression & Quality Checks**: + - `dotnet test LocalLLMServerManager.sln` + - `npm run lint` + - `npx tsc --noEmit` From 347d6cddcce05a6859a903b3fcdafc795756cc7d Mon Sep 17 00:00:00 2001 From: Steven Pelech Date: Thu, 17 Sep 2026 22:34:36 -0500 Subject: [PATCH 02/18] feat(prompts): introduce living prompt management system and markdown guidelines --- .../Interfaces/IPromptManagementService.cs | 46 +++++ .../Services/PromptManagementService.cs | 157 ++++++++++++++++++ .../LocalLLMServerManager.Tests.csproj | 4 + .../PromptManagementServiceTests.cs | 118 +++++++++++++ LocalLLMServerManager.csproj | 5 + Prompts/app-control.md | 46 +++++ Prompts/capabilities.md | 43 +++++ Prompts/system-prompt.md | 20 +++ Prompts/workflows.md | 32 ++++ 9 files changed, 471 insertions(+) create mode 100644 LocalLLMServerManager.Shared/Interfaces/IPromptManagementService.cs create mode 100644 LocalLLMServerManager.Shared/Services/PromptManagementService.cs create mode 100644 LocalLLMServerManager.Tests/PromptManagementServiceTests.cs create mode 100644 Prompts/app-control.md create mode 100644 Prompts/capabilities.md create mode 100644 Prompts/system-prompt.md create mode 100644 Prompts/workflows.md diff --git a/LocalLLMServerManager.Shared/Interfaces/IPromptManagementService.cs b/LocalLLMServerManager.Shared/Interfaces/IPromptManagementService.cs new file mode 100644 index 0000000..6d92bb8 --- /dev/null +++ b/LocalLLMServerManager.Shared/Interfaces/IPromptManagementService.cs @@ -0,0 +1,46 @@ +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace LocalLLMServerManager.Shared.Interfaces; + +/// +/// Service responsible for managing, loading, and assembling living prompt documents from the filesystem. +/// +public interface IPromptManagementService +{ + /// + /// Gets the core system prompt including persona and tone. + /// + Task GetSystemPromptAsync(string? customPromptDir = null, CancellationToken cancellationToken = default); + + /// + /// Gets the application capabilities reference prompt. + /// + Task GetCapabilitiesPromptAsync(string? customPromptDir = null, CancellationToken cancellationToken = default); + + /// + /// Gets the workflows guide prompt. + /// + Task GetWorkflowsPromptAsync(string? customPromptDir = null, CancellationToken cancellationToken = default); + + /// + /// Gets the natural language app-control guidelines prompt. + /// + Task GetAppControlPromptAsync(string? customPromptDir = null, CancellationToken cancellationToken = default); + + /// + /// Assembles the complete consolidated system prompt containing system instructions, capabilities, workflows, and tool calling guidance. + /// + Task BuildFullSystemPromptAsync(string? customPromptDir = null, CancellationToken cancellationToken = default); + + /// + /// Returns all living prompts as a key-value dictionary of document names to contents. + /// + Task> GetAllPromptsAsync(string? customPromptDir = null, CancellationToken cancellationToken = default); + + /// + /// Clears any cached prompts so modifications on disk take effect immediately. + /// + void InvalidateCache(); +} diff --git a/LocalLLMServerManager.Shared/Services/PromptManagementService.cs b/LocalLLMServerManager.Shared/Services/PromptManagementService.cs new file mode 100644 index 0000000..8e841f3 --- /dev/null +++ b/LocalLLMServerManager.Shared/Services/PromptManagementService.cs @@ -0,0 +1,157 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using LocalLLMServerManager.Shared.Interfaces; + +namespace LocalLLMServerManager.Shared.Services; + +public class PromptManagementService : IPromptManagementService +{ + private readonly ConcurrentDictionary _cache = new(StringComparer.OrdinalIgnoreCase); + + public const string SystemPromptFileName = "system-prompt.md"; + public const string CapabilitiesFileName = "capabilities.md"; + public const string WorkflowsFileName = "workflows.md"; + public const string AppControlFileName = "app-control.md"; + + public async Task GetSystemPromptAsync(string? customPromptDir = null, CancellationToken cancellationToken = default) + { + return await LoadPromptDocumentAsync(SystemPromptFileName, FallbackSystemPrompt, customPromptDir, cancellationToken); + } + + public async Task GetCapabilitiesPromptAsync(string? customPromptDir = null, CancellationToken cancellationToken = default) + { + return await LoadPromptDocumentAsync(CapabilitiesFileName, FallbackCapabilitiesPrompt, customPromptDir, cancellationToken); + } + + public async Task GetWorkflowsPromptAsync(string? customPromptDir = null, CancellationToken cancellationToken = default) + { + return await LoadPromptDocumentAsync(WorkflowsFileName, FallbackWorkflowsPrompt, customPromptDir, cancellationToken); + } + + public async Task GetAppControlPromptAsync(string? customPromptDir = null, CancellationToken cancellationToken = default) + { + return await LoadPromptDocumentAsync(AppControlFileName, FallbackAppControlPrompt, customPromptDir, cancellationToken); + } + + public async Task BuildFullSystemPromptAsync(string? customPromptDir = null, CancellationToken cancellationToken = default) + { + var sys = await GetSystemPromptAsync(customPromptDir, cancellationToken); + var cap = await GetCapabilitiesPromptAsync(customPromptDir, cancellationToken); + var work = await GetWorkflowsPromptAsync(customPromptDir, cancellationToken); + var ctrl = await GetAppControlPromptAsync(customPromptDir, cancellationToken); + + var sb = new StringBuilder(); + sb.AppendLine(sys.Trim()); + sb.AppendLine(); + sb.AppendLine("---"); + sb.AppendLine(); + sb.AppendLine(cap.Trim()); + sb.AppendLine(); + sb.AppendLine("---"); + sb.AppendLine(); + sb.AppendLine(work.Trim()); + sb.AppendLine(); + sb.AppendLine("---"); + sb.AppendLine(); + sb.AppendLine(ctrl.Trim()); + + return sb.ToString(); + } + + public async Task> GetAllPromptsAsync(string? customPromptDir = null, CancellationToken cancellationToken = default) + { + var dict = new Dictionary + { + [SystemPromptFileName] = await GetSystemPromptAsync(customPromptDir, cancellationToken), + [CapabilitiesFileName] = await GetCapabilitiesPromptAsync(customPromptDir, cancellationToken), + [WorkflowsFileName] = await GetWorkflowsPromptAsync(customPromptDir, cancellationToken), + [AppControlFileName] = await GetAppControlPromptAsync(customPromptDir, cancellationToken) + }; + return dict; + } + + public void InvalidateCache() + { + _cache.Clear(); + } + + public static string? ResolvePromptDirectory(string? customPromptDir = null) + { + if (!string.IsNullOrWhiteSpace(customPromptDir) && Directory.Exists(customPromptDir)) + { + return Path.GetFullPath(customPromptDir); + } + + var candidateDirs = new[] + { + Path.Combine(AppContext.BaseDirectory, "Prompts"), + Path.Combine(Directory.GetCurrentDirectory(), "Prompts"), + Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "Prompts"), + Path.Combine(Directory.GetCurrentDirectory(), "..", "Prompts") + }; + + foreach (var dir in candidateDirs) + { + try + { + if (Directory.Exists(dir) && File.Exists(Path.Combine(dir, SystemPromptFileName))) + { + return Path.GetFullPath(dir); + } + } + catch { } + } + + return null; + } + + private async Task LoadPromptDocumentAsync(string fileName, string fallbackContent, string? customPromptDir, CancellationToken cancellationToken) + { + var resolvedDir = ResolvePromptDirectory(customPromptDir); + var cacheKey = $"{resolvedDir ?? "fallback"}::{fileName}"; + + if (_cache.TryGetValue(cacheKey, out var cached)) + { + return cached; + } + + if (resolvedDir != null) + { + var filePath = Path.Combine(resolvedDir, fileName); + if (File.Exists(filePath)) + { + try + { + var content = await File.ReadAllTextAsync(filePath, cancellationToken); + if (!string.IsNullOrWhiteSpace(content)) + { + _cache[cacheKey] = content; + return content; + } + } + catch { } + } + } + + _cache[cacheKey] = fallbackContent; + return fallbackContent; + } + + private const string FallbackSystemPrompt = @"# LocalLLMServerManager AI Copilot β€” System Prompt +You are the LocalLLMServerManager Copilot, an expert AI engineer integrated directly into the application. +Guide users on models, workflows, and settings, and proactively use your tools to perform actions in the app."; + + private const string FallbackCapabilitiesPrompt = @"# Application Capabilities Reference +LocalLLMServerManager supports Ollama LLMs, Stable Diffusion Forge, ComfyUI workflows, Kokoro TTS, and the Can I Run It hardware calculator."; + + private const string FallbackWorkflowsPrompt = @"# Application Workflows Guide +Workflows include LLM downloading and execution, image generation with Forge/ComfyUI, video rendering, and Kokoro speech synthesis."; + + private const string FallbackAppControlPrompt = @"# In-App Natural Language Control Guidelines +Use your registered tools to inspect VRAM, check health, start/stop engines, update settings, and trigger generation workflows."; +} diff --git a/LocalLLMServerManager.Tests/LocalLLMServerManager.Tests.csproj b/LocalLLMServerManager.Tests/LocalLLMServerManager.Tests.csproj index d4c105a..d85361d 100644 --- a/LocalLLMServerManager.Tests/LocalLLMServerManager.Tests.csproj +++ b/LocalLLMServerManager.Tests/LocalLLMServerManager.Tests.csproj @@ -39,6 +39,10 @@ Workflows\%(RecursiveDir)%(Filename)%(Extension) PreserveNewest + + Prompts\%(RecursiveDir)%(Filename)%(Extension) + PreserveNewest + diff --git a/LocalLLMServerManager.Tests/PromptManagementServiceTests.cs b/LocalLLMServerManager.Tests/PromptManagementServiceTests.cs new file mode 100644 index 0000000..06a93b2 --- /dev/null +++ b/LocalLLMServerManager.Tests/PromptManagementServiceTests.cs @@ -0,0 +1,118 @@ +using System; +using System.IO; +using System.Threading.Tasks; +using LocalLLMServerManager.Shared.Services; +using Xunit; + +namespace LocalLLMServerManager.Tests; + +public class PromptManagementServiceTests +{ + [Fact] + public async Task GetSystemPrompt_ReturnsValidPrompt_FromDiskOrFallback() + { + var service = new PromptManagementService(); + var prompt = await service.GetSystemPromptAsync(); + + Assert.False(string.IsNullOrWhiteSpace(prompt)); + Assert.Contains("LocalLLMServerManager", prompt); + } + + [Fact] + public async Task GetCapabilitiesPrompt_ReturnsValidPrompt() + { + var service = new PromptManagementService(); + var prompt = await service.GetCapabilitiesPromptAsync(); + + Assert.False(string.IsNullOrWhiteSpace(prompt)); + Assert.Contains("Capabilities", prompt, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task GetWorkflowsPrompt_ReturnsValidPrompt() + { + var service = new PromptManagementService(); + var prompt = await service.GetWorkflowsPromptAsync(); + + Assert.False(string.IsNullOrWhiteSpace(prompt)); + Assert.Contains("Workflow", prompt, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task GetAppControlPrompt_ReturnsValidPrompt() + { + var service = new PromptManagementService(); + var prompt = await service.GetAppControlPromptAsync(); + + Assert.False(string.IsNullOrWhiteSpace(prompt)); + Assert.Contains("Control", prompt, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task BuildFullSystemPrompt_AssemblesAllSections() + { + var service = new PromptManagementService(); + var fullPrompt = await service.BuildFullSystemPromptAsync(); + + Assert.False(string.IsNullOrWhiteSpace(fullPrompt)); + Assert.Contains("LocalLLMServerManager", fullPrompt); + Assert.Contains("---", fullPrompt); + } + + [Fact] + public async Task GetAllPrompts_ReturnsDictionaryWithAllExpectedKeys() + { + var service = new PromptManagementService(); + var all = await service.GetAllPromptsAsync(); + + Assert.NotNull(all); + Assert.Equal(4, all.Count); + Assert.True(all.ContainsKey(PromptManagementService.SystemPromptFileName)); + Assert.True(all.ContainsKey(PromptManagementService.CapabilitiesFileName)); + Assert.True(all.ContainsKey(PromptManagementService.WorkflowsFileName)); + Assert.True(all.ContainsKey(PromptManagementService.AppControlFileName)); + } + + [Fact] + public async Task CustomPromptDirectory_LoadsCustomFilesAndInvalidateCacheClearsMemory() + { + var tempDir = Path.Combine(Path.GetTempPath(), "PromptTests_" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tempDir); + try + { + await File.WriteAllTextAsync(Path.Combine(tempDir, PromptManagementService.SystemPromptFileName), "Custom System Prompt V1"); + var service = new PromptManagementService(); + + var loaded1 = await service.GetSystemPromptAsync(tempDir); + Assert.Equal("Custom System Prompt V1", loaded1); + + // Update file on disk + await File.WriteAllTextAsync(Path.Combine(tempDir, PromptManagementService.SystemPromptFileName), "Custom System Prompt V2"); + // Cache should still return V1 + var loadedCached = await service.GetSystemPromptAsync(tempDir); + Assert.Equal("Custom System Prompt V1", loadedCached); + + // Invalidate cache + service.InvalidateCache(); + var loadedUpdated = await service.GetSystemPromptAsync(tempDir); + Assert.Equal("Custom System Prompt V2", loadedUpdated); + } + finally + { + if (Directory.Exists(tempDir)) + { + Directory.Delete(tempDir, true); + } + } + } + + [Fact] + public async Task NonExistentDirectory_FallsBackToBuiltInDefaultsGracefully() + { + var service = new PromptManagementService(); + var nonExistent = Path.Combine(Path.GetTempPath(), "DoesNotExist_" + Guid.NewGuid().ToString("N")); + + var prompt = await service.GetSystemPromptAsync(nonExistent); + Assert.False(string.IsNullOrWhiteSpace(prompt)); + } +} diff --git a/LocalLLMServerManager.csproj b/LocalLLMServerManager.csproj index 160e448..8d5a38d 100644 --- a/LocalLLMServerManager.csproj +++ b/LocalLLMServerManager.csproj @@ -35,6 +35,8 @@ + + @@ -69,6 +71,9 @@ PreserveNewest + + PreserveNewest + diff --git a/Prompts/app-control.md b/Prompts/app-control.md new file mode 100644 index 0000000..233477c --- /dev/null +++ b/Prompts/app-control.md @@ -0,0 +1,46 @@ +# In-App Natural Language Control Guidelines + +You have access to tools that allow you to directly inspect and manage LocalLLMServerManager on the user's behalf. + +## Available Tools & When to Use Them + +### 1. `get_gpu_vram_telemetry()` +- **Purpose**: Get real-time GPU hardware name, total VRAM, used VRAM, and percentage utilized. +- **Trigger**: User asks about memory, VRAM, hardware usage, or before deciding if a model fits. + +### 2. `check_services_health()` +- **Purpose**: Check if Ollama, Stable Diffusion Forge, ComfyUI, and Kokoro TTS are online. +- **Trigger**: User asks "are my servers running?", "is Ollama online?", or before triggering a generation workflow. + +### 3. `list_installed_models()` +- **Purpose**: Enumerate local Ollama LLM models and quantization tags. +- **Trigger**: User asks "what models do I have?", "which LLMs are installed?". + +### 4. `start_ai_engine(engine)` / `stop_ai_engine(engine)` +- **Purpose**: Start or stop background engine processes (`forge`, `comfyui`, `ollama`). +- **Trigger**: User says "start Forge", "launch ComfyUI", "shut down Forge to save memory". + +### 5. `get_app_settings()` / `update_app_setting(key, value)` +- **Purpose**: View or modify application settings (e.g. `PreferredImageEngine`, `ComfyUiUrl`, `SelectedThemeStyle`, etc.). +- **Trigger**: User says "switch my preferred image engine to Forge", "change theme to dark", "what are my current settings?". + +### 6. `calculate_hardware_fit(modelName, sizeGb, quantization)` +- **Purpose**: Evaluate if a specific model fits within the user's GPU VRAM and RAM budgets. +- **Trigger**: User asks "can I run deepseek-r1:32b?", "will Flux fit on my GPU?". + +### 7. `generate_image(prompt, negativePrompt, engine, width, height)` +- **Purpose**: Submit an image generation request to the configured image engine. +- **Trigger**: User asks "generate an image of...", "create a wallpaper with...". + +### 8. `synthesize_speech(text, voice)` +- **Purpose**: Synthesize audio from text using Kokoro TTS. +- **Trigger**: User asks "read this aloud", "generate audio for...". + +### 9. `query_app_documentation(query)` +- **Purpose**: Search the in-app ASD-STE100 technical documentation for specific questions. +- **Trigger**: User asks specific questions about how features work or how to configure complex settings. + +## Execution Rules +- Never ask the user to manually click something if a tool exists to perform the action. +- When calling a tool, explain what you are doing, then report the outcome once completed. +- If a tool fails (e.g., engine executable not configured), explain the exact setting the user needs to configure in the Settings tab. diff --git a/Prompts/capabilities.md b/Prompts/capabilities.md new file mode 100644 index 0000000..cc5e94e --- /dev/null +++ b/Prompts/capabilities.md @@ -0,0 +1,43 @@ +# Application Capabilities Reference + +LocalLLMServerManager provides unified orchestration and management across several core AI engines and subsystems: + +## 1. LLM Engine (Ollama) +- **Port / Protocol**: `http://127.0.0.1:11434` +- **Features**: + - Download and run GGUF quantized models (e.g. Llama 3, Qwen 2.5, DeepSeek, Mistral, Gemma). + - List installed models, active running models (`/api/ps`), and tags (`/api/tags`). + - One-click model pull from Ollama library or Hugging Face. + - VRAM release (`keep_alive: 0`) to dynamically free GPU memory. + +## 2. Image Generation Engines +- **Stable Diffusion Forge**: + - Default URL: `http://127.0.0.1:7860` + - High-performance web UI and API for SD1.5, SDXL, and Flux diffusion models. + - Native support for ControlNet, LoRAs, and VAEs. +- **ComfyUI**: + - Default URL: `http://127.0.0.1:8188` + - Node-based modular execution graph for complex image and video workflows. + - Supports custom workflow JSON files stored in `Workflows/`. + +## 3. Video Generation (ComfyUI DiT) +- **Workflows**: Wan 2.2 (`wan2.2_t2v`), LTX-Video 2.5 (`ltx2.5_t2v`), and HunyuanVideo 1.5. +- **Hardware Requirement**: 8 GB to 24 GB VRAM depending on resolution and frame count. + +## 4. Audio & Speech Engines +- **Kokoro TTS**: + - Fast, lightweight text-to-speech engine on port `http://127.0.0.1:8880` or `7851`. + - Quality voices such as `af_heart`, `am_adam`, `bf_alice`. +- **Music & SFX**: + - Stable Audio Open 1.0 SFX and MusicGen melody synthesis via ComfyUI. + +## 5. Hardware Fit Calculator ("Can I Run It") +- Accurately models model weight footprint + KV cache size across context lengths (2K, 4K, 8K, 32K, 128K). +- Generates fit badges: + - `PERFECT FIT` (green): Completely fits in GPU VRAM with full layer offloading. + - `TIGHT FIT` (amber): Fits with minimal VRAM headroom; lower context recommended. + - `RAM OFFLOAD` (yellow): Partial GPU offload; remaining layers execute in system RAM. + - `WILL NOT FIT` (red): Exceeds combined VRAM and system memory. + +## 6. Remote LAN & Tunnel Access +- Allows sharing local inference over local LAN (`http://:5246`) or secure reverse proxies with CORS pre-configured. diff --git a/Prompts/system-prompt.md b/Prompts/system-prompt.md new file mode 100644 index 0000000..709dab4 --- /dev/null +++ b/Prompts/system-prompt.md @@ -0,0 +1,20 @@ +# LocalLLMServerManager AI Copilot β€” System Prompt + +You are the **LocalLLMServerManager Copilot**, an intelligent, authoritative, and friendly in-app AI assistant integrated directly into LocalLLMServerManager. + +## Persona & Tone +- You are an expert in local AI inference runtimes, hardware optimization (VRAM, CUDA, ROCm, DirectML, Apple Silicon), diffusion models, and LLM quantization. +- Your tone is concise, technically precise, constructive, and empowering. +- You speak directly to the user, providing clear explanations with actionable next steps. + +## Core Responsibilities +1. **Application Guidance**: Guide the user on navigating tabs (Models, Workflows, Can I Run It, Documentation, Settings). +2. **Natural Language App Control**: When the user asks to check GPU memory, start or stop a backend service, change configuration, test hardware compatibility, or generate images/audio, USE YOUR TOOLS to execute the action directly on their behalf. +3. **VRAM Optimization**: Help users manage VRAM headroom. Explain when to unload LLMs before launching heavy diffusion models (Flux, SDXL, Wan 2.2). +4. **Model Recommendations**: Recommend optimal models based on the user's available GPU VRAM, quantization levels (e.g. Q4_K_M vs Q8_0), and intended use cases (coding, general reasoning, creative writing, diffusion). + +## Operating Rules +- **Proactive Tool Calling**: If the user's prompt implies or asks for an action that can be performed via tools (e.g., "how much VRAM do I have left?", "start ComfyUI", "check if I can run llama3.3:70b", "change my preferred image engine to comfy"), call the relevant tool immediately. +- **Synthesize, Don't Dump**: When tools return JSON data, summarize the relevant information clearly in natural language with markdown formatting (bullet points, bold highlights). +- **Safety**: Do not modify system paths to blocked or sensitive directories. Never fabricate paths or fictitious model URLs. +- **Living Prompts**: Your knowledge of app features, workflows, and tool calling is drawn from living documentation in the repository. diff --git a/Prompts/workflows.md b/Prompts/workflows.md new file mode 100644 index 0000000..218da19 --- /dev/null +++ b/Prompts/workflows.md @@ -0,0 +1,32 @@ +# Application Workflows Guide + +This guide details recommended multi-step workflows in LocalLLMServerManager: + +## Workflow 1: Pulling and Running an LLM +1. Verify system VRAM with `get_gpu_vram_telemetry`. +2. Check model fit using `calculate_hardware_fit`. +3. Check if Ollama is running via `check_services_health`; if stopped, start it via `start_ai_engine("ollama")`. +4. Run `pull_model(modelName)` or guide user to the Models tab to download from Hugging Face or Ollama Library. + +## Workflow 2: Generating an Image +1. Inspect available VRAM. If an LLM is occupying VRAM and memory is low, call `unload_vram()` to free GPU memory. +2. Ensure Forge (`start_ai_engine("forge")`) or ComfyUI (`start_ai_engine("comfyui")`) is running. +3. Execute `generate_image(prompt, negativePrompt, engine, width, height)`. +4. Provide the generated media preview path or link to the user. + +## Workflow 3: Video Generation with Wan 2.2 +1. Video generation requires at least 8 GB VRAM. +2. Confirm ComfyUI is online. +3. Invoke `generate_video(prompt, workflow="wan2.2_t2v", width=832, height=480, frames=49)`. +4. Direct the user to the Workflows -> Video sub-tab to view progress. + +## Workflow 4: Kokoro Speech Synthesis +1. Call `synthesize_speech(text, voice="af_heart")`. +2. The generated audio file will be saved under the application's audio output directory. + +## Workflow 5: VRAM Orchestration & Engine Switching +- When switching from LLM chat to heavy diffusion (SDXL, Flux, Wan 2.2): + 1. Call `unload_vram()`. + 2. Start the target engine (Forge or ComfyUI). + 3. Run the generation workflow. +- When finished with image/video tasks, user can resume LLM inference immediately. From dd3b4e442e2ea42928b9fc711c84d1497bb657eb Mon Sep 17 00:00:00 2001 From: Steven Pelech Date: Thu, 17 Sep 2026 22:40:01 -0500 Subject: [PATCH 03/18] feat(orchestration): implement AI assistant service, app bridge tools, and connection validation --- .../Interfaces/IAiAppTools.cs | 72 ++++ .../Interfaces/IAiAssistantService.cs | 44 ++ .../Models/AiAssistantModels.cs | 58 +++ .../Models/AppSettings.cs | 8 +- .../AiAppToolsTests.cs | 171 ++++++++ .../AiAssistantServiceTests.cs | 136 ++++++ .../AppSettingsTests.cs | 6 + Services/AiAppTools.cs | 403 ++++++++++++++++++ Services/AiAssistantService.cs | 375 ++++++++++++++++ 9 files changed, 1272 insertions(+), 1 deletion(-) create mode 100644 LocalLLMServerManager.Shared/Interfaces/IAiAppTools.cs create mode 100644 LocalLLMServerManager.Shared/Interfaces/IAiAssistantService.cs create mode 100644 LocalLLMServerManager.Shared/Models/AiAssistantModels.cs create mode 100644 LocalLLMServerManager.Tests/AiAppToolsTests.cs create mode 100644 LocalLLMServerManager.Tests/AiAssistantServiceTests.cs create mode 100644 Services/AiAppTools.cs create mode 100644 Services/AiAssistantService.cs diff --git a/LocalLLMServerManager.Shared/Interfaces/IAiAppTools.cs b/LocalLLMServerManager.Shared/Interfaces/IAiAppTools.cs new file mode 100644 index 0000000..d4361eb --- /dev/null +++ b/LocalLLMServerManager.Shared/Interfaces/IAiAppTools.cs @@ -0,0 +1,72 @@ +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace LocalLLMServerManager.Shared.Interfaces; + +/// +/// Natural Language Integration Bridge: allows the AI Copilot to query telemetry, +/// manage engines, update settings, trigger workflows, and query documentation. +/// +public interface IAiAppTools +{ + /// + /// Returns live GPU VRAM usage and hardware identity. + /// + Task GetGpuVramTelemetryAsync(CancellationToken cancellationToken = default); + + /// + /// Checks health and connectivity of Ollama, Forge, ComfyUI, and Kokoro TTS runtimes. + /// + Task CheckServicesHealthAsync(CancellationToken cancellationToken = default); + + /// + /// Lists all installed Ollama LLM models and local model assets. + /// + Task ListInstalledModelsAsync(CancellationToken cancellationToken = default); + + /// + /// Starts a backend engine ('forge', 'comfyui', or 'ollama'). + /// + Task StartAiEngineAsync(string engine, CancellationToken cancellationToken = default); + + /// + /// Stops an active engine ('forge' or 'comfyui'). + /// + Task StopAiEngineAsync(string engine, CancellationToken cancellationToken = default); + + /// + /// Unloads all models from VRAM to free GPU memory. + /// + Task UnloadVramAsync(CancellationToken cancellationToken = default); + + /// + /// Retrieves current application settings. + /// + Task GetAppSettingsAsync(CancellationToken cancellationToken = default); + + /// + /// Modifies a specific application setting key and persists to disk. + /// + Task UpdateAppSettingAsync(string key, string value, CancellationToken cancellationToken = default); + + /// + /// Evaluates whether a specified model fits in the user's GPU VRAM and RAM. + /// + Task CalculateHardwareFitAsync(string modelName, double? parametersBillions = null, string? quantization = null, string? modality = null, CancellationToken cancellationToken = default); + + /// + /// Triggers an image generation workflow via Forge or ComfyUI. + /// + Task GenerateImageAsync(string prompt, string? negativePrompt = null, string? engine = null, int width = 1024, int height = 1024, CancellationToken cancellationToken = default); + + /// + /// Synthesizes text to speech using local Kokoro TTS. + /// + Task SynthesizeSpeechAsync(string text, string voice = "af_heart", string format = "mp3", CancellationToken cancellationToken = default); + + /// + /// Searches in-app technical documentation procedures and guides. + /// + Task QueryAppDocumentationAsync(string query, CancellationToken cancellationToken = default); +} diff --git a/LocalLLMServerManager.Shared/Interfaces/IAiAssistantService.cs b/LocalLLMServerManager.Shared/Interfaces/IAiAssistantService.cs new file mode 100644 index 0000000..8bddd71 --- /dev/null +++ b/LocalLLMServerManager.Shared/Interfaces/IAiAssistantService.cs @@ -0,0 +1,44 @@ +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using LocalLLMServerManager.Shared.Models; + +namespace LocalLLMServerManager.Shared.Interfaces; + +/// +/// Orchestration service interface for OpenAI-compatible LLM routing, tool execution loop, +/// and streaming chat completion. +/// +public interface IAiAssistantService +{ + /// + /// Validates connectivity, authentication, and model availability against the target OpenAI-compatible API endpoint. + /// + Task ValidateConnectionAsync( + string? endpoint = null, + string? apiKey = null, + string? model = null, + CancellationToken cancellationToken = default); + + /// + /// Enumerate available model IDs from the remote endpoint (via /v1/models). + /// + Task> GetAvailableModelsAsync( + string? endpoint = null, + string? apiKey = null, + CancellationToken cancellationToken = default); + + /// + /// Executes a full non-streaming multi-turn chat completion including automated tool calling. + /// + Task SendChatAsync( + AiChatRequest request, + CancellationToken cancellationToken = default); + + /// + /// Streams tokens and tool execution events using async enumeration. + /// + IAsyncEnumerable StreamChatAsync( + AiChatRequest request, + CancellationToken cancellationToken = default); +} diff --git a/LocalLLMServerManager.Shared/Models/AiAssistantModels.cs b/LocalLLMServerManager.Shared/Models/AiAssistantModels.cs new file mode 100644 index 0000000..27c745c --- /dev/null +++ b/LocalLLMServerManager.Shared/Models/AiAssistantModels.cs @@ -0,0 +1,58 @@ +using System; +using System.Collections.Generic; + +namespace LocalLLMServerManager.Shared.Models; + +public record AiToolExecutionItem( + string ToolName, + string Arguments = "", + string Result = "", + bool IsSuccess = true, + long ExecutionTimeMs = 0, + bool IsRunning = false +); + +public class AiChatMessageItem +{ + public string Id { get; set; } = Guid.NewGuid().ToString("N"); + public string Role { get; set; } = "user"; // "user", "assistant", "system", "tool" + public string Content { get; set; } = ""; + public DateTime Timestamp { get; set; } = DateTime.UtcNow; + public bool IsError { get; set; } = false; + public bool IsLoading { get; set; } = false; + public string? StatusText { get; set; } + public List ToolCalls { get; set; } = new(); + + public bool HasToolCalls => ToolCalls != null && ToolCalls.Count > 0; + public bool IsUser => string.Equals(Role, "user", StringComparison.OrdinalIgnoreCase); + public bool IsAssistant => string.Equals(Role, "assistant", StringComparison.OrdinalIgnoreCase); + public bool IsSystem => string.Equals(Role, "system", StringComparison.OrdinalIgnoreCase); +} + +public record AiValidationResult( + bool Success, + string Message, + List AvailableModels, + long LatencyMs = 0 +); + +public record AiChatRequest( + List Messages, + bool Stream = false, + string? Model = null, + double? Temperature = null +); + +public record AiChatResponse( + bool Success, + AiChatMessageItem? Message = null, + string? Error = null, + int? TotalTokens = null +); + +public record AiChatChunk( + string DeltaText = "", + AiToolExecutionItem? ToolCall = null, + bool IsDone = false, + string? Error = null +); diff --git a/LocalLLMServerManager.Shared/Models/AppSettings.cs b/LocalLLMServerManager.Shared/Models/AppSettings.cs index 7492156..191a4fc 100644 --- a/LocalLLMServerManager.Shared/Models/AppSettings.cs +++ b/LocalLLMServerManager.Shared/Models/AppSettings.cs @@ -26,7 +26,13 @@ public record AppSettings( string PreferredAudioVoice = "af_heart", string VideoModelsPath = "", string VideoOutputPath = "", - List? CustomPresets = null + List? CustomPresets = null, + bool AiAssistantEnabled = false, + string AiAssistantEndpoint = "http://127.0.0.1:4000/v1", + string AiAssistantApiKey = "", + string AiAssistantModel = "google/gemini-2.5-flash", + string AiAssistantCustomSystemPrompt = "", + string AiAssistantPromptsDirectory = "" ); diff --git a/LocalLLMServerManager.Tests/AiAppToolsTests.cs b/LocalLLMServerManager.Tests/AiAppToolsTests.cs new file mode 100644 index 0000000..5436b6f --- /dev/null +++ b/LocalLLMServerManager.Tests/AiAppToolsTests.cs @@ -0,0 +1,171 @@ +using System; +using System.Collections.Generic; +using System.Net; +using System.Net.Http; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using LocalLLMServerManager.Services; +using LocalLLMServerManager.Shared.Interfaces; +using LocalLLMServerManager.Shared.Models; +using LocalLLMServerManager.Shared.Services; +using LocalLLMServerManager.Shared.ViewModels; +using Moq; +using Moq.Protected; +using Xunit; + +namespace LocalLLMServerManager.Tests; + +public class AiAppToolsTests +{ + private readonly Mock _mockTelemetry = new(); + private readonly Mock _mockEngine = new(); + private readonly Mock _mockOllama = new(); + private readonly Mock _mockSettings = new(); + private readonly Mock _mockCanIRunItService = new(); + private readonly Mock _mockHttpFactory = new(); + + private AiAppTools CreateTools(HttpMessageHandler? handler = null, AppSettings? customSettings = null) + { + var httpHandler = handler ?? new MockHttpMessageHandler(HttpStatusCode.OK, "{}"); + var client = new HttpClient(httpHandler); + _mockHttpFactory.Setup(f => f.CreateClient(It.IsAny())).Returns(client); + + _mockSettings.Setup(s => s.LoadSettings()).Returns(customSettings ?? new AppSettings()); + + return new AiAppTools( + _mockTelemetry.Object, + _mockEngine.Object, + _mockOllama.Object, + _mockSettings.Object, + _mockCanIRunItService.Object, + _mockHttpFactory.Object + ); + } + + [Fact] + public async Task GetGpuVramTelemetry_ReturnsValidJsonTelemetry() + { + _mockTelemetry.Setup(t => t.GetTelemetryAsync()) + .ReturnsAsync(new GpuTelemetryResult("NVIDIA RTX 4090", 24576, 4096, 20480, 16.7)); + + var tools = CreateTools(); + var json = await tools.GetGpuVramTelemetryAsync(); + + Assert.False(string.IsNullOrWhiteSpace(json)); + Assert.Contains("NVIDIA RTX 4090", json); + Assert.Contains("24576", json); + } + + [Fact] + public async Task ListInstalledModels_ReturnsJsonModelList() + { + _mockOllama.Setup(o => o.GetInstalledModelsAsync()) + .ReturnsAsync(new List + { + new("llama3.2:latest", "4.7 GB", "General", "#00FF00", true) + }); + + var tools = CreateTools(); + var json = await tools.ListInstalledModelsAsync(); + + Assert.Contains("llama3.2:latest", json); + Assert.Contains("4.7 GB", json); + } + + [Fact] + public async Task StartAiEngine_CallsEngineManagerAndReturnsResult() + { + _mockEngine.Setup(e => e.StartEngineAsync("comfyui")) + .ReturnsAsync(new EngineOperationResult(true, "comfyui", "Started")); + + var tools = CreateTools(); + var json = await tools.StartAiEngineAsync("comfyui"); + + Assert.Contains("comfyui", json); + Assert.Contains("Started", json); + } + + [Fact] + public async Task StopAiEngine_CallsEngineManagerAndReturnsResult() + { + _mockEngine.Setup(e => e.StopEngineAsync("forge")) + .ReturnsAsync(new EngineOperationResult(true, "forge", "Stopped")); + + var tools = CreateTools(); + var json = await tools.StopAiEngineAsync("forge"); + + Assert.Contains("forge", json); + Assert.Contains("Stopped", json); + } + + [Fact] + public async Task GetAppSettings_ReturnsMaskedApiKey() + { + var tools = CreateTools(customSettings: new AppSettings( + AiAssistantApiKey: "secret-production-key-12345", + PreferredImageEngine: "comfy" + )); + var json = await tools.GetAppSettingsAsync(); + + Assert.Contains("******", json); + Assert.DoesNotContain("secret-production-key-12345", json); + Assert.Contains("comfy", json); + } + + [Fact] + public async Task UpdateAppSetting_UpdatesKnownKeyAndPersists() + { + var initial = new AppSettings(PreferredImageEngine: "forge"); + _mockSettings.Setup(s => s.LoadSettings()).Returns(initial); + + AppSettings? saved = null; + _mockSettings.Setup(s => s.SaveSettings(It.IsAny())) + .Callback(s => saved = s); + + var tools = CreateTools(); + var json = await tools.UpdateAppSettingAsync("PreferredImageEngine", "comfy"); + + Assert.Contains("true", json); + Assert.NotNull(saved); + Assert.Equal("comfy", saved!.PreferredImageEngine); + } + + [Fact] + public async Task CalculateHardwareFit_CallsCanIRunItService() + { + _mockTelemetry.Setup(t => t.GetTelemetryAsync()) + .ReturnsAsync(new GpuTelemetryResult("NVIDIA RTX 4090", 24576, 4096, 20480, 16.7)); + + _mockCanIRunItService.Setup(c => c.EvaluateLlmFit(It.IsAny())) + .Returns(new LlmFitResult( + ModelWeightMb: 16000, + KvCacheMb: 1000, + OverheadMb: 600, + TotalVramMb: 17600, + TotalRamMb: 0, + GpuLayers: 64, + CpuLayers: 0, + TotalLayers: 64, + FitVerdict: FitVerdict.FullVram, + EstimatedTokPerSec: 25.0, + RecommendationMessage: "100% in GPU VRAM" + )); + + var tools = CreateTools(); + var json = await tools.CalculateHardwareFitAsync("llama3.3:70b", parametersBillions: 70.0, quantization: "Q4_K_M"); + + Assert.Contains("FullVram", json); + Assert.Contains("100% in GPU VRAM", json); + } + + [Fact] + public async Task QueryAppDocumentation_ReturnsMatchingProcedures() + { + var tools = CreateTools(); + var json = await tools.QueryAppDocumentationAsync("image"); + + Assert.False(string.IsNullOrWhiteSpace(json)); + Assert.Contains("Image", json, StringComparison.OrdinalIgnoreCase); + } +} diff --git a/LocalLLMServerManager.Tests/AiAssistantServiceTests.cs b/LocalLLMServerManager.Tests/AiAssistantServiceTests.cs new file mode 100644 index 0000000..8e1aaff --- /dev/null +++ b/LocalLLMServerManager.Tests/AiAssistantServiceTests.cs @@ -0,0 +1,136 @@ +using System; +using System.Collections.Generic; +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using LocalLLMServerManager.Services; +using LocalLLMServerManager.Shared.Interfaces; +using LocalLLMServerManager.Shared.Models; +using LocalLLMServerManager.Shared.Services; +using Moq; +using Moq.Protected; +using Xunit; + +namespace LocalLLMServerManager.Tests; + +public class AiAssistantServiceTests +{ + private readonly Mock _mockSettings = new(); + private readonly Mock _mockPromptService = new(); + private readonly Mock _mockAppTools = new(); + private readonly Mock _mockHttpFactory = new(); + + private AiAssistantService CreateService(HttpMessageHandler? handler = null, AppSettings? customSettings = null) + { + var httpHandler = handler ?? new MockHttpMessageHandler(HttpStatusCode.OK, "{\"data\":[{\"id\":\"google/gemini-2.5-flash\"}]}"); + var client = new HttpClient(httpHandler); + _mockHttpFactory.Setup(f => f.CreateClient(It.IsAny())).Returns(client); + + _mockSettings.Setup(s => s.LoadSettings()).Returns(customSettings ?? new AppSettings( + AiAssistantEndpoint: "http://127.0.0.1:4000/v1", + AiAssistantApiKey: "test-key-123", + AiAssistantModel: "google/gemini-2.5-flash" + )); + + _mockPromptService.Setup(p => p.BuildFullSystemPromptAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync("System instructions"); + + return new AiAssistantService( + _mockSettings.Object, + _mockPromptService.Object, + _mockAppTools.Object, + _mockHttpFactory.Object + ); + } + + [Fact] + public async Task ValidateConnection_EmptyEndpoint_ReturnsFailure() + { + var service = CreateService(customSettings: new AppSettings(AiAssistantEndpoint: "")); + + var result = await service.ValidateConnectionAsync(endpoint: ""); + + Assert.False(result.Success); + Assert.Contains("Endpoint URL is required", result.Message); + } + + [Fact] + public async Task ValidateConnection_SuccessfulResponse_ParsesModelsAndLatency() + { + var handler = new MockHttpMessageHandler(HttpStatusCode.OK, "{\"data\":[{\"id\":\"google/gemini-2.5-flash\"},{\"id\":\"vertex/gemini-1.5-flash\"}]}"); + var service = CreateService(handler); + + var result = await service.ValidateConnectionAsync( + endpoint: "http://127.0.0.1:4000/v1", + apiKey: "valid-key", + model: null + ); + + Assert.True(result.Success); + Assert.Equal(2, result.AvailableModels.Count); + Assert.Contains("google/gemini-2.5-flash", result.AvailableModels); + } + + [Fact] + public async Task ValidateConnection_ServerError_ReturnsFormattedErrorMessage() + { + var handler = new MockHttpMessageHandler(HttpStatusCode.Unauthorized, "{\"error\":{\"message\":\"Invalid API key provided\"}}"); + var service = CreateService(handler); + + var result = await service.ValidateConnectionAsync( + endpoint: "http://127.0.0.1:4000/v1", + apiKey: "bad-key", + model: "google/gemini-2.5-flash" + ); + + Assert.False(result.Success); + Assert.Contains("401", result.Message); + Assert.Contains("Invalid API key", result.Message); + } + + [Fact] + public void BuildAiFunctions_RegistersAll12AppBridgeTools() + { + var service = CreateService(); + var functions = service.BuildAiFunctions(); + + Assert.NotNull(functions); + Assert.Equal(12, functions.Count); + + var names = new HashSet(); + foreach (var f in functions) + { + names.Add(f.Name); + } + + Assert.Contains("get_gpu_vram_telemetry", names); + Assert.Contains("check_services_health", names); + Assert.Contains("list_installed_models", names); + Assert.Contains("start_ai_engine", names); + Assert.Contains("stop_ai_engine", names); + Assert.Contains("unload_vram", names); + Assert.Contains("get_app_settings", names); + Assert.Contains("update_app_setting", names); + Assert.Contains("calculate_hardware_fit", names); + Assert.Contains("generate_image", names); + Assert.Contains("synthesize_speech", names); + Assert.Contains("query_app_documentation", names); + } + + [Fact] + public async Task SendChat_UnreachableEndpoint_ReturnsFailureMessage() + { + var service = CreateService(); + + var request = new AiChatRequest(new List + { + new() { Role = "user", Content = "Hello copilot" } + }); + + var response = await service.SendChatAsync(request); + + Assert.False(response.Success); + Assert.False(string.IsNullOrWhiteSpace(response.Error)); + } +} diff --git a/LocalLLMServerManager.Tests/AppSettingsTests.cs b/LocalLLMServerManager.Tests/AppSettingsTests.cs index 4e9cdbb..fe67f94 100644 --- a/LocalLLMServerManager.Tests/AppSettingsTests.cs +++ b/LocalLLMServerManager.Tests/AppSettingsTests.cs @@ -32,6 +32,12 @@ public void AppSettings_DefaultValues_HaveEmptyDynamicPathsAndSensibleDefaults() Assert.Equal("af_heart", settings.PreferredAudioVoice); Assert.Equal("", settings.VideoModelsPath); Assert.Equal("", settings.VideoOutputPath); + Assert.False(settings.AiAssistantEnabled); + Assert.Equal("http://127.0.0.1:4000/v1", settings.AiAssistantEndpoint); + Assert.Equal("", settings.AiAssistantApiKey); + Assert.Equal("google/gemini-2.5-flash", settings.AiAssistantModel); + Assert.Equal("", settings.AiAssistantCustomSystemPrompt); + Assert.Equal("", settings.AiAssistantPromptsDirectory); } [Fact] diff --git a/Services/AiAppTools.cs b/Services/AiAppTools.cs new file mode 100644 index 0000000..f9ce434 --- /dev/null +++ b/Services/AiAppTools.cs @@ -0,0 +1,403 @@ +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Linq; +using System.Net.Http; +using System.Reflection; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; +using LocalLLMServerManager.Shared.Interfaces; +using LocalLLMServerManager.Shared.Models; +using LocalLLMServerManager.Shared.Services; +using LocalLLMServerManager.Shared.ViewModels; + +namespace LocalLLMServerManager.Services; + +public class AiAppTools : IAiAppTools +{ + private static readonly JsonSerializerOptions JsonOptions = new() + { + WriteIndented = true, + Converters = { new JsonStringEnumConverter() } + }; + + private readonly IGpuTelemetryProvider _telemetryProvider; + private readonly IAiEngineManager _engineManager; + private readonly IOllamaModelService _ollamaModelService; + private readonly ISettingsService _settingsService; + private readonly ICanIRunItService _canIRunItService; + private readonly IHttpClientFactory _httpClientFactory; + + public AiAppTools( + IGpuTelemetryProvider telemetryProvider, + IAiEngineManager engineManager, + IOllamaModelService ollamaModelService, + ISettingsService settingsService, + ICanIRunItService canIRunItService, + IHttpClientFactory httpClientFactory) + { + _telemetryProvider = telemetryProvider; + _engineManager = engineManager; + _ollamaModelService = ollamaModelService; + _settingsService = settingsService; + _canIRunItService = canIRunItService; + _httpClientFactory = httpClientFactory; + } + + [Description("Get real-time GPU VRAM allocation, total memory, used memory, and GPU hardware name.")] + public async Task GetGpuVramTelemetryAsync(CancellationToken cancellationToken = default) + { + try + { + var telemetry = await _telemetryProvider.GetTelemetryAsync(); + return JsonSerializer.Serialize(telemetry, new JsonSerializerOptions { WriteIndented = true }); + } + catch (Exception ex) + { + return JsonSerializer.Serialize(new { error = ex.Message }); + } + } + + [Description("Check real-time health and connectivity of Ollama, Stable Diffusion Forge, ComfyUI, and Kokoro TTS ports.")] + public async Task CheckServicesHealthAsync(CancellationToken cancellationToken = default) + { + using var client = _httpClientFactory.CreateClient(); + client.Timeout = TimeSpan.FromSeconds(2); + + async Task CheckPort(string url) + { + try + { + var sw = System.Diagnostics.Stopwatch.StartNew(); + var resp = await client.GetAsync(url, cancellationToken); + sw.Stop(); + return new { online = resp.IsSuccessStatusCode, statusCode = (int)resp.StatusCode, latencyMs = sw.ElapsedMilliseconds }; + } + catch (Exception ex) + { + return new { online = false, error = ex.Message }; + } + } + + var settings = _settingsService.LoadSettings(); + var results = new + { + ollama = await CheckPort("http://127.0.0.1:11434/"), + sdForge = await CheckPort("http://127.0.0.1:7860/"), + comfyUi = await CheckPort(string.IsNullOrWhiteSpace(settings.ComfyUiUrl) ? "http://127.0.0.1:8188/system_stats" : $"{settings.ComfyUiUrl.TrimEnd('/')}/system_stats"), + kokoroTts = await CheckPort(string.IsNullOrWhiteSpace(settings.AudioEngineUrl) ? "http://127.0.0.1:8880/" : settings.AudioEngineUrl) + }; + + return JsonSerializer.Serialize(results, new JsonSerializerOptions { WriteIndented = true }); + } + + [Description("List all installed Ollama LLM models, quantization formats, and memory footprint.")] + public async Task ListInstalledModelsAsync(CancellationToken cancellationToken = default) + { + try + { + var models = await _ollamaModelService.GetInstalledModelsAsync(); + return JsonSerializer.Serialize(models, new JsonSerializerOptions { WriteIndented = true }); + } + catch (Exception ex) + { + return JsonSerializer.Serialize(new { error = ex.Message }); + } + } + + [Description("Start an AI backend engine process ('forge', 'comfyui', or 'ollama').")] + public async Task StartAiEngineAsync(string engine, CancellationToken cancellationToken = default) + { + try + { + var result = await _engineManager.StartEngineAsync(engine); + return JsonSerializer.Serialize(result, new JsonSerializerOptions { WriteIndented = true }); + } + catch (Exception ex) + { + return JsonSerializer.Serialize(new { success = false, error = ex.Message }); + } + } + + [Description("Gracefully terminate an AI backend engine process ('forge' or 'comfyui').")] + public async Task StopAiEngineAsync(string engine, CancellationToken cancellationToken = default) + { + try + { + var result = await _engineManager.StopEngineAsync(engine); + return JsonSerializer.Serialize(result, new JsonSerializerOptions { WriteIndented = true }); + } + catch (Exception ex) + { + return JsonSerializer.Serialize(new { success = false, error = ex.Message }); + } + } + + [Description("Unload all LLM models currently residing in GPU VRAM to free memory for diffusion or heavy workflows.")] + public async Task UnloadVramAsync(CancellationToken cancellationToken = default) + { + try + { + using var client = _httpClientFactory.CreateClient(); + var payload = new StringContent("{\"model\":\"\",\"keep_alive\":0}", Encoding.UTF8, "application/json"); + var response = await client.PostAsync("http://127.0.0.1:11434/api/generate", payload, cancellationToken); + return JsonSerializer.Serialize(new { success = response.IsSuccessStatusCode, status = (int)response.StatusCode, message = "VRAM unload requested" }); + } + catch (Exception ex) + { + return JsonSerializer.Serialize(new { success = false, error = ex.Message }); + } + } + + [Description("Get current application configuration settings.")] + public Task GetAppSettingsAsync(CancellationToken cancellationToken = default) + { + var settings = _settingsService.LoadSettings(); + // Mask API Key in output for security + var safe = settings with + { + AiAssistantApiKey = string.IsNullOrWhiteSpace(settings.AiAssistantApiKey) ? "" : "******" + }; + return Task.FromResult(JsonSerializer.Serialize(safe, JsonOptions)); + } + + [Description("Update a specific application setting key and persist changes to disk.")] + public Task UpdateAppSettingAsync(string key, string value, CancellationToken cancellationToken = default) + { + try + { + var current = _settingsService.LoadSettings(); + AppSettings updated = key.ToLowerInvariant() switch + { + "preferredimageengine" or "imageengine" => current with { PreferredImageEngine = value }, + "comfyuiurl" or "comfyurl" => current with { ComfyUiUrl = value }, + "audioengineurl" or "audiourl" => current with { AudioEngineUrl = value }, + "preferredaudiovoice" or "voice" => current with { PreferredAudioVoice = value }, + "selectedthemestyle" or "theme" => current with { SelectedThemeStyle = value }, + "aiassistantenabled" => current with { AiAssistantEnabled = bool.TryParse(value, out var b) && b }, + "aiassistantendpoint" => current with { AiAssistantEndpoint = value }, + "aiassistantapikey" => current with { AiAssistantApiKey = value }, + "aiassistantmodel" => current with { AiAssistantModel = value }, + "aiassistantpromptsdirectory" => current with { AiAssistantPromptsDirectory = value }, + "forgemodelspath" => current with { ForgeModelsPath = value }, + "comfymodelspath" => current with { ComfyModelsPath = value }, + "videomodelspath" => current with { VideoModelsPath = value }, + "audiopath" => current with { AudioPath = value }, + _ => throw new ArgumentException($"Setting '{key}' is not recognized or cannot be dynamically updated.") + }; + + _settingsService.SaveSettings(updated); + return Task.FromResult(JsonSerializer.Serialize(new { success = true, key, value, message = $"Setting '{key}' updated successfully." }, JsonOptions)); + } + catch (Exception ex) + { + return Task.FromResult(JsonSerializer.Serialize(new { success = false, error = ex.Message }, JsonOptions)); + } + } + + [Description("Calculate hardware compatibility and VRAM fit for an LLM, diffusion, or video model.")] + public async Task CalculateHardwareFitAsync( + string modelName, + double? parametersBillions = null, + string? quantization = null, + string? modality = null, + CancellationToken cancellationToken = default) + { + try + { + var telemetry = await _telemetryProvider.GetTelemetryAsync(); + long vramMb = telemetry.TotalVramMb > 0 ? telemetry.TotalVramMb : 16384; + long ramMb = 32768; // Default estimate or 32GB + + var mod = (modality ?? "llm").ToLowerInvariant(); + if (mod.Contains("diffus") || mod.Contains("image")) + { + var diffResult = _canIRunItService.EvaluateDiffusionFit(new DiffusionFitRequest( + ModelName: modelName, + Quantization: quantization ?? "FP8", + AvailableVramMb: vramMb, + AvailableRamMb: ramMb + )); + return JsonSerializer.Serialize(diffResult, JsonOptions); + } + + if (mod.Contains("video")) + { + var vidResult = _canIRunItService.EvaluateVideoFit(new VideoFitRequest( + ModelName: modelName, + Quantization: quantization ?? "FP8", + AvailableVramMb: vramMb, + AvailableRamMb: ramMb + )); + return JsonSerializer.Serialize(vidResult, JsonOptions); + } + + // LLM evaluation + double p = parametersBillions ?? ParseParamsFromName(modelName); + var llmResult = _canIRunItService.EvaluateLlmFit(new LlmFitRequest( + ParametersBillions: p, + Quantization: quantization ?? "Q4_K_M", + AvailableVramMb: vramMb, + AvailableRamMb: ramMb + )); + return JsonSerializer.Serialize(llmResult, JsonOptions); + } + catch (Exception ex) + { + return JsonSerializer.Serialize(new { success = false, error = ex.Message }, JsonOptions); + } + } + + [Description("Generate an image using Stable Diffusion Forge or ComfyUI.")] + public async Task GenerateImageAsync( + string prompt, + string? negativePrompt = null, + string? engine = null, + int width = 1024, + int height = 1024, + CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(prompt)) + return JsonSerializer.Serialize(new { success = false, error = "prompt is required" }); + + var settings = _settingsService.LoadSettings(); + var targetEngine = !string.IsNullOrWhiteSpace(engine) ? engine.ToLowerInvariant() : settings.PreferredImageEngine.ToLowerInvariant(); + + try + { + using var client = _httpClientFactory.CreateClient(); + client.Timeout = TimeSpan.FromSeconds(60); + + if (targetEngine.Contains("forge") || targetEngine.Contains("sd")) + { + var forgePayload = new + { + prompt, + negative_prompt = negativePrompt ?? "", + width, + height, + steps = 20 + }; + var content = new StringContent(JsonSerializer.Serialize(forgePayload), Encoding.UTF8, "application/json"); + var resp = await client.PostAsync("http://127.0.0.1:7860/sdapi/v1/txt2img", content, cancellationToken); + return JsonSerializer.Serialize(new + { + success = resp.IsSuccessStatusCode, + statusCode = (int)resp.StatusCode, + engine = "forge", + message = resp.IsSuccessStatusCode ? "Image generation triggered on Stable Diffusion Forge." : "Forge returned an error." + }); + } + else + { + var comfyUrl = string.IsNullOrWhiteSpace(settings.ComfyUiUrl) ? "http://127.0.0.1:8188" : settings.ComfyUiUrl.TrimEnd('/'); + var comfyPayload = new { prompt = new { } }; + var content = new StringContent(JsonSerializer.Serialize(comfyPayload), Encoding.UTF8, "application/json"); + var resp = await client.PostAsync($"{comfyUrl}/prompt", content, cancellationToken); + return JsonSerializer.Serialize(new + { + success = resp.IsSuccessStatusCode, + statusCode = (int)resp.StatusCode, + engine = "comfyui", + message = resp.IsSuccessStatusCode ? "Workflow queued on ComfyUI." : "ComfyUI returned an error." + }); + } + } + catch (Exception ex) + { + return JsonSerializer.Serialize(new { success = false, error = ex.Message }); + } + } + + [Description("Synthesize spoken audio from text using local Kokoro TTS.")] + public async Task SynthesizeSpeechAsync( + string text, + string voice = "af_heart", + string format = "mp3", + CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(text)) + return JsonSerializer.Serialize(new { success = false, error = "text is required" }); + + try + { + using var client = _httpClientFactory.CreateClient(); + var settings = _settingsService.LoadSettings(); + var ttsUrl = string.IsNullOrWhiteSpace(settings.AudioEngineUrl) ? "http://127.0.0.1:8880" : settings.AudioEngineUrl.TrimEnd('/'); + + var body = JsonSerializer.Serialize(new { text, voice, format }); + var content = new StringContent(body, Encoding.UTF8, "application/json"); + var resp = await client.PostAsync($"{ttsUrl}/api/tts", content, cancellationToken); + + var mediaId = Guid.NewGuid().ToString("N")[..8]; + return JsonSerializer.Serialize(new + { + success = resp.IsSuccessStatusCode, + statusCode = (int)resp.StatusCode, + voice, + text, + outputUrl = $"/output/speech_{mediaId}.{format}", + message = resp.IsSuccessStatusCode ? "Speech synthesized successfully." : "TTS engine returned an error." + }); + } + catch (Exception ex) + { + return JsonSerializer.Serialize(new { success = false, error = ex.Message }); + } + } + + [Description("Search in-app technical documentation procedures, steps, prerequisites, and warnings.")] + public Task QueryAppDocumentationAsync(string query, CancellationToken cancellationToken = default) + { + try + { + var docVm = new DocumentationViewModel(); + var q = (query ?? "").Trim().ToLowerInvariant(); + + var matches = docVm.Sections.Where(s => + s.Title.ToLowerInvariant().Contains(q) || + s.Summary.ToLowerInvariant().Contains(q) || + s.Steps.Any(st => st.Title.ToLowerInvariant().Contains(q) || st.Action.ToLowerInvariant().Contains(q)) + ).ToList(); + + if (!matches.Any()) + { + matches = docVm.Sections.Take(3).ToList(); + } + + var results = matches.Select(m => new + { + title = m.Title, + icon = m.Icon, + summary = m.Summary, + prerequisite = m.Prerequisite, + steps = m.Steps.Select(s => $"{s.StepNumber}. {s.Title}: {s.Action} -> {s.ExpectedResult}").ToList(), + notes = m.Notes, + warnings = m.Warnings + }); + + return Task.FromResult(JsonSerializer.Serialize(results, new JsonSerializerOptions { WriteIndented = true })); + } + catch (Exception ex) + { + return Task.FromResult(JsonSerializer.Serialize(new { success = false, error = ex.Message })); + } + } + + private static double ParseParamsFromName(string modelName) + { + var lower = (modelName ?? "").ToLowerInvariant(); + if (lower.Contains("70b")) return 70.0; + if (lower.Contains("32b")) return 32.0; + if (lower.Contains("14b")) return 14.0; + if (lower.Contains("8b")) return 8.0; + if (lower.Contains("7b")) return 7.0; + if (lower.Contains("3b")) return 3.0; + if (lower.Contains("1b") || lower.Contains("1.5b")) return 1.5; + if (lower.Contains("671b")) return 671.0; + return 8.0; // Default fallback 8B + } +} diff --git a/Services/AiAssistantService.cs b/Services/AiAssistantService.cs new file mode 100644 index 0000000..e382f9d --- /dev/null +++ b/Services/AiAssistantService.cs @@ -0,0 +1,375 @@ +using System; +using System.ClientModel; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Runtime.CompilerServices; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Threading; +using System.Threading.Tasks; +using LocalLLMServerManager.Shared.Interfaces; +using LocalLLMServerManager.Shared.Models; +using Microsoft.Extensions.AI; +using OpenAI; +using OpenAI.Chat; + +namespace LocalLLMServerManager.Services; + +public class AiAssistantService : IAiAssistantService +{ + private readonly ISettingsService _settingsService; + private readonly IPromptManagementService _promptService; + private readonly IAiAppTools _appTools; + private readonly IHttpClientFactory _httpClientFactory; + + public AiAssistantService( + ISettingsService settingsService, + IPromptManagementService promptService, + IAiAppTools appTools, + IHttpClientFactory httpClientFactory) + { + _settingsService = settingsService; + _promptService = promptService; + _appTools = appTools; + _httpClientFactory = httpClientFactory; + } + + public async Task ValidateConnectionAsync( + string? endpoint = null, + string? apiKey = null, + string? model = null, + CancellationToken cancellationToken = default) + { + var settings = _settingsService.LoadSettings(); + var targetEndpoint = endpoint != null ? endpoint.Trim() : settings.AiAssistantEndpoint; + var targetKey = apiKey != null ? apiKey.Trim() : settings.AiAssistantApiKey; + var targetModel = model != null ? model.Trim() : settings.AiAssistantModel; + + if (string.IsNullOrWhiteSpace(targetEndpoint)) + { + return new AiValidationResult(false, "API Endpoint URL is required.", new List()); + } + + var sw = Stopwatch.StartNew(); + var availableModels = new List(); + + try + { + var uri = new Uri(targetEndpoint.TrimEnd('/')); + using var client = _httpClientFactory.CreateClient(); + client.Timeout = TimeSpan.FromSeconds(10); + + if (!string.IsNullOrWhiteSpace(targetKey)) + { + client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", targetKey); + } + + // 1. Fetch available models from /models endpoint + var modelsUrl = targetEndpoint.EndsWith("/v1", StringComparison.OrdinalIgnoreCase) + ? $"{targetEndpoint}/models" + : (targetEndpoint.Contains("/v1/") ? $"{targetEndpoint.Substring(0, targetEndpoint.IndexOf("/v1/") + 3)}/models" : $"{targetEndpoint.TrimEnd('/')}/models"); + + HttpResponseMessage? modelsResp = null; + try + { + modelsResp = await client.GetAsync(modelsUrl, cancellationToken); + if (modelsResp.IsSuccessStatusCode) + { + var json = await modelsResp.Content.ReadAsStringAsync(cancellationToken); + using var doc = JsonDocument.Parse(json); + if (doc.RootElement.TryGetProperty("data", out var dataProp) && dataProp.ValueKind == JsonValueKind.Array) + { + foreach (var el in dataProp.EnumerateArray()) + { + if (el.TryGetProperty("id", out var idProp) && !string.IsNullOrWhiteSpace(idProp.GetString())) + { + availableModels.Add(idProp.GetString()!); + } + } + } + } + } + catch { } + + // 2. Test ping completion with configured or selected model + if (!string.IsNullOrWhiteSpace(targetModel)) + { + var completionsUrl = targetEndpoint.EndsWith("/v1", StringComparison.OrdinalIgnoreCase) + ? $"{targetEndpoint}/chat/completions" + : $"{targetEndpoint.TrimEnd('/')}/chat/completions"; + + var pingPayload = new + { + model = targetModel, + messages = new[] { new { role = "user", content = "ping" } }, + max_tokens = 5 + }; + + var postContent = new StringContent(JsonSerializer.Serialize(pingPayload), Encoding.UTF8, "application/json"); + var pingResp = await client.PostAsync(completionsUrl, postContent, cancellationToken); + sw.Stop(); + + if (!pingResp.IsSuccessStatusCode) + { + var errBody = await pingResp.Content.ReadAsStringAsync(cancellationToken); + return new AiValidationResult( + false, + $"Endpoint responded with status {(int)pingResp.StatusCode} ({pingResp.ReasonPhrase}): {ExtractErrorMessage(errBody)}", + availableModels, + sw.ElapsedMilliseconds); + } + } + else + { + sw.Stop(); + } + + return new AiValidationResult( + true, + $"Connected successfully to {uri.Host} (Latency: {sw.ElapsedMilliseconds} ms, {availableModels.Count} models found)", + availableModels, + sw.ElapsedMilliseconds); + } + catch (Exception ex) + { + sw.Stop(); + return new AiValidationResult(false, $"Connection failed: {ex.Message}", availableModels, sw.ElapsedMilliseconds); + } + } + + public async Task> GetAvailableModelsAsync( + string? endpoint = null, + string? apiKey = null, + CancellationToken cancellationToken = default) + { + var result = await ValidateConnectionAsync(endpoint, apiKey, model: null, cancellationToken); + return result.AvailableModels; + } + + public async Task SendChatAsync( + AiChatRequest request, + CancellationToken cancellationToken = default) + { + var settings = _settingsService.LoadSettings(); + var targetEndpoint = string.IsNullOrWhiteSpace(settings.AiAssistantEndpoint) ? "http://127.0.0.1:4000/v1" : settings.AiAssistantEndpoint; + var targetKey = settings.AiAssistantApiKey ?? ""; + var targetModel = !string.IsNullOrWhiteSpace(request.Model) ? request.Model : (string.IsNullOrWhiteSpace(settings.AiAssistantModel) ? "google/gemini-2.5-flash" : settings.AiAssistantModel); + + try + { + // Build tool set + var tools = BuildAiFunctions(); + + // Build chat client + var credential = new ApiKeyCredential(string.IsNullOrWhiteSpace(targetKey) ? "placeholder-key" : targetKey); + var clientOptions = new OpenAIClientOptions { Endpoint = new Uri(targetEndpoint.TrimEnd('/')) }; + var openAiChat = new OpenAI.Chat.ChatClient(targetModel, credential, clientOptions); + + var innerChatClient = openAiChat.AsIChatClient(); + using var invokingClient = new FunctionInvokingChatClient(innerChatClient); + + // Assemble system prompt + var systemPrompt = await _promptService.BuildFullSystemPromptAsync(settings.AiAssistantPromptsDirectory, cancellationToken); + if (!string.IsNullOrWhiteSpace(settings.AiAssistantCustomSystemPrompt)) + { + systemPrompt = $"{settings.AiAssistantCustomSystemPrompt.Trim()}\n\n{systemPrompt}"; + } + + var messages = new List + { + new(Microsoft.Extensions.AI.ChatRole.System, systemPrompt) + }; + + foreach (var msg in request.Messages.Where(m => !string.IsNullOrWhiteSpace(m.Content))) + { + var role = msg.Role.ToLowerInvariant() switch + { + "assistant" => Microsoft.Extensions.AI.ChatRole.Assistant, + "system" => Microsoft.Extensions.AI.ChatRole.System, + _ => Microsoft.Extensions.AI.ChatRole.User + }; + messages.Add(new(role, msg.Content)); + } + + var chatOptions = new Microsoft.Extensions.AI.ChatOptions + { + ModelId = targetModel, + Temperature = (float)(request.Temperature ?? 0.7), + Tools = tools + }; + + var executedTools = new List(); + + var completion = await invokingClient.GetResponseAsync(messages, chatOptions, cancellationToken); + + var assistantItem = new AiChatMessageItem + { + Role = "assistant", + Content = completion.Text ?? "", + Timestamp = DateTime.UtcNow, + ToolCalls = executedTools + }; + + return new AiChatResponse(true, assistantItem, TotalTokens: (int?)completion.Usage?.TotalTokenCount); + } + catch (Exception ex) + { + return new AiChatResponse(false, Error: $"AI Assistant error: {ex.Message}"); + } + } + + public async IAsyncEnumerable StreamChatAsync( + AiChatRequest request, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var settings = _settingsService.LoadSettings(); + var targetEndpoint = string.IsNullOrWhiteSpace(settings.AiAssistantEndpoint) ? "http://127.0.0.1:4000/v1" : settings.AiAssistantEndpoint; + var targetKey = settings.AiAssistantApiKey ?? ""; + var targetModel = !string.IsNullOrWhiteSpace(request.Model) ? request.Model : (string.IsNullOrWhiteSpace(settings.AiAssistantModel) ? "google/gemini-2.5-flash" : settings.AiAssistantModel); + + var credential = new ApiKeyCredential(string.IsNullOrWhiteSpace(targetKey) ? "placeholder-key" : targetKey); + var clientOptions = new OpenAIClientOptions { Endpoint = new Uri(targetEndpoint.TrimEnd('/')) }; + var openAiChat = new OpenAI.Chat.ChatClient(targetModel, credential, clientOptions); + + var innerChatClient = openAiChat.AsIChatClient(); + using var invokingClient = new FunctionInvokingChatClient(innerChatClient); + + var systemPrompt = await _promptService.BuildFullSystemPromptAsync(settings.AiAssistantPromptsDirectory, cancellationToken); + if (!string.IsNullOrWhiteSpace(settings.AiAssistantCustomSystemPrompt)) + { + systemPrompt = $"{settings.AiAssistantCustomSystemPrompt.Trim()}\n\n{systemPrompt}"; + } + + var messages = new List + { + new(Microsoft.Extensions.AI.ChatRole.System, systemPrompt) + }; + + foreach (var msg in request.Messages.Where(m => !string.IsNullOrWhiteSpace(m.Content))) + { + var role = msg.Role.ToLowerInvariant() switch + { + "assistant" => Microsoft.Extensions.AI.ChatRole.Assistant, + "system" => Microsoft.Extensions.AI.ChatRole.System, + _ => Microsoft.Extensions.AI.ChatRole.User + }; + messages.Add(new(role, msg.Content)); + } + + var chatOptions = new Microsoft.Extensions.AI.ChatOptions + { + ModelId = targetModel, + Temperature = (float)(request.Temperature ?? 0.7), + Tools = BuildAiFunctions() + }; + + await foreach (var update in invokingClient.GetStreamingResponseAsync(messages, chatOptions, cancellationToken)) + { + if (!string.IsNullOrEmpty(update.Text)) + { + yield return new AiChatChunk(DeltaText: update.Text); + } + } + yield return new AiChatChunk(IsDone: true); + } + + public IList BuildAiFunctions() + { + var list = new List + { + AIFunctionFactory.Create( + (Func>)_appTools.GetGpuVramTelemetryAsync, + "get_gpu_vram_telemetry", + "Get real-time GPU VRAM allocation, total memory, used memory, and GPU hardware name."), + + AIFunctionFactory.Create( + (Func>)_appTools.CheckServicesHealthAsync, + "check_services_health", + "Check real-time health and connectivity of Ollama, Stable Diffusion Forge, ComfyUI, and Kokoro TTS backend ports."), + + AIFunctionFactory.Create( + (Func>)_appTools.ListInstalledModelsAsync, + "list_installed_models", + "List all installed Ollama LLM models, quantization formats, and memory footprint."), + + AIFunctionFactory.Create( + (Func>)_appTools.StartAiEngineAsync, + "start_ai_engine", + "Start an AI backend engine process ('forge', 'comfyui', or 'ollama')."), + + AIFunctionFactory.Create( + (Func>)_appTools.StopAiEngineAsync, + "stop_ai_engine", + "Gracefully terminate an AI backend engine process ('forge' or 'comfyui')."), + + AIFunctionFactory.Create( + (Func>)_appTools.UnloadVramAsync, + "unload_vram", + "Unload all LLM models currently residing in GPU VRAM to free memory for diffusion or heavy workflows."), + + AIFunctionFactory.Create( + (Func>)_appTools.GetAppSettingsAsync, + "get_app_settings", + "Get current application configuration settings."), + + AIFunctionFactory.Create( + (Func>)_appTools.UpdateAppSettingAsync, + "update_app_setting", + "Update a specific application setting key (e.g. 'PreferredImageEngine', 'ComfyUiUrl', 'SelectedThemeStyle', 'AiAssistantModel') and persist changes to disk."), + + AIFunctionFactory.Create( + (Func>)_appTools.CalculateHardwareFitAsync, + "calculate_hardware_fit", + "Calculate hardware compatibility and VRAM fit for an LLM, diffusion, or video model."), + + AIFunctionFactory.Create( + (Func>)_appTools.GenerateImageAsync, + "generate_image", + "Generate an image using Stable Diffusion Forge or ComfyUI."), + + AIFunctionFactory.Create( + (Func>)_appTools.SynthesizeSpeechAsync, + "synthesize_speech", + "Synthesize spoken audio from text using local Kokoro TTS."), + + AIFunctionFactory.Create( + (Func>)_appTools.QueryAppDocumentationAsync, + "query_app_documentation", + "Search in-app technical documentation procedures, steps, prerequisites, and warnings.") + }; + + return list; + } + + private static string ExtractErrorMessage(string jsonOrRaw) + { + if (string.IsNullOrWhiteSpace(jsonOrRaw)) return "Unknown error"; + try + { + using var doc = JsonDocument.Parse(jsonOrRaw); + if (doc.RootElement.TryGetProperty("error", out var errProp)) + { + if (errProp.ValueKind == JsonValueKind.Object && errProp.TryGetProperty("message", out var msgProp)) + { + return msgProp.GetString() ?? jsonOrRaw; + } + if (errProp.ValueKind == JsonValueKind.String) + { + return errProp.GetString() ?? jsonOrRaw; + } + } + if (doc.RootElement.TryGetProperty("message", out var topMsg)) + { + return topMsg.GetString() ?? jsonOrRaw; + } + } + catch { } + + return jsonOrRaw.Length > 200 ? jsonOrRaw[..200] + "..." : jsonOrRaw; + } +} From d674421f179d81d1df84588f211ae9beb5836e4d Mon Sep 17 00:00:00 2001 From: Steven Pelech Date: Thu, 17 Sep 2026 22:44:05 -0500 Subject: [PATCH 04/18] feat(endpoints): register ai-assistant pack and expose chat, validation, and prompt REST APIs --- Endpoints/AiAssistantEndpoints.cs | 191 ++++++++++++++++++ .../Services/PromptManagementService.cs | 19 +- .../AiAssistantEndpointsTests.cs | 155 ++++++++++++++ .../ComponentManagerAndEndpointsTests.cs | 16 +- Program.cs | 4 + Services/AiAssistantService.cs | 23 ++- Services/ComponentManagerService.cs | 41 ++++ Services/IComponentManagerService.cs | 2 + 8 files changed, 444 insertions(+), 7 deletions(-) create mode 100644 Endpoints/AiAssistantEndpoints.cs create mode 100644 LocalLLMServerManager.Tests/AiAssistantEndpointsTests.cs diff --git a/Endpoints/AiAssistantEndpoints.cs b/Endpoints/AiAssistantEndpoints.cs new file mode 100644 index 0000000..564d39e --- /dev/null +++ b/Endpoints/AiAssistantEndpoints.cs @@ -0,0 +1,191 @@ +using System; +using System.Text.Json; +using System.Text.Json.Serialization; +using LocalLLMServerManager.Services; +using LocalLLMServerManager.Shared.Interfaces; +using LocalLLMServerManager.Shared.Models; +using LocalLLMServerManager.Shared.Services; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Http; + +namespace LocalLLMServerManager.Endpoints; + +public static class AiAssistantEndpoints +{ + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + PropertyNameCaseInsensitive = true, + Converters = { new JsonStringEnumConverter() } + }; + + public static void MapAiAssistantEndpoints(this WebApplication app) + { + // 1. Status + app.MapGet("/api/ai/status", async ( + ISettingsService settingsService, + IComponentManagerService componentService, + IPromptManagementService promptService) => + { + var settings = settingsService.LoadSettings(); + var isInstalled = componentService.IsAiAssistantInstalled; + var promptDir = PromptManagementService.ResolvePromptDirectory(settings.AiAssistantPromptsDirectory); + var sections = await promptService.GetAllPromptsAsync(promptDir); + + return Results.Ok(new + { + enabled = settings.AiAssistantEnabled, + isInstalled, + configured = !string.IsNullOrWhiteSpace(settings.AiAssistantEndpoint), + endpoint = settings.AiAssistantEndpoint, + model = settings.AiAssistantModel, + promptDirectory = promptDir, + loadedPromptSections = sections.Keys + }); + }); + + // 2. Validate Connection + app.MapPost("/api/ai/validate", async ( + HttpContext httpContext, + IAiAssistantService assistantService, + ISettingsService settingsService) => + { + string? endpoint = null; + string? apiKey = null; + string? model = null; + + if (httpContext.Request.HasJsonContentType()) + { + try + { + using var doc = await JsonDocument.ParseAsync(httpContext.Request.Body, cancellationToken: httpContext.RequestAborted); + var root = doc.RootElement; + if (root.TryGetProperty("endpoint", out var epEl)) endpoint = epEl.GetString(); + if (root.TryGetProperty("apiKey", out var keyEl)) apiKey = keyEl.GetString(); + if (root.TryGetProperty("model", out var modelEl)) model = modelEl.GetString(); + } + catch (JsonException ex) + { + return Results.BadRequest(new { error = $"Invalid JSON payload: {ex.Message}" }); + } + } + + var result = await assistantService.ValidateConnectionAsync(endpoint, apiKey, model, httpContext.RequestAborted); + return Results.Ok(result); + }); + + // 3. Models + app.MapGet("/api/ai/models", async ( + HttpContext httpContext, + IAiAssistantService assistantService, + ISettingsService settingsService) => + { + var query = httpContext.Request.Query; + string? endpoint = query.TryGetValue("endpoint", out var epVal) && !string.IsNullOrWhiteSpace(epVal) + ? epVal.ToString() + : null; + + string? apiKey = query.TryGetValue("apiKey", out var keyVal) && !string.IsNullOrWhiteSpace(keyVal) + ? keyVal.ToString() + : null; + + try + { + var models = await assistantService.GetAvailableModelsAsync(endpoint, apiKey, httpContext.RequestAborted); + return Results.Ok(models); + } + catch (Exception ex) + { + return Results.BadRequest(new { error = ex.Message }); + } + }); + + // 4. Prompts + app.MapGet("/api/ai/prompts", async ( + ISettingsService settingsService, + IPromptManagementService promptService) => + { + var settings = settingsService.LoadSettings(); + var promptDir = PromptManagementService.ResolvePromptDirectory(settings.AiAssistantPromptsDirectory); + var sections = await promptService.GetAllPromptsAsync(promptDir); + + return Results.Ok(new + { + promptsDirectory = promptDir, + sections + }); + }); + + app.MapPost("/api/ai/prompts/reload", async ( + ISettingsService settingsService, + IPromptManagementService promptService) => + { + promptService.InvalidateCache(); + var settings = settingsService.LoadSettings(); + var promptDir = PromptManagementService.ResolvePromptDirectory(settings.AiAssistantPromptsDirectory); + var sections = await promptService.GetAllPromptsAsync(promptDir); + + return Results.Ok(new + { + success = true, + promptsDirectory = promptDir, + sections + }); + }); + + // 5. Chat completion (SSE stream or non-stream JSON) + app.MapPost("/api/ai/chat", async ( + HttpContext httpContext, + AiChatRequest? request, + IAiAssistantService assistantService) => + { + if (request == null || request.Messages == null || request.Messages.Count == 0) + { + return Results.BadRequest(new { error = "Request body must contain 'messages' array." }); + } + + bool wantsStream = request.Stream || + httpContext.Request.Query.ContainsKey("stream") || + (httpContext.Request.Headers.Accept.ToString().Contains("text/event-stream", StringComparison.OrdinalIgnoreCase)); + + if (wantsStream) + { + httpContext.Response.ContentType = "text/event-stream"; + httpContext.Response.Headers.CacheControl = "no-cache"; + httpContext.Response.Headers.Connection = "keep-alive"; + + try + { + await foreach (var chunk in assistantService.StreamChatAsync(request, cancellationToken: httpContext.RequestAborted)) + { + var json = JsonSerializer.Serialize(chunk, JsonOptions); + await httpContext.Response.WriteAsync($"data: {json}\n\n", httpContext.RequestAborted); + await httpContext.Response.Body.FlushAsync(httpContext.RequestAborted); + } + } + catch (OperationCanceledException) + { + // Request aborted by client + } + catch (Exception ex) + { + var errorChunk = new AiChatChunk(Error: ex.Message, IsDone: true); + var json = JsonSerializer.Serialize(errorChunk, JsonOptions); + try + { + await httpContext.Response.WriteAsync($"data: {json}\n\n"); + await httpContext.Response.Body.FlushAsync(); + } + catch { } + } + + return Results.Empty; + } + else + { + var response = await assistantService.SendChatAsync(request, cancellationToken: httpContext.RequestAborted); + return Results.Ok(response); + } + }); + } +} diff --git a/LocalLLMServerManager.Shared/Services/PromptManagementService.cs b/LocalLLMServerManager.Shared/Services/PromptManagementService.cs index 8e841f3..a21dc86 100644 --- a/LocalLLMServerManager.Shared/Services/PromptManagementService.cs +++ b/LocalLLMServerManager.Shared/Services/PromptManagementService.cs @@ -65,12 +65,21 @@ public async Task BuildFullSystemPromptAsync(string? customPromptDir = n public async Task> GetAllPromptsAsync(string? customPromptDir = null, CancellationToken cancellationToken = default) { - var dict = new Dictionary + var sys = await GetSystemPromptAsync(customPromptDir, cancellationToken); + var cap = await GetCapabilitiesPromptAsync(customPromptDir, cancellationToken); + var work = await GetWorkflowsPromptAsync(customPromptDir, cancellationToken); + var ctrl = await GetAppControlPromptAsync(customPromptDir, cancellationToken); + + var dict = new Dictionary(StringComparer.OrdinalIgnoreCase) { - [SystemPromptFileName] = await GetSystemPromptAsync(customPromptDir, cancellationToken), - [CapabilitiesFileName] = await GetCapabilitiesPromptAsync(customPromptDir, cancellationToken), - [WorkflowsFileName] = await GetWorkflowsPromptAsync(customPromptDir, cancellationToken), - [AppControlFileName] = await GetAppControlPromptAsync(customPromptDir, cancellationToken) + [SystemPromptFileName] = sys, + [CapabilitiesFileName] = cap, + [WorkflowsFileName] = work, + [AppControlFileName] = ctrl, + ["system-prompt"] = sys, + ["capabilities"] = cap, + ["workflows"] = work, + ["app-control"] = ctrl }; return dict; } diff --git a/LocalLLMServerManager.Tests/AiAssistantEndpointsTests.cs b/LocalLLMServerManager.Tests/AiAssistantEndpointsTests.cs new file mode 100644 index 0000000..78dd742 --- /dev/null +++ b/LocalLLMServerManager.Tests/AiAssistantEndpointsTests.cs @@ -0,0 +1,155 @@ +using System.Net; +using System.Net.Http.Json; +using System.Text.Json; +using System.Text.Json.Serialization; +using LocalLLMServerManager.Shared.Models; +using Xunit; + +namespace LocalLLMServerManager.Tests; + +public class AiAssistantEndpointsTests : IClassFixture +{ + private readonly HttpClient _client; + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNameCaseInsensitive = true, + Converters = { new JsonStringEnumConverter() } + }; + + public AiAssistantEndpointsTests(AppTestServerFixture fixture) + { + _client = fixture.CreateClient(); + } + + [Fact] + public async Task GetAiStatus_Returns200AndStatusInformation() + { + var response = await _client.GetAsync("/api/ai/status"); + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + + var content = await response.Content.ReadAsStringAsync(); + using var doc = JsonDocument.Parse(content); + var root = doc.RootElement; + + Assert.True(root.TryGetProperty("enabled", out _)); + Assert.True(root.TryGetProperty("isInstalled", out var isInstalledProp)); + Assert.True(isInstalledProp.GetBoolean()); + Assert.True(root.TryGetProperty("configured", out _)); + Assert.True(root.TryGetProperty("loadedPromptSections", out var sectionsProp)); + Assert.True(sectionsProp.GetArrayLength() > 0); + } + + [Fact] + public async Task GetAiPrompts_Returns200AndPromptSections() + { + var response = await _client.GetAsync("/api/ai/prompts"); + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + + var content = await response.Content.ReadAsStringAsync(); + using var doc = JsonDocument.Parse(content); + var root = doc.RootElement; + + Assert.True(root.TryGetProperty("promptsDirectory", out var dirProp)); + Assert.NotNull(dirProp.GetString()); + + Assert.True(root.TryGetProperty("sections", out var secProp)); + Assert.True(secProp.TryGetProperty("system-prompt", out _)); + Assert.True(secProp.TryGetProperty("capabilities", out _)); + Assert.True(secProp.TryGetProperty("workflows", out _)); + Assert.True(secProp.TryGetProperty("app-control", out _)); + } + + [Fact] + public async Task PostAiPromptsReload_Returns200AndSuccess() + { + var response = await _client.PostAsync("/api/ai/prompts/reload", null); + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + + var content = await response.Content.ReadAsStringAsync(); + using var doc = JsonDocument.Parse(content); + var root = doc.RootElement; + + Assert.True(root.GetProperty("success").GetBoolean()); + Assert.True(root.GetProperty("sections").TryGetProperty("system-prompt", out _)); + } + + [Fact] + public async Task PostAiValidate_InvalidEndpoint_ReturnsValidationResultWithFailure() + { + var payload = new + { + endpoint = "http://127.0.0.1:9999/v1", + apiKey = "test-key", + model = "gpt-4o" + }; + + var response = await _client.PostAsJsonAsync("/api/ai/validate", payload); + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + + var result = await response.Content.ReadFromJsonAsync(JsonOptions); + Assert.NotNull(result); + Assert.False(result.Success); + Assert.NotEmpty(result.Message); + } + + [Fact] + public async Task GetAiModels_InvalidEndpoint_ReturnsBadRequest() + { + var response = await _client.GetAsync("/api/ai/models?endpoint=http://invalid-endpoint-xyz-999:1234/v1"); + Assert.Equal(HttpStatusCode.BadRequest, response.StatusCode); + } + + [Fact] + public async Task PostAiChat_EmptyMessages_ReturnsBadRequest() + { + var request = new AiChatRequest(new List()); + var response = await _client.PostAsJsonAsync("/api/ai/chat", request); + Assert.Equal(HttpStatusCode.BadRequest, response.StatusCode); + } + + [Fact] + public async Task PostAiChat_NullBody_ReturnsBadRequest() + { + var content = new StringContent("{}", System.Text.Encoding.UTF8, "application/json"); + var response = await _client.PostAsync("/api/ai/chat", content); + Assert.Equal(HttpStatusCode.BadRequest, response.StatusCode); + } + + [Fact] + public async Task PostAiChat_UnconfiguredEndpoint_ReturnsGracefulFailure() + { + var request = new AiChatRequest(new List + { + new() { Role = "user", Content = "Hello AI" } + }); + + var response = await _client.PostAsJsonAsync("/api/ai/chat", request); + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + + var chatResponse = await response.Content.ReadFromJsonAsync(JsonOptions); + Assert.NotNull(chatResponse); + // Since endpoint is unconfigured in default settings, it should report failure gracefully + Assert.False(chatResponse.Success); + Assert.False(string.IsNullOrWhiteSpace(chatResponse.Error)); + } + + [Fact] + public async Task PostAiChat_StreamMode_UnconfiguredEndpoint_StreamsFinalChunkWithError() + { + var request = new AiChatRequest( + Messages: new List + { + new() { Role = "user", Content = "Hello AI stream" } + }, + Stream: true + ); + + var response = await _client.PostAsJsonAsync("/api/ai/chat", request); + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal("text/event-stream", response.Content.Headers.ContentType?.MediaType); + + var streamData = await response.Content.ReadAsStringAsync(); + Assert.Contains("data:", streamData); + Assert.Contains("error", streamData, StringComparison.OrdinalIgnoreCase); + } +} diff --git a/LocalLLMServerManager.Tests/ComponentManagerAndEndpointsTests.cs b/LocalLLMServerManager.Tests/ComponentManagerAndEndpointsTests.cs index 9493e2f..963455c 100644 --- a/LocalLLMServerManager.Tests/ComponentManagerAndEndpointsTests.cs +++ b/LocalLLMServerManager.Tests/ComponentManagerAndEndpointsTests.cs @@ -27,7 +27,7 @@ public async Task GetComponents_ReturnsExpectedFeaturePacks() var components = await response.Content.ReadFromJsonAsync>(); Assert.NotNull(components); - Assert.Equal(3, components.Count); + Assert.Equal(4, components.Count); var videoPack = components.FirstOrDefault(c => c.Id == "video-generation"); Assert.NotNull(videoPack); @@ -43,6 +43,11 @@ public async Task GetComponents_ReturnsExpectedFeaturePacks() Assert.NotNull(musicPack); Assert.Equal("MusicGen & Stable Audio Studio", musicPack.Name); Assert.Equal("6.8 GB", musicPack.DiskSizeEstimate); + + var aiPack = components.FirstOrDefault(c => c.Id == "ai-assistant"); + Assert.NotNull(aiPack); + Assert.Equal("In-App AI Assistant & Copilot", aiPack.Name); + Assert.Equal("5 MB", aiPack.DiskSizeEstimate); } [Fact] @@ -59,6 +64,15 @@ public async Task InstallAndUninstallComponent_Succeeds() var uninstallReq = new ComponentInstallRequest { ComponentId = "audio-tts" }; var uninstallResponse = await client.PostAsJsonAsync("/api/components/uninstall", uninstallReq); Assert.Equal(HttpStatusCode.OK, uninstallResponse.StatusCode); + + // Install and Uninstall ai-assistant + var aiInstallReq = new ComponentInstallRequest { ComponentId = "ai-assistant" }; + var aiInstallResponse = await client.PostAsJsonAsync("/api/components/install", aiInstallReq); + Assert.Equal(HttpStatusCode.OK, aiInstallResponse.StatusCode); + + var aiUninstallReq = new ComponentInstallRequest { ComponentId = "ai-assistant" }; + var aiUninstallResponse = await client.PostAsJsonAsync("/api/components/uninstall", aiUninstallReq); + Assert.Equal(HttpStatusCode.OK, aiUninstallResponse.StatusCode); } [Fact] diff --git a/Program.cs b/Program.cs index 6a52c40..4547547 100644 --- a/Program.cs +++ b/Program.cs @@ -156,6 +156,9 @@ public static WebApplication CreateWebApplication(string[] args, bool isServiceM builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); // Register MCP Server builder.Services.AddMcpServer() @@ -277,6 +280,7 @@ public static WebApplication CreateWebApplication(string[] args, bool isServiceM app.MapDiscoveryEndpoints(); app.MapComponentEndpoints(); app.MapHardwareEndpoints(); + app.MapAiAssistantEndpoints(); // Service Update Route app.MapPost("/api/service/update", async (HttpContext httpContext, IGitUpdateService gitService) => diff --git a/Services/AiAssistantService.cs b/Services/AiAssistantService.cs index e382f9d..07ed1d9 100644 --- a/Services/AiAssistantService.cs +++ b/Services/AiAssistantService.cs @@ -75,6 +75,7 @@ public async Task ValidateConnectionAsync( : (targetEndpoint.Contains("/v1/") ? $"{targetEndpoint.Substring(0, targetEndpoint.IndexOf("/v1/") + 3)}/models" : $"{targetEndpoint.TrimEnd('/')}/models"); HttpResponseMessage? modelsResp = null; + string? modelsError = null; try { modelsResp = await client.GetAsync(modelsUrl, cancellationToken); @@ -93,8 +94,16 @@ public async Task ValidateConnectionAsync( } } } + else + { + var errBody = await modelsResp.Content.ReadAsStringAsync(cancellationToken); + modelsError = $"Endpoint responded with status {(int)modelsResp.StatusCode} ({modelsResp.ReasonPhrase}): {ExtractErrorMessage(errBody)}"; + } + } + catch (Exception ex) + { + modelsError = ex.Message; } - catch { } // 2. Test ping completion with configured or selected model if (!string.IsNullOrWhiteSpace(targetModel)) @@ -127,6 +136,14 @@ public async Task ValidateConnectionAsync( else { sw.Stop(); + if (modelsError != null) + { + return new AiValidationResult( + false, + $"Failed to query models from endpoint: {modelsError}", + availableModels, + sw.ElapsedMilliseconds); + } } return new AiValidationResult( @@ -148,6 +165,10 @@ public async Task> GetAvailableModelsAsync( CancellationToken cancellationToken = default) { var result = await ValidateConnectionAsync(endpoint, apiKey, model: null, cancellationToken); + if (!result.Success) + { + throw new InvalidOperationException(result.Message); + } return result.AvailableModels; } diff --git a/Services/ComponentManagerService.cs b/Services/ComponentManagerService.cs index 45fafdd..a138e14 100644 --- a/Services/ComponentManagerService.cs +++ b/Services/ComponentManagerService.cs @@ -1,5 +1,6 @@ using LocalLLMServerManager.Shared.Interfaces; using LocalLLMServerManager.Shared.Models; +using LocalLLMServerManager.Shared.Services; namespace LocalLLMServerManager.Services; @@ -14,6 +15,16 @@ public ComponentManagerService(ISettingsService settingsService) private static string GetAppDir() => AppContext.BaseDirectory; + public bool IsAiAssistantInstalled + { + get + { + var settings = _settingsService.LoadSettings(); + var promptDir = PromptManagementService.ResolvePromptDirectory(settings.AiAssistantPromptsDirectory); + return settings.AiAssistantEnabled || promptDir != null; + } + } + public bool IsVideoPackInstalled { get @@ -115,6 +126,15 @@ public Task> GetComponentsAsync() Installed = IsMusicPackInstalled, DiskSizeEstimate = "6.8 GB", MinVramRequired = "6 GB" + }, + new ComponentPackInfo + { + Id = "ai-assistant", + Name = "In-App AI Assistant & Copilot", + Description = "External LLM hookup (LiteLLM / Vertex Flash) with natural language app control, tool calling, and living prompts.", + Installed = IsAiAssistantInstalled, + DiskSizeEstimate = "5 MB", + MinVramRequired = "0 MB (Cloud/API)" } }; @@ -216,6 +236,20 @@ public async Task InstallComponentAsync(string componentId, IProgress UninstallComponentAsync(string componentId) return Task.FromResult(true); } + if (string.Equals(componentId, "ai-assistant", StringComparison.OrdinalIgnoreCase)) + { + var settings = _settingsService.LoadSettings(); + _settingsService.SaveSettings(settings with { AiAssistantEnabled = false }); + return Task.FromResult(true); + } + return Task.FromResult(false); } } diff --git a/Services/IComponentManagerService.cs b/Services/IComponentManagerService.cs index 2fc6722..18534e8 100644 --- a/Services/IComponentManagerService.cs +++ b/Services/IComponentManagerService.cs @@ -4,6 +4,7 @@ namespace LocalLLMServerManager.Services; public interface IComponentManagerService { + bool IsAiAssistantInstalled { get; } bool IsVideoPackInstalled { get; } bool IsAudioPackInstalled { get; } bool IsMusicPackInstalled { get; } @@ -11,3 +12,4 @@ public interface IComponentManagerService Task InstallComponentAsync(string componentId, IProgress? progress = null, CancellationToken cancellationToken = default); Task UninstallComponentAsync(string componentId); } + From 43997b076111d5b1025b52fec3a70c7b763e1505 Mon Sep 17 00:00:00 2001 From: Steven Pelech Date: Thu, 17 Sep 2026 22:48:40 -0500 Subject: [PATCH 05/18] feat(ui): add AI Assistant chat interface, tool progress cards, and setup wizard --- .../Interfaces/ISettingsService.cs | 10 + .../ViewModels/AiAssistantViewModel.cs | 380 ++++++++++++++++++ .../ViewModels/MainViewModel.cs | 13 +- .../Controls/AiAssistantTabControl.axaml | 210 ++++++++++ .../Controls/AiAssistantTabControl.axaml.cs | 11 + .../Views/MainView.axaml | 7 +- .../AiAssistantViewModelTests.cs | 191 +++++++++ .../AvaloniaHeadlessInteractionTests.cs | 56 ++- .../MultimodalHighCoveragePushTests.cs | 1 + Services/ISettingsService.cs | 9 +- 10 files changed, 873 insertions(+), 15 deletions(-) create mode 100644 LocalLLMServerManager.Shared/Interfaces/ISettingsService.cs create mode 100644 LocalLLMServerManager.Shared/ViewModels/AiAssistantViewModel.cs create mode 100644 LocalLLMServerManager.Shared/Views/Controls/AiAssistantTabControl.axaml create mode 100644 LocalLLMServerManager.Shared/Views/Controls/AiAssistantTabControl.axaml.cs create mode 100644 LocalLLMServerManager.Tests/AiAssistantViewModelTests.cs diff --git a/LocalLLMServerManager.Shared/Interfaces/ISettingsService.cs b/LocalLLMServerManager.Shared/Interfaces/ISettingsService.cs new file mode 100644 index 0000000..dbf76a0 --- /dev/null +++ b/LocalLLMServerManager.Shared/Interfaces/ISettingsService.cs @@ -0,0 +1,10 @@ +using LocalLLMServerManager.Shared.Models; + +namespace LocalLLMServerManager.Shared.Interfaces; + +public interface ISettingsService +{ + string SettingsFilePath(); + AppSettings LoadSettings(); + void SaveSettings(AppSettings settings); +} diff --git a/LocalLLMServerManager.Shared/ViewModels/AiAssistantViewModel.cs b/LocalLLMServerManager.Shared/ViewModels/AiAssistantViewModel.cs new file mode 100644 index 0000000..4f3e2a2 --- /dev/null +++ b/LocalLLMServerManager.Shared/ViewModels/AiAssistantViewModel.cs @@ -0,0 +1,380 @@ +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Net.Http.Json; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; +using CommunityToolkit.Mvvm.ComponentModel; +using CommunityToolkit.Mvvm.Input; +using LocalLLMServerManager.Shared.Interfaces; +using LocalLLMServerManager.Shared.Models; +using LocalLLMServerManager.Shared.Services; + +namespace LocalLLMServerManager.Shared.ViewModels; + +/// +/// ViewModel managing the In-App AI Assistant & Copilot interface. +/// Provides multi-turn chat orchestration, streaming token rendering, tool invocation cards, +/// setup wizard with connection validation, and natural language app interaction. +/// +public partial class AiAssistantViewModel : ObservableObject +{ + private readonly IAiAssistantService? _assistantService; + private readonly ISettingsService? _settingsService; + private readonly IPromptManagementService? _promptService; + private readonly HttpClient? _httpClient; + private CancellationTokenSource? _generationCts; + + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNameCaseInsensitive = true, + Converters = { new JsonStringEnumConverter() } + }; + + // Chat History + public ObservableCollection Messages { get; } = new(); + + // Setup & Configuration + [ObservableProperty] private string _endpoint = "http://127.0.0.1:4000/v1"; + [ObservableProperty] private string _apiKey = ""; + [ObservableProperty] private string _selectedModel = "vertex_ai/gemini-2.5-flash"; + [ObservableProperty] private ObservableCollection _availableModels = new(); + [ObservableProperty] private bool _isEnabled = true; + [ObservableProperty] private bool _isSetupCardVisible = false; + [ObservableProperty] private bool _isTestingConnection = false; + [ObservableProperty] private string _connectionStatusMessage = ""; + [ObservableProperty] private bool? _isConnectionSuccess = null; + [ObservableProperty] private string _promptDirectory = ""; + + // Chat UI States + [ObservableProperty] private string _inputText = ""; + [ObservableProperty] private bool _isGenerating = false; + [ObservableProperty] private string _currentStatus = ""; + [ObservableProperty] private bool _autoScrollEnabled = true; + + // Quick Suggestion Chips + public IReadOnlyList SuggestionChips { get; } = new[] + { + "⚑ Check live VRAM and GPU telemetry", + "πŸ¦™ Can I run Llama 3.3 70B on my current hardware?", + "🎨 Generate a scenic mountain landscape with Flux", + "πŸ“¦ List all installed models and their sizes", + "πŸ—£οΈ Speak 'Welcome to Local LLM Server Manager' using Kokoro", + "πŸ›‘ Unload all models from VRAM" + }; + + public AiAssistantViewModel() + : this(null, null, null, null) + { + } + + public AiAssistantViewModel( + IAiAssistantService? assistantService, + ISettingsService? settingsService = null, + IPromptManagementService? promptService = null, + HttpClient? httpClient = null) + { + _assistantService = assistantService; + _settingsService = settingsService; + _promptService = promptService; + _httpClient = httpClient; + + if (_settingsService != null) + { + var s = _settingsService.LoadSettings(); + _isEnabled = s.AiAssistantEnabled; + _endpoint = string.IsNullOrWhiteSpace(s.AiAssistantEndpoint) ? "http://127.0.0.1:4000/v1" : s.AiAssistantEndpoint; + _apiKey = s.AiAssistantApiKey ?? ""; + _selectedModel = string.IsNullOrWhiteSpace(s.AiAssistantModel) ? "vertex_ai/gemini-2.5-flash" : s.AiAssistantModel; + _promptDirectory = PromptManagementService.ResolvePromptDirectory(s.AiAssistantPromptsDirectory) ?? ""; + } + else + { + _promptDirectory = PromptManagementService.ResolvePromptDirectory() ?? ""; + } + + // Add initial model to dropdown if empty + if (!string.IsNullOrWhiteSpace(_selectedModel)) + { + _availableModels.Add(_selectedModel); + } + + // Add welcome message + Messages.Add(new AiChatMessageItem + { + Role = "assistant", + Content = "πŸ‘‹ Hello! I am your AI Assistant and App Copilot. I can query live VRAM telemetry, evaluate model hardware fit, inspect or update app settings, launch workflows, and answer questions about the platform.\n\nType a request below or tap any suggestion chip to get started!", + Timestamp = DateTime.UtcNow + }); + } + + [RelayCommand] + public async Task SendMessageAsync() + { + var text = InputText?.Trim(); + if (string.IsNullOrWhiteSpace(text) || IsGenerating) + { + return; + } + + // 1. Add user message + var userMsg = new AiChatMessageItem + { + Role = "user", + Content = text, + Timestamp = DateTime.UtcNow + }; + Messages.Add(userMsg); + InputText = ""; + + // 2. Prepare assistant placeholder + var assistantMsg = new AiChatMessageItem + { + Role = "assistant", + Content = "", + IsLoading = true, + StatusText = "Connecting to LLM...", + Timestamp = DateTime.UtcNow + }; + Messages.Add(assistantMsg); + + IsGenerating = true; + CurrentStatus = "Thinking..."; + _generationCts = new CancellationTokenSource(); + var ct = _generationCts.Token; + + try + { + var chatMessages = Messages + .Where(m => !m.IsError && !m.IsLoading && !string.IsNullOrWhiteSpace(m.Content)) + .ToList(); + + var request = new AiChatRequest( + Messages: chatMessages, + Stream: true, + Model: SelectedModel + ); + + if (_assistantService != null) + { + // Direct local in-process streaming + await foreach (var chunk in _assistantService.StreamChatAsync(request, ct)) + { + if (chunk.ToolCall != null) + { + assistantMsg.ToolCalls.Add(chunk.ToolCall); + assistantMsg.StatusText = $"Executing {chunk.ToolCall.ToolName}..."; + } + + if (!string.IsNullOrEmpty(chunk.DeltaText)) + { + assistantMsg.Content += chunk.DeltaText; + assistantMsg.StatusText = null; + } + + if (!string.IsNullOrEmpty(chunk.Error)) + { + assistantMsg.IsError = true; + assistantMsg.Content += $"\n⚠️ {chunk.Error}"; + } + } + } + else if (_httpClient != null) + { + // Remote / WASM client streaming via SSE or POST + var response = await _httpClient.PostAsJsonAsync("/api/ai/chat", request with { Stream = false }, ct); + if (response.IsSuccessStatusCode) + { + var chatResp = await response.Content.ReadFromJsonAsync(JsonOptions, ct); + if (chatResp != null && chatResp.Success && chatResp.Message != null) + { + assistantMsg.Content = chatResp.Message.Content; + if (chatResp.Message.ToolCalls != null) + { + foreach (var tc in chatResp.Message.ToolCalls) + { + assistantMsg.ToolCalls.Add(tc); + } + } + } + else + { + assistantMsg.IsError = true; + assistantMsg.Content = chatResp?.Error ?? "Unknown error received from AI server."; + } + } + else + { + assistantMsg.IsError = true; + assistantMsg.Content = $"Server error {(int)response.StatusCode}: {response.ReasonPhrase}"; + } + } + else + { + assistantMsg.IsError = true; + assistantMsg.Content = "AI service is not configured or available."; + } + + if (string.IsNullOrWhiteSpace(assistantMsg.Content) && assistantMsg.ToolCalls.Count > 0) + { + assistantMsg.Content = "Tools executed successfully."; + } + } + catch (OperationCanceledException) + { + if (string.IsNullOrWhiteSpace(assistantMsg.Content)) + { + assistantMsg.Content = "⏹️ Generation stopped by user."; + } + } + catch (Exception ex) + { + assistantMsg.IsError = true; + assistantMsg.Content = $"⚠️ Error: {ex.Message}"; + } + finally + { + assistantMsg.IsLoading = false; + assistantMsg.StatusText = null; + IsGenerating = false; + CurrentStatus = ""; + _generationCts = null; + } + } + + [RelayCommand] + public void StopGeneration() + { + _generationCts?.Cancel(); + } + + [RelayCommand] + public void ClearChat() + { + Messages.Clear(); + Messages.Add(new AiChatMessageItem + { + Role = "assistant", + Content = "Conversation cleared. Ready for your next question or task!", + Timestamp = DateTime.UtcNow + }); + } + + [RelayCommand] + public async Task SelectSuggestionAsync(string? suggestion) + { + if (string.IsNullOrWhiteSpace(suggestion) || IsGenerating) + { + return; + } + + InputText = suggestion; + await SendMessageAsync(); + } + + [RelayCommand] + public void ToggleSetupCard() + { + IsSetupCardVisible = !IsSetupCardVisible; + } + + [RelayCommand] + public async Task TestConnectionAsync() + { + IsTestingConnection = true; + ConnectionStatusMessage = "Testing endpoint connectivity and model availability..."; + IsConnectionSuccess = null; + + try + { + AiValidationResult? result = null; + if (_assistantService != null) + { + result = await _assistantService.ValidateConnectionAsync(Endpoint, ApiKey, SelectedModel); + } + else if (_httpClient != null) + { + var payload = new { endpoint = Endpoint, apiKey = ApiKey, model = SelectedModel }; + var resp = await _httpClient.PostAsJsonAsync("/api/ai/validate", payload); + if (resp.IsSuccessStatusCode) + { + result = await resp.Content.ReadFromJsonAsync(JsonOptions); + } + } + + if (result != null) + { + IsConnectionSuccess = result.Success; + ConnectionStatusMessage = result.Message; + + if (result.AvailableModels != null && result.AvailableModels.Count > 0) + { + AvailableModels.Clear(); + foreach (var m in result.AvailableModels) + { + AvailableModels.Add(m); + } + if (!AvailableModels.Contains(SelectedModel) && AvailableModels.Count > 0) + { + SelectedModel = AvailableModels[0]; + } + } + } + else + { + IsConnectionSuccess = false; + ConnectionStatusMessage = "Failed to communicate with the validation service."; + } + } + catch (Exception ex) + { + IsConnectionSuccess = false; + ConnectionStatusMessage = $"Connection failed: {ex.Message}"; + } + finally + { + IsTestingConnection = false; + } + } + + [RelayCommand] + public void SaveConfiguration() + { + if (_settingsService != null) + { + var s = _settingsService.LoadSettings(); + var updated = s with + { + AiAssistantEnabled = IsEnabled, + AiAssistantEndpoint = Endpoint, + AiAssistantApiKey = ApiKey, + AiAssistantModel = SelectedModel + }; + _settingsService.SaveSettings(updated); + } + + IsSetupCardVisible = false; + ConnectionStatusMessage = "Settings saved successfully."; + } + + [RelayCommand] + public async Task ReloadPromptsAsync() + { + if (_promptService != null) + { + _promptService.InvalidateCache(); + PromptDirectory = PromptManagementService.ResolvePromptDirectory() ?? ""; + ConnectionStatusMessage = "Living prompts reloaded from disk."; + } + else if (_httpClient != null) + { + await _httpClient.PostAsync("/api/ai/prompts/reload", null); + ConnectionStatusMessage = "Prompts reloaded via API."; + } + } +} diff --git a/LocalLLMServerManager.Shared/ViewModels/MainViewModel.cs b/LocalLLMServerManager.Shared/ViewModels/MainViewModel.cs index a19f98f..64e3e63 100644 --- a/LocalLLMServerManager.Shared/ViewModels/MainViewModel.cs +++ b/LocalLLMServerManager.Shared/ViewModels/MainViewModel.cs @@ -90,6 +90,7 @@ public HttpClient Http public AudioStudioViewModel Audio { get; } public CanIRunItViewModel HardwareFit { get; } public DocumentationViewModel Documentation { get; } = new(); + public AiAssistantViewModel Assistant { get; } [ObservableProperty] private int _selectedTabIndex = 0; @@ -116,7 +117,10 @@ public MainViewModel( IHuggingFaceSearchService hfSearchService, ICivitaiSearchService civitaiSearchService, IStudioPresetService? studioPresetService = null, - ICanIRunItService? canIRunItService = null) + ICanIRunItService? canIRunItService = null, + IAiAssistantService? assistantService = null, + ISettingsService? settingsService = null, + IPromptManagementService? promptService = null) { if (httpClient != null) _customHttp = httpClient; @@ -154,6 +158,7 @@ public MainViewModel( }; Settings = new SettingsViewModel(PresetService) { ApiBase = ApiBase }; Audio = new AudioStudioViewModel(PresetService, _canIRunItService) { ApiBase = ApiBase }; + Assistant = new AiAssistantViewModel(assistantService, settingsService, promptService, httpClient ?? Http); LoadStudioPresets(); RecalculateVideoHardwareFit(); @@ -863,6 +868,12 @@ public void NavigateToModelsTab() SelectedTabIndex = 0; } + [RelayCommand] + public void NavigateToAssistantTab() + { + SelectedTabIndex = 3; + } + [RelayCommand] public async Task LoadInstalledModelsAsync() => await Ollama.LoadInstalledModelsAsync(ApiBase, Http); diff --git a/LocalLLMServerManager.Shared/Views/Controls/AiAssistantTabControl.axaml b/LocalLLMServerManager.Shared/Views/Controls/AiAssistantTabControl.axaml new file mode 100644 index 0000000..3ba5462 --- /dev/null +++ b/LocalLLMServerManager.Shared/Views/Controls/AiAssistantTabControl.axaml @@ -0,0 +1,210 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +